
    +sg6                        d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	c m
Z d dlmZm	Z	 d dlmZ d dlmZ d dlmZ  G d	 d
      Z G d de	j*                        Zy)    )annotationsN)Iterable)Any)Tensornn)SentenceTransformer)CachedGISTEmbedLoss)"CachedMultipleNegativesRankingLossc                  ,    e Zd ZddZddZddZddZy)	ForwardDecoratorc                J    || _         d | _        g | _        d | _        d| _        y Nr   )fndimcache	cache_dimidx)selfr   s     ^/var/www/html/venv/lib/python3.12/site-packages/sentence_transformers/losses/MatryoshkaLoss.py__init__zForwardDecorator.__init__   s%    
    c                     || _         d| _        y r   )r   r   )r   r   s     r   set_dimzForwardDecorator.set_dim   s    r   c                    |j                   d   }| j                  |kD  rt        d| j                   d|       |dd | j                  f   }t        j                  |dd      }|S )Nz
Dimension zL in matryoshka_dims cannot be greater than the model's embedding dimension: .   )pr   )shaper   
ValueErrorF	normalize)r   tensor
tensor_dims      r   shrinkzForwardDecorator.shrink   so    \\"%
88j TXXJ&rs}r~  ZtxxZ(Vqb1r   c                   | j                   | j                   | j                  k(  r>| j                  |      }| j                  j	                  |       | j                  | _         n| j                  | j
                     }d|v r| j                  |d         |d<   | j                  |d         |d<   | xj
                  dz  c_        |S )Ntoken_embeddingssentence_embedding   )r   r   r   r   appendr   r$   )r   featuresoutputs      r   __call__zForwardDecorator.__call__'   s    >>!T^^txx%?WWX&FJJf%!XXDN ZZ)F')-V<N5O)PF%&'+{{6:N3O'P#$Ar   N)returnNone)r"   r   r-   r   )r*   dict[str, Tensor]r-   r/   )__name__
__module____qualname__r   r   r$   r,    r   r   r   r      s    r   r   c                  ^     e Zd Z	 	 d	 	 	 	 	 	 	 	 	 	 	 d fdZddZddZed	d       Z xZS )
MatryoshkaLossc                f   t         |           || _        || _        t	        |t
              rt        j                  dd       t	        |t              rt        j                  dd       |dgt        |      z  }t        ||      }t        t        |d d	       \  | _        | _        || _        y)
a?  
        The MatryoshkaLoss can be seen as a loss *modifier* that allows you to use other loss functions at various
        different embedding dimensions. This is useful for when you want to train a model where users have the option
        to lower the embedding dimension to improve their embedding comparison speed and costs.

        Args:
            model: SentenceTransformer model
            loss: The loss function to be used, e.g.
                :class:`MultipleNegativesRankingLoss`,
                :class:`CoSENTLoss`, etc.
            matryoshka_dims: A list of embedding dimensions to be used
                for the loss function, e.g. [768, 512, 256, 128, 64].
            matryoshka_weights: A list of weights to be used for the
                loss function, e.g. [1, 1, 1, 1, 1]. If None, then the
                weights will be set to 1 for all dimensions.
            n_dims_per_step: The number of dimensions to use per step.
                If -1, then all dimensions are used. If > 0, then a
                random sample of n_dims_per_step dimensions are used per
                step. The default value is -1.

        References:
            - The concept was introduced in this paper: https://arxiv.org/abs/2205.13147
            - `Matryoshka Embeddings <../../examples/training/matryoshka/README.html>`_

        Requirements:
            1. The base loss cannot be :class:`CachedMultipleNegativesRankingLoss` or :class:`CachedGISTEmbedLoss`.

        Inputs:
            +---------------------------------------+--------+
            | Texts                                 | Labels |
            +=======================================+========+
            | any                                   | any    |
            +---------------------------------------+--------+

        Relations:
            - :class:`Matryoshka2dLoss` uses this loss in combination with :class:`AdaptiveLayerLoss` which allows for
                layer reduction for faster inference.

        Example:
            ::

                from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
                from datasets import Dataset

                model = SentenceTransformer("microsoft/mpnet-base")
                train_dataset = Dataset.from_dict({
                    "anchor": ["It's nice weather outside today.", "He drove to work."],
                    "positive": ["It's so sunny.", "He took the car to the office."],
                })
                loss = losses.MultipleNegativesRankingLoss(model)
                loss = losses.MatryoshkaLoss(model, loss, [768, 512, 256, 128, 64])

                trainer = SentenceTransformerTrainer(
                    model=model,
                    train_dataset=train_dataset,
                    loss=loss,
                )
                trainer.train()
        zIMatryoshkaLoss is not compatible with CachedMultipleNegativesRankingLoss.r   )
stacklevelz:MatryoshkaLoss is not compatible with CachedGISTEmbedLoss.Nr(   c                    | d   S r   r3   )xs    r   <lambda>z)MatryoshkaLoss.__init__.<locals>.<lambda>   s    `abc`d r   T)keyreverse)superr   modelloss
isinstancer
   warningswarnr	   lenzipsortedmatryoshka_dimsmatryoshka_weightsn_dims_per_step)r   r>   r?   rF   rG   rH   dims_weights	__class__s          r   r   zMatryoshkaLoss.__init__8   s    F 	
	d>?MMerstd/0MMVcde%"#s?';!;?,>?8;VLVdnr=s8t5d5.r   c                &   | j                   j                  }	 t        |      }|| j                   _        t        t	        | j
                              }| j                  dkD  r8| j                  t	        |      k  r t        j                  || j                        }d}|D ]I  }| j
                  |   }| j                  |   }	|j                  |       ||	| j                  ||      z  z  }K 	 || j                   _        |S # || j                   _        w xY w)Nr   g        )r>   forwardr   rangerC   rF   rH   randomsamplerG   r   r?   )
r   sentence_featureslabelsoriginal_forwarddecorated_forwarddim_indicesr?   r   r   weights
             r   rL   zMatryoshkaLoss.forward   s    ::--	2 01A B!2DJJD$8$8 9:K##a'D,@,@3{CS,S$mmK9M9MND" F**3/005!))#.+<f!EEE	F "2DJJ "2DJJs   CC= =Dc                    | j                   j                  j                  | j                  | j                  | j
                  dS )N)r?   rF   rG   rH   )r?   rJ   r0   rF   rG   rH   r   s    r   get_config_dictzMatryoshkaLoss.get_config_dict   s:    II''00#33"&"9"9#33	
 	
r   c                     y)Na  
@misc{kusupati2024matryoshka,
    title={Matryoshka Representation Learning},
    author={Aditya Kusupati and Gantavya Bhatt and Aniket Rege and Matthew Wallingford and Aditya Sinha and Vivek Ramanujan and William Howard-Snyder and Kaifeng Chen and Sham Kakade and Prateek Jain and Ali Farhadi},
    year={2024},
    eprint={2205.13147},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}
r3   rW   s    r   citationzMatryoshkaLoss.citation   s    	r   )Nr   )r>   r   r?   z	nn.ModulerF   z	list[int]rG   zlist[float | int] | NonerH   intr-   r.   )rP   zIterable[dict[str, Tensor]]rQ   r   r-   r   )r-   zdict[str, Any])r-   str)	r0   r1   r2   r   rL   rX   propertyrZ   __classcell__)rJ   s   @r   r5   r5   7   sp     8<!P/"P/ P/ #	P/
 5P/ P/ 
P/d(
 
 
r   r5   )
__future__r   rN   rA   collections.abcr   typingr   torch.nn.functionalr   
functionalr    torchr   sentence_transformersr   0sentence_transformers.losses.CachedGISTEmbedLossr	   ?sentence_transformers.losses.CachedMultipleNegativesRankingLossr
   r   Moduler5   r3   r   r   <module>ri      sC    "   $     5 P n$ $NzRYY zr   