
    +sg                        d dl mZ d dlmZ d dlmZ d dlZd dlmZmZ d dl	m
Z
mZ d dlmZ  G d d	ej                        Zy)
    )annotations)Iterable)AnyN)Tensornn)StaticEmbeddingTransformer)SentenceTransformerc                  \     e Zd Z	 d	 	 	 	 	 	 	 d fdZddZd	dZd
dZedd       Z xZ	S )GISTEmbedLossc                   t         |           || _        || _        || _        t        j                  d      | _        t        |d   t              rt        |d   t              st        d      |j                  j                  |j                  j                  k7  xs |j                  |j                  k  | _        | j                  rD| j                  j                  | _        t        | j                  d   t              rt        d      yy)a
  
        This loss is used to train a SentenceTransformer model using the GISTEmbed algorithm.
        It takes a model and a guide model as input, and uses the guide model to guide the
        in-batch negative sample selection. The cosine similarity is used to compute the loss
        and the temperature parameter is used to scale the cosine similarities.

        Args:
            model: SentenceTransformer model based on a `transformers`
                model.
            guide: SentenceTransformer model to guide the in-batch
                negative sample selection.
            temperature: Temperature parameter to scale the cosine
                similarities.

        References:
            - For further details, see: https://arxiv.org/abs/2402.16829

        Requirements:
            1. (anchor, positive, negative) triplets
            2. (anchor, positive) pairs

        Inputs:
            +---------------------------------------+--------+
            | Texts                                 | Labels |
            +=======================================+========+
            | (anchor, positive, negative) triplets | none   |
            +---------------------------------------+--------+
            | (anchor, positive) pairs              | none   |
            +---------------------------------------+--------+

        Recommendations:
            - Use ``BatchSamplers.NO_DUPLICATES`` (:class:`docs <sentence_transformers.training_args.BatchSamplers>`) to
              ensure that no in-batch negatives are duplicates of the anchor or positive samples.

        Relations:
            - :class:`MultipleNegativesRankingLoss` is similar to this loss, but it does not use
              a guide model to guide the in-batch negative sample selection. `GISTEmbedLoss` yields
              a stronger training signal at the cost of some training overhead.

        Example:
            ::

                from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
                from datasets import Dataset

                model = SentenceTransformer("microsoft/mpnet-base")
                guide = SentenceTransformer("all-MiniLM-L6-v2")
                train_dataset = Dataset.from_dict({
                    "anchor": ["It's nice weather outside today.", "He drove to work."],
                    "positive": ["It's so sunny.", "He took the car to the office."],
                })
                loss = losses.GISTEmbedLoss(model, guide)

                trainer = SentenceTransformerTrainer(
                    model=model,
                    train_dataset=train_dataset,
                    loss=loss,
                )
                trainer.train()
        dimr   z_Both the training model and the guiding model must be based on the `transformers` architecture.zIf we must retokenize because the guide model has a different tokenizer, then the Sentence Transformer model must not be based on a StaticEmbedding.N)super__init__modelguidetemperaturer   CosineSimilaritysimilarity_fct
isinstancer	   
ValueError	tokenizervocabmax_seq_lengthmust_retokenizer   )selfr   r   r   	__class__s       ]/var/www/html/venv/lib/python3.12/site-packages/sentence_transformers/losses/GISTEmbedLoss.pyr   zGISTEmbedLoss.__init__   s    D 	

& 11b9%(K0
58[8Yq  OO!!U__%:%::ie>R>RUZUiUi>i 	 !ZZ11DN$**Q-9 b  :      c                b    | j                  |j                  d      |j                  d            S )N   r   )r   	unsqueeze)r   embed1embed2s      r    
sim_matrixzGISTEmbedLoss.sim_matrixe   s+    ""6#3#3A#68H8H8KLLr!   c                   |D cg c]  }| j                  |      d    }}t        j                         5  | j                  r|D cg c]"  }| j                  j                  |d   d      $ }}|D cg c]  }| j                  j                  |       }}|D cg c]I  }|j                         D ci c]+  \  }}||j                  | j                  j                        - c}}K }}}}|D cg c]  }| j                  |      d    }	}d d d        d }
d }t        |      dk(  r|\  }}	\  }}n2t        |      dk(  r|\  }}}
	\  }}}nt        dt        |             | j                  ||      }| j                  ||      }| j                  ||      }| j                  ||      }| j                  ||      }| j                  ||      }|j                         j                  dd	      }t        j                    |||kD  <   t        j                    |||kD  <   t        j                    |||kD  <   |||g}|
L| j                  ||
      }| j                  ||      }t        j                    |||kD  <   |j#                  |       t        j$                  |d	
      | j&                  z  }t        j(                  |j+                  d            j-                         j                  |j                        } t/        j0                         ||      S c c}w c c}w c c}w c c}}w c c}}}w c c}w # 1 sw Y   (xY w)Nsentence_embedding	input_idsT)skip_special_tokens      z Expected 2 or 3 embeddings, got r   r#   r   r   )r   torchno_gradr   r   batch_decoder   tokenizeitemstodevicelenr   r'   diagonalviewinfappendcatr   arangesizelongr   CrossEntropyLoss)r   sentence_featureslabelssentence_feature
embeddingsdecoded	sentenceskeyvalueguide_embeddingsnegativenegative_guideanchorpositiveanchor_guidepositive_guideap_simaa_simpp_simguided_ap_simguided_aa_simguided_pp_sim
guided_simscoresan_simguided_an_sims                             r    forwardzGISTEmbedLoss.forwardh   s5   arsM]djj!123GHs
s]]_ 	## ->( NN//0@0Mcg/h  V]$]	TZZ%8%8%C$]!$] ->% %( IYH^H^H`a*#uS%((4::#4#455a%! % \m GW

+,-AB   	  z?a)FH+;(L._!)3&FHh;K8L..?J?PQRR 2084 nElCG #++-222q9
 /4iiZ}z)*.3iiZ}z)*.3iiZ}z)*&&) __VX6F OOL.IM27))F=:-.MM&!6q)D,<,<< fkk!n-22477F$r""$VV44E t %^a%
 	 	s^   LL7'L.L74"L L7L+
60L%&L+
,L74L2L7
L7%L+
+L77Mc                4    | j                   | j                  dS )Nr   r   rZ   r   s    r    get_config_dictzGISTEmbedLoss.get_config_dict   s    ZZ++
 	
r!   c                     y)Na  
@misc{solatorio2024gistembed,
    title={GISTEmbed: Guided In-sample Selection of Training Negatives for Text Embedding Fine-tuning},
    author={Aivin V. Solatorio},
    year={2024},
    eprint={2402.16829},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}
 r[   s    r    citationzGISTEmbedLoss.citation   s    	r!   )g{Gz?)r   r
   r   r
   r   floatreturnNone)r%   r   r&   r   ra   r   )r?   zIterable[dict[str, Tensor]]r@   r   ra   r   )ra   zdict[str, Any])ra   str)
__name__
__module____qualname__r   r'   rX   r\   propertyr_   __classcell__)r   s   @r    r   r      s_    
 "	U"U #U 	U
 
UnMC5J
 
 
r!   r   )
__future__r   collections.abcr   typingr   r.   r   r   sentence_transformers.modelsr   r	   )sentence_transformers.SentenceTransformerr
   Moduler   r^   r!   r    <module>ro      s,    " $    E IqBII qr!   