
    +sg?                    4    d dl mZ d dlZd dlZ G d d      Zy)    )annotationsNc                      e Zd Zd Zd Zd Zy)NoDuplicatesDataLoaderc                z    || _         d| _        d| _        || _        t	        j
                  | j                         y)z
        A special data loader to be used with MultipleNegativesRankingLoss.
        The data loader ensures that there are no duplicate sentences within the same batch
        r   N)
batch_sizedata_pointer
collate_fntrain_examplesrandomshuffle)selfr
   r   s      h/var/www/html/venv/lib/python3.12/site-packages/sentence_transformers/datasets/NoDuplicatesDataLoader.py__init__zNoDuplicatesDataLoader.__init__   s4    
 %,t**+    c              #    K   t        | j                               D ]a  }g }t               }t        |      | j                  k  r| j
                  | j                     }d}|j                  D ]&  }|j                         j                         |v s$d} n |rO|j                  |       |j                  D ]/  }|j                  |j                         j                                1 | xj                  dz  c_        | j                  t        | j
                        k\  r&d| _        t        j                  | j
                         t        |      | j                  k  r| j                  | j                  |      n| d y w)NTF   r   )range__len__setlenr   r
   r   textsstriplowerappendaddr   r   r	   )r   _batchtexts_in_batchexamplevalid_exampletexts          r   __iter__zNoDuplicatesDataLoader.__iter__   sB    t||~& 	SAE UNe*t.--d.?.?@ $#MM Dzz|))+~=(-
 !LL) ' A&**4::<+=+=+?@A !!Q&!$$D,?,?(@@()D%NN4#6#67# e*t.& -1OO,G$//%(UR/	Ss   BFCF%Fc                l    t        j                  t        | j                        | j                  z        S )N)mathfloorr   r
   r   )r   s    r   r   zNoDuplicatesDataLoader.__len__-   s%    zz#d112T__DEEr   N)__name__
__module____qualname__r   r"   r    r   r   r   r      s    	,S4Fr   r   )
__future__r   r$   r   r   r)   r   r   <module>r+      s    "  'F 'Fr   