+sg(1ddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZe rddlmZej"eZGd d e Zy) ) annotationsN) defaultdict) nullcontext) TYPE_CHECKING)SentenceEvaluator)paraphrase_mining)SentenceTransformerceZdZdZ d dfd Z d ddZedZxZS) ParaphraseMiningEvaluatora Given a large set of sentences, this evaluator performs paraphrase (duplicate) mining and identifies the pairs with the highest similarity. It compare the extracted paraphrase pairs with a set of gold labels and computes the F1 score. Example: :: from datasets import load_dataset from sentence_transformers.SentenceTransformer import SentenceTransformer from sentence_transformers.evaluation import ParaphraseMiningEvaluator # Load a model model = SentenceTransformer('all-mpnet-base-v2') # Load the Quora Duplicates Mining dataset questions_dataset = load_dataset("sentence-transformers/quora-duplicates-mining", "questions", split="dev") duplicates_dataset = load_dataset("sentence-transformers/quora-duplicates-mining", "duplicates", split="dev") # Create a mapping from qid to question & a list of duplicates (qid1, qid2) qid_to_questions = dict(zip(questions_dataset["qid"], questions_dataset["question"])) duplicates = list(zip(duplicates_dataset["qid1"], duplicates_dataset["qid2"])) # Initialize the paraphrase mining evaluator paraphrase_mining_evaluator = ParaphraseMiningEvaluator( sentences_map=qid_to_questions, duplicates_list=duplicates, name="quora-duplicates-dev", ) results = paraphrase_mining_evaluator(model) ''' Paraphrase Mining Evaluation of the model on the quora-duplicates-dev dataset: Number of candidate pairs: 250564 Average Precision: 56.51 Optimal threshold: 0.8325 Precision: 52.76 Recall: 59.19 F1: 55.79 ''' print(paraphrase_mining_evaluator.primary_metric) # => "quora-duplicates-dev_average_precision" print(results[paraphrase_mining_evaluator.primary_metric]) # => 0.5650940787776353 c t|g|_g|_|j D];\}}|jj ||jj |=| |_| |_| |_||_ ||_ ||_ ||_ | |_ ||n td|_|8|D]3\}}||vs ||vsd|j||<d|j||<5|r |j!|j|_t#}|jD]j}|j|D]V}||vs||vs |j||s|j||s2|j%t't)||gXlt+||_| rd| z} d| zdz|_gd|_| |_d|_y) a Initializes the ParaphraseMiningEvaluator. Args: sentences_map (Dict[str, str]): A dictionary that maps sentence-ids to sentences. For example, sentences_map[id] => sentence. duplicates_list (List[Tuple[str, str]], optional): A list with id pairs [(id1, id2), (id1, id5)] that identifies the duplicates / paraphrases in the sentences_map. Defaults to None. duplicates_dict (Dict[str, Dict[str, bool]], optional): A default dictionary mapping [id1][id2] to true if id1 and id2 are duplicates. Must be symmetric, i.e., if [id1][id2] => True, then [id2][id1] => True. Defaults to None. add_transitive_closure (bool, optional): If true, it adds a transitive closure, i.e. if dup[a][b] and dup[b][c], then dup[a][c]. Defaults to False. query_chunk_size (int, optional): To identify the paraphrases, the cosine-similarity between all sentence-pairs will be computed. As this might require a lot of memory, we perform a batched computation. query_chunk_size sentences will be compared against up to corpus_chunk_size sentences. In the default setting, 5000 sentences will be grouped together and compared up-to against 100k other sentences. Defaults to 5000. corpus_chunk_size (int, optional): The corpus will be batched, to reduce the memory requirement. Defaults to 100000. max_pairs (int, optional): We will only extract up to max_pairs potential paraphrase candidates. Defaults to 500000. top_k (int, optional): For each query, we extract the top_k most similar pairs and add it to a sorted list. I.e., for one sentence we cannot find more than top_k paraphrases. Defaults to 100. show_progress_bar (bool, optional): Output a progress bar. Defaults to False. batch_size (int, optional): Batch size for computing sentence embeddings. Defaults to 16. name (str, optional): Name of the experiment. Defaults to "". write_csv (bool, optional): Write results to CSV file. Defaults to True. truncate_dim (Optional[int], optional): The dimension to truncate sentence embeddings to. `None` uses the model's current truncation dimension. Defaults to None. Nc ttS)N)rboolm/var/www/html/venv/lib/python3.12/site-packages/sentence_transformers/evaluation/ParaphraseMiningEvaluator.pyz4ParaphraseMiningEvaluator.__init__..s bmnrbsrT_paraphrase_mining_evaluationz _results.csv)epochsteps precisionrecallf1 thresholdaverage_precisionr)super__init__ sentencesidsitemsappendnameshow_progress_bar batch_sizequery_chunk_sizecorpus_chunk_size max_pairstop_k truncate_dimr duplicatesadd_transitive_closuresetaddtuplesortedlentotal_num_duplicatescsv_file csv_headers write_csvprimary_metric)self sentences_mapduplicates_listduplicates_dictr+r%r&r'r(r#r$r"r4r)idsentenceid1id2positive_key_pairskey1key2 __class__s rrz"ParaphraseMiningEvaluator.__init__As^ )//1 LB NN ! !( + HHOOB   !2$ 0!2" (-<-H/kZsNt  &+ 5S-'C=,@04DOOC(-04DOOC(- 5 ""99$//JDO UOO HD- HM) -.t48Md8S&**5t 1E+FG  H H%((:$;! :D;dB^S l"1rc |dk7r|dk(rd|}n d|d|d}nd}|j|d|jdz }tjd |jd |d |j t n|j |j5t ||j|j|j|j|j|j|j}dddtjd ttzd x}}d } d x} x} } d } t!t|D]}||\}}}|j"|}|j"|}|dz }|j$||s|j$||sT|dz }||z }||j&z }d|z|z||zz }| |z } || kDs|} |} |} ||d |t)|dzt|dz d zdz } | |j&z } tjd| dzdtjd| dtjd| dzdtjd| dzdtjd| dzdd||j*rt,j.j1||j2}t,j.j5|sat7|ddd5}t9j:|}|j=|j>|j=||| | | | | gdddnEt7|ddd5}t9j:|}|j=||| | | | | gddd| | | | | d}|jA||j}|jC|||S#1swYxYw#1swYNxYw#1swYZxYw)Nz after epoch z in epoch z after z stepsz (truncated to )z1Paraphrase Mining Evaluation of the model on the z dataset:zNumber of candidate pairs: rzAverage Precision: dz.2fzOptimal threshold: z.4fz Precision: zRecall: zF1:  wzutf-8)newlinemodeencodinga)rrrrr)"r)loggerinfor"rtruncate_sentence_embeddingsrrr#r$r%r&r'r(strr0rangerr*r1minr4ospathjoinr2isfileopencsvwriterwriterowr3prefix_name_to_metrics store_metrics_in_model_card_data)r6model output_pathrrout_txt pairs_list n_extract n_correctrbest_f1 best_recallbest_precisionridxscoreijr<r=rrrcsv_pathfr\metricss r__call__z"ParaphraseMiningEvaluator.__call__s B;{)%1&ugWUG6BG    ( ):):(;1= =G G {RZ[bZccdef#//7[]U=_=_`d`q`q=r *&&%%&&  J   1CJ4HHI!"! I 1222+Z) lC$S/KE1a((1+C((1+C NIs#C(DOOC,@,EQ % 1 "T%>%>>]V+y6/AB!Y.!< G%.N"(K!+C!3jS1WcR\o`aNaAb6cde6f!fjk kI# l&.0I0II )*;c*A#)FGH ))C9: k.3"6s!;<= h{S0567 d7S=-R01  "t~~ww||K?H77>>(+(BS7Kxq ZZ]FOOD$4$45OOUE>;PWYbdu$vwxx (BS7Kxq ZZ]FOOUE>;PWYbdu$vwx "3'!"  --gtyyA --eW=K  fxx xxs&AO :A O.O" OO"O+ctt}t|jD]}||vs t}|j|t||}t |dkDrI|j d}||vr%|j||j ||t |dkDrIt|}tt |dz D]c}t|dzt |D]F}d|||||<d|||||<|j|||j||He|S)NrrGT)r,listkeysr-r0popextendrT)graph nodes_visitedrOconnected_subgraph_nodesneighbor_nodes_queuenoderkrls rr+z0ParaphraseMiningEvaluator.add_transitive_closuresb ejjl# GA %+.5((,,Q/(,E!H~$./!3/33A6D#;;044T:,33E$K@ ./!3,00H+I(s#;rsG"  #" P8M   8 $} 1}r