sg[NddlZddlZddlZddlmZmZmZmZddl m Z m Z m Z m Z mZer ddlZddlmZmZer ddlZddlmZmZGdd e Zeed Gd d e Zy)N)add_end_docstringsis_tf_availableis_torch_availablerequires_backends)ArgumentHandlerDatasetPipelinePipelineExceptionbuild_pipeline_init_args),MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES0MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES)/TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES3TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMESceZdZdZddZy)%TableQuestionAnsweringArgumentHandlerzB Handles arguments for the TableQuestionAnsweringPipeline Nc t|dddl}| td|t|tr&|j d|j d|g}nt|t rt|dkDrrtd|Dstdd|D|dj d|dj d|}nptd |djd tt|tst|tjr|Std t|d ||d g}|D]C}t|d|jr|d td|j|d|d<E|S)Npandasrz(Keyword argument `table` cannot be None.querytablec3<K|]}t|tywN) isinstancedict.0ds b/var/www/html/venv/lib/python3.12/site-packages/transformers/pipelines/table_question_answering.py zATableQuestionAnsweringArgumentHandler.__call__..5s>1:a.>sz:Keyword argument `table` should be a list of dict, but is c32K|]}t|ywr)typers rr zATableQuestionAnsweringArgumentHandler.__call__..7sUmbcVZ[\V]UmszIf keyword argument `table` is a list of dictionaries, each dictionary should have a `table` and `query` key, but only dictionary has keys z `table` and `query` keys.zZInvalid input. Keyword argument `table` should be either of type `dict` or `list`, but is ))rrzTable cannot be None.)rr ValueErrorrrgetlistlenallkeysr types GeneratorTyper" DataFrame)selfrrkwargspdtqa_pipeline_inputstqa_pipeline_inputs r__call__z.TableQuestionAnsweringArgumentHandler.__call__%s $) =GH H ]%&599W+=+IeiiX_N`Nl',g#E4(SZ!^>>>$TUmglUmTno8<<(4qg9N9Z*/'$JJOPQ(--/IZZtv$E7)CzRWY^YlYlGm  u+a) .3U#C"D "5 X 092<<H%g.6$%<==.0ll;Mg;V.W"7+  X#")NN)__name__ __module__ __qualname____doc__r2r3rrr s -#r3rT) has_tokenizercheZdZdZdZeffd ZdZdZfdZ d dZ d dZ d d Z d Z xZS)TableQuestionAnsweringPipelinea Table Question Answering pipeline using a `ModelForTableQuestionAnswering`. This pipeline is only available in PyTorch. Example: ```python >>> from transformers import pipeline >>> oracle = pipeline(model="google/tapas-base-finetuned-wtq") >>> table = { ... "Repository": ["Transformers", "Datasets", "Tokenizers"], ... "Stars": ["36542", "4512", "3934"], ... "Contributors": ["651", "77", "34"], ... "Programming language": ["Python", "Python", "Rust, Python and NodeJS"], ... } >>> oracle(query="How many stars does the transformers repository have?", table=table) {'answer': 'AVERAGE > 36542', 'coordinates': [(0, 1)], 'cells': ['36542'], 'aggregator': 'AVERAGE'} ``` Learn more about the basics of using a pipeline in the [pipeline tutorial](../pipeline_tutorial) This tabular question answering pipeline can currently be loaded from [`pipeline`] using the following task identifier: `"table-question-answering"`. The models that this pipeline can use are models that have been fine-tuned on a tabular question answering task. See the up-to-date list of available models on [huggingface.co/models](https://huggingface.co/models?filter=table-question-answering). z table,queryc,t||i|||_|jdk(r*t j }|j tn)tj }|j t|j|tt|jjddxr*tt|jjdd|_t!|jjdrd|_yd|_y)Ntfaggregation_labelsnum_aggregation_labelstapas)super__init__ _args_parser frameworkrcopyupdaterrrcheck_model_typeboolgetattrmodelconfig aggregatehasattrr")r- args_parserargsr.mapping __class__s rrBz'TableQuestionAnsweringPipeline.__init__ws $)&)' >>T !INNPG NNJ KFKKMG NNG H g&gdjj&7&79MtTU Z^ DJJ%%'? F[  'tzz'8'8:NOG UY r3c &|jdi|S)Nr8)rJ)r-inputss rbatch_inferencez.TableQuestionAnsweringPipeline.batch_inferencestzz#F##r3c @|jdk(rg}g}d}|djd}|dj|j}|dj|j}|dj|j}d} t |D]+} |'| dddf} t j | jj} || } t | jdD]{} | dddfj| }| dddfj| dz }| ddd fj| dz }|dk\s]|dk\sc|dk(sit|||f| | <}tj| jtjj|j| dddf<|| }|| }|| } |j|j!d|j!d| j!d }|j"}|j$r|j'|j(|j'|tj*j-| }|j.|jtj0j|j.jz}t3j4t6}t9|j;jD]\} }| dddfj| }| dddfj| dz }| ddd fj| dz }|dk\s`|dk\sf|dk(sl|||fj'||Dcic],}|t j<||j?d kD.}}.tj@tC|d}|j$s|fS|tj@tC|dfSg}g}d}|djd}|d}|d}|dj}d} t |D]} || dddf} t j | t jD } || } t | jdD]{} | dddfj| }| dddfj| dz }| ddd fj| dz }|dk\s]|dk\sc|dk(sit|||f| | <}| | dddf<|| }|| }|| } |jt jF|dt jF|dt jF| d }|j"}|j$r|j'|j(|j'|tHjJjMtIjN|tHj0tIjN|tHj0z}t3j4t6}| } t9tIj:|jjD]\} }| dddfj| }| dddfj| dz }| ddd fj| dz }|dk\s`|dk\sf|dk(sl|||fj'||Dcic],}|t j<||j?d kD.}}tIjPtC|d}|j$s|fS|tIjPtC|dfScc}wcc}w)z Inference used for models that need to process sequences in a sequential fashion, like the SQA models which handle conversational query related to a table. ptN input_idsrattention_masktoken_type_idsrr)rWrXrY)logitsg?)dtype)axis))rDshapetodevicerangenp zeros_likecpunumpytolistinttorch from_numpyr"longrJ unsqueezer[rLappendlogits_aggregation distributions Bernoulliprobsfloat32 collections defaultdictr& enumeratesqueezearraymeancattupleint32 expand_dimsr=mathsigmoidcastconcat)r-rS all_logitsall_aggregations prev_answers batch_sizerWrXrYtoken_type_ids_exampleindexprev_labels_example model_labelsi segment_idcol_idrow_idinput_ids_exampleattention_mask_exampleoutputsr[dist_per_token probabilitiescoords_to_probspcolrowkey logits_batchs rsequential_inferencez3TableQuestionAnsweringPipeline.sequential_inferences >>T !J! L ,2215J{+..t{{;I#$4588EN#$4588EN%) "z*. m +*@A*F'#%==1D1H1H1J1P1P1R#SL-;E-B*"<#5#5a#89R%;AqD%A%H%H%J1%M !71!=!D!D!Fq!IA!M!71!=!D!D!Fq!IA!M!Q;6Q;:?.1,?O2P.QLO R493C3CL3Q3V3VW\WaWa3b3e3efjfqfq3r*1a40$-e$4!)7)>&)7)>&**/99!<#9#C#CA#F#9#C#CA#F% !>>$++G,F,FG!!&)!&!4!4!>!>f!>!M . 4 47M7R7RSXS`S`7a7d7d"((//8! #."9"9$"?%m&;&;&=&D&D&FG>DAq!71!=!D!D!Fq!IJ0A6==?BQFC0A6==?BQFCaxC1Hq'c 3::1= >]llUXRXXoc.B%C%H%H%JS%P Pl l]. m`!99U:%6:L*...L? s|UYYW\]mWnpqMr>s sJ! L ,2215J{+I#$45N#$45;;=N%) "z*. m +*@A*F'#%==1DBHH#UL-;E-B*"<#5#5a#89R%;AqD%A%H%H%J1%M !71!=!D!D!Fq!IA!M!71!=!D!D!Fq!IA!M!Q;6Q;:?.1,?O2P.QLO R4@*1a40$-e$4!)7)>&)7)>&** nn->QG#%>>2Hq#Q#%>>2Hq#Q% !>>$++G,F,FG!!&) " 0K Lrww*BJJP! #."9"9$"?)?&%bjj&?&E&E&G&N&N&PQ>DAq!71!=!D!D!Fq!IJ0A6==?BQFC0A6==?BQFCaxC1Hq'c 3::1= >]llUXRXXoc.B%C%H%H%JS%P Pl l]. m`99U:%6:L*...L? s|RYYW\]mWnpqMr>s sE m| ms 1^1^cr|j|i|}t||fi|}t|dk(r|dS|S)a Answers queries according to a table. The pipeline accepts several types of inputs which are detailed below: - `pipeline(table, query)` - `pipeline(table, [query])` - `pipeline(table=table, query=query)` - `pipeline(table=table, query=[query])` - `pipeline({"table": table, "query": query})` - `pipeline({"table": table, "query": [query]})` - `pipeline([{"table": table, "query": query}, {"table": table, "query": query}])` The `table` argument should be a dict or a DataFrame built from that dict, containing the whole table: Example: ```python data = { "actors": ["brad pitt", "leonardo di caprio", "george clooney"], "age": ["56", "45", "59"], "number of movies": ["87", "53", "69"], "date of birth": ["7 february 1967", "10 june 1996", "28 november 1967"], } ``` This dictionary can be passed in as such, or can be converted to a pandas DataFrame: Example: ```python import pandas as pd table = pd.DataFrame.from_dict(data) ``` Args: table (`pd.DataFrame` or `Dict`): Pandas DataFrame or dictionary that will be converted to a DataFrame containing all the table values. See above for an example of dictionary. query (`str` or `List[str]`): Query or list of queries that will be sent to the model alongside the table. sequential (`bool`, *optional*, defaults to `False`): Whether to do inference sequentially or as a batch. Batching is faster, but models like SQA require the inference to be done sequentially to extract relations within sequences, given their conversational nature. padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`): Activates and controls padding. Accepts the following values: - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence if provided). - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum acceptable input length for the model if that argument is not provided. - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different lengths). truncation (`bool`, `str` or [`TapasTruncationStrategy`], *optional*, defaults to `False`): Activates and controls truncation. Accepts the following values: - `True` or `'drop_rows_to_fit'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum acceptable input length for the model if that argument is not provided. This will truncate row by row, removing rows from the table. - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths greater than the model maximum admissible input size). Return: A dictionary or a list of dictionaries containing results: Each result is a dictionary with the following keys: - **answer** (`str`) -- The answer of the query given the table. If there is an aggregator, the answer will be preceded by `AGGREGATOR >`. - **coordinates** (`List[Tuple[int, int]]`) -- Coordinates of the cells of the answers. - **cells** (`List[str]`) -- List of strings made up of the answer cell values. - **aggregator** (`str`) -- If the model has an aggregator, this returns the aggregator. rr)rCrAr2r')r-rOr.pipeline_inputsresultsrQs rr2z'TableQuestionAnsweringPipeline.__call__ sLV,$++Ti}|||d<|||d<i}|||d<||ifS)Npadding truncation sequentialr8)r-rrrr.preprocess_paramsforward_paramss r_sanitize_parametersz3TableQuestionAnsweringPipeline._sanitize_parameters_sL  +2 i (  !.8 l +  !+5N< ( ."44r3c||jdk(rd}nd}|d|d}}|jr td||dk(r td|j|||j|| }||d<|S) Nr@drop_rows_to_fitdo_not_truncaterrztable is emptyzquery is empty)return_tensorsrr)r"emptyr$ tokenizerrD)r-pipeline_inputrrrrrrSs r preprocessz)TableQuestionAnsweringPipeline.preprocessks  yyG#/ . %g.w0Gu ;;-. . =ERK-. .uT^^Xblstw r3c |jd}|jdk(r(|r|jdi|}nD|jdi|}n1d|vr|j|d<|j j di||}|||d}|S)Nrr@generation_config) model_inputsrrr8)popr"rrTrrJgenerate)r-rrgenerate_kwargsrr model_outputss r_forwardz'TableQuestionAnsweringPipeline._forward{s  ) 99 3$33ClC.$..>>#/97;7M7M 34)djj))LLLOLG)5RYZ r3c>|d}|d}|d}|jdk(r|jr|dd\}}|jj|||}|\}} t | D cic])\} } | |j j j| +} } } |j j j} t | D cic]\} } | | k7s | | | dz}} } n*|d}|jj||}|d}i} i}g}t |D]\}}|Dcgc]}|j|}}| j|d}|j|d}|d j|z||Dcgc]}|j|c}d }|r||d <|j|tdk(r7td |jj|d Dcgc]}d|i}}t|dkDr|S|dScc} } wcc} } wcc}wcc}wcc}w)Nrrrr@rz > rrz, )answer coordinatescells aggregatorz Empty answerT)skip_special_tokensrr)r"rLrconvert_logits_to_predictionsrtrJrKr>no_aggregation_label_indexiatr%joinrlr'r batch_decode)r-rrSrrr[ logits_agg predictionsanswer_coordinates_batchagg_predictionsrpred aggregatorsno_agg_label_indexaggregators_prefixanswersrr coordinaterraggregator_prefixrs r postprocessz*TableQuestionAnsweringPipeline.postprocesssS~.g& * 99 ~~%,Ra[" "nnJJ6SY[ef ( %'"G&/0H&I '"{ALM::.MM(__UB7 $6$:$:5"$E!/$))E2BB#.FQR eii 3R +5F<(v& '6{a'778<8S8ST[qu8S8vwf&)wGwg,*w: :?x&N Sxs$$.H H  H !H5H " H)NNN)NTN)F)r4r5r6r7default_input_namesrrBrTrr2rrrr __classcell__)rQs@rr;r;UsD<(#H#JZ"$@tDPd 5 "(;r3r;)rrr*rerbutilsrrrrbaser r r r r rhmodels.auto.modeling_autorr tensorflowr=models.auto.modeling_tf_autorrrr;r8r3rrsx  ba  2#O2#j,4@A^;X^;B^;r3