sg!dZddlZddlmZmZmZddlmZddl Z ddl m Z ddl Z ddlmZmZmZddlmZmZddlmZddlmZddlmZmZdd lmZd d lmZmZm Z m!Z!m"Z"d d l#m$Z$m%Z%m&Z&m'Z'm(Z(d d l)m*Z*m+Z+m,Z,m-Z-ddl.m/Z/e,j`e1Z2dZ3dZ4ejjZ5de jlde7de7de jlfdZ8GddejrZ:GddejrZ;GddejrZ<GddejrZ=Gdd ejrZ>Gd!d"ejrZ?Gd#d$ejrZ@Gd%d&ejrZAGd'd(ejrZBGd)d*ejrZCGd+d,ejrZDd-ZEd.ZFd/ZGGd0d1e%ZHd2ZIe*d3eIGd4d5ejrZJGd6d7eHZKe&eKe3e"e4d8ZLe(eKeGeLze'eKe!e49e*d:eIGd;deHZNe*d?eIGd@dAejrZOGdBdCeHZPdDZQe(ePeGeQze'ePe!e49y)EzFlax T5 model.N)CallableOptionalTuple) FrozenDictfreezeunfreeze) combine_masksmake_causal_mask) partitioning)dot_product_attention_weights) flatten_dictunflatten_dict)PRNGKey)FlaxBaseModelOutput-FlaxBaseModelOutputWithPastAndCrossAttentions%FlaxCausalLMOutputWithCrossAttentionsFlaxSeq2SeqLMOutputFlaxSeq2SeqModelOutput)ACT2FNFlaxPreTrainedModelappend_call_sample_docstring append_replace_return_docstringsoverwrite_call_docstring)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)T5Configzgoogle-t5/t5-smallr input_ids pad_token_iddecoder_start_token_idreturnctj|}|jddddfj|ddddf}|jdddfj|}tj|dk(||}|S)z1 Shift input ids one token to the right. Nrri)jnp zeros_likeatsetwhere)r!r"r#shifted_input_idss Z/var/www/html/venv/lib/python3.12/site-packages/transformers/models/t5/modeling_flax_t5.pyshift_tokens_rightr.:sy1),,QU377 !SbS&8IJ),,QT2667MN "3t";\K\] ceZdZUeed<ej Zejed<dZe ed<e jjjZedej"fed<dZdZy ) FlaxT5LayerNorm hidden_sizedtypegư>eps. weight_initc^|jd|j|jf|_y)Nweight)paramr5r2r7selfs r-setupzFlaxT5LayerNorm.setupLs%jj4+;+;d>N>N=PQ r/ctj|jddjdd}|tj||j zz }|j |zS)zc Construct a layernorm module in the T5 style; No bias and no subtraction of mean. f4r&T)axiskeepdims)r'powerastypemeansqrtr4r7)r: hidden_statesvariances r-__call__zFlaxT5LayerNorm.__call__Os[ 99]11$7;@@bSW@X%DHH1D(EE {{]**r/N)__name__ __module__ __qualname__int__annotations__r'float32r3r4floatjaxnn initializersonesr5rnpndarrayr;rGr/r-r1r1FsV{{E399"C-0VV-@-@-E-EK#rzz/*ER+r/r1c\eZdZUeed<ej Zejed<dZddZ y)FlaxT5DenseActDenseconfigr3c|jj|jjdzz}|jj|jjdzz}t j |jjdt jjj||j|_ t j |jjdt jjj||j|_ t j|jj|_t|jj |_yNFuse_bias kernel_initr3)rXinitializer_factord_modeld_ffrPDenserOrQnormalr3wiwoDropout dropout_ratedropoutr dense_act_fnactr: wi_init_std wo_init_stds r-r;zFlaxT5DenseActDense.setup^skk44 8K8KT8QR kk44 8H8H$8NO (( KK  ++22;?**   (( KK  ++22;?**   zz$++":":; $++223r/c|j|}|j|}|j||}|j|}|SN deterministic)rdrjrhre)r:rErqs r-rGzFlaxT5DenseActDense.__call__qsD . /  ]- P  . r/NT rHrIrJr rLr'rMr3r;rGrUr/r-rWrWZs$ {{E399"4&r/rWcZeZdZUeed<ej Zejed<dZdZ y)FlaxT5DenseGatedActDenserXr3c|jj|jjdzz}|jj|jjdzz}t j |jjdt jjj||j|_ t j |jjdt jjj||j|_ t j |jjdt jjj||j|_ t j|jj|_t |jj"|_yrZ)rXr_r`rarPrbrOrQrcr3wi_0wi_1rerfrgrhrrirjrks r-r;zFlaxT5DenseGatedActDense.setup}s;kk44 8K8KT8QR kk44 8H8H$8NO HH KK  ++22;?**   HH KK  ++22;?**   (( KK  ++22;?**   zz$++":":; $++223r/c|j|j|}|j|}||z}|j||}|j |}|Sro)rjrwrxrhre)r:rErq hidden_gelu hidden_linears r-rGz!FlaxT5DenseGatedActDense.__call__sWhhtyy78  -0 #m3  ]- P  . r/NrsrUr/r-ruruys$ {{E399"42r/ruc\eZdZUeed<ej Zejed<dZddZ y) FlaxT5LayerFFrXr3c|jjr't|j|j|_n&t |j|j|_t |jj|jj|j|_ tj|jj|_ y)Nr3r4r3)rX is_gated_actrur3DenseReluDenserWr1r`layer_norm_epsilon layer_normrPrfrgrhr9s r-r;zFlaxT5LayerFF.setups ;; # #":4;;djj"YD "5dkk"TD )$++*=*=4;;CaCaimisistzz$++":":; r/cz|j|}|j||}||j||z}|Sro)rrrh)r:rErqforwarded_statess r-rGzFlaxT5LayerFF.__call__sG??=9../?}.]% 5EUb (cc r/NrrrsrUr/r-r}r}s$ {{E399"<r/r}ceZdZUeed<dZeed<dZeed<ejZ ejed<dZ e ddZ dZd Zd Zej$d Zd Z ddZy )FlaxT5AttentionrXFhas_relative_attention_biascausalr3c*|jj|_|jj|_|jj|_|jj|_|jj |_|jj|_ |j|j z|_ |jj|j|j zdzz}|jj|jdzz}|jj|jdzz}tj|jdtjjj!||j"|_tj|jdtjjj!||j"|_tj|jdtjjj!||j"|_tj|jdtjjj!||j"|_|j,rdtj.|j|jtjjj!||j"|_yy)Nr[Fr\embedding_initr3)rXrelative_attention_num_bucketsrelative_attention_max_distancer`d_kvkey_value_proj_dim num_headsn_headsrgrh inner_dimr_rPrbrOrQrcr3qkvorEmbedrelative_attention_bias)r: q_init_std kv_init_std o_init_stds r-r;zFlaxT5Attention.setups.2kk.X.X+/3{{/Z/Z,{{** "&++"2"2{{,, {{// (?(??[[33I`I`8`ei7ij kk448LM [[33t~~t7KL  NN++22:>**    NN++22;?**    NN++22;?**    LL++22:>**    + ++-8833 "vv2299+Fjj ,D ( ,r/cd}|r&|dz}||dkD|zz }tj|}ntj|d }|dz}||k}|tj||z tj||z z ||z zz}tj||dz }|tj|||z }|j dS)av Adapted from Mesh Tensorflow: https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 Translate relative position to a bucket number for relative attention. The relative position is defined as memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for small absolute relative_position and larger buckets for larger absolute relative_positions. All relative positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. This should allow for more graceful generalization to longer sequences than the model has been trained on rr>)a_maxri4)r'abscliplogr+rB)relative_position bidirectional num_buckets max_distancerelative_buckets max_exactis_smallrelative_position_if_larges r-_relative_position_bucketz)FlaxT5Attention._relative_position_buckets  A K !2Q!6+ E E  #(9 : !$*;1!E E  1$ $y0&/ GG% 1 2SWW\I=U5V VZehqZq r& "&)XX.HP[^_P_%`"CIIh0AC]^^&&t,,r/cNtj|ddddf}tj|ddddf}||z }|j||j |j|j }|j |}|jddddddddf}|S)z%Compute binned relative position biasrrN)rrr)r>rr)r'arangerrrrr transpose)r: query_length key_lengthcontext_positionmemory_positionrrelative_position_bucketvaluess r- compute_biaszFlaxT5Attention.compute_bias s::l$?4H**Zt>#'#A#A #{{?;;== $B$  --.FG!!),T1a]; r/cp|j|jdd|j|jfzSNr>)reshapeshaperrr:rEs r- _split_headszFlaxT5Attention._split_headss4$$]%8%8!%< dNeNe?f%fggr/cZ|j|jdd|jfzSr)rrrrs r- _merge_headszFlaxT5Attention._merge_headss,$$]%8%8!%<?P%PQQr/cR|jdd}|jddtj|j|j }|jddtj|j|j }|jddd}|r |j j^} } } } |j } dt| z| ddfz}tjj|j ||}tjj|j ||}||_||_|jd}|j |z|_tjtj| | |zkt| d|| fz}t||}|||fS) a[ This function takes projected key, value states from a single input token and concatenates the states to cached states from previous steps. This function is slighly adapted from the official Flax repository: https://github.com/google/flax/blob/491ce18759622506588784b4fca0e4bf05f8c8cd/flax/linen/attention.py#L252 cache cached_key cached_value cache_indexcLtjdtjS)Nrr)r'arrayint32rUr/r-z7FlaxT5Attention._concatenate_to_cache..-sCIIaWZW`W`Dar/)rrr) has_variablevariabler'zerosrr3valuelenrOlaxdynamic_update_slice broadcast_tortupler )r:keyrqueryattention_maskis_initializedrrr batch_dims max_lengthrdepth_per_head cur_indexindicesnum_updated_cache_vectorspad_masks r-_concatenate_to_cachez%FlaxT5Attention._concatenate_to_cache"s|**7LA]]7L#))SYYPSPYPYZ }}WnciiV[VaVab mmG]ZY#))IS_, 1a/@@G''..z/?/?gNCGG001C1CUGTE"J !&L (- A % + 1 14M MK '' :&5N)NNj!Q(A:$NNH+8^DNE>))r/c|jxr|jddxr| }|jd}|r|n|jd} |jr|j | |} nG|t j |} n/t jd|j| |f|j} |rR|jddjd} tjj| dd|dfd|j|| f} | S)Nrrrrr)rrrrrr'r(rrr3 variablesrOr dynamic_slice) r: key_states query_statesr init_cache seq_lengthcausal_attention_mask_shiftcache_is_filledrr position_biasmax_decoder_lengths r-_create_position_biasz%FlaxT5Attention._create_position_biasCs++g$*;*;G\*Rg\fXf%%a( %4z,:L:LQ:O  + + --lJGM  'NN>:MIIq$,, j&QY]YcYcdM !%!8!F!L!LQ!O GG11A2A6DLL*.@AM r/Nc "|jdd\} } |j|} ||j|n|j|} ||j|n|j|} |j | } |j | } |j | } | t j | jdz} |jddr|jr|jddnd}|jrt|d }|jddrH|jddjd }tjj|dd|dfd d | |f}t j|| f|jd dz}t jt j|d |j}t!||}n|t j|d }|jr,|jdds|r|j#| | | |\} } }|t j$|j&j(}tjj+|dkDt j,|jd j/|j&t j,|j|j/|j&}||j1| | ||| |}|||z}d}|s |j2d kDr|j5d}t7| | |||j2d||j&}t j8d|| }|j;|}|j=|}||f}|r||fz}|S)z Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states). Nr>r&rrrrboolrr))r?grhT)bias dropout_rngrgbroadcast_dropoutrqr3z...hqk,...khd->...qhd)rrrrrr'rDrrrr rOrrr expand_dimsr rfinfor3minselectfullrBrrhmake_rngr einsumrr)r:rErkey_value_statesr use_cacheoutput_attentionsrqr batch_sizerrr value_statesrcausal_attention_maskr mask_valuer attn_weights attn_outputoutputss r-rGzFlaxT5Attention.__call__[s|"/!4!4Ra!8 Jvvm, .>.FTVVM*DFFScLd 0@0Htvvm,dffUeNf ((6 &&z2 ((6  !3!3B!788 8<7H7HR^7_dhdodoDNN7 #M 2vw $ ;;$4^6$R !  ,7%)^^G%<\%J%P%PQR%S"(+(=(=)6::'9:)%%($4$4% }7L7R7RSTSU7V'V% !!--X>@U@[@[N+>;PQN  ' __^(KN ;;D--g|D 7;7Q7QL,8 4J n  %4::.22J WW^^"--s3::4::F--z:AA$**MN   66L.*jRmM) - >  !3-- 2K5  #"'**  jj!8, U '' 4 ff[)  . /Gr/)T )NNNFFTF)rHrIrJr rLrrrr'rMr3r; staticmethodrrrrrPcompactrrrGrUr/r-rrs (--FD{{E399",\!-!-F"hRZZ**@6qr/rcteZdZUeed<dZeed<ejZ ejed<dZ ddZ y) FlaxT5LayerSelfAttentionrXFrr3cvt|j|j|jj|j|_t |jj|jj|j|_ tj|jj|_ y)Nrrr3r)rrXrrr3 SelfAttentionr1r`rrrPrfrgrhr9s r-r;zFlaxT5LayerSelfAttention.setupsz, KK(,(H(H;;%%**   *$++*=*=4;;CaCaimisistzz$++":":; r/Nc|j|}|j||||||}||j|d|z}|f|ddz} | S)Nrrrrqrrrpr)rr rh) r:rErrrrqrnormed_hidden_statesattention_outputrs r-rGz!FlaxT5LayerSelfAttention.__call__su $}=-- )'/'! . & 5Ea5HXe (ff  "%5ab%99r/)NNFTF rHrIrJr rLrrr'rMr3r;rGrUr/r-rrs@ (--{{E399"<r/rcdeZdZUeed<ej Zejed<dZ ddZ y)FlaxT5LayerCrossAttentionrXr3c:t|jdd|j|_t |jj |jj |j|_tj|jj|_ y)NFrr) rrXr3EncDecAttentionr1r`rrrPrfrgrhr9s r-r;zFlaxT5LayerCrossAttention.setupsj. KKU5PTPZPZ *$++*=*=4;;CaCaimisistzz$++":":; r/Nc|j|}|j|||||}||j|d|z}|f|ddz} | S)N)rrrrrrpr)rrrh) r:rErrrrrqr r rs r-rGz"FlaxT5LayerCrossAttention.__call__sr $}=// )-'/ 0 & 5Ea5HXe (ff  "%5ab%99r/)NNFTrsrUr/r-rrs2 {{E399"<r/rc|eZdZUeed<dZeed<ejZ ejed<dZ ddZ y) FlaxT5BlockrXFrr3c|jj|_t|j|jt d|j f|_d}|jrD|xj t|jt d|j fz c_|dz }|xj t|jt ||j fz c_y)Nr)rnamer3r)rr3) rXrrrstrr3layerrr})r:feed_forward_indexs r-r;zFlaxT5Block.setupskk(( $ ,0,L,LVjj     ;; JJ4T[[s1vUYU_U_`b bJ ! #  }T[[s;M7NVZV`V`acc r/Nc |jd||||| | } | d}| dd} |jxr|du} | r(|jd|||||| }|d}| |ddz} |jd|| }|f}|| z}|S)Nrr r)rrrrrqr&rp)rr)r:rErrencoder_hidden_statesencoder_attention_maskencoder_decoder_position_biasr return_dictrqrself_attention_outputsattention_outputsdo_cross_attentioncross_attention_outputsrs r-rGzFlaxT5Block.__call__,s"/A )'/'! " /q1 2126![[N-B$-N &3djjm!65;"3+ ' #4A6M!24KAB4O O ' 2}MR  "--r/) NNNNNFTTFrrUr/r-rrsM (--{{E399"d("#&*0r/rcveZdZUeed<eed<ejZejed<dZ ddZ y)FlaxT5LayerCollectionrXrr3cft|j|j|j|_y)N)rr3)rrXrr3rr9s r-r;zFlaxT5LayerCollection.setupds& KKT5U5U]a]g]g  r/Nc 6|j|||||||||  S)N)rrrrrrrqr)r) r:rErrrrrrrqrs r-rGzFlaxT5LayerCollection.__call__is5zz )'"7#9*G/'!  r/)NNNNNFTF) rHrIrJr rLrr'rMr3r;rGrUr/r-r%r%_sD !%%{{E399" "#&* r/r%c eZdZUeed<ej Zejed<dZe ed<dZ d de de d e d e fd Z y) FlaxT5BlockCollectionrXr3Fgradient_checkpointingc |jj|_|jrnttd}t |jj Dcgc].}||j|dk(|jt|0c}|_ yt |jj Dcgc]1}t |j|dk(|jt|3c}|_ ycc}wcc}w)N))static_argnumsr)rr3r) rXrr*rematr%range num_layersr3rblocks)r:FlaxT5CheckpointLayeris r-r;zFlaxT5BlockCollection.setupskk((  & &$)*?PY$Z !t{{556 &KK12a**Q DK"t{{556 &KK12a**Q DKs 3C676C;Nroutput_hidden_statesrqrc T|rdnd} |rdnd} |r|jrdnd} d} d} t|jD]`\}}|r| |fz} |||| ||| ||| }|d}|d} |jr | ||rdnd} |sB| |dfz} |jsX| |dfz} bt|| | | S)NrUrrrr>last_hidden_staterE attentionscross_attentions)r enumerater3r)r:rErrrrr6rqrall_hidden_statesall_attentionsall_cross_attentionsrrr5 layer_module layer_outputss r-rGzFlaxT5BlockCollection.__call__s#7BD0d&7DKKrd (,%(5 VOA|#$58H$H!(%&-! M*!,M *!,M{{4@0=CTaZ[0\- !/=3C2E!E;;+?=QRCSBU+U(; V>=++%1   r/)NNNNFFTF rHrIrJr rLr'rMr3r*rr;rGrUr/r-r)r)sq {{E399"#(D(4"#"'%*" 6  6 #6 6 6 r/r)c eZdZUeed<ej ed<ejZ ejed<dZ e ed<dZ dde d e d e d e d e f d Z y) FlaxT5StackrX embed_tokensr3Fr*c|jj|_t|j|j|j|_t |jj|jj|j|_ tj|jj|_ y)Nr3r*r)rXrr)r3r*blockr1r`rfinal_layer_normrPrfrgrhr9s r-r;zFlaxT5Stack.setups~kk(( * KKtzz$B]B]  !0 KK  T[[%C%C4::! zz$++":":; r/Nrr6rrqrc n|j|} |j| |} |j| ||||||| } | d} |j| } |j| |} d} |r| j} | | fz} |s|r | | f| ddzS| f| ddzSt | | | j | jS)Nrp)rrrrr6rqrrr>rr9)rFrhrIrJrErr;r<) r:r!rrrrr6rrqrrErr>s r-rGzFlaxT5Stack.__call__s)))4  ]- P ** )"7#9/!5'!    --m<  ]- P !  ' 5 5  1]4D D #!%ABK  "#gabk1 1<++))$55   r/) NNNNFFTTF)rHrIrJr rLrPrr'rMr3r*rr;rGrUr/r-rErEs (({{E399"#(D( <"#"'%* " 3  3 #3 3 3 3 r/rEa Args: input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. a1 Args: decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) For training, `decoder_input_ids` should be provided. encoder_outputs (`tuple(tuple(jnp.ndarray)`): Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. past_key_values (`Dict[str, np.ndarray]`, *optional*, returned by `init_cache` or when passing previous `past_key_values`): Dictionary of pre-computed hidden-states (key and values in the attention blocks) that can be used for fast auto-regressive decoding. Pre-computed key and value hidden-states are of shape *[batch_size, max_length]*. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. a Args: input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. [What are input IDs?](../glossary#input-ids) To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training). decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. encoder_outputs (`tuple(tuple(jnp.ndarray)`, *optional*): Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. past_key_values (`tuple(tuple(jnp.ndarray))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. ceZdZUdZeZdZdZeje d<dde jddfd ed e ed ed e jd edef fd ZdZd%dej(j*d e dedefdZee d&de j4dee j4de j4dee j4deedeedeedededefdZdZee e!e"e  d'de j4dee j4deedeedeedededefd!Z#ee$e!e%e  d&d"ee j4dee j4d#edeedeedeedededefd$Z&xZ'S)(FlaxT5PreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. transformerN module_class)rrrTFrX input_shapeseedr3_do_initr*c \|jd|||d|}t | ||||||y)NrXr3r*)rPrQr3rRrU)rOsuper__init__) r:rXrPrQr3rRr*kwargsmodule __class__s r-rVzFlaxT5PreTrainedModel.__init__sA#""w&Vlwpvw [tSXcklr/c^|j|j|jd|_y)NTrT)rOrXr3_moduler9s r-enable_gradient_checkpointingz3FlaxT5PreTrainedModel.enable_gradient_checkpointings*((;;**#')  r/rngparamsr$c`tj|d}tj|}||g}|jtfvr=tj|}tj|}|j ||gt jj|\} } | | d} |jj| g|d} |dtt| } tt|}|jD] } | | || < t|_ tt!|S| S)Nrr)r^rhr^)r'r ones_likerOFlaxT5EncoderModuleextendrOrandomsplitrXinitr r _missing_keysr*rr)r:r]rPr^r!rargsdecoder_input_idsdecoder_attention_mask params_rngrrngs random_params missing_keys r- init_weightsz"FlaxT5PreTrainedModel.init_weightss'IIk6 y1>*   %8$9 9 # i 8 %(]]9%= " KK*,BC D"%**"2"23"7 K$=( ((      (-)@AM!(6"23F#11 A &3K&@{# A!$D .01 1 r/r!rrhrirr6rtrainrc 8||n|jj}||n|jj}||n|jj}| t d|t j |}|t j |}| d| ini} |jjd| xs |jit j|dt j|dt j|dt j|d|||| |  S)NzfMake sure to provide both `input_ids` and `decoder_input_ids`. `decoder_input_ids` is not passed here.rhr^rr) r!rrhrirr6rrqrk) rXrr6r ValueErrorr'r`rXapplyr^r) r:r!rrhrirr6rror^rrks r-rGzFlaxT5PreTrainedModel.__call__s#2C1N-TXT_T_TqTq$8$D $++JjJj &1%._decoder_forward-#779N!!& r/rT)rhrirrmethodr) r'rRr`rXrerOrcrr)r:rrencoder_outputsrhrirzinit_variabless r-rz FlaxT5PreTrainedModel.init_cache s HHj*%=TJ!$/@!A )) JJ  q !/#9"1!"4# * w/00r/ output_type config_classc ||n|jj}||n|jj}||n|jj}|t j |}i} ||| d<d} |j jd|xs |jit j|dt j|d|||| | |  S)a Returns: Example: ```python >>> from transformers import AutoTokenizer, FlaxT5ForConditionalGeneration >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small") >>> model = FlaxT5ForConditionalGeneration.from_pretrained("google-t5/t5-small") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, return_tensors="np") >>> encoder_outputs = model.encode(**inputs) ```rhc6|j}|||fi|Sru)_get_encoder_module)rXr!rrW encode_modules r-_encoder_forwardz6FlaxT5PreTrainedModel.encode.._encoder_forwardYs""668M NEfE Er/r^rr)r!rrr6rrqrkr| rXrr6rr'r`rXrrr^r) r:r!rrr6rror^rrkrs r-encodezFlaxT5PreTrainedModel.encode/s82C1N-TXT_T_TqTq$8$D $++JjJj &1%>> from transformers import AutoTokenizer, FlaxT5ForConditionalGeneration >>> import jax.numpy as jnp >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small") >>> model = FlaxT5ForConditionalGeneration.from_pretrained("google-t5/t5-small") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, return_tensors="np") >>> encoder_outputs = model.encode(**inputs) >>> decoder_start_token_id = model.config.decoder_start_token_id >>> decoder_input_ids = jnp.ones((inputs.input_ids.shape[0], 1), dtype="i4") * decoder_start_token_id >>> outputs = model.decode(decoder_input_ids, encoder_outputs) >>> logits = outputs.logits ```Nrr>rhr^rFc6|j}|||fi|Srurvrxs r-rzz6FlaxT5PreTrainedModel.decode.._decoder_forwardr{r/rr rhrirrrr6rrqrkmutabler|rr) rXrr6rrr'rRr^rXrrrr)r:rhr}rrirrr6rror^rrrsequence_lengthrkinputsrrzrpasts r-decodezFlaxT5PreTrainedModel.decodeisL2C1N-TXT_T_TqTq$8$D $++JjJj &1%>LGr/ru NNNNNNFNNNNNNFNN)(rHrIrJ__doc__r rbase_model_prefixrOrPModulerLr'rMrrKr3rrVr\rOrcrrrnrT5_INPUTS_DOCSTRINGrTrdictrGrrT5_ENCODE_INPUTS_DOCSTRINGrrrT5_DECODE_INPUTS_DOCSTRINGrr __classcell__)rYs@r-rMrMs L%"L"))" #);;', m m3Z m m yy m  m!% m ! 2 2!!PZ!fp!:++>?15)-8<,0/3&*#/ ;;/ !-/ ;; / !) 5 / $D> / 'tn/ d^/ / / / @/ b"1H45+>XV15,0/3&*#6 ;;6 !-6 $D> 6 'tn 6 d^ 6 6 6 6 W66 p45+Xgop 9=8< $,0/3&*#c!) 5 c !) 5 c  c$D>c'tncd^ccccq6cr/rMa The T5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a text-to-text denoising generative setting. This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior. Finally, this model supports inherent JAX features such as: - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit) - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation) - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap) - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap) Parameters: config ([`T5Config`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights. dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`): The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and `jax.numpy.bfloat16` (on TPUs). This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If specified all the computation will be performed with the given `dtype`. **Note that this only specifies the dtype of the computation and does not influence the dtype of model parameters.** If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`]. z[The bare T5 Model transformer outputting raw hidden-stateswithout any specific head on top.ceZdZUeed<ej Zejed<dZe ed<dZ dZ dZ d d e fd Z y) FlaxT5ModulerXr3Fr*c|jSruencoderr9s r-rz FlaxT5Module._get_encoder_module ||r/c|jSrudecoderr9s r-rwz FlaxT5Module._get_decoder_modulerr/ctj|jj|jjt jj j|jjdz|j|_ tj|j}d|_ t||j|j|j|_tj|j}d|_ |jj"|_t||j|j|j|_y)N?rFrFr3r*T)rPrrX vocab_sizer`rOrQrcr_r3sharedcopydeepcopyrrEr*rnum_decoder_layersr2rr:encoder_configdecoder_configs r-r;zFlaxT5Module.setupshh KK " " KK  66..55dkk6T6TWZ6Z[**   t{{3 %" **#'#>#>   t{{3 $$(KK$B$B!" **#'#>#>   r/Nrqc p||n|jj}|j|||||| }|j|||d||||| } |s| |zSt | j | j | j| j| j|j |j|jS)Nr!rrr6rrqrr!rrrrr6rrq)r:rdecoder_hidden_statesdecoder_attentionsr<encoder_last_hidden_staterencoder_attentions) rXuse_return_dictrrrr:rrEr;r<) r:r!rrhrir}rr6rrqdecoder_outputss r-rGzFlaxT5Module.__call__#s&1%>> from transformers import AutoTokenizer, FlaxT5Model >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small") >>> model = FlaxT5Model.from_pretrained("google-t5/t5-small") >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="np" ... ).input_ids >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="np").input_ids >>> # preprocess: Prepend decoder_input_ids with start token which is pad token for T5Model. >>> # This is not needed for torch's T5ForConditionalGeneration as it does this internally using labels arg. >>> decoder_input_ids = model._shift_right(decoder_input_ids) >>> # forward pass >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) >>> last_hidden_states = outputs.last_hidden_state ``` rzfThe bare T5 Model transformer outputting encoder's raw hidden-states without any specific head on top.ceZdZUeed<ej Zejed<dZe ed<dZ d de de fd Z y) rarXr3Fr*ctj|jj|jjt jj j|jjdz|j|_ tj|j}d|_ d|_d|_t!||j|j|j"|_y)NrrFr)rPrrXrr`rOrQrcr_r3rrr is_decoderis_encoder_decoderrrEr*r)r:rs r-r;zFlaxT5EncoderModule.setupshh KK " " KK  66..55dkk6T6TWZ6Z[**   t{{3$)!,1) %" **#'#>#>   r/Nrrqc4|j||||||}|S)Nrr)r:r!rrr6rrqr}s r-rGzFlaxT5EncoderModule.__call__s1,,)/!5#' ' r/)NNFFTTrCrUr/r-rara{sY {{E399"#(D( *" "  r/raceZdZeZee d dejde ejde e de e de e de de d e fd Z y) FlaxT5EncoderModelNr!rrr6rror^rc ||n|jj}||n|jj}||n|jj}|t j |}|d|ini} |j jd|xs |jit j|dt j|d|||| | S)Nrhr^rr)r!rrr6rrqrkr) r:r!rrr6rror^rrks r-rGzFlaxT5EncoderModel.__call__s2C1N-TXT_T_TqTq$8$D $++JjJj &1% ! 'tn ! d^ ! ! ! ! G! r/rz0T5 Model with a `language modeling` head on top.ceZdZUeed<ej Zejed<dZe ed<dZ dZ dZ d d e fd Z y) $FlaxT5ForConditionalGenerationModulerXr3Fr*c|jSrurr9s r-rz8FlaxT5ForConditionalGenerationModule._get_encoder_modulerr/c|jSrurr9s r-rwz8FlaxT5ForConditionalGenerationModule._get_decoder_modulerr/c|jj|_tj|jj |jjt jjj|jj|j|_ tj|j}d|_d|_d|_t#||j|j|j$|_tj|j}d|_d|_|jj(|_t#||j|j|j$|_tj.|jj dt jjj|jj|j|_y)NrFrHTr\)rXr` model_dimrPrrrOrQrcr_r3rrrrrrrEr*rrr2rrblm_headrs r-r;z*FlaxT5ForConditionalGenerationModule.setups[,,hh KK " " KK  66..55dkk6T6TU**   t{{3 %#( ,1)" DKKtzzRVRmRm  t{{3 $,1)$(KK$B$B!" DKKtzzRVRmRm  xx KK " "++224;;3Q3QR**   r/Nrqc ||n|jj}|j|||||| }|d} |j||| ||||| } | d} |jjr| |j dzz} |jjrG|j jdd} |jjdd| jii| }n|j| }|s |f| ddz|zSt|| j| j| j| j|j |j|j S) Nrrrr[r^ embeddingkernelr)logitsrrrr<rrr)rXrrrtie_word_embeddingsrrrrrrTrrrEr;r<r:)r:r!rrhrir}rr6rrqrErsequence_outputshared_embedding lm_logitss r-rGz-FlaxT5ForConditionalGenerationModule.__call__sp&1%{K  **HxAQASAS6T+UWfgI _5I>> from transformers import AutoTokenizer, FlaxT5ForConditionalGeneration >>> import jax.numpy as jnp >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small") >>> model = FlaxT5ForConditionalGeneration.from_pretrained("google-t5/t5-small") >>> text = "summarize: My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, return_tensors="np") >>> encoder_outputs = model.encode(**inputs) >>> decoder_start_token_id = model.config.decoder_start_token_id >>> decoder_input_ids = jnp.ones((inputs.input_ids.shape[0], 1), dtype="i4") * decoder_start_token_id >>> outputs = model.decode(decoder_input_ids, encoder_outputs) >>> logits = outputs.logits ```Nrr>rhr^rFc|j}|||fi|}|d} jjr| jjdzz} jjrJ|jj dd}|j jdd|jii|}||fS|j |}||fS)Nrr[r^rr) rwrXrr`rrrrrr) rXrhrirWryrrrrr:s r-rzz?FlaxT5ForConditionalGeneration.decode.._decoder_forwards#779N,!&O .a0O{{..#2T[[5H5H$5N"O{{..#)==#:#:8#D[#Q "NN00(XGWGYGY>LGr/rc |j\}}|j|||} tj||fd} |!tj j | |d} | ||| dS)Nrr)rr)rr}rri)rrr'rRrOrr) r:rhrrrir}rWrrrextended_attention_masks r-prepare_inputs_for_generationz'tnd^i6J/36:  !+  !) 3  :r/ra Returns: Example: ```python >>> from transformers import AutoTokenizer, FlaxT5ForConditionalGeneration >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small") >>> model = FlaxT5ForConditionalGeneration.from_pretrained("google-t5/t5-small") >>> ARTICLE_TO_SUMMARIZE = "summarize: My friends are cool but they eat too many carbs." >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], return_tensors="np") >>> # Generate Summary >>> summary_ids = model.generate(inputs["input_ids"]).sequences >>> print(tokenizer.decode(summary_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False)) ``` )Rrrtypingrrr flax.linenlinenrPrO jax.numpynumpyr'rSflax.core.frozen_dictrrrr r r nn_partitioningflax.linen.attentionr flax.traverse_utilr r jax.randomrmodeling_flax_outputsrrrrrmodeling_flax_utilsrrrrrutilsrrrrconfiguration_t5r get_loggerrHlogger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCr0rTrKr.rr1rWrur}rrrrr%r)rErrrrMT5_START_DOCSTRINGrrFLAX_T5_MODEL_DOCSTRINGrarrr(FLAX_T5_CONDITIONAL_GENERATION_DOCSTRINGrUr/r-rs ,, >>66>;ut&   H %* #++ S Z] bebmbm +bii+("))>#ryy#LBII(YbiiYx#ryy#L BF"))FR  BII  FS BIIS lD "))D N6'T7tp/pf %PaV 299V V r ' [*=?UWfg6&9