sgjdZddlZddlmZmZmZddlZddlZddlmZddl m Z ddl m Z m Z mZddlmZdd lmZdd lmZdd lmZmZdd lmZmZmZmZdd lmZddlm Z ejBe"Z#dZ$dZ%dZ&gdZ'dZ(dZ)GddejTZ+GddejTZ,GddejTZ-GddejTZ.GddejTZ/Gdd ejTZ0Gd!d"ejTZ1Gd#d$ejTZ2Gd%d&ejTZ3Gd'd(eZ4d)Z5d*Z6Gd+d,e4Z7e d-e5Gd.d/e4Z8e d0e5Gd1d2e4Z9y)3zPyTorch M-CTC-T model.N)OptionalTupleUnion)nn)ACT2FN)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forward)is_deepspeed_zero3_enabled)is_fsdp_managed_module)_prepare_4d_attention_mask)BaseModelOutputCausalLMOutput)PreTrainedModelapply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)logging) MCTCTConfigrzspeechbrain/m-ctc-t-large)rizY"Mr. Quilter is the apostle of the middle classes, and we're glad to welcome his gospel."gv@c(eZdZdZfdZdZxZS)MCTCTConv1dSubsamplerz Convolutional subsampler: a stack of 1D convolution (along temporal dimension) followed by non-linear activation via gated linear units (https://arxiv.org/abs/1911.08460) cPt|_|j_t j |j_|j_ |j|jz_ jdkDr)|j td|j_nd_|j"dz_|j&_|j*_t j.fdt1j(D_y)NrzbNeed to specify `conv_channels` configuration in `MCTCTConfig` to use multiple convolution layers.c3K|]w\}}tj|dk(r jnj||jdz krj|n j |j |dyyw)rrvalid) kernel_sizestridepaddingN)rConv1d in_channels mid_channels num_layers out_channelsr ).0ikselfs f/var/www/html/venv/lib/python3.12/site-packages/transformers/models/deprecated/mctct/modeling_mctct.py z1MCTCTConv1dSubsampler.__init__..Ys ) 1 II$%F  0A0A!0D()DOOa,?(?!!!$TEVEV{{1~    ) sA=B)super__init__config conv_glu_dimglu_dimrDropout conv_dropoutdropoutnum_conv_layersr%input_feat_per_channelinput_channelsr# conv_channels ValueErrorr$ hidden_sizer& conv_kernelr conv_strider ModuleList enumerate conv_layersr*r/ __class__s` r+r.zMCTCTConv1dSubsampler.__init__=s  ** zz&"5"56  00!886;P;PP ??Q ##+  !' 4 4D  $D "..2!--(( == ) "$"2"23 )  ct|jDcgc]}|dz c}}tjjj |dd||fdd}|j ddj}|jD]F}||}tjj||j}|j|}H|j ddj}|Scc}w)Nrrconstantrdim) sumrtorchr functionalpad transpose contiguousr?glur1r4)r*input_featuressizer! hidden_statesconvs r+forwardzMCTCTConv1dSubsampler.forwarddsT-=-=>Ttqy>?,,00!QQXAY[eghi&00A6AAC $$ 8D /MMM--m-NM LL7M 8 &//15@@B ?s C.__name__ __module__ __qualname____doc__r.rR __classcell__rAs@r+rr7s % N rBrc,eZdZdZfdZ ddZxZS)MCTCTEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.ct|tj|j|j |j |_tj|j|j |_ tj|j|j |_ t|_ tj|j|_|j#dt%j&|jj)dd|j#dt%j*|j,j/t$j0|j,j2dy)N) padding_idx position_ids)rF) persistenttoken_type_idsdtypedevice)r-r.r Embedding vocab_sizer: pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddingsMCTCTLayerNorm LayerNormr2hidden_dropout_probr4register_bufferrHarangeexpandzerosr^rOlongrdr@s r+r.zMCTCTEmbeddings.__init__ws !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2H2H&J\J\%]" ()zz&"<"<=   ELL)G)GHOOPWXej     KK))..0 4K\K\KcKc d  rBc||jn|jdd}|d}||jdd|||zf}|st|dr-|jddd|f}|j |d|} | }n:t j |t j|jj}||j|}|j|} || z} |j| } |j| } | S)Nr_rrarrb) rOr^hasattrrarrrHrsrtrdrhrlrnr4) r*rNrar^ inputs_embedspast_key_values_length input_shape seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedrl embeddingss r+rRzMCTCTEmbeddings.forwards0>/In))+}OaOaOcdgegOh  ^  ,,Q0FVlIl0l-lmL  !t-.*.*=*=a*n*M'3J3Q3QR]^_R`bl3m0!A!&[ SWSdSdSkSk!l   00@M $ : :> J"%:: ^^J/ \\*- rB)NNNNrrSrYs@r+r[r[tsQ .wxrBr[c>eZdZfdZdZdZdZ ddZxZS)MCTCTSelfAttentioncZt||j|jzdk7r2t |ds&t d|jd|jd|j|_|j |_|j|jz|_tj|j|jd|_ tj|j|jd|_ tj|j|jd|_ tj|j|_|j"|_tj$d|j"zd z |j|_|j(|_y) Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()Fbiasrr)r-r.r:num_attention_headsrvr9attention_head_dimattention_head_size all_head_sizerLinearquerykeyvaluer2attention_probs_dropout_probr4riredistance_embedding is_decoderr@s r+r.zMCTCTSelfAttention.__init__sX     : : :a ?PVXhHi#F$6$6#78 445Q8  $*#=#= #)#<#< !558P8PPYYv1143E3EER 99V//1C1C%PYYv1143E3EER zz&"E"EF '-'E'E$"$,,q63Q3Q/QTU/UW[WoWo"p ++rBc|jdd|j|jfz}|j|}|j ddddS)Nr_rrr)rOrrviewpermute)r*x new_x_shapes r+transpose_for_scoresz'MCTCTSelfAttention.transpose_for_scoressNffhsmt'?'?AYAY&ZZ AFFK yyAq!$$rBc t|jdkDr4|jtt t|j}|j t|jtt t|S)Nr)lenshaperreversedrangereshape)r*rrs r+reshape_fortranz"MCTCTSelfAttention.reshape_fortransd qww bclerrrErr) start_dim)rmathsqrtrrrrrHmatmulrKrweighteinsumrrrIsoftmaxr4rflatten)r*rPattention_mask head_maskoutput_attentionsmixed_query_layer key_layer value_layer query_layerattention_scorespositional_embeddingrelative_position_scoresattention_probs context_layeroutputss r+rRzMCTCTSelfAttention.forwards!JJ}5- $:R:R0SS--dhh}.EF // =0IJ //0AB !<< Y5H5HR5PQ $66==#(<<0BDXZeZoZopqstZu#v #'#J#JKc#d +.FF  %/.@ --//0@b/I,,7  - 9O _kB %--aAq9AABAO 6G=/2O\M]rBNNF) rTrUrVr.rrrrRrXrYs@r+rrs(,.% Q *8 .rBrc$eZdZfdZdZxZS)rmct|tjt j d|_tjt jd|_yNr) r-r.r ParameterrHonessingleton_weightrssingleton_bias)r*rAs r+r.zMCTCTLayerNorm.__init__s@  " UZZ] ; ll5;;q>:rBc:||jz|jzSN)rrr*rPs r+rRzMCTCTLayerNorm.forward s 5 559L9LLLrBrTrUrVr.rRrXrYs@r+rmrms; MrBrmc$eZdZfdZdZxZS)MCTCTSelfOutputc:t|||_tj|j |j d|_tj|j |j|_tj|j|_ yNFr)eps) r-r.r/rrr:densernlayer_norm_epsr2ror4r@s r+r.zMCTCTSelfOutput.__init__%si  YYv1163E3EER f&8&8f>S>STzz&"<"<= rBcr|j|}|j|}|j||z}|Srrr4rnr*rP input_tensors r+rRzMCTCTSelfOutput.forward,7 =1  ]3 }|'CD rBrrYs@r+rr$s >rBrc2eZdZfdZdZ ddZxZS)MCTCTAttentionct|t||_t ||_t |_yr)r-r.rr*routputset pruned_headsr@s r+r.zMCTCTAttention.__init__4s0 &v. %f- ErBc>t|dk(ryt||jj|jj|j \}}t |jj||j_t |jj||j_t |jj||j_ t |jj|d|j_ |jjt|z |j_|jj|jjz|j_ |j j||_y)NrrrE)rrr*rrrrrrrrrrunion)r*rindexs r+ prune_headszMCTCTAttention.prune_heads:s u:? 7 49900$))2O2OQUQbQb u -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(EE (R %"&))"?"?$))B_B_"_  --33E:rBcj|j||||}|j|d|}|f|ddz}|S)Nrr)r*r)r*rPrrr self_outputsattention_outputrs r+rRzMCTCTAttention.forwardLsNyy       ;;|A F#% QR(88rBr)rTrUrVr.rrRrXrYs@r+rr3s" ;* rBrc$eZdZfdZdZxZS)MCTCTIntermediatec t|tj|j|j d|_t|jtrt|j|_ y|j|_ y)NFr) r-r.rrr:intermediate_sizer isinstance hidden_actstrrintermediate_act_fnr@s r+r.zMCTCTIntermediate.__init__`s` YYv1163K3KRWX f'' -'-f.?.?'@D $'-'8'8D $rBcJ|j|}|j|}|Sr)rrrs r+rRzMCTCTIntermediate.forwardhs& =1 00? rBrrYs@r+rr_s 9rBrc$eZdZfdZdZxZS) MCTCTOutputc,t|tj|j|j d|_tj|j |j|_tj|j|_ yr) r-r.rrrr:rrnrr2ror4r@s r+r.zMCTCTOutput.__init__osc YYv779K9KRWX f&8&8f>S>STzz&"<"<= rBcr|j|}|j|}|j||z}|Srrrs r+rRzMCTCTOutput.forwardurrBrrYs@r+rrns > rBrc8eZdZdeffd Z ddZdZxZS) MCTCTLayerr/ct|d|_|j|_t ||_t ||_|j|_t||_ yr) r-r. seq_len_dimchunk_size_feed_forwardr intermediater attentionrrrr@s r+r.zMCTCTLayer.__init__}sV '-'E'E$-f5'/ ++!&) rBc|j||||}|d}|dd}t|j|j|j|}|f|z}|S)N)rrr)rrfeed_forward_chunkrr) r*rPrrrself_attention_outputsrr layer_outputs r+rRzMCTCTLayer.forwardst"& >9HY"0" 2!4(,0  # #T%A%A4CSCSUe  /G+rBcL|j|}|j||}|Sr)rr)r*rintermediate_outputrs r+rzMCTCTLayer.feed_forward_chunks,"//0@A{{#68HI rBr)rTrUrVrr.rRrrXrYs@r+rr|s$ *{ * *rBrcLeZdZdZeZdZdZdZdZ de jfdZ dZ y ) MCTCTPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. mctctrNTc|jj}t|tjrZ|j j jd||jb|jj jn  - KK   " " $ MM   $ $S )  /  # # ( ( . .s 3  ! ! & & , , . fryy"))4 5 MM   & &CS & 9{{&   &&(' 6rB input_lengthsc(d}tt|jj|jj|jj D]:\}}}|dz}|d|zz||dz zz dz }t j||ddz}<|S)zH Computes the output length of the convolutional layers rrtrunc) rounding_mode)ziprr/r5r;r<rHdiv)r*r dilation_ kernel_szr r!s r+ _get_feat_extract_output_lengthsz5MCTCTPreTrainedModel._get_feat_extract_output_lengthss$' $++-- . 0G0GI`I`%  X Ay& 1nG)AK7(iRSm:TTWXXM!IImV7SVWWM  XrBct|jdkDr |dddddf}|j|jd}|j d}t j ||f|j|j}d|t j||j|dz f<|jdgjdjdgj}|S)Nrr_rrbrr) rrrrGrOrHrsrcrdrqflipcumsumrt)r*feature_vector_lengthrsubsampled_lengthsbszs r+"_get_feature_vector_attention_maskz7MCTCTPreTrainedModel._get_feature_vector_attention_masks ~## $q (+Aq"H5N"BB>CUCUVXCYZ!!#A& ' (0D0D^MbMb  ef S1F1FGI[^_I_`a',,bT299"=BBB4HMMOrBN)rTrUrVrWr config_classbase_model_prefixmain_input_namesupports_gradient_checkpointingr rH LongTensorrrrBr+rrs; L&O&*#)0 e>N>N rBraH This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`MCTCTConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_features (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`Wav2Vec2CTCTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. ceZdZdeffd Z d dej dej dej dededed ee e ffd Z xZ S) MCTCTEncoderr/c$t|||j|_t|_t ||_tjt|jDcgc] }t|c}|_ d|_ ycc}w)NF)r-r.rorm layer_normrrQrr=rnum_hidden_layersrlayersgradient_checkpointing)r*r/rrAs r+r.zMCTCTEncoder.__init__sm  #)#=#= (*)&1 mmvG_G_A`$aAZ%7$ab &+#%bs'B rNrrroutput_hidden_states return_dictreturnc||n|jj}||n|jj}||n|jj}|j |}|j |}||j |jd|}tjj||j|j}|t||j}|rdnd} |rdnd} |_|jdt!|j"k7r6t%dt!|j"d|jddt'xs t)|} t+|j"D]\} } |r| |fz} t-j.g}|jr||jj0krdnd }|r| rO|j2r3|jr'|j5| j6||||| nd|}n | ||| }|d}|rd }|s| dfz} |r| |fz} |st9d || | fDSt;|| | S)Nr)ptrainingr!rz&The head_mask should be specified for z layers, but it is for .TF)rPrr)NNc3&K|] }|| ywrr!)r'vs r+r,z'MCTCTEncoder.forward..mseqWXWdeslast_hidden_staterP attentions)r/rr)use_return_dictr%rQrrrrIr4ror.rrcrOrr'r9r r r>rHrand layerdropr(_gradient_checkpointing_func__call__tupler)r*rNrrrr)r*rwrPencoder_statesall_attentions synced_gpusidx encoder_layerdropout_probabilityskip_the_layer layer_outputss r+rRzMCTCTEncoder.forwards2C1N-TXT_T_TqTq$8$D $++JjJj &1%>Ne]NN$See e+>Vd  rB)FFT) rTrUrVrr.rHTensorboolrrrrRrXrYs@r+r#r#s,{,#(%* R  R  R << R  R # R R  uo% &R rBr#zaThe bare M-CTC-T Model transformer outputting raw hidden-states without any specific head on top.ceZdZfdZeej deee e de  d de jdee jdee jdeed eed eed eee ffd ZxZS) MCTCTModelcrt||||_t||_|j yr)r-r.r/r#encoder post_initr@s r+r.zMCTCTModel.__init__xs/   #F+  rBzbatch_size, sequence_lengthaudio) checkpoint output_typermodalityexpected_outputrNrrrr)r*r+cJ||n|jj}||n|jj}||n|jj}| t d|j ||||||}|d}|s |f|ddzSt ||j|jS)Nz#You have to specify input_features.rrrr)r*rrr2) r/rr)r5r9rHrrPr4) r*rNrrrr)r*encoder_outputssequence_outputs r+rRzMCTCTModel.forwards"2C1N-TXT_T_TqTq$8$D $++JjJj &1% # 'tn # d^#  uo% &# i# rBrFzcMCTCT Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).ceZdZfdZeeeeee e e  d de jdee jdee jdeedeedeed ee j d eeeffd ZxZS) MCTCTForCTCct||t||_|jt d|j d|j}tj||j|_ |jy)NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `MCTCTForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.) r-r.rFrrfr9rAr:rrctc_headrI)r*r/output_hidden_sizerAs r+r.zMCTCTForCTC.__init__s  '    $00@AHH  $// "4f6G6GH  rB)rKrLrrN expected_lossrNrrrr)r*labelsr+c |I|j|jjk\r"td|jj||n|jj}|j ||||||}|d} |j | } d} |o||n1tj|jddtj}|j|jdjtj} |dk\} | jd}|j| }tj j#| dtj$j'dd}tj(j*j-d 5tj j/||| ||jj0|jj2|jj4 } ddd|s| f|t6dz}| | f|zS|St9| | |j:|j< S#1swYExYw) a labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`. Nz$Label values must be <= vocab_size: rPrr_)rc)rFrcrF)enabled)blank reduction zero_infinity)losslogitsrPr4)maxr/rfr9r5rr[rHrrrtrrGto masked_selectrrI log_softmaxfloat32rKbackendscudnnflagsctc_lossrgctc_loss_reductionctc_zero_infinity_HIDDEN_STATES_START_POSITIONrrPr4)r*rNrrrr)r*r^rrPrerdr  labels_masktarget_lengthsflattened_targets log_probsrs r+rRzMCTCTForCTC.forwards)2  &**,$++2H2H"HCDKKDZDZC[\] ]%0% E 'tn E d^E ))*E  un$ %E CE rBrY):rWrtypingrrrrHtorch.utils.checkpointr activationsr file_utilsr r r integrations.deepspeedr integrations.fsdpr modeling_attn_mask_utilsrmodeling_outputsrrmodeling_utilsrrrrutilsrconfiguration_mctctr get_loggerrTloggerrqrVrUrWrvrwModulerr[rrmrrrrrrMCTCT_START_DOCSTRINGrSr#rFrYr!rBr+rs )) "rrA8C@ ,   H % !2't:BII:z7bii7tiiXMRYYM bii )RYY)X   ")) $$NB?BJ @] '] @g5 %5 5 pma &a a rB