sg4 dZddlZddlZddlmZmZmZddlZddl Z ddl m cm Z ddlZ ddl m Z ddlmZddlmZddlmZdd lmZdd lmZmZmZmZmZmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%d dl&m'Z'e%jPe)Z*dZ+dZ,dZ-gdZ.dZ/dZ0dZ1ddgZ2dZ3dZ4 dXdee5e5fde6de5dee jnde5dejpf dZ9Gdd e jtZ;Gd!d"e jtZ<Gd#d$e jtZ=Gd%d&e jtZ>Gd'd(e jtZ?Gd)d*e jtZ@Gd+d,e@ZAGd-d.e jtZBGd/d0e jtZCGd1d2e jtZDGd3d4e jtZEGd5d6e jtZFGd7d8e jtZGGd9d:e jtZHGd;de jtZJGd?d@e jtZKGdAdBeZLdCZMdDZNe"dEeMGdFdGeLZOe"dHeMGdIdJeLZPe"dKeMGdLdMeLZQe"dNeMGdOdPeLZRGdQdRe jtZSGdSdTe jtZTe"dUeMGdVdWeLZUy)YzPyTorch WavLM model.N)OptionalTupleUnion)nn)CrossEntropyLoss)ACT2FN)is_deepspeed_zero3_enabled)is_fsdp_managed_module)BaseModelOutputCausalLMOutputSequenceClassifierOutputTokenClassifierOutputWav2Vec2BaseModelOutput XVectorOutput)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardis_peft_availablelogging) WavLMConfigrz1patrickvonplaten/wavlm-libri-clean-100h-base-plus)ri$izZ'mister quilter is the aposle of the middle classes and we are glad to welcome his gospel'gQ)@zmicrosoft/wavlm-base-plus-sdzmicrosoft/wavlm-base-plus-svg ףp= ?shape mask_prob mask_lengthattention_mask min_masksreturnc|\}dkr tdkDrtdddtjjdj fd}|-|j dj jnt|Dcgc]}c}}tj|ft } g} |} | d k(r| S|D]} || } tjjtj| dz z | d }t|d k(rdz }n|d }tj|tj| | z tj |zg}| j#|tj$| } tj&| dddddf|| f} | j)|| z} tjddddf}tj&||| fj)|| z}| |z} | j+dz kDrdz | | dz kD<tj,| | dd| Scc}w) af Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `ct|zz z}t|}|zkDrz}|dz z |krt|dz z d}|S)z;Given input length, compute how many spans should be maskedrr)intmax) input_lengthnum_masked_spanepsilonrrrsequence_lengths [/var/www/html/venv/lib/python3.12/site-packages/transformers/models/wavlm/modeling_wavlm.pycompute_num_masked_spanz6_compute_mask_indices..compute_num_masked_spanrsoi,6DwNOoy9 [ (? :-s) #:Q#>!#Car,cV|jdkDr|ddddd|j f}|SNr)rrks r*rmzWavLMSamePadLayer.forwardBs6    ")!Q0F43F3F2F0F*FGMr,rorts@r*rr=sKr,rc.eZdZdZfdZdZdZxZS)WavLMFeatureEncoderz.Construct the features from raw audio waveformc t||jdk(rDt|dgt |j dz Dcgc]}t ||dzc}z}nV|jdk(r.t |j Dcgc]}t||}}ntd|jdtj||_ d|_ d |_ ycc}wcc}w) Ngroupr)rgrlayerz`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)rXrYfeat_extract_normrr9num_feat_extract_layersrRrvr1r ModuleList conv_layersgradient_checkpointing_requires_grad)rerfirrhs r*rYzWavLMFeatureEncoder.__init__Ls   # #w .26AFGKPQWQoQorsQsKtKFG)&1q5AKK % % 0PUV\VtVtPuv126AFvKv01I1I0JJst ==5&+#"Kws C" C'cJ|jD] }d|_ d|_y)NF) parameters requires_gradrreparams r*_freeze_parametersz&WavLMFeatureEncoder._freeze_parameters]s(__& (E"'E  (#r,c |dddf}|jr|jrd|_|jD]K}|jr5|jr)|jr|j |j |}D||}M|S)NT)rtrainingrrr_gradient_checkpointing_func__call__)re input_valuesrl conv_layers r*rmzWavLMFeatureEncoder.forwardbs$QW-    4==*.M '** :J""t'B'Bt}} $ A A''!! !+= 9  :r,)rprqrr__doc__rYrrmrsrts@r*rrIs8#"$ r,rceZdZfdZxZS)WavLMFeatureExtractorct||tjd|jj d|jj dj dty)Nz The class `zD` has been depreciated and will be removed in Transformers v5. Use `rz ` instead.)rXrYwarningswarnrhrp __bases__ FutureWarningrerfrhs r*rYzWavLMFeatureExtractor.__init__vs[   $..1123NN,,Q/889 E   r,)rprqrrrYrsrts@r*rrus   r,rc$eZdZfdZdZxZS)WavLMFeatureProjectionc4t|tj|jd|j |_tj|jd|j|_ tj|j|_ y)Nr-eps) rXrYrryrZlayer_norm_epsrzLinearr projectionDropoutfeat_proj_dropoutdropoutrs r*rYzWavLMFeatureProjection.__init__sf ,,vr':@U@UV))FOOB$79K9KLzz&":":; r,cp|j|}|j|}|j|}||fSrj)rzrr)rerlnorm_hidden_statess r*rmzWavLMFeatureProjection.forwards:!__];(:;  ]3 000r,rorts@r*rrs < 1r,rceZdZdZ ddedededededef fd Z dd ejd e ejd e ejd ed e eje eje e ejff dZ d ejd eejej fdejd ed ejejff dZdeded ejfdZdejd ejfdZxZS)WavLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paper embed_dim num_headsr num_buckets max_distancehas_relative_position_biasct|||_||_||_||z|_|j |z|jk7rt d|jd|d|j dz|_tj|||_ tj|||_ tj|||_ tj|||_ ||_||_tj t#j$d|jdd|_tj|j d|_|r0tj*|j|j|_yy)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gr)rXrYrrrhead_dimr1scalingrrk_projv_projq_projout_projrr Parametertorchr@gru_rel_pos_constgru_rel_pos_linear Embeddingrel_attn_embed)rerrrrrrrhs r*rYzWavLMAttention.__init__s7 "" !Y. MMI %$.. 8MdnnM]$YKr3 }}d* ii 95 ii 95 ii 95  )Y7 &(!#ejjDNNAq.Q!R"$))DMM1"= %"$,,t/?/?"PD  &r,rlr position_biasoutput_attentionsr c|j\}}}|S|j||}|jdj|dddj ||j z||}|j |j dd|j dfz} | jdddd} |j| } | j | j dddzjd} tj| jdd\} } | | |jzd z zd z} | j ||j zdd|z}|j d||f}|j||||\}}|||fS) z'Attention layer with relative attentionNrrr-rr)rr?g@)size compute_bias unsqueezerepeatviewrrpermuterr6rsigmoidchunkrtorch_multi_head_self_attention)rerlrrrindexbsztgt_lenrHgated_hidden_statesrelative_position_projgate_agate_b gate_outputgated_position_bias attn_output attn_weightss r*rmzWavLMAttention.forwards(,,.Wa   --gw?M''*11#q!Q?DDS4>>EY[bdkl  ,001D1DSb1IT^^]_L`1`a199!Q1E"&!8!89L!M!7!DDQBDO)?)? ?# EFL *..sT^^/CRKm[166GW7MN$($H$H >+>@Q% ! \L-77r,rcX|jddx}x}}||jdnd}dx} } d} tj||||j|j t jdgt j|jj|jj|jjf| | | |j|jj|jj|j |||d|jj|jj|jj\} } | jdd} | C| dddfj#| j$dd|j fz| j$ddz} | | fS)zCsimple wrapper around torch's multi_head_attention_forward functionrrNFT)use_separate_proj_weight q_proj_weight k_proj_weight v_proj_weight)r}neFmulti_head_attention_forwardrrremptycatrrWrrrrrrrDr)rerlrrrquerykeyvaluekey_padding_maskbias_kbias_v add_zero_attnrrs r*rz.WavLMAttention.torch_multi_head_self_attentions,55a;;;e3A3M>,,Q/SW %&$B$B    NN NN KK  IIt{{'')9)94;;;K;KL M    LL MM MM   MM   %)++,,++,,++,,+% ! \2"++Aq1  #(40==""2A&$..)::\=O=OPQPR=SSLL((r, query_length key_lengthctj|tjdddf}tj|tjdddf}||z }|j|}|j |j j j}|j |}|jgd}|S)Nr.)rrr) rr=long_relative_positions_buckettorrdevicer)rerrcontext_positionmemory_positionrelative_positionrelative_position_bucketvaluess r*rzWavLMAttention.compute_biass << EJJG4P,,zDT1WM+.>>#'#B#BCT#U #;#>#>t?R?R?Y?Y?`?`#a $$%=> * r,relative_positionsc$|jdz}|dkDjtj|z}tj|}|dz}||k}tj |j |z }|tj |j|z z }|||z z}||zjtj}tj|tj||dz }|tj|||z }|Sr) rrrrabslogfloatmathrmin full_likewhere)rer!rrelative_buckets max_exactis_smallrelative_positions_if_largerelative_position_if_larges r*rz)WavLMAttention._relative_positions_bucket!s &&!+ .266uzzB[P"YY'9:1$ % 1&+ii0B0H0H0JY0V&W#&ADHHTM^M^ajMjDk&k#&A[S\E\&]#&/2M&M%Q%QRWR\R\%]"%*YY &8RT_bcTc(d& " EKK2DF`aar,)i@i TNNFr)rprqrrrr$r%r;rYrTensorrrrm FloatTensorr LongTensor BoolTensorrrrrsrts@r*rrsG +/"Q"Q"Q "Q  "Q  "Q%)"QN2604"' '8||'8!.'8  - '8  '8 u||Xell3XeELL>Q5RR S'8R5)((5)e..0@0@@A5)#.. 5)  5)   U.. / 5)n#%BSBS U=N=N SXSdSd r,rc$eZdZfdZdZxZS)WavLMFeedForwardct|tj|j|_tj |j|j|_ t|jtrt|j|_n|j|_tj |j|j|_tj|j |_yrj)rXrYrractivation_dropoutintermediate_dropoutrrintermediate_sizeintermediate_dense isinstance hidden_actstrr intermediate_act_fn output_densehidden_dropoutoutput_dropoutrs r*rYzWavLMFeedForward.__init__8s $&JJv/H/H$I!"$))F,>,>@X@X"Y f'' -'-f.?.?'@D $'-'8'8D $IIf&>&>@R@RS jj)>)>?r,c|j|}|j|}|j|}|j|}|j |}|Srj)r;r?r9r@rBrks r*rmzWavLMFeedForward.forwardEsX// > 00? 11-@ ))-8 ++M: r,rorts@r*r6r67s @r,r6c2eZdZddedeffd ZddZxZS)WavLMEncoderLayerrfrct|t|j|j|j |j |j||_tj|j|_ tj|j|j|_t!||_tj|j|j|_yN)rrrrrrrrXrYrrnum_attention_headsattention_dropoutrmax_bucket_distance attentionrrrArryrrzr6 feed_forwardfinal_layer_normrerfrrhs r*rYzWavLMEncoderLayer.__init__P '((00,,**33'A  zz&"7"78 ,,v'9'9v?T?TU,V4 " V-?-?VEZEZ [r,c|}|j|||||\}}}|j|}||z}|j|}||j|z}|j |}||f}|r||fz }|S)Nrrrr)rLrrzrMrN) rerlrrrr attn_residualroutputss r*rmzWavLMEncoderLayer.forward_s% 59^^ )'/ 6D6 2 |] ]3 % 5  6 %(9(9-(HH --m<  -0    &Gr,Tr0rprqrrrr;rYrmrsrts@r*rErEOs \{ \ \r,rEc2eZdZddedeffd ZddZxZS) WavLMEncoderLayerStableLayerNormrfrct|t|j|j|j |j |j||_tj|j|_ tj|j|j|_t!||_tj|j|j|_yrGrHrOs r*rYz)WavLMEncoderLayerStableLayerNorm.__init__yrPr,c|}|j|}|j||||\}}}|j|}||z}||j|j |z}||f}|r||fz }|S)N)rrr)rzrLrrMrN)rerlrrrrSrrTs r*rmz(WavLMEncoderLayerStableLayerNorm.forwards%  6 59^^ )'/ 6D6 2 |]  ]3 % 5 %(9(9$:O:OP]:^(__  -0    &Gr,rU)NNFrVrts@r*rXrXxs \{ \ \r,rXc.eZdZfdZ ddZxZS) WavLMEncoderc t|||_t||_t j |j|j|_ t j|j|_ t jt|jDcgc]}t!||dk(c}|_d|_ycc}wNrr)rF)rXrYrfrpos_conv_embedrryrrrzrrArrr9num_hidden_layersrElayersrrerfrrhs r*rYzWavLMEncoder.__init__s  :6B,,v'9'9v?T?TUzz&"7"78 mmUZ[a[s[sUt uPQ v16 K u  ',# v!C c|rdnd}|rdnd}|d||<|j|}||z}|j|}|j|}txs t |} d} t |j D]\} } |r||fz}tjg} |jxr | dkDxr| |jjk}|r| rM|jr,|jr |j| j||| |}n | ||| || }|dd\}} |rd}|s|dfz}|r||fz}|std|||fDSt!|||S) Nr/rrRrNNNc3&K|] }|| ywrjre.0vs r* z'WavLMEncoder.forward..mq_`_lmlast_hidden_staterl attentions)r_rzrr r enumeraterarr4rrf layerdroprrrtupler rerlrroutput_hidden_states return_dictall_hidden_statesall_self_attentionsposition_embeddings synced_gpusrrrdropout_probabilityskip_the_layer layer_outputss r*rmzWavLMEncoder.forwards#7BD$5b4  %-0M>/ *"11-@%(;;  6  ]3 02R6LT6R  !$++.! PHAu#$58H$H!#(**R. !]]fq1uf:MPTP[P[PePe:eN![..4==$($E$E%&%) %M%*%'5&3*; %M0=Ra/@, } 2  &9]1=M.<rlrmrn)r_rr r rqrarr4rrfrrrrrrzrsr rts r*rmz#WavLMEncoderStableLayerNorm.forwards#7BD$5b4  %-.M>/ *"11-@%(;;  ]3 02R6LT6R  !$++. PHAu#$58H$H!#(**R. !]]fq1uf:MPTP[P[PePe:eN![..4==$($E$E%&%) %M%*%'5*;&3 %M 0=Ra/@, } 2  &9]1=Mr-)meanrexpr6r$)probsmarginal_probs perplexitys r*_compute_perplexityz.WavLMGumbelVectorQuantizer._compute_perplexity]sR*YY .599^VZEZ;[*[ac ddeiik r,c|j\}}}|j|}|j||z|jzd}|jrt j j|j|jd}|j|}tj|j||z|jdjd}|j|}n}|jd}|j|jj!d|jddd}|j||z|jd}|j|}|j||zd}|j#d|j$z} | j||z|j|j&d} | j)dj||d} | |fS)Nr-T)tauhardrrrr|)rrrrrr functionalgumbel_softmaxr%rtype_asrsoftmaxrargmax new_zerosscatter_rrrr6) rerlrGr)rcodevector_probscodevector_soft_distrcodevector_idxcodevectors_per_grouprs r*rmz"WavLMGumbelVectorQuantizer.forwardcs3@3F3F0 O[((7 %**:+G$//+Y[]^ ==!}};;MAQAQ Q+00o1Mt`d`m`moqr !oob)..z?BO J&&r,) rprqrrrrY staticmethodrrmrsrts@r*rrBs& * "'r,rc$eZdZfdZdZxZS) WavLMAdapterct|jjk7rTt j jj|_t jj|_ndx|_|_t jfdtjD|_ j|_ y)Nc34K|]}tywrj)WavLMAdapterLayer)rirHrfs r*rkz(WavLMAdapter.__init__..s#h!$5f$=#hs)rXrYoutput_hidden_sizerrrprojryproj_layer_normrr9num_adapter_layersrarrrs `r*rYzWavLMAdapter.__init__s   $ $(:(: : &"4"4f6O6OPDI#%<<0I0I#JD /3 3DI,mm#huVMfMfGg#hh ))r,ch|j.|j"|j|}|j|}|jdd}|jD]D}tj j }|j r||jkDs=||}F|jdd}|Sr)rrr}rar2r3rrr)rerlrlayerdrop_probs r*rmzWavLMAdapter.forwards 99 T%9%9%E IIm4M 00?M%//15 [[ 5EYY--/N==^dnn%D %m 4  5 &//15 r,rorts@r*rrs  *r,rc$eZdZfdZdZxZS)rct|tj|jd|jz|j |j d|_y)Nrr)rVr)rXrYrr]radapter_kernel_sizeadapter_striderars r*rYzWavLMAdapterLayer.__init__sJ II  % % )) )  & &((   r,cj|j|}tjj|d}|S)Nrr)rarrglurks r*rmzWavLMAdapterLayer.forwards/ -0  ))-Q)? r,rorts@r*rrs  r,rceZdZdZeZdZdZdZdZ d de e je fdeefd Z d d e d e jfd Zy)WavLMPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. wavlmrTc zt|tr|jjjj dd|jj jjtjj|jyt|trtjj |jjddtjd|jj d|jj"zz ztjj%|jj dyt|t&rtjd|j(j*z }tjj|j(j| |tjj|j(j | |yt|tj,rm|jjj d|j.j0|j %|j jjyyt|tj2tj4frJ|j jj|jjj7dyt|tj8rtjj;|j|j jtj|j<|j"|j dzz }tjj|j | |yyy) zInitialize the weightsr/r)rstdrr)abNr)r<rrrdatanormal_rWzero_rinituniform_rrrar&sqrtrU in_channels constant_rr in_featuresrrfinitializer_rangeryrfill_r]kaiming_normal_r)remoduleks r* _init_weightsz"WavLMPreTrainedModel._init_weightss f8 9    % % * * 2 2 2 C    # # ( ( . . 0 GG  V// 0  < = GGOO "" !v{{'>'>q'AFKKD[D['["\]]   GG  fkk.. 2  6 7 !f//;;;tj||z |ddzS)Nfloor) rounding_moder)rdivr&rUrVs r*_conv_out_lengthzOWavLMPreTrainedModel._get_feat_extract_output_lengths.._conv_out_lengths"99\K7wWZ[[ [r,r)rfrzipr^r_r9rr)rerIrrrUrVrHs r* _get_feat_extract_output_lengthsz5WavLMPreTrainedModel._get_feat_extract_output_lengthss2=1Ddkk--+  \ $'t{{'>'> @W@W#X Q K,]KPM Q 4;;99: _ 04;;C]C] ^  _r,feature_vector_lengthrc|jddddf}|j||}|jtj}|j d}tj ||f|j|j}d|tj|j d|j|dz f<|jdgjdjdgj}|S)Nr-rrr)r/rr)r) cumsumrrrrrr:r/rr=flipr;)rerrrnon_padded_lengthsoutput_lengthsrGs r*"_get_feature_vector_attention_maskz7WavLMPreTrainedModel._get_feature_vector_attention_masks ,22r2:1b5A>>?Q_j>k'**5::6#))!,  . /~7K7KTbTiTi uv ^%9%9!%<^EZEZ[]kno]opq',,bT299"=BBB4HMMOr,rj)rprqrrrr config_classbase_model_prefixmain_input_namesupports_gradient_checkpointingrrrr3r$rr;rrrer,r*rrsx L$O&*#9DZ^"5#3#3S#89HPQU0Y]%(:?:J:Jr,ra WavLM was proposed in [WavLM: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Xiangzhan Yu, Furu Wei. This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving etc.). This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`WavLMConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. aI Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should **not** be passed to avoid degraded performance when doing batched inference. For such models `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly different results depending on whether `input_values` is padded or not. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. z_The bare WavLM Model transformer outputting raw hidden-states without any specific head on top.cbeZdZdeffd ZdZdZ ddejde ejde ejfdZ e e eeeed e  dd e ej&de ej&de ejd e ed e ede edeeeffdZxZS) WavLMModelrfct||||_t||_t ||_|jdkDs|jdkDrEtjtj|jj|_|j rt#||_nt'||_|j(r t+|nd|_|j/y)Nr/)rXrYrfrfeature_extractorrfeature_projectionmask_time_probmask_feature_probrrrr1rrmasked_spec_embeddo_stable_layer_normrencoderr\rradapter post_initrs r*rYzWavLMModel.__init__Rs   !4V!<"8"@  3 &&*B*BS*H%'\\%,,v?Q?Q2R2[2[2]%^D "  & &6v>DL'/DL/5/A/A|F+t  r,cXtjdt|jyz Calling this function will disable the gradient computation for the feature encoder so that its parameters will not be updated during training. The method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5. Please use the equivalent `freeze_feature_encoder` method instead.Nrrrfreeze_feature_encoderres r*freeze_feature_extractorz#WavLMModel.freeze_feature_extractorf'   Q  ##%r,c8|jjy Calling this function will disable the gradient computation for the feature encoder so that its parameter will not be updated during training. N)rrrs r*rz!WavLMModel.freeze_feature_encoderrs 113r,rlmask_time_indicesrct|jdds|S|j\}}}|)|jj |j ||<n|jj dkDr|jrt||f|jj |jj||jj}tj||jtj}|jj |j ||<|jjdkDr|jrt||f|jj|jj |jj"}tj||jtj}|dddfj%d|d}d||<|S) z Masks extracted features along time axis and/or along feature axis according to [SpecAugment](https://arxiv.org/abs/1904.08779). apply_spec_augmentTNr)rrrr)rr/)rrrr-)getattrrfrrrr/rrrPmask_time_lengthmask_time_min_masksrtensorrr;rmask_feature_lengthmask_feature_min_masksexpand)rerlrrrGr)rmask_feature_indicess r*_mask_hidden_stateszWavLMModel._mask_hidden_statesyst{{$8$? 4A3E3E3G0 O[  (/3/E/E/H/HI\I\/]M+ , [[ ' '! + 5_-++44 KK88-++99 ! !& ->}G[G[chcmcm n /3/E/E/H/HI\I\/]M+ , ;; ( (1 ,#8[)++77 KK;;++<< $ $)<<0D]MaMainisis#t #74#@#G#GO]_#` 23M. /r,audio checkpoint output_typermodalityexpected_outputrrrurvr cH||n|jj}||n|jj}||n|jj}|j |}|j dd}|!|j |jd|d}|j|\}}|j|||}|j|||||} | d}|j|j|}|s ||f| ddzSt||| j| jS) NrrFr)rrrrrurvr)roextract_featuresrlrp)rfrruuse_return_dictrr}rrrrrrrrlrp) rerrrrrurvr rlencoder_outputss r*rmzWavLMModel.forwardsb"2C1N-TXT_T_TqTq$8$D $++JjJj &1%uEN+/*A*ABR*S' '00 ->~1 ,, )/!5# ' (* << # LL7M!#34qr7JJ J&+-)77&11   r,)NNNNNNN)rprqrrrrYrrrr2rr3rrWAVLM_INPUTS_DOCSTRINGr_CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr1r;rrrmrsrts@r*rrLs {( &4:>59 ,((,$E$5$56,!!1!12 ,\++AB&+$. 269=,0/3&*2 u||,2 !.2 $E$5$56 2 $D> 2 'tn 2 d^2  u-- .2 C2 r,rzcWavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).ceZdZddeeffd ZdZdZdZdZ e e e e eeee ddeej&d eej&d eed eed eed eej&deeeffdZxZS) WavLMForCTC target_langct||t||_t j |j |_||_|jtd|jdt|dr|jr |jn |j}t j ||j|_|j%y)NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `WavLMForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.r)rXrYrrrr final_dropoutrr vocab_sizer1rhrrrrrlm_headr)rerfrrrhs r*rYzWavLMForCTC.__init__s  ' zz&"6"67 &    $00@AHH *1)GFL^L^F % %djdvdv yy!3V5F5FG  r,c|j}|&t|jddtd|d|-t|jddtj dy||j |dyy)a' This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when passing `target_lang=...` to `from_pretrained(...)`. This method is **not** supposed to be called by the user and is prone to be changed in the future. Nadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T) force_load)rrrfr1loggerinfo load_adapter)rers r* tie_weightszWavLMForCTC.tie_weightss&&  "wt{{rYr!rrr*rrrrr r_CTC_EXPECTED_OUTPUT_CTC_EXPECTED_LOSSrr1r;rrrmrsrts@r*rrs HSM.<* &:(++AB&"$,( 26,0/3&*)-D u||,D !.D $D> D 'tn D d^ D &D  un$ %D CD r,rz WavLM Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. ceZdZfdZdZdZdZeee e e e d dde ejde ejd e ed e ed e ed e ejd eee ffdZxZS)WavLMForSequenceClassificationct||t|dr|jr t dt ||_|jdz}|jr0tjtj||z |_ tj|j|j |_tj|j |j$|_|j)y)Nrz\Sequence classification does not support the use of WavLM adapters (config.add_adapter=True)r)rXrYrrr1rrr`use_weighted_layer_sumrrrr@ layer_weightsrrclassifier_proj_size projector num_labels classifierrrerf num_layersrhs r*rYz'WavLMForSequenceClassification.__init__s  6= )f.@.@n  ' --1  ( (!#ejj.Dz.Q!RD 6#5#5v7R7RS))F$?$?ARARS r,cXtjdt|jyrrrs r*rz7WavLMForSequenceClassification.freeze_feature_extractorrr,cL|jjjyrr%rs r*rz5WavLMForSequenceClassification.freeze_feature_encoderr&r,cP|jjD] }d|_ yr(r)rs r*r*z0WavLMForSequenceClassification.freeze_base_modelr+r,r)rrrrrrrrurvr-r c||n|jj}|jjrdn|}|j|||||}|jjrr|t}t j |d}tjj|jd} || jdddzjd}n|d}|j|}||jd} nZ|j|j d|} d|| <|jd| jdjddz } |j#| } d} |Ft%}|| jd|jj&|jd} |s| f|tdz}| | f|zS|St)| | |j*|j, S)  labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NTr rrr-rr/r3)rfr rLrrArstackrrrrMrr6rOrrrrQrrPrrlrp)rerrrrurvr-rTrl norm_weights pooled_output padding_maskr5r4loss_fctrFs r*rmz&WavLMForSequenceClassification.forwards0&1%->r1a-HHMMRSMTM#AJM}5  !)..1.5MBB=CVCVWXCY[ijL+.M<- ()--!-4|7G7GA7G7N7S7STVXY7ZZM/  ')HFKKDKK,B,BCV[[QS_UDY)F)G!HHF)-)9TGf$ Ev E'!//))   r,r)rprqrrrYrrr*rrrrrrrrr1r;rrrmrsrts@r*rJrJs$ &:(++AB&,$ 26,0/3&*)-; u||,; !.; $D> ; 'tn ; d^ ; &;  u.. /; C; r,rJza WavLM Model with a frame classification head on top for tasks like Speaker Diarization. ceZdZfdZdZdZdZeee e e e de  ddeej deej d eej d eed eed eed eee ffdZxZS) WavLMForAudioFrameClassificationct||t|dr|jr t dt ||_|jdz}|jr0tjtj||z |_ tj|j|j |_|j |_|j%y)Nrz_Audio frame classification does not support the use of WavLM adapters (config.add_adapter=True)r)rXrYrrr1rrr`rLrrrr@rMrrrPrQ init_weightsrRs r*rYz)WavLMForAudioFrameClassification.__init__s  6= )f.@.@q  ' --1  ( (!#ejj.Dz.Q!RD ))F$6$68I8IJ ++ r,cXtjdt|jyr#rrs r*rz9WavLMForAudioFrameClassification.freeze_feature_extractorrr,cL|jjjyrr%rs r*rz7WavLMForAudioFrameClassification.freeze_feature_encoder!r&r,cP|jjD] }d|_ yr(r)rs r*r*z2WavLMForAudioFrameClassification.freeze_base_model(r+r,rrrrr-rrurvr c ||n|jj}|jjrdn|}|j|||||}|jjrr|t}t j |d}tjj|jd} || jdddzjd}n|d}|j|} d} |\t} | | jd|jt j |jd|jd} |s| f|tdz} | St#| | |j$|j& S) rXNTr rrr-r)axisr3)rfr rLrrArrYrrrrMrr6rQrrPrrrlrp)rerrr-rrurvrTrlrZr5r4r]rFs r*rmz(WavLMForAudioFrameClassification.forward0sh0&1%->r1a-HHMMRSMTM#AJM/  ')HFKKDOOtuDY)F)G!HHFM$!//))   r,r)rprqrrrYrrr*rrr_FRAME_CLASS_CHECKPOINTrr_FRAME_EXPECTED_OUTPUTrrr1r;rrrmrsrts@r*r_r_s &:(++AB*)$. 26)-,0/3&*3 u||,3 !.3 & 3 $D> 3 'tn 3 d^3  u++ ,3 C3 r,r_c&eZdZdfd ZdZxZS) AMSoftmaxLossctt| ||_||_||_t jtj||d|_ t j|_ y)NT)r) rXrjrYscalemarginrPrrrrandnrrr4)re input_dimrPrlrmrhs r*rYzAMSoftmaxLoss.__init__psS mT+-  $ll5;;y*#EUYZ '') r,c|j}tjj|jd}tjj|d}t j ||}||jz }tjj||j}|jt j|j||z}|j||}|S)Nrrr)flattenrr normalizerrmmrmone_hotrPrlr)r;r4) rerlr-r cos_thetapsionehotr5r4s r*rmzAMSoftmaxLoss.forwardxs!((!(< // 1/E HH]F3 $++%&&vt?ekk&++-iHHyy( r,)g>@g?rorts@r*rjrjos * r,rjcXeZdZdfd Zdej dej fdZxZS) TDNNLayerct||dkDr|j|dz n|j||_|j||_|j ||_|j||_tj|j|j z|j|_ tj|_ y)Nrr)rXrYtdnn_dimr[r\ tdnn_kernelrU tdnn_dilationdilationrrkernelReLUrcrds r*rYzTDNNLayer.__init__s ryrtdnnxvector_output_dimrrQrjrP objectivera)rerfrSr tdnn_layersrhs r*rYzWavLMForXVector.__init__s  ' --1  ( (!#ejj.Dz.Q!RD 6#5#5vq7IJ5:3v;O5PQy+Q QMM+. !#6??2+>+BFD]D]!^))F$=$=v?X?XY&v'@'@&BSBST Rs>FcXtjdt|jyr#rrs r*rz(WavLMForXVector.freeze_feature_extractorrr,cL|jjjyrr%rs r*rz&WavLMForXVector.freeze_feature_encoderr&r,cP|jjD] }d|_ yr(r)rs r*r*z!WavLMForXVector.freeze_base_modelr+r,rIcVd}|jjD] }|||d}|S)z? Computes the output length of the TDNN layers c||z |zdzS)Nrrers r*rzBWavLMForXVector._get_tdnn_output_lengths.._conv_out_lengths!;.69A= =r,r)rfr|)rerIrrUs r*_get_tdnn_output_lengthsz(WavLMForXVector._get_tdnn_output_lengthss:  >  ;;22 LK,]KKM Lr,rrrrrrurvr-r c||n|jj}|jjrdn|}|j|||||}|jjrr|t}t j |d}tjj|jd} || jdddzjd}n|d}|j|}|jD] } | |} |%|jd} |j!d} n|j#|jd} |j%| }g} g} t'|D]U\}}| j)||d|fjd| j)||d|fj!dWt j | } t j | } t j*| | gd}|j-|}|j/|}d}||j1||}|s||f|tdz}||f|zS|St3||||j4|j6S) rXNTr rrr-r)r4r5 embeddingsrlrp)rfr rLrrArrYrrrrMrr6rOrrrrrrqrBr rrQrrrlrp)rerrrrurvr-rTrlrZ tdnn_layer mean_features std_featuresfeat_extract_output_lengthstdnn_output_lengthsrlengthstatistic_poolingoutput_embeddingsr5r4rFs r*rmzWavLMForXVector.forwards0&1%->r1a-HHMMRSMTM#AJM}5 )) 6J&}5M 6  !)..1.5M(,,,3L*.*O*OP^PbPbghPbPi*j '"&"?"?@["\ ML&':; J 6$$]1gvg:%>%C%C%C%JK##M!WfW*$=$A$Aa$A$HI J"KK 6M ;;|4L!II}l&CL 223DE!23  >>&&1D/07;X;Y3ZZF)-)9TGf$ Ev E(!//))   r,r)rprqrrrYrrr*rrr3r$rrrr_XVECTOR_CHECKPOINTrr_XVECTOR_EXPECTED_OUTPUTrr1r;rrmrsrts@r*rrs& &:( eE I 'tn I d^ I &I  um# $I CI r,rr)Vrr&rtypingrrrnumpyr2rtorch.nn.functionalrrr torch.utils.checkpointtorch.nnr activationsr integrations.deepspeedr integrations.fsdpr modeling_outputsr r rrrrmodeling_utilsrrrrrrrconfiguration_wavlmr get_loggerrprrArrrrGrHrgrhrrr$r%r3ndarrayrPModulerRrvrrrrrrrr6rErXr\rrrrrWAVLM_START_DOCSTRINGrrrrJr_rjryrrer,r*rsW )) %!@7.-   H %!" J&t9Q526 t c?tttU--. t  t ZZ tp ,bii8bii2*299*\ )")))X /  1RYY 1c RYYc Nryy0& &R"ryy"JO 299O dP "))P fC'C'N299@ $U?Up("Je P %P  P fm T &T  T n s %9s s l g ';g g VBII0 > O *O O r,