
    sgI                         d Z ddlZddlmZmZmZmZmZ ddlZ	ddl
Z
ddlmZmZmZ ddlmZ ddlmZ ddlmZmZ  ej,                  e      Z G d	 d
e      Zy)z!Feature extractor class for CLAP.    N)AnyDictListOptionalUnion   )mel_filter_bankspectrogramwindow_function)SequenceFeatureExtractor)BatchFeature)
TensorTypeloggingc                       e Zd ZdZddgZ	 	 	 	 	 	 	 	 	 	 	 	 ddededededef
 fd	Zd
e	ee
f   fdZddej                  deej                     d
ej                  fdZd Zdej                  d
ej                  fdZ	 	 	 	 	 ddeej                  ee   eej                     eee      f   dedee   dee   dee   deeeef      d
efdZ xZS )ClapFeatureExtractora  
    Constructs a CLAP feature extractor.

    This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
    most of the main methods. Users should refer to this superclass for more information regarding those methods.

    This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the *Short Time
    Fourier Transform* (STFT) which should match pytorch's `torch.stft` equivalent.

    Args:
        feature_size (`int`, *optional*, defaults to 64):
            The feature dimension of the extracted Mel spectrograms. This corresponds to the number of mel filters
            (`n_mels`).
        sampling_rate (`int`, *optional*, defaults to 48000):
            The sampling rate at which the audio files should be digitalized expressed in hertz (Hz). This only serves
            to warn users if the audio fed to the feature extractor does not have the same sampling rate.
        hop_length (`int`,*optional*, defaults to 480):
            Length of the overlaping windows for the STFT used to obtain the Mel Spectrogram. The audio will be split
            in smaller `frames` with a step of `hop_length` between each frame.
        max_length_s (`int`, *optional*, defaults to 10):
            The maximum input length of the model in seconds. This is used to pad the audio.
        fft_window_size (`int`, *optional*, defaults to 1024):
            Size of the window (in samples) on which the Fourier transform is applied. This controls the frequency
            resolution of the spectrogram. 400 means that the fourrier transform is computed on windows of 400 samples.
        padding_value (`float`, *optional*, defaults to 0.0):
            Padding value used to pad the audio. Should correspond to silences.
        return_attention_mask (`bool`, *optional*, defaults to `False`):
            Whether or not the model should return the attention masks coresponding to the input.
        frequency_min (`float`, *optional*, defaults to 0):
            The lowest frequency of interest. The STFT will not be computed for values below this.
        frequency_max (`float`, *optional*, defaults to 14000):
            The highest frequency of interest. The STFT will not be computed for values above this.
        top_db (`float`, *optional*):
            The highest decibel value used to convert the mel spectrogram to the log scale. For more details see the
            `audio_utils.power_to_db` function
        truncation (`str`, *optional*, defaults to `"fusion"`):
            Truncation pattern for long audio inputs. Two patterns are available:
                - `fusion` will use `_random_mel_fusion`, which stacks 3 random crops from the mel spectrogram and a
                  downsampled version of the entire mel spectrogram.
            If `config.fusion` is set to True, shorter audios also need to to return 4 mels, which will just be a copy
            of the original mel obtained from the padded audio.
                - `rand_trunc` will select a random crop of the mel spectrogram.
        padding (`str`, *optional*, defaults to `"repeatpad"`):
               Padding pattern for shorter audio inputs. Three patterns were originally implemented:
                - `repeatpad`: the audio is repeated, and then padded to fit the `max_length`.
                - `repeat`: the audio is repeated and then cut to fit the `max_length`
                - `pad`: the audio is padded.
    input_features	is_longerfrequency_minfrequency_maxtop_db
truncationpaddingc           	      ^   t        |   d||||d| |
| _        || _        || _        || _        |dz	  dz   | _        || _        || _        ||z  | _	        || _
        || _        |	| _        t        | j                  |||	|d d      | _        t        | j                  |||	|dd      | _        y )N)feature_sizesampling_ratepadding_valuereturn_attention_mask   htk)num_frequency_binsnum_mel_filtersmin_frequencymax_frequencyr   norm	mel_scaleslaney )super__init__r   r   r   fft_window_sizenb_frequency_bins
hop_lengthmax_length_snb_max_samplesr   r   r   r	   mel_filtersmel_filters_slaney)selfr   r   r,   r-   r*   r   r   r   r   r   r   r   kwargs	__class__s                 c/var/www/html/venv/lib/python3.12/site-packages/transformers/models/clap/feature_extraction_clap.pyr)   zClapFeatureExtractor.__init__T   s      	 	
%''"7		

 	
 $."1Q"6!!;$(*]:****#55('''
 #2#55('''#
    returnc                     t        j                  | j                        }| j                  j                  |d<   d|v r|d= d|v r|d= |S )a)  
        Serializes this instance to a Python dictionary.

        Returns:
            `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance, excpet for the
            mel filter banks, which do not need to be saved or printed as they are too long.
        feature_extractor_typer/   r0   )copydeepcopy__dict__r3   __name__)r1   outputs     r4   to_dictzClapFeatureExtractor.to_dict   sQ     t}}-+/>>+B+B'(F"}%6)+,r5   waveformr/   c           	          t        |t        | j                  d      | j                  | j                  d|d      }|j                  S )a  
        Compute the log-mel spectrogram of the provided `waveform` using the Hann window. In CLAP, two different filter
        banks are used depending on the truncation pattern:
            - `self.mel_filters`: they correspond to the default parameters of `torchaudio` which can be obtained from
              calling `torchaudio.transforms.MelSpectrogram().mel_scale.fb`. These filters are used when `truncation`
              is set to `"fusion"`.
            - `self.mel_filteres_slaney` : they correspond to the default parameters of `librosa` which used
              `librosa.filters.mel` when computing the mel spectrogram. These filters were only used in the original
              implementation when the truncation mode is not `"fusion"`.
        hanng       @dB)frame_lengthr,   powerr/   log_mel)r
   r   r*   r,   T)r1   r?   r/   log_mel_spectrograms       r4   _np_extract_fbank_featuresz/ClapFeatureExtractor._np_extract_fbank_features   sI     *D00&9--#
 #$$$r5   c           	         t        j                  t        t        d||z
  dz               d      }t	        |d         dk(  rdg|d<   t	        |d         dk(  rdg|d<   t         j
                  j                  |d         }t         j
                  j                  |d         }t         j
                  j                  |d         }||||z   d d f   }||||z   d d f   }	||||z   d d f   }
t        j                  |d d d d f         }t        j                  j                  j                  ||dgdd      }|d   d   j                         }t        j                  |||	|
gd	      }|S )
Nr   r   r      @   bilinearF)sizemodealign_cornersaxis)nparray_splitlistrangelenrandomchoicetorchtensornn
functionalinterpolatenumpystack)r1   meltotal_frameschunk_framesranges	idx_front
idx_middleidx_backmel_chunk_frontmel_chunk_middlemel_chunk_back
mel_shrink
mel_fusions                r4   _random_mel_fusionz'ClapFeatureExtractor._random_mel_fusion   ss   U1l\.IA.M%N OQRSvay>QF1Ivay>QF1III$$VAY/	YY%%fQi0
99##F1I.i)l*BBAEFzJ,EEqHIX<(??BCll3tT1}-.XX((44|R(z 5 

  ]1%++-
XXz?<Ln]def
r5   c                 4   |j                   d   |kD  r|dk(  rbd}t        |      |z
  }t        j                  j	                  d|dz         }||||z    }| j                  || j                        dddf   }||fS |dk(  r|| j                  || j                        }	|| j                  z  dz   }
|	j                   d   }|
|k(  r!t        j                  |	|	|	|	gd      }d}||fS | j                  |	||
      }d}||fS t        d	| d
      d}|j                   d   |k  r|dk(  r3t        |t        |      z        }t        j                  ||dz         d| }|dk(  r-t        |t        |      z        }t        j                  ||      }t        j                  |d||j                   d   z
  fdd      }|dk(  r;| j                  || j                        }t        j                  ||||gd      }||fS | j                  || j                        dddf   }||fS )a  
        Extracts the mel spectrogram and prepares it for the mode based on the `truncation` and `padding` arguments.
        Four different path are possible:
            - `truncation="fusion"` and the length of the waveform is greater than the max length: the mel spectrogram
              will be computed on the entire audio. 3 random crops and a dowsampled version of the full mel spectrogram
              are then stacked together. They will later be used for `feature_fusion`.
            - `truncation="rand_trunc"` and the length of the waveform is smaller than the max length: the audio is
              padded based on `padding`.
            - `truncation="fusion"` and the length of the waveform is smaller than the max length: the audio is padded
              based on `padding`, and is repeated `4` times.
            - `truncation="rand_trunc"` and the length of the waveform is greater than the max length: the mel
              spectrogram will be computed on a random crop of the waveform.

        r   
rand_truncTr   NfusionrP   Fzdata_truncating z not implementedrepeat	repeatpadconstant)rN   constant_values)shaperV   rR   rW   randintrH   r0   r/   r,   r_   rl   NotImplementedErrorinttilepad)r1   r?   
max_lengthr   r   longeroverflowidx	input_melr`   rb   ra   n_repeats                r4   _get_input_melz#ClapFeatureExtractor._get_input_mel   s_    >>!z)\)x=:5ii''8a<8#C#
*:; ;;HdF]F]^_cef_fg	D &  C x'55h@P@PQ)T__<q@"yy|</ !##sC)=A FI"F2 &  / !% 7 7\< XI!F, &  ) *,<ZLHX*YZZ F~~a :-h&":H#=>H!wwxA>{
KHk)":H#=>H!wwx:H66(Q
X^^A=N0N,OV`rstX% ;;HdFVFVW	HHiIy%QXYZ	 &   !;;HdF]F]^_cef_fg	&  r5   
raw_speechrz   r   return_tensorsc                    ||n| j                   }|r|n| j                  }|O|| j                  k7  rUt        d| j                  j
                   d| j                   d| j                   d| d	      t        j                  d       t        |t        j                        xr t        |j                        dkD  }|r&t        |j                        dkD  rt        d	|        |xs@ t        |t        t        f      xr( t        |d
   t        j                  t        t        f      }	|	r3|D 
cg c]'  }
t        j                  |
t        j                         ) }}
n|	s@t        |t        j                        s&t        j                  |t        j                         }nht        |t        j                        rN|j"                  t        j"                  t        j                         u r|j%                  t        j                         }|	st        j                  |      g}|D cg c]$  }| j'                  ||r|n| j(                  ||      & }}g }g }|D ]'  \  }}|j+                  |       |j+                  |       ) |dk(  r<t-        |      d
k(  r.t        j.                  j1                  d
t        |            }d||<   t        |d
   t2              r2|D cg c]'  }t        j                  |t        j                         ) }}|D cg c]  }|g }}||d}t5        |      }||j7                  |      }|S c c}
w c c}w c c}w c c}w )af  
        Main method to featurize and prepare for the model one or several sequence(s).

        Args:
            raw_speech (`np.ndarray`, `List[float]`, `List[np.ndarray]`, `List[List[float]]`):
                The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
                values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
                stereo, i.e. single float per timestep.
            truncation (`str`, *optional*):
                Truncation pattern for long audio inputs. Two patterns are available:
                    - `fusion` will use `_random_mel_fusion`, which stacks 3 random crops from the mel spectrogram and
                      a downsampled version of the entire mel spectrogram.
                If `config.fusion` is set to True, shorter audios also need to to return 4 mels, which will just be a
                copy of the original mel obtained from the padded audio.
                    - `rand_trunc` will select a random crop of the mel spectrogram.
            padding (`str`, *optional*):
               Padding pattern for shorter audio inputs. Three patterns were originally implemented:
                    - `repeatpad`: the audio is repeated, and then padded to fit the `max_length`.
                    - `repeat`: the audio is repeated and then cut to fit the `max_length`
                    - `pad`: the audio is padded.
            return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.np.array` objects.
                - `'np'`: Return Numpy `np.ndarray` objects.
            sampling_rate (`int`, *optional*):
                The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
                `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
                pipeline.
        z3The model corresponding to this feature extractor: z& was trained using a sampling rate of zI. Please make sure that the provided `raw_speech` input was sampled with z	 and not .zIt is strongly recommended to pass the `sampling_rate` argument to this function. Failing to do so can result in silent errors that might be hard to debug.r   rJ   z2Only mono-channel audio is supported for input to r   )dtypero   T)r   r   )r   r   r   
ValueErrorr3   r<   loggerwarning
isinstancerR   ndarrayrV   rt   rT   tupleasarrayfloat64r   astyper   r.   appendsumrW   ru   r   r   convert_to_tensors)r1   r   r   r   rz   r   r   r2   is_batched_numpy
is_batchedspeechr?   padded_inputsr~   r   r`   r{   rand_idxfeaturer   s                       r4   __call__zClapFeatureExtractor.__call__  s   R $.#9Zt
$'$,,$ 2 22 I$..JaJaIb c))-););(< =))-););(<Im_TUW  NN\
 &j"**=[#jFVFVBWZ[B[J$4$4 5 9QRVQWXYY% 
zD%=1lz*Q-RTR\R\^ceiQj7k 	 MWX6"**V2::>XJXJz2::$FJbjjAJ
BJJ/J4D4DQSQ[Q[H\4\#**2::6J **Z01J
 '
 
*H[H[]gipq
 

 		( 	%KCS!V$	% !c)n&9yy((C	N;H"&IhilD)NWX7G2::>XIX -66&fX6	6,5IN%n5%+>>~NNO Y
" Y 7s   -,L8")L=,M
M)rK   i  i  
   i   g        Fr   i6  Nro   rq   )N)NNNNN)r<   
__module____qualname____doc__model_input_namesfloatrw   strr)   r   r   r>   rR   arrayr   r   rH   rl   r   r   r   r   r   r   __classcell__)r3   s   @r4   r   r       s   /b *;7 # %""3
 3
 3
 3
 3
 3
jc3h  %288 %(SUS[S[J\ %hjhrhr %,28!rxx 8!UWU]U] 8!z !%$('+;?h"**d5k4

3CT$u+EVVWh h #	h
 SMh  }h !sJ!78h 
hr5   r   )r   r9   typingr   r   r   r   r   r^   rR   rY   audio_utilsr	   r
   r   !feature_extraction_sequence_utilsr   feature_extraction_utilsr   utilsr   r   
get_loggerr<   r   r   r'   r5   r4   <module>r      sK    (  3 3   H H I 4 ( 
		H	%J3 Jr5   