
    sg'                     .   d Z ddlmZmZ ddlZddlmZ ddlm	Z	  G d d	ej                  j                  j                        Z G d
 dej                  j                  j                        Z G d dej                  j                  j                        Zy)a  

Generic interface to various configurations of the Perceiver Resampler, that simply takes in a series of (potentially
time-indexed) contextual embeddings, and "resamples" (compresses) them down to a pre-specified number of latents! Note
that the Perceiver in general resamples based solely off the *long-range* context; there's a nice opportunity here to
prime the Perceiver Resampler with say a single layer's worth of language embeddings (the target domain), and use that
to softly "retrieve & compress" what we need --> this would be a novel contribution we should explore.

References:
    - DeepMind's Flamingo: https://www.deepmind.com/blog/tackling-multiple-tasks-with-a-single-visual-language-model
    - Code borrowed w/ love from: https://github.com/lucidrains/flamingo-pytorch

    )OptionalTupleN   )
shape_list   )IdeficsConfigc                   ~     e Zd Zdededededededdf fd	Z fd
Zdej                  dej                  fdZ	 xZ
S )TFIdeficsPerceiverResamplerconfig	embed_dimdepthn_headshead_dim	n_latentsreturnNc                    t        	|   di | ||||f\  | _        | _        | _        | _        |j                  j                  | _        t        |j                  d      s| j                  dz  n|j                  j                  dz  | _        g | _        t        |      D ]s  }| j                  j                  t        | j                  | j                  | j                  | j                  d| d      t!        | j                  |d| d      g       u t"        j$                  j&                  j)                  dd	      | _        y
)ao  
        Instantiates a Perceiver Resampler that operates over a sequence of embeddings (say from a ResNet or ViT or
        MAE) of a given dimension, performs `depth` blocks of cross-attention with a fixed `n_latents` inputs, then
        returns a Tensor of shape [bsz, n_latents, embed_dim]. :param embed_dim: Dimensionality of embeddings being fed
        to the Perceiver Resampler (also dimensionality of latent embeddings *returned* by the Perceiver Resampler.
        Could be e.g., VIT embed_dim, ResNet pool dim, and so on.

        Args:
            config (`IdeficsConfig`): config object
            embed_dim (`int`): The size of each embedding vector
            depth (`int`): Depth of the Perceiver Resampler (Transformer w/ cross attention). Should be shallow (< 3).
            n_heads (`int`): Number of heads in each Transformer block (for multi-headed self-attention).
            head_dim (`int`): Dimensionality of each head projection in the Transformer block.
            n_latents (`int`):
                Number of latent embeddings to resample ("compress") the input sequence to (usually < 128).

        r      zblocks.z.0namez.1h㈵>
layer_normepsilonr   N )super__init__r   r   r   r   perceiver_configqk_layer_norms_perceiverqk_layer_normshasattrvision_configintermediate_dimblocksrangeappendTFIdeficsPerceiverAttentionTFIdeficsMLPtfkeraslayersLayerNormalizationr   )
selfr   r   r   r   r   r   kwargsi	__class__s
            [/var/www/html/venv/lib/python3.12/site-packages/transformers/models/idefics/perceiver_tf.pyr   z$TFIdeficsPerceiverResampler.__init__1   s+   ( 	"6"FOQXZbdmFmCdmT^$55NN 6//= NNQ%%//!3 	 u 	AKK/dmmTEXEXahijhkkm_n !!6!6wqcQS_U		 ((//<<TP\<]    c                     | j                  | j                  | j                  fddd      | _        t        |   |       y )Nrandom_normalTlatents)shapeinitializer	trainabler   )
add_weightr   r   r4   r   build)r,   input_shaper/   s     r0   r9   z!TFIdeficsPerceiverResampler.build\   s>    >>4>>2[_fo ' 
 	k"r1   contextc                    t        j                  | j                  d      }t        j                  |t        j                  |      d   ddg      }| j
                  D ]  \  }} |||      |z   } ||      |z   } | j                  |      S )zWResample arbitrary length context & *compress* down to self.n_latents latent embeddingsr   axisr   )r(   expand_dimsr4   tiler5   r#   r   )r,   r;   r4   attnffs        r0   callz TFIdeficsPerceiverResampler.callc   s     ..A6'''BHHW$5a$8!Q#?@ 	,HD"7G,w6GkG+G	, w''r1   )__name__
__module____qualname__r   intr   r9   r(   TensorrC   __classcell__r/   s   @r0   r
   r
   0   si    )^#)^03)^<?)^JM)^Y\)^il)^	)^V#	(BII 	(")) 	(r1   r
   c            
            e Zd Zdededededdf
 fdZdej                  d	ej                  dej                  fd
Z xZ	S )r&   r   r   r   r   r   Nc                 0   t        |   di | |||c| _        | _        | _        || _        t        j                  j                  j                  dd      | _
        t        j                  j                  j                  dd      | _        | j
                  r`t        j                  j                  j                  dd      | _        t        j                  j                  j                  dd      | _        | j                  dz  | _        t        j                  j                  j                  | j                  | j                  z  dd	
      | _        t        j                  j                  j                  | j                  | j                  z  dd
      | _        t        j                  j                  j                  | j                  | j                  z  dd
      | _        t        j                  j                  j                  |dd
      | _        y)ziPerceiver Cross-Attention Module --> let long-form inputs be `context`, resampled embeddings be `latents`r   context_layer_normr   latents_layer_normq_layer_normk_layer_normg      Fq_projuse_biasr   k_projv_projoutput_projNr   )r   r   r   r   r   r   r(   r)   r*   r+   rM   rN   rO   rP   qk_scaleDenserQ   rT   rU   rV   )r,   r   r   r   r   r-   r/   s         r0   r   z$TFIdeficsPerceiverAttention.__init__p   s   "6"6?(3dm,"$((//"D"DTXl"D"m"$((//"D"DTXl"D"m " B B4Vd B eD " B B4Vd B eDt+ hhoo++DLL4==,HSX_g+hhhoo++DLL4==,HSX_g+hhhoo++DLL4==,HSX_g+h88??00UQ^0_r1   r;   r4   c                    | j                  |      }| j                  |      }t        |      \  }}}| j                  |      }| j	                  t        j                  ||gd            }| j                  t        j                  ||gd            }|||fD 	cg c]T  }	t        j                  t        j                  |	||	j                  d   | j                  | j                  f      g d      V c}	\  }}}| j                  r"| j                  |      }| j                  |      }t        j                   d|| j"                  z  |      }
|
t        j$                  |
dd	      z
  }t
        j&                  j)                  |d      }t        j                   d
||      }| j+                  t        j                  t        j                  |g d      |d| j                  | j                  z  f            S c c}	w )a=  
        Runs Perceiver Self-Attention, with special (context, latents) appended along the `seq` dimension!

        Args:
            context (`tf.Tensor`):
                Tensor of shape `[bsz, seq, embed_dim]` representing long-form context to resample.
            latents (`tf.Tensor`):
                Tensor of shape `[bsz, n_latents, embed_dim]` representing fixed length latents to compress to.

        Returns:
            `tf.Tensor`: Tensor of shape `[bsz, n_latents, embed_dim]` representing attention over latents w/ cross
            from context.
        r=   r   )r      r   r   )permz... i d, ... j d -> ... i jT)r>   keepdimsz... i j, ... j d -> ... i d)rM   rN   r   rQ   rT   r(   concatrU   	transposereshaper5   r   r   r   rO   rP   einsumrW   
reduce_maxnnsoftmaxrV   )r,   r;   r4   
batch_size
seq_lengthr   qkvxscoresstabilized_scoresrA   	resampleds                 r0   rC   z TFIdeficsPerceiverAttention.call   s    ))'2))'2,6w,?)
J	 KK KK		7G"42>?KK		7G"42>? AY
 LLA
AGGAJdmm'\]dpq
1a
 !!!$A!!!$A8!dmm:KQO"R]]6T%RRuu}}.R}8 II;T1E	JJr||ILAJPRTXT`T`cgcpcpTpCqr
 	

s   AG5)
rD   rE   rF   rG   boolr   r(   rH   rC   rI   rJ   s   @r0   r&   r&   o   sY    `# ` `s `TX `gk `*+
BII +
		 +
bii +
r1   r&   c                   h     e Zd Zdef fdZdeeej                        dej                  fdZ	 xZ
S )r'   r   c                    t        |   di | |j                  j                  | _        t        j
                  j                  j                  dd      | _        t        j
                  j                  j                  |dd      | _
        t        j
                  j                  j                  d      | _        t        j
                  j                  j                  | j                  dd	      | _        y
)z:Simple MLP block with intermediate_size and embedding sizer   lnr   FfcrR   actr   c_projNr   )r   r   r!   r   r(   r)   r*   r+   rr   rX   rs   ReLUrt   ru   )r,   intermediate_sizer   r-   r/   s       r0   r   zTFIdeficsMLP.__init__   s    "6"--77((//44T4M((//''(9EPT'U88??''U'3hhoo++DNNUQY+Zr1   hidden_statesr   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|S )N)rr   rs   rt   ru   )r,   rx   s     r0   rC   zTFIdeficsMLP.call   s@    ../M2r1   )rD   rE   rF   r   r   r   r   r(   rH   rC   rI   rJ   s   @r0   r'   r'      s6    [- [(5+;"<  r1   r'   )__doc__typingr   r   
tensorflowr(   modeling_tf_utilsr   configuration_ideficsr   r)   r*   Layerr
   r&   r'   r   r1   r0   <module>r      sj   4 #  + 0<("((//"7"7 <(~A
"((//"7"7 A
H288??(( r1   