
    sg>                        d Z ddlmZmZmZ ddlmZ ddlmZm	Z	 ddl
mZmZmZmZmZmZ ddlmZmZmZ ddlmZ  ej.                  e      Zd	Z ed
      D  cg c]	  } d| dd c}  ed      D  cg c]	  } d| dd c} z   Z G d de      Z G d de      Z G d ded      Zde fdZ!d Z"d Z#d Z$deee      fdZ% G d d e      Z&y!c c} w c c} w )"z 
Processor class for PaliGemma.
    )ListOptionalUnion   )BatchFeature)
ImageInputis_valid_image)ImagesKwargsProcessingKwargsProcessorMixin
TextKwargsUnpack!_validate_images_text_input_order)
AddedTokenPreTokenizedInput	TextInput)loggingz<image>i   z<locz0>4>   z<segz0>3c                   8    e Zd ZU eeeeee   ee   f      ed<   y)PaliGemmaTextKwargssuffixN)	__name__
__module____qualname__r   r   r   r   r   __annotations__     e/var/www/html/venv/lib/python3.12/site-packages/transformers/models/paligemma/processing_paligemma.pyr   r   -   s&    U9&7i$O`Jaabccr   r   c                       e Zd ZU ee   ed<   y)PaliGemmaImagesKwargsdo_convert_rgbN)r   r   r   r   boolr   r   r   r   r!   r!   1   s    TN"r   r!   c                   4    e Zd ZU eed<   eed<   ddiddidZy)	PaliGemmaProcessorKwargstext_kwargsimages_kwargspaddingFdata_formatchannels_first)r&   r'   N)r   r   r   r   r   r!   	_defaultsr   r   r   r%   r%   5   s.    $$(( u
 +
	Ir   r%   F)totalreturnc                 H    t        | t              xr | j                  d      S )Nhttp)
isinstancestr
startswith)vals    r   is_urlr4   C   s    c3:CNN6$::r   c                 2    t        |       xs t        |       S N)r4   r	   elems    r   is_image_or_image_urlr9   H   s    $</>$//r   c                 <    t        | t              xs t        |       S r6   )r0   r1   r9   r7   s    r   _is_str_or_imager;   L   s    dS"A&;D&AAr   c                      ||z  |z   | |  dS )aZ  
    Builds a string from the input prompt and image tokens.
    For example, for the call:
    build_string_from_input(
        prompt="Prefix str"
        bos_token="<s>",
        image_seq_len=3,
        image_token="<im>",
    )
    The output will be:
    "<im><im><im><s>Initial str"
    Args:
        prompt (`List[Union[str, ImageInput]]`): The input prompt.
        bos_token (`str`): The beginning of sentence token.
        image_seq_len (`int`): The length of the image sequence.
        image_token (`str`): The image token.
        num_images (`int`): Number of images in the prompt.
    
r   prompt	bos_tokenimage_seq_lenimage_token
num_imagess        r   build_string_from_inputrD   P   s$    & M)J67	{6("MMr   c                 D   t        | t        t        f      rCt        | d   t        t        f      r*t        | d   d         r| D cg c]  }|D ]  }|  c}}S t        | t        t        f      rt        | d         r| S t        |       r| gS t	        d|        c c}}w )a  
    Accepts images in list or nested list format, and makes a list of images for preprocessing.

    Args:
        images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`):
            The input image.

    Returns:
        list: A list of images.
    r   z"Could not make batched video from )r0   listtupler	   
ValueError)imagesimg_listimgs      r   make_batched_imagesrL   g   s     &4-(Zq	D%=-QVdeklmenopeqVr$*?h?s???	FT5M	*~fQi/H		x
9&B
CC @s   Bc            
            e Zd ZdZddgZdgZdZdZ	 	 	 d fd	Z	 	 	 	 dde	d	e
eeee   ee   f   d
ee   defdZd Zd Zed        Z xZS )PaliGemmaProcessora  
    Constructs a PaliGemma processor which wraps a PaliGemma image processor and a PaliGemma tokenizer into a single processor.

    [`PaliGemmaProcessor`] offers all the functionalities of [`SiglipImageProcessor`] and [`LlamaTokenizerFast`]. See the
    [`~PaliGemmaProcessor.__call__`] and [`~PaliGemmaProcessor.decode`] for more information.

    Args:
        image_processor ([`SiglipImageProcessor`], *optional*):
            The image processor is a required input.
        tokenizer ([`LlamaTokenizerFast`], *optional*):
            The tokenizer is a required input.
        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
            in a chat into a tokenizable string.
    image_processor	tokenizerchat_templateSiglipImageProcessor)GemmaTokenizerGemmaTokenizerFastc                    |t        d      |t        d      t        |d      st        d      |j                  | _        t        |d      sCt        t        dd      }d	|gi}|j                  |       |j                  t              | _        n|j                  | _        |j                  t               d|_
        d|_        t        | 5  |||
       y )Nz)You need to specify an `image_processor`.z"You need to specify a `tokenizer`.image_seq_lengthz;Image processor is missing an `image_seq_length` attribute.rB   FT)
normalizedspecialadditional_special_tokens)rQ   )rH   hasattrrV   r   IMAGE_TOKENadd_special_tokensconvert_tokens_to_idsimage_token_id
add_tokensEXTRA_TOKENSadd_bos_tokenadd_eos_tokensuper__init__)selfrO   rP   rQ   kwargsrB   tokens_to_add	__class__s          r   rd   zPaliGemmaProcessor.__init__   s     "HIIABB(:;Z[[ / @ @y-0$[UDQK8;-HM((7"+"A"A+"ND"+":":D\*"'	"'	)=Qr   rI   textrf   r-   c                 z   t        ||      \  }} | j                  t        fd| j                  j                  i|}|d   j                  dd      }|dnd}|t        d      |t        j                  d       d	}t        |      r|g}nt        |t              rt        |d
         r	 |T|Qt        d |D              st        j                  d       t        |t              rKt        |t              r;t        |      t        |      k7  r$t        dt        |       dt        |       d      t!        |      r|gg}nnt        |t              rt!        |d
         r|D 	cg c]  }	|	g }}	n?t        |t              r$t        |d
   t              rt!        |d
   d
         st        d      |t        |      r|g}|&|D 
cg c]  }
|
| j                  j"                  z    }}
t%        ||      D cg c]R  \  }}t'        || j                  j(                  | j*                  t,        t        |t              rt        |      nd      T }}}t/        |      }ng }|D ]  }|j1                  t,        t,        | j*                  z        }|j3                  t,              }|dk7  r|t        t,              z   nd
}|d| | j                  j(                  z   ||d z   }|j5                  |        |D cg c]  }| d	 }} | j6                  |fi |d   d   }|d   j9                  dd      |d   dxx   | j*                  z  cc<    | j                  f||d|d   }i |d|i}|r.|d   j;                  |d   d
k(  d      }|j=                  d|i       t?        |      S c c}	w c c}
w c c}}w c c}w )ah  
        Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
        and `kwargs` arguments to LlamaTokenizerFast's [`~LlamaTokenizerFast.__call__`] if `text` is not `None` to encode
        the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to
        SiglipImageProcessor's [`~SiglipImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring
        of the above two methods for more information.

        The usage for PaliGemma fine-tuning preparation is slightly different than usual. suffix passed are suffixes to
        the prompt in `text`, and will be placed after the prompt. This is because attention is handled differently for
        the prefix and the suffix. For instance,
        ```python
        image = PIL_cow_image
        prompt = "answer en Where is the cow standing?"
        suffix = "on the beach"
        inputs = processor(text=prompt, images=image, suffix=suffix)
        ```
        Here `inputs` will contain the `input_ids` and `token_type_ids` that follow
        ```python
        inputs["input_ids"][:, 256:]
        # tensor([[     2,   6006,    603,    573,  13910,   9980, 235336,    108,    477,   573,   8318]])
        inputs["token_type_ids"][:, 256:]
        tensor([[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1]])
        ```
        Meaning the last three tokens are of "label" ("suffix") type while the other ones are of "prefix" type.


        Args:
            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
                tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a
                number of channels, H and W are image height and width.
            text (`str`, `List[str]`, `List[List[str]]`):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors of a particular framework. Acceptable values are:

                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return NumPy `np.ndarray` objects.
                - `'jax'`: Return JAX `jnp.ndarray` objects.
            suffix (`str`, `List[str]`, `List[List[str]]`):
                The suffixes or batch of suffixes to be encoded. Only necessary for finetuning. See https://github.com/google-research/big_vision/blob/main/big_vision/configs/proj/paligemma/README.md
                for more information. If your prompt is "<image> What is on the image", the suffix corresponds to the expected prediction "a cow sitting on a bench".

        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. If `suffix`
              is provided, the `input_ids` will also contain the suffix input ids.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
            - **labels** -- Labels compatible with training if `suffix` is not None
        tokenizer_init_kwargsr&   r   NTFzF`images` are expected as arguments to a `PaliGemmaProcessor` instance.z]You are using PaliGemma without a text prefix. It will perform as a picture-captioning model. r   c              3   ,   K   | ]  }t         |v   y wr6   )r[   ).0samples     r   	<genexpr>z.PaliGemmaProcessor.__call__.<locals>.<genexpr>  s     @{f,@s   aL  You are passing both `text` and `images` to `PaliGemmaProcessor`. The processor expects special image tokens in the text, as many tokens as there are images per each text. It is recommended to add `<image>` tokens in the very beginning of your text. For this call, we will infer how many images each text has and add special tokens.z	Received z images for zK prompts. Each prompt should be associated with an image or list of images.zAimages must be an image, list of images or list of list of images   r>   r=   r'   pixel_values
max_length)	text_pairreturn_token_type_ids	input_idstoken_type_idsilabels)data) r   _merge_kwargsr%   rP   init_kwargspoprH   loggerwarning_oncer;   r0   rF   anywarningr   lenr	   	eos_tokenziprD   r@   rV   r[   rL   replacerfindappendrO   getmasked_fillupdater   )re   rI   ri   audiovideosrf   output_kwargsr   rv   imagesfxr?   
image_listinput_stringsexpanded_samplesro   expanded_samplebos_rfind_index	bos_indexrs   inputsreturn_datary   s                          r   __call__zPaliGemmaProcessor.__call__   s   D 9F***$
"&.."<"<
 

 }-11(DA(.(:>eff<o DD!6Dd#(8a(A 2@4@@< dD)j.F6{c$i/('F}LT  LW  X 
 "&)%hZF-.2K39:%ug:F:$VT2z&)T7RWeflmnfopqfrWs$%hii%*:6*B$XF%HNOcDNN$<$<<OFO /2$.?	! +
 ,%"&..":":&*&;&;$/6@T6R3z?XY	! 	! -V4#% " =F&,nn[+PTPePeBe&fO&5&;&;K&HOFUY[F[#k2B BabI'
3dnn6N6NNQ`ajakQll $ %++O<= >N N6F82 N N+t++FUmO6TUVde '++L$?K-(6$:O:OO6
"7
 M*	
 ?>> K(44V<L5MQR5RTXYF&12--c ; P	!* !Os   '
N(	 N-:AN25N8c                 :     | j                   j                  |i |S )z
        This method forwards all its arguments to GemmaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
        refer to the docstring of this method for more information.
        )rP   batch_decodere   argsrf   s      r   r   zPaliGemmaProcessor.batch_decodeR  s     
 +t~~**D;F;;r   c                 :     | j                   j                  |i |S )z
        This method forwards all its arguments to GemmaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
        the docstring of this method for more information.
        )rP   decoder   s      r   r   zPaliGemmaProcessor.decodeZ  s     
 %t~~$$d5f55r   c                     | j                   j                  }| j                  j                  }t        t        j                  ||z               S r6   )rP   model_input_namesrO   rF   dictfromkeys)re   tokenizer_input_namesimage_processor_input_namess      r   r   z$PaliGemmaProcessor.model_input_namesa  sA     !% @ @&*&:&:&L&L#DMM"7:U"UVWWr   )NNN)NNNN)r   r   r   __doc__
attributesvalid_kwargsimage_processor_classtokenizer_classrd   r   r   r   r   r   r   r%   r   r   r   r   propertyr   __classcell__)rh   s   @r   rN   rN   ~   s     $[1J#$L2>O 	R@ "^b^.^. I0$y/4HYCZZ[^. 12^. 
^.B<6 X Xr   rN   N)'r   typingr   r   r   feature_extraction_utilsr   image_utilsr   r	   processing_utilsr
   r   r   r   r   r   tokenization_utils_baser   r   r   utilsr   
get_loggerr   r~   r[   ranger`   r   r!   r%   r#   r4   r9   r;   rD   rL   rN   )is   0r   <module>r      s    ) ( 4 5  
  
		H	%).t5A$qgQ5RWX[R\8]Q4#wa8]]d* d#L #
/u 
;4 ;
0BN.D4Z(8#9 D.hX hXi 68]s   C3C