sgP ddlZddlmZmZmZmZddlmZmZddl m Z m Z m Z m Z mZddlmZmZe rddlmZdd lmZmZe r dd lmZdd lmZe j6eZd ZGd dej>Z GddZ!de"deee#ee#dedffdZ$e edGddeZ%y)N)DictListOptionalUnion)ProcessingKwargsUnpack)add_end_docstringsis_torch_availableis_vision_availableloggingrequires_backends)Pipelinebuild_pipeline_init_args)Image) load_images valid_images)*MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES) KeyDatasetzceZdZdZdZdZy) ReturnTyperrrN)__name__ __module__ __qualname__TENSORSNEW_TEXT FULL_TEXT\/var/www/html/venv/lib/python3.12/site-packages/transformers/pipelines/image_text_to_text.pyrr-sGHIr rc :eZdZdZdedeeeededffdZy)Chata This class is intended to just be used internally in this pipeline and not exposed to users. We convert chats to this format because the rest of the pipeline code tends to assume that lists of messages are actually a batch of samples rather than messages in the same conversation.messagesimages Image.Imagecl|D]}d|vrd|vr tdt||}||_||_y)NrolecontentzQWhen passing chat dicts as input, each dict must have a 'role' and 'content' key.) ValueErrorretrieve_images_in_messagesr$r%)selfr$r%messages r!__init__z Chat.__init__8sJ vGg%)w*> !tuu v-Xv>   r N) rrr__doc__rrstrrr.rr r!r#r#3s4RuS$s)]TXYfTg5g/hr r#r$r%r&c<|g}d}g}|D]}|dD]}t|ts|jddk(rTdD]}||vs|j||H|t |kr|j|||dz }qt d|jddk(st|jdtr2d |dvr+|j|dd d|d<|dd |d<|d=t d |t |k7r t d|S) zS Retrieve and combine images from the chat and the images passed as input. rr)typeimage)r3urlpathbase64rzlThe number of images in the chat messages should be the same as the number of images passed to the pipeline. image_urlr4zhWrong format for 'image_url' content type. The content should have an 'image_url' dict with a 'url' key.) isinstancedictgetappendlenr*)r$r% idx_imagesretrieved_imagesr-r)keys r!r+r+Bsh ~Jy) G'4(;;v&'1A '>,33GCLA!  &F 3,33F:4FG&!OJ",!O#[[(K7!'++k":DAewWbOcFc(// 0DU0KL*1+2;+?+F(#K0(G1 <S[  z   r T) has_processorc eZdZdZdZdZdZdZfdZ dde e fdZ dde e eeeeeed ed eed fd e e eeeeefffd Zdd Zdd Zej*dfdZxZS)ImageTextToTextPipelineac Image-text-to-text pipeline using an `AutoModelForImageTextToText`. This pipeline generates text given an image and text. When the underlying model is a conversational model, it can also accept one or more chats, in which case the pipeline will operate in chat mode and will continue the chat(s) by adding its response(s). Each chat takes the form of a list of dicts, where each dict contains "role" and "content" keys. Example: ```python >>> from transformers import pipeline >>> pipe = pipeline(task="image-text-to-text", model="Salesforce/blip-image-captioning-base") >>> pipe("https://huggingface.co/datasets/Narsil/image_dummy/raw/main/parrots.png", text="A photo of") [{'generated_text': 'a photo of two birds'}] ``` ```python >>> from transformers import pipeline >>> pipe = pipeline("image-text-to-text", model="llava-hf/llava-interleave-qwen-0.5b-hf") >>> messages = [ >>> { >>> "role": "user", >>> "content": [ >>> { >>> "type": "image", >>> "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg", >>> }, >>> {"type": "text", "text": "Describe this image."}, >>> ], >>> }, >>> { >>> "role": "assistant", >>> "content": [ >>> {"type": "text", "text": "There is a dog and"}, >>> ], >>> }, >>> ] >>> pipe(text=messages, max_new_tokens=20, return_full_text=False) [{'input_text': [{'role': 'user', 'content': [{'type': 'image', 'url': 'https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg'}, {'type': 'text', 'text': 'Describe this image.'}]}, {'role': 'assistant', 'content': [{'type': 'text', 'text': 'There is a dog and'}]}], 'generated_text': ' a person in the image. The dog is sitting on the sand, and the person is sitting on'}] ``` Learn more about the basics of using a pipeline in the [pipeline tutorial](../pipeline_tutorial) This image-text to text pipeline can currently be loaded from pipeline() using the following task identifier: "image-text-to-text". See the list of available models on [huggingface.co/models](https://huggingface.co/models?pipeline_tag=image-text-to-text). TFcft||i|t|d|jty)Nvision)superr.rcheck_model_typer)r,argskwargs __class__s r!r.z ImageTextToTextPipeline.__init__s. $)&)$) HIr NrHc @i} i} i} || d<||| d<||| d<||| d<|#d| vri| d<d| dvr td|| dd<|1|/| td|rtjntj}||tj}||| d<||| d<| | | fS) Nprocessing_kwargstimeoutcontinue_final_messagegenerate_kwargsmax_new_tokenszp'max_new_tokens' is defined twice, once in 'generate_kwargs' and once as a direct parameter, please use only onez>`return_full_text` is mutually exclusive with `return_tensors` return_type)r*rrrr) r,rOrNrLreturn_full_textreturn_tensorsrPrMrHforward_kwargspreprocess_paramspostprocess_paramss r!_sanitize_parametersz,ImageTextToTextPipeline._sanitize_parameterss17-.  +2 i ( ! -:P 6 7  &0?N, -  % 64601>2C#DD +CQN, -.> ?  'K,?) !abb2B*.. H[H[K  %+*=$,,K  "0; } - ! -;Q 7 8 .2DDDr r%r&textc | | td||t|s t| |fi|St |t t tfrt |dt t tfrvt |dtrt| t||fi|S|dgt|z}t||Dcgc]\}}t||}}}t| |fi|St|jddtjd|t| |fi|S| tdt| ||dfi|Scc}}w)a Generate a text given text and the image(s) passed as inputs. Args: images (`str`, `List[str]`, `PIL.Image or `List[PIL.Image]`): The pipeline handles three types of images: - A string containing a HTTP(s) link pointing to an image - A string containing a local path to an image - An image loaded in PIL directly The pipeline accepts either a single image or a batch of images. text (str, List[str], `List[Dict[str, Union[str, PIL.Image]]]`): The text to be used for generation. If a list of strings is passed, the length of the list should be the same as the number of images. Text can also follow the chat format: a list of dictionaries where each dictionary represents a message in a conversation. Each dictionary should have two keys: 'role' and 'content'. 'role' should be one of 'user', 'system' or 'assistant'. 'content' should be a list of dictionary containing the text of the message and the type of the message. The type of the message can be either 'text' or 'image'. If the type is 'image', no text is needed. return_tensors (`bool`, *optional*, defaults to `False`): Returns the tensors of predictions (as token indices) in the outputs. If set to `True`, the decoded text is not returned. return_text (`bool`, *optional*): Returns the decoded texts in the outputs. return_full_text (`bool`, *optional*, defaults to `True`): If set to `False` only added text is returned, otherwise the full text is returned. Cannot be specified at the same time as `return_text`. continue_final_message( `bool`, *optional*): This indicates that you want the model to continue the last message in the input chat rather than starting a new one, allowing you to "prefill" its response. By default this is `True` when the final message in the input chat has the `assistant` role and `False` otherwise, but you can manually override that behaviour by setting this flag. Return: A list or a list of list of `dict`: Each result comes as a dictionary with the following key (cannot return a combination of both `generated_text` and `generated_token_ids`): - **generated_text** (`str`, present when `return_text=True`) -- The generated text. - **generated_token_ids** (`torch.Tensor`, present when `return_tensors=True`) -- The token ids of the generated text. - **input_text** (`str`) -- The input text. Nz0You must at least provide either text or images.r chat_templatea The input data was not formatted as a chat with dicts containing 'role' and 'content' keys, even though this model supports chat. Consider using the chat format for better results. For more information, see https://huggingface.co/docs/transformers/en/chat_templatingz(You must provide text for this pipeline.)r%rW)r*rrE__call__r8listtuplerr9r#r<zipgetattr processorlogger warning_once)r,r%rWrHchatr3chatsrIs r!rZz ImageTextToTextPipeline.__call__s_b >dlOP P  $,|F7K 7#F5f5 5 dT5*5 6:d1gPTV[]aOb;c$q'4(w'T6(:EfEE>"Vc$i/F>A$>OP{tUdE*PPw'888 4>>?D 9 E   [  >7#D3F3 3 <GH Hw64 @KFKK!Qs7D>c>t|tttfrd}|}|}nt|trZ||j dddk(}|j j|j | ||j}|}|j}n|d}|d}|d}t|}t|ttfr t|dkDr|jdd |j d |||jd d |j|j }||d<|S)Nr( assistant)add_generation_promptrMrRrWr%rpaddingTF)r%rWrRlegacy)dtyper)r8r[r\r0r#r$r_apply_chat_template frameworkr%rr< setdefaultto torch_dtype) r,inputsrLrMrKr%rW inputs_text model_inputss r! preprocessz"ImageTextToTextPipeline.preprocess>s. ftUC0 1FD K&$'*1-3__R-@-HK-W*~~99OO.D*D+A#'>> : % f~$Vn ) (F dT5M *s4y1}  ( (D 9%t~~ T^^E Uf "4##" $  + Vr c|in|}|jd}d|vr|dn|d}|jjdi||}|||dS)NrW input_idsdecoder_input_ids)generated_sequence prompt_textrur)popmodelgenerate)r,rrrNrxrurws r!_forwardz ImageTextToTextPipeline._forwarddsi / 7"_"&&v. )4 )DL %,WjJk 1TZZ00S<S?S&8clmmr c|d}t|ttfr|gn|}|d}|d}|tjk(r+t t |Dcgc] }||||dc}S|jj|}|jj|} |tjtjhvrig} t|| D]V\} } | j| } d| cxkrdkr$nn!| j| | t | zdF| j| X| }|tjk(rg}t||D]\}}t|tr||z}nt|tr||jddd k(}|rt|jdd dj!}|d xx|z cc<t#|jdd|jdd|jdd dd|gzd gz}nt#|jd |d gz}|j||}t||Dcgc]&\}}t|tr |jn||d (}}}|Scc}wcc}}w)Nrxrwru) input_textgenerated_token_idsrrrer(rfr)rW)r(r))r~generated_text)r8r0r#rrranger<r_post_process_image_text_to_textrrr]findr;r$r9itemsr[)r, model_outputsrPrM input_textsrwruigenerated_textsdecoded_inputsnew_generated_textstext_generated decoded_inputindex_input_text full_textsrxrnew_textr~recordss r! postprocessz#ImageTextToTextPipeline.postprocessns#M2 '1+T{'K{mQ\ *+?@!+. *,, ,s;/0 +1~FXYZF[\  ..HHI[\GG R :.. 0D0DE E#% 14_n1U ?- #1#6#6}#E (-A-'..~>NQTUbQc>c>e/fg'..~> ?2O *.. .J/2;/P 2+ ^k3/%0>%AN T2-52=1E1Eb1I&1QU`1`.-#' (<(rs  ..757V$   H %   --$U3S =$}J]+]%^_-`,4@A}h}B}r