o íîÔiˆã@s¸ddlZddlmZmZddlmZddlmZddl m Z m Z m Z m Z mZddlmZmZddlmZGd d „d e ƒZGd d „d e d d�ZGdd„de ƒZe de¡e de¡dS)éN)ÚAutoImageProcessorÚ AutoProcessor)Ú BatchFeature)Ú ImageInput)Ú ImagesKwargsÚMultiModalDataÚProcessingKwargsÚProcessorMixinÚUnpack)ÚPreTokenizedInputÚ TextInputé)ÚVectorLLMImageProcessorc@seZdZUeed<eed<dS)ÚVectorLLMImagesKwargsÚ resized_sizeÚ patch_sizeN)Ú__name__Ú __module__Ú __qualname__ÚintÚ__annotations__©rrúA/home/zhangtao/hf_model/vectorllm_hf_0407/processing_vectorllm.pyr s  rc@s$eZdZUeed<ddddœiZdS)ÚVectorLLMProcessorKwargsÚ images_kwargsÚ text_kwargsF)ÚpaddingÚreturn_mm_token_type_idsN)rrrrrÚ _defaultsrrrrrs þÿrF)Útotalc sŠeZdZddgZdZdZd‡fdd„ Z  dded ee Be eBe e Bd e e d e fd d „Zddd„Z  ddd„Zedd„ƒZ‡ZS)ÚVectorLLMProcessorÚimage_processorÚ tokenizerr)ÚQwen2TokenizerZQwen2TokenizerFastNc s2d|_| |j¡|_tƒj||fd|i|¤ŽdS)NzÚ chat_template)Ú image_tokenÚconvert_tokens_to_idsÚimage_token_idÚsuperÚ__init__)Úselfr!r"r$Úkwargs©Ú __class__rrr)szVectorLLMProcessor.__init__ÚimagesÚtextr+Úreturnc KsR|jtfd|jji|¤Ž}i}|dur|jd d|i|d¤Ž}t|tƒs'|g}| ¡}|duri|jj|jj d}t t |ƒƒD])}|j ||vr]||  |j d|d¡||<|j ||vsH||  d|j ¡||<q?|d dd¡}|d d d¡} |j|fi|d¤Ž} | rŸt | d ¡} t | ¡} d| | |jk<|  ¡| d <ti| ¥|¥|d �S)NÚtokenizer_init_kwargsr.réz<|placeholder|>r rÚreturn_tensorsrÚ input_idsÚmm_token_type_ids)ÚdataÚ tensor_typer)Ú _merge_kwargsrr"Ú init_kwargsr!Ú isinstanceÚlistÚcopyrrÚrangeÚlenr%ÚreplaceÚpopÚnpÚarrayÚ zeros_liker'Útolistr) r*r.r/r+Ú output_kwargsZ image_inputsÚnum_image_tokensÚindexr3rÚ text_inputsZ array_idsr5rrrÚ__call__$sFÿþý þýÿ  zVectorLLMProcessor.__call__c s~i}|dur8tj di¡}| |¡| dd¡p|jj‰| dd¡p%|jj‰‡‡fdd„|Dƒ}| ||dœ¡tdi|¤ŽS)Nrrrcsg|]}ˆˆd‘qS)r2r)Ú.0Ú_©rrrrÚ XszAVectorLLMProcessor._get_num_multimodal_tokens..)rFÚnum_image_patchesr)rrÚgetÚupdater!rrr)r*Ú image_sizesr+Z vision_datarrNrrLrÚ_get_num_multimodal_tokensQs ÿz-VectorLLMProcessor._get_num_multimodal_tokensTFcKs|jj|f||dœ|¤ŽS)N)Úskip_special_tokensÚclean_up_tokenization_spaces)r"Ú batch_decode)r*Úgenerated_outputsrSrTr+rrrÚpost_process_image_text_to_text^sÿýüz2VectorLLMProcessor.post_process_image_text_to_textcCs"|jj}|jj}tt ||¡ƒS©N)r"Úmodel_input_namesr!r;ÚdictÚfromkeys)r*Ztokenizer_input_namesZimage_processor_input_namesrrrrYlsz$VectorLLMProcessor.model_input_names)NNN)NNrX)TF)rrrÚ attributesÚimage_processor_classÚtokenizer_classr)rr r r;r rrrIrRrWÚpropertyrYÚ __classcell__rrr,rr s,ýþýü û - ür r)ÚnumpyrAÚ transformersrrZ%transformers.feature_extraction_utilsrZtransformers.image_utilsrZtransformers.processing_utilsrrrr r Z$transformers.tokenization_utils_baser r Úimage_processing_vectorllmrrrr ÚregisterrrrrÚs    Y