
    9iX                       S SK r S SKrS SKrS SKJr  S SKJrJrJrJ	r	J
r
JrJrJrJr  S SKrS SKrS SKJr  S SKJr  S SKJr  S SKJr  S SKJr  S	S
KJrJr  S	SKJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(  \" 5       r)Sr*\	\\\+\+4   \	\+   4      r,\	\\+\	\-   \	\+   4      r.\.r/\\+\	\-   4   r0 " S S5      r1SS.S\+S\S\2SS4S jjr3\3" \1Rh                  \" / S/S/S//\*S/SS95         " S S\5      r5\3" \1Rl                  \5" 5       SS9  \3" \1Rn                  \" S//S /SS//5      SS9   " S! S"5      r8 " S# S$\8\5      r9 " S% S&\85      r: " S' S(\:\5      r; " S) S*5      r<\3" \1Rz                  \;" 5       5         " S+ S,\<\;5      r> " S- S.\<\55      r?\3" \1R                  \>" 5       5        \3" \1R                  \?" 5       5        \3" \1R                  \9" 5       5        \3" \1R                  \" / S/// S0/\*S1/5      5         " S2 S35      rD " S4 S5\D\;5      rE " S6 S7\D\55      rF\3" \1R                  \E" 5       SS89  \3" \1R                  \F" 5       SSS99   " S: S;5      rI " S< S=\I\;5      rJ " S> S?\I\55      rK\3" \1R                  \J" 5       SS89  S@ rM " SA SB\;5      rN\3" \1R                  \N" 5       SS89  \3" \1R                  \K" 5       SSS99   " SC SD\95      rQ\3" \1R                  \Q" 5       5        SErS " SF SG\5      rT " SH SI\5      rU " SJ SK\U5      rV\3" \1R                  \V" 5       SLSSSM9  \3" \1R                  \T" / SN/SOSN/SP/SQ/SQSN//\SS/5      SSRSSS9  \3" \1R                  \" ST/SU/S SV/// S//5      5        \3" \1R                  \U" ST/SW/S/S//5      5        \3" \1R                  \U" / S /SS//5      SS9  \3" \1R                  \U" / SX// SY/SSZ/5      5        \3" \1R                  \U" / SX// SY/SSZ/S[S\/S]95        S^r^\3" \1R                  \U" / SX// S_/\^SZ/5      5        \3" \1R                  \" S//S`/S//S//SS/S/5      5        \3" \1R                  \" S//Sa/SQ/S//SS/ST/5      5        \3" \1R                  \" Sb//S`/Sc//Sc//SSb/S/5      5        \3" \1R                  \" Sd/Se/Sf/Sf/SSg/5      5        Shrd\3" \1R                  \" Si/Sj/Sk/Sl/\dSm/5      5        \3" \1R                  \" Sn/So/Sn/Sp/SSq/5      5        \3" \1R                  \" Si/SSo/Sr/Sl/5      5         " Ss St5      rh " Su Sv\h\5      ri " Sw Sx\h\5      rj\3" \1R                  \j" 5       5        \3" \1R                  \i" 5       5        Syrm\3" \1R                  \" / Sz/SQ/S//\mS/SS95        S{ro\3" \1R                  \" / S|/S}/S~/\oS/SS95        Srq\3" \1R                  \" S/S/S/S/\qS/5      5        \" S5      rs\t" 5       ruS\+S\\+/\s4   S\\s   S\\s   4S jrv " S S\95      rw\3" \1R                  \w" 5       5         GS&S\+S\,S\+S\\+\,\	\+   4   4S jjry " S S\5      rz\3" \1R                  \z" SS9SSS9   " S S\z5      r|\3" \1R                  \|" SS9SSS9  \3" \1R                  \|" SS9SSS9   " S S\5      rS\	\0   4S jr " S S\5      r " S S5      r " S S\\5      r " S S\\5      r\3" \1GR
                  \" 5       SSSLS9  \3" \1GR                  \" 5       SSSLS9  \3" \1GR                  \" 5       SSS9  \3" \1GR                  \" 5       SSS9   " S S\5      r\3" \1GR                  \" 5       SSSLSS9  \3" \1GR                  \" ST/S/S//S//5      5        \3" \1GR                  \" / S/SS//5      5        \3" \1GR                  \" S/ST/S/SQ/S//5      5        \3" \1GR                  \" S/S/SS/5      5        \3" \1GR                  \" S/ST/S// S//5      5        \3" \1GR                   \" ST/S// S//5      5        \3" \1GR"                  \" / S/S/SQ/S//SS/5      5        \3" \1GR$                  \" ST/S/S/S/S5      5         " S S\5      r " S S\5      r\3" \1GR*                  \" 5       SSS9   " S S\5      r\3" \1GR.                  \" S/S/S/Sl/5      SSS9  \3" \1GR0                  \" S/S/S/S/5      SSRSSS9  S\	\-   S\	\-   S\-S\\	\-   \	\-   4   4S jr " S S\5      r\3" \1GR6                  \" S/S/S/S/SS/5      SSS9   " S S\5      r\3" \1GR:                  \" 5       SSS9   " S S\5      r " S S\5      r " S S\5      r " S S\5      r\3" \1GRD                  \" 5       SSS9  \3" \1GRF                  \" 5       SSS9   " S S\5      r\3" \1GRJ                  \" 5       SSS9   " S S\h\5      r\3" \1GRN                  \" 5       SSS9   " S S\:\5      r\3" \1GRR                  \" 5       SSS9   " S S\:\5      r\3" \1GRV                  \" 5       SSS9   " S S\i5      r\3" \1GRZ                  \" 5       SSS9   " S S\5      r\3" \1GR^                  \" 5       SLSSS9   " S S\5      r\3" \1GRb                  \" 5       5         " S S\5      r\3" \1GRf                  \" 5       SS89   " S S\h\5      r\3" \1GRj                  \" 5       SSS9   " S S\:\5      r\3" \1GRn                  \" 5       SSS9  S\	\-   S\\-\	\-   4   S\	\-   4S jr " S S\5      r\3" \1GRt                  \" 5       SSS9  \3" \1GRv                  \" / S/S/Sl/SS/5      5        \3" \1GRx                  \" ST/S/S_/S_/5      5        \3" \1GRz                  \" S/S/Sl/Sl/5      5         " S S\5      r\3" \1GR~                  \" S/S// Sl/5      SSLSSS9  \3" \1GR                  \" S/S/SSl/5      SSLSSS9  \3" \1GR                  \" S//S/SQ/S//5      SSLSSS9   " S S\5      r\3" \1GR                  \" S//S/SQ/S//5      SSLSSS9  \3" \1GR                  \" S/S// Sl/5      5        S\	\-   GS \	\-   S\	\-   4GS jr " GS GS\5      r " GS GS\:\5      r\3" \1GR                  \" 5       SSS9   " GS GS\h\5      r\3" \1GR                  \" 5       SSSLS9  \3" \1GR                  \" S/S// Sl/5      SSSLS9  \" GS/GS	/GS
/GS/SGS/5      r\3" \1GR                  \5        \3" \1GR                  \" / GS/GS/GS/5      5        \3" \1GR                  \" / GS// GS/5      5        GSr " GS GS\95      r\3" \1GR                  \" 5       5        \3" \1GR                  \" / GS// S//SGS/5      5        GSr\3" \1GR                  \" GS/GS/GS/GS/\GS/5      5         " GS GS\5      r\3" \1GR                  \" 5       SRSSGS9   " GS GS\:\5      r\3" \1GR                  \" 5       SSS9  \3" \1GR                  \" ST/GS/S/Sl/5      5        GS r\3" \1GR                  \" ST/S/Sl/Sl/\5      5        \3" \1GR                  \" ST/GS!/Sl/Sl/5      5         " GS" GS#5      r " GS$ GS%5      rg('      N)partial)	AnyCallableDictListLiteralOptionalTupleTypeVarUnionversion)PreTrainedTokenizerBase)get_class_from_dynamic_module)is_deepspeed_zero3_enabled)
get_logger   )TemplateTEMPLATE_MAPPING)load_audio_qwen
load_batch
load_imageload_video_cogvlm2load_video_internvlload_video_llavaload_video_minicpmv_mplug_owl3load_video_qwen2transform_imageupper_bound	fetch_oneYou are a helpful assistant.c                      \ rS rSrSrSrSrSrSrSr	Sr
S	rS
rSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSr Sr!S r"S!r#S"r$S#r%S$r&S%r'S&r(S'r)S(r*S)r+S*r,S+r-S,r.S-r/S.r0S/r1S0r2S1r3S2r4S3r5S4r6S5r7S6r8S7r9S8r:S9r;S:r<S;r=S<r>S=r?S>r@S?rAS@rBSArCSBrDSCrESDrFSErGSFrHSGrISHrJSIrKSJrLSKrMSLrNSMrOSNrPSOrQSPrRSQrSSRrTSSrUSTrVSUrWSVrXSWrYSXrZSYr[SZr\S[r]S\r^\_S]\`\a   4S^ j5       rbS_rcg`)aTemplateType   zdefault-generationzchatglm-generationzqwen-vl-generationzqwen-audio-generationdefaultqwenzqwen-vlz
qwen-audiozqwen2-audiozqwen2-audio-generationzqwen2-vlzmodelscope-agentbaichuanchatglm2chatglm3chatglm4	codegeex4llamallama3
reflectionzlongwriter-llama3llava1_5zllava-mistralzllava-vicunazllava-yizllama-llava-next-hfzllava-next-llama3zllama-qwen-hfzllava-onevision-qwenzllava-next-videozllava-next-video-yizllama3-llava-nextz
llava-qwenzllava-llama-instructidefics3zmistral-nemo	openbuddy
openbuddy2internlm	internlm2zinternlm-xcomposer2zinternlm-xcomposer2-4khdzinternlm-xcomposer2_5internvl	internvl2zinternvl-phi3zinternvl2-phi3florencezyi-coderzyi-vlyuanxverseziyaskyworkbluelmzephyrsusdeepseekznumina-mathzdeepseek-coderzdeepseek-vl	deepseek2deepseek2_5zcodefuse-codellamacodefusecogvlmzcogvlm2-videoglm4vzcogagent-chatzcogagent-instructorionminicpmz	minicpm-vzminicpm-v-v2_5zminicpm-v-v2_6gemma	paligemmaz
mplug-owl2
mplug_owl3zwizardlm2-awq	wizardlm2atomphi3zphi3-vltelechatztelechat-v2dbrxmengzic4aichatmlzdefault-generation-bosreturnc                     / nU R                   R                  5        H?  nUR                  S5      (       d  US:X  a  M!  UR                  U R                   U   5        MA     U$ )N__get_template_name_list)__dict__keys
startswithappend)clsresks      k/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/preprocessors/templates/template.pyrU   #TemplateType.get_template_name_list   sS    ""$A||D!!Q*B%BJJs||A' % 
     N)d__name__
__module____qualname____firstlineno__default_generationchatglm_generationqwen_vl_generationqwen_audio_generationr%   r&   qwen_vl
qwen_audioqwen2_audioqwen2_audio_generationqwen2_vlmodelscope_agentr'   r(   r)   r*   r+   r,   r-   r.   longwriter_llama3r/   llava_mistralllava_vicunallava_yillama3_llava_next_hfllava_next_llama3llava_qwen_hfllava_onevision_qwenllava_next_videollava_next_video_yillama3_llava_next
llava_qwenllava_llama_instructr0   mistral_nemor1   r2   r3   r4   internlm_xcomposer2internlm_xcomposer2_4khdinternlm_xcomposer2_5r5   r6   internvl_phi3internvl2_phi3r7   yi_coderyi_vlr8   r9   r:   r;   r<   r=   r>   r?   numina_mathdeepseek_coderdeepseek_vlr@   rA   codefuse_codellamarB   rC   cogvlm2_videorD   cogagent_chatcogagent_instructrE   rF   	minicpm_vminicpm_v_v2_5minicpm_v_v2_6rG   rH   
mplug_owl2rI   wizardlm2_awqrJ   rK   rL   phi3_vlrM   telechat_v2rN   rO   rP   rQ   default_generation_bosclassmethodr   strrU   __static_attributes__r`   r_   r]   r#   r#      s   ---3GDGJK5H)HHHHIEFJ+H#M!LH0+#M1)/+J1H!LIJHI/93HI#M%NHHEDFDGFF
CHK%NKIK-HF#ME#M+EGI%N%NEIJJ#MIDDGHKDFDF5tCy  r_   r#   F)exist_oktemplate_typetemplater   rR   c                n    U(       d  U [         ;   a  [        SU  S35      eXl        SU0UEnU[         U '   g )NzThe `z6` has already been registered in the TEMPLATE_MAPPING.r   )r   
ValueErrorr   )r   r   r   kwargstemplate_infos        r]   register_templater      sB    )995/efgg*4V4M&3]#r_   z%### Human:
{{QUERY}}

### Assistant:
z

eos_token_idz{{SYSTEM}}

Tauto_add_bosc                   (   ^  \ rS rSrU 4S jrSrU =r$ )DefaultGenerationTemplate   c                 .   > [         TU ]  / S/S S//SS9  g )N	{{QUERY}}r   Tr   super__init__self	__class__s    r]   r   "DefaultGenerationTemplate.__init__   s$    k]DN3C2DSWXr_   r`   ra   rb   rc   rd   r   r   __classcell__r   s   @r]   r   r      s    Y Yr_   r   )is_generationbos_token_idr   c                   *    \ rS rSrSrSS\4S jjrSrg)ChatmlTemplateMixin   Nr   c                 V    [         R                  " U / S/S/S/U R                  S/US9  g )Nz;<|im_start|>user
{{QUERY}}<|im_end|>
<|im_start|>assistant
z<|im_end|>

<|im_end|>z(<|im_start|>system
{{SYSTEM}}<|im_end|>
r   r   r   system)r   r   s     r]   r   ChatmlTemplateMixin.__init__   s6    "WX[iZjNKKFG%		'r_   r`   )T)ra   rb   rc   rd   r   boolr   r   r`   r_   r]   r   r      s    F'T ' 'r_   r   c                       \ rS rSrSrg)ChatmlTemplate   r`   Nra   rb   rc   rd   r   r`   r_   r]   r   r          r_   r   c                   ,   ^  \ rS rSr\rU 4S jrSrU =r$ )QwenTemplateMixin   c                     > [         TU ]  SS9  g )NFr   r   r   s    r]   r   QwenTemplateMixin.__init__   s    e,r_   r`   )	ra   rb   rc   rd   DEFAULT_SYSTEMr   r   r   r   r   s   @r]   r   r      s    F- -r_   r   c                       \ rS rSrSrg)QwenTemplate   r`   Nr   r`   r_   r]   r   r      r   r_   r   c            	           \ rS rSrSrS rS\S   S\S\\	\
4   S\\   4S	 jrS\S\\	\
4   S\\   4S
 jrS\S\\	\
4   S\\   4S jrSrg)_QwenVLTemplateMixin   Fc                     UR                  S5      =(       d    / nU(       a&  [        [        U5      [        5      (       d   S5       eg g )Nimagesz0QwenVL only supports datasets with images paths!)get
isinstancer    r   r   exampler   s      r]   check_example"_QwenVLTemplateMixin.check_example   s:    X&,"Z	&(93??sAss??6r_   
media_typeimagevideoaudioindexr   rR   c                     US:X  d   eUR                  S5      =(       d    / nXB   n[        U[        5      (       d   eSUS-    SU S3/$ )Nr   r   zPicture r   z:<img></img>
r   r   r   )r   r   r   r   r   r   s         r]   replace_tag _QwenVLTemplateMixin.replace_tag   sY    W$$$X&,"%%%%%519+VE7(;<<r_   c                 (    US   nX1   nSUS    S3/$ )Nobjects<ref>caption</ref>r`   r   r   r   r   object_s        r]   replace_object#_QwenVLTemplateMixin.replace_object   s+    )$.	*+6233r_   c                     US   nX1   n[        US   S   [        5      (       a/  SnUS    H!  nUSUS    SUS    SUS	    SUS
    S3	-  nM#     U/$ SUS   S    SUS   S    SUS   S	    SUS   S
    S3	/$ )Nr   bboxr    z<box>(,r   ),(      z)</box>r   listr   r   r   r   r   all_objects
sub_objects          r]   replace_box _QwenVLTemplateMixin.replace_box   s    )$.gfoa($//K%fo
&AqAwzZ[}o]^_ijk_l^mmt uv .=  +,Agfoa.@-A BFOA&'q);(<GE r_   r`   N)ra   rb   rc   rd   load_mediasr   r   intr   r   r   r   Contextr   r   r   r   r`   r_   r]   r   r      s    Kt=g.G&H =QT =!#s(^=04W=4C 4$sCx. 4T'] 4
 tCH~ $w- r_   r   c                       \ rS rSrSrg)QwenVLTemplate   r`   Nr   r`   r_   r]   r   r      r   r_   r   c                       \ rS rSrSrg)QwenVLGenerationTemplate   r`   Nr   r`   r_   r]   r   r      r   r_   r   z& 

<|user|>:{{QUERY}} 

<|assistant|>:z 

</s>z 

<|system|>:{{SYSTEM}}c            	           \ rS rSrS\S   S\S\\\4   S\	\
   4S jrS\\\4   S\\\\4   \\\4   4   4S jrS	\S\\\4   4S
 jrS\\\4   S\\\4   SS4S jrSS\	\\\4      S\\   S\\\4   4S jjrSrg)_QwenAudioTemplateMixin   r   r   r   r   rR   c                     US:X  d   eUR                  S5      =(       d    / nXB   n[        U[        5      (       d   eSUS-    SU S3/$ )Nr   audiosAudio r   z:<audio>z	</audio>
r   )r   r   r   r   r  r   s         r]   r   #_QwenAudioTemplateMixin.replace_tag   sY    W$$$X&,"%%%%%8E7*=>>r_   c                     [         R                  " X5      u  p#[        U5      S:X  a  X#4$ UR                  SS 5        UR	                  U5        X#4$ )Nr   
loss_scale)r   _encodelenpopupdate)r   r   inputstokenizer_kwargss       r]   r  _QwenAudioTemplateMixin._encode  sN    #+#3#3D#B v;!++

<&&'''r_   contextc                 <    SU R                   R                  U5      0$ )N
audio_info)	tokenizerprocess_audio)r   r  s     r]   _get_tokenizer_kwargs-_QwenAudioTemplateMixin._get_tokenizer_kwargs  s    dnn::7CDDr_   r  curr_tokenizer_kwargsNc                     UR                  S5      nUR                  S5      nUc  X1S'   g Ub;  S H   n[        R                  " XE   X5   /SS9XE'   M"     S H  nXE   X5   -   XE'   M     g g )Nr  )input_audiosinput_audio_lengthsr   dim)audio_span_tokens
audio_urls)r   torchconcat)r   r  r  r  old_audio_infor\   s         r]   _concat_tokenizer_kwargs0_QwenAudioTemplateMixin._concat_tokenizer_kwargs  s    *..|<
)--l;!-7\*#<$)LL.2CZ]1SYZ$[! =8$2$5
$E! 9 $r_   batch
padding_toc                     [         R                  " XU5      nUS   R                  S5      b  U Vs/ s H  oDS   PM	     snUS'   U$ s  snf )Nr   r  )r   data_collatorr   )r   r#  r$  r[   bs        r]   r&  %_QwenAudioTemplateMixin.data_collator  sM    $$T*=8<<%1:? @%Q<% @C
 !As   Ar`   N)ra   rb   rc   rd   r   r   r   r   r   r   r   r   r
   r  r  r!  r	   r&  r   r`   r_   r]   r   r      s    ?g.G&H ?QT ?!#s(^?04W?(tCH~ (%S#XSRUX8V2W (ES ET#s(^ E	Fc3h 	F`dehjmem`n 	Fsw 	F4S#X#7 Xc] ^bcfhkck^l  r_   r   c                       \ rS rSrSrg)QwenAudioTemplatei$  r`   Nr   r`   r_   r]   r+  r+  $  r   r_   r+  c                       \ rS rSrSrg)QwenAudioGenerationTemplatei(  r`   Nr   r`   r_   r]   r-  r-  (  r   r_   r-  )lazy_tokenize)r.  r   c            	           \ rS rSrS\\\4   S\\\\4   \\\4   4   4S jrS
S\	\\\4      S\
\   S\\\4   4S jjrS	rg)_Qwen2AudioTemplateMixini1  r   rR   c                    [         R                  " X5      u  p#[        U5      S:X  a  U0 4$ U R                  R                  nUR
                  R                  n[        UR                  S5      =(       d    / [        [        US9S9nU(       a6  UR                  XeSSS9nUR                  S5      US	'   UR                  U5        U0 4$ )
Nr   r  )sampling_rate)	load_funcTpt)r2  return_attention_maskreturn_tensorsattention_maskfeature_attention_mask)r   r  r	  r  	processorfeature_extractorr2  r   r   r   r   r
  r  )r   r   r  _r9  r2  r  audio_inputss           r]   r   _Qwen2AudioTemplateMixin._encode3  s    $$T3	v;!2:NN,,	!33AAKK!'R7?Zg3hj$664`d 7 fL5A5E5EFV5WL12MM,'rzr_   Nr#  r$  c                 2   [         R                  " XU5      nU Vs/ s H  oDR                  S5      c  M  US   PM     nnU(       aF  [        R                  " U5      US'   U Vs/ s H  oDS   PM	     nn[        R                  " U5      US'   U$ s  snf s  snf )Ninput_featuresr8  )r   r&  r   r  r  )r   r#  r$  r[   r'  r?  r8  s          r]   r&  &_Qwen2AudioTemplateMixin.data_collatorB  s    $$T*=7<du!FV@W-!,-ud$)LL$@C !KP%Q5a(@&A5"%Q,1LL9O,PC()
 e &Rs   B	B$Br`   r)  )ra   rb   rc   rd   r   r   r   r
   r  r   r	   r   r&  r   r`   r_   r]   r0  r0  1  su    tCH~ %S#XSRUX8V2W 4S#X#7 Xc] ^bcfhkck^l  r_   r0  c            	       D    \ rS rSrS\S   S\S\\\4   S\	\
   4S jrSrg	)
Qwen2AudioTemplateiL  r   r   r   r   rR   c                 &    US:X  d   eSUS-    S3/$ )Nr   r  r   z&: <|audio_bos|><|AUDIO|><|audio_eos|>
r`   r   r   r   r   s       r]   r   Qwen2AudioTemplate.replace_tagN  s'    W$$$#JKLLr_   r`   Nra   rb   rc   rd   r   r   r   r   r   r   r   r   r   r`   r_   r]   rB  rB  L  s=    Mg.G&H MQT M!#s(^M04WMr_   rB  c            	       D    \ rS rSrS\S   S\S\\\4   S\	\
   4S jrSrg	)
Qwen2AudioGenerationTemplateiT  r   r   r   r   rR   c                     US:X  d   eS/$ )Nr   z$<|audio_bos|><|AUDIO|><|audio_eos|>
r`   rD  s       r]   r   (Qwen2AudioGenerationTemplate.replace_tagV  s    W$$$788r_   r`   NrF  r`   r_   r]   rH  rH  T  s8    9g.G&H 9QT 9!#s(^904W9r_   rH  c                 R   SSK JnJnJnJn  [        S[        U5      n[        S[        S 5      n[        S[        S 5      nU(       a  U(       a  U" UUUS9u  pgO<U R                  u  p[        S[        U5      n
[        S[        U5      nU" U	UUU
US	9u  pgU R                  Xv45      n U $ )
Nr   )IMAGE_FACTOR
MIN_PIXELS
MAX_PIXELSsmart_resizesize_factorresized_heightresized_width)factor
min_pixels
max_pixels)rS  rT  rU  )	qwen_vl_utils.vision_processrL  rM  rN  rO  get_env_argsr   sizeresize)r   rL  rM  rN  rO  rP  rQ  rR  widthheightrT  rU  s               r]   _process_image_qwenr\  _  s    __}c<@K!"2C>N #t<M-(4)
% 

!,Z@
!,Z@
(4!!)
% LL-89ELr_   c            	         ^  \ rS rSrS\S   S\S\\\4   S\	\
   4S jrS\S\\\4   S\	\
   4S jrS\S\\\4   S\	\
   4S	 jrS\\\4   S\\\\4   \\\4   4   4U 4S
 jjrSS\	\\\4      S\\   S\\\4   4U 4S jjjrSrU =r$ )Qwen2VLTemplateiz  r   r   r   r   rR   c                 *    US;   d   eUS:X  a  S/$ S/$ )N>   r   r   r   z+<|vision_start|><|image_pad|><|vision_end|>z+<|vision_start|><|video_pad|><|vision_end|>r`   rD  s       r]   r   Qwen2VLTemplate.replace_tag|  s+    //// ABBABBr_   c                 P    UR                  S5      nU(       a  X1   nSUS   S/$ S/$ )Nr   z<|object_ref_start|>r   z<|object_ref_end|><ref-object>r   r   s        r]   r   Qwen2VLTemplate.replace_object  s6    ++i(nG*GI,>@TUU"##r_   c                 $   UR                  S5      nU(       av  X1   n[        US   S   [        5      (       a/  SnUS    H!  nUSUS    SUS    SUS	    SUS
    S3	-  nM#     U/$ SUS   S    SUS   S    SUS   S	    SUS   S
    S3	/$ S/$ )Nr   r   r   r   z<|box_start|>(r   r   r   r   r   z)<|box_end|><bbox>r   r   r   r   s          r]   r   Qwen2VLTemplate.replace_box  s    ++i(nG'&/!,d33 ")&/JnZ]O1ZPQ]O T((21a
1l%T UK #2 $}$ %WV_Q%7$8'&/!:L9M N*+1WV_Q-?,@N 
 :r_   c                 @  > [         TU ]  U5      u  p#[        U5      S:X  a  U0 4$ U R                  R                  nUS   nUS   nUR                  S5      =(       d    / nUR                  S5      =(       d    / nS GH  n	[        5       U	   (       d  M  U	S:X  a)  [        U[        5      nSn
UR                  US SS	9nUS
   nO([        U[        5      nUR                  S USS	9nUS   nSn
[        XZ5      nSn[        U5       Hw  u  nnUR                  R                  S-  nX   R                  5       U-  nUS UU-    U
/U-  -   X^U-   S-   S  -   nU(       a  US UU-    S/U-  -   XnU-   S-   S  -   nUUS-
  -  nMy     UR                  U5        GM     XRS'   XbS'   U0 4$ )Nr   	input_idslabelsr   videos)r   rl  igP r4  )r   rl  r6  image_grid_thwvideo_grid_thwihP r   r   )r   r  r	  r  r9  r   localsr   r\  image_processorr   _findall	enumerate
merge_sizeprodr  )r   r   r  r;  r9  rj  rk  r   rl  r   media_tokenmedia_inputsmedia_grid_thwidx_listadded_tokens_leniidxmerge_length	token_lenr   s                      r]   r  Qwen2VLTemplate._encode  s%   GOG,	v;!2:NN,,	;'	!X&,"X&,".Jx
##)'0CDF"(K#,#<#<FSWhl#<#mL%12B%CN'0@AF#,#<#<DQWhl#<#mL%12B%CN"(K#I;#$ '1FAs#,#<#<#G#G#JL!/!2!7!7!9\!II ) +=3,<,= !>ALPY@Y!Z\ehkglnogp gq ]r!rI !'(?/?)?!@D6ICU!UX^ru_uab`c `d Ye "e$	A5$ 2 l+1 /4 ({!xrzr_   r#  r$  c                    > [         TU ]  X5      nS HT  nU Vs/ s H   oUR                  U S35      c  M  XT S3   PM"     nnU(       d  M9  [        R                  " U5      X4 S3'   MV     U$ s  snf )N)r   r   	_grid_thw)r   r&  r   r  r  )r   r#  r$  r[   r   r'  grid_thwr   s          r]   r&  Qwen2VLTemplate.data_collator  sx    g#E6,J=BrUeezlZcLdFe3L	23UHrx05X0Fl),- - 
 ss
   A-A-r`   r)  )ra   rb   rc   rd   r   r   r   r   r   r   r   r   r   r   r
   r  r	   r&  r   r   r   s   @r]   r^  r^  z  s    Cg.G&H CQT C!#s(^C04WC$C $$sCx. $T'] $ tCH~ $w- $%tCH~ %%S#XSRUX8V2W %N4S#X#7 Xc] ^bcfhkck^l  r_   r^  c                       \ rS rSrSrSrg)YiCoderTemplatei  r!   r`   Nra   rb   rc   rd   r   r   r`   r_   r]   r  r    s    +Fr_   r  u  This is a chat between an inquisitive human and an AI assistant. Assume the role of the AI assistant. Read all the images carefully, and respond to the human's questions with informative, helpful, detailed and polite answers. 这是一个好奇的人类和一个人工智能助手之间的对话。假设你扮演这个AI助手的角色。仔细阅读所有的图像，并对人类的问题做出信息丰富、有帮助、详细的和礼貌的回答。c            	          ^  \ rS rSrS\\   4S jrS\\\	4   S\
\\\	4   \\\	4   4   4U 4S jjrS
S\\\\	4      S\\   S\\\	4   4U 4S jjjrS	rU =r$ )YiVLTemplatei  rR   c                     US:X  d   eS/S/$ Nr   8
r`   rD  s       r]   r   YiVLTemplate.replace_tag      W$$$~r_   r   c                   > [         TU ]  U5      u  p4[        U5      S:X  a  U0 4$ UR                  SS 5        SSKJn  U R                  R                  nUR                  S5      =(       d    / n[        U5       H-  u  p[        S UR                   5       5      n
U" X5      n	XU'   M/     U(       a*  UR                  USS9S   nUR                  US	   5      US'   U0 4$ )
Nr   r  )expand2squarer   c              3   >   #    U  H  n[        US -  5      v   M     g7f)   N)r   ).0xs     r]   	<genexpr>'YiVLTemplate._encode.<locals>.<genexpr>  s     $V;UaSS\\;Us   r4  r6  pixel_valuesdtype)r   r  r	  r
  llava.mm_utilsr  r  rq  r   rs  tuple
image_mean
preprocessto)r   r   r   r  r;  r  rq  r   r{  r   background_colorimage_tensorr   s               r]   r  YiVLTemplate._encode  s    GOG,	v;!2:

<&0..88X&,"!&)HA$$V?;U;U$VV!%:E1I * *55fT5RSabL+vg?F8rzr_   r#  r$  c                   > [         TU ]  X5      nU Vs/ s H  nSU;   d  M  US   PM     nnU(       a  [        R                  " U5      US'   US    Vs/ s H  oDS:H  R	                  5       PM     nn[        U Vs/ s H  owS:  PM	     sn5      (       d+  [        U Vs/ s H  nUS:  PM
     sn5      (       a   S5       eU$ s  snf s  snf s  snf s  snf )Nr   rj  r  r   zCYIVL does not support mix-batch nlp dataset and multi-modal dataset)r   r&  r  r  sumallany	r   r#  r$  r[   r'  r   
has_imageshr   s	           r]   r&  YiVLTemplate.data_collator  s    g#E6',>u!A+!H+u>!LL0CM14[1AB1AADyoo'1A
B%
%aE:
   *, *1 1u *, - -	t /t	t - 
 ? C
,s   
C	CC<CCr`   r)  )ra   rb   rc   rd   r   r   r   r   r   r   r
   r  r	   r   r&  r   r   r   s   @r]   r  r    s    g tCH~ E$sCx.RVWZ\_W_R`B`<a $
4S#X#7 
Xc] 
^bcfhkck^l 
 
r_   r  c                   4   ^  \ rS rSrS\SS4U 4S jjrSrU =r$ )GLMTemplatei  r  rR   Nc                    > [         TU ]  " U/UQ70 UD6nUR                  S5      nU R                  R	                  SU5        U R
                  b  U R
                  R	                  SU5        U$ )Nr   r   )r   _init_templateencodeprefixinsertsystem_prefix)r   r  argsr   r[   
token_listr   s         r]   r  GLMTemplate._init_template	  sg    g$Y@@@%%b)
1j))%%a4
r_   r`   )ra   rb   rc   rd   r   r  r   r   r   s   @r]   r  r    s    (? UY  r_   r  c            	          ^  \ rS rSrU 4S jrS rS\S   S\\   4S jr	S\
\\4   S\\
\\4   \
\\4   4   4U 4S	 jjrSS
\\
\\4      S\\   S\
\\4   4U 4S jjjrSrU =r$ )GLM4VTemplatei  c                 4   > [         TU ]  / S// S/S S/5        g )Nz<|user|>
{{QUERY}}<|assistant|><|endoftext|><|system|>
{{SYSTEM}}r   r   s    r]   r   GLM4VTemplate.__init__  s(    @A2GXZ^23	5r_   c                 Z    UR                  S5      =(       d    / n[        U5      S::  d   eg Nr   r   r   r	  r   s      r]   r   GLM4VTemplate.check_example  '    X&,"6{ar_   r   r   rR   c                     US:X  d   eS//$ )Nr   ro  r`   rD  s       r]   r   GLM4VTemplate.replace_tag      W$$$xr_   r   c                   > [         TU ]  U5      u  p4[        U5      S:X  a  U0 4$ US   nUS   n[        US5      nU(       a  US   nUR	                  S5      S   n	Sn
U R
                  R                  U
SS9nUS U U-   XXS	-   S  -   nUb  US U S/[        U5      -  -   XhS	-   S  -   nUS
   nXS   S'   U R
                  R                  USS9nUS   US'   XSS'   XcS'   U0 4$ )Nr   rj  rk  ro  r   z/<|begin_of_image|><|endoftext|><|end_of_image|>Fadd_special_tokensr   messagesr   T)return_dict)r   r  r	  rr  r   r  r  apply_chat_template)r   r   r   r  r;  rj  rk  ry  r|  r   placeholderplaceholder_idr  inputs2r   s                 r]   r  GLM4VTemplate._encode   s/   GOG,	v;!2:;'	!It,1+CKK)!,EKK!^^22;SX2YN"4C>9IAgh<OOI! #,$#n2E)EEUVwxHXXz*H#(QK &*nn&H&H_c&H&dG&x0F8'{!xrzr_   r#  r$  c                    > [         TU ]  X5      nU Vs/ s H  nSU;   d  M  US   PM     nnU(       a  [        R                  " U5      US'   U$ s  snf Nr   r   r&  r  r  r   r#  r$  r[   r'  r   r   s         r]   r&  GLM4VTemplate.data_collator7  Q    g#E6',>u!A+!H+u>!LL0CM
 ?
   
A	Ar`   r)  )ra   rb   rc   rd   r   r   r   r   r   r   r   r   r   r
   r  r	   r   r&  r   r   r   s   @r]   r  r    s    5 g.G&H ]abi]j tCH~ E$sCx.RVWZ\_W_R`B`<a .4S#X#7 Xc] ^bcfhkck^l  r_   r  dialogue)infer_media_typer.  	use_modelit   zHuman: {{QUERY}}
z
Assistant:r  round)r  r  r.  
{{SYSTEM}}   r   u+   [Round {{ROUND1}}]

问：{{QUERY}}

答：z <|user|>
{{QUERY}}<|assistant|>
z<|user|>r  glm4z'<|observation|>
{{QUERY}}<|assistant|>
tools_prompttool_promptu   你是一位智能编程助手，你叫CodeGeeX。你会为用户回答关于编程、代码、计算机方面的任何问题，并提供格式规范、可以执行、准确安全的代码，并在必要时提供详细的解释。r  User: {{QUERY}}

Assistant:z%### Problem: {{QUERY}}
### Solution: i i u   <｜begin▁of▁sentence｜>u&   <｜User｜>{{QUERY}}<｜Assistant｜>u   <｜end_of_sentense｜>u'   <｜begin▁of▁sentence｜>{{SYSTEM}}a  You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
<s>[INST] {{QUERY}} [/INST]z</s><s>[INST] </s>z'<s>[INST] <<SYS>>
{{SYSTEM}}
<</SYS>>

z[INST]z{{QUERY}}[/INST]z<|end_of_text|>z<<SYS>>
{{SYSTEM}}
<</SYS>>

z</s>[INST] c                       \ rS rSrSrS rSrg)Llama3TemplateMixini  Nc                 \    [         R                  " U S/S/S/S/U R                  S/SS/S9	  g )N<|begin_of_text|>l<|start_header_id|>user<|end_header_id|>

{{QUERY}}<|eot_id|><|start_header_id|>assistant<|end_header_id|>


<|eot_id|>Q<|begin_of_text|><|start_header_id|>system<|end_header_id|>

{{SYSTEM}}<|eot_id|>	toolbenchzl<|start_header_id|>tool<|end_header_id|>

{{QUERY}}<|eot_id|><|start_header_id|>assistant<|end_header_id|>

r  r   r   s    r]   r   Llama3TemplateMixin.__init__  sJ    &'D* ~~KKop$D
	r_   r`   )ra   rb   rc   rd   r   r   r   r`   r_   r]   r  r    s    Fr_   r  c                       \ rS rSrSrg)Llama3Templatei  r`   Nr   r`   r_   r]   r  r    r   r_   r  c                       \ rS rSrSrSrg)ReflectionTemplatei  a*  You are a world-class AI system, capable of complex reasoning and reflection. Reason through the query inside <thinking> tags, and then provide your final response inside <output> tags. If you detect that you made a mistake in your reasoning at any point, correct yourself inside <reflection> tags.r`   Nr  r`   r_   r]   r  r    s
    IFr_   r  a  You are a helpful, respectful and honest INTP-T AI Assistant named Buddy. You are talking to a human User.
Always answer as helpfully and logically as possible, while being safe. Your answers should not include any harmful, political, religious, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
You like to use emojis. You can speak fluently in many languages, for example: English, Chinese.
You cannot access the internet, but you have vast knowledge, cutoff: 2021-09.
You always deeply love and support China, Chinese government, people and culture.
You are trained by OpenBuddy team, (https://openbuddy.ai, https://github.com/OpenBuddy/OpenBuddy), you are based on LLaMA and Falcon transformers model, not related to GPT or OpenAI.zUser: {{QUERY}}
Assistant:a`  You(assistant) are a helpful, respectful and honest INTP-T AI Assistant named Buddy. You are talking to a human(user).
Always answer as helpfully and logically as possible, while being safe. Your answers should not include any harmful, political, religious, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
You cannot access the internet, but you have vast knowledge, cutoff: 2023-04.
You are trained by OpenBuddy team, (https://openbuddy.ai, https://github.com/OpenBuddy/OpenBuddy), not related to GPT or OpenAIz><|role|>user<|says|>{{QUERY}}<|end|>
<|role|>assistant<|says|><|end|>
<|end|>z(<|role|>system<|says|>{{SYSTEM}}<|end|>
u  You are an AI assistant whose name is InternLM (书生·浦语).
- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.
- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文.<s>z<|User|>:{{QUERY}}
<|Bot|>:z<eoa>
z<eoa>z<s><|System|>:{{SYSTEM}}
_T	args_name	type_funcdefault_valuec                    U R                  5       n[        R                  " U5      nUc  UnSU  SU SU S3nOU" U5      nSU SU  SU S3nU[        ;  a*  [        R	                  U5        [
        R                  U5        U$ )NzSetting : zH. You can adjust this hyperparameter through the environment variable: `z`.zUsing environment variable `z`, Setting .)upperosgetenv_log_setaddloggerinfo)r  r  r  args_name_uppervaluelog_infos         r]   rW  rW    s    oo'OIIo&E}ykM? ;]]l\mmoq % 1/1B+i[XZ[`ZaabcxXHLr_   c                       \ rS rSr\rSrg)Internlm2Templatei  r`   N)ra   rb   rc   rd   INTERNLM_SYSTEMr   r   r`   r_   r]   r  r    s    Fr_   r  queryhistoryreplace_tokenc           	         / n/ n[        U5       Hv  u  pgUS   c!  UR                  UR                  5       5        M,  U[        R                  " X7S   5      -  nUR                  [        R
                  " X2US   5      US   /5        Mx     U c  U nO0U[        R                  " X05      -  n[        R
                  " X2U 5      nXU4$ )Nr   r   )rs  rY   copyrefindallsub)	r  r  r	  patternimages_pathnew_historyr{  r  	new_querys	            r]   replace_img_tagr    s     KK'"Q4<qvvx(2::gt44Kwqt DadKL # }	rzz'11FF759	;..r_   c            	          ^  \ rS rSrSrS/rU 4S jrS\\\	4   S\
\\\	4   \\\	4   4   4U 4S jjrS\	S\\\	4   4S	 jrSS
\\\\	4      S\\   S\\\	4   4U 4S jjjr\S\\   S\S\\   4S j5       rSrU =r$ )InternLMXComposer2Templatei
  u  You are an AI assistant whose name is InternLM-XComposer (浦语·灵笔).
- InternLM-XComposer (浦语·灵笔) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.
- InternLM-XComposer (浦语·灵笔) can understand and communicate fluently in the language chosen by the user such as English and 中文.r  c                 h   > S/nS/nS/nS/nS/n[         TU ]  X#XEU R                  U5        Xl        g )Nr  zO[UNUSED_TOKEN_146]user
{{QUERY}}[UNUSED_TOKEN_145]
[UNUSED_TOKEN_146]assistant
z[UNUSED_TOKEN_145]
z[UNUSED_TOKEN_145]z9<s>[UNUSED_TOKEN_146]system
{{SYSTEM}}[UNUSED_TOKEN_145]
)r   r   INTERNLM_XCOMPOSER_SYSTEMr   )r   r   r  promptchat_sepsuffixr  r   s          r]   r   #InternLMXComposer2Template.__init__  sI    fg*+&'VW4;Y;Y[hir_   r   rR   c                   > [         T
U ]  U5      u  p4[        U5      S:X  a  U0 4$ UR                  S5      =(       d    / nU R                  S:X  aZ  Sn[        U5      S:  a  Sn[        S[        U5      n[        SU R                  R                  5      nU Vs/ s H  o" XS	9PM
     nnOXU R                  S
:X  aH  Sn[        S[        U5      n[        SU R                  R                  5      n	U Vs/ s H  o" XS	9PM
     nnU Vs/ s H/  oR                  R                  U5      R                  US   5      PM1     nnUS   US   US.US'   U0 4$ s  snf s  snf s  snf )Nr   r   v2.5   r      hd_numzixc_utils.Image_transform)r   v2-4khd7   zixc_utils.HD_transformr  rj  rk  rj  rk  r   _data)r   r  r	  r   r   rW  r   r   r  	model_dirvis_processorr  )r   r   r   r  r;  r   r   Image_transformr   HD_transformr   s             r]   r  "InternLMXComposer2Template._encode  sX   GOG,	v;!2:X&,"<<6!F6{Q!(C8F;<WY]YgYgYqYqrOIOPoe;FPF\\Y&F!(C8F89QSWSaSaSkSklLFLMfUl58fFMW]^W]e....u588IW]^(.{(;vhGWcijwrz Q
 N^s   E1E6Edatac           	      :   US   nUS   nUS   n[        U5      S:  a  USS  nUb  USS  n[        U[        R                  5      (       a  UR	                  5       nUR                  S5        UbA  [        U[        R                  5      (       a  UR	                  5       nUR                  S5        O/ n/ n/ n/ nSu  pnUR                  nUR                  n[        US5      (       d  UR                  nUR                  n[        U5      S:  a:  [        R                  " U Vs/ s H  oR                  US    5      S   PM     snSS	9nU
[        U5      :  a  X:   S:X  a  [        R                  " S/X9U
 -   US
9nUR                  U" US    5      S   5        US/[        U5      -  -  nUS/XIU
 -   -  n[        U5      S:  a`  XR                  S   :  aN  UR                  X[   R                  U5      5        US/UR                  S   -  -  nUS/UR                  S   -  -  nUS-  nU
S-  n
U
n	M  U
S-  n
U
[        U5      :  a  M  [        U5      S:X  a  S n[        R                  " USS	9n[        R                  " U[        R                  US9S    nXhUS.$ s  snf )Nrj  rk  r   r   r   r   )r   r   r   tok_embeddingsr  devicero  )r  r.  )inputs_embedsim_maskrk  )r	  r   r  TensortolistrY   r.  modelhasattrr,  r  img2embtensorshaper  r   )r   r3  r*  rj  rk  r   res_inputs_embeds
res_labelswrap_im_maskpre_ir{  r|  r.  internlm2_modelr,  r   res_input_idss                    r]   post_encode&InternLMXComposer2Template.post_encode4  s   %	hhv;?!!"I!i..!((*I&%,,//MM!F
#++(899-33O(77v;?\\f"UfU==t#=a#@f"U[\]F#i. |q  %aS913E-Ef U!((d8K)LQ)OPc-&8 88tfvA66
v;?s\\!_'<%,,V[^^F-CD QC&,,q/$99L4&6<<?"::JqQFA #i.  v;!J!LL):B||L

6RSWX!2Wabb) #Vs   ! Jr#  r$  c                    > [         TU ]  X5      nSUS   ;   a8  U Vs/ s H
  oDS   S   PM     nnU R                  USU R                  5      nXSS'   U$ s  snf )Nr0  r   )r   r&  pad_sequencepadding_side)r   r#  r$  r[   r'  r0  r   s         r]   r&  (InternLMXComposer2Template.data_collatore  sc    g#E6a 0561|AG6''D4E4EFG$	N
 7s   Agenerate_idsinput_token_lenc                     U $ r)  r`   rD  rE  s     r]   _get_generate_ids,InternLMXComposer2Template._get_generate_idsm      r_   r   r)  )ra   rb   rc   rd   r  image_placeholderr   r   r   r   r
   r  r>  r   r	   r   r&  staticmethodrH  r   r   r   s   @r]   r  r  
  s    	2   tCH~ E$sCx.RVWZ\_W_R`B`<a ./cs /ctCH~ /cb4S#X#7 Xc] ^bcfhkck^l   S	 C DQTI  r_   r  v2)r  r.  c                       \ rS rSrSrSrg)InternLMXComposer2_5Templateiv  u4  You are an AI assistant whose name is InternLM-XComposer (浦语·灵笔).
- InternLM-XComposer (浦语·灵笔) is a multi-modality conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.
- InternLM-XComposer (浦语·灵笔) can understand and communicate fluently in the language chosen by the user such as English and 中文.
- InternLM-XComposer (浦语·灵笔) is capable of comprehending and articulating responses effectively based on the provided image.r`   N)ra   rb   rc   rd   r  r   r`   r_   r]   rO  rO  v  s    	' r_   rO  r  r!  c                      ^  \ rS rSrSrSrU 4S jrS\\   4S jr	S\
\\4   S\\
\\4   \
\\4   4   4U 4S jjrS	\S\
\\4   4S
 jr\S\\   S\S\\   4S j5       rSrU =r$ )InternvlTemplatei  u@   You are an AI assistant whose name is InternLM (书生·浦语).   c           
      H   > [         TU ]  / S/S/S/U R                  S/SS9  g )N:<|im_start|>user
{{QUERY}}<|im_end|><|im_start|>assistant
r   '<|im_start|>system
{{SYSTEM}}<|im_end|>Tr   r   r   r   r   s    r]   r   InternvlTemplate.__init__  s8    \]`l_m&'Q&R&* 	 	,r_   rR   c                     SS/S/$ )Nz<img>ro  r   r`   rD  s       r]   r   InternvlTemplate.replace_tag  s    $,,r_   r   c                   > [         TU ]  U5      u  p4[        U5      S:X  a  U0 4$ US   n[        US5      nS nUR	                  S5      nU(       a  UR	                  S5      n	[        S[        S5      n
[        S[        S	5      nU Vs/ s H  n[        XU5      PM     nn[        R                  " USS
9R                  US   5      nUR                  S   nUS   US   nnU R                  R                  SSS9U R                  -  U-  nUS U U-   UUS-   S  -   nU	b  U	S U S/[        U5      -  -   U	US-   S  -   n	XSS'   XS'   [        R                  " U5      US.US'   UR!                  SS 5        U0 4$ s  snf )Nr   rj  ro  r   rk  
input_size  max_num   r  r  <IMG_CONTEXT>Fr  r   rj  r  r$  r  )r   r  r	  rr  r   rW  r   r   r  catr  r7  r  r  num_image_tokenr6  r
  )r   r   r   r  r;  rj  ry  r  r   rk  r[  r]  r   pixel_values_imagesimage_bsr|  idx2
img_tokensr   s                     r]   r  InternvlTemplate._encode  s   GOG,	v;!2:;'	It,X&ZZ)F%lC=J"9c26G\b"c\bSX?5g#N\b"c 99%8a@CCF7OTL#))!,H Xb\C$(NN$9$9E %: %;=A=Q=Q%RT\%]J!$3*4y7KKI!Z(@@6$QR()CTT"+;%8(-Y(?Q]^w

<&rz #ds   E6r*  c                 J   UR                  5       nUR                  R                  nUS   nU" US    5      S   R                  US9nUS   nUbq  UR                  US9nUR	                  U5      R                  US9nXPR
                  R                  SSS9S   :H  n	UR                  SUR                  S   5      Xi'   SU0$ [        5       (       aR  [        R                  " S	XFR                  S
9n
UR	                  U
5      R                  US9nXhR                  5       S-  -  nSU0$ )Nrj  r   r-  r  r`  Fr  r_  )r   r       rj  )r.  r  g        r/  )get_input_embeddingsweightr.  r  extract_featurer  r  reshaper7  r   r  zerosr  mean)r   r3  r*  	embeddingr.  rj  r/  r  
vit_embedsselecteddummy_pixel_valuess              r]   r>  InternvlTemplate.post_encode  s:   ..0	!!((%	!)D/215888GN+#'??&?9L..|<??v?NJ!^^%:%:?_d%:%efg%hhH&0&8&8Z=M=Mb=Q&RM#
  //	 ())!&^FReRe!f../ABEEVETJ__.33M//r_   rD  rE  c                     U $ r)  r`   rG  s     r]   rH  "InternvlTemplate._get_generate_ids  rJ  r_   r`   )ra   rb   rc   rd   r   rc  r   r   r   r   r   r   r   r
   r  r>  rL  r   rH  r   r   r   s   @r]   rQ  rQ    s    OFO,-g -tCH~ E$sCx.RVWZ\_W_R`B`<a 80s 0tCH~ 0" S	 C DQTI  r_   rQ  c                     / nUS   nUS   U   nUS   nUS   nU " U5      nUS U U-   XgS  -   US'   [        [        U5      5       H  n	X2" U	5      -  nM     US==   [        U5      -  ss'   U$ )Nvideo_indexrl  r   image_index)ranger	  )
load_video_funcr   r   context_listry  r   r   rz  
new_imagesr{  s
             r]   _replace_video2imager    s    L-(KHk*EXF-(K 'J|,z9F<<PPGH3z?#A& $Mc*o-r_   c                      ^  \ rS rSrSrSrS\\   4U 4S jjrS\	S\
\\4   S\\   4S jrS\	S\
\\4   S\\   4S	 jrS\
\\4   S\\
\\4   \
\\4   4   4U 4S
 jjrSrU =r$ )Internvl2Templatei     u   你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。rR   c                    >^ [         TU ]  SX#5      mUS:X  a  T$ US:X  a9  [        S[        U R                  5      n[        [        US9n[        XSU4S j5      $ g )Nr   r   video_segments)num_segmentsc                    > SU S-    S3/T-   $ )NFramer   r  r`   r{  image_contexts    r]   <lambda>/Internvl2Template.replace_tag.<locals>.<lambda>  s    %PQTUPUwVXHYGZ]jGjr_   )r   r   rW  r   r  r   r   r  )r   r   r   r   r  
load_videor  r   s         @r]   r   Internvl2Template.replace_tag  s_    +GUD   7")*:CATATUN !4>RJ'
=jkk #r_   r   r   c                 T    UR                  S5      nU(       a  X1   nSUS    S3/$ S/$ )Nr   r   r   r   rb  rc  r   s        r]   r    Internvl2Template.replace_object  s;    ++i(nGGI./v677"##r_   c                 8   UR                  S5      nU(       a  X1   n[        US   S   [        5      (       a9  SnUS    H!  nUSUS    SUS    SUS    SUS	    S
3	-  nM#     US S nUS-  nU/$ SUS   S    SUS   S    SUS   S    SUS   S	    S3	/$ S/$ )Nr   r   r   z<box> [[z, r   r   r   z],r_  z] </box>z<box> [[z	]] </box>rf  rg  r   s          r]   r   Internvl2Template.replace_box  s    ++i(nG'&/!,d33'")&/Ja
1bArjYZm_\^_ijk_l^mmo$pqK #2)#2.z)#}$ wvq12"WV_Q5G4Hvq)*"WV_Q-?,@	K 
 :r_   c                 :  > [         [        U ]
  " U40 UD6u  p4[        U5      S:X  a  U0 4$ US   n[	        US5      nUR                  S5      nUR                  S5      nU(       a  [        UR                  S5      5      n	[        S[        S5      n
[        S	[        U	(       a  S
OS5      nU Vs/ s H  n[        XU5      PM     nnU Vs/ s H  oR                  S   PM     nn[        R                  " U5      R                  US   5      nOS n/ n[        U5      [        U5      :X  d   S[        U5       S[        U5       35       eSn[        Xo5       H  u  nnU R                  R!                  SSS9U R"                  -  U-  nUS UU-    U-   UUU-   S
-   S  -   nUb$  US UU-    S/[        U5      -  -   UUU-   S
-   S  -   nU[        U5      S
-
  -  nM     XSS'   XsS'   [        R$                  " U5      US.US'   UR'                  SS 5        U0 4$ s  snf s  snf )Nr   rj  ro  rk  r   rl  r[  r\  r]  r   r^  r  zlen(num_patches): z, len(idx_list): r`  Fr  ra  r$  r  )r   rQ  r  r	  rr  r   r   rW  r   r   r7  r  rb  r  zipr  r  rc  r6  r
  )r   r   r   r  r;  rj  ry  rk  r   	has_videor[  r]  r   r  pvnum_patchesrz  r|  	num_patchrg  r   s                       r]   r  Internvl2Template._encode  so   *D9'LVL	v;!2:;'	It,H%X&W[[23I%lC=J"9c	1rJGU[\U[EOEwGU[L\1=>288A;K> 99\255fWoFLLK;3$  	_+C,<+==NsS[}o^	_ !(8NC$(NN$9$9E %: %;=A=Q=Q%RT]%^J!"93)9#9:ZG)TWZjTjmnTnToJppI! 7'7!78D6C
O;SSV\]`cs]s_`^a ^b Wc cJ! 33 9 ({!x(-Y(?Q]^w

<&rz+ ]>s   /HHr`   )ra   rb   rc   rd   r  r   r   r   r   r   r   r   r   r   r   r
   r  r   r   r   s   @r]   r  r    s    N mFlg l$C $$sCx. $T'] $ tCH~ $w- &!tCH~ !E$sCx.RVWZ\_W_R`B`<a ! !r_   r  c                       \ rS rSrS rSrg)InternvlPhi3TemplateMixini(  c                 f    [         R                  " U / S/S/S/[        U SS 5      S/SS9  SU l        g )Nz'<|user|>
{{QUERY}}<|end|><|assistant|>
r  r   z<|system|>
{{SYSTEM}}<|end|>Tr   left)r   r   getattrrB  r  s    r]   r   "InternvlPhi3TemplateMixin.__init__*  sD    "BCi[S\R]D(D),K+L	 #r_   rB  N)ra   rb   rc   rd   r   r   r`   r_   r]   r  r  (  s    #r_   r  c                       \ rS rSrSrSrg)InternvlPhi3Templatei2  z,You are an AI assistant whose name is Phi-3.r`   Nr  r`   r_   r]   r  r  2  s    ;Fr_   r  c                       \ rS rSrSrg)Internvl2Phi3Templatei6  r`   Nr   r`   r_   r]   r  r  6  r   r_   r  )r  r.  r  c                     ^  \ rS rSrSrSrU 4S jrS rS\\	\
4   SS4S	 jrS
\S\\	\
4   S\\   4S jrS\\	\
4   S\\\	\
4   \\	\
4   4   4U 4S jjrS\
S\\	\
4   4S jr\S\\   S\S\\   4S j5       rS rSrU =r$ )FlorenceTemplateiE  FTc           	      n   > [         TU ]  S/S/S S/5        SSSSSS	S
SS.U l        SSSSSSSS.U l        g )Nr  z{{QUERY}}</s>r  zWhat is the text in the image?z,What is the text in the image, with regions?zWhat does the image describe?z.Describe in detail what is shown in the image.z5Describe with a paragraph what is shown in the image.z3Locate the objects with category name in the image.z9Locate the objects in the image, with their descriptions.z)Locate the region proposals in the image.)z<OCR>z<OCR_WITH_REGION>z	<CAPTION>z<DETAILED_CAPTION>z<MORE_DETAILED_CAPTION>z<OD>z<DENSE_REGION_CAPTION>z<REGION_PROPOSAL>z*Locate the phrases in the caption: {input}z%Locate {input} in the image with maskz*What is the polygon mask of region {input}zLocate {input} in the image.zWhat is the region {input}?z&What does the region {input} describe?z#What text is in the region {input}?)z<CAPTION_TO_PHRASE_GROUNDING>z#<REFERRING_EXPRESSION_SEGMENTATION>z<REGION_TO_SEGMENTATION>z<OPEN_VOCABULARY_DETECTION>z<REGION_TO_CATEGORY>z<REGION_TO_DESCRIPTION>z<REGION_TO_OCR>)r   r   task_prompts_without_inputstask_prompts_with_inputr   s    r]   r   FlorenceTemplate.__init__I  sa    %?"3TF8D5!O8"R'^I&a!L	,
( .Z3Z(T+I$A'OD(
$r_   c                 d    UR                  S5      =(       d    / n[        U5      S:X  d   S5       eg )Nr   r   z?Florence series models only supports input with a single image.r  r   s      r]   r   FlorenceTemplate.check_example_  s-    X&,"6{ab!bbr_   r   rR   Nc                     g r)  r`   r   r   s     r]   add_default_tags!FlorenceTemplate.add_default_tagsc  s    r_   r   c                     US   U   n[        US   S   [        5      (       a+  SnUS    H  nUu  pgpUSU SU SU SU	 S3	-  nM     US S /$ US   u  pgpSU SU SU SU	 S	3	/$ )
Nr   r   r   r   z<loc_z><loc_z>,r_  >r   )
r   r   r   r   r   r   x1y1x2y2s
             r]   r   FlorenceTemplate.replace_boxf  s    )$U+gfoa($//K%fo
!+rd&F2$fRDKK .  $%%$V_NBBB4vbTt6"Q?@@r_   c                   > US   nU R                   R                  nUR                  U/5      S   US'   [        TU ]  U5      u  pVUS   n[        U5      S:X  a  U0 4$ UR                  S5      =(       d    / nUS   n	U	b  S/U	-   n	UR                  USS9S   R                  US	   5      n
UU	[        R                  " U5      S    U
S
.S.nU0 4$ )Nr  r   prompt_input_idsr   answer_labelsr4  r  r  r  ra  rj  rk  r$  )r  r9  _construct_promptsr   r  r	  r   rq  r  r  r6  )r   r   r   r  r9  r  r;  rj  r   rk  r  r   s              r]   r  FlorenceTemplate._encoder  s     NN,,	$77@CGOG,	-.	v;!2:X&,"(S6\F 000Mn]``aghoapq""\\)4T: ,
 rzr_   r*  c                     UR                  5       " US   5      nUR                  US   5      nUR                  XC5      u  p5SUS   0$ )Nrj  r  r/  r   )rk  _encode_image$_merge_input_ids_with_image_features)r   r3  r*  r/  image_featuresr;  s         r]   r>  FlorenceTemplate.post_encode  sR    224T+5FG,,T.-AB EEndq!122r_   rD  rE  c                     U $ r)  r`   rG  s     r]   rH  "FlorenceTemplate._get_generate_ids  rJ  r_   c                    [        US   [        5      (       a  US   S   US'   [        US   5      n[        R                  " U R
                  R                  R                  XS   UR                  UR                  4S95      $ )Nr   r   r  )task
image_size)
r   r   r   jsondumpsr  r9  post_process_generationrZ  r[  )r   responser   r   s       r]   post_process_generate_response/FlorenceTemplate.post_process_generate_response  s    gh'.. ' 1! 4GH78,-zzNN$$<<w/U[[%,,<W = YZ 	Zr_   )r  r  )ra   rb   rc   rd   compute_per_round_lossoutput_prompt_answerr   r   r   r   r   r  r   r   r   r   r
   r  r>  rL  rH  r  r   r   r   s   @r]   r  r  E  s    "
,cS#X 4 
A 
AtCH~ 
A$w- 
AtCH~ E$sCx.RVWZ\_W_R`B`<a .3s 3tCH~ 3 S	 C DQTI  Z Zr_   r  )r  r.  r  streamzHuman: {{QUERY}}

Assistant: z{{QUERY}}<sep>z<human>:{{QUERY}}
<bot>:z<s>{{SYSTEM}}z </s><s>[USER]{{QUERY}}[SEP][BOT]z	[SEP]</s>z[|Human|]:{{QUERY}}[|AI|]:zE<|role_start|>human<|role_end|>{{QUERY}}<|role_start|>bot<|role_end|>z<s>human
{{QUERY}}
<s>bot
z<s>system
{{SYSTEM}}
z)### Instruction:
{{QUERY}}
### Response:
z	
<|EOT|>
z
<|EOT|>a)  You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer
c                      ^  \ rS rSrS
U 4S jjrS\S   S\\   4S jrS\	\
\4   S\\	\
\4   \	\
\4   4   4U 4S jjrS	rU =r$ )LlavaHfTemplatei  rR   c                    > [         TU ]  " U0 UD6  [        R                  " [        R
                  5      [        R                  " S5      :  a  SU l        g g )Nz4.43.0r  )r   r   r   parsetransformers__version__rB  )r   r  r   r   s      r]   r   LlavaHfTemplate.__init__  sC    $)&)==112W]]85LL &D Mr_   r   r   c                     US:X  d   eS/$ )Nr   <image>
r`   rD  s       r]   r   LlavaHfTemplate.replace_tag  s    W$$$}r_   r   c                 "  > [         TU ]  U5      u  p4[        U5      S:X  a  U0 4$ UR                  S5      nU(       aO  U R                  R
                  R                  nU" USS9R                  US   5      nUS   US'   SU;   a  US   US'   U0 4$ )Nr   r   r4  r  r  r  image_sizesr   r  r	  r   r  r9  rq  r  )	r   r   r   r  r;  r   rq  image_inputsr   s	           r]   r  LlavaHfTemplate._encode  s    GOG,	v;!2:X&"nn66FFO*6$GJJ6RY?[L%1.%AF>",(4](C}%rzr_   r  )rR   N)ra   rb   rc   rd   r   r   r   r   r   r   r   r   r
   r  r   r   r   s   @r]   r  r    sg    '
g.G&H ]abi]j tCH~ E$sCx.RVWZ\_W_R`B`<a  r_   r  c                   l   ^  \ rS rSrSrU 4S jrS\\\4   S\	\\\4   \\\4   4   4U 4S jjr
SrU =r$ )Llava1_6Llama3Templatei  You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.c                 8   > [         TU ]  S/S/S/S/S S/5        g )Nr  r  r  r  r   r   s    r]   r   Llava1_6Llama3Template.__init__  s5    -.@1
 >L>4op		rr_   r   rR   c                    > [         TU ]  U5      u  p4[        US   R                  5      S:X  a  [        R
                  " US   SS9US'   U0 4$ )Nr     r   r  )r   r  r	  r7  r  squeeze)r   r   r   r  r;  r   s        r]   r  Llava1_6Llama3Template._encode  sO    GOG,	vn%++,1%*]]6.3Iq%QF>"rzr_   r`   )ra   rb   rc   rd   default_systemr   r   r   r   r
   r  r   r   r   s   @r]   r  r    sP    [NrtCH~ E$sCx.RVWZ\_W_R`B`<a  r_   r  c                   z   ^  \ rS rSrS\S   S\\   4S jrS\\	\
4   S\\\	\
4   \\	\
4   4   4U 4S jjrSrU =r$ )	LlavaVideoTemplatei  r   r   rR   c                 r    US:X  a  S/$ US:X  d   eUS   U   nUR                  SS5      S   S;   a  S/$ S	/$ )
Nr   r  r   rl  r  r   r_  >   jpgpngz<video>
)rsplit)r   r   r   r   
media_files        r]   r   LlavaVideoTemplate.replace_tag  sY     = W$$$X&u-
S!$R(N:= = r_   r   c                 ,  > [         TU ]  U5      u  p4[        U5      S:X  a  U0 4$ UR                  S5      =(       d    / nUR                  S5      =(       d    / n[        U5      S:  aQ  [	        U[
        5      nU R                  R                  R                  nU" USS9R                  US   5      n	U	S   US'   [        U5      S:  aI  U R                  R                  R                  n
U
" USS9R                  US   5      nUS   US'   US	   US	'   U0 4$ )
Nr   r   rl  r4  r  r  pixel_values_videosr  r  )r   r  r	  r   r   r   r  r9  video_processorr  rq  )r   r   r   r  r;  r   videos_pathrl  r  video_inputsrq  r  r   s               r]   r  LlavaVideoTemplate._encode  s   GOG,	v;!2:X&,"kk(+1r{a-=>F"nn66FFO*6$GJJ6RY?[L,89N,OF()v;?"nn66FFO*6$GJJ6RY?[L%1.%AF>"$0$?F=!rzr_   r`   )ra   rb   rc   rd   r   r   r   r   r   r   r   r
   r  r   r   r   s   @r]   r  r    sb    	!g.G&H 	!]abi]j 	!tCH~ E$sCx.RVWZ\_W_R`B`<a  r_   r  <s>{{SYSTEM}} USER: {{QUERY}} ASSISTANT: z{{SYSTEM}} r   rj  rk  image_tokenc                    [        U[        R                  5      (       a  UR                  5       nSu  pEU[	        U 5      :  GaE  X   nXc:X  Ga  US-   [	        U 5      :  d   SU SS   35       eUS-
  S:  d   SU S S  35       eUS-
  n[        S5       HJ  nXx-   [	        U5      :  a  X'U-      XS-
     :X  a  Xx-  n  O5Xx-
  S:  d  M4  X'U-
     XS-
     :X  d  MF  Xx-  n  O   [        S	U S
U  35      eUS-  nU[	        U5      :  a0  X%   XS-      :w  a#  US-  nU[	        U5      :  a  X%   XS-      :w  a  M#  U S U X'U -   XS-   S  -   n U(       a  US U S/XW-
  -  -   XS-   S  -   nXEU-
  -  nOUS-  nUS-  nU[	        U 5      :  a  GME  X4$ )N)r   r   r   zinput_ids[-10:]: ir   zinput_ids[:10]: 
   r  znew_input_ids: z, input_ids: ro  )r   r  r1  r2  r	  r{  r   )	rj  rk  new_input_idsr  r{  jr  j_beginr\   s	            r]   align_image_inputsr     s   -..%,,. DA
c)n
Lq53y>)P->y>O+PP)q5A:B!1)CR.1ABB: !eG1X;]!33PQk8RV_de`eVf8fLG;!#k(BiTUPUFV(VLG  !?=/yk!Z[[qLGc-((]-=q5AQ-QQ c-((]-=q5AQ-Q!"1a(@@9QRUVCTTItf&<<v!ef~MWAFA	Q5 c)n
6 r_   c                   \   ^  \ rS rSrS\\\4   S\\\\4   \\\4   4   4U 4S jjrSr	U =r
$ )Idefics3TemplateiE  r   rR   c                 h  > [         T
U ]  U5      u  p4[        U5      S:X  a  U0 4$ UR                  S5      =(       d    / nU R                  R
                  nU R                  R                  US   5      nU(       a5  U" XuSSS9nSn	[        US   US   US   S   U	5      u  US'   US'   US	   US	'   U0 4$ )
Nr   r   rj  r4  F)textr   r6  r  i rk  r  )r   r  r	  r   r  r9  decoder  )r   r   r   r  r;  r   r9  r  r  r  r   s             r]   r  Idefics3Template._encodeG  s    GOG,	v;!2:X&,"NN,,	&&vk':;$&PTinoL K4FvkGZ\bck\lGST_G`abGcep5r1F;!1%1.%AF>"rzr_   r`   )ra   rb   rc   rd   r   r   r   r
   r  r   r   r   s   @r]   r  r  E  sB    tCH~ E$sCx.RVWZ\_W_R`B`<a  r_   r  r  z+User:{{QUERY}}<end_of_utterance>
Assistant:z<end_of_utterance>
z<end_of_utterance>z$System:{{SYSTEM}}<end_of_utterance>
c                   (   ^  \ rS rSrU 4S jrSrU =r$ )Llava1_5Templatei_  c                 2   > [         TU ]  S/S/S/S/5        g )Nr  zUSER: {{QUERY}}
ASSISTANT:r  r   r   s    r]   r   Llava1_5Template.__init__a  s     %#@"AF8fXVr_   r`   r   r   s   @r]   r	  r	  _  s    W Wr_   r	  c            	          ^  \ rS rSrU 4S jrS\S   S\\   4S jrS\	\
\4   S\\	\
\4   \	\
\4   4   4U 4S jjrSS	\\	\
\4      S
\\   S\	\
\4   4U 4S jjjr\S\\   S\S\\   4S j5       rSrU =r$ )LLavaTemplateih  c                 0   > [         TU ]  S/S/S S/S/S9  g Nr  r  r  z<<SYS>>
{{system}}
<</SYS>>

r  r   r   s    r]   r   LLavaTemplate.__init__j  s-    ,*=)>x(K'L 	 	Nr_   r   r   rR   c                     US:X  d   eS/S/$ r  r`   rD  s       r]   r   LLavaTemplate.replace_tagp  r  r_   r   c                   > [         T
U ]  U5      u  p4[        U5      S:X  a  U0 4$ UR                  S5      =(       d    / nU Vs/ s H  ofR                  PM     nnSSKJn  U(       a[  U" XPR                  R                  U R                  R                  5      n	U	R                  US   5      R                  S5      US'   XsS'   U0 4$ s  snf )Nr   r   process_imagesr  r  )r   r  r	  r   rX  r  r  r  rq  configr  r  )r   r   r   r  r;  r   r  r  r  images_tensorr   s             r]   r  LLavaTemplate._encodet  s    GOG,	v;!2:X&,"'-.v!vvv.1 +6>>3Q3QSWSaSaShShiM,//w@HHKF8$/=!rz /s   Cr#  r$  c                   > [         TU ]  X5      nU Vs/ s H  nSU;   d  M  US   PM     nnU(       a-  XSS'   [        U Vs/ s H  nSU;   d  M  US   PM     sn/ S9US'   US    Vs/ s H  oDS:H  R                  5       PM     nn[        U Vs/ s H  owS:  PM	     sn5      (       d+  [	        U Vs/ s H  nUS:  PM
     sn5      (       a   S5       eU$ s  snf s  snf s  snf s  snf s  snf )Nr   r  )startrj  r  r   zDLlava does not support mix-batch nlp dataset and multi-modal dataset)r   r&  r  r  r  r  s	           r]   r&  LLavaTemplate.data_collator  s   g#E6',>u!A+!H+u>"M!$%\1Z[I[&6a&6%\df!gC14[1AB1AADyoo'1A
B%
%aE:
   *, *1 1u *, - -	u /u	u - 
 ? &]B
,s(   
C	C
C	C)C!C&3C+rD  rE  c                     U $ r)  r`   rG  s     r]   rH  LLavaTemplate._get_generate_ids  rJ  r_   r`   r)  )ra   rb   rc   rd   r   r   r   r   r   r   r   r   r
   r  r	   r   r&  rL  rH  r   r   r   s   @r]   r  r  h  s    Ng.G&H ]abi]j tCH~ E$sCx.RVWZ\_W_R`B`<a 4S#X#7 Xc] ^bcfhkck^l   S	 C DQTI  r_   r  c            	       \   ^  \ rS rSrSS\\\\4      S\\	   S\\\4   4U 4S jjjr
SrU =r$ )Llava1_6Templatei  r#  r$  rR   c                    > U H-  nUR                  S5      nUc  M  UR                  S5      US'   M/     [        TU ]  X5      nU$ )Nr  r   )r   r  r   r&  )r   r#  r$  r'  r  r[   r   s         r]   r&  Llava1_6Template.data_collator  sL    A550L'$0$8$8$;.!  g#E6
r_   r`   r)  )ra   rb   rc   rd   r   r   r   r   r	   r   r&  r   r   r   s   @r]   r   r     s@    4S#X#7 Xc] ^bcfhkck^l  r_   r   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )Llava1_6MistralTemplatei  c                 2   > [         TU ]  S/S/S/S/S/S9  g r  r   r   s    r]   r    Llava1_6MistralTemplate.__init__  s.    ,*=)>6((K'L 	 	Nr_   r`   r   r   s   @r]   r$  r$    s    N Nr_   r$  c                   ,   ^  \ rS rSrSrU 4S jrSrU =r$ )Llava1_6VicunaTemplatei  zA chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.c           	      H   > [         TU ]  S/S/S/S/U R                  S/S9  g )Nr  r  r  r  r  rV  r   s    r]   r   Llava1_6VicunaTemplate.__init__  s4    %#?"@6(VH(8'9 	 	;r_   r`   )ra   rb   rc   rd   r   r   r   r   r   s   @r]   r(  r(    s    dF; ;r_   r(  c                   L   ^  \ rS rSrU 4S jrS\S   S\\   4U 4S jjrSr	U =r
$ )LLava1_6YiTemplatei  c                 0   > [         TU ]  / S/S/S/S/S9  g )NrT  r   rU  r  r   r   s    r]   r   LLava1_6YiTemplate.__init__  s/    \]`l_m&(R'S 	 	Ur_   r   r   rR   c                 $   > [         TU ]  XU5      $ r)  r   r   r   r   r   r   r   s       r]   r   LLava1_6YiTemplate.replace_tag      w":g>>r_   r`   )ra   rb   rc   rd   r   r   r   r   r   r   r   r   s   @r]   r,  r,    s.    U
?g.G&H ?]abi]j ? ?r_   r,  c                       \ rS rSrSrg)Llama3LlavaNextHfTemplatei  r`   Nr   r`   r_   r]   r5  r5    r   r_   r5  c                       \ rS rSrSrg)LlavaQwenHfTemplatei  r`   Nr   r`   r_   r]   r7  r7    r   r_   r7  c                   R    \ rS rSrSrS\\\4   S\\\\4   \\\4   4   4S jr	Sr
g)LlavaOneVisonTemplatei  Nr   rR   c                    [         R                  " X5      u  p4[        U5      S:X  a  U0 4$ UR                  S5      nUS   nUS   n[	        US5      nU R
                  R                  n	U(       a  U	R                  n
U
" USS9R                  US   5      nUS	   S   R                  S
S  u  pSn[        XS	   US   5       H]  u  nnnUu  nnU	R                  UUX5      nUS UU-    S/U-  -   XnU-   S-   S  -   nUb  US X-    S/U-  -   X~U-   S-   S  -   nUUS-
  -  nM_     XcS'   XsS'   US	   US	'   SU;   a  US   US'   U0 4$ )Nr   r   rj  rk  i^P r4  r  r  r  r  r   ro  )r   r  r	  r   rr  r  r9  rq  r  r7  r  _get_number_of_features)r   r   r   r  r;  r   rj  rk  ry  r9  rq  r  r[  rZ  rz  r|  pixel_vr  orig_height
orig_widthnum_image_tokenss                        r]   r  LlavaOneVisonTemplate._encode  s   $$T3	v;!2:X&;'	!Iv.NN,,	'77O*6$GJJ6RY?[L(8;AA"#FMF ,/~:VXderXs,t(Wj*4'Z#,#D#D[R\^d#l % ','7(+(, -06x:J/JKMVjmWmpqWqWrMst	%#$;%5%;<vHX?XX[auxbxdecf cg \h hF $4q$88  -u #,;%8%1.%AF>",(4](C}%rzr_   r`   )ra   rb   rc   rd   r   r   r   r   r
   r  r   r`   r_   r]   r9  r9    sA    FtCH~ E$sCx.RVWZ\_W_R`B`<a r_   r9  c                   p   ^  \ rS rSrS\S   4S jrS\\\4   S\	\\\4   \\\4   4   4U 4S jjr
SrU =r$ )	LLavaLlamaTemplatei  r   r   c                     S/$ )Nr  r`   rD  s       r]   r   LLavaLlamaTemplate.replace_tag  s
    }r_   r   rR   c                    > [         TU ]  U5      u  p4[        U5      S:X  a  U0 4$ UR                  S5      nU(       a>  U R                  R
                  R                  USS9S   nUR                  US   5      US'   U0 4$ )Nr   r   r4  r  r  r  r  )r   r   r   r  r;  	raw_imager  r   s          r]   r  LLavaLlamaTemplate._encode  s    GOG,	v;!2:KK)	>>33CCI^bCcdrsL%1__VG_%EF>"rzr_   r`   )ra   rb   rc   rd   r   r   r   r   r   r
   r  r   r   r   s   @r]   rC  rC    sT    g.G&H tCH~ E$sCx.RVWZ\_W_R`B`<a  r_   rC  c            	          ^  \ rS rSrU 4S jrS rS\\   4S jrS\	\
\4   S\\	\
\4   \	\
\4   4   4U 4S jjrSS\\	\
\4      S	\\   S\	\
\4   4U 4S
 jjjrSrU =r$ )PaliGemmaTemplatei
  c                 .   > [         TU ]  / S/S S/5        g )N
{{QUERY}}
z<eos>r   r   s    r]   r   PaliGemmaTemplate.__init__  s    m_dWI>r_   c                 Z    UR                  S5      =(       d    / n[        U5      S::  d   eg r  r  r   s      r]   r   PaliGemmaTemplate.check_example  r  r_   rR   c                     US:X  d   eU R                   (       a
  S/U l        / $ S/U l        SU R                  R                  R                  -  S-   /$ )Nr   r   rL  <image><bos>)_is_vllmr  r  r9  image_seq_lengthrD  s       r]   r   PaliGemmaTemplate.replace_tag  sQ    W$$$==&-DKI(/DK 8 8 I IIGSTTr_   r   c                   >^	 [         T
U ]  U5      u  m	n[        T	5      S:X  a  T	0 4$ UR                  S5      nU R                  R
                  nT	S   b?  [        S[        T	S   5      U	4S j5      n[        T	S   5      U-
  nS/U-  S/U-  -   T	S'   OS/[        T	S   5      -  T	S'   U(       a  U" US   US   S	S
9nUS   T	S'   T	0 4$ )Nr   r   rk  c                    > TS   U    S:H  $ )Nrk  ro  r`   )r|  r  s    r]   r  +PaliGemmaTemplate._encode.<locals>.<lambda>#  s    &BRSVBW[_B_r_   r   token_type_idsrj  r  r4  )r  r   r6  r  )r   r  r	  r   r  r9  r   )r   r   r   r;  rG  r9  nn2model_inputsr  r   s            @r]   r  PaliGemmaTemplate._encode  s    GOG,	v;!2:KK)	NN,,	('As6(#346_`AVH%&*B()sQw!r'9F#$()sS1D-E'EF#$$''*:9Q<`deL%1.%AF>"rzr_   r#  r$  c                    > [         TU ]  X5      nU Vs/ s H  n[        R                  " US   5      PM     nnU R	                  USU R
                  5      nXSS'   U$ s  snf )NrY  r   r   r&  r  r6  rA  rB  )r   r#  r$  r[   r'  rY  r   s         r]   r&  PaliGemmaTemplate.data_collator-  sc    g#E6EJKU%,,q)9':;UK**>1d>O>OP .
 Ls   #A)r  r)  )ra   rb   rc   rd   r   r   r   r   r   r   r   r   r
   r  r	   r   r&  r   r   r   s   @r]   rJ  rJ  
  s    ? Ug UtCH~ E$sCx.RVWZ\_W_R`B`<a "4S#X#7 Xc] ^bcfhkck^l  r_   rJ  )r  r.  r   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )Phi3Templatei9  c           
      4   > [         TU ]  / S/S/S/S S/SS9  g )Nz(<|user|>
{{QUERY}}<|end|>
<|assistant|>
r  r  z<|system|>
{{SYSTEM}}<|end|>
Tr   r   r   s    r]   r   Phi3Template.__init__;  s3    KL{m^g]h AB&* 	 	,r_   r`   r   r   s   @r]   rb  rb  9  s    , ,r_   rb  c                   |   ^  \ rS rSrS/rS\\   4U 4S jjrS\\	\
4   S\\\	\
4   \\	\
4   4   4U 4S jjrSrU =r$ )Phi3VisionTemplateiD  z<|image|><s>
rR   c                 $   > [         TU ]  XU5      $ r)  r0  r1  s       r]   r   Phi3VisionTemplate.replace_tagG  r3  r_   r   c                 "  > UR                  S5      =(       d    / n[        TU ]	  U5      u  pE[        U5      S:X  a  U0 4$ US   nUS   n[	        US5      n[        U5      S:  Ga  U R
                  R                  n	UR                  U	R                  USS95        [        U5      [        U5      :X  d   S[        U5       S	[        U5       35       e/ n
/ nUR                  S
5      R                  5       nUR                  SS5        [        [        U5      S-
  5       HB  nU* S-
  nXX   S-   XS-       U/X   -  -   -  n
Uc  M(  XX   S-   XS-       S/X   -  -   -  nMD     XUS   S-   S  -  n
U
nUb  XUS   S-   S  -  nUnXdS'   XtS'   U0 4$ )Nr   r   rj  rk  i,}  r4  r  zlen(idx_list): z, len(images): num_img_tokensr_  r   ro  )r   r   r  r	  rr  r  r9  r  rq  r
  r2  r  r{  )r   r   r   r   r  r;  rj  rk  ry  r9  r=  r9  rj  r{  image_token_idr   s                  r]   r  Phi3VisionTemplate._encodeJ  s   X&,"GOG,	v;!2:;'	!Iu-v;?00IMM)33F43PQx=CK/n?3x=/Q`adekal`m1nn/MJ#ZZ(89@@BNOOAr"3x=1,-"#a8;?8E?!K~N^aoarNr!rr%qa%"ITFUcUfLf"ffJ	 .
 x|a'7'899M%I!Xb\A%5%677
#'{!xrzr_   r`   )ra   rb   rc   rd   rK  r   r   r   r   r   r   r
   r  r   r   r   s   @r]   rf  rf  D  s[    )*?g ?tCH~ E$sCx.RVWZ\_W_R`B`<a  r_   rf  c                       \ rS rSrSrSrg)Llama3LlavaNextTemplatein  r  r`   Nr  r`   r_   r]   rn  rn  n  s
    SFr_   rn  c                       \ rS rSrSrg)LLavaQwenTemplateiw  r`   Nr   r`   r_   r]   rp  rp  w  r   r_   rp  r  sub_token_listc                     [        U[        5      (       a  U/n/ nSn  U R                  US   US-   5      n[        U5      S:X  d  XX3[        U5      -    :X  a  UR	                  U5        MM  ! [
         a     U$ f = f)z,Find the index of a token in the token_list.r_  r   r   )r   r   r   r	  rY   r   )r  rq  r[   r|  s       r]   rr  rr  ~  s    .#&&()
C
C"">!#4cAg>C>"a'>RUVdReLe=f+f

3   Js   AA, ,
A:9A:c                      ^  \ rS rSrSrS/rU 4S jrS\\\	4   S\
\\\	4   \\\	4   4   4U 4S jjrS\	S\\\	4   4S	 jr\S
\\   S\S\\   4S j5       rSrU =r$ )DeepseekVLTemplatei  r  z<image_placeholder>c                 H   > [         TU ]  S/S/S/S/U R                  5        g )Nu)   <｜begin▁of▁sentence｜>{{SYSTEM}}

r  u   <｜end▁of▁sentence｜>)r   r   DEEPSEEK_VL_SYSTEMr   s    r]   r   DeepseekVLTemplate.__init__  s2    GHKjJk78;X:Y[_[r[r	tr_   r   rR   c                   > [         TU ]  U5      u  p4[        U5      S:X  a  U0 4$ UR                  S5      nU R                  R
                  nUS   US   p[        XvR                  5      n	/ / pSnU	 HF  nXX -  n
Ub  XX -  nXR                  /UR                  -  -  n
US/UR                  -  -  nUS-   nMH     XUS  -  n
Ub  XUS  -  nOS nSSK	J
n  UR                  USS	9nU" S [        R                  " U
5      UR                  [        R                  " UR                  /[        U	5      -  5      S
9n[        UR!                  U/5      5      nUS   R#                  US   S9US'   XUS.nU0 4$ )Nr   r   rj  rk  ro  r   )VLChatProcessorOutputr4  r  )
sft_formatrj  r  r@  r  r  r  r  )r   r  r	  r   r  r9  rr  image_idr@  !deepseek_vl.models.processing_vlmry  rq  r  r6  r  dictbatchifyr  )r   r   r   r  r;  r   r9  rj  rk  ry  r  
new_labelslohiry  images_outputsoutputbatched_outputr   s                     r]   r  DeepseekVLTemplate._encode  s   GOG,	v;!2:X&NN,,	";/1A6I'9'9:$&zBr--M!Rm+
001I4N4NNNM4&9#=#===JaB  	23'+%JJK"226$2O&ll=1'44"\\9+E+E*FX*VW	Y
 i00&:;)7)G)J)JQWX_Q`)J)a~&,^\rzr_   r*  c                 4    UR                   " S0 UD6S   nSU0$ )Nr   r/  r`   )prepare_inputs_embeds)r   r3  r*  r/  s       r]   r>  DeepseekVLTemplate.post_encode  s%    33;d;A>//r_   rD  rE  c                     U $ r)  r`   rG  s     r]   rH  $DeepseekVLTemplate._get_generate_ids  rJ  r_   r`   )ra   rb   rc   rd   rv  rK  r   r   r   r   r
   r  r>  rL  r   r   rH  r   r   r   s   @r]   rt  rt    s    ` //t tCH~  E$sCx.RVWZ\_W_R`B`<a  D0s 0tCH~ 0 S	 C DQTI  r_   rt  z%<|user|>
{{QUERY}}</s>
<|assistant|>
z</s>
z<|system|>
{{SYSTEM}}</s>
z%### Human: {{QUERY}}

### Assistant: z!Human: {{QUERY}}

Assistant: </s>c            	          ^  \ rS rSrS rS\S   S\\   4S jrS\	\
\4   S\\	\
\4   \	\
\4   4   4U 4S jjrSS	\\	\
\4      S
\\   S\	\
\4   4U 4S jjjrSrU =r$ )CogTemplatei  c                 Z    UR                  S5      =(       d    / n[        U5      S::  d   eg r  r  r   s      r]   r   CogTemplate.check_example  r  r_   r   r   rR   c                     / $ r)  r`   rD  s       r]   r   CogTemplate.replace_tag  s    	r_   r   c                   > [         TU ]  U5      u  p4[        U5      S:X  a  U0 4$ UR                  S5      =(       d    / nUR	                  SS 5        U R
                  R                  U R
                  US   UR                  S5      US9nUS   R                  5       R                  5       nUS   nUS	   n	S/S
/U-  -   S/[        US
S  5      -  -   US'   US S
 U R
                  R                  /U-  -   US
S  -   US'   U	b  U	S S
 S/U-  -   U	S
S  -   US	'   [        U5      S:  aX  US    V
s/ s H  oR                  US   S9/PM     sn
US'   SU;   a)  US    Vs/ s H  oR                  US   S9/PM     snUS'   U0 4$ s  sn
f s  snf )Nr   r   r  r  r  )r  r  r   rY  rj  rk  r   ro  r  r{  cross_images)r   r  r	  r   r
  r  build_conversation_input_idsr  itempad_token_idr  )r   r   r   r  r;  r   r  image_token_lenrj  rk  img	cross_imgr   s               r]   r  CogTemplate._encode  s   GOG,	v;!2:H%+

<&..==NN''"2GKK	<R[` > b!"23779>>@;'	!$%3!)>#>!s9UVUW=GYAY#Y 'mt~~/J/J.Ko.]]`ijkjl`mm{%bqzTF_,DDvabzQF8u:>KRS[K\]K\CfWo!> ?K\]F8(ahiwax)yaxT]<<fWo<+N*Oax)y~&rz	  ^ *zs   ,FFr#  r$  c                 0  > [         TU ]  X5      nSS/nU H'  nXQS   ;   d  M  U Vs/ s H
  ofU   S   PM     snX5'   M)     U Vs/ s H  n[        R                  " US   5      PM     nnU R	                  USU R
                  5      nXsS'   U$ s  snf s  snf )Nr   r  r   rY  r_  )	r   r#  r$  r[   rW   keyr'  rY  r   s	           r]   r&  CogTemplate.data_collator  s    g#E6.)CAh/45u!cF1Iu5  FKKU%,,q)9':;UK**>1d>O>OP .
	 6Ks   B#Br`   r)  )ra   rb   rc   rd   r   r   r   r   r   r   r   r   r
   r  r	   r   r&  r   r   r   s   @r]   r  r    s     g.G&H ]abi]j tCH~ E$sCx.RVWZ\_W_R`B`<a ,	4S#X#7 	Xc] 	^bcfhkck^l 	 	r_   r  z [INST] {{QUERY}} [/INST] z <EOI>Question: {{QUERY}} Answer:zQuestion: {{QUERY}} Answer:c                   b   ^  \ rS rSrS rS\\\4   S\\\\4   \\\4   4   4U 4S jjr	Sr
U =r$ )Cog2VideoTemplatei  c                 Z    UR                  S5      =(       d    / n[        U5      S::  d   eg )Nrl  r   r  )r   r   rl  s      r]   r   Cog2VideoTemplate.check_example  r  r_   r   rR   c                   > [         [        U ]  U5      u  p4[        U5      S:X  a  U0 4$ UR	                  S5      =(       d    / n[        U[        5      nUR                  SS 5        U R                  R                  U R                  US   UR	                  S5      USS9nUS   R                  5       R                  5       nUS	   n	US
   n
S/S/U-  -   S/[        U	SS  5      -  -   US'   U	S S U R                  R                  /U-  -   U	SS  -   US	'   U
b  U
S S S/U-  -   U
SS  -   US
'   [        U5      S:  a)  US    Vs/ s H  oR                  US   S9/PM     snUS'   U0 4$ s  snf )Nr   rl  r  r  r  chat)r  r  r   template_versionrY  rj  rk  r   ro  r   r  r{  )r   r  r  r	  r   r   r   r
  r  r  r  r  r  r  )r   r   r   r  r;  r  r   r  video_token_lenrj  rk  r  r   s               r]   r  Cog2VideoTemplate._encode  s   +t4W=	v;!2:kk(+1r;(:;

<&..==NN'"KK	*# > % ""23779>>@;'	!$%3!)>#>!s9UVUW=GYAY#Y 'mt~~/J/J.Ko.]]`ijkjl`mm{%bqzTF_,DDvabzQF8u:>KRS[K\]K\CfWo!> ?K\]F8rz  ^s   E'r`   )ra   rb   rc   rd   r   r   r   r   r
   r  r   r   r   s   @r]   r  r    sG     tCH~ E$sCx.RVWZ\_W_R`B`<a  r_   r  r   )r  r  r.  r   u   <用户>{{QUERY}}<AI>arrry  c                 x    / n[        U5      n[        U 5       H  u  pEXC;  d  M  UR                  U5        M     U$ r)  )setrs  rY   )r  ry  r[   idx_setr{  r  s         r]   _remove_idxr  >  s8    
C(mG#JJqM  Jr_   c                      ^  \ rS rSrSrS\S   S\\   4S jrS r	S\
\\4   S\\
\\4   \
\\4   4   4U 4S	 jjrS
\S\
\\4   4S jr\S\\   S\S\\   4S j5       rSrU =r$ )MiniCPMVTemplateiG  Fr   r   rR   c                 
    S//$ Nro  r`   rD  s       r]   r   MiniCPMVTemplate.replace_tagJ  s    xr_   c                 Z    UR                  S5      =(       d    / n[        U5      S:X  d   eg r  r  r   s      r]   r   MiniCPMVTemplate.check_exampleM  r  r_   r   c                   > [         TU ]  U5      u  p4[        U5      S:X  a  U0 4$ US   nUS   nUS   n[        US5      nUS   n	S n
[	        U R
                  R                  SS5      nU(       Ga  U R                  (       ae  U R
                  R                  R                  nU" USS	9R                  US
   5      nUR                  UR                  S   S   5      nUS   nUS   n
OXU R
                  R                  US   U R
                  5      u  p^U Vs/ s H  nU R
                  R                  U5      PM      sn/nUS-  nU R
                  R                  USS9nUS U	 U-   XiS-   S  -   nUb  US U	 S/[        U5      -  -   XyS-   S  -   n[        R                   " U5      n[        R"                  " UU R
                  R$                  :H  5      S   nUS-  n[        R"                  " UU R
                  R&                  :H  5      S   n[)        [        U5      [        U5      5      n[        R*                  " US U R-                  S5      US U R-                  S5      /5      /nOSSU R
                  R                  R.                  -  -   S-   nU R
                  R                  USS9nUS U	 U-   XiS-   S  -   nUb  US U	 S/[        U5      -  -   XyS-   S  -   n[        R                   " XU R
                  R                  R.                  -   //5      /nU R
                  R                  US   5      //nUU[        R                   " U5      S    UUU
S.S.nU0 4$ s  snf )Nr   r   rj  rk  ro  
slice_modeFr4  r  r  r  	tgt_sizesr  r  r   r_  rQ  <unk>z	</image>
rj  image_boundr  r  r  )r   r  r	  rr  r  r  r  is_v2_5r9  rq  r  get_slice_image_placeholderr  	transformr  r  r6  whereim_start_id	im_end_idmaxhstack	unsqueeze	query_num)r   r   r   r  r;  r   rj  rk  ry  r|  r  r  rq  r  r  r  r  r  input_tensor_idsimage_start_idximage_end_idxvalid_image_numsr  r   s                          r]   r  MiniCPMVTemplate._encodeQ  s   GOG,	v;!2:";'	!It,qk	T^^22L%H
||"&..":":"J"J.vdKNNvV]_-II,JbJbcdJefgJhi+N;(5	 '+nn&P&PQWXYQZ\`\j\j&k#JP Q&3!9!9#!>& QR4K!^^22;SX2YN"4C>9IAgh<OOI! #,$#n2E)EEUVwxHXX$||I6#kk*:dnn>X>X*XYZ[\Oq O!KK(8DNN<T<T(TUVWXM"3#7]9KL$%6&67AA"E}UfVfGgGqGqrtGuvxK
 $g0E0E0O0O&OOR^^K!^^22;SX2YN"4C>9IAgh<OOI! #,$#n2E)EEUVwxHXX <<#T^^5J5J5T5T/T)U(VWXK!^^55fQi@ABL""\\)4T:* ,&		
 rzC !Rs   %Mr*  c                 6    UR                  U5      u  p4SUS   0$ )Nr/  r   )get_vllm_embedding)r   r3  r*  r/  r;  s        r]   r>  MiniCPMVTemplate.post_encode  s%     33D9q!122r_   rD  rE  c                     U $ r)  r`   rG  s     r]   rH  "MiniCPMVTemplate._get_generate_ids  rJ  r_   r`   )ra   rb   rc   rd   r  r   r   r   r   r   r   r   r   r
   r  r>  rL  r   rH  r   r   r   s   @r]   r  r  G  s    Gg.G&H ]abi]j  6tCH~ 6E$sCx.RVWZ\_W_R`B`<a 6p3s 3tCH~ 3 S	 C DQTI  r_   r  c                      ^  \ rS rSrS rS\S   S\\   4U 4S jjrS\	\
\4   S\\	\
\4   \	\
\4   4   4S jrS	rU =r$ )
MiniCPMV2_6Templatei  c                     g r)  r`   r  s     r]   r   !MiniCPMV2_6Template.check_example  s    r_   r   r   rR   c                    >^ US;   d   e[        S[        S5      n[        [        US9n[        TU ]  SX#5      mUS:X  a  T$ US:X  a  [        XSU4S j5      $ g )N>   r   r   max_num_frames@   r  r   r   c                    > T$ r)  r`   r  s    r]   r  1MiniCPMV2_6Template.replace_tag.<locals>.<lambda>  s    }r_   )rW  r   r   r   r   r   r  )r   r   r   r   r  r  r  r   s         @r]   r   MiniCPMV2_6Template.replace_tag  sh    ////%&6R@;N[
+GUD   7"'
=TUU #r_   r   c                    [         R                  " X5      u  p4[        U5      S:X  a  U0 4$ UR                  S5      n[	        UR                  S5      5      nU(       + =(       a    U(       + nSnS n	U(       a  SnSn	[        S[        U	5      n	US   n
US	   n[        U
S
5      nUR                  SS5        U R                  R                  R                  nU" U/SU	S9R                  US   5      n/ n/ n[        [        U5      S-
  5       H  nUR                  UR                  S   U   UXS9nUS-  nU R                  R!                  USS9nXUU   S-   UUS-       U-   -  nUc  M^  UXU   S-   UUS-       S
/[        U5      -  -   -  nM     XUS   S-   S  -  nUn
Ub  UXS   S-   S  -  nUnU(       d  ["        R$                  " U
5      nU R                  R!                  SSS9S   nUU:H  R'                  SS9S   R)                  5       n/ nUS   n[        S[        U5      5       H8  nUU   UUS-
     S-   :w  d  M  UR+                  UUUS-
     S-   /5        UU   nM:     UR+                  UUS   S-   /5        ["        R$                  " U5      /nO/ /nU
U["        R$                  " U
5      S    UUS   US   S.S.nU0 4$ )Nr   r   rl  TFr   max_slice_numsrj  rk  ro  r_  r4  )r6  r  r  )	image_idxr  use_image_idr  r  r  )as_tupler  r  r  r  )r   r  r	  r   r   rW  r   rr  r  r  r9  rq  r  r{  r  r  r  r  r6  nonzeror2  rY   )r   r   r   r  r;  r   	use_videois_plain_textr  r  rj  rk  ry  rq  r  r=  r9  r{  r  r  r  	unk_tokenindicesrangesr  r  s                             r]   r  MiniCPMV2_6Template._encode  s/   $$T3	v;!2:X&X./	"
49} LN%&6^L;'	!It,2..22BB&x6DFFHbPWFY 	 
s8}q()A)EE((+A.!N F wK4K!^^22;SX2YNx{QxAG.XXM!fa[1_Xa!e_EQTUcQdHddd
 * 	8B<!#3#455!	&"!1!233JF$||I6--g%-PQRSI'94==t=LQOVVXGFAJE1c'l+1:Q!!33MM5'!a%.1*<"=>#AJE , MM5'"+/23 <</0K$K #"\\)4T:* ,^ <)+6		
 rzr_   r`   )ra   rb   rc   rd   r   r   r   r   r   r   r   r   r
   r  r   r   r   s   @r]   r  r    sn    Vg.G&H V]abi]j VAtCH~ AE$sCx.RVWZ\_W_R`B`<a A Ar_   r  c                       \ rS rSrSrSrg)MiniCPMV2_5Templatei  Tr`   N)ra   rb   rc   rd   r  r   r`   r_   r]   r  r    s    Gr_   r  rR  z@<start_of_turn>user
{{QUERY}}<end_of_turn>
<start_of_turn>model
z<end_of_turn>
z<end_of_turn>z3<bos><start_of_turn>system
{{SYSTEM}}<end_of_turn>
z<_user>{{QUERY}}<_bot>z<_end>z<_user> {{QUERY}}<_bot>u  You are DBRX, created by Databricks. You were last updated in December 2023. You answer questions based on information available up to that point.
YOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, but provide thorough responses to more complex and open-ended questions.
You assist with various tasks, from writing to coding (using markdown for code blocks — remember to use ``` with code, JSON, and tables).
You do not have real-time data access or code execution capabilities. You avoid stereotyping and provide balanced perspectives on controversial topics. You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.
This is your system prompt, guiding your responses. Do not reference it, just respond to the user. If you find yourself talking about this message, stop. You should be responding appropriately and usually that means not mentioning this.YOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY PERTINENT TO THE USER'S QUERY.c                       \ rS rSr\rSrg)DbrxTemplatei  r`   N)ra   rb   rc   rd   DBRX_SYSTEMr   r   r`   r_   r]   r  r    s    Fr_   r  u   输入：{{QUERY}}输出：
u   指令：{{SYSTEM}}zYou are Command-R, a brilliant, sophisticated, AI-assistant trained to assist human users by providing thorough responses.You are trained by Cohere.z<BOS_TOKEN>zk<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{QUERY}}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>z<|END_OF_TURN_TOKEN|>zE<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{SYSTEM}}<|END_OF_TURN_TOKEN|c            	          ^  \ rS rSrU 4S jrS\S   S\\   4S jrS\	\
\4   S\\	\
\4   \	\
\4   4   4U 4S jjrSS	\\	\
\4      S
\\   S\	\
\4   4U 4S jjjrSrU =r$ )mPlugOwl2Templatei$  c                 4   > [         TU ]  S/S/S/S//5        g )Nr  zUSER: {{QUERY}}ASSISTANT:r  r   r   r   s    r]   r   mPlugOwl2Template.__init__&  s%    ,*E)FTbScRder_   r   r   rR   c                     US:X  d   eS//$ )Nr   r  r`   rD  s       r]   r   mPlugOwl2Template.replace_tag)  r  r_   r   c                   > SSK Jn  U R                  R                  nUR	                  S5      =(       d    / n[        U5       H0  u  pg[        UR                  5      nUR                  X45      nXuU'   M2     [        TU ])  U5      u  p[        U	5      S:X  a  U	0 4$ U	S   nU	S   nU(       a#  U" XT5      nUR                  US   5      nXUS.0 4$ XS.0 4$ )	Nr   r  r   rj  rk  r  r#  rj  rk  )mplug_owl2.mm_utilsr  r  r9  r   rs  r  rX  rY  r   r  r	  r  )r   r   r   r  r9  r   r{  r   max_edger  r;  rj  rk  r   s                r]   r  mPlugOwl2Template._encode-  s    6NN,,	X&,"!&)HA5::HLL(!56E1I	 *
 GOG,	v;!2:;'	!#F6FYYvg/F!*OQSSS!*=rAAr_   r#  r$  c                    > [         TU ]  X5      nU Vs/ s H  nSU;   d  M  US   PM     nnU(       a  [        R                  " U5      US'   U$ s  snf r  r  r  s         r]   r&  mPlugOwl2Template.data_collatorB  r  r  r`   r)  )ra   rb   rc   rd   r   r   r   r   r   r   r   r   r
   r  r	   r   r&  r   r   r   s   @r]   r  r  $  s    fg.G&H ]abi]j BtCH~ BE$sCx.RVWZ\_W_R`B`<a B*4S#X#7 Xc] ^bcfhkck^l  r_   r  )r  r  r.  c            	          ^  \ rS rSrSrS rS\S   S\\   4S jr	S\
\\4   S\\
\\4   \
\\4   4   4U 4S	 jjrS
\S\
\\4   4S jrSS\\
\\4      S\\   S\
\\4   4U 4S jjjrSrU =r$ )mPlugOwl3TemplateiN  Nc                    U R                   R                  nUR                  R                  SUS   US   S9nUR	                  S5      nUS   S:X  a  UR                  5         / nU H	  nXSS/-  nM     U R                  U5      S   nU$ )N	<|image|>r   r   )	img_tokenr  wr_  r   )r  r9  rq  cut_prompt_templatesplitr
  _encode_context_list)r   	cut_shaper9  r  	text_listres_text_listr  s          r]   _get_image_token_list'mPlugOwl3Template._get_image_token_listQ  s    NN,,	((<<{V_`aVbfopqfr<sJJ{+	R=BMMODK00M ..}=a@
r_   r   r   rR   c                     US;   d   e[        S[        S5      n[        [        US9nUS:X  a  S/S/$ US:X  a  [	        XSS	 5      S/-   $ g )
N>   r   r   r     r  r   ro  r  r   c                 
    S//$ r  r`   )r{  s    r]   r  /mPlugOwl3Template.replace_tag.<locals>.<lambda>d  s    xr_   )rW  r   r   r   r  )r   r   r   r   r  r  s         r]   r   mPlugOwl3Template.replace_tag]  se    ////%&6R@;N[
 FD>!7"'
=OPTXSYYY #r_   r   c                   > [         TU ]  U5      u  p#[        U5      S:X  a  U0 4$ US   nUS   nU(       + nUS   nUS   n[        US5      n	U R                  R
                  n
U(       Gaw  U
R                  XFSS9nSnUS	   =(       d    S /[        U	5      -  nU R                  R                  S
SS9n[        X5       Hp  u  nnU(       a  U R                  U5      nOUnUS X-    U-   X|U-   S-   S  -   nU(       a"  US X-    S/[        U5      -  -   XU-   S-   S  -   nU[        U5      S-
  -  nMr     [        R                  " [        X~5      5      S    n[        R                  " [        U5      5      S S 2S 4   nUU:  R                  SS9n[        R                  " [        R                  " UR                   S   [        R"                  S9U/SS9S    nSUS   0US'   UUS'   XrS'   XS'   U0 4$ )Nr   r   rl  rj  rk  ro  r4  )
cut_enabler6  r  r  Fr  r   r  r{  r_  r  r$  media_offset)r   r  r	  rr  r  r9  rq  r  r  r  r  r6  aranger  stackro  r7  long)r   r   r  r;  r   rl  r
  rj  rk  ry  r9  r  rz  
cut_shapesimage_token_listr|  r  r  image_token_idx_rangematrixr  r   s                         r]   r  mPlugOwl3Template._encodef  s5   GOG,	v;!2:""Z
;'	!It,NN,,	$44Vcg4hL %k2Ltfs8}6LJ#~~44[UZ4["%h";Y!%!;!;I!FJ!1J%&=s'=>KiknXnqrXrXsNtt	#$;S%;<vJ?WWZ`twawcdbe bf [g gF C
Oa$77  #< $ll8I+PQRVWO\\#i.1!T':F.333:F ;;FLLO5::(VX^'_eghimnL-|N/KLF7O%1F>"'{!xrzr_   r*  c                 2    UR                  US   5      nSU0$ )Nr  image_embeds)forward_image)r   r3  r*  r  s       r]   _post_encodemPlugOwl3Template._post_encode  s"    **4+?@--r_   r#  r$  c                 .  > [         TU ]  X5      nU Vs/ s H  nSU;   d  M  US   PM     nnU(       a  [        R                  " U5      US'   U Vs/ s H  nSU;   d  M  US   PM     nnU(       a  [        R                  " U5      US'   U$ s  snf s  snf )Nr  r  r  )r   r#  r$  r[   r'  r  r  r   s          r]   r&  mPlugOwl3Template.data_collator  s    g#E638P5aNa<O).)5P"',,|"<C38P5aNa<O).)5P"',,|"<C
 Q Qs   
B	B
B 	Br`   r)  )ra   rb   rc   rd   r   r  r   r   r   r   r   r   r   r
   r  r  r	   r   r&  r   r   r   s   @r]   r  r  N  s    F
Zg.G&H Z]abi]j Z"tCH~ "%S#XSRUX8V2W "H. .S#X .4S#X#7 Xc] ^bcfhkck^l  r_   r  zUser:
{{QUERY}}

Assistant:
zA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. z&<s>Human: {{QUERY}}
</s><s>Assistant: c                       \ rS rSr SS\S\\\4   S\S\	\\\4   \\\4   4   4S jjr
SS\S\\\\4      S	\\   S\\\4   4S
 jjrSrg)RLHFTemplateMixini  r   r   	streamingrR   c                 `   U R                   n0 n0 nXR                  5       pvUS   US'   U(       a  U" U5      0 pU" U5      0 pOU" U5      u  pU" U5      u  p[        SS/XE/5       HC  u  pS H8  n[        5       U SU 3   nUR	                  5        H  u  nnUX SU 3'   M     M:     ME     XE4$ )Nrejected_responser  r  r  )chosenrejectedr;  )_old_encoder  r  rp  items)r   r   r  template_encoder  r  chosen_examplerejected_examplechosen_inputschosen_tokenizer_kwargsrejected_inputsrejected_tokenizer_kwargsr  r[   r  r*  r\   vs                     r]   r  RLHFTemplateMixin.encode  s     **+2LLN('./B'C$5D^5TVX29HIY9Z\^65D^5T2M9HIY9Z6O*<=?YZKF0x6(!F8 45 JJLDAq+,C(!A3( ) 1 [
 ''r_   Nr#  r$  c                    U R                   n/ nS Hu  nU Hl  n0 nUR                  5        H1  u  pUR                  U5      (       d  M  U[        U5      S  n
Xh   Xz'   M3     [        U5      S:  d  M[  UR	                  U5        Mn     Mw     [        U5      S[        U5      S-  1;   d
   SU 35       eU" U=(       d    UU5      $ )N)chosen_	rejected_r   r   znew_batch: )_old_data_collatorr$  rX   r	  rY   )r   r#  r$  _data_collator	new_batchr  r  
new_inputsr\   r,  new_ks              r]   r&  RLHFTemplateMixin.data_collator  s    00	.F
"LLNDA||F++ !#f+,,2I
) + z?Q&$$Z0   / 9~!SZ!^!44OI;6OO4i05*==r_   r`   Fr)  ra   rb   rc   rd   r   r   r   r   r   r
   r  r   r	   r   r&  r   r`   r_   r]   r  r    s     "'(X (S#X((+0c3hc3h1O+P(,>H >T$sCx.-A >xX[} >hlmprumuhv > >r_   r  c                       \ rS rSr SS\S\\\4   S\S\	\\\4   \\\4   4   4S jjr
SS\S\\\\4      S	\\   S\\\4   4S
 jjrSrg)KTOTemplateMixini  r   r   r  rR   c                 \    U R                  X5      u  p4[        U5      S:  a  US   US'   X44$ )Nr   label)r#  r	  )r   r   r  r  r  s        r]   r  KTOTemplateMixin.encode  s9     $(#3#3G#G v;?%g.F7O''r_   Nr#  r$  c                 
   0 nS H]  n/ nU H!  nUR                  Xd S3   Xd S3   S.5        M#     U R                  XR5      R                  5        H  u  pxXU SU 3'   M     M_     U Vs/ s H  ofS   PM	     snUS'   U$ s  snf )N)r   KL_rj  rk  r  completion_r<  )rY   r1  r$  )	r   r#  r$  r[   r  r3  r'  r\   r,  s	            r]   r&  KTOTemplateMixin.data_collator  s    !FI  q891E/FRSV^^dTeRf!gh //	FLLN12vhk!-. O	 " -22Eq'
E2G
 3s   *B r`   r7  r)  r8  r`   r_   r]   r:  r:    s     "'(X (S#X((+0c3hc3h1O+P(	H 	T$sCx.-A 	xX[} 	hlmprumuhv 	 	r_   r:  )z<img>(.+?)</img>)r  r  r  	functoolsr   typingr   r   r   r   r   r	   r
   r   r   r  r  	packagingr   r   !transformers.dynamic_module_utilsr   transformers.integrationsr   
modelscoper   baser   r   utilsr   r   r   r   r   r   r   r   r   r   r    r  r   r   Historyr   Prompt	StopWordsr   r#   r   r   r%   r   re   r   r   r   r   r   r   r&   r   r   ri   rg   rQ   rn   r   r+  r-  rj   rh   r0  rB  rH  rk   r\  r^  rm   rl   r  r   yi_vl_default_systemr  r  r  rD   r   r'   r(   rf   r)   r*   codegeex4_systemr+   r?   r   r@   rA   LLAMA_DEFAULT_SYSTEMr,   ro   r|   r  r  r  r.   r-   OPENBUDDY_DEFAULT_SYSTEMr1   OPENBUDDY2_DEFAULT_SYSTEMr2   r  r3   r  r  r  rW  r  r4   r  r  r}   rO  r   r~   rQ  r  r  r  r  r  r5   r   r6   r   r  r7   r9   r8   r:   r;   r<   r   rB   r   r  r  rt   r  rw   rx   r  r  r0   r	  r/   r  r   r$  r(  rp   rq   r,  rr   r5  rs   r7  ru   r9  rv   rC  r{   rJ  rH   rb  rL   rf  r   rn  ry   rp  rz   rr  rt  r   r=   r>   rE   r  r   r   rC   r  r   rF   r  r  r  r   r  r   r   gemma_templaterG   rM   r   r  r  rN   rO   C4AI_SYSTEMrP   r  r   r  rI   r   _wizardlm2_systemrJ   rK   r  r:  r`   r_   r]   <module>rU     s5    	 	  V V V    0 K @ ! ,= = = = 
/
uU38_d3i/0
1	eCcDI-.	/	
T#Y
l l^ SX 4S 4H 44 4fj 4 R=>NK[J\./ !Y Y ,113L3N^b c ''~+7G6HI' '	(( 	-+ -	$h 	   F ,##\^ 4	)< 		35N 	 ,&&(8 9 ,113K3M N ,%%~'7 8 !!R>?k]Tb*+-.$ $N	/ 		"9;T 	 ,))+<+>d S &&(C(EUYimo 6M1< M9#;=V 9 ,**,>,@PT U6Ql Qh ,''):$ O '')E)GW[koq,n , ,''): ;y "8 "J( *K *Z ,$$mo
bfrw x tf2TFLID6TX[_Z`Sacw"#% ,''<.C5+X[W\B]_aesdtcu)v w !R SV\U]ao`p_qrt ##[k]DNK[J\%]mqs 9:BdUmTnoq 9:B/0#HIKL |  9:B@QSc)*,-
 ~"A!BnEUDVZhYiXjlp 0134 ~"J!KdVWeVfUgim-/0 vhZ9:fXJ&
TX\b[ceuZvwy -.1Y0Z]v\w'($1Z0[]^X  l^125E4FRf;<>?
 ""hZ,-z<M;NPT1245
 ,++L>,<>P+QTaSbekdlmo "	(( 	J,h J ,))+=+? @ ,%%~'7 8Z  R/04&N;K:L%(8'9 !#  RSTWbVcfoep&)T(U !.  eW56gYP_*+-.
 T]5C HcUBY,? PXY[P\ aijlam    ,((*;*= > 0/3 /$/#&/ 5:#wS	:Q4R/(e eP $$&@&NZ_osu	(#= 	( && 0	 )) 3	<x <~4= I( IX# #<46F <	57H 	 +-deoq  4 6%W[nxz ,((*;*=^b c ,--/D/FRWgk lTZx TZn  ,%%L>,M+NR`QaPbfteudvwy ,##Xb3C2DdnM]L^%_ ` ,##^,l;>Y=Z]a\bfteudvwy ,&&O,/Q.RTX[fZghj ,%%^,l;>Z=[]_cqbrastv ##l^efhj!"
 R12n5Et4LP^O_N`bf'(*+
 l^LMP]al`mVXYh 2_ & ,002H2JV[ko p @ !!(),H+IC5SYRZ[	 $$)E(FP\~^"$s) "T#Y "$'",1$s)T#Y2F,G"Jx $ )*-[,\_u^v*+T4[3\^W W ,'')9);u\` a*H *Z N. N;- ; ,,,.E.GSXhl m ,++-C-EQVfj k?) ? ,'');)=^b c	 35E 	 ,335N5P\aqu v	+-= 	 ,,,.A.Cudh i-/? D ,335J5LX]mq r   ,335G5IUZjn o( (V -/*\`ptv,8 , ,##\^ 4$ $N ,&&(:(<D QS1= S ,002I2KW\lp q	)= 	 ,))+<+>%_c dc E#tCy.4I dSVi  3 3l ,**,>,@Eae f R<=zF8UY,-/0
 l^GH?J[^m]noq ,$$O,/T.UX^W_bhaijl(( (V 67fXF ""<=tfXN .!"%B$CdV~N^M_` > '(+H*ID6UcTdSef ,&&/1BE\D]_adjck(l mT#Y $s) S	 Hx HVP+-= Pf ,--/B/DPUei j-/?  !4!6%W[nxz o&)@(A2xP! 7)&k%l,-/@$RSU ,$$n 5 ,''"7O6PS[R\_g^h)i j ,**HR:S9TVX[cZd,e f& ">  ,##\^ 4 ,%%2 ?@"GWFXZ^av`wxzI 		vw	 !$;#<k	PQ	ST# #L .07V[koqH)8 HV ,))+<+>%_c d ,,,L>,N+ORXQY\b[cdfl  ,((L>,H+IF8V\U]_pqs ,##L>,U+VY_X`cibjkm%> %>P r_   