
    9iv                     Z   S SK r S SKJr  S SKJr  S SKJrJrJrJ	r	J
r
JrJrJrJr  S SKrS SKrS SKrS SKJrJrJr  S SKJrJrJrJr  S SKJr  S SKJr  S S	K J!r!J"r"  S S
K#J$r$  S SK%J&r&  S SK'J(r(  S SK)J*r*J+r+  S SK,J-r-  S SK.J/r/J0r0J1r1J2r2  S SK3J4r4J5r5  S SK6J7r7  S SK8J9r9  S SK:J;r;J<r<J=r=  \7" 5       r>Sr? " S S5      r@\(R                  " \2R                  SS9\(R                  " \2R                  SS9 " S S\\;5      5       5       rD\@R                  S5      S 5       rF\@R                  S5      \@R                  S5      S 5       5       rH\@R                  S5      \@R                  S 5      S! 5       5       rI\@R                  S"5      \@R                  S#5      S$ 5       5       rJ\@R                  S%5      S& 5       rK\@R                  S'5      S( 5       rL\@R                  S5      S) 5       rM\@R                  S*5      S+ 5       rN\@R                  S\!5        \@R                  S\"5        \@R                  S \"5        g),    N)contextmanager)Lock)	AnyCallableDict	GeneratorIteratorListOptionalTupleUnion)
AutoConfigPreTrainedModelPreTrainedTokenizer)AutoModelForCausalLMAutoTokenizerPipelinesnapshot_download)model_file_download)Model)ChatGLM2TokenizerLlama2Tokenizer)
OutputKeys)Input)	PIPELINES)is_modelis_official_hub_path)Config)
FrameworksInvoke	ModelFileTasks)create_devicedevice_placement)
get_logger)ModelTypeHelper)PipelineStreamingOutputMixinStreamingOutputMixinadd_stream_generateswiftc                       \ rS rSrS/ SQ0r\SS\S\4S jj5       r\SS\S\4S jj5       r	\SS\4S	 jj5       r
\SS\4S
 jj5       r\SS\4S jj5       r\S\S\4S j5       r\S\S\4S j5       rSrg)LLMAdapterRegistry$   qwenNNNN
model_typevalue_indexc                     U(       d  [         R                  (       d   eUc  [         R                  nXR                  ;  a  / SQU R                  U'   U R                  U   U   b   eX0R                  U   U'   U$ )Nr/   )r&   current_model_typellm_format_map)clsr0   r1   members       e/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/pipelines/nlp/llm_pipeline.py_add_to_mapLLMAdapterRegistry._add_to_map(   st    _????(;;J///-?Cz*!!*-k:BBB6<:&{3    c                 H   ^ ^^ Ub  T R                  TTU5      $ U UU4S jnU$ )Nc                 *   > TR                  TTU 5      $ Nr8   )r6   r5   r0   r1   s    r7   	_register.LLMAdapterRegistry._wrapper.<locals>._register8   s    ??:{FCCr:   r>   )r5   r0   r1   r6   r?   s   ```  r7   _wrapperLLMAdapterRegistry._wrapper3   s*    ??:{FCC	D r:   c                 (    U R                  USU5      $ )Nr   rA   r5   r0   functions      r7   register_format_messages+LLMAdapterRegistry.register_format_messages=       ||J844r:   c                 (    U R                  USU5      $ )N   rD   rE   s      r7   register_format_output)LLMAdapterRegistry.register_format_outputA   rI   r:   c                 (    U R                  USU5      $ )N   rD   )r5   r0   tokenizer_classs      r7   register_tokenizer%LLMAdapterRegistry.register_tokenizerE   s    ||J?;;r:   
model_namereturnc                     XR                   ;   $ r=   r4   r5   rS   s     r7   containsLLMAdapterRegistry.containsI   s    ////r:   c                      U R                   U   $ r=   rV   rW   s     r7   getLLMAdapterRegistry.getM   s    !!*--r:    )r   N)NN)__name__
__module____qualname____firstlineno__r4   classmethodstrintr8   rA   rG   rL   rQ   boolrX   r[   __static_attributes__r]   r:   r7   r,   r,   $   s    01NS s   # C   5# 5 5 5 5 5 <C < < 0# 0$ 0 0 .S .T . .r:   r,   llm)module_namec                     ^  \ rS rSrS rS\\\4   S\4S jr	S)S jr
     S*S\\\4   S\S	\S
\S\\   4
U 4S jjjrS+S jr\S\\\4   4S j5       rS\\\4   4S jrS\\\\   4   S\4S jrS\\\4   S\\\4   S\\\4   S\4S jrS\\\4   4S jrS rS rS,S jr\S\\\\\\4      4   S	\S\\\R>                  4   4S j5       r \S\4S j5       r!\S\\\\\\4      4   S\"\#\\4      4S  j5       r$\S!\\%   S"\S#\\\\\\4      4   S	\S\\%   4
S$ j5       r&\S	\S#\\\\\\4      4   4S% j5       r'\S!\\%   S&\\%\\%   4   S\\%   4S' j5       r(S(r)U =r*$ )-LLMPipelineR   c           	         SSK Jn  [        U[        5      (       a  [        R                  SU 35        U R                  U5      (       a  U R                  b#  [        R                  SU R                   S35        U R                  R                  S5      nUc   S5       eU R                  R                  SS	5      n[        R                  " UU[        R                  U R                  U R                   U R"                  S
9nUR                  X1S9nU$ [        U[        5      (       Gae  [%        U5      (       GaT  [        R                  SU S35        U R                  (       as  [&        R(                  R+                  U5      (       a  UO
[-        U5      n U R/                  UU R                  5      n[        R                  SU R                   S35        U$ [3        U5      (       aD  [        R                  " U[        R                  U R                  U R                   U R4                  S9$ [&        R(                  R+                  U5      (       a  UO
[-        U5      n[6        R                  " UU R                  U R"                  S9nXal        U$ U$ ! [0         a9  n[        R                  SU SU R                   SU 35        S U l         S nANS nAff = f)Nr   )Swiftzinitiate model from z.Cannot use swift with llm_framework, ignoring .zadapter_cfg.model_id_or_pathzECannot get adapter_cfg.model_id_or_path from configuration.json file.zadapter_cfg.model_revisionmaster)
invoked_by
device_maptorch_dtypetrust_remote_code)model_idzinitiate model from location zinitiate model with z Cannot using llm_framework with z, ignoring llm_framework=z : )rp   rq   rr   ignore_file_pattern)rq   rs   )r*   rm   
isinstancerc   loggerinfo_is_swift_modelllm_frameworkwarningcfgsafe_getr   from_pretrainedr    PIPELINErq   rr   rs   r   ospathexistsr   _wrap_infer_framework	Exceptionr   ru   r   	model_dir)selfmodelrm   
base_modelrevisionswift_modelr   es           r7   initiate_single_model!LLMPipeline.initiate_single_modelV   s   eS!!KK.ug67&&!!-DTEWEWDXXYZ **+IJJ)r+rr)xx(()E)13H..!???? ,,"&"8"8:J  //
/KKeS!!&:5&A&AKK7wa@A!!%'WW^^& &E 1% 8 	. 66y7;7I7IKEKK"6t7I7I6J! LM L ,,%# $ 0 0(,(@(@B B &(WW^^& &E 1% 8  -<<#&*&<&<> #,L1 ! .NN:5' B2262D2D1ESMN *.D&&	.s   A J 
K/KKr   rT   c                    [        U[        5      (       d  g[        R                  R	                  U5      (       a/  [        R                  R                  U[        R                  5      nO [        U[        R                  5      n[        R                  " U5      U l        U R                  R                  S5      [        :H  $ ! [         a     gf = f)NFzadapter_cfg.tuner_backend)rv   rc   r   r   r   joinr!   CONFIGURATIONr   r   r   	from_filer|   r}   SWIFT_FRAMEWORK)r   r   cfg_files      r7   ry   LLMPipeline._is_swift_model   s    %%%77>>%  ww||E9+B+BCH.ui6M6MN ##H-xx  '),;< 	<	  s   +C 
CCc                 0    SSK Jn  UR                  X5      $ )Nr   )InferFramework)$modelscope.pipelines.accelerate.baser   r~   )r   r   	frameworkr   s       r7   r   !LLMPipeline._wrap_infer_framework   s    G--iCCr:   format_messagesformat_output	tokenizerrz   rs   c                   > UR                  SS 5      U l        XPl        X@l        [        R
                  R                  US   5      (       a  [        R                  " US   U R                  S9nUR                  R                  SS 5      n	U	(       a@  U	R                  SS5      S:X  a*  [        R                  R                  5       (       a  SU l        UR                  SS 5      U l        UR                  S	S 5      U l        U[         :X  a&  U R#                  US   UR                  S
S5      5        g U R%                  U5         [&        TU ]P  " U0 UD6  S S S 5        [+        U R,                  [.        5      (       a  [1        U R,                  5      U l        S n
[+        U[2        5      (       a=  [4        R7                  U5      (       d   SU S35       e[4        R                  U5      u  pn
Uc[  [8        R                  " U R,                  R:                  SS9n[4        R7                  U5      (       a  [4        R                  U5      u  pn
Ub  Xl        Ub  X l        Uc  U RA                  U
5      U l!        g UU l!        g ! , (       d  f       GN.= f)Nrq   r   rs   quantization_configquant_methodgptqcudarr   ru   devicegpuzCan not find function for `z`!-)split)"poprq   rs   rz   r   r   r   r   r~   __dict__r[   torchr   device_countrr   ru   r   _init_swift_temp_configuration_filesuper__init__rv   r   r   r)   rc   r,   rX   r&   r   r   r   _get_tokenizerr   )r   r   r   r   rz   rs   argskwargsconfigq_configrP   r0   	__class__s               r7   r   LLMPipeline.__init__   s.    !**\48!2*77>>&/**//w43I3IKF**+@$GH<<& #)*.3jj.E.E.G.G&,DO!::mT:#)::.CT#J O+VG_fjj5.IJ**62Gd-f- 3djj/22,TZZ8DJos++%..?? B-o->bAB? #&&7 <OO "(,,TZZ-A-AMJ!**:66&**:6 @ &#2 $!. ) 1 ,,7@ 	- 32s    I88
Jc                   ^  SSK Jn  SSK JnJn  S[        [
        [        [        [
        [
        4      4   S[        S[        [
        [        R                  4   4U 4S jjnS[        [
        [        [        [
        [
        4      4   S[        [
        [
        4   4U 4S jjnU" US	9nU" UT R                  S
9u  p[        U	5      T l        T R                  U
l        U
T l        U
R                  T l        UT l        ST l        ["        R                  T l        UT l        [)        U5      T l        ST l        [/        5       T l        ST l        ST l        g )Nr   )prepare_model_template)InferArgumentsget_model_info_metamessagesr   rT   c                   > TR                   R                  U 5      nUR                  SS 5        SU;   a/  [        R                  " US   5      S    nXCS'   UR
                  S   nSU;   a  US   S    nXcS'   UR
                  S   n[        R                  " W5      S    US'   SU;   a  [        R                  " US   5      S    US'   U$ )Nlabels	input_idsrK   inputs_embedsattention_masktoken_type_ids)templateencoder   r   tensorshapeones)r   r   r   inputsr   	token_lenr   r   s          r7   r   0LLMPipeline._init_swift.<locals>.format_messages   s     ]]))(3FJJx&f$!LL)<=dC	&/{#%OOA.	&( & 7 =*7')//2	',zz)'<T'BF#$6)+0<<+,,..2,4'(Mr:   c           	      p  > U S   n [        U 5      S:  d   S5       eS nU S   S   S:X  a  U S   S   nU SS  n [        U 5      S-  S:X  d   S	5       eU  Vs/ s H  o"S   PM	     nnUS
   n[        [        US S S2   USS S2   5      5      nTR                  [        :X  a
  [        XUS9$ [        XUS9$ s  snf )Nr   r   zmessages cannot be empty!rolesystemcontentrK   rO   zUnsupported messages format!)r   queryhistory)r   promptr   )lenlistziprz   r   dict)r   r   messagecontentsr   r   r   s         r7   get_example,LLMPipeline._init_swift.<locals>.get_example   s    
+Hx=1$A&AA$F{6"h.!!Y/#AB<x=1$)I+II):BC(w	*(HCb\F3x!}hqt!tn=>G!!_46II6'JJ Ds   B3)r   )rq   FT)	swift.llmr   r   r   r   rc   r
   r   r   Tensorrq   r)   r   r   r   r   has_multiple_modelsr   r   device_namer#   r   _model_preparer   _model_prepare_lock_auto_collate_compile)r   rt   r   r   r   r   r   r   r   r   r   s   `          r7   r   LLMPipeline._init_swift   s   4A	d3T#s(^0D+D&E 	':	)-c5<<.?)@	&	KsDc3h$889	K>B38n	K" H-0T__.(/
 !++.#( #))!#F+##'6 !r:   r   c              #     #    U R                  US   5      =US'   n[        U[        5      (       a  UOUR                  n[        R
                  R                  U[        R                  5      n[        R
                  R                  U5      (       a  S v   g [        US5       n[        R                  " SSS.U5        S S S 5        S v   [        R                  " U5        g ! , (       d  f       N)= f7f)Nr   wpytorchchat)r   task)r   rv   rc   r   r   r   r   r!   r   r   openjsondumpremove)r   r   r   r   configuration_pathfs         r7   r   $LLMPipeline._temp_configuration_file  s     "&"<"<VG_"MMw%'s33E	WW\\)Y5L5LM77>>,--(#.!			6BAF /II() /.s   B C1"C =#C1 
C.*C1c                    UR                  S0 5      nUR                  S0 5      nUR                  S0 5      n[        U[        5      =(       a    SU;   =US'   US'   U R                  " U40 UD6nU R                  S [
        4;   a  [        U R                  S5      (       a  U R                  R                  " S0 UDUD6nO[        U R                  S5      (       aN  [        U R                  R                  S5      (       a)  U R                  R                  R                  " S0 UDUD6nOL[        S5      e[        US	   R                  5       R                  5       5      /nU R                  " U40 UD6S
   nU R                  S [
        4;   a%  UR                  5       S
   [        US	   S
   5      S  nU R                  " U40 UD6n	U	$ )Npreprocess_paramsforward_paramspostprocess_paramsr   is_messagesgenerater   z"model does not support `generate`!r   r   r]   )r[   rv   r   
preprocessrz   r   hasattrr   r   
ValueErrorr   flattennumpytolistr   postprocess)
r   r   r   r   r   r   r   tokensoutputsresponses
             r7   _process_singleLLMPipeline._process_single"  s   "JJ':B?$4b9#ZZ(<bA &?:+?	@-(+=m+L=+<=$!88tzz:..**--II.IW--'JJ$$j32 32****33OfOO !EFF6(+335;;=>?Fjj:>:1=G$!88nn&q)#fX.>q.A*B*CDG##GB/ABr:   r   c                    [        U R                  [        5      (       d   S5       eU R                  (       d%  U R                  (       a5  U R                  S   (       a!  U R
                  (       d  U R                  5         U R                  " S0 UD6u  pEn[        U[        5      =(       a    SU;   =US'   US'   [        U[        5      (       aO  U Vs/ s H  nU R                  Xt5      PM     nn/ n	U H$  n
U	R                  U R                  XU5      5        M&     U	$ U R                  UU5      nU R                  XU5      n	U	$ s  snf )Nz,pipeline.model must be StreamingOutputMixin!r   r   r   r]   )rv   r   r(   r   modelsr   prepare_model_sanitize_parametersr   r   _preprocess_with_checkappend_stream_single)r   r   r   r   r   r   r   imodel_input_listoutputelemodel_inputs               r7   stream_generateLLMPipeline.stream_generate>  sQ   $**&:   	LK	L JJ433A&&""$@D@Y@Y AA=+= &?:+?	@-(+=m+L fd##   A ++AA    F'''(:<= ( 	 55f6GIK(();=F s   Er  r   r   c              #     #    [        U R                  U R                  5         U R                  [        R                  :X  a_  [        R
                  " 5          U R                  (       a  U R                  U5      nU R                  R                  " S0 UDUD6nS S S 5        OU R                  R                  " S0 UDUD6nW HP  nUR                  5       S   [        US   S   5      S  nU R                  " U40 UD6nU R                  U5        Uv   MR     S S S 5        g ! , (       d  f       Nm= f! , (       d  f       g = f7f)Nr   r   r]   )r$   r   r   r   r   no_gradr   _collate_fnr   r  r   r   r   _check_output)r   r  r   r   streamouts         r7   r  LLMPipeline._stream_single\  s     dnnd.>.>?~~!1!11]]_))&*&6&6{&C!ZZ77 9%9)79F %_ 33 Fk F6DF jjl1oc+h*?*B&C&DE&&sA.@A""3'		  @?$_ @?s6   !E 4D/ADA=D/	E 
D,	(D//
D=9E c                    UR                  S5      nU(       a  U R                  " XR                  40 UD6nOU R                  " U4SS0UD6nUR                  S5      US'   [        U R                  S5      (       a  U R                  R
                  nO[        U R                  S5      (       aF  [        U R                  R                  S5      (       a!  U R                  R                  R
                  nO)[        U R                  S5      (       a  S	nO[        S
5      eUR                  5        VVs0 s H4  u  pgU[        R                  " U5      (       a  UR                  U5      OU_M6     snn$ s  snnf )Nr   return_tensorsptr   r   r   r   rz   cpuz'model does not have `device` attribute!)r   r   r   r   r   r   r   itemsr   	is_tensorto)r   r   r   r   r   r   kvs           r7   r   LLMPipeline.preprocessq  s(   jj/))&..KFKF^^FJ4J6JF!::k2x4::x((ZZ&&FTZZ))gdjj6F6F6>/@ /@ZZ%%,,FTZZ11FFGG 
&  2 2V9&
 	
 
s   /;E.c                    UR                  S5      n[        U[        5      (       dn  [        R                  [
        R                  4n[        X5      (       a  [        UR                  5      S:  a  US   nU R                  R                  " U4SS0UD6nOUnU(       a  U R                  " U40 UD6nU$ [        R                  U0nU$ )Nr   rK   r   skip_special_tokensT)r   rv   rc   r   r   npndarrayr   r   r   decoder   r   TEXT)r   r   r   r   
shape_typer   s         r7   r   LLMPipeline.postprocess  s    jj/'3'',,

3J'..3w}}3E3I!!*~~,,=-1=5;=H H))(=f=H  #2Hr:   c                     0 U0 4$ )a  
this method should sanitize the keyword args to preprocessor params,
forward params and postprocess params on '__call__' or '_process_single' method
considered to be a normal classmethod with default implementation / output

Default Returns:
    Dict[str, str]:  preprocess_params = {}
    Dict[str, str]:  forward_params = {}
    Dict[str, str]:  postprocess_params = pipeline_parameters
r]   )r   generate_parameters     r7   r   LLMPipeline._sanitize_parameters  s     %r))r:   c                     [        U R                  [        5      (       a  U R                  nOU R                  R                  nUc  [        nUR                  X R                  S9$ )Nr   )rv   r   rc   r   r   r~   rs   )r   rP   r   s      r7   r   LLMPipeline._get_tokenizer  sY    djj#&&

I

,,I"+O..)?)? / A 	Ar:   r   c                     / n[         R                  U 5       H  u  pE[         R                  X4XQ5      nM     S[        R                  " U/[        R
                  S90$ )Nr   )dtype)rj   _message_iter_concat_with_special_tokensr   r   int64)r   r   r   r   r   r   s         r7   r   LLMPipeline.format_messages  sT    
 (66x@MD <<g2F A U\\6(%++FGGr:   r   c                 4    U R                  5       n SSU S.0nU$ )Nr   	assistantr   r   )strip)r   r   r   s      r7   r   LLMPipeline.format_output  s"    >>#{xHIr:   datac              #   <   #    U S    H  nUS   US   4v   M     g 7f)Nr   r   r   r]   )r7  pairs     r7   r.  LLMPipeline._message_iter  s)      $Dv,Y// %s   idsr   r   c           	          UR                   nUR                  nUR                  S5      nUR                  UR                  5       5      n[        R                  X25      n[        R                  XXUXV5      $ )N
)im_start_id	im_end_idr   r5  rj   _encode_concat)r;  r   r   r   im_startim_endnl_tokens          r7   r/  'LLMPipeline._concat_with_special_tokens  sl    
 (($$##D)

-%%i9""3$'#)5 	5r:   c                    [        U[        5      (       a  U R                  UR                  5       5      $ / nU H  nUR	                  5       u  u  pEUS:X  aw  [        U SS5      nU R                  nUS-   nUS-   n	[        [        USS95      n
[        U
5      U::  d   S5       eU	/U[        U
5      -
  -  n[        R                  X'XU5      nM  UR                  U R                  U5      5        M     U$ )	Nimageimg_token_span   rK   rO   zutf-8)encodingzImage url is too long.)rv   rc   r   rstripr  getattrimg_start_idr   bytesr   rj   rA  extend)r   r   encodedr9  modalvaluerH  rM  
img_end_id
img_pad_idlist_int_urlpad_idss               r7   r@  LLMPipeline._encode  s    gs####GNN$455D"jjlONU!(4Dc!J(55)A-
)A-
#E%'$BC "%34 N5MN 4%,.3|;L*LM%--g.:.8: y//67   r:   r   c                     U H<  n[        U[        5      (       a  U R                  U5        M+  U R                  U5        M>     U $ r=   )rv   r   rO  r  )r;  r   items      r7   rA  LLMPipeline._concat  s8    D$%%

4 

4 	 
 
r:   )r   r   r   r   r|   r   rq   r   r   r   r   r   ru   rz   r   r   r   rr   rs   )vllm)NNNNN)rT   Nr=   )+r^   r_   r`   ra   r   r   rc   r   re   ry   r   r   r   r   r   r   r   r   r   r   r   r
   r   r  r  r   r   r  r   staticmethodr   r   r   r   r	   r   r.  rd   r/  r@  rA  rf   __classcell__)r   s   @r7   rj   rj   R   s   ;z<U38_ < <D
 :>+/26&*593A"'#"63A (3A 03A !$	3A
 %-TN3A 3Aj8t 
*tCH~ 
* 
*$sCx. 8eE4;,>&? %.<$sCx. '+CH~+/S>>G*
sDy!1 
."*A H$sDc3h,@'@"A H#6H%)#u||*;%<H H   
 0sDc&)'* "+ , , -019%S/1J0 0 
5c
5"%
505c4SEHFI AJ <K 7K 1L
5 +
5 04Cy
5 
5 . sDc3h$889 . T#Y uS$s)^'< c  r:   rj   chatglm2c                 *    S nU" U 40 UD6nU" USSS9$ )Nc           	          SnU S   n U S   S   S:X  d   S5       e[        S[        U 5      S-
  S5       H,  nUS	R                  US-  S-   X   S
   XS-      S
   5      -  nM.     USR                  [        U 5      S-  S-   U S   S
   5      -  nU$ )N r   r   r   userz&chatglm2 does not have system messagesrK   rO   u    [Round {}]

问：{}

答：{}

r   u   [Round {}]

问：{}

答：r   )ranger   format)r   r   r   r  s       r7   build_chatglm2_prompt7chatglm2_format_messages.<locals>.build_chatglm2_prompt  s    J'{ 	HG	H  q#h-!+Q/A>EEQ
HK	2HUOI4NP PF 0 	4;;MQ"HRL$;= 	=r:   Fr  return_token_type_idsr  r]   )r   r   r   re  r   s        r7   chatglm2_format_messagesri    s'     #86v6FV5NNr:   chatglmc                 \    U R                  5       n U R                  SS5      n SU S.nSU0nU$ )Nu   [[训练时间]]u   2023年r3  r4  r   )r5  replace)r   r   r   r   s       r7   chatglm2_format_outputrm    s@     ~~H 2I>H#9H8G Nr:   llamallama2c                 >    SSK Jn  S nU" X40 UD6u  pVU" SU05      $ )Nr   BatchEncodingc                    UR                  SS5      nSnU S   n U S   S   S:w  a	  SUS./U -   n U S   S	   nS
U S3nU" USS9R                  nU S   S	   nUR                  5        S3n	U" U	SS9R                  n
UR                  S   U
R                  S   -   nX:  a  [	        SU SU 35      eSn/ n[        [        U 5      S-
  SS5       H~  nX   S	   XS-      S	   nnUR                  5        SUR                  5        S3nU" USS9R                  nUUR                  S   -   U:  a    O!UU-   nU/U-   nUUR                  S   -  nM     XlU	/nU/U-   U
/-   nSR                  U5      [        R                  " USS94$ )N
max_length   zyou are a helpful assistant!r   r   r   r   r4  r   z<s>[INST] <<SYS>>
z
<</SYS>>

r  r  r   z [/INST]zprepend prompt length z is bigger than max_length ra  rO   rK   z	 [/INST] z </s><s>[INST] )dim)
r[   r   r5  r   RuntimeErrorrc  r   r   r   cat)r   r   r   rt  default_system_messager   system_prompt
system_idstexttext_prompttext_idsprompt_lengthhistory_prompthistory_ids_listr  rb  r3  round_prompt	round_idsprompt_listprompt_ids_lists                        r7   build_llama2_prompt3llama2_format_messages.<locals>.build_llama2_prompt  s   ZZd3
!?J'A;v(* 1  H
 !Y'-fX^D}TBLL
|I&h/[>HH"((,x~~b/AA%(7RS]R^_ 
 s8}q(!R0A&k)4hAB@C 7DDM7O)D"jjl^9Y__5F4GWL!,tDNNIyr22Z?!-!>$-;1A#A !44 1 %kB%,)99XJFww{#UYYB%GGGr:   r   )transformersrr  )r   r   r   rr  r  r   r   s          r7   llama2_format_messagesr    s1     +*HX )GGNF+v.//r:   baichuan	baichuan2c                    SSK Jn  SS jnU S   n SnSnUR                  SS 5      =(       d    S	nS
nX-
  n	U" U SS9u  pUR                  U
5      nU	[	        U5      -
  n/ nUS S S2    H  n/ nU HR  nUS   S:X  a  UR                  U5        OUR                  U5        UR                  UR                  US   5      5        MT     [	        U5      S:X  d  [	        U5      [	        U5      -   U::  a  UU-   n[	        U5      U:  a  M    O   X-   nU S   S   S:w  a  UR                  U5        UU	* S  n[        R                  " U/5      nU" SU05      $ )Nr   rq  rb  c                    S/ p2/ n[        U 5       HV  u  pVUS   S:X  a  US:X  d   S5       eUS   nM"  US   U:X  a  U(       a  UR                  U5        / nUR                  U5        MX     U(       a  UR                  U5        X#4$ )Nra  r   r   r   'first message should be system message.r   )	enumerater  )r   
split_roler   roundsroundr  r   s          r7   _parse_messages1baichuan_format_messages.<locals>._parse_messagesR  s    R#H-JAv(*AvHHHv +v*,e$LL! . MM% ~r:   r         max_new_tokensru  i   )r  r   r   r   r3  r   )rb  )	r  rr  r[   r   r   r  rO  r   
LongTensor)r   r   r   rr  r  assistant_token_iduser_token_idr  model_max_lengthmax_input_tokensr   r  system_tokensmax_history_tokenshistory_tokensr  round_tokensr   input_tokenss                      r7   baichuan_format_messagesr  M  s    +  
#HMZZ 0$7?4N'8$X&ANF$$V,M)C,>>N"Gv&(##M2##$67	 0 01C DE  ~!#s>':S> (!3(4)N:N>"%77  !1L|F{*./!1 1 23L##\N3L+|455r:   wizardlmc                 *    S nU" X40 UD6nU" USSS9$ )Nc                 8   Sn U S   n U S   S   S:w  a	  SUS./U -   n U S   S   nU/n[        U SS  5       HN  u  pgUS   S	:X  a  US   nUR                  S
U 35        M)  US   S:X  d  M4  US   nUR                  SU S35        MP     SR                  U5      n	U	$ )NzGA chat between a curious user and an artificial intelligence assistant.r   r   r   r   r4  r   rK   rb  zUSER: r3  zASSISTANT: z</s> )r  r  r   )
r   r   r   r{  r|  r  r  r   user_promptpromptss
             r7   build_wizardlm_prompt7wizardlm_format_messages.<locals>.build_wizardlm_prompt  s    !j]J'A;v(* 1  H
 !I.$o#HQRL1JAv&(%i0""VK=#9:K/%i0""[T#BC 2 ((;'r:   Fr  rg  r]   )r   r   r   r  r  s        r7   wizardlm_format_messagesr    s'    0 $HB6BGWE$OOr:   
wizardcodec                    U S   n [        U 5      S:X  d   S5       eSu  p4[        U 5       H=  u  pVUS   S:X  a  US:X  d   S5       eUS	   nUS   S
:X  d  M+  US:X  d   S5       eUS	   nM?     US-   U-   S-   nU" USSSSS9nU$ )Nr   rO   z&wizard code only support two messages.)ra  ra  r   r   r   r  r   rb  rK   z&second message should be user message.z

### Instruction:
z

### Response:Fr  )rh  paddingadd_special_tokensr  )r   r  )	r   r   r   r   rb  r  r   r   r   s	            r7   wizardcode_format_messagesr    s    
#Hx=AGGGLF)
6?h&6DDD6Y'F6?f$6CCC69%D * ..58KKF# F Mr:   c                     U S   n U S   S   U S S pCUR                  X4S9nUR                  UR                  S5      UR                  S5      /nXeS'   U$ )Nr   r   r   )r   z<|user|>z<|observation|>eos_token_id)build_chat_inputr  get_command)r   r   r   r   r   r   r  s          r7   chatglm3_format_messagesr    sq    
#Hb\),hsm7'''?Fj)/0L
 *>Mr:   qwen2c                 >    U S   n UR                  U SSS9nU" U/SS9$ )Nr   FT)tokenizeadd_generation_promptr  rv  )apply_chat_template)r   r   r   r~  s       r7   qwen2_format_messagesr    s7    
#H((5 ) >DdVD11r:   )Pr   
contextlibr   	threadingr   typingr   r   r   r   r	   r
   r   r   r   r   r   r!  r   r  r   r   r   
modelscoper   r   r   r   modelscope.hub.file_downloadr   modelscope.models.baser   modelscope.models.nlpr   r   modelscope.outputsr   modelscope.pipelines.baser   modelscope.pipelines.builderr   modelscope.pipelines.utilr   r   modelscope.utils.configr   modelscope.utils.constantr   r    r!   r"   modelscope.utils.devicer#   r$   modelscope.utils.loggerr%   "modelscope.utils.model_type_helperr&   !modelscope.utils.streaming_outputr'   r(   r)   rw   r   r,   register_moduler   text_generationrj   rG   ri  rL   rm  r  r  r  r  r  r  rQ   r]   r:   r7   <module>r     s=   	 % " " "    I I+ + < ( D ) + 2 D * J J C . >D D 
+. +.\ 5::59
500eD`(8 ` E :`F ,,Z8O 9O( **95**:6 7 6 ,,W5,,X600 7 600f ,,Z8,,[926 : 926j ,,Z8P 9P< ,,\: ;, ,,Y7
 8
 ,,W52 62  % %j2C D  % %g ?  % %h @r:   