
    9i!f                        S SK r S SKrS SKJr  S SKJrJrJrJrJ	r	J
r
  S SKrS SKJr  S SKrS SKJr  S SKJr  S SKJr  S SKJr  S SKJrJrJrJrJrJr  S S	KJr  S S
KJr  S SK J!r!   " S S5      r" " S S\"5      r# " S S\"5      r$S\4S jr%g)    N)contextmanager)AnyDict	GeneratorListOptionalUnion)version)nn)PreTrainedModel)GreedySearchDecoderOnlyOutput) GreedySearchEncoderDecoderOutputLogitsProcessorListSampleDecoderOnlyOutputSampleEncoderDecoderOutputStoppingCriteriaListvalidate_stopping_criteria)Input)
Frameworks)device_placementc                   "    \ rS rSrS\4S jrSrg)StreamingOutputMixin   returnc                     [         e)z
Support the input of Model and Pipeline.
The output is a `Generator` type,
which conforms to the output standard of modelscope.
)NotImplementedError)selfargskwargss      a/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/utils/streaming_output.pystream_generate$StreamingOutputMixin.stream_generate   s
     "!     N)__name__
__module____qualname____firstlineno__r   r!   __static_attributes__r$   r#   r    r   r      s    ") "r#   r   c            
           \ rS rSrS\\\\   4   S\4S jrS\S\	\
\4   S\	\
\4   4S jrS\	\
\4   S\	\
\4   S	\	\
\4   S\4S
 jrS\\	\
\4      S\S\	\
\4   S	\	\
\4   S\4
S jrSrg)PipelineStreamingOutputMixin%   inputr   c                    [        U R                  [        5      (       d   S5       eU R                  (       d%  U R                  (       a5  U R                  S   (       a!  U R
                  (       d  U R                  5         UR                  SS5      nU R                  " S0 UD6u  pVn[        U[        5      (       ag  U Vs/ s H  nU R                  X5      PM     n	nUc.  / n
U	 H$  nU
R                  U R                  XU5      5        M&     U
$ U R                  XXg5      n
 U
$ U R                  X5      nU R                  XU5      n
U
$ s  snf )z
Similar to the `Pipeline.__call__` method.
it supports the input that the pipeline can accept,
and also supports batch input.

self.model must be a subclass of StreamingOutputMixin
and implement the stream method.
z,pipeline.model must be StreamingOutputMixin!r   
batch_sizeNr$   )
isinstancemodelr   has_multiple_modelsmodels_model_prepareprepare_modelpop_sanitize_parameterslist_preprocess_with_checkappend_stream_single_stream_batch)r   r-   r   r   r/   preprocess_paramsforward_paramspostprocess_paramsimodel_input_listoutputelemodel_inputs                r    r!   ,PipelineStreamingOutputMixin.stream_generate'   sY    $**&:   	LK	L JJ433A&&""$ZZd3
@D@Y@Y AA=+= eT""  A ++AA   
 !+CMM++C,>@A ,  ++,<,:P  55eOK(();=F' s   >Er=   c                 J    U R                  U5        U R                  " U40 UD6$ N)_check_input
preprocess)r   r-   r=   s      r    r9   3PipelineStreamingOutputMixin._preprocess_with_checkQ   s'     	% u:(9::r#   rD   r>   r?   c              #   6  #    [        U R                  U R                  5         U R                  [        R                  :X  a^  [        R
                  " 5          U R                  (       a  U R                  U5      nU R                  R                  " U40 UD6nS S S 5        OU R                  R                  " U40 UD6nW H+  nU R                  " U40 UD6nU R                  U5        Uv   M-     S S S 5        g ! , (       d  f       NH= f! , (       d  f       g = f7frG   )r   	frameworkdevice_namer   torchno_grad_auto_collate_collate_fnr1   r!   postprocess_check_output)r   rD   r>   r?   streamouts         r    r;   +PipelineStreamingOutputMixin._stream_singleW   s      dnnd.>.>?~~!1!11]]_))&*&6&6{&C!ZZ77#7'57F %_ 33K F6DF &&sA.@A""3'	  @?$_ @?s6   !D4DA C7AD.	D7
D	D
DDrA   r/   c           
   #   v  ^#    / n/ n[        U R                  U R                  5         [        S[	        U5      U5       H  n[        Xr-   [	        U5      5      nX-
  n	UR                  U	5        U R                  XU 5      n
U R                  [        R                  :X  an  [        R                  " 5          U R                  (       a  U R                  U
5      n
UR                  U R                  R                  " U
40 UD65        S S S 5        M  UR                  U R                  R                  " U
40 UD65        M     S /[	        U5      -  nSnU[	        U5      :  Ga  Sn[        [!        XV5      5       H  u  nu  p [#        U5      n
[        U	5       H  m0 nU
R%                  5        Hz  u  nnUc  M  ['        U[(        [*        45      (       aJ  ['        US   [        R,                  5      (       a  [/        U5      " U4S jU 5       5      X'   Mg  UT   X'   Mp  UTTS-    X'   M|     U R0                  " U40 UD6nU R3                  U5        Xr-  T-   nXU'   M     M     Uv   U[	        U5      :  a  GM  S S S 5        U$ ! , (       d  f       GMN  = f! [4         a
    US-  n GM/  f = f! , (       d  f       W$ = f7f)Nr   c              3   4   >#    U  H  nUTTS -    v   M     g7f)   Nr$   ).0e	batch_idxs     r    	<genexpr>=PipelineStreamingOutputMixin._stream_batch.<locals>.<genexpr>   s%      CB9@A 12)IM0J9@s   rY   )r   rL   rM   rangelenminr:   _batchr   rN   rO   rP   rQ   r1   r!   	enumeratezipnextitemsr0   tupler8   TensortyperR   rS   StopIteration)r   rA   r/   r>   r?   stream_listreal_batch_sizesr@   endreal_batch_sizebatched_outoutput_liststop_streamsrT   rU   kelementoutput_indexr\   s                     @r    r<   *PipelineStreamingOutputMixin._stream_batchk   s     dnnd.>.>?1c"23Z@!.#.>*?@"%' ''8"kk*:S*AB>>Z%5%55--*.*:*:;*GK#** JJ66{ I9GIJ )  &&

22; E5CEF A$  &3'7#88KL[!11 4=K:5<0A0*&*6l).)?I"$C.9.?.?.A
7#*#6'1'E4='I'I+5gaj6;ll,D ,D59'] CB9@CB 6BCF
 6=Y5GCF189;<D= 2> /B #'"2"23"M:L"MC ..s3+,>I+EL8;5' *@	5<4 "!9 [!11+ @f W )N ) *$)*_ @?f sb   %J9BJ'4AI=A9J'=2J3B&JJ'3
J9=
J	J'J$J'#J$$J''
J61J9r$   N)r%   r&   r'   r(   r	   r   r   r   r!   r   strr   r9   r;   intr<   r)   r$   r#   r    r+   r+   %   s    (U5$u++=%> (%.(T;;#CH~;26sCx.;$sCx. '+CH~+/S>>G(9d4S>.B 9"%97;CH~9*.sCx.9=F9r#   r+   c                      \ rS rSrS\4S jr\S\S\4S j5       r\	          SS\
R                  S\\   S	\\   S
\\   S\\   S\\\\\   4      S\\   S\\   S\\   S\\   S\S\4S jj5       r\	           SS\
R                  S\\   S	\\   S\\   S
\\   S\\   S\\\\\   4      S\\   S\\   S\\   S\\   S\S\4S jj5       rSrg)#PretrainedModelStreamingOutputMixin   r   c                     [        U [        5      (       a  U OU R                  n[        U[        5      (       d   S5       eU R                  U5         UR                  " U0 UD6sS S S 5        $ ! , (       d  f       g = f)Nz-self or self.model must be `PretrainedModel`!)r0   r   r1   _replace_generategenerate)r   r   r   r1   s       r    r!   3PretrainedModelStreamingOutputMixin.stream_generate   sa    "499tzz%11 	<;	<1##E*>>4262 +**s   A--
A;r1   c              #   (  #    [         R                  " [        R                  5      [         R                  " S5      :  a  SnSnOE[         R                  " [        R                  5      [         R                  " S5      :  a  SnSnOSnSn[	        X5      n[	        X5      n[        X[        R                  " U R                  U5      5        [        X[        R                  " U R                  U5      5        S v   [        XU5        [        XU5        g 7f)Nz4.43.0stream_greedy_search_samplez4.39.0_greedy_searchgreedy_searchsample)
r
   parsetransformers__version__getattrsetattrtypes
MethodTyper   stream_sample)r   r1   greedy_search_namesample_nameorigin_greedy_searchorigin_samples         r    r|   5PretrainedModelStreamingOutputMixin._replace_generate   s     ==112gmmH6MM!7#K]]((*-4]]8-DE!1#K!0"K&uA3  !:!:EB	DE$4$4T5G5G5:%< 	=+?@M2s   DDN	input_idslogits_processorstopping_criteria
max_lengthpad_token_ideos_token_idoutput_attentionsoutput_hidden_statesoutput_scoresreturn_dict_in_generatesynced_gpusc           
   +   	  #    Ub  UO	[        5       nUb  UO	[        5       nUb&  [        R                  " S[        5        [        X45      nUb  UOU R                  R                  nUb  UOU R                  R                  n[        U[        5      (       a  U/nUb/  [        R                  " U5      R                  UR                  5      OS nU	b  U	OU R                  R                  n	Ub  UOU R                  R                   nUb  UOU R                  R"                  nU
b  U
OU R                  R$                  n
U
(       a	  U	(       a  SOS nU
(       a	  U(       a  SOS nU
(       a	  U(       a  SOS nU
(       a	  U(       a  SOS nU
(       aU  U R&                  R(                  (       a:  U(       a  US   R+                  S5      OS nU(       a  US   R+                  S5      OS n[        R,                  " UR.                  S   [        R0                  UR                  S9nSn U(       az  [        R                  " U(       a  S
OS5      R                  UR                  5      n[2        R4                  " U[2        R6                  R8                  S9  UR;                  5       S
:X  a  g U R<                  " U40 UD6nU " S0 UDS	UUS.D6nU(       a	  U(       a  M  UR>                  S S 2SS S 24   nU" UU5      nU
(       a  U	(       a  UU4-  nU(       ac  UU R&                  R(                  (       a  UR@                  4OURB                  4-  nU R&                  R(                  (       a  UURD                  4-  nU(       a8  UU R&                  R(                  (       a  URF                  4OURH                  4-  n[        RJ                  " USS9nUb  Uc  [M        S5      eUU-  USU-
  -  -   n[        RN                  " UUS S 2S 4   /SS9nU
(       a<  U R&                  R(                  (       a  [Q        UUWWUUUS9v   O[S        UUUUS9v   OUv   U RU                  UUU R&                  R(                  S9nUbo  URW                  URY                  UR.                  S   S5      R[                  UR]                  S5      5      R_                  SS95      nURa                  5       S:X  a  S	nU" X5      (       a  S	nU(       a  U(       d  g GM  7f)Nz`max_length` is deprecated in this function, use `stopping_criteria=StoppingCriteriaList([MaxLengthCriteria(max_length=max_length)])` instead.r$   encoder_outputs
attentionshidden_statesr   dtypedeviceFT              ?opreturn_dictr   r   dimGIf `eos_token_id` is defined, make sure that `pad_token_id` is defined.rY   	sequencesscoresencoder_attentionsencoder_hidden_statesdecoder_attentionscross_attentionsdecoder_hidden_statesr   r   r   r   is_encoder_decoder)1r   r   warningswarnUserWarningr   generation_configr   r   r0   rw   rN   tensortor   r   r   r   r   configr   getonesshapelongdist
all_reduceReduceOpSUMitemprepare_inputs_for_generationlogitsr   r   r   r   r   argmax
ValueErrorcatr   r   #_update_model_kwargs_for_generationmultilene	unsqueezeprodmax)r   r   r   r   r   r   r   r   r   r   r   r   model_kwargseos_token_id_tensorr   r   r   r   r   r   unfinished_sequencesthis_peer_finishedthis_peer_finished_flagmodel_inputsoutputsnext_token_logitsnext_tokens_scoresnext_tokenss                               r    r   8PretrainedModelStreamingOutputMixin.stream_greedy_search   s      0@/K+Qd R
1B1N-Th U
!MMq
 !;!!/'3'?|TE[E[EhEh'3'?|TE[E[EhEhlC(((>L!-!9 $ll<8;;?C 	)6)BH^H^HlHl!2!>""44 	 %9$D ""77 	 (?'J#"":: 	 
 0M$;(9  @D 	"9&7 >B 	'>+? !#FJ 	 #t{{'E'E!2 "..?!@!D!D"8<  ( ./33OD-1 "
  %zzOOAejj9I9I K # +0,,-C3+88:9;K;K8L (  7DMM<M<MN*//1S8  ==+)+L   "3%9	G 1 'q"ax 8 "2)=N!O ' 144F$&+/;;+I+I -4,F,F+I,3,>,>+AC& {{55(W-E-E,HH(')151O1O 07/L/L.O4;4I4I3LN)
  ,,'9rBK ''$a  *,@@<,,D. . 		9k!T'.B"CLI ';;11:"+%+=.C+=)9.C  8"+%#5&;	   CC#';;#A#A D CL #.';'?'?$$%8%>%>q%A1EHH+55a8::>$1$+(G$
 (++-2)-& !33%)"!+S s   S7S9logits_warperc           
   +   
  #    Ub  UO	[        5       nUb  UO	[        5       nUb&  [        R                  " S[        5        [        X55      nUb  UO	[        5       nUb  UOU R                  R                  nUb  UOU R                  R                  n[        U[        5      (       a  U/nUb/  [        R                  " U5      R                  UR                  5      OS nU
b  U
OU R                  R                  n
Ub  UOU R                  R                   nU	b  U	OU R                  R"                  n	Ub  UOU R                  R$                  nU(       a	  U
(       a  SOS nU(       a	  U(       a  SOS nU(       a	  U(       a  SOS nU(       a	  U	(       a  SOS nU(       aU  U R&                  R(                  (       a:  U(       a  US   R+                  S5      OS nU	(       a  US   R+                  S5      OS n[        R,                  " UR.                  S   [        R0                  UR                  S9nSn U(       az  [        R                  " U(       a  S
OS5      R                  UR                  5      n[2        R4                  " U[2        R6                  R8                  S9  UR;                  5       S
:X  a  g U R=                  X5      nU R>                  " U40 UD6nU " S0 UDS	UU	S.D6nU(       a	  U(       a  M  UR@                  S S 2SS S 24   nU" UU5      nU" UU5      nU(       a  U
(       a  UU4-  nU(       ac  UU R&                  R(                  (       a  URB                  4OURD                  4-  nU R&                  R(                  (       a  UURF                  4-  nU	(       a8  UU R&                  R(                  (       a  URH                  4OURJ                  4-  n[L        RN                  RQ                  USS9n[        RR                  " USS9RU                  S5      nUb  Uc  [W        S5      eUU-  USU-
  -  -   n[        RX                  " UUS S 2S 4   /SS9nU(       a<  U R&                  R(                  (       a  [[        UUWWUUUS9v   O[]        UUUUS9v   OUv   U R_                  UUU R&                  R(                  S9nUbo  URa                  URc                  UR.                  S   S5      Re                  URg                  S5      5      Ri                  SS95      nURk                  5       S:X  a  S	nU" X5      (       a  S	nU(       a  U(       d  g GM&  7f)Nz`max_length` is deprecated in this function, use `stopping_criteria=StoppingCriteriaList(MaxLengthCriteria(max_length=max_length))` instead.r$   r   r   r   r   r   FTr   r   r   r   r   r   rY   )num_samplesr   r   r   r   )6r   r   r   r   r   r   r   r   r   r0   rw   rN   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   _get_initial_cache_positionr   r   r   r   r   r   r   r   
functionalsoftmaxmultinomialsqueezer   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   next_token_scoresprobsr   s                                 r    r   1PretrainedModelStreamingOutputMixin.stream_samplet  sk    " 0@/K+Qd R
1B1N-Th U
!MMo
 !;!!/)6)BH[ I
'3'?|TE[E[EhEh'3'?|TE[E[EhEhlC(((>L!-!9 $ll<8;;?C 	)6)BH^H^HlHl!2!>""44 	 %9$D ""77 	 (?'J#"":: 	 
 0M$;(9  @D 	"9&7 >B 	'>+? !#FJ 	 #t{{'E'E!2 "..?!@!D!D"8<  ( ./33OD-1 "
  %zzOOAejj9I9I K # +0,,-C3+88:9;K;K8L (  7DMM<M<MN*//1S8  ;;)L==+)+L   "3%9	G 1 'q"ax 8 !1<M N -i9J K ' 033F$&+/;;+I+I -4,F,F+I,3,>,>+AC& {{55(W-E-E,HH(')151O1O 07/L/L.O4;4I4I3LN)
 MM))*;)DE++EqAII!LK ''$a  *,@@<,,D. . 		9k!T'.B"CLI ';;114"+%+=.C+=)9.C  2"+%#5&;	   CC#';;#A#A D CL #.';'?'?$$%8%>%>q%A1EHH+55a8::>$1$+(G$
 (++-2)-& !33%)"!+[ s   UUr$   )
NNNNNNNNNF)NNNNNNNNNNF)r%   r&   r'   r(   r   r!   r   r   r|   staticmethodrN   
LongTensorr   r   r   rw   r	   r   boolr   r   r)   r$   r#   r    ry   ry      s-   3) 3 3 39 3 3*  ;?<@$(&*8<,0/3(,26!k##k ##67k $$89	k
 SMk smk uS$s)^45k $D>k 'tnk  ~k "*$k k 
k kZ  ;?<@7;$(&*8<,0/3(,26!s##s ##67s $$89	s
   34s SMs sms uS$s)^45s $D>s 'tns  ~s "*$s s 
s sr#   ry   r1   c                     [        U 5      nU[        4n[        UR                  U0 5      " U R                  5      nUR                  R                  U R                  5        U$ rG   )ri   ry   r%   r   __dict__update)r1   pretrained_classparent_classes	new_models       r    add_stream_generater   +  sV    E{&(KLN%..CIenn-r#   )&r   r   
contextlibr   typingr   r   r   r   r   r	   rN   torch.distributeddistributedr   r   	packagingr
   r   r   transformers.generationr   r   r   r   r   r   r   modelscope.pipelines.baser   modelscope.utils.constantr   modelscope.utils.devicer   r   r+   ry   r   r$   r#   r    <module>r      sv      % > >       ( AA A , 0 4" "#7 DA*> AH r#   