
    9i              
          S r SSKrSSKrSSKrSSKrSSKJrJrJrJ	r	J
r
Jr  SSKrSSKJs  Jr  SSKrSSKJr  SSKJrJr  SSKJr  SSKJr  SSKJrJrJrJr  SS	KJrJ r   SS
K!J"r"  SSK#J$r$J%r%  SSK&J'r'  SSK(J)r)  SSK*J+r,  SSK-J.r.  SSK/J0r0  SSK1J2r2  \Rf                  S:w  al  \Rh                  Rk                  S5        \Rh                  Rm                  S5        \Rh                  Ro                  S5        \Rh                  Rq                  S5        \,Rr                  " 5       r+Sr:Sr;S/r<S r= " S S\5      r> " S S\R                  R~                  5      r@ S?S\R                  S \BS!\CS"\	\R                     4S# jjrD " S$ S%\R~                  5      rE\R                  R                  S&\R                  S'\R                  S"\R                  4S( j5       rH " S) S*\R                  R~                  5      rI " S+ S,\R                  R~                  5      rJ " S- S.\R                  R~                  5      rKS/ rL " S0 S1\R                  R~                  5      rM " S2 S3\R                  R~                  5      rN " S4 S5\R                  R~                  5      rO " S6 S7\%\"5      rP " S8 S9\R                  R~                  5      rQ " S: S;\P5      rR\0R                  " \.R                  \'R                  S<9 " S= S>\P5      5       rVg)@zPyTorch ChatGLM model.     N)AnyCallableDictListOptionalTuple)nn)CrossEntropyLoss	LayerNorm)	skip_init)LogitsProcessor)GenerationConfigLogitsProcessorListModelOutputStoppingCriteriaList)BaseModelOutputWithPastCausalLMOutputWithPast)PreTrainedModel)Model
TorchModel)Models)
OutputKeys)logger)Tasks   )MODELS   )ChatGLM2ConfigdarwinFTzTHUDM/ChatGLM2-6BChatGLM6BConfigzTHUDM/chatglm2-6bc                     U " U0 UD6$ N )clsargskwargss      n/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/models/nlp/chatglm2/text_generation.pydefault_initr(   2   s        c                   f    \ rS rSrS\R
                  S\R                  S\R                  4S jrSrg)InvalidScoreLogitsProcessor6   	input_idsscoresreturnc                     [         R                  " U5      R                  5       (       d)  [         R                  " U5      R                  5       (       a  UR	                  5         SUS'   U$ )Ng     j@).   )torchisnananyisinfzero_)selfr-   r.   s      r'   __call__$InvalidScoreLogitsProcessor.__call__8   sH    ;;v""$$F(;(?(?(A(ALLN F6Nr)   r#   N)	__name__
__module____qualname____firstlineno__r2   
LongTensorFloatTensorr8   __static_attributes__r#   r)   r'   r+   r+   6   s0    %"2"2 **/4/@/@r)   r+   c                   V   ^  \ rS rSrSrS\4U 4S jjrS\R                  4S jr	Sr
U =r$ )PrefixEncoder@   z
The torch.nn model to encode the prefix
Input shape: (batch-size, prefix-length)
Output shape: (batch-size, prefix-length, 2*layers*hidden)
configc                   > [         TU ]  5         UR                  U l        U R                  (       a  UR                  UR                  -  UR
                  -  S-  n[        R                  R                  UR                  U5      U l
        [        R                  R                  [        R                  R                  X!R                  5      [        R                  R                  5       [        R                  R                  UR                  U5      5      U l        g [        R                  R                  UR                  UR                  UR                  -  UR
                  -  S-  5      U l
        g )N   )super__init__prefix_projection
num_layerskv_channelsmulti_query_group_numr2   r	   	Embeddingpre_seq_len	embedding
SequentialLinearhidden_sizeTanhtrans)r7   rD   kv_size	__class__s      r'   rH   PrefixEncoder.__init__G   s   !'!9!9!!''&*<*<<v?[?[[^__G"XX//0B0BGLDN,,););<ehhmmo 2 2G<>DJ #XX//""F$5$58J8J$J..%/12%34DNr)   prefixc                     U R                   (       a$  U R                  U5      nU R                  U5      nU$ U R                  U5      nU$ r"   )rI   rO   rT   )r7   rX   prefix_tokenspast_key_valuess       r'   forwardPrefixEncoder.forwardV   sE    !! NN62M"jj7O  #nnV4Or)   )rO   rI   rT   )r:   r;   r<   r=   __doc__r   rH   r2   Tensorr\   r@   __classcell__rV   s   @r'   rB   rB   @   s)    4~ 4ell  r)   rB   tensornum_partitionscontiguous_split_chunksr/   c                     U R                  5       S-
  nU R                  5       U   U-  n[        R                  " XUS9nU(       a  [	        S U 5       5      $ U$ )a  Split a tensor along its last dimension.

Arguments:
    tensor: input tensor.
    num_partitions: number of partitions to split the tensor
    contiguous_split_chunks: If True, make each chunk contiguous
                             in memory.

Returns:
    A list of Tensors
r   dimc              3   @   #    U  H  oR                  5       v   M     g 7fr"   )
contiguous).0chunks     r'   	<genexpr>.split_tensor_along_last_dim.<locals>.<genexpr>v   s     A[E%%''[s   )rg   sizer2   splittuple)rb   rc   rd   last_dimlast_dim_sizetensor_lists         r'   split_tensor_along_last_dimrt   _   sS    " zz|aHKKM(+~=M++fBKA[AAAr)   c                      ^  \ rS rSr    SU 4S jjr SS\S\S\R                  S\R                  S\4
S jjr	SS	 jr
S
rU =r$ )RotaryEmbedding{   c           	         > [         TU ]  5         SS[        R                  " SUSUS9R	                  US9U-  -  -  nU R                  SU5        Xl        X0l        X l        g )N      ?'  r   rF   device)dtypeinv_freq)	rG   rH   r2   arangetoregister_bufferrg   original_impl
rope_ratio)r7   rg   r   r   r|   r}   r~   rV   s          r'   rH   RotaryEmbedding.__init__}   sh     	%LLC6255E5BSHJ KZ2*$r)   seq_lenn_elemr}   r|   basec           
         SU[         R                  " SUSX4S9U-  -  -  n[         R                  " XUS9U R                  -  n[         R                  " Xv5      R	                  5       n[         R
                  " [         R                  " U5      [         R                  " U5      /SS9n	U[         R                  [         R                  [         R                  4;   a4  U[         R                  :X  a  U	R                  5       OU	R                  5       n	U	$ )a-  Enhanced Transformer with Rotary Position Embedding.

Derived from: https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/
transformers/rope/__init__.py. MIT License:
https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/master/license.
ry   r   rF   r}   r|   rf   )r2   r   r   outerfloatstackcossinfloat16bfloat16int8half)
r7   r   r   r}   r|   r   thetaseq_idx	idx_thetacaches
             r'   forward_implRotaryEmbedding.forward_impl   s     5<<61EI 
 ,,137??C KK/557	YYy!599Y#78bB U]]ENNEJJ??%..( NN .3jjl r)   c                     U R                  UU R                  U R                  R                  U R                  R                  S9$ Nr   )r   rg   r~   r}   r|   )r7   max_seq_lenoffsets      r'   r\   RotaryEmbedding.forward   s>      HH--%%==''	 ! ) 	)r)   )rg   r   r   )r   FNN)rz   )r   )r:   r;   r<   r=   rH   intr2   r}   r|   r   r\   r@   r`   ra   s   @r'   rv   rv   {   sa     $%& "'!  "KK #\\	
 B) )r)   rv   x
rope_cachec                 &   U R                  S5      U R                  S5      U R                  S5      U R                  S5      4u  p#pCUR                  S   S-  nU SS U24   U SUS 24   p`US U nU R                  USXES-  S5      nUR                  USSUR                  S5      S5      n[        R
                  " US   US   -  US	   US	   -  -
  US	   US   -  US   US	   -  -   /S5      nUR                  S5      n[        R                  " X4SS
9$ )Nr   r   rF   r   .r   ).r   ).r   rf   )rn   shapereshapeviewr2   r   flattencat)	r   r   sq_nprot_dimx_passxshapedx_out2s	            r'   apply_rotary_pos_embr      s5    66!9affQiAFF1I=LB2r"Q&G#xx- !CM"2vCRJiiBqL!4GRGLLOQ?J[[FOj00fo
6 223FOj00fo
6 223	
 	F ^^AF99f%2..r)   c                   T   ^  \ rS rSr   SU 4S jjrS\R                  4S jrSrU =r	$ )RMSNorm   c                    > [         TU ]  5         [        R                  R	                  [        R
                  " XUS95      U l        X l        g )Nr|   r}   )rG   rH   r2   r	   	Parameteremptyweighteps)r7   normalized_shaper   r|   r}   r&   rV   s         r'   rH   RMSNorm.__init__   s;     	hh((KK(uEGr)   hidden_statesc                    UR                   nUR                  [        R                  5      R	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  U-  R                  U5      $ )NrF   r   T)keepdim)	r}   r   r2   float32powmeanrsqrtr   r   )r7   r   input_dtypevariances       r'   r\   RMSNorm.forward   su    #)) ##EMM266q9>> ? %Hxx4G(HHm+//<<r)   )r   r   )gh㈵>NN)
r:   r;   r<   r=   rH   r2   r_   r\   r@   r`   ra   s   @r'   r   r      s(     		=U\\ = =r)   r   c                   6   ^  \ rS rSrS\4U 4S jjrS rSrU =r$ )CoreAttention   rD   c                 `  > [         [        U ]  5         UR                  U l        UR                  U l        U R                  (       a  SU l        [        SU5      U l        UR                  UR                  -  nX0l	        X1R                  -  U l
        UR                  U l        S n[        R                  " U R                  5      U l        U R                  (       a!  U R                  nU =R                  U-  sl        X@l        [         R"                  R%                  UR&                  5      U l        g )NTr   )rG   r   rH   apply_query_key_layer_scalingattention_softmax_in_fp32maxlayer_numberrK   num_attention_headshidden_size_per_partitionhidden_size_per_attention_head!num_attention_heads_per_partitionmathsqrtnorm_factorcoeffr2   r	   Dropoutattention_dropout)r7   rD   r   projection_sizer   rV   s        r'   rH   CoreAttention.__init__   s    mT+--3-Q-Q*)/)I)I&---1D*<0 ,,v/I/II *9&.=A[A[.[+171K1K.99T%H%HI--%%E%
!&!1!1&2J2J!Kr)   c           	         [        [        R                  R                  S5      S   5      nUS:  a  XU4 Vs/ s H  nUR	                  SSSS5      PM     snu  pnUcJ  UR
                  S   UR
                  S   :X  a*  [        R                  R                  R                  XUSS9nO0Ub  U) n[        R                  R                  R                  XX45      nUR	                  SSSS5      nUR                  5       S S U R                  4-   nUR                  " U6 nU$ UR                  S5      UR                  S5      UR                  S5      UR                  S5      4n	UR                  U	S   U	S   U	S   -  S	5      nUR                  U	S   U	S   U	S   -  S	5      n[        R                  " U	S   U	S   -  U	S   U	S   UR                  UR                  S
9n
[        R                   " U
UR#                  SS5      UR#                  SS5      R#                  SS5      SSU R$                  -  S9nUR                  " U	6 nU R&                  (       a  UR)                  5       nU R*                  b  XR*                  -  nUcm  UR
                  S   UR
                  S   :X  aM  [        R,                  " U	S   SU	S   U	S   UR                  [        R.                  S9nUR1                  5         U) nUb  UR3                  U[)        S5      5      n[4        R6                  " US	S9nUR9                  U5      nU R;                  U5      nUR                  S5      UR                  S5      UR                  S5      UR                  S5      4n	UR                  UR                  S5      U	S   U	S   -  S	5      nUR                  U	S   U	S   -  U	S   S	5      n[        R<                  " UUR#                  SS5      5      nUR                  " U	6 nUR	                  SSSS5      R?                  5       nUR                  5       S S U R                  4-   nUR                  " U6 nU$ s  snf )N.r   rF   r   r   T)	is_causalr   r   r   g        ry   )betaalphar   z-infrf   ) r   r2   __version__ro   permuter   r	   
functionalscaled_dot_product_attentionrn   r   r   r   r   r}   r|   baddbmm	transposer   r   r   r   onesbooltril_masked_fillFsoftmaxtype_asr   bmmri   )r7   query_layer	key_layervalue_layerattention_maskpytorch_major_versionkcontext_layernew_context_layer_shapeoutput_sizematmul_input_buffermatmul_resultattention_scoresattention_probss                 r'   r\   CoreAttention.forward   s    #E$5$5$;$;C$@$C D A% &+>3>A 		!Q1%>3/KK %+*;*;+#//!,+- % 3 3 P PK4 !Q !I "-&4_N % 3 3 P PK!I)11!Q1=M&3&8&8&:3B&?..C2 '2#)113JKMx o '++A.0@0@0C&++A.	q0ACK &**;q>+6q>KN+JBPK "{1~'21~A'FLI #(++AQ/AA!''"))#+ "MM#%%a+##Aq)33Aq9T---M  -11;? --#3#9#9#; zz%#3jj#@ %*:*@*@+*003+4!&NNN+22**"& $$&"0)#3#?#?"E&M$3 ii(8bAO-55kBO #44_EO '++A.0@0@0C&++A.0@0@0CEK &**  #[^k!n%DbJK .22AQ/QEO "IIo&1&;&;Aq&ACM *..<M)11!Q1=HHJM&3&8&8&:3B&?..C2 '2#)..0GHMY3s   Q)	r   r   r   r   r   r   r   r   r   )	r:   r;   r<   r=   r   rH   r\   r@   r`   ra   s   @r'   r   r      s    L~ L2o or)   r   c                   T   ^  \ rS rSrSrSS\4U 4S jjjr  S	S jr  S
S jrSr	U =r
$ )SelfAttentionio  zParallel self-attention layer abstract class.

Self-attention layer takes input with size [s, b, h]
and returns output of the same size.
rD   c                 F  > [         [        U ]  5         [        SU5      U l        UR
                  UR                  -  U l        U R                  UR                  -  U l        UR                  U l	        UR                  U l
        SU R                  -  U l        U R                  (       a?  UR                  U l        U R                  SU R                  -  UR                  -  -   U l        [        R                  " UR                   U R                  4UR"                  =(       d    UR$                  US.['        U5      D6U l        [+        XR                  5      U l        [        R                  " U R                  UR                   4UR"                  US.['        U5      D6U l        g )Nr   r   rF   biasr|   )rG   r   rH   r   r   rK   r   r   r   r   multi_query_attentionqkv_hidden_sizerL   $num_multi_query_groups_per_partitionr	   rQ   rR   add_bias_linearadd_qkv_bias_config_to_kwargsquery_key_valuer   core_attentiondense)r7   rD   r   r|   rV   s       r'   rH   SelfAttention.__init__v  sw   mT+-<0%11F4N4NN /3.B.BfF`F`.`+171K1K.%+%A%A" 4#7#77%%8>8T8TD5$$q4+N+N'N..(/ /    "yy   ) ''>6+>+>	 )
  ' ) ,F4E4EF YY  ) ''	)
  ')
r)   c           	          U R                   (       a  U R                  nOU R                  n[        R                  " UUUU R
                  UUS9$ r   )r   r  r   r2   r   r   )r7   inference_max_sequence_len
batch_sizer|   r}   r   s         r'   _allocate_memorySelfAttention._allocate_memory  sN    
 %%"&"K"K"&"H"H{{&//
 	
r)   c                 6   U R                  U5      nU R                  (       Ga  UR                  U R                  U R                  -  U R
                  U R                  -  U R
                  U R                  -  /SS9u  pxn	UR                  UR                  5       S S U R                  U R                  4-   5      nUR                  UR                  5       S S U R
                  U R                  4-   5      nU	R                  U	R                  5       S S U R
                  U R                  4-   5      n	OMUR                  5       S S U R                  SU R                  -  4-   n
UR                  " U
6 n[        US5      u  pxn	Ub  [        Xs5      n[        X5      nUb0  Uu  p[        R                  " X4SS9n[        R                  " X4SS9n	U(       a  X4nOS nU R                  (       Ga  UR                  S5      nUR                  SSSU R                  U R
                  -  S5      nUR                  5       R                  UR                  5       S S U R                  U R                  4-   5      nU	R                  S5      n	U	R                  SSSU R                  U R
                  -  S5      n	U	R                  5       R                  U	R                  5       S S U R                  U R                  4-   5      n	U R                  XxX5      nU R!                  U5      nX4$ )Nr   rf   r   r   r   rF   )r  r   ro   r   r   r  r   rn   rt   r   r2   r   	unsqueezeexpandri   r  r  )r7   r   r   rotary_pos_embkv_cache	use_cachemixed_x_layerr   r   r   new_tensor_shapecache_kcache_vr   outputs                  r'   r\   SelfAttention.forward  sd     ,,];%%%4A4G4G::99:==99:==99:  5H 
51[[ &**;+;+;+=cr+B6633F5 ,5 6K "y~~'7'<9933@5 (5 6I &**;+;+;+=cr+B9933F5 ,5 6K  -113CR8 $ F F !D$G$G G I I *..0@AM 8qI[ %.{KK,YGI 'G		7"6A>I))W$:BK!/HH%%%!++B/I!((BDBB<<=>@BI ",,.33 !$(N(N(,(K(K(M MNI &//3K%,,BDBB<<=>@BK &00277  "2A&9966889K ++K,7I M*r)   )
r  r  r   r   r   r   r  r   r  r  r"   )NNNT)r:   r;   r<   r=   r^   r   rH   r  r\   r@   r`   ra   s   @r'   r   r   o  s;     )~  )  )J !%#	
, ^  ^ r)   r   c                 "    SU R                   0nU$ )Nr}   )torch_dtype)r%   common_kwargss     r'   r  r    s    !!M r)   c                   >   ^  \ rS rSrSrSS\4U 4S jjjrS rSrU =r	$ )MLPi  zMLP.

MLP will take the input with h hidden state, project it to 4*h
hidden dimension, perform nonlinear transformation, and project the
state back into h hidden dimension.
rD   c                   > [         [        U ]  5         UR                  U l        [
        R                  " UR                  UR                  S-  4U R                  US.[        U5      D6U l
        S nX0l        [
        R                  " UR                  UR                  4U R                  US.[        U5      D6U l        g )NrF   r   c                 l    [         R                  " U SSS9n [        R                  " U S   5      U S   -  $ )NrF   r   rf   r   r   )r2   rk   r   silu)r   s    r'   swigluMLP.__init__.<locals>.swiglu'  s/    Aqb)A66!A$<!A$&&r)   )rG   r   rH   r  add_biasr	   rQ   rR   ffn_hidden_sizer  dense_h_to_4hactivation_funcdense_4h_to_h)r7   rD   r|   r$  rV   s       r'   rH   MLP.__init__  s    c4!#..  YY""Q&) 	)
  ')	'  &  YY"") 	)
  ')r)   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ r"   )r(  r)  r*  )r7   r   intermediate_parallelr  s       r'   r\   MLP.forward5  s;     $ 2 2= A $ 4 45J K##$9:r)   )r)  r&  r*  r(  r"   
r:   r;   r<   r=   r^   r   rH   r\   r@   r`   ra   s   @r'   r   r     s#    )~ ) )6 r)   r   c                   F   ^  \ rS rSrSrSS\4U 4S jjjr  SS jrSrU =r	$ )	GLMBlocki>  zwA single transformer layer.

Transformer layer takes input with size [s, b, h] and returns an
output of the same size.
rD   c                   > [         [        U ]  5         X l        UR                  U l        UR
                  U l        UR                  (       a  [        O[        nU" UR                  UR                  UUR                  S9U l        [        XUS9U l        UR                  U l        U" UR                  UR                  UUR                  S9U l        [#        XS9U l        g )Nr   r|   r}   r{   )rG   r1  rH   r   (apply_residual_connection_post_layernormfp32_residual_connectionrmsnormr   r   rR   layernorm_epsilonr  input_layernormr   self_attentionhidden_dropoutpost_attention_layernormr   mlp)r7   rD   r   r|   LayerNormFuncrV   s        r'   rH   GLMBlock.__init__E  s    h&((8>8g8g5(.(G(G%#)>>y,(($$	 & ,1$33 )6(($$	)&% v-r)   c                    U R                  U5      nU R                  UUUUUS9u  ptU R                  (       a  UnOUn[        R                  R
                  R                  XpR                  U R                  S9n	X-   n	U R                  U	5      nU R                  U5      n
U R                  (       a  UnOU	n[        R                  R
                  R                  XR                  U R                  S9nX-   nX4$ )Nr  r  )ptraining)r8  r9  r4  r2   r	   r   dropoutr:  rB  r;  r<  )r7   r   r   r  r  r  layernorm_outputattention_outputresiduallayernorm_input
mlp_outputr  s               r'   r\   GLMBlock.forwardd  s     //>%)%8%8 &9 &!" 88'H$H((--55 3 3dmm 6 M"4  88I XX./
 88'H&H$$,,-- - G"r)   )r4  r5  r:  r8  r   r<  r;  r9  r"   r  r/  ra   s   @r'   r1  r1  >  s,    .~ . .H .  . r)   r1  c                   f   ^  \ rS rSrSrS
S\4U 4S jjjrS r   SS\\	   S\\	   4S jjr
S	rU =r$ )GLMTransformeri  zTransformer class.rD   c           	        >^^ [         [        U ]  5         TR                  U l        TR                  U l        TR
                  U l        UU4S jn[        R                  R                  [        U R
                  5       Vs/ s H  oC" US-   5      PM     sn5      U l
        U R                  (       aI  TR                  (       a  [        O[        nU" TR                  TR                  TTR                   S9U l        SU l        g s  snf )Nc                    > [        TU TS9$ )Nr{   )r1  )r   rD   r|   s    r'   build_layer,GLMTransformer.__init__.<locals>.build_layer  s    FL@@r)   r   r3  F)rG   rK  rH   r5  post_layer_normrJ   r2   r	   
ModuleListrangelayersr6  r   r   rR   r7  r  final_layernormgradient_checkpointing)r7   rD   r|   rN  ir=  rV   s    ``   r'   rH   GLMTransformer.__init__  s    nd,.(.(G(G%%55 !++	A hh))).t)?@)?A[Q)?@B '-~~G9M#0"",,((	$*D  ',# As    Dc                      U R                   U   $ r"   )rS  )r7   r   s     r'   
_get_layerGLMTransformer._get_layer  s    {{<((r)   r  output_hidden_statesc           	         U(       d%  [        U R                  5       Vs/ s H  nS PM     nnU(       a  SOS nU R                  (       a/  U R                  (       a  U(       a  [        R                  S5        SnS n	U(       a  SOS n
[        U R                  5       H  nU(       a  X4-   n
U R                  U5      nU R                  (       a@  U R                  (       a/  [        R                  R                  R                  XX#XK   U5      nOU" UUUXK   US9nUu  pU(       d  M  X4-   nM     U(       a  X4-   n
U R                  (       a  U R                  U5      nXX4$ s  snf )Nr#   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr@  )rR  rJ   rU  rB  r   warning_oncerY  r2   utils
checkpointrP  rT  )r7   r   r   r  	kv_cachesr  r[  r   presentsall_self_attentionsall_hidden_statesindexlayer	layer_retr  s                  r'   r\   GLMTransformer.forward  sD    ',T__'=>'=!'=I>"2&&4==##p "	""6BD4??+E#$58I$I!OOE*E**t}}!KK22==.$i1	 "!""&-')	 '0#My#l2% ,(   14E E  00?M(9NNM ?s   E)rT  r5  rU  rS  rJ   rP  r"   )NTF)r:   r;   r<   r=   r^   r   rH   rY  r   r   r\   r@   r`   ra   s   @r'   rK  rK    sQ    ,~ , ,6) $(/40O D>0O 'tn0O 0Or)   rK  c                      ^  \ rS rSrSrSrSr\rSr	S/r
U 4S jrS\R                  4S	 jrSS
 jrS rSS jr\U 4S j5       rSrU =r$ )ChatGLMPreTrainedModeli  zz
An abstract class to handle weights initialization and
a simple interface for downloading and loading pretrained models.
FTtransformerr1  c                 b   > [         TU ]  " UR                  40 UD6  [         [        U ]  U5        g r"   )rG   rH   name_or_pathr   )r7   rD   r&   rV   s      r'   rH   ChatGLMPreTrainedModel.__init__  s*    ,,77eT#F+r)   modulec                     g)zInitialize the weights.Nr#   )r7   rn  s     r'   _init_weights$ChatGLMPreTrainedModel._init_weights  s    r)   c           	         UR                   u  pE[        R                  " XEXQR                  S9nUR	                  5         SnU(       a  US   S   R                   S   nU(       a6  [        R
                  " [        R                  " UUUUR                  S9U4SS9nUb  XcR                  S5      -  nU(       d  Ub  XcR                  S5      S-
  -  nUS:  R                  5       nUR                  S5        U$ )Nr{   r   r   rf   r   g      ?)	r   r2   r   r|   r   r   r  r   
unsqueeze_)r7   r-   r[   padding_maskr  
seq_lengthfull_attention_maskpast_lengths           r'   	get_masks ChatGLMPreTrainedModel.get_masks  s    !*
#jjJ7G7GI!!#)!,Q/55a8K"'))JJ""#(//	1
 () # #"58N8N9 #|7#9#9"#=#AA2S8>>@&&q)""r)   c                     UR                   u  p4[        R                  " U[        R                  US9R	                  S5      R                  US5      nU$ )Nr   r   r   )r   r2   r   longr  repeat)r7   r-   r|   r  ru  position_idss         r'   get_position_ids'ChatGLMPreTrainedModel.get_position_ids  sG    !*
||ejj$9Q<z1(= 	 r)   c                 <    [        U[        5      (       a  X!l        g g r"   )
isinstancerK  rU  )r7   rn  values      r'   _set_gradient_checkpointing2ChatGLMPreTrainedModel._set_gradient_checkpointing  s    fn--,1) .r)   c                    > UR                  SS5      nUR                  SS5        [        [        U ]  " SSU0UD6nX#l        U$ )zInstantiate the model.

Args:
    kwargs: Input args.
            model_dir: The model dir used to load the checkpoint and the label information.

Returns:
    The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained
	model_dirNcfgpretrained_model_name_or_pathr#   )poprG   r   from_pretrainedr  )r$   r&   r  modelrV   s       r'   _instantiate#ChatGLMPreTrainedModel._instantiate#  sN     JJ{D1	

5$eS1 ?*3?7=?#r)   r#   r"   F)r:   r;   r<   r=   r^   is_parallelizablesupports_gradient_checkpointingr   config_classbase_model_prefix_no_split_modulesrH   r	   Modulerp  rx  r~  r  classmethodr  r@   r`   ra   s   @r'   ri  ri    sa    
 &*#!L%#,BII #62  r)   ri  c                   >   ^  \ rS rSrSrSS\4U 4S jjjrS rSrU =r	$ )rM   i7  zLanguage model embeddings.rD   c                    > [         [        U ]  5         UR                  U l        [        R                  " UR
                  U R                  UR                  US9U l        UR                  U l        g r   )	rG   rM   rH   rR   r	   padded_vocab_sizer  word_embeddingsr5  )r7   rD   r|   rV   s      r'   rH   Embedding.__init__:  s]    i')!--!||$$$$	 
 )/(G(G%r)   c                     U R                  U5      nUnUR                  SS5      R                  5       nU R                  (       a  UR	                  5       nU$ )Nr   r   )r  r   ri   r5  r   )r7   r-   words_embeddings
embeddingss       r'   r\   Embedding.forwardF  sP    //	:%
))!Q/::<
((#))+Jr)   )r5  rR   r  r"   r/  ra   s   @r'   rM   rM   7  s$    $
H~ 
H 
H	 	r)   rM   c                   b  ^  \ rS rSrSS\4U 4S jjjrS r\R                  4S jr	        SS\
\R                     S\
\R                     S\
\R                     S	\
\\\R                  \R                  4   S
4      S\
\R                     S\
\   S\
\   S\
\   4S jjrS\4S jrSrU =r$ )ChatGLMModeliR  rD   c                   > [         TU ]  U5        U(       a  [        nO[        n0 nUb  X%S'   U" [        U40 UD6U l        UR                  U l        UR                  U l        UR                  U l        UR                  U l	        UR                  c  UR                  UR                  -  OUR                  n[        US-  UR                  UR                  UUR                  S9U l        U" ["        U40 UD6U l        U" [&        R(                  UR                  UR*                  4SUR                  S.UD6U l        UR.                  U l        UR0                  U l        U R.                  b  U R3                  5        H
  nSUl        M     [6        R8                  " U R.                  5      R;                  5       U l        [?        U5      U l         [6        R&                  RC                  S5      U l"        g g )Nr|   rF   )r   r   r|   r}   F)r   r}   g?)#rG   rH   r   r(   rM   rO   rJ   rL   rK   ru  rR   r   rv   r   original_roper  r  rK  encoderr	   rQ   r  output_layerrN   rI   
parametersrequires_gradr2   r   r{  rZ   rB   prefix_encoderr   rC  )	r7   rD   r|   
empty_initinit_methodinit_kwargs
rotary_dimparamrV   s	           r'   rH   ChatGLMModel.__init__T  s    #K&K$*!$YF+F ++%+%A%A"!-- !++ !!) &"<"<</5/A/A 	 .!O(( ..$$& #>6I[I'II$$ $$  "--!'!9!9'*&+# +!&d.>.>!?!D!D!FD"/"7D 88++C0DL (r)   c                 .    U R                   R                  $ r"   )rO   r  )r7   s    r'   get_input_embeddings!ChatGLMModel.get_input_embeddings  s    ~~---r)   c                    U R                   R                  S5      R                  US5      R                  U5      nU R	                  U5      R                  U5      nUR                  XR                  U R                  S-  U R                  U R                  5      nU R                  U5      nUR                  / SQ5      R                  S5      nU$ )Nr   r   rF   )rF   r   r   r      )rZ   r  r  r   r  typer   rN   rJ   rL   rK   rC  r   ro   )r7   r  r|   r}   rZ   r[   s         r'   
get_promptChatGLMModel.get_prompt  s    **44Q7>>z?ACCE2f: 	--m<AA%H)..z;K;K/3/B/3/I/I/3/?/?A
 ,,7)11/BHHKr)   r}  r   rv  r[   .inputs_embedsr  r[  return_dictc
           	      B   Ub  UOU R                   R                  nUb  UOU R                   R                  nU	b  U	OU R                   R                  n	UR                  u  pUc  U R                  U5      nU R                  b\  Uc%  U R                  U
UR                  UR                  S9nUb1  [        R                  " UR                  XR                  45      U/SS9nUc5  Ub  UR                  5       (       a  U(       a  US:w  a  U R                  XUS9nU R                  U R                   5      nUb  X   nO	US S U24   nUR#                  SS5      R%                  5       nU R'                  UUUUUUS9u  pnnU	(       d  [)        S XUU4 5       5      $ [+        UUUUS	9$ )
N)r  r|   r}   r   rf   r   )rt  r   )r  r`  r  r[  c              3   0   #    U  H  nUc  M  Uv   M     g 7fr"   r#   )rj   vs     r'   rl   'ChatGLMModel.forward.<locals>.<genexpr>  s         %q  %s   	)last_hidden_stater[   r   
attentions)rD   r[  r  use_return_dictr   rO   rN   r  r|   r}   r2   r   new_onesallrx  r  ru  r   ri   r  rp   r   )r7   r-   r}  r   rv  r[   r  r  r[  r  r  ru  r  r   ra  rc  rb  s                    r'   r\   ChatGLMModel.forward  s    %9$D KK,, 	 "+!6IDKK<Q<Q	%0%<k$++B]B]!*
  NN95M'&"&//)$++'-- #2 #/ )!&&//')9)9:<&
 " &**..00o:D/&*nn^ '5 'M# ,,T__=#+9N+D+:+,=>N'11!Q7BBD KO,,)%!5 KW K7G!24G   ):<O%       '+$+*	
 	
r)   weight_bit_widthc                 8    SSK Jn  U" U R                  U5        U $ )Nr   quantize)quantizationr  r  )r7   r  r  s      r'   r  ChatGLMModel.quantize  s    */0r)   )rC  rO   r  rK   rL   rJ   r  rN   r  rI   rZ   r  ru  r  )NNNNNNNN)r:   r;   r<   r=   r   rH   r  r2   r   r  r   r_   
BoolTensorr   r   r\   r   r  r@   r`   ra   s   @r'   r  r  R  s   )1~ )1 )1V. 49::   0459:>0404$(/3&*I
 u||,I
 !!1!12	I

 &e&6&67I
 "%ellELL.H(I(+), #- .I
  -I
 D>I
 'tnI
 d^I
V  r)   r  )module_namec                     ^  \ rS rSrS.S\4U 4S jjjr  S/S\S\\\	4   S\
S\
S\\\	4   4
S	 jjr    S0S
\R                  S\\R                     S\\R                     S\\R                     S\
S\4S jjr           S1S
\\R                     S\\R                     S\\R                     S\\\R&                        S\\R                     S\\R                     S\\
   S\\
   S\\
   S\\
   S\\
   4S jjr\S\\\R                  \R                  4   S4   S\R                  S\\\R                  \R                  4   S4   4S j5       rS r S2S\S\\\\4      4S jjr S2S\S\\\\4      4S  jjr\R6                  " 5              S3S\S\\\\4      S!\4S" jj5       r\R6                  " 5               S4S\S\\\\4      S!\4S# jj5       r\R6                  " 5            S5S$\\   S%\\    S&\\!   S'\\"\\R                  /\\   4      4S( jj5       r#S6S)\4S* jjr$S+\S\4S, jr%S-r&U =r'$ )7 ChatGLM2ForConditionalGenerationi  rD   c                    > [         TU ]  U5        UR                  U l        [	        XUS9U l        Xl        SU l        U R                  R                  (       a%  U R                  U R                  R                  SS9  g g )Nr  r|   FT)r  )
rG   rH   
max_lengthmax_sequence_lengthr  rj  rD   	quantizedquantization_bitr  )r7   rD   r  r|   rV   s       r'   rH   )ChatGLM2ForConditionalGeneration.__init__  sh     #)#4#4 '&:;;''MM$++664MH (r)   outputsmodel_kwargsis_encoder_decoderstandardize_cache_formatr/   c                 :   U R                  XS9US'   SU;   a=  US   n[        R                  " UUR                  UR                  S   S45      /SS9US'   SU;   a:  US   nUS	SS 24   R                  5       nUS-  n[        R                  " Xg/SS9US'   S
US'   U$ )N)r  r[   r   r   r   r   rf   r}  .Fis_first_forward)_extract_past_from_model_outputr2   r   r  r   clone)r7   r  r  r  r  r   r}  new_position_ids           r'   #_update_model_kwargs_for_generationDChatGLM2ForConditionalGeneration._update_model_kwargs_for_generation  s     +/*N*N +O +H&' |+)*:;N-2YY""++'--a0!46
 .L)* \)'7L*384::<Oq O+099/R,9L( ,1'(r)   r-   r[   r   r}  r  c                     Uc  U R                  XR                  S9nU(       d  USSS 24   nUS S 2SS 24   nUUUUSS.$ )Nr{   .r   T)r-   r[   r}  r   return_last_logit)r~  r|   )r7   r-   r[   r   r}  r  r&   s          r'   prepare_inputs_for_generation>ChatGLM2ForConditionalGeneration.prepare_inputs_for_generation  sd     00"2"2 1 4L'RS1L!!RS&)I".(,!%
 	
r)   r  labelsr  output_attentionsr[  r  r  c                    Ub  UOU R                   R                  nU
b  U
OU R                   R                  n
U R                  UUUUUUU	U
S9nUS   nU(       a  USS  nU R                  R	                  U5      nUR                  SS5      R                  5       nS nUb  UR                  [        R                  5      nUSS S2S S 24   R                  5       nUSSS 24   R                  5       n[        SS9nUR                  UR                  5      nU" UR                  SUR                  S5      5      UR                  S5      5      nUR                  UR                  5      nUR                  UR                  5      nU
(       d  U4USS  -   nUb  U4U-   $ U$ [        UUUR                   UR"                  UR$                  S9$ )	N)r-   r}  r   r[   r  r  r[  r  r   r   r   .i)ignore_index)losslogitsr[   r   r  )rD   r  r  rj  r  r   ri   r   r2   r   r
   r|   r   rn   r}   r   r[   r   r  )r7   r-   r}  r   r[   r  r  r  r  r[  r  r  transformer_outputsr   	lm_logitsr  shift_logitsshift_labelsloss_fctr  s                       r'   r\   (ChatGLM2ForConditionalGeneration.forward'  s    "+!6IDKK<Q<Q	%0%<k$++B]B]"..%)+'!5# / 	
 ,A.)"#.M$$11-@	''1-88:	!U]]3I %S#2#q[1<<>L!#qr'?557L'T:H'??<+>+>?L!!"l&7&7&;<!!"%'D "]%8%89I77=../D]%8%<<F*.*:THv%FF%/??-;;*55
 	
r)   past.beam_idxc                 .   ^ [        U4S jU  5       5      $ )a$  
This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
[`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
beam_idx at every generation step.

Output shares the same memory storage as `past`.
c           	   3      >#    U  Hg  nUS    R                  STR                  US    R                  5      5      US   R                  STR                  US   R                  5      5      4v   Mi     g7f)r   r   N)index_selectr   r|   )rj   
layer_pastr  s     r'   rl   BChatGLM2ForConditionalGeneration._reorder_cache.<locals>.<genexpr>r  sl      " !j qM&&q(++jm6J6J*KLqM&&q(++jm6J6J*KL
 !s   A/A2)rp   )r  r  s    `r'   _reorder_cache/ChatGLM2ForConditionalGeneration._reorder_cachef  s!      " !" " 	"r)   c                 J    UR                  5       nUR                  SS5      nU$ )Nu   [[训练时间]]u   2023年)stripreplace)r7   responses     r'   process_response1ChatGLM2ForConditionalGeneration.process_responsew  s&    >>###$6	Br)   queryhistoryc                 j    UR                  X#S9nU" U/SS9nUR                  U R                  5      nU$ )Nr  ptreturn_tensors)build_promptr   r|   )r7   	tokenizerr  r  promptinputss         r'   build_inputs-ChatGLM2ForConditionalGeneration.build_inputs|  s<     '''?F8D94;;'r)   c                 &   U(       aH  SR                  [        U5      S-   U5      nUR                  USS9nUSS  nUR                  US 4/SSS9nO&SR                  [        U5      S-   U5      nU" U/SS9nUR	                  U R
                  5      nU$ )	Nu   

[Round {}]

问：{}

答：r   F)add_special_tokensr  )r  r  u   [Round {}]

问：{}

答：r  )formatlenencodebatch_encode_plusr   r|   )r7   r  r  r  r  r-   r  s          r'   build_stream_inputs4ChatGLM2ForConditionalGeneration.build_stream_inputs  s     ;BBGq %)F!((E(JI!!"I009d2C1D@DDI 1 KF 8>>s7|a?OQVWFx=F4;;'r)   r  c
                 v   Uc  / nU	c
  [        5       n	U	R                  [        5       5        Uc  U R                  nUUUUUU	S.U
EnU R	                  XUS9nU R
                  " S0 UDUD6nUR                  5       S   [        US   S   5      S  nUR                  U5      nU R                  U5      nX2U4/-   nX4$ )N)r  	num_beams	do_sampletop_ptemperaturelogits_processorr   r   r-   r#   )
r   appendr+   ru  r  generatetolistr  decoder  )r7   r  r  r  r  r  r  r  r  r  r&   
gen_kwargsr  r  r  s                  r'   _chat&ChatGLM2ForConditionalGeneration._chat  s     ?G#24 ; =>J$""& 0
 

 ""9W"E--7&7J7.."1%c&*=a*@&A&BC##G,((2X.//  r)   c              +   X  #    Uc  / nU	c
  [        5       n	U	R                  [        5       5        Uc  U R                  nUUUUU	S.UEnUc  U
(       d  U R	                  XUS9nOU R                  XUS9nUb  US   S   R                  S   nU R                  R                  b  XR                  R                  -  nU=R                  U-  sl	        UR                  n[        R                  " UR                  SU5      U4SS9nXS'   U R                  " S0 UDUU
S.DUD6 H  nU
(       a  Uu  nnUR                  5       S   [!        US   S   5      S  nUR#                  U5      nU(       d  MN  US	   S
:w  d  MY  U R%                  U5      nX2U4/-   nU
(       a	  UUU4v   M  UU4v   M     g 7f)N)r  r  r  r  r  r   r   r   rf   r   )r[   return_past_key_valuesr-   r   u   �r#   )r   r  r+   ru  r  r  r   rj  rN   r}  r   r2   r   r  stream_generater  r  r  r  )r7   r  r  r  r[   r  r  r  r  r  r   r&   r  r  rw  r   r  r  new_historys                      r'   stream_chat,ChatGLM2ForConditionalGeneration.stream_chat  s     ?G#24 ; =>J$"& 0
 

 "+A&&y&IF--' . 3F&)!,Q/55a8K++7//;;;;.#22N"YY((K8.IN (6#$++  /'= 	G
 &+2(nn&q)#f[.A!.D*E*FGG ''0HxHRLE100:%):(;;)"K@@"K//s   E&F*,F*73F*generation_configr  stopping_criteriaprefix_allowed_tokens_fnc           	   +     ^#    UR                   S   UR                   S   pUc  U R                  n[        R                  " U5      nUR                  " S0 UD6n
UR
                  UR                  p[        U[        5      (       a  U/nUR                  S5      S L =(       a    UR                  S LnU(       a7  UR                  c*  [        R                  " SUR                   S3[        5        O]UR                  bP  UR                  U	-   Ul
        U(       d5  [        R                  SUR                   SUR                   S3[        5        XR                  :  aH  U R                   R"                  (       a  S	OS
n[        R%                  SU SU	 SUR                   S35        Ub  UO	['        5       nUb  UO	[)        5       nU R+                  UU	UUUS9nU R-                  UUS9nU R/                  U5      nUR1                  UR                   S   5      R3                  S5      nS n U R4                  " U40 U
D6nU " S0 UDSSSS.D6nUR6                  S S 2SS S 24   nU" UU5      nU" UU5      n[8        R:                  R=                  USS9nUR>                  (       a%  [@        RB                  " USS9RE                  S5      mO[@        RF                  " USS9m[@        RH                  " UTS S 2S 4   /SS9nU RK                  UU
U R                   R"                  S9n
URM                  [O        U4S jU 5       5      RQ                  5       5      nU(       a  UURR                  4v   OUv   URU                  5       S:X  d  U" UU5      (       a  g GMi  7f)Nr   r   r  zUsing `max_length`'s default (z) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.zBoth `max_new_tokens` (=z) and `max_length`(=z) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)decoder_input_idsr-   zInput length of z is z, but `max_length` is set to zX. This can lead to unexpected behavior. You should consider increasing `max_new_tokens`.)r%  input_ids_seq_lengthencoder_input_idsr'  r  )r%  r&  r   TF)r  r  r[  rf   )num_samples)r  c              3   .   >#    U  H
  nTU:g  v   M     g 7fr"   r#   )rj   rV  next_tokenss     r'   rl   CChatGLM2ForConditionalGeneration.stream_generate.<locals>.<genexpr>^  s     <|![A%|s   r#   )+r   r%  copydeepcopyupdatebos_token_ideos_token_idr  r   getr  max_new_tokenswarningswarnUserWarningr   rD   r  warningr   r   _get_logits_processor_get_stopping_criteria_get_logits_warpernewfill_r  r  r	   r   r   r  r2   multinomialsqueezeargmaxr   r  mulsumr{  r[   r   )r7   r-   r%  r  r&  r'  r   r&   r   r*  r  r4  has_default_max_lengthinput_ids_stringlogits_warperunfinished_sequencesr.   model_inputsr  next_token_logitsnext_token_scoresprobsr.  s                         @r'   r!  0ChatGLM2ForConditionalGeneration.stream_generate  s     #,//!"4ioob6I$ $ 6 6 MM*;<(//9&9+88:K:X:X<lC(((>L!'"!"" "O&7&B&B$&N 	!&7&F&F&NMM01B1M1M0N Oe e 	 --9+<+K+KNb+b()./@/O/O.PPd(334 5ff    #?#??6:kk6T6T2ZeNN"#3"4D9M8N O%001 2001 0@/K+Qd R
1B1N-Th U
  55/!5'%=- 6 
 !77// 8 1 //0AB(}}Y__Q-?@FFqI==+)+L   "'%*	G !(q"ax 8 !1<M N -i9J K MM))*;)DE **#//q**1'!*  $ll5b9 		9k!T'.B"CLICC#';;#A#A D CL $8#;#;<|<<BBD$F %!8!888#'')Q.2Cv3' 3'Q s   N	Nbitsc                     US:X  a  g SSK Jn  U R                  (       a  [        R	                  S5        U $ SU l        XR
                  l        U" U R                  R                  U4UUS.UD6U R                  l        U $ )Nr   r   r  zAlready quantized.Tr  )	r  r  r  r   inforD   r  rj  r  )r7   rN  r  r|   r&   r  s         r'   r  )ChatGLM2ForConditionalGeneration.quantizeh  s|    19*>>KK,-K'+$#+$$$ "	$
 $  r)   inputc           
      R   US   nUS   nSU;   a  US   nOSnSU;   a  US   nOSnSU;   a  US   nOSnS	U;   a  US	   nOS
n[        U5      [        R                  :X  a  UR                  5       nU R	                  UUUUUUUS9u  p[
        R                  U	[
        R                  U0$ )Ntextr  r  i   r  gffffff?r  r   r  T)r  r  r  r  )r  r2   r_   r  r  r   RESPONSEHISTORY)
r7   rR  r  rT  r  r  r  r  r  r  s
             r'   chat%ChatGLM2ForConditionalGeneration.chat~  s    V}	"5 |,JJE!.KK%k*II%k*II=ELL(nn&G JJ!# ' ! ##Xz/A/A7KKr)   )rD   r  r  rj  )TN)FF)NNNT)NNNNNNNNNNFr"   )NNr   T皙?rY  N)NNNTrY  rY  NF)NNNNF)FN)(r:   r;   r<   r=   r   rH   r   r   strr   r   r  r2   r>   r   r_   dictr  r   r?   r\   staticmethodr  r  r   r  r  no_gradr   r  r#  r   r   r   r   r!  r  rW  r@   r`   ra   s   @r'   r  r    s   
I~ 
I 
I  $)). 38n !	
 #' 
c3hH 7;5937%)
''
 &ell3
 %U\\2	

 #5<<0
 #
 
2 -1/315>B04)-$(,0/3&*,1=
ELL)=
 u||,=
 !.	=

 "%(9(9":;=
  -=
 &=
 D>=
 $D>=
 'tn=
 d^=
 $D>=
~ "E%,,45 "$)$4$4" 
uU\\5<</0#5	6" "  7; #5c?3 >B#& &*%S/%:$ ]]_ 04 $#!!!! E#s(O,!! 	!! !!F ]]_ 6:$(&*" #%)+09090 "%S/290
 !$90 90v ]]_ 9=:><@BF$n $$45n ##67	n
 $$89n #+8S%,,4G48I5> ,? #@n n`S ,"L$ "Ld "L "Lr)   r  r  )Wr^   r0  r   sysr7  typingr   r   r   r   r   r   r2   torch.nn.functionalr	   r   r   torch.utils.checkpointtorch.nnr
   r   torch.nn.utilsr   &transformers.generation.logits_processr   transformers.generation.utilsr   r   r   r   transformers.modeling_outputsr   r   transformers.modeling_utilsr   
modelscoper   r   modelscope.metainfor   modelscope.outputsr   modelscope.utilsr   loggingmodelscope.utils.constantr    r   configurationr   platform_C_jit_set_profiling_mode_jit_set_profiling_executor_jit_override_can_fuse_on_cpu_jit_override_can_fuse_on_gpu
get_logger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOC(CHATGLM_6B_PRETRAINED_MODEL_ARCHIVE_LISTr(   r+   r  rB   r_   r   r   rt   rv   jitscriptr   r   r   r   r  r   r1  rK  ri  rM   r  register_modulerW  chatglm2_6br  r#   r)   r'   <module>r~     s      
  = =      0 $ BA AC 7 ( & ) . +  ) <<8	HH$$U+	HH((/	HH**40	HH**40				) # , ( / EHHOO D %*LL " 
%,,	86)bii 6)r /ELL /%*\\/6;ll/ /.=ehhoo =,JEHHOO JZY EHHOO Y x)%((// )XT uxx T nQOUXX__ QOhKZ K\ 6K) K\ 

0B0BCL'= L DLr)   