
    9i                        S SK r S SKrS SKJs  Jr  S r " S S\R                  R                  5      r " S S\R                  R                  5      r	 " S S\R                  R                  5      r
 " S	 S
\R                  R                  5      r " S S\R                  R                  5      r " S S\R                  R                  5      r " S S\R                  R                  5      r " S S\R                  R                  5      r " S S\R                  R                  5      r " S S\R                  R                  5      rg)    Nc                     U S[         R                  " S[         R                  " U 5      -  5      -   -  [         R                  " SU [         R                  " U 5      -
  -  5      -  $ )z%Mindspore's fast gelu implementation.   gZd;gZd;?)torchexpabs)xs    g/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/models/nlp/codegeex/codegeex.py	fast_gelur
      sO    EIIfuyy|3445		UYYq\!"9$ $ $    c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )MLP   zMLP.

MLP will take the input with h hidden state, project it to 4*h
hidden dimension, perform nonlinear transformation, and project the
state back into h hidden dimension. At the end, dropout is also
applied.
c                 <  > [         [        U ]  5         Xl        [        R
                  R                  U R                  SU R                  -  5      U l        [        U l	        [        R
                  R                  SU R                  -  U R                  5      U l
        g )N   )superr   __init__hidden_sizer   nnLineardense_h_to_4hr
   activation_funcdense_4h_to_h)selfr   	__class__s     r	   r   MLP.__init__   s|     	c4!#&"XX__   

  ) #XX__   
r   c                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ N)r   r   r   )r   hidden_statesintermediate_paralleloutputs       r	   forwardMLP.forward+   s;     $ 2 2= A $ 4 45J K##$9:r   )r   r   r   r   	__name__
__module____qualname____firstlineno____doc__r   r!   __static_attributes____classcell__r   s   @r	   r   r      s    
( r   r   c                   F   ^  \ rS rSrSr  SU 4S jjr    SS jrSrU =r$ )SelfAttention5   zself-attention layer abstract class.

Self-attention layer takes input with size [b, s, h]
and returns output of the same size.
c                 ^  > [         [        U ]  5         Xl        X l        X@l        XPl        [        SU5      U l        U R                  U R                  -  S:X  d   e[        U R                  U R                  -  5      U l
        [        R                  R                  U R                  U R                  5      U l        [        R                  R                  U R                  U R                  5      U l        [        R                  R                  U R                  U R                  5      U l        ["        R$                  " U R                  5      U l        [        R                  R)                  SS9U l        [        R                  R                  U R                  U R                  5      U l        g Nr   r   dim)r   r-   r   r   num_attention_headsfp16attention_softmax_in_fp32maxlayer_numberinthidden_size_per_attention_headr   r   r   querykeyvaluemathsqrtnorm_factorSoftmaxsoftmaxdenser   r   r4   r8   r5   r6   r   s         r	   r   SelfAttention.__init__<   s6    	mT+-&#6 	)B&<0$":"::a???.1$2B2B595M5M3N /O+ XX__T%5%5t7G7GH
88??4#3#3T5E5EFXX__T%5%5t7G7GH
99T%H%HIxx''B'/XX__T%5%5t7G7GH
r   c                 d	   U R                  U5      nU R                  U5      nU R                  U5      n	UR                  5       S S U R                  U R
                  4-   n
UR                  " U
6 nUR                  5       S S U R                  U R
                  4-   n
UR                  " U
6 nU	R                  5       S S U R                  U R
                  4-   n
U	R                  " U
6 n	UbP  Uu  p[        R                  " UR                  U5      U4SS9n[        R                  " UR                  U	5      U	4SS9n	U(       a  X4nUR                  S5      UR                  S5      UR                  S5      UR                  S5      4nUR                  5       R                  US   US   US   -  S5      nUR                  5       R                  US   US   US   -  S5      n[        R                  " UR                  SS5      UR                  SS5      R                  SS5      5      U R                  -  nUR                  " U6 nU(       a  [        R                  " 5          Ub;  USUR                  S5      S-
  S UR                  S5      24   R                  S5      nO*USS UR                  S5      2S UR                  S5      24   nS S S 5        Ub'  [        R                   " U5      nSUS S 2S S 2US 2S S 24'   UUS	-  -
  nU R"                  (       a.  U R%                  UR'                  5       5      R)                  5       nOU R%                  U5      nU	R                  S5      U	R                  S5      UR                  S5      U	R                  S5      4nU	R                  U	R                  S5      US   US   -  S5      n	UR                  US   US   -  US   S5      n[        R*                  " UU	R                  S5      R                  SS5      R-                  S5      5      nUR                  " U6 nUR/                  SSSS5      R                  5       nUR                  5       S S
 U R0                  4-   nUR                  " U6 nU R3                  U5      nU(       a  UW/nU$ ! , (       d  f       GN= fNr1   r   r2   r         .Tg     @r;   r<   r=   sizer4   r:   viewr   cattype_as
contiguousmatmul	transposer@   no_grad	unsqueezecloner6   rB   floathalfbmmsqueezepermuter   rC   )r   r   attention_mask
layer_pastget_key_valueprompt_lengthcontext_lengthquery_layer	key_layervalue_layernew_query_layer_shapepast_key
past_valuepresentoutput_sizematmul_resultattention_scoresattention_probscontext_layernew_context_layer_shaper    s                        r	   r!   SelfAttention.forwardX   s    jj/HH]+	jj/ + 0 0 23B 7$$d&I&I;
 !
 "&&(=> ) 0" 5$$d&I&I9K !KNN$9:	 + 0 0 23B 7$$d&I&I;
 !
 "&&(=> !#- H		8#3#3I#>	"J&')I))##K0+>AGK .G #''*K,<,<Q,?"''*INN1,=? ",,.33NKN[^;RA((*//NKN[^;RA	 !!!Q'1%//1579=9I9IJ
 )--{; )%3(--a06.33A667&8 9B	! #
 &46.33A66 9 9IQ9   &!N ! %"[[8N7;N1a!34 ,nw.FF))"ll+;+A+A+CDIIKO"ll+;<O #''*K,<,<Q,?"''*K,<,<Q,?A "&&QQ+a.!@"F *..{1~A/N/:1~rC 		!!!$..q!4<<Q?A
 &**K8 &--aAq9DDF #0"4"4"6s";?  # %**,CD M*g&FE !   A)R  
R/r6   rC   r5   r   r:   r<   r8   r@   r4   r;   rB   r=   TTNFNNr#   r+   s   @r	   r-   r-   5   s1     "&I@ K Kr   r-   c                   F   ^  \ rS rSrSr  SU 4S jjr    SS jrSrU =r$ )TopQuerySelfAttention   zTop query self-attention layer abstract class.

Self-attention layer takes input with size [b, s, h]
and returns output of the same size.
c                 ^  > [         [        U ]  5         Xl        X l        X@l        XPl        [        SU5      U l        U R                  U R                  -  S:X  d   e[        U R                  U R                  -  5      U l
        [        R                  R                  U R                  U R                  5      U l        [        R                  R                  U R                  U R                  5      U l        [        R                  R                  U R                  U R                  5      U l        ["        R$                  " U R                  5      U l        [        R                  R)                  SS9U l        [        R                  R                  U R                  U R                  5      U l        g r0   )r   rs   r   r   r4   r5   r6   r7   r8   r9   r:   r   r   r   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   s         r	   r   TopQuerySelfAttention.__init__   s7    	#T35&#6 	)B&<0$":"::a???.1$2B2B595M5M3N /O+ XX__T%5%5t7G7GH
88??4#3#3T5E5EFXX__T%5%5t7G7GH
99T%H%HIxx''B'/XX__T%5%5t7G7GH
r   c                 d	   U R                  U5      nU R                  U5      n	U R                  U5      n
UR                  5       S S U R                  U R
                  4-   nUR                  " U6 nU	R                  5       S S U R                  U R
                  4-   nU	R                  " U6 n	U
R                  5       S S U R                  U R
                  4-   nU
R                  " U6 n
UbP  Uu  p[        R                  " UR                  U	5      U	4SS9n	[        R                  " UR                  U
5      U
4SS9n
U(       a  X4nUR                  S5      UR                  S5      UR                  S5      U	R                  S5      4nUR                  5       R                  US   US   US   -  S5      nU	R                  5       R                  US   US   US   -  S5      n	[        R                  " UR                  SS5      U	R                  SS5      R                  SS5      5      U R                  -  nUR                  " U6 nU(       a  [        R                  " 5          Ub;  USUR                  S5      S-
  S UR                  S5      24   R                  S5      nO*USS UR                  S5      2S UR                  S5      24   nS S S 5        Ub'  [        R                   " U5      nSUS S 2S S 2US 2S S 24'   UUS	-  -
  nU R"                  (       a.  U R%                  UR'                  5       5      R)                  5       nOU R%                  U5      nU
R                  S5      U
R                  S5      UR                  S5      U
R                  S5      4nU
R                  U
R                  S5      US   US   -  S5      n
UR                  US   US   -  US   S5      n[        R*                  " UU
R                  S5      R                  SS5      R-                  S5      5      nUR                  " U6 nUR/                  SSSS5      R                  5       nUR                  5       S S
 U R0                  4-   nUR                  " U6 nU R3                  U5      nU(       a  UW/nU$ ! , (       d  f       GN= frG   rK   )r   r   query_hidden_stater[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   r    s                         r	   r!   TopQuerySelfAttention.forward	  s    jj!34HH]+	jj/ + 0 0 23B 7$$d&I&I;
 !
 "&&(=> ) 0" 5$$d&I&I9K !KNN$9:	 + 0 0 23B 7$$d&I&I;
 !
 "&&(=> !#- H		8#3#3I#>	"J&')I))##K0+>AGK .G #''*K,<,<Q,?"''*INN1,=? ",,.33NKN[^;RA((*//NKN[^;RA	 !!!Q'1%//1579=9I9IJ
 )--{; )%3(--a06.33A667&8 9B	! #
 &46.33A66 9 9IQ9   &!N ! %"[[8N7;N1a!34 ,nw.FF))"ll+;+A+A+CDIIKO"ll+;<O #''*K,<,<Q,?"''*K,<,<Q,?A "&&QQ+a.!@"F *..{1~A/N/:1~rC 		!!!$..q!4<<Q?A
 &**K8 &--aAq9DDF #0"4"4"6s";#'#3#3"5#6%**,CD M*g&FG !rn   ro   rp   rq   r#   r+   s   @r	   rs   rs      s1     "&IB I Ir   rs   c                   H   ^  \ rS rSrSr   SU 4S jjr    SS jrSrU =r$ )TransformerLayeri  zxA single transformer layer.

Transformore layer takes input with size [b, s, h] and returns an
output of the same size.
c                 l  > [         [        U ]  5         Xl        X@l        X0l        [        R                  R                  XR                  S9U l	        [        XX5U5      U l        [        R                  R                  U R                  U R                  S9U l        [        U R                  5      U l        g Neps)r   r{   r   r   layernorm_epsilonr8   r   r   	LayerNorminput_layernormr-   	attentionpost_attention_layernormr   mlp)r   r   r4   r8   r   r5   r6   r   s          r	   r   TransformerLayer.__init__  s     	.0&!2(  %xx1133  2  5 '{'3'@B
 ).(:(:$"8"8 ); ):%t''(r   c           	          U R                  U5      nU R                  UUUUUUS9nU(       a  Uu  pUn
X-   nU R                  U5      nU R                  U5      nX-   nU(       a  UW	/nU$ Nr\   r]   r^   r_   r   r   r   r   )r   r   r[   r\   r]   r^   r_   layernorm_outputattention_outputpresentsresiduallayernorm_input
mlp_outputr    s                 r	   r!   TransformerLayer.forward  s      //>  >>!'') * + )9& !*5  88IXX./
-h'Fr   )r   r   r   r8   r   r   r   )h㈵>TTrq   r#   r+   s   @r	   r{   r{     s1     "&)@ ' 'r   r{   c                   D   ^  \ rS rSrSr SU 4S jjr    SS jrSrU =r$ )TopQueryLayeri  zsA single top query layer.

Top query layer takes input with size [b, s, h] and returns an
output of the same size.
c                   > [         [        U ]  5         Xl        X l        X@l        X0l        [        R                  R                  U R                  U R
                  S9U l
        [        U R                  U R                  U R                  5      U l        [        R                  R                  U R                  U R
                  S9U l        [        U R                  5      U l        g r}   )r   r   r   r   r4   r   r8   r   r   r   r   rs   r   r   r   r   )r   r   r4   r8   r   r   s        r	   r   TopQueryLayer.__init__  s     	mT+-&#6 !2(  %xx11$"8"8  2  : /t/?/?/3/G/G/3/@/@B ).(:(:$"8"8 ); ):% t''(r   c           
          US :w  d   eU R                  U5      nU R                  UUUUUUUS9n	U(       a  U	u  pUnX-   nU R                  U5      nU R                  U5      nUnX-   nU(       a  UW
/nU$ r   r   )r   r   rx   r[   r\   r]   r^   r_   r   r   r   r   r   r   r    s                  r	   r!   TopQueryLayer.forward  s     "T)))  //>  >>!'') * + )9& !*5  88I XX./
 #&h'Fr   )r   r   r   r8   r   r   r4   r   r   rq   r#   r+   s   @r	   r   r     s+     )B 0 0r   r   c                   `   ^  \ rS rSrSr S	U 4S jjrS rS r    S
S jr   SS jr	Sr
U =r$ )Transformeri8  zTransformer class.c           	        >^  [         [        T ]  5         UT l        UT l        UT l        UT l        S T l        T R                  b   eT R                  c  T R                  T l        T R                  T R                  -  S:X  d   S5       eU 4S jn[        R                  R                  [        T R                  5       Vs/ s H  oe" US-   5      PM     sn5      T l        [        T R                  T R                  T R                  5      T l        [        R                  R                  T R                  T R
                  S9T l        g s  snf )Nr   z?number of layers should be divisible by number of unique layersc                 F   > [        TR                  TR                  U 5      $ r   )r{   r   r4   )r8   r   s    r	   build_layer)Transformer.__init__.<locals>.build_layerT  s#    #D$4$4d6N6N$02 2r   r   r~   )r   r   r   r   r4   r   
num_layersnum_unique_layersr   r   
ModuleListrangelayersr   topQueryLayerr   final_layernorm)r   r   r4   r   r   r   ir   s   `      r	   r   Transformer.__init__;  s5    	k4)+&#6 !2$!% %%--- !!)%)__D"!7!771< 	NM	N<	2 hh))).t/E/E)FG)FA[Q)FGI +4+;+;+/+C+C+/+A+AC  %xx11$"8"8  2  : Hs   ?Ec                     XR                   -  $ r   )r   r   r8   s     r	   _get_layer_indexTransformer._get_layer_indexb  s    4444r   c                 >    U R                   U R                  U5         $ r   )r   r   r   s     r	   
_get_layerTransformer._get_layere  s    {{400>??r   c           
      V   UR                  SS5      R                  5       nUR                  SS5      R                  5       nU(       a  / n[        U R                  5       HF  n	U R	                  U	5      n
S nUb  XI   nU
" UUUUUUS9nU(       d  M1  Uu  pWR                  U5        MH     U R                  U5      nS nUb  X@R                     nU R                  UUUUUUUS9nU(       a  Uu  pWR                  U5        UR                  SS5      R                  5       nU(       a  UW/nU$ )Nr   r   r   )rR   rP   r   r   r   appendr   r   )r   r   rx   r[   r\   r]   r^   r_   r   indexlayerpastrf   hidden_states_r    s                  r	   r!   Transformer.forwardh  sK    &//15@@B/99!Q?JJLH4??+EOOE*ED%!(!++-/M })6&( ,$ --m<
 !oo.D**'') + + %2"MOOG$ ((A.99;h'Fr   c                 &    U R                  XU5      $ r   )
state_dict)r   destinationprefix	keep_varss       r	   state_dict_for_save_checkpoint*Transformer.state_dict_for_save_checkpoint  s     {I>>r   )r   r   r   r   r4   r   r   r   r   rq   N F)r$   r%   r&   r'   r(   r   r   r   r!   r   r)   r*   r+   s   @r	   r   r   8  sE     %:N5@ =@ 48.016? ?r   r   c                   L   ^  \ rS rSrSrU 4S jrS r   SS jrS	S jrSr	U =r
$ )
	Embeddingi  Language model embeddings.

Arguments:
    hidden_size: hidden size
    vocab_size: vocabulary size
    max_sequence_length: maximum size of sequence. This
                         is used for positional embedding
c                   > [         [        U ]  5         Xl        X l        X0l        [        R                  R                  U R                  U R                  5      U l        SU l	        [        R                  R                  U R
                  U R                  5      U l
        U R                  R                  5       U l
        SU l        g )Nword_embeddingsposition_embeddings)r   r   r   r   
vocab_sizemax_sequence_lengthr   r   r   _word_embeddings_keyr   rW   _position_embeddings_keyr   r   r   r   r   s       r	   r   Embedding.__init__  s     	i')&$#6   %xx11$//262B2B D$5! $)88#5#5d6N6N6:6F6F$H #'#;#;#@#@#B (=%r   c                 R    U R                  U5      nU R                  U5      nX4-   nU$ r   )r   r   )r   	input_idsposition_idswords_embeddingsr   
embeddingss         r	   r!   Embedding.forward  s2    //	:"66|D%;
r   c                     0 nU R                   R                  XU5      X@R                  '   U R                  R                  XU5      X@R                  '   U$ zFor easy load.)r   r   r   r   r   r   r   r   r   state_dict_s        r	   r   (Embedding.state_dict_for_save_checkpoint  s^     ""--k9M 	--. &&11, 	112 r   c                    U R                   U;   a  XR                      nO90 nUR                  5        H#  nSU;   d  M  X   X4R                  S5      S   '   M%     US   SU R                   US'   U R                  R                  X2S9  U R                  U;   a  XR                     nO90 nUR                  5        H#  nSU;   d  M  X   X4R                  S5      S   '   M%     U R                  R                  X2S9  g)	Customized load.r   zword_embeddings.r   weightNstrictr   zposition_embeddings.)r   keyssplitr   r   load_state_dictr   r   r   r   r   r   r<   s        r	   r   Embedding.load_state_dict  s    $$
2$%>%>?K K!($+$/  		*< =a @A ) !,H 56Ft GH,,[,H ((J6$%B%BCK K!((C/$/  		*@ A! DE ) 	  000Lr   )r   r   r   r   r   r   r   r   Tr$   r%   r&   r'   r(   r   r!   r   r   r)   r*   r+   s   @r	   r   r     s.    >, 48.016M Mr   r   c                   L   ^  \ rS rSrSrU 4S jrS r   SS jrS	S jrSr	U =r
$ )
QueryEmbeddingi  r   c                   > [         [        U ]  5         Xl        X l        X0l        [        R                  R                  U R
                  U R                  5      U l	        U R                  R                  5       U l	        SU l        g )Ntop_query_embeddings)r   r   r   r   r   r   r   r   r   r   rW   _top_query_embeddings_keyr   s       r	   r   QueryEmbedding.__init__  sl     	nd,.&$#6  %*HH$6$6$$d&6&6%8!$($=$=$B$B$D!)?&r   c                 (    U R                  U5      nU$ r   )r   )r   r   r   s      r	   r!   QueryEmbedding.forward  s    ..|<
r   c                 Z    0 nU R                   R                  XU5      X@R                  '   U$ r   )r   r   r   r   s        r	   r   -QueryEmbedding.state_dict_for_save_checkpoint$  s8     ''22, 	223 r   c                     U R                   U;   a  XR                      nO90 nUR                  5        H#  nSU;   d  M  X   X4R                  S5      S   '   M%     U R                  R	                  X2S9  g)r   r   ztop_query_embeddings.r   r   N)r   r   r   r   r   r   s        r	   r   QueryEmbedding.load_state_dict1  su     ))Z7$%C%CDK K!()S0$/  		*A B1 EF ) 	!!11+1Mr   )r   r   r   r   r   r   r   r   r+   s   @r	   r   r     s/    @$ 48.016N Nr   r   c                   X   ^  \ rS rSrSrU 4S jr    SS jr   S	S jrS
S jrSr	U =r
$ )TransformerLanguageModeliA  ai  Transformer language model.

Arguments:
    transformer_hparams: transformer hyperparameters
    attention_mask_func: a function that takes `unmaksed-attention-scores`
        with size [b, np, s, s] and an `attention-mask` and will apply
        the masking. The function should return a masked score of the
        same size [b, np, s, s].
      masked-attention-scores = attention_mask_func(
                                 unmaksed-attention-scores, attention-mask)
    vocab_size: vocabulary size
    max_sequence_length: maximum size of sequence. This
                         is used for positional embedding
c                   > [         [        U ]  5         Xl        X l        X0l        X@l        XPl        [        U R                  U R                  U R                  5      U l	        SU l
        [        U R                  U R                  U R                  5      U l        SU l        [        U R                  U R
                  U R                  5      U l        SU l        g )N	embeddingtopQueryEmbeddingtransformer)r   r   r   r   r   r4   padded_vocab_sizemax_position_embeddingsr   r   _embedding_keyr   r   _topQueryEmbedding_keyr   r   _transformer_keyr   r   r   r4   r   r   r   s         r	   r   !TransformerLanguageModel.__init__Q  s     	&68&$#6 !2'>$ #4#3#3T5K5K#'#?#?A) "00@0@040F0F040L0L"N ':# 't'7'7'+'?'?'+8 !.r   c           
      x    U R                  X5      nUn	U R                  U	5      n
U R                  UU
UUUUUS9nU$ r   )r   r   r   )r   r   r   r[   r\   r]   r^   r_   embedding_outputquery_position_idsqueryEmbedding_outtransformer_outputs               r	   r!    TransformerLanguageModel.forwardq  s`      >>)B)!334FG "--!'') . + "!r   c                     0 nU R                   R                  XU5      X@R                  '   U R                  R                  XU5      X@R                  '   U R
                  R                  XU5      X@R                  '   U$ r   )r   r   r   r   r   r   r   r   s        r	   r   7TransformerLanguageModel.state_dict_for_save_checkpoint  s     nn;;, 	''( $$CC, 	//0 ==, 	))* r   c                 b   U R                   U;   a  XR                      nO'0 nUR                  5        H  nSU;   d  M  X   X4'   M     U R                  R                  X2S9  U R                  U;   a  XR                     nO'0 nUR                  5        H  nSU;   d  M  X   X4'   M     U R
                  R                  X2S9  U R                  U;   a  XR                     nO90 nUR                  5        H#  nSU;   d  M  X   X4R                  S5      S   '   M%     U R                  R                  X2S9  g)r   _embeddingsr   ztransformer.r   N)	r   r   r   r   r   r   r   r   r   r   s        r	   r   (TransformerLanguageModel.load_state_dict  s/    *,$%8%89K K!( C''1K$ ) 	&&{&B&&*4$%@%@AK K!( C''1K$ ) 	..{.J   J.$%:%:;K K!(!S(@JK		. 9! <= ) 	(((Dr   )r   r   r   r   r   r   r4   r   r   r   r   rq   r   r   r   r+   s   @r	   r   r   A  s=    .J ": 48.016&!E !Er   r   c                   X   ^  \ rS rSrSrU 4S jr    SS jr   S	S jrS
S jrSr	U =r
$ )CodeGeeXModeli  z/CodeGeeX: A Multilingual Code Generation Model.c                 \   > [         [        U ]  5         [        XX4U5      U l        SU l        g )Nlanguage_model)r   r  r   r   r
  _language_model_keyr   s         r	   r   CodeGeeXModel.__init__  s2     	mT+-6%8#% $4 r   c           
          U R                  UUUUUUUS9nU(       a  Uu  p[        R                  " UU R                   R                  R                  R
                  R                  5       5      n
U(       a  U
W	/n
U
$ r   )r
  Flinearr   r   r   rW   )r   r   r   r[   r\   r]   r^   r_   	lm_outputr   r    s              r	   r!   CodeGeeXModel.forward  s     ''!'') ( +	 "+I))99@@EEGI h'Fr   c                 Z    0 nU R                   R                  XU5      X@R                  '   U$ r   )r
  r   r  r   s        r	   r   ,CodeGeeXModel.state_dict_for_save_checkpoint  s8    
 !!@@, 	,,- r   c                 r    U R                   U;   a  XR                      nU R                  R                  XS9  g)r   r   N)r  r
  r   )r   r   r   s      r	   r   CodeGeeXModel.load_state_dict  s7     ##z1#$<$<=J++J+Fr   )r  r
  rq   r   r   r   r+   s   @r	   r  r    s;    94( B 48.016	G Gr   r  )r>   r   torch.nn.functionalr   
functionalr  r
   Moduler   r-   rs   r{   r   r   r   r   r   r   r   r	   <module>r     s       $$%((// $NnEHHOO nblEHHOO l^Juxx JZSEHHOO Sls?%((// s?lQM QMh<NUXX__ <N~@Euxx @EFBGEHHOO BGr   