
    9i                         S SK r S SKrS SKJr  S r " S S\R
                  5      r " S S\R
                  5      r " S S\R
                  5      r " S	 S
\R
                  5      r	 " S S\R
                  5      r
g)    Nc                     SU -  S[         R                  " [        R                  " S[        R                  -  5      U S[         R
                  " U S5      -  -   -  5      -   -  $ )Ng      ?      gHm?   )torchtanhmathsqrtpipow)xs    e/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/models/nlp/use/transformer.pygelur   	   sV    7a%**		!dgg+!h1a&@"@AC C D D    c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )PositionwiseFeedForward   c                 L  > [         [        U ]  5         [        R                  " X5      U l        [        R                  " X!5      U l        [        R                  " USS9U l        [        U l
        [        R                  " U5      U l        [        R                  " U5      U l        g )Nư>eps)superr   __init__nnLinearw_1w_2	LayerNorm
layer_normr   actvDropout	dropout_1	dropout_2)selfd_modeld_ffdropout	__class__s       r   r    PositionwiseFeedForward.__init__   sj    %t5799W+99T+,,wD9	G,G,r   c           	          U R                  U R                  U R                  U R                  U5      5      5      5      nU R	                  U R                  U5      5      nX1-   $ N)r"   r    r   r   r#   r   )r$   r   interoutputs       r   forwardPositionwiseFeedForward.forward   sI    tyy$//!2D)EFG0zr   )r    r"   r#   r   r   r   g?__name__
__module____qualname____firstlineno__r   r.   __static_attributes____classcell__r(   s   @r   r   r      s    - r   r   c                   6   ^  \ rS rSrSU 4S jjrSS jrSrU =r$ )MultiHeadedAttention   c                   > X!-  S:X  d   eX!-  U l         X l        [        [        U ]  5         Xl        [        R                  " X!U R                   -  5      U l        [        R                  " X!U R                   -  5      U l	        [        R                  " X!U R                   -  5      U l
        [        R                  " SS9U l        [        R                  " U5      U l        [        R                  " X"5      U l        g )Nr   )dim)dim_per_head	model_dimr   r:   r   
head_countr   r   linear_klinear_vlinear_qSoftmaxsoftmaxr!   r'   linear)r$   rA   r@   r'   r(   s       r   r   MultiHeadedAttention.__init__!   s    %***%3""D24$		)$:K:K-KL		)$:K:K-KL		)$:K:K-KLzzb)zz'*ii	5r   c                   ^^^ UR                  S5      mU R                  mU R                  mUUU4S jnUUU4S jnU R                  U5      R	                  TSTT5      R                  SS5      nU R                  U5      R	                  TSTT5      R                  SS5      nU R                  U5      R	                  TSTT5      R                  SS5      nU[        R                  " T5      -  n[        R                  " X1R                  SS5      5      nUb2  UR                  S5      R                  U5      nUR                  US5      nU R                  U5      nU R!                  U5      n	[        R                  " U	U5      R                  SS5      R#                  5       R	                  TSTT-  5      n
U R%                  U
5      nU$ )	Nr   c                 L   > U R                  TSTT5      R                  SS5      $ )zprojection r=   r   r   )view	transposer   
batch_sizer?   rA   s    r   shape+MultiHeadedAttention.forward.<locals>.shape5   s%    66*b*lC1a!r   c                 l   > U R                  SS5      R                  5       R                  TSTT-  5      $ )zcompute context r   r   r=   )rL   
contiguousrK   rM   s    r   unshape-MultiHeadedAttention.forward.<locals>.unshape:   s3    ;;q!$//1j"j<&?@Ar   r=   r   r   r   g    _)sizer?   rA   rB   rK   rL   rC   rD   r	   r
   r   matmul	unsqueeze	expand_asmasked_fillrF   r'   rR   rG   )r$   keyvaluequerymaskrO   rS   scoresattn	drop_attncontextr-   rN   r?   rA   s               @@@r   r.   MultiHeadedAttention.forward0   s   XXa[
((__
	!
	A
 mmC %%j"j&244=IaO 	e$))*b**688A	!Q 	e$))*b**688A	!Q 	 		,//e]]1a%89>>!$..v6D''e4F||F#LL&	,,y$&&/i1ojjl44#-r:3LDN 	 W%r   )	r?   r'   rA   rG   rB   rD   rC   r@   rF   r0   r+   r1   r8   s   @r   r:   r:      s    6$ $r   r:   c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )PositionalEncodingW   c           	      T  > [         [        U ]  5         [        R                  " X!5      n[        R
                  " SU5      R                  S5      n[        R                  " [        R
                  " SUS[        R                  S9[        R                  " S5      U-  * -  5      n[        R                  " UR                  5       U-  5      US S 2SS S24'   [        R                  " UR                  5       U-  5      US S 2SS S24'   UR                  S5      nU R                  SU5        g )Nr   r   r   )dtypeg     @pe)r   rd   r   r   zerosarangerW   expfloatr	   logsincosregister_buffer)r$   r>   max_lenrh   positiondiv_termr(   s         r   r   PositionalEncoding.__init__Y   s     $02[[&<<7+55a899ell1c1EKKH"&((7"3c"9 :; =ii 08 ;<1add7ii 08 ;<1add7\\!_T2&r   c                 Z    UR                  S5      nU R                  S S 2S U24   nX-   nU$ )Nr   )rU   rh   )r$   r   Lpos_embs       r   r.   PositionalEncoding.forwardd   s0    FF1I''!RaR%.Kr    )i   r1   r8   s   @r   rd   rd   W   s    	' r   rd   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )TransformerEncoderLayerk   c                    > [         [        U ]  5         [        X!US9U l        [        XU5      U l        [        R                  " USS9U l	        [        R                  " U5      U l        g )N)r'   r   r   )r   r{   r   r:   	self_attnr   feed_forwardr   r   r   r!   r'   )r$   r%   headsr&   r'   r(   s        r   r    TransformerEncoderLayer.__init__m   sP    %t57-egN3G7K,,wD9zz'*r   c                     US:w  a  U R                  U5      nOUnUR                  S5      nU R                  XUXTS9nU R                  U5      U-   nU R	                  U5      $ )Nr   r   )r]   )r   rW   r~   r'   r   )r$   iterr\   inputsr]   
input_normra   outs           r   r.   TransformerEncoderLayer.forwardu   s`    190JJ~~a ...Oll7#f,  %%r   )r'   r   r   r~   r1   r8   s   @r   r{   r{   k   s    +	& 	&r   r{   c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )TransformerEncoder   c                    > [         [        U ]  5         Xl        X@l        [        U5      U l        [        R                  " [        U5       Vs/ s H  n[        XX%5      PM     sn5      U l        [        R                  " U5      U l        g s  snf r+   )r   r   r   r%   layersrd   rw   r   
ModuleListranger{   transformer_interr!   r'   )r$   r%   r&   r   r   r'   _r(   s          r   r   TransformerEncoder.__init__   su     $02)'2!#6]0
" $GDB"0
 " zz'*	0
s   Bc           	          U R                  U5      nU R                  U5      n[        U R                  5       H(  nU R                  U   " X1XR                  S5      5      nM*     U$ )Nr   )rw   r'   r   r   r   eq)r$   r   r]   is       r   r.   TransformerEncoder.forward   sS    LLOLLOt{{#A&&q)!771:>A $r   )r%   r'   r   rw   r   r0   r1   r8   s   @r   r   r      s    
+ r   r   )r	   r   torch.nnr   r   Moduler   r:   rd   r{   r   ry   r   r   <module>r      sg      D
bii "5299 5p (&bii &, r   