
    9i#0                        S SK Jr  S SKJrJr  S SKrS SKJr  S SKJ	r	J
r
  S SKJr  SSKJrJr  S r " S	 S
\R"                  5      r " S S\R"                  5      rSS jr " S S\R"                  5      r " S S\5      r " S S\5      r " S S\R"                  5      rS r " S S\R"                  5      r " S S\5      r " S S\5      rg)    )partialmethod)ListOptionalN)	LayerNormsoftmax_dropout)permute_final_dims   )Linearchunk_layerc                 P    US:  d   e[         R                  " U 5      nXU S:H  '   U$ )Ng     r   )torch
zeros_like)maskneg_inf	attn_masks      t/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/models/science/unifold/modules/attentions.pygen_attn_maskr      s0    T>>  &I"dai    c                      ^  \ rS rSr SS\S\S\S\S\S\4U 4S jjjr  SS	\R                  S
\R                  S\R                  S\R                  S\	\R                     S\R                  4S jjr
S rSrU =r$ )	Attention   q_dimk_dimv_dimhead_dim	num_headsgatingc                 >  > [         [        U ]  5         XPl        X@R                  -  nX`l        [        XSSS9U l        [        X'SSS9U l        [        X7SSS9U l        [        XqSS9U l	        S U l
        U R                  (       a  [        XSS9U l
        US-  U l        g )NFglorotbiasinitfinalr"   r         )superr   __init__r   r   r
   linear_qlinear_klinear_vlinear_olinear_gnorm)	selfr   r   r   r   r   r   	total_dim	__class__s	           r   r'   Attention.__init__   s     	i')"~~-	ue(Kue(Kue(Kyg>;;"5(CDMdN	r   qkvr   r!   returnc                 :   S nU R                   b  U R                  U5      nU R                  U5      nXR                  -  nU R                  U5      nU R	                  U5      nUR                  UR                  S S U R                  S4-   5      R                  SS5      R                  5       nUR                  UR                  S S U R                  S4-   5      R                  SS5      R                  5       nUR                  UR                  S S U R                  S4-   5      R                  SS5      n[        R                  " XR                  SS5      5      nAA[        USU R                  XES9n[        R                  " Xs5      nAAUR                  SS5      R                  5       nUR
                  " / UR                  S S QSP76 nUb  [        R                  " U5      U-  n[        R                   R#                  XR$                  R&                  5      nU$ )Nr   )r   r!   )r,   r(   r-   r)   r*   viewshaper   	transpose
contiguousr   matmulr   trainingsigmoidnn
functionallinearr+   weight)	r.   r2   r3   r4   r   r!   gattnos	            r   forwardAttention.forward0   s    ==$a AMM!	YYMM!MM!FF1773B<4>>2"667AAJL 	
FF1773B<4>>2"667AAJL 	
FF1773B<4>>2"667AA"bI||A{{2r23qtQDLLL!!KKB**,FF%AGGCRL%"%=a 1$A MM  MM$8$89r   c                 .    U R                   R                  $ N)r+   r!   r.   s    r   get_output_biasAttention.get_output_biasY   s    }}!!!r   )r   r,   r)   r+   r(   r*   r-   r   TNN)__name__
__module____qualname____firstlineno__intboolr'   r   Tensorr   rH   rM   __static_attributes____classcell__r0   s   @r   r   r      s     ## # 	#
 # # # #: "'+'<<' <<' <<	'
 ll' u||$' 
'R" "r   r   c                   z   ^  \ rS rSrU 4S jrS\R                  S\R                  S\R                  4S jrSrU =r	$ )GlobalAttention]   c                 D  > [         [        U ]  5         X0l        X@l        XPl        [        XU-  SSS9U l        [        XSSS9U l        [        XSSS9U l	        [        XU-  SS9U l
        [        X#-  USS9U l        [        R                  " 5       U l        US-  U l        g )NFr   r    r   r$   r#   r%   )r&   r\   r'   r   infepsr
   r(   r)   r*   r,   r+   rA   Sigmoidr@   r-   )r.   	input_dimr   r   r_   r`   r0   s         r   r'   GlobalAttention.__init___   s    ot-/")+%hHyXNyXNyY*>XNx3YWMzz|dN	r   xr   r5   c                    U R                  U R                  U5      5      nU R                  U5      nU R                  U5      n[        R
                  " XR                  S5      -  SS9[        R
                  " USSS9U R                  -   -  nU R                  U5      nX`R                  -  nUR                  UR                  S S U R                  S4-   5      n[        R                  " XdR                  SS5      5      nAA[        X R                   * 5      SS S 2S S S 24   n[#        USU R$                  US9n[        R                  " UU5      n	AAUR                  UR                  S S U R                  S4-   5      nU	R                  S	5      U-  n	AU	R'                  U	R                  S S S
-   5      n	U R)                  U	5      $ )Nr7   r8   dimT)rg   keepdims.r   r   r9   )r7   )r@   r,   r)   r*   r   sum	unsqueezer`   r(   r-   r:   r;   r   r>   r<   r   r_   r   r?   reshaper+   )
r.   rd   r   rE   r3   r4   r2   rF   r   rG   s
             r   rH   GlobalAttention.forwardo   s    LLq)*MM!MM!IIr"",		$B6AC MM!	YYFF1773B<4>>2"667||A{{2r23q!$	234?C	tQIFLL
 !FF1773B<4>>2"667KKOa IIaggcrlV+,}}Qr   )
r`   r_   r,   r)   r+   r(   r*   r-   r   r@   )
rQ   rR   rS   rT   r'   r   rW   rH   rX   rY   rZ   s   @r   r\   r\   ]   s3    # !  ! U\\ ! ell !  ! r   r\   c                     [        X* 5      SS S 2S S S S 24   nU(       a,  [        U R                  SS5      U* 5      SS S 2S S S S 24   nX44$ U$ N.r7   r8   r   r<   )r   r_   gen_col_maskrow_maskcol_masks        r   gen_msa_attn_maskrt      s`    T4(atQ)>?H B!7#>sAt?CQ@G H!!r   c                   ^  ^  \ rS rSr  SU 4S jjr\R                  R                     SS\R                  S\	\R                     S\	\R                     S\
S\R                  4
S jj5       r\R                  R                     SS\R                  S\	\R                     S\	\R                     S\	\
   S\R                  4
S	 jj5       rSS\	\R                     4S
 jjr   SS\R                  S\	\R                     S\	\R                     S\	\
   S\R                  4
S jjrS rSrU =r$ )MSAAttention   c                    > [         [        U ]  5         X@l        [	        U5      U l        S U l        S U l        U R                  (       a   [	        U5      U l        [        XSSSS9U l        [        XXU5      U l
        g NFnormalr    )r&   rv   r'   	pair_biasr   layer_norm_mlayer_norm_zlinear_zr
   r   mha)r.   d_ind_hidr   r{   d_pairr0   s         r   r'   MSAAttention.__init__   sk     	lD*,"%dO >> )& 1D"H>DM Ti@r   mr   r!   
chunk_sizer5   c           
      b    [        U R                  UUUS.U[        UR                  S S 5      S9$ )N)r   r   r!   r8   r   num_batch_dimsr   _attn_forwardlenr;   )r.   r   r   r!   r   s        r   _chunkMSAAttention._chunk   s?     
 "qwws|,	
 		
r   c                 l   U R                  U5      nUR                  S   U-   S-
  U-  n/ n[        U5       Hf  nXt-  n[        UR                  S   X-   5      n	USX2S S 2S S 24   n
Ub  USX2S S 2S S 2S S 24   OS nUR	                  U R                  XXUS95        Mh     [        R                  " USS9$ )Nr9   r	   .r2   r3   r4   r   r!   rf   )r|   r;   rangeminappendr   r   cat)r.   r   r   r!   r   	num_chunkoutputsichunk_start	chunk_endcur_mcur_masks               r   _attn_chunk_forward MSAAttention._attn_chunk_forward   s     a WWR[:-1j@	y!A.KAGGBK)ABIc;0!Q67E # S+/Aq89)-  NN5UMO " yyb))r   c                 F    U R                  U5      nU R                  XXUS9$ )Nr   )r|   r   )r.   r   r   r!   s       r   r   MSAAttention._attn_forward   s'    a xx!Atx<<r   zr   c                 Z   S nU R                   (       aI  U R                  U5      n[        U R                  U5      S5      R	                  S5      R                  5       nUb  U R                  XXT5      nU$ SnUR                  S   U::  a  U R                  XU5      nU$ U R                  XXVS9$ )N   r   r	    
  r9   )r   )
r{   r}   r   r~   rk   r=   r   r;   r   r   )r.   r   r   r   r   r!   attn_chunk_sizes          r   rH   MSAAttention.forward   s     >>!!!$A"4==#3#,..7imJJL  !A$;A  #Owwr{o-&&qT:  //$ 0 D Dr   c                 6    U R                   R                  5       $ rK   r   rM   rL   s    r   rM   MSAAttention.get_output_bias       xx''))r   )r|   r}   r~   r   r{   )FNNNN)NNr   rK   )rQ   rR   rS   rT   r'   r   jitignorerW   r   rU   r   r   r   rH   rM   rX   rY   rZ   s   @r   rv   rv      sx    A* YY (,'+
<<
 u||$
 u||$	

 
 

 
& YY (,'+$(*<<* u||$* u||$	*
 SM* 
* **=8ELL+A = %),0$(<< ELL! ELL)	
 SM 
8* *r   rv   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )MSARowAttentionWithPairBiasi  c                 0   > [         [        U ]  UUUSUS9  g )NT)r{   r   )r&   r   r'   )r.   d_msar   r   r   r0   s        r   r'   $MSARowAttentionWithPairBias.__init__  s'    )49 	: 	
r    )rQ   rR   rS   rT   r'   rX   rY   rZ   s   @r   r   r     s    
 
r   r   c            	          ^  \ rS rSrU 4S jr  S	S\R                  S\\R                     S\\   S\R                  4U 4S jjjr	Sr
U =r$ )
MSAColumnAttentioni  c                 0   > [         [        U ]  UUUSS S9  g )NF)r   r   r   r{   r   )r&   r   r'   )r.   r   r   r   r0   s       r   r'   MSAColumnAttention.__init__  s'     $0 	1 	
r   r   r   r   r5   c                 l   > UR                  SS5      n[        TU ]	  XUS9nUR                  SS5      nU$ )Nr8   r9   )r   r   )r<   r&   rH   )r.   r   r   r   r0   s       r   rH   MSAColumnAttention.forward  s<     KKBGOAzOJKKBr   r   rP   )rQ   rR   rS   rT   r'   r   rW   r   rU   rH   rX   rY   rZ   s   @r   r   r     sU    
 -1$(	
<<
 ELL)
 SM	

 

 
r   r   c            	       (  ^  \ rS rSr  SU 4S jjr\R                  R                  S\R                  S\R                  S\	S\R                  4S j5       r
S r  SS\R                  S\\R                     S\\	   S\R                  4S	 jjrS
rU =r$ )MSAColumnGlobalAttentioni&  c                 n   > [         [        U ]  5         [        U5      U l        [        UUUUUS9U l        g )N)r_   r`   )r&   r   r'   r   r|   r\   global_attention)r.   r   r   r   r_   r`   r0   s         r   r'   !MSAColumnGlobalAttention.__init__(  s;     	&68%dO /!
r   r   r   r   r5   c           
      `    [        U R                  UUS.U[        UR                  S S 5      S9$ )N)r   r   r8   r   r   r.   r   r   r   s       r   r   MSAColumnGlobalAttention._chunk;  s<      "qwws|,
 	
r   c                 B    U R                  U5      nU R                  XS9$ )Nri   )r|   r   )r.   r   r   s      r   r   &MSAColumnGlobalAttention._attn_forwardL  s&    a $$Q$22r   c                     UR                  SS5      nUR                  SS5      nUb  U R                  XU5      nOU R                  XS9nUR                  SS5      nU$ )Nr8   r9   r7   ri   )r<   r   r   r   s       r   rH    MSAColumnGlobalAttention.forwardP  sb     KKB~~b"%!AZ0A""1"0AKKBr   )r   r|   )g    eAg|=rP   )rQ   rR   rS   rT   r'   r   r   r   rW   rU   r   r   r   rH   rX   rY   rZ   s   @r   r   r   &  s     
& YY
<<
 ll
 	

 

 
 3 (,$(	<< u||$ SM	
 
 r   r   c                     [        X* 5      SS S 2S S S S 24   n[        U R                  SS5      U* 5      SS S 2S S S S 24   nX#4$ ro   rp   )r   r_   
start_maskend_masks       r   gen_tri_attn_maskr   c  sW    tT*34q+@AJT^^B3cT:34;?<C DHr   c                   N  ^  \ rS rSrU 4S jr\R                  R                     SS\R                  S\	\R                     S\	\R                     S\
S\R                  4
S jj5       r  SS\R                  S	\	\R                     S\	\
   S\R                  4S
 jjrS rSrU =r$ )TriangleAttentionij  c                    > [         [        U ]  5         X@l        [	        U5      U l        [        XSSS9U l        [        XXU5      U l	        g ry   )
r&   r   r'   startingr   
layer_normr
   rC   r   r   )r.   r   r   r   r   r0   s        r   r'   TriangleAttention.__init__l  sC     	/1 #D/T5xHTi@r   rd   r   r!   r   r5   c           
      f    [        U R                  UUUUUS.U[        UR                  S S 5      S9$ )Nr   r8   r   )r   r   r   r;   )r.   rd   r   r!   r   s        r   r   TriangleAttention._chunky  sC     HH "qwws|,
 	
r   r   c                 r   U R                   (       d  UR                  SS5      nU R                  U5      n[        U R	                  U5      S5      R                  S5      R                  5       nUb  U R                  XXC5      nOU R                  XXUS9nU R                   (       d  UR                  SS5      nU$ )Nr8   r9   r   r   r   )	r   r<   r   r   rC   rk   r=   r   r   )r.   rd   r   r   triangle_biass        r   rH   TriangleAttention.forward  s     }}B#AOOAt{{1~(**3)B-

 	 !A-DA1Q]KA}}B#Ar   c                 6    U R                   R                  5       $ rK   r   rL   s    r   rM   !TriangleAttention.get_output_bias  r   r   )r   rC   r   r   r   rP   )rQ   rR   rS   rT   r'   r   r   r   rW   r   rU   r   rH   rM   rX   rY   rZ   s   @r   r   r   j  s    A YY (,'+
<<
 u||$
 u||$	

 
 

 
. -1$(	<< ELL) SM	
 
.* *r   r   c                   6    \ rS rSr\" \R                  SS9rSrg)TriangleAttentionStartingi  Tr   r   NrQ   rR   rS   rT   r   r   r'   rX   r   r   r   r   r     s    .77$GHr   r   c                   6    \ rS rSr\" \R                  SS9rSrg)TriangleAttentionEndingi  Fr   r   Nr   r   r   r   r   r     s    .77%HHr   r   rO   )	functoolsr   typingr   r   r   torch.nnrA   unicore.modulesr   r   unicore.utilsr   commonr
   r   r   Moduler   r\   rt   rv   r   r   r   r   r   r   r   r   r   r   <module>r      s    $ !   6 , 'D"		 D"N3 bii 3 lb*299 b*J	
, 	
 0:ryy :z <*		 <*~H 1 HI/ Ir   