
    Si%                        S SK Jr  S SKrS SKrS SKJr  S
SS jjrS rS r	           S               SS jjr
 " S S	\5      rg)    )annotationsN)OpRunc                    [         R                  " XSS9n[         R                  " X-
  5      n[         R                  " X1SS9nX4-  $ )NT)axiskeepdims)npmaxexpsum)xr   x_maxtmpss        ^/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/onnx/reference/ops/op_attention.py_softmaxr      s9    FF1$/E
&&
C
s-A7N    c                N    US:  a  X-  n[         R                  " U5      nX!-  $ U $ )Nr   )r   tanh)XsoftcapYs      r   _softcapr      s*    {KGGAJ{Hr   c                    U R                   SS u  p#[        R                  " [        R                  " X#U-
  4U R                  S9SS9n[        R
                  * XDS:H  '   U SSS2US24==   U-  ss'   U $ )zApplies a causal mask on the input `mask`:
``mask[i, j] = -inf if past_sequence_length + i > j else 0``.
Because a softmax is applied on the mask, -inf becomes 0 and 0 becomes 1.
The modification is done inplace.
Ndtype   )k.)shaper   triuonesr   inf)maskpast_sequence_lengthq_sequence_lengthtotal_sequence_lengthr    s        r   _apply_causalr'      s|     04zz"#,77
8L LM**	
 D vvgDOa%&	&'4/'Kr   c                b   [        U R                  5      [        UR                  5      s=:X  a  [        UR                  5      :X  d   e   e[        U R                  5      nU R                  S   n[        U R                  5      S:X  Ga  U R                  S   nUR                  S   nUR                  S   nU	b  U
c   e[        UU	-  5      nXR                  S   U	U/n[        R                  " U U5      n [        R
                  " U S5      n [        UU
-  5      nXR                  S   U
U/n[        R                  " UU5      n[        R
                  " US5      n[        UU
-  5      nXR                  S   U
U/n[        R                  " UU5      n[        R
                  " US5      n[        U R                  5      S:X  a2  [        UR                  5      S:X  a  [        UR                  5      S:X  d   eUc(  U R                  S   nS[        R                  " U5      -  n[        R                  " U5      nUb  [        R                  " XA4SS9nOUnUb  [        R                  " XR4SS9nOUnUnUnU R                  S   nUR                  S   n[        R                  " UU4U R                  S9nUby  UUR                  S	   -
  nUS:  aa  S
/UR                  S-
  -  SU4/-   n UR                  [        R                  :X  a  SO[        R                  * n![        R                  " UU SU!S9nU(       a  Uc>  [        R                  " UU4U R                  S9n"[        U"Ub  UR                  S   OSS9nOUR                  [        R                  :X  a0  SU-
  R                  U R                  5      [        R                  * -  n[        UR!                  5       Ub  UR                  S   OSS9nOZUbW  UR                  [        R                  :X  a4  SU-
  R                  U R                  5      n[        R                  * X3S:H  '   UU-   nUb  UR	                  SSUR                  -
  -  UR                  -   5      n[        R"                  " U5      US S 2[        R$                  4   :  n#U#R	                  USSU5      n#[        R&                  " U#S[        R                  * 5      n#UU#-  nU	c  U R                  S   n	U
c  UR                  S   n$UR                  S   n%OU
n$U
n%U	U$:w  a@  U	U$-  S:X  a7  U$U%:X  a1  U	U$-  n&[        R(                  " UU&SS9n[        R(                  " UU&SS9n[        R
                  " US5      n'[        R*                  " X-  U'U-  5      n(U(U-   n)US:X  a  U)R!                  5       n(Ub  [-        U)U5      n)US:X  a  U)n(Ub.  U)R                  [.        R0                  R3                  U5      5      n)[5        U)5      n*US:X  a  U*n(U(R                  U R                  5      n([        R*                  " U*U5      R                  U R                  5      n+US:X  aK  [        R
                  " U+S5      n+[        R                  " U+U+R                  S   U+R                  S   S	45      n+U+UUU(4$ )Nr         r   )r   r*   r   r)      )r   r   )r   r   Fconstant)modeconstant_values)r$   )r   )repeatsr   )r   r   r)   r*   )lenr   intr   reshape	transposesqrtconcatenatezerosr   ndimbool_r"   padr'   astypecopyarangenewaxiswhererepeatmatmulr   onnxhelpertensor_dtype_to_np_dtyper   ),QKV	attn_maskpast_key
past_valuenonpad_kv_seqlenscale	is_causalq_num_headskv_num_headssoftmax_precisionr   qk_matmul_output_modeinput_shape_len
batch_sizehidden_size_qhidden_size_khidden_size_vhead_size_qintermediate_shape_qhead_size_kintermediate_shape_khead_size_vintermediate_shape_vq_head_sizepresent_keypresent_valuer%   kv_sequence_length	attn_bias	pad_width	pad_shape	pad_value	temp_maskpadding_maskk_num_headsv_num_headsseq_repsk_transposeqk_matmul_outputqk_with_bias
qk_softmaxoutputs,                                               r   _compute_attentionro   .   s     qww<3qww<73qww<77777
 !''lOJ
177|q


&<+CCC-+56 *GGAJ[QJJq./LLL)-,67 *GGAJkRJJq./LLL)-,67 *GGAJkRJJq./LLL)qww<1QWW!2s177|q7HHH }ggajBGGK((GGENE nnh];
Q?AA 
+-?@PI &)<<	q=INNQ$67Ay>:JJI!*BHH!<266'I9:yI "35G!HPQPWPWXI%:B:NX^^A%6TUI
 "((*]22177;wG	% :B:NX^^A%6TUI 
	??bhh&Y..qww7I)+I1n%	)	#%%A	&')//9
	 yy!347G2::7VV#++J1>PQxxa"&&9\!	 ggajggajggaj""		#;&!+K'+-IIa2IIa2( ,,q,/KyyK%,?@#i/L!',,. g6 A%+$#**KK001BC
 ,'J!%'..qww7YYz1%,,QWW5F!fl3FV\\!_fll1or$JK;/???r   c                  V    \ rS rSr           S               SS jjrSrg)	Attention   Nc                .    [        UUUUUUUUU	U
UUUUS9$ )N)rH   rI   rJ   rK   rL   rM   rN   rO   rP   r   rQ   )ro   )selfrE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   r   rQ   s                  r   _runAttention._run   s;    " "!-#%/"7
 	
r    NNNNNFNNNNNrE   
np.ndarrayrF   rz   rG   rz   rH   np.ndarray | NonerI   r{   rJ   r{   rK   r{   returnrz   )__name__
__module____qualname____firstlineno__ru   __static_attributes__rw   r   r   rq   rq      s}     (,&*(,.2" 
 
  
 	 

 % 
 $ 
 & 
 , 
  
! 
  
r   rq   )r,   )r   rz   r   r2   r|   rz   rx   ry   )
__future__r   numpyr   rB   onnx.reference.op_runr   r   r   r'   ro   rq   rw   r   r   <module>r      s    #   '. $("&$(*.
v@v@v@ v@ !	v@
  v@ "v@ (v@ v@r!
 !
r   