
    i7                        S SK r S SKrS SKrS SKrS SKrS SKJrJrJrJ	r	  S SK
Jr  S SKJr  \R                  R                  5       =(       a    \R                   R                  r\R                  R                  5       =(       a    \R                   R$                  rS SKJrJrJr  S SKJrJrJr  S SKJr   " S S	\5      r\" \5        \R>                  RA                  \(       + S
S9S 5       r!/ SQr"/ SQr#/ SQr$\RJ                  \RL                  /r'\(" \ RR                  " \"\#\$\'5      5      r*\RV                  4S jr,\R>                  RA                  S\RZ                  R]                  S5      ;  SS9\R>                  R                  S\*\/S9\R>                  R                  S\RV                  \R`                  /5      S 5       5       5       r1\R>                  RA                  S\RZ                  R]                  S5      ;  SS9\R>                  R                  S\*\/S9\R>                  R                  S\RV                  \Rd                  /5      S 5       5       5       r3\R>                  RA                  S\RZ                  R]                  S5      ;  =(       d    S\RZ                  R]                  S5      ;  SS9\R>                  RA                  \" S5      (       + SS9\R>                  R                  SSS/5      \R>                  R                  S S!S"/5      \R>                  R                  S\RV                  \Rh                  \Rj                  /5      \R>                  R                  S#S$S%/5      \R>                  R                  S&\" 5       \" 5       \" S'5      /5      \R>                  R                  S(\" 5       \" 5       \" S'5      /5      S) 5       5       5       5       5       5       5       5       r6\7S*:X  a  \Rp                  " \Rr                  5        gg)+    N)IS_LINUXTestCaseinstantiate_parametrized_testsparametrize)opcheck)torch_version_at_least)PerGroupPerRow	PerTensor)_choose_scale_float8_dequantize_affine_float8_quantize_affine_float8)get_block_sizec                   t   \ rS rSr             SS jr        S S jr\R                  R                  \	" S5      (       + SS9\R                  R                  \
(       + SS9\R                  R                  S	\R                  R                  S
5      ;  SS9\" S\R                  \R                   /5      \" SSS/5      \" SSS/5      \" SSS/5      \" SSS/5      \" SSS/5      \" SS\R"                  \R$                  /5      S 5       5       5       5       5       5       5       5       5       5       rSrg)!TestOps&   Nc                 d   UR                  [        R                  5      U-
  U-  nUR                  [        R                  5      U
-
  U	-  nUR                  [        R                  5      U-
  U-  nS[        R                  " UR                  S5      5      -  nXR                  SS5      -  nUU-  nUb"  UUR                  [        R                  5      -   nUR                  SSS9R                  nUU-
  n[        R                  " U5      n[        R                  " USSS9nUU-  n[        R                  " [        R                  " UU-  5      U-   SSS9nUU-
  U-  nUU-  n[        R                  " [        R                  " UU-  5      U-   SSS9nUR                  [        R                  5      $ )	N   Tdimkeepdimr      minmax)totorchfloatmathsqrtsize	transposer   valuesexpsumclamprounduint8)selfqkv	attn_mask	dropout_p	is_causalq_scaleq_zpk_scalek_zpv_scalev_zpa_scalea_zpo_scaleo_zpscale_factorattnattn_maxattn_sumouts                         L/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/test/test_ops.py_scaled_dot_product_int8_op_ref'TestOps._scaled_dot_product_int8_op_ref'   sh   & TT%++%0TT%++%0TT%++%0499QVVBZ00;;r2&&l" ),,u{{33D88D8188hyy99Tr48h{{5;;tg~6=1#Ntw&Qhkk%++cGm4t;Lvvekk""    c                 Z   UR                  [        R                  5      U-  nUR                  [        R                  5      U-  nUR                  [        R                  5      U	-  nS[        R                  " UR                  S5      5      -  nXR                  SS5      -  nX-  nUb!  XR                  [        R                  5      -   nUR                  SSS9R                  nX-
  n[        R                  " U5      n[        R                  " USSS9nX-  n[        R                  " X-  SSS9nUR                  [        R                  5      R                  [        R                  5      nX-  nX-  n[        R                  " UU-  SSS9nUR                  [        R                  5      $ )	Nr   r   r   Tr   @  r   )r   r   r    r!   r"   r#   r$   r   r%   r&   r'   r(   float8_e4m3fn)r+   r,   r-   r.   r/   r0   r1   r2   r4   r6   r8   r:   r<   r=   r>   r?   r@   s                    rA   _scaled_dot_product_fp8_op_ref&TestOps._scaled_dot_product_fp8_op_refM   sO    DD'DD'DD'499QVVBZ00;;r2&&" ,,u{{33D88D8188yy99Tr48{{4>t=wwu**+..u{{;~hkk#-Ts;vve))**rD   z2.7.0z*quantized sdpa requires torch 2.7 or laterreasonzonly support on linuxCPUztorchao::qscaled_dot_productcpp kernels not builtinput_dtype
batch_size8   x   n_head      	q_seq_len   Y   
kv_seq_lend      head_dim    @   
mask_dtypec                 B   [         R                  " S5        SnU[         R                  :X  at  [        S5      n	[        S5      n
[        S5      n[        S5      n[        S5      n[	        S5      n[	        S	5      n[	        S5      n[	        S
5      n[	        S5      nSu  nnO<[        S5      n	[        S5      n
[        S5      n[        S5      n[        S5      nSu  nnX$X6/nX%X6/nUSSU/n[         R
                  " U[         R                  US9R                  SS5      n[         R
                  " U[         R                  US9R                  SS5      n[         R
                  " U[         R                  US9R                  SS5      nU[         R                  :X  a  US-  nUS-  nUS-  nUR                  U5      nUR                  U5      nUR                  U5      nUb  [         R
                  " UXxS9OS nUR                  5       UR                  5       UR                  5       Ub  UR                  5       OS 4u  nnnnU[         R                  :X  aU  U R                  UUUUSSU	WU
WUWUWUWS9n [         R                  R                  R                  UUUUSSU	UU
UUUUUUUS9n!OJU R                  UUUUSSU	U
UUUS9n [         R                  R                  R                  UUUUSSU	U
UUUS9n!U R                  U!R                  5       U R                  5       UUS9  g )Ni  cpug    Φ?g   ?g    l?g    p?g   `P?   }   rR      )      ?h㈵>g     @g      @g     `?g     `@g     `	@)g      ?rf   r   )dtypedevicerT   rZ   g        F)r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   )r/   r0   r1   r2   r4   r6   r8   r:   atolrtol)r   manual_seedr*   r    intrandnr$   r   clonerB   opstorchaoqscaled_dot_productrI   assertEqual)"r+   rO   rP   rS   rV   rY   r\   r_   rh   r2   r4   r6   r8   r:   r3   r5   r7   r9   r;   rj   rk   q_shapekv_shape
mask_shaper,   r-   r.   r/   q2k2v2attn_mask_2math_refactuals"                                     rA   $test_quantized_scaled_dot_product_op,TestOps.test_quantized_scaled_dot_product_opp   sH   4 	$%++%./G./G-.G01G./Gs8Ds8Ds8Ds8Ds8D"JD$GnGGnGJ'GGnGHoG$JD$&;F= !Q
3
KKu{{6BLLQPQRKKFCMMaQRSKKFCMMaQRS%++%HAHAHADDDDDD % KK
*D 	 GGIGGIGGI!+!7IOOT	#
BK %++%;;#! < H$ YY&&::%! ; F& ::# ; H YY&&::% ; F 	)94PrD    )Nr   Fre   r   re   r   re   r   re   r   re   r   )Nr   Fre   re   re   re   re   )__name__
__module____qualname____firstlineno__rB   rI   pytestmarkskipifr   r   r   _C_dispatch_dumpr   r*   rH   float32bfloat16r}   __static_attributes__r   rD   rA   r   r   &   s    #$#V !+F [["7++;   [[H-DE[[UXX,,-KLL&   e.A.A BCCy)Ar7#r2h'Sz*b"X&emmU^^DExQ F ' + ( $ * D	 F	 xQrD   r   zROCm not availablerK   c                  &   / SQn U R                  S5        [        R                  " SS[        R                  SSS9n[        R                  " SS[        R                  SSS9n[	        [        R
                  R                  R                  XS	S	4U S
9  g )N)test_schematest_autograd_registrationtest_faketensortest_aot_dispatch_dynamicr   rU   )rU   r]   cuda)rg   r#   rh   )r]   rU   F)
test_utils)appendr   randintr    r   rp   rq   
swizzle_mm)r   mat1mat2s      rA   test_swizzle_mmr      sv    J 12==BekkPD==BekkPD		$$	UE"rD   )r   rT      
   )r   rT   rd   i   )r   rd   i   c                    U[         R                  [         R                  [         R                  4;   d   eSnSnUS:X  a  SnOUS:X  a  SnOUS:X  a  Sn[         R                  " SX-  45      R                  U5      n	[         R                  " SUS-   U -  U 5      R                  U5      n
[         R                  R                  SUU[         R                  US	9nU[         R                  :X  a^  S
UR                  R                  R                  5       R                  5       -  nUR                  R                  U-  R                  U5      nO<[         R                  " S/5      nUR                  R                  R                  U5      nUR                  UR                  R                  5      UR                  l
        SnU[         R                  [         R                  4;   a  Sn[         R                  " 5          UR!                  X5      U-  nU[         R                  :X  az  [         R"                  " X-  5      R                  [         R$                  5      n[         R&                  " USS5      R                  U5      R                  [         R                  5      nOZU[         R                  :X  aF  [         R&                  " X-  SS5      R                  U5      R                  [         R                  5      n[         R(                  R*                  R-                  UU	U
UUWUU5      R                  [         R                  5      n[         R.                  R1                  UUSSS9  S S S 5        g ! , (       d  f       g = f)NTr'   r   meanr   r   rT   i  )moderg   include_last_offsetg     _@g       @re   irb   rF   rG   gh㈵>ri   )r   r    int8rH   r   r   arangennEmbeddingBagweightdataabsr   tensorrg   no_gradforwardr)   int32r(   rp   rq   _scaled_embedding_bagtestingassert_close)	multi_hotrP   vector_size
index_typeqtype	out_dtyper   r   	mode_enumindicesoffsetsmweight_scaleqweight	out_scalerefe_outtest_outs                    rA   %_test_scaled_embedding_bag_cpu_helperr   !  s    ejj%2E2EFFFFDu}					mmD:#9";<??
KGll1zA~:IFII*UGkk/ 	 	A 

qxx}}002668888==</33E:||SE*((--""5)JJqxx~~.AHHMIUZZ!4!455		99W.=

"{{8#78;;EKKHH{{8T37::9EHHUH%---H0$<IEKK 
 99$$::	
 "U[[/ 	 	""	 	# 	
+ 
s   8E*M++
M9rM   ztorchao::_scaled_embedding_bagrN   z.multi_hot, batch_size, vector_size, index_type)idsr   c                 @    [        U UUU[        R                  U5        g N)r   r   r   r   rP   r   r   r   s        rA   "test_scaled_embedding_bag_int8_cpur   g  s"     *

rD   c                 @    [        U UUU[        R                  U5        g r   )r   r   rH   r   s        rA   !test_scaled_embedding_bag_fp8_cpur   ~  s$     *rD   z"torchao::float8_linear_prepack_cpuztorchao::float8_linear_cpuz2.6.0zTest only enabled for 2.6+shape)r^   r^   )   r   bsr      biasTFx_granularityrd   w_granularityc                 Z   U u  pg[        U[        5      (       a'  UR                  U:  a  g [        U[        5      (       d  g [        U[        5      (       a  UR                  U:  a  g [        R                  R                  XgUS9R                  5       nUR                  n	[        R                  " X5      n
[        U
R                  U5      n[        U
[        R                  US9n[        X[        R                  5      nUR                  R                  5       n[        UR                  U5      n[        U[        R                  US9n[        UU[        R                  5      n[!        X5      n[!        UU5      n[        R                  R"                  R%                  UUU	5      R'                  U5      n[        R(                  R*                  R-                  UU5      u  nn[        R(                  R*                  R/                  XUUX5      n[        R0                  R3                  UUSSS9  g )N)r   )float8_dtype
block_sizeg{Gz?ri   )
isinstancer	   
group_sizer   r   Linearevalr   rn   r   r   r   rH   r   r   detachr   
functionallinearr   rp   rq   float8_linear_prepack_cpufloat8_linear_cpur   r   )r   r   r   r   r   r   
in_featureout_featurer   bxx_block_sizex_scalex_fp8ww_block_sizew_scalew_fp8x_dqw_dqrefpacked_wpacked_scaleys                           rA   test_float8_linear_cpur     s    $J-**##z1-22-**##z1
d;@@BA	AB#A!!''=9L"	((G
 $A0C0CDE	A!!''=9L"	((G
 $Aw0C0CDE$U4D$UG4D
((


$
$T4
3
6
6y
AC"YY..HHPWXHl		++,	A 
MMq#Dt<rD   __main__):	itertoolsr!   sysr   r   $torch.testing._internal.common_utilsr   r   r   r   torch.testing._internal.optestsr   torchao.utilsr   r   is_availableversionIS_CUDAhipIS_ROCMtorchao.quantizationr	   r
   r   %torchao.quantization.quant_primitivesr   r   r   torchao.quantization.utilsr   r   r   r   r   EMBEDINGBAG_MULTIHOT_SIZESEMBEDINGBAG_BAG_SIZESEMBEDINGBAG_VECTOR_SIZESint64r   EMBEDINGBAG_INDEX_DTYPESlistproductEMBEDINGBAG_TEST_PARAMSr    r   r   r   strr   r   rH   r   r   halfr   r   mainargvr   rD   rA   <module>r     s     
    4 **
!
!
#
:(:(:
**
!
!
#
9(9(9 < < 

RQh RQj w ' K(<= >& + ) ( !KK5 "  	   kkC
L 	(()IJJ"   4  
 u{{EJJ&?@
 A	
 	(()IJJ"   4  
 u{{E4G4G&HI
 J	
 	(()MNN JEHH++,HII"  
 w''0L   8Z"893x(u{{ENNEJJ&OP$/9;(3-*PQ9;(3-*PQ'= R R 0 Q ) :'=T z
KK rD   