
    i?'                        S SK r S SKrS SKJr  S SKJs  Jr   " S S\R                  R                  5      r " S S\R                  R                  5      r	 " S S\R                  5      r
 " S S	\R                  5      r " S
 S\R                  5      r " S S\R                  R                  5      r\R                  SS4S\S\S\S\S\R"                  S\S\4S jjr " S S\R                  5      rg)    Nc                   >   ^  \ rS rSr SU 4S jjrSS jrS rSrU =r$ )ToySingleLinearModel   c                    > [         TU ]  5         X0l        X@l        [        R
                  R                  XXSUS9U l        g N)biasdtypedevice)super__init__r	   r
   torchnnLinearlinear1)self	input_dim
output_dimr	   r
   has_bias	__class__s         b/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/torchao/testing/model_architectures.pyr   ToySingleLinearModel.__init__   s:     	
xxf ' 
    c                     [         R                  " UU R                  R                  U R                  U R
                  S94$ N)r	   r
   r   randnr   in_featuresr	   r
   r   
batch_sizes     r   example_inputs#ToySingleLinearModel.example_inputs   6    KK((jj{{	
 	
r   c                 (    U R                  U5      nU$ N)r   r   xs     r   forwardToySingleLinearModel.forward(   s    LLOr   )r
   r	   r   F   	__name__
__module____qualname____firstlineno__r   r    r'   __static_attributes____classcell__r   s   @r   r   r      s     

 r   r   c                   >   ^  \ rS rSr SU 4S jjrSS jrS rSrU =r$ )ToyTwoLinearModel.   c                    > [         TU ]  5         X@l        XPl        [        R
                  R                  XXdUS9U l        [        R
                  R                  X#XdUS9U l        g r   )	r   r   r	   r
   r   r   r   r   linear2)r   r   
hidden_dimr   r	   r
   r   r   s          r   r   ToyTwoLinearModel.__init__/   s\     	
xxf ' 
 xxv ' 
r   c                     [         R                  " UU R                  R                  U R                  U R
                  S94$ r   r   r   s     r   r     ToyTwoLinearModel.example_inputsC   r"   r   c                 J    U R                  U5      nU R                  U5      nU$ r$   )r   r8   r%   s     r   r'   ToyTwoLinearModel.forwardM   s!    LLOLLOr   )r
   r	   r   r8   r)   r*   r,   r3   s   @r   r5   r5   .   s     
(
 r   r5   c                   \   ^  \ rS rSr    S SU 4S jjjrS\R                  4S jrSrU =r	$ )#ConvWithSharedWeightInExportedModelS   returnc           	         > [         TU ]  5         Xl        [        R                  " X#XEXgS9U l        [        R                  " U5      U l        [        R                  " SS9U l	        g )Nr   T)inplace)
r   r   n_chunksr   Conv2dconvBatchNorm2dbnReLUrelu)	r   rF   in_channelsout_channelskernel_sizestridepaddingr   r   s	           r   r   ,ConvWithSharedWeightInExportedModel.__init__T   sN     	 II{G
	 ...GGD)	r   c                    [         R                  " XR                  SS9n/ nU HG  nU R                  U5      nU R	                  U5      nU R                  U5      nUR                  U5        MI     [         R                  " USS9$ )Nr+   dim)r   chunkrF   rH   rJ   rL   appendcat)r   r&   chunksoutputsrV   outs         r   r'   +ConvWithSharedWeightInExportedModel.forwardf   sl    Q15E))E"C''#,C))C.CNN3	 
 yya((r   )rJ   rH   rF   rL   )   r+   r+   T)rB   N)
r-   r.   r/   r0   r   r   Tensorr'   r1   r2   r3   s   @r   r@   r@   S   s:     * 
* *$)ELL ) )r   r@   c                   J   ^  \ rS rSr\R
                  S4U 4S jjrS rSrU =r	$ )LNLinearActivationModelq   sigmoidc                   > [         TU ]  5         UR                  5       n[        R                  " 5       [        R
                  " 5       [        R                  " 5       [        R                  " 5       [        R                  " 5       [        R                  " 5       [        R                  " 5       S.nXE;  a  [        SU 35      e[        R                  " USS9U l        [        R                  " XSS9R                  US9U l        XT   U l        g )N)rL   rb   	leakyrelurelu6gelusilu	hardswishzUnsupported activation: F)elementwise_affinerD   )r	   )r   r   lowerr   rK   Sigmoid	LeakyReLUReLU6GELUSiLU	Hardswish
ValueError	LayerNormlnr   tofc
activation)r   fc_dim1fc_dim2r	   rv   activation_mapr   s         r   r    LNLinearActivationModel.__init__r   s    %%'
GGIzz|XXZGGIGGI
 +7
|DEE,,w5A))G59<<5<I(4r   c                 h    U R                  U5      nU R                  U5      nU R                  U5      $ r$   )rs   ru   rv   r%   s     r   r'   LNLinearActivationModel.forward   s+    GGAJGGAJq!!r   )rv   ru   rs   
r-   r.   r/   r0   r   bfloat16r   r'   r1   r2   r3   s   @r   r`   r`   q   s    /4~~) 5*" "r   r`   c                   x   ^  \ rS rSrS
S\S\4U 4S jjjrS rS\R                  S\R                  4S jr
S	rU =r$ )RMSNorm   rU   epsc                    > [         TU ]  5         X l        [        R                  " [
        R                  " U5      5      U l        g r$   )r   r   r   r   	Parameterr   onesweight)r   rU   r   r   s      r   r   RMSNorm.__init__   s+    ll5::c?3r   c           	      z    U[         R                  " [         R                  " X-  SSS9U R                  -   5      -  $ )NT)rU   keepdim)r   rsqrtmeanr   r%   s     r   _normRMSNorm._norm   s.    5;;uzz!%RFQRRRr   r&   rB   c                 z    U R                  UR                  5       5      R                  U5      nX R                  -  $ r$   )r   floattype_asr   )r   r&   outputs      r   r'   RMSNorm.forward   s.    AGGI&..q1##r   )r   r   )gh㈵>)r-   r.   r/   r0   intr   r   r   r   r^   r'   r1   r2   r3   s   @r   r   r      sA    4C 4e 4 4
S$ $%,, $ $r   r   c                   L   ^  \ rS rSrSS\R
                  4U 4S jjrS rSrU =r	$ )TransformerBlock         c                    > [         TU ]  5         Xl        X l        X-  U l        [
        R                  R                  USU-  SS9R                  U5      U l	        [
        R                  R                  XSS9R                  U5      U l
        X0l        [        X-  5      U l        [
        R                  R                  XR                  SS9R                  U5      U l        [
        R                  R                  U R                  USS9R                  U5      U l        [!        U5      R                  U5      U l        [!        U5      R                  U5      U l        [
        R                  R'                  5       U l        g )Nr]   FrD   )r   r   r9   	num_headshead_dimr   r   r   rt   qkvproj	mlp_ratior   mlp_hidden_dimmlp_fc1mlp_fc2r   norm1norm2rn   rv   )r   r9   r   r   r	   r   s        r   r   TransformerBlock.__init__   s&   $""/ 88??:q:~E?JMMeTHHOOJOGJJ5Q	 #!*"89xxz3F3FUSVV
 xxt':':JUSVV

 Z(++E2
Z(++E2
  ((--/r   c                    UR                   u  p#nUnU R                  U5      nU R                  U5      nUR                  X#SU R                  U R
                  5      nUR                  SSSSS5      nUu  pxn	UR                  X R                  -  X0R
                  5      nUR                  X R                  -  X0R
                  5      nU	R                  X R                  -  X0R
                  5      n	XxR                  SS5      -  SU R
                  S	-  -  -  n
[        R                  " U
SS
9n
X-  nUR                  X R                  X0R
                  5      nUR                  SS5      R                  X#U R                  5      nU R                  U5      nXQ-   nUnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nXQ-   nU$ )Nr]      r   r+   r   r   g      ?g      ?rT   )shaper   r   reshaper   r   permute	transposer   softmaxr9   r   r   r   rv   r   )r   r&   r   seq_len_residualr   qkvattns              r   r'   TransformerBlock.forward   s   !"
Q JJqM hhqkkk*q$..$--Pkkq!Q
 a IIj>>17MMJIIj>>17MMJIIj>>17MMJ KKB''C4==#3E,FG}}Tr* H IIj..'==IKK1%%j4??K IIaLL JJqMLLOOOALLOLr   )rv   r   r9   r   r   r   r   r   r   r   r   r   r}   r3   s   @r   r   r      s    -.!5>> *6, ,r   r   cudarL   
model_typemr   nhigh_precision_dtyper
   rv   c                    U S:X  a  [        X#XTS9nUR                  US9S   nXx4$ SU ;   ax  [        R                  " SU 5      n	U	(       a'  U	R	                  S5      (       a  U	R	                  S5      OSn[        X#XFS	9R                  U5      n[        R                  " XXTS9nXx4$ U S
:X  a5  [        USSUS9R                  U5      n[        R                  " USX%US9nXx4$ [        SU  35      e)aF  Create a model and input data for benchmarking.

Args:
    model_type (str): type of the model to be created
    batch_size (int): batch size of the input data
    device (str): device to run the model on
    high_precision_dtype (torch.dtype): data type of the model
    m, k, n (int): dimensions of the model and input data
linear)r
   r	   )r   r   	ln_linearzln_linear_?(\w+)?r+   rL   )rv   transformer_blockr   r   )r   r   r	      zUnknown model type: )r   r    researchgroupr`   rt   r   r   r   rq   )
r   r   r   r   r   r
   rv   model
input_datamatchs
             r   create_model_and_input_datar      s   $ X$Q&U))Q)7:
$ # 

	"		.
;',QU[[^V
'&

"V* 	 [[fQ
  
*	* a/C

"V* 	 [[B@TU
  /
|<==r   c                      ^  \ rS rSrS\S\S\S\R                  S\R                  SS4U 4S	 jjrS
\R                  S\R                  4S jr
SrU =r$ )LlamaModelsLlama4Expertsi  num_local_expertsrU   r9   r	   r
   rB   Nc           
      `  > [         TU ]  5         Xl        X l        [        R
                  " [        R                  " UUUUUS95      U l        [        R
                  " [        R                  " UUUUUS95      U l	        [        R
                  " [        R                  " UUUUUS95      U l
        g r   )r   r   r   rU   r   r   r   r   w1w2w3)r   r   rU   r9   r	   r
   r   s         r   r   !LlamaModelsLlama4Experts.__init__  s     	!2 "KK!!
 !#KK!!
 !#KK!!
r   routed_in_egDc                 j   U R                   nU R                  nUR                  USU5      n[        R                  " [
        R                  " X@R                  5      5      [
        R                  " X@R                  5      -  n[
        R                  " XPR                  5      nUR                  SU5      nU$ )Nr   )
r   rU   viewFrg   r   bmmr   r   r   )r   r   eDx_egDmiddle_out_egFout_egDs          r   r'    LlamaModelsLlama4Experts.forward<  s     ""HH""1b!,		% 9:UYYugg=VV))NGG4,,r1%r   )rU   r   r   r   r   )r-   r.   r/   r0   r   r   r	   r
   r   r^   r'   r1   r2   r3   s   @r   r   r     so    )
)
 )
 	)

 {{)
 )
 
)
V|| 
 r   r   )r   r   torch.nnr   torch.nn.functional
functionalr   Moduler   r5   r@   r`   r   r   r~   strr   r	   r   r    r   r   <module>r      s    
    588?? @" "J)")) )<"bii "8$bii $Huxx H` ).&&
& & 	&
  ++& & &T9ryy 9r   