
    9i                      S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKJr  S SKJr  S SKJrJrJrJrJrJr  S SKrS SKJr  S SKrS SKrS SKrS SKJrJr  S SKJr  S S	KJrJ r   S S
K!J"r"  S SK#J$r$  S SKJ%r%J&r&  S SK'J(r(J)r)  S SK*J+r+  S SK,J-r-  S SK.J/r/  S SK0J1r1J2r2  S SK3J4r4J5r5J6r6J7r7J8r8J9r9J:r:J;r;  S SK<J=r=  S SK>J?r?  S SK@JArA  S SKBJCrC  SSKDJErEJFrFJGrG  SSKHJIrIJJrJJKrKJLrLJMrMJNrNJOrOJPrP  SSKQJRrRJSrSJTrTJUrU  SSKVJWrW  SSKFJXrXJYrYJZrZJ[r[J\r\J]r]J^r^J_r_J`r`JaraJbrbJcrc  SSKdJereJfrfJgrgJhrhJiriJjrjJkrkJlrlJmrmJnrnJoroJprp  SSKqJrrr  SSKsJtrt  SS KuJvrv  SS!KwJxrxJyryJzrzJ{r{J|r|J}r}J~r~JrJrJrJrJr  SS"KJrJr  \(       a;  S S#KJrJrJr  S S$KJr  S S%KJr  S S&KJr  S S'KJr  SS(KJr  SS)KJr  SS*KJr  \\\K4   rS S+KJr  \GR6                  " \5      r\GR<                  GR?                  \S,5      r\GRB                  GRD                  r\GRF                  " 5       r\EGRJ                  " 5       (       a  S S-KJr  OS8S. jrS9S/ jrS:S0 jr      S;S1 jrS<S2 jr      S=S3 jr " S4 S5\GRZ                  GR\                  5      r " S6 S7\5      rg)>    )annotationsN)defaultdict)contextmanager)AnyCallableNoReturnOptionalTYPE_CHECKINGUnion)Expr)deviceTensor)get_decompositions)defakedynamo_timed)FakeScriptObject)get_layout_constraint_tag)
LazyStringtrace_structured)compute_required_storage_lengthmake_channels_last_strides_for)
FakeTensor)full_aoti_runtime_assert)BackwardState)magic_methodsmethod_to_operator)_get_placeholder_exprfree_unbacked_symbolshas_free_symbolsresolve_unbacked_bindingsRuntimeAssertShapeEnvSympyBooleanSymTypes)Node)no_dispatch)
OrderedSet)int_oo   )configirmetrics)BackendFeatureDeviceOpOverridesFileBackedGraphModuleget_backend_featuresget_device_op_overridesget_wrapper_codegen_for_deviceinit_backend_registrationWorkspaceArg)CppWrapperCodegenErrorLoweringExceptionMissingOperatorWithDecompMissingOperatorWithoutDecomp)count_flops_fx)ConstantDonatedBufferFixedLayoutget_device_typeGraphPartitionSignatureInputBuffer	Pointwise	ReductionShapeAsConstantBuffer
StorageBox	TensorBoxTorchBindObject)constrain_to_fake_tensorsconstrain_to_fx_stridesFALLBACK_ALLOW_LISTfallback_handler%fallback_node_due_to_unsupported_type	loweringsmake_fallbackmaybe_layout_constraintsneeds_realized_inputsrequire_contiguoustag_to_layout_constraintunsupported_output_tensor)autotune_cache)AutotuneCacheBundler)SizeVarAllocator)convert_shape_to_inductorgather_origins get_cloned_parameter_buffer_nameget_donated_idxsget_sympy_Expr_dtypeGraphPartitionMapis_same_tensor#maybe_get_suppress_shape_guards_ctxnormalize_nameshould_assume_input_alignedSUPPORTED_MKLDNN_DEVICESValueWithLineMap)NullHandlerV)IterableIteratorSequence)
ModuleType)_EffectType)GraphModule)Graph)PythonWrapperCodegen)Dep)BaseSchedulerNode)output_code_log
perf_hints)log_module_codec                     g N )argskwargss     U/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/torch/_inductor/graph.pyro   ro      s        c                   [        U [        R                  [        R                  [        R                  R
                  R                  45      (       d   S5       e[        U [        R                  R
                  R                  5      (       a  [        R                  $ [        U [        R                  5      (       a  [        U 5      $ U R                  (       a  [        R                  $ U R                  (       a  [        R                  $ g )Nzgget_constant_buffer_dtype only supports input of sympy.Symbol, sympy.Expr or sympy.core.numbers.Integer)
isinstancesympySymbolr   corenumbersIntegertorchint64rY   
is_integeris_floatfloat32)constant_buffers    ru   may_get_constant_buffer_dtyper      s    %,,

EJJ4F4F4N4NO   	r 
 /5::#5#5#=#=>>{{/5::..#O44!!{{		!	!}}rv   c                6    [        S [         5       5      nX;   $ )Nc              3  8   #    U  H  n[        U5      v   M     g 7frq   )r   ).0ms     ru   	<genexpr>"is_magic_method.<locals>.<genexpr>   s     H-Q-a00-s   )r'   r   )op	magic_opss     ru   is_magic_methodr      s    H-HHI?rv   c           	         UR                  S5      nU n[        U5       H@  u  pE[        X55      (       d   [        SSR	                  US U 5       35      e[        X55      nMB     U$ )N.z#Node referenced nonexistent target )split	enumeratehasattrRuntimeErrorjoingetattr)objtargettarget_atomsattr_itriatoms         ru   getattr_recursiver      sn     <<$LH\*x&&5chh|BQ?O6P5QR  8* + Orv   c                n   0 nU R                  SS9S   nSUR                  ;  a  U$ [        UR                  S   [        R
                  R                  5      (       d  UR                  S   nOUR                  n[        U5       H-  u  pEXBR                  S   ;   d  M  UR                  S   U   X'   M/     U$ )Noutputr   r   user_visible_output_idxsoriginal_output_strides)
find_nodesmetarx   rs   r~   fxr%   r   )gretoutput_nodeoutput_node_argsidxnodes         ru   get_user_visible_output_stridesr      s    ')C,,(,+A.K!)9)99
k&&q)588==99&++A.&++/0	""#=>>#(()BCCHCI 1 Jrv   c                d   [         R                  (       d  g[        [        R                  [        R
                  [        R                  /5      n[        [        R                  [        R                  [        R                  [        R                  [        R                  [        R                  [        R                  [        R                  [        R                  [        R                   [        R"                  [        R$                  /5      n    SS jn['        U R(                  5       GH  n[+        UR,                  [.        R0                  R2                  R4                  5      (       a  SUR6                  S'   MR  [+        UR,                  [.        R8                  R:                  5      (       aL  [=        UR,                  5      [.        R>                  R@                  RB                  :X  a  SUR6                  S'   M  U" U5      nU(       d  M  Xb;   a  SUR6                  S'   UR6                  RE                  SS5      (       a:  URF                   H*  nU" U5      nU(       d  M  X;  d  M  SUR6                  S'   M,     [         RH                  (       a  GMi  XQ;   d  GMq  SUR6                  S'   GM     g)ar  
Nodes like convolution/convolution_backward want its input to be dense.
If we pad their inputs, we result in extra calls to copy kernels!  On the other hand, padding usually helps reduction.

The pass finds nodes that dislike padding. These are nodes that can be reached
from a convolution/convolution_backward in the backward direction without
going thru a reduction.
Nc                    U R                   S:X  ad  [        U R                  [        R                  R
                  5      (       a1  [        U R                  S5      (       a  U R                  R                  $ S $ )Ncall_function_overloadpacket)r   rx   r   r~   _ops
OpOverloadr   r   )r   s    ru   _get_overload_packet8mark_nodes_dislike_padding.<locals>._get_overload_packet   s[    
 ww/)4;;

(=(=>>%677	 KK''	
 	
rv   Tdislike_paddingF)r   torch.fx.Nodereturnz%Optional[torch._ops.OpOverloadPacket])%r*   comprehensive_paddingr'   atenconvolutionconvolution_backward
_scaled_mmvar_meansummeanprodanyaminamaxminmaxargminargmaxscatter_reducereversednodesrx   r   r~   _higher_order_opstriton_kernel_wrapTritonKernelWrapperMutationr   r   r   r   _CTagneeds_exact_stridesgetall_input_nodespad_outputs)	r   user_visible_output_stridesops_dislike_paddingops_like_paddingr   curr   priorprior_ops	            ru   mark_nodes_dislike_paddingr      s    ''$%%OO	
 "MMHHIIIIHHIIIIHHHHKKKK	
"



	.

  JJ##66RR
 
 +/CHH&' szz5::#8#899)#**5xx||//0 +/CHH&'!#&$*.CHH&'88<<)511,,/6348EJJ01 - !!!c&H*.CHH&'A !rv   c                  t  ^  \ rS rSr% S\S'                    SF                                     SGU 4S jjjrSHS jr    SIS jr    SJS	 jr    SKS
 jr	    SLS jr
      SMS jrSNS jrSOS jr\R                  SPS j5       rSQS jr\SRS j5       rSSS jr        STS jrSUS jrSVS jrSWS jr\SXS j5       r    SYS jrSZS jr    S[S jrS\S jrS]S jrS^U 4S jjr S_S jr!SS.S`S  jjr"SaS! jr#    SbS" jr$SVS# jr%ScS$ jr&      SdS% jr' Se     SfS& jjr(SgS' jr)        ShU 4S( jjr*SiU 4S) jjr+\SjS* j5       r,        SkS+ jr-SlS, jr.SlS- jr/        SmU 4S. jjr0SHS/ jr1\SnS0 j5       r2\SoS1 j5       r3            SpS2 jr4SqU 4S3 jjr5      SrS4 jr6SHS5 jr7    Ss         StS6 jjr8    SuS7 jr9  SvS8 jr:SHS9 jr;SvS: jr<SwS; jr=  SxS< jr>Sr?S=\S>'   SyS? jr@SyS@ jrA    SzSA jrBS{SB jrCS|SC jrDS}SD jrESErFU =rG$ )~GraphLoweringi#  list[ir.IRNode]graph_outputsNFc                
  > [         TU ]  U5        X l        Ub  UOU R                  XS9U l        SU l        Xl        Xl        Xl        Xl	        Xl
        Xl        UU l        SU l        Uc  [        5       nSU l        OSU l        X0l        UR"                  R%                  5       U l        [(        [*        R,                     " 5       U l        [1        U5      U l        / U l        0 U l        0 U l        S U l        [)        5       U l        U(       a  UR>                  O	[)        5       U l        U(       a  UR@                  O	[)        5       U l         SU l!        0 U l"        / U l#        / U l$        U(       a  UO0 U l%        U(       a  [)        URM                  5       5      O	[)        5       U l'        U(       a  URP                  O0 U l(        U(       a  URR                  O0 U l)        U(       a  URT                  O0 U l*        0 U l+        0 U l,        0 U l-        [)        5       U l.        [)        5       U l/        [)        5       U l0        [)        5       U l1        [)        5       U l2        [)        5       U l3        S U l4        S U l5        SSK6J7n  [p        Rr                  " 5       (       a	  U(       a  UOUU l:        S U l;        0 U l<        [)        5       U l=        / U l>        0 U l?        [        [        5      U lB        0 U lC        [        R                  " 5       U lE        UU lF        XPlG        UU lH        XPlI        0 U lJ        X`lK        X@lL        [        [        5      U lO        S U lP        S U lQ        S U lR        S U lS        S U lT        U R                  (       a  U R                  5       O	[)        5       U lV        [)        S/5      U lW        [        UR                  5      U lZ        [        UR                  U R                  5        SU l\        SU l]        / U l^        S U l_        0 U l`        UR                  5       U lb        U R                  RS                  5        H  u  nnUU RR                  U'   M     U R                  RU                  5        H  u  nnUU RT                  U'   M     U R                  R                  R                  S	0 5      U lf        Ub  UR                  O0 U lg        [        5         [        R                  " S 5      " [        5      U lk        0 U ll        [)        5       U lm        [)        5       U ln        [)        5       U lo        [)        5       U lp        [)        5       U lq        [        R                  " 5       U lt        S
U lu        [        5       U lw        0 U lx        g )N)is_inferencer   FTcpu)extern_node_json_serializerzaten.convolution_backward  dynamo_flat_name_to_original_fqn)ysuper__init__example_inputsdecide_layout_opt
layout_optnum_channels_last_convr   is_backwardis_const_graphconst_wrapper_codeconst_kernel_codeconst_moduleinputs_to_checkextra_tracebackr"   reuse_shape_env
_shape_envdeferred_runtime_assertscopyras_by_symbolr'   ry   rz   bound_unbacked_symbolsrT   sizevarsgraph_input_namesgraph_inputsgraph_inputs_originalpartition_mapszero_dim_cpu_tensor_listdevice_typesdevice_idxsdevice_typebuffer_to_padded_sizebuffers
operationsconst_output_indexkeysfolded_constants	constantsnamed_buffersnamed_parameterstorchbind_constantsseen_subgraphsconstant_reprsremoved_operationsremoved_buffersremoved_inplace_buffersmutated_buffersnever_reuse_buffersinplaced_to_remove
device_opswrapper_code&torch._inductor.extern_node_serializerr   r*   	is_fbcodeextern_node_serializercurrent_nodelistsmutated_inputsmutated_input_idxsname_to_bufferr   listname_to_users
name_to_optimecreation_timenamecpp_wrapper
fx_wrapperrecord_multi_kernel_choicemulti_kernel_to_choiceaot_modegraph_idnext_post_grad_graph_counterpost_grad_graph_id	schedulerautotuning_inputsautotuning_mappingautotuning_gridscurrent_devicefind_nodes_prefer_channels_lastnodes_prefer_channels_last_warned_fallbackr   graphr   r   	cache_key
cache_pathcache_linemapdisable_cudagraphs_reasondevice_node_mapping__copy__orig_gmmoduler   r   r   allocated_constant_namer3   	functools	lru_cacher0   effectful_opsunaligned_buffersno_fuse_buffer_nameslow_precision_codegen_opsinvoke_quant_opsall_codegen_kernel_names	itertoolscountworkspace_idplaceholder_idxrX   bw_donated_idxsdep_size_hint_cache)selfgmr   	shape_envr(  r#  r'  r   r  r   r   r   r  r   r   r   r"  r   r$  r   kv	__class__s                         ru   r   GraphLowering.__init__&  s   . 	, % '''F 	
 '(#(&,"4!2(.$ 
I#(D #'D # ..335 	 '1&>&@#(3,.VX=?"AE9C%)5L%%:< 	 )5L$$*, 	 ! <>"(*.0"4" 	
 " )..01 	 '3L"" 	 +7L&&B 	 .:L))r 	
  	  79.03=<0:8B$0:4>L 3=<-126V !!&< #, 	# ,0+-
/9|-/46@KD@Q35!YY[	&$
 +6'68#  "&'?"@>B @DGK:> 7; 7;ooD002:< 	' !+,G+H I+J288+T("288T-M-MN !  	 9=& GI -/[[]LL..0DAq$%Dq! 1LL113DAq'(D!!!$ 4040@0@0D0D.1
- 5A4LL00RT 	$ 	"#$-$7$7$=>R$S!;= 3=,5?\!:D,&1; :D% &OO-  "/1 46 rv   c                8    U R                   R                  5         g rq   )r   freeze_runtime_assertsrL  s    ru   rT  $GraphLowering.freeze_runtime_asserts  s    ..0rv   c                Z   U R                   (       a2  [        UR                  5       5      [        UR                  5       5      4$ SSKJn  U" S[        U R                  R                  5       35      nU R                  R                  UU5      u  nnnU Vs/ s H:  n[        U[        R                  5      (       a  UR                  R                  OUPM<     nnU Vs/ s H:  n[        U[        R                  5      (       a  UR                  R                  OUPM<     n	nX4$ s  snf s  snf )z
Support dynamic shapes and dynamic strides by assigning variables
to each dimension.  We duck-shape tensors, so if two tensors
have the same size they get assigned the same symbolic variable.
r   )ConstantSource__inductor_unknown_tensor_)r   rU   sizestridetorch._dynamo.sourcerX  lenr   
var_to_val,create_symbolic_sizes_strides_storage_offsetrx   r~   SymIntr   expr)
rL  exrX  sourcerZ  r[  _r   r_sizer_strides
             ru   symbolic_sizes_strides$GraphLowering.symbolic_sizes_strides  s    ,RWWY79R		:   < $,S1K1K-L,MNF LL	 NRRTAu||!<!<!&&++!CTROUVv!:a#>#>AFFKKAEvV SVs   AD#AD(c                    UR                  5        Vs/ s H  n[        R                  " U5      PM     nnUR                  5        Vs/ s H  n[        R                  " U5      PM     nnX44$ s  snf s  snf )z
Primarily used to weights
)rZ  ry   r}   r[  )rL  rb  r   rZ  r[  s        ru   static_sizes_strides"GraphLowering.static_sizes_strides  sZ     +-'')4)Qa )4,.IIK8Kq%--"K8| 58s    A, A1c                r   [        U[        R                  5      (       a  UR                  n[        U[        R                  5      (       a  UR                  n[        U[        R
                  5      (       a3  UR                  U R                  ;   a  U R                  UR                     $ UR                  5       $ rq   )	rx   r+   rD   datarC   ComputedBufferr"  r  get_size)rL  r   s     ru   get_allocation_size!GraphLowering.get_allocation_size  s~     dBLL))99DdBMM**99DtR..//		T777--dii88==?"rv   c                    UR                  5       nU R                  U5      nUR                  nUR                  n[	        X4U5      $ rq   )
get_layoutrp  r[  offsetr   )rL  r   layoutrZ  r[  rt  s         ru   get_allocation_storage_size)GraphLowering.get_allocation_storage_size1  s?     "''-.tVDDrv   c                r    [        U[        5      (       d   U5       eX R                  [        U5      5      ;   $ rq   )rx   r-   r0   r=   )rL  r   features      ru   has_featureGraphLowering.has_feature:  s5    
 '>22;G;233OF4KLLLrv   c                    XR                   ;  a6  Sn UR                  5       (       d  UR                  5       nX R                   U'   U R                   U   $ ! [         a     N)f = f)zS
Get the size hint for a dependency with caching to avoid expensive recomputation.
r   )rK  has_unbacked_symbolsnumbytes_hintKeyError)rL  depress      ru   get_dep_size_hintGraphLowering.get_dep_size_hintB  sn     ...C//11++-C -0$$S)'',,   	s   %A 
A"!A"c                B    U R                   =n(       a  U$ [        S5      e)NzNo current device)r0  r   rL  r   s     ru   get_current_device_or_throw)GraphLowering.get_current_device_or_throwS  s$    (((6(M233rv   c              #  \   #    U R                   nXl          S v   X l         g ! X l         f = f7frq   )r0  )rL  r   r   s      ru   set_current_device GraphLowering.set_current_deviceY  s*     ##$	("'%s   ,! ,),c                L    U R                   (       a  gU R                  (       a  gg)N	inferencebackwardforward)r   r   rU  s    ru   get_training_phase GraphLowering.get_training_phaseb  s    rv   c                  [         R                  (       d  g[         R                  (       a  gU R                  R                   Vs/ s HB  o"R
                  [        R                  R                  R                  R                  :X  d  M@  UPMD     nn[        U5      nUS:X  a  g[        R                  R                  R                  (       aE  [        R                  R                  R                  5       (       a  [!        S U 5       5      (       a  g[        [#        U R                  R                  5      5      SU-  :  a  [$        R'                  S5        g[)        S U 5       5      (       a  [$        R'                  S5        gSS	 jnSS
 jnSS jnU(       a  [+        [,        5      nU HQ  n	[/        U	5      n
U
c  M  U" U	5      (       a  SnO"U" U	5      (       a  SnOU" U	5      (       a  SnOSnX==   U
-  ss'   MS     [$        R'                  S5        SnSnSnSn[1        UR3                  5       5      nUS   U-  US   U-  -   US   U-  -   US   U-  -   nUU:*  nU(       d  [$        R'                  SUU5        U$ [)        [5        XS5      5      (       a  [$        R'                  S5        g[)        [5        Xc5      5      (       a  [$        R'                  S5        g[!        [5        Xs5      5      (       a  [$        R'                  S5        ggs  snf )zT
Decide if we should enable layout optimization for this graph based on
heuristics.
FTr   c              3     #    U  HF  nS   H<  nUR                   U   R                  S   R                  R                  [        ;   v   M>     MH     g7f)r   r)   valN)rs   r   r   typer_   r   nr   s      ru   r   2GraphLowering.decide_layout_opt.<locals>.<genexpr>  sJ      #A!C s  '..337OO! P#s   AAi,  z*Skipped layout opt because only a few convc              3     #    U  H4  nS   H*  n[        UR                  U   R                  S   5      v   M,     M6     g7fr  )r   rs   r   r  s      ru   r   r    s?      
 QVVC[--e455 6s   <>zeSee perf regression with dynamic shape. Follow up in https://github.com/pytorch/pytorch/issues/102670c                    U R                   S   R                  S   n[        U[        R                  5      (       d   eU R                   S   S:  =(       a    UR                  S5      S:  $ )Nr)   r  r   )rs   r   rx   r~   r   rZ  )r  meta_vals     ru   
is_grouped3GraphLowering.decide_layout_opt.<locals>.is_grouped  sT    vvay~~e,Hh555566":>:hmmA&6&::rv   c                    U R                   S   R                  S   R                  S5      S-  U R                   S   R                  S   R                  S5      :*  =(       a.    U R                   S   R                  S   R                  S5      S:  $ )Nr)   r  r      rs   r   rZ  r  s    ru   is_in_out_channel:GraphLowering.decide_layout_opt.<locals>.is_in_out_channel  sv    q	u%**1-1QVVAY^^E5J5O5OPQ5RR 6FF1INN5)..q1A5rv   c                    U R                   S   R                  S   R                  S5      S:*  =(       a.    U R                   S   R                  S   R                  S5      S:*  $ )Nr)   r  r   @   r  r  s    ru   is_small_channel9GraphLowering.decide_layout_opt.<locals>.is_small_channel  sT    q	u%**1-3 8FF1INN5)..q1R7rv   groupedsmallin_outdefaultzConv inputs meta not foundg|?5^?gtV?g333333?guV?zhSkipped layout opt in inference because weighted flops indicate slowdown, default: %d, channels last: %dzFSkip layout opt because found grouped convolution with >1 in_channels!zBSkip layout opt because some convolutions have smaller out_channelz>Skip layout opt because all convolution channels are too small)r  r   r   bool)r  r   r   r  )r*   layout_optimizationforce_layout_optimizationr4  r   r   r~   opsr   r   r  r]  backendsmkldnnenabledis_availableallr  logdebugr   r   floatr9   r   valuesmap)rM  r   r  
conv_nodesnconvr  r  r  flop_countsr   counted_flops	node_typeGROUPED_MULTIPLIERDEFAULT_MULTIPLIERIN_OUT_MULTIPLIERSMALL_MULTIPLIERtotal_flopsweighted_flopsdo_layout_opts                      ru   r   GraphLowering.decide_layout_opti  s    ))++ xx~~
%!UYY^^5O5O5W5W)WA~ 	 
 JA: NN!!))%%2244 #   
 tBHHNN#$e3IIBC 

 
 

 IIw 	;
		 ,7,>K" .t 4 (d## )I%d++ 'I&t,, (I )I&-7& #  		67
 "'!& %$k0023K I&);;g&)99:h'*;;< i(+==>  +k9M 		~"
 ! & s:*++IIX  s$122IIT  s#011IIVW]
s   ?L
L
c                B    U R                   b  U R                    SU 3$ U$ )z2Prepend the given name with the graph name if any.rd  )r"  rL  r"  s     ru   qualify_nameGraphLowering.qualify_name  s&    99 ii[$((rv   c                    [        U UUU R                  U R                  U R                  U R                  U R
                  U R                  U R                  U5      S9
$ )a  
Make a subgraph of the current graph with all inherited parts, except
the graph module (`gm`) and `example_inputs`.  The subgraphs are lowered
separately and lifted into a separate function in the parent output
wrapper code.  The subgraph name is qualified by the parent graph's
name. Note that the lifting of subgraph is supported for python wrapper
only. For cpp wrapper, we inline the subgraphs in the parent wrapper.
)
parentrM  r   rN  r#  r'  r  r   r   r"  )SubgraphLoweringr   r#  r'  r  r   r   r  )rL  rM  r   subgraph_names       ru   make_subgraphGraphLowering.make_subgraph  sZ      )oo((]]#'#>#>**((""=1
 	
rv   c                   [         [           " 5       n[        U R                  R                  R
                  5       H~  nUR                  [        R                  R                  R                  R                  :X  a  UR                  U5        MR  UR                   H  nX1;   d  M
  UR                  U5          M|     M     U R                  R                  R
                   H%  nX!;   d  M
  UR                  UR                  5        M'     U$ )a  
The rule to decide if an node prefer channels last is simple.
1. if it's input/output of a convolution
2. if one of its user prefers channels last

We have rule 1 because cudnn runs a faster convolution kernel for channels last inputs;
Rule 2 is also important. It makes sure that indirect inputs to convolution also prefers
channels last.

Consider the scenario: conv -> batch-norm -> relu -> conv
Without rule 2, batch-norm output may use a contiguous layout. That will cause 2 extra copies:
1. the output of batch-norm should be channels last initially since its input is a conv's output.
   Forcing the batch-norm's output to be contiguous results in the first copy
2. The second conv's input is initially contiguous. This layout is propagated from the batch-norm's output.
   We need convert it to channels last layout which results in the second copy.
With rule 2, we makes sure all the tensors in the chain uses channels last layout. So both copies
can be saved.
)r'   r%   r   r<  r4  r   r   r~   r  r   r   r  addusersupdate)rL  
output_setr  users       ru   r1  -GraphLowering.find_nodes_prefer_channels_last&  s    &  %'
$++++112Axx599>>55===q!%NN1%   30 ""((A!!!''* ) rv   c                    XR                   ;  a2  U R                   R                  U5        [        R                  SU5        g g )NzUsing FallbackKernel: %s)r3  r  perf_hint_loginfor  s     ru   warn_fallbackGraphLowering.warn_fallbackX  s8    ,,,!!%%d+94@ -rv   c                `   U R                   R                  UR                  5        UR                  b%  U R                  R                  UR                  5        [
        R                  R                  (       a8  XR                  ;  a(  [
        R                  R                  U R                  U'   g g g rq   )	r   r  r  indexr   rb   r4  r  r9  r  s     ru   add_device_infoGraphLowering.add_device_info]  sw    fkk*<<#  .77F2J2J$J/0ww/C/CD$$V, %Krv   c                "    [         R                  $ rq   )rb   	fake_moderU  s    ru   r  GraphLowering.fake_moded  s    {{rv   c           	        XR                   ;   a  U R                   U   $ XR                  ;   a  U R                  U   $ XR                  ;   ay  [        R                  R                  U   n[
        R                  " U[
        R                  " UR                  UR                  /[        R                  R                  U5      Q76 S9$ g Nr"  ru  )r  r   r  rb   r4  r+   ConstantBufferr<   r   dtyperj  )rL  buffer_namerm  s      ru   try_get_bufferGraphLowering.try_get_bufferh  s     ---&&{33+++$$[11..(77$$[1D$$ ~~KK./gg.J.J4.P  rv   c                    [        S5      e)Nz'Should not be called for the main graph)r   )rL  symbols     ru   add_symbol_graph_input$GraphLowering.add_symbol_graph_inputz  s    DEErv   c                J    U R                  U5      nUb  U$ [        SU 35      e)Nz$Failed to find buffer matching name )r  r   rL  r  bufs      ru   
get_bufferGraphLowering.get_buffer}  s1     !!+.?JA+OPPrv   c                   XR                   ;   a  U R                   U   R                  $ [        U R                  S5      (       a  XR                  R                  ;   aq  U R                  R                  U   nX R
                  ;   a  U R
                  U   R                  5       $ X R                  ;   a  U R                  U   R                  5       $ XR
                  ;   a  U R
                  U   R                  5       $ XR                  ;   a  U R                  U   R                  5       $ [        R                  " SU5      nU(       a   U R                  UR                  S5      5      $ [        SU 35      e)Nmutation_real_namez1(as_strided|reinterpret_tensor)\(([a-zA-Z0-9_]+),r)   could not find )r  r  r   r,  r  r  	get_dtyper   rematchgroupr  )rL  r  mutated_bufr   s       ru   r  GraphLowering.get_dtype  s.   ..(>>+.444 DNN$899~~@@@..;;KHK111**;7AACC///((5??AA---&&{3==??+++$$[1;;==HHI;W>>!''!*--677rv   c                V   XR                   ;   a  U R                   U   R                  5       $ XR                  ;   a5  U R                  U   nUR                  5       (       d  gUR	                  5       $ XR
                  ;   a  U R
                  U   R	                  5       $ [        SU 35      e)Nr)   r  )r  numelr  has_tensor_output	get_numelr   r  r  s      ru   r  GraphLowering.get_numel  s    ..(>>+.4466---%%k2C((**==?"+++$$[1;;==677rv   c                j   > [        S5         [        TU ]  " U6 sS S S 5        $ ! , (       d  f       g = f)NzGraphLowering.run)r   r   run)rL  rs   rQ  s     ru   r  GraphLowering.run  s$    -.7;% /..s   $
2c                "   UR                   b
   SU 35       e[        U[        R                  5      (       d   eU R	                  S[        U R                  5       35      nU R                  R                  U5        XR                  U'   X!l         U$ )NzOperation registered twice: r   )	operation_namerx   r+   	Operationr  r]  r  appendr  )rL  r   r"  s      ru   register_operation GraphLowering.register_operation  s      (M,H*MM("bll++++  2c$//&:%;!<=r" " rv   set_namec                  U R                  S[        U R                  5       35      nU R                  R                  U5        XR                  U'   UR                  5       nUb_  [        U[        R                  5      (       a/  UR                  5       (       a  U[        R                  " S5      :X  d  U R                  U5        U(       a  X1l        U$ )Nr  r   )r  r]  r  r	  r  
get_devicerx   r+   rn  is_zero_elementsr~   r   r  r"  )rL  bufferr  r"  r   s        ru   register_bufferGraphLowering.register_buffer  s      3s4<<'8&9!:;F#$*D!""$ 62#4#455++--ell511   (Krv   c                h    U R                  SSR                  U5      -   5      nXR                  U'   U$ )Nlist_rd  )r  r   r  )rL  operation_namesr"  s      ru   register_operation_list%GraphLowering.register_operation_list  s1      388O+D!DE*

4rv   c                *   ^ ^ SUU 4S jjmT" U5        g )Nc                  > [        U [        [        45      (       a  U  H  nT" U5        M     [        U [        R                  5      (       a6  U R                  5        H!  nTR                  U   R                  U 5        M#     g g rq   )rx   r  tupler+   rD   get_read_namesr  r	  )valuex	read_nameregisterrL  s      ru   r   1GraphLowering.register_users_of.<locals>.register  sg    %$//AQK %..!&!5!5!7I&&y188? "8 /rv   )r  %Union[Iterable[ir.IRNode], ir.IRNode]r   Nonerr   )rL  node_outputr   s   ` @ru   register_users_ofGraphLowering.register_users_of  s    	@ 	@ 	rv   c                    [        U[        5      (       d   eU R                  R                  U5        XR                  ;  a  gU R                  U    H  nUR                  5         M     g)zz
When a buffer is mutated we need to make sure all the reads to
the old version are realized before the mutation happens.
N)rx   strr  r  r  realize)rL  r"  r  s      ru   mark_buffer_mutated!GraphLowering.mark_buffer_mutated  sX    
 $$$$$  &)))&&t,DLLN -rv   c                   XR                   ;   a  XR                  ;   d
   SU-   5       e[        U R                   U   5      nX R                  R                  ;   a  U R                  R                  U   $ U R                  U   $ )z
In AOTI, module buffers may have been mutated during the tracing and compilation.
Thus we need to read from previously stored original buffers, to make sure the
generated model.so uses correct initial values.
z$Can not find the original value for )r=  r  rW   r<  r   )rL  r"  	orig_names      ru   get_original_value_of_constant,GraphLowering.get_original_value_of_constant  s     3338N 	
2T9	
N 5T5Q5QRV5WX	 KK,,, KKY'	
 %	
rv   c                   [         R                  R                  (       d7  U R                  R	                  5        H  u  p4[        X$5      (       d  M  Us  $    Uc  S[        U R                  5       3nUnUS   R                  5       (       a  SU 3nU R                  U5      n[        U5      nUnSnXR                  ;   a  U SU 3nUS-  nXR                  ;   a  M  X R                  U'   UR                  < SUR                  < S[        UR                  5       5      < S[        UR                  5       5      < S[        U5      S 3	U R                   U'   XPR"                  U'   U$ )Nconstantr   	constant_rd  r)    r  )r*   aot_inductoruse_runtime_constant_foldingr  itemsr[   r]  isdigitr  r]   r   r  r  rZ  r[  hashr  r=  )rL  r"  rm  constant_namer  r-  prefixcnts           ru   allocate_non_dup_const_name)GraphLowering.allocate_non_dup_const_name  sO    ""??(,(<(<(>$!$..(( )? <c$..123D	7??tf%D  &  %nn$XQse$D1HC nn$  $t{{oQtzznATYY[!$AeDKKM&:%=QDz!n 	D!
 .7$$T*rv   c                    U R                  X!5      n[        R                  " [        R                  " U[        UR                  UR                  /U R                  U5      Q76 S95      $ r  )	r<  rD   creater+   r  r<   r   r  rj  )rL  rm  r"  new_names       ru   add_tensor_constant!GraphLowering.add_tensor_constant  s`     33D?"KK.2.G.G.M
 	
rv   c                p   U R                   U   R                  U:X  d  Uc  U$ [        R                  R                  R                  5          U R                  U SUR                   UR                  =(       d    S 3U R                   U   R                  U5      5      sSSS5        $ ! , (       d  f       g= f)z
We AOT copy constants to the devices they are needed on.
If device_override doesn't match the constant's device, then
copy it and return a different name.
Nrd  r   )
r  r   r~   utils_python_dispatch_disable_current_modesr<  r  r  to)rL  r"  device_overrides      ru   r9  GraphLowering.constant_name$  s     >>$&&/9_=TK[[))@@B 33&/../0E0E0J/KLt$''8 CBBs   AB''
B5c                	  > U =R                   S-  sl         [        TU ]	  XU5      nU R                  U5      n[	        U[
        5      (       av  [        R                  R                  (       d  [        UR                  5      nOUR                  R                  nXPR                  U'   U R                  R                  U5        U$ [	        U[        [         ["        45      (       aA  [$        R&                  " U5      nXPR                  U'   U R                  R                  U5        U$ [	        U[(        5      (       a4  [+        XS9nX`R                  U'   U R                  R                  U5        U$ Uc  U R                  R                  U5        g [	        U[,        5      (       a  U R                  R                  U5        g [	        U[.        R0                  5      (       Ga  [3        [        R                  R4                  R6                  5      S:X  a  [9        [;        [        R                  R4                  R6                  5      5      R<                  [.        R>                  R@                  RB                  [.        RD                  RF                  RH                  4;   d   e[J        RL                  " XRN                  S9nXpR                  U'   U R                  R                  U5        U$ [	        U[.        RP                  5      (       d   U5       eURR                  (       d  U RU                  U5      u  pOU RW                  U5      u  pU R                  (       ai  U RX                  (       aX  U R                   U RX                  ;   a>  [Z        R\                  " [_        U[a        URN                  URb                  X5      S95      n
O=[Z        R\                  " [e        U[a        URN                  URb                  X5      S95      n
XR                  U'   U R                  R                  U5        U
Rf                  Rf                  U Rh                  U'   U R4                  R6                  (       a  U Rk                  URN                  5        [m        5          [o        U5      (       d  U Rp                  Rs                  U5        S S S 5        U
$ ! , (       d  f       U
$ = f)Nr)   r"  r  )r"  r   r  ):rI  r   placeholderr  rx   r$   rb   r4  r   r   r   ra  r   r   r	  intr  r  ry   sympifyr   rE   r   r~   	Generatorr]  r  r  r)  iterr   _prims	rng_primsgraphsafe_run_with_rng_stater  higher_orderinvoke_subgraphr+   GeneratorStater   r   _has_symbolic_sizes_stridesrj  rg  rJ  rD   r?  r;   r<   r  r?   rm  r   r  r\   r^   rA  r  )rL  r   rs   rt   examplera  r   gensizesstridestensorrQ  s              ru   rL  GraphLowering.placeholder4  s    	!'%fF;""6*gx(( 77&&,W\\:||(((,f%""))&1K#tU!344==)D(,f%""))&1K!122!v=C(+f%""))&1J_""))&1g}-- ""))&111qww++112a7DQWW))//0=f&&CC		&&66=   ##GC(+f%""))&1J'5<<009'90
 22!66w?NE7!88ANE $$$$(<(<<%%&w~~w}}eUF %%&w~~w}}eUF %+&!%%f--3[[-=-=""6*""  0 12.w77&&**62 3  32 s   ,R::
S	c                `  >^ T[         R                  L a3  [        US   [        [        [
        45      (       a  [        TU ]  TX#5      $ [        T[        R                  R                  5      (       d  [        TS5      (       a  T" U0 UD6$ T[        ;  Ga_  [        T[        R                  R                  5      (       d
   T S35       eTR                  5       R                  S5      S   nU[         ;   a  [#        TSSS9  O[$        R&                  (       a  [)        T/5      (       a  [*        O[,        n[.        R1                  SUR3                  TX#5      5        [5        TSS	9nUcF  [        R6                  R8                  R;                  T5      (       a  U R<                  (       a  [>        nO[5        TSS	9n[A        U5      n[#        TUS
9  O)[)        T/5      (       a  [+        TX#5      e[-        TX#5      e [.        RC                  S[        T   5        U RD                  n	[G        T5      n
U
(       a  X#pU
[H        L a  SU	RJ                  ;   ap  U	RJ                  S   u  p[        T[        R                  R                  5      (       d   eSU4S jjnU" X5      u  pU" X#5      u  p#U" X5      u  p[I        X#X5      u  p#OU
" U	/UQ70 UD6u  p#[        T   " U0 UD6nU
(       a  U RM                  U	WWX#5        U$ ! [N         a,  n[Q        UTX#5      RS                  URT                  5      S eS nAff = f)Nr   _inductor_lowering_functionz is not an OpOverloadr   FT)warnoverride_decompz"Creating implicit fallback for:
%s)with_default)layout_constraintz  via %seager_input_valsc                v   > [         R                  R                  R                  TX5      nUc   eUS   US   4$ )Nr   r)   )r~   r   operator_schemasnormalize_function)rs   rt   resultr   s      ru   	normalize.GraphLowering.call_function.<locals>.normalize  sC    %*XX%>%>%Q%Q &&F $*#55#5#)!9fQi#77rv   )rs   r   rt   r   r   ztuple[Any, Any])+operatorgetitemrx   r  r  dictr   r   r~   r   OpOverloadPacketr   rK   r   r"  r   rH   rL   r*   implicit_fallbacksr   r7   r8   r  r  operator_strr   _libraryrD  
is_builtinr   rO   rP   r  r  rM   rF   r   propagate_mutation	Exceptionr6   with_traceback__traceback__)rL  r   rs   rt   	base_nameerrortagdecided_constraintdefault_tagr  layout_constraintsold_args
old_kwargs	fake_argsfake_kwargsri  outerQ  s    `                ru   r   GraphLowering.call_function  s   X%%%*T!WtUD>Q*R*R7(>> &%**"="=>>71D
 D
 4*6**"fejj&;&;<< (/0< ++C03I//f5$G** *6(33 .5 
 9&&vt<
 /H/ K,,77??(( *<&0IT1K *B+)N&f8JK#VH-- 0EE264HH0	IIj)F"34!!A!9&!A!'+*%)BB
 *QVV3128J1K.	  *&%**2G2GHHHH8 2;91R.	'0'>/8/N,'@ )( $6a#I$#I&#ILDF#T4V4C! ''8ZNJ 	#Avt<KK	s   6D K7 7
L-'L((L-c                d    [        U R                  5      S:H  =(       a    U R                  S   S:*  $ )z=
True if this is a small constant attr that will be inlined.
r)   r      )r]  shape)ts    ru   can_inline_constant!GraphLowering.can_inline_constant  s(    
 177|q 4QWWQZ1_4rv   c                   [        U R                  U5      n[        U[        R                  R
                  5      (       aB  XR                  ;   a  U R                  U   $ [        R                  " XS9nXPR                  U'   U$ [        U[        R                  R                  5      (       a&  X@R                  U'   SU R                  U'   [        XS9$ [        U[        5      (       a&  X@R                  U'   SU R                  U'   [        XS9$ [        U[        R                  5      (       d   e[         R"                  R$                  (       d%  [         R&                  (       d  [)        U5      (       a  U R+                  XA5      $ [-        5          UR.                  S:X  a6  [1        UR3                  5       UR4                  UR6                  S9sS S S 5        $ U R9                  U5      (       aX  [:        R=                  S[?        U5      5        SSK J!n  U" URE                  5       UR4                  UR6                  S	9sS S S 5        $  S S S 5        U R+                  XA5      $ ! , (       d  f       N= f)
N)r"  graph_moduler   rK  rr   )r  r  r   zInlining constant: %s r)   )r\  )r  r   )#r   r<  rx   r~   r   rh   r  r+   Subgraphr   ScriptObjectr
  r  rE   r   r   r*   r4  r5  always_keep_tensor_constantsrQ   rA  r&   r  r:   itemr  r   r  r  r  r(  loweringr\  tolist)rL  r   rs   rt   r  r  r\  s          ru   get_attrGraphLowering.get_attr  s    "$++v6eUXX1122,,,**622++6>C*-'JeUXX2233/4$$V,*,D'"<</00/4$$V,*,D'"<<%....<<22(//++E::]{{b **,ekk%,, ]
 ''..		2CK@,ellnEKKU ]
 /  ''66 ]s   =IA$I
I)c                    [         erq   AssertionErrorrL  r   rs   rt   s       ru   call_moduleGraphLowering.call_moduleG      rv   c                    [         erq   r  r  s       ru   call_methodGraphLowering.call_methodJ  r  rv   c                	  > [         TU ]  XU5      n[        U[        [        45      (       d  U4n[        U[        [        45      (       d   [        U5      5       e[        S U 5       5      (       d   U5       e[        R                  R                  R                  S   n[        U[        [        45      (       d  U4nU Vs/ s H"  n[        R                  R                  U5      PM$     nn/ n[        U5      [        U5      :X  d   e[        XE5       GH\  u  p[        U[        R                   [        R"                  45      (       d  UR%                  U5        MH  [        UR'                  5       [        R(                  5      (       a0  UR%                  [        R                  R+                  U5      5        M  [,        R.                  R                  R1                  U5      (       d   eU	R2                  S   R5                  5        V
s/ s H:  n
[        U
[,        R6                  5      (       a  U
R8                  R:                  OU
PM<     nn
UR%                  [        R<                  " X5      5        GM_     Xpl        U R@                  RC                  5        GH  u  p[        U[D        5      (       a  M  [        U[         [F        RH                  [,        R.                  R                  RJ                  45      (       d   S[        U5       35       e[        U[         5      (       d  M  URM                  5         [        U[         5      (       d   eURN                  n[        U[        RP                  5      (       d   eUnURN                  n[        U[R        5      (       a  URU                  5       U:w  d  GM  [        RV                  RY                  XRZ                  U   5         U R>                  R]                  U5      nU RZ                  U   U R>                  U'   GM     U Ra                  5         [b        Re                  SU Rf                  U Rh                  b  U Rh                  5        g S5        g s  snf s  sn
f ! [^         a     GM  f = f)Nc              3  >  #    U  H  n[        U[        [        R                  [	        S 5      [        R
                  [        R                  [        R                  R                  R                  [        [        R                  [        R                  4	5      v   M     g 7frq   )rx   rD   r+   r:   r  r  ry   r   logicboolalgBooleanrM  EffectfulKernelrB   )r   r  s     ru   r   'GraphLowering.output.<locals>.<genexpr>X  sv      
  KKJ%%JJKK''//&&,,
  s   BBr   r  z'Unsupported inductor graph input type: zGForce channels last inputs for %d conv for the current graph with id %dr   )5r   r   rx   r  r  r  r  rb   r4  r  rs   r+   ExternKernelrealize_inputr]  ziprD   BaseViewr	  get_output_specCommBufferLayout
copy_inputr~   	_inductoris_storage_and_layoutr   r[  r`  r   ra  try_match_insignificant_stridesr   r   r6  rE   ry   r   rV  r)  rm  rC   r?   get_nameMutationLayoutSHOULDREMOVErealize_intor   r  
ValueErrorfinalizer  r  r   r(  )rL  r   rs   rt   rh  fx_node_argsr  result_correct_stridesrfx_nodesmeta_stridesr"  r  value_storage_boxindrQ  s                   ru   r   GraphLowering.outputM  s    f5&5$-00YF&5$-00>$v,>0 
 
 
 
 	  !	 
$ ww++003,66(?L<BCFq"////2FC!#< CK///f3JAa",,!<==&--a0A--/1D1DEE '--boo.H.H.KL ))??BBBB %\\%0779 9 $.a#>#>AFFKKAE9    '--66qG# 4* 4,,224KD%11	5::u/A/A/P/PQ  G8eFG  eY//MMOeY////JJEeR]]3333 %JJEe[11U^^5E5M--::55d;,,223DEC.2.H.H.ND&&s+- 54 			U''!]]6DMM	
 =?	
m D" B " s   )Q+AQ07Q55
RRc                J    U R                    H  nUR                  5         M     g rq   )r  decide_layout)rL  r  s     ru   r  GraphLowering.finalize  s    <<C  rv   c              #  \   #    U R                   n Xl         S v   X l         g ! X l         f = f7frq   )r  )rL  r   olds      ru   set_current_nodeGraphLowering.set_current_node  s*     	$ $ #s   ,
! ,),c              #  P   #    U R                   n S v   Xl         g ! Xl         f = f7frq   r  )rL  r  s     ru   set_current_wrapper_code&GraphLowering.set_current_wrapper_code  s$     	$ #s   & &#&c                  ^  [        U5      [        U5      :X  d   e[        U5      [        U5      :X  d   eUR                  [        R                  R                  R
                  L Ga  UR                  S   n[        U[        5      (       d   e[        R                  R                  R                  US   US   UR                  5        VVs0 s H@  u  pxU[        U[        R                  R                  5      (       a  UR                  S   OU_MB     snnUS   5      n	U	 HY  n
US   U
   nUS   U
   nXL a  M  T R!                  [        R                  R"                  R$                  R&                  X40 5        M[     g[        UR                  [        R(                  R*                  5      (       d   e        SU 4S jjnUR                  R,                  n[/        [1        X$5      5       H   u  nu  pUR2                  U   nU" UX5        M"     UR2                   Vs0 s H  nUR4                  U_M     nnUR7                  5        H  nUU   nUU   nUU   nU" UX5        M     gs  snnf s  snf )	aP  Propagate mutations on new_args/new_kwargs back to old_args/old_kwargs.

Assumes we may have cloned old_args/old_kwargs into new_args/new_kwargs
and then called fx_node(*new_args, **new_kwargs).

If fx_node mutates any of new_args/new_kwargs, and they are different from
old_args/old_kwargs, then we need to update the original tensor.
rt   
kernel_idxconstant_args_idxr  tma_descriptor_metadataNc                b  > XL a  g U R                   b  U R                   R                  (       a  [        U[        R                  5      (       a  U4nU4n[        X5       HK  u  p4X4L a  M  TR                  [        R                  R                  R                  R                  X440 5        MM     g g g rq   )
alias_infois_writerx   r+   IRNoder  r   r~   r  r   copy_r  )
schema_argold_argnew_argold_arg_itemnew_arg_itemrL  s        ru   maybe_propagate9GraphLowering.propagate_mutation.<locals>.maybe_propagate  s     !$$0Z5J5J5S5S gryy11&jG&jG25g2G.L#3 &&		,,44|6RTV 3H 6T0rv   )r  ztorch._C.Argumentr  	ir.IRNoder  r  r   r#  )r]  r   r~   r  rT  triton_kernel_wrapper_mutationrt   rx   rm  r   r   get_mutated_tensorsr6  r   r%   r   r   r   r  r  r   r   _schemar   r  	argumentsr"  r  )rL  r  r}  r~  new_args
new_kwargsrt   rO  rP  mutatedr"  r  r  r  schemar   r  argschema_kwargskeys   `                   ru   rs   GraphLowering.propagate_mutation  sH     8}H---:#j/111>>UYY33RRR^^H-Ffd++++--@@TT<(./ !' . 
1ehhmm(D(Dqvve}!K . 45G  $X.t4$X.t4%""599>>#7#7#?#?'ASUWX   '..%***?*?@@@@	)	4=	HQ		( '''0X1H'I#C#'))#.JJ9 (J 392B2BC2B332BC??$C oG oG&s+JJ9	 %YT Ds   AI) I/c                 &  >^ ^^$^%^& S&U4S jjnSSK Jn  [        T R                  5      m%[        T R                  5      m&[        T/5      nTR                  S:H  nU(       a!  T R                  T5      u  pgU[        Xg5      -  n[        R                  R                  U5         T R                  T5         [        R                  " T5         TR                  S:X  a  TR                  (       a  [        TR                  [         R"                  R$                  5      (       a  [         R&                  R(                  R+                  TR                  5      (       aP  [-        T5      (       d  UR/                  SSU4S j5      (       a$  U" S5        [1        TR                  S	S
9" W0 WD6nGOTR                  S:X  a  TR                  [         R2                  R4                  R6                  L a  [8        R:                  S:w  a  U" S5        [8        R:                  S:X  a  Wn	Wn
TR<                  R?                  S5      =n(       a  US   nUS   n[A        XgX5      u  pgO[C        T/UQ70 UD6u  pgT RE                  TR                  Xg5      nT RG                  TXXg5        O[I        S[8        R:                   35      e[K        TR                  5      (       a  U" S5        [        TR<                  S   [         RL                  [         RN                  [         RP                  45      (       a$  TR<                  S   RR                  RT                  nO'[V        T'T ]  T5      nOU" S5        [V        T'T ]  T5      n[         R2                  RZ                  R\                  R^                  [         R2                  RZ                  R`                  R^                  [         R2                  RZ                  Rb                  R^                  [         R2                  RZ                  Rd                  R^                  [         R2                  RZ                  Rf                  R^                  /m$[i        S TRj                   5       5      nTT Rl                  ;   n[i        U$4S jTRj                   5       5      nTR<                  R?                  SS	5      (       a  [        U[n        5      (       a  URq                  5         TR<                  S   Rs                  5       n[         Rt                  R(                  Rv                  " U6 nURy                  5       U:w  a=  U(       d6  [        Rz                  " U5      n[        R|                  R                  UU5      nU(       aN  [        U[n        5      (       a9  [        UR                  [        R                  5      (       a  URq                  5         U(       d  U(       Gaq  [        TR<                  S   [         R                  5      (       GaD  U(       a  T Rl                  R?                  T5      nOTR<                  S   Rs                  5       nUGb   [        U5      S:  Ga  [8        R                  =(       d    U(       + =(       a    U(       + n[         R                  R                  TR<                  S   5      n[        [        U5      5      S:  nU(       d~  U(       aw  [        UR                  5       5      S:X  aZ  TT R                  ;   aJ  U(       dC  U(       d<  [        R                  R                  UR                  5       [         R                  5      nU(       d  [        U5      (       a  TR<                  S   R                  5       (       d)  [        UR                  [        R                  5      (       a4  [        R|                  R                  U[        Rz                  " U5      US9nOfU Vs/ s H:  n[        U[         RL                  5      (       a  URR                  RT                  OUPM<     nn[        R|                  R                  UUUS9n[        [        TRj                  5      5      nUS:  Ga:  [        U[n        5      (       Ga$  TRj                   GH  nUR                  [        ;   Ga  UR                  5         [         R2                  RZ                  R                  R^                  [         R2                  RZ                  R                  R^                  [         R2                  RZ                  R                  R^                  /n/ nT R                  (       d=  UR                  [         R2                  RZ                  R                  R^                  5        [         R                  R                  (       Ga  U[         R2                  R                  R                  R^                  [         R2                  R                  R                  R                  [         R2                  RZ                  R                  R^                  [         R2                  R                  R                  R^                  [         R2                  R                  R                  R                  [         R2                  R                  R                  R                  [         R2                  R                  R                  R                  /-  nU[         R2                  R                  R                  R^                  [         R2                  R                  R                  R                  [         R2                  R                  R                  R                  [         R2                  R                  R                  R^                  [         R2                  R                  R                  R^                  [         R2                  R                  R                  R                  /-  n[         R                  R                  (       a2  U[         R2                  R                  R                  R^                  /-  nUR                  U;   aN  [        R|                  R                  U[        Rz                  " TR<                  S   Rs                  5       5      SS9nUR                  U;   af  TUR                  S   L aT  [        R|                  R                  U[        Rz                  " [        TR<                  S   R                  5      5      5      nUR                  S:X  d  GM  [        UR                  R                  [        [        45      (       d  GM  URq                  5         GM     UR                  [        TRj                  5      5        [        U[n        5      (       a%  UR                  5       (       a  UR                  5         [        U[n        5      (       an  [        UR                  [        5      (       aO  UR                  R                  n[        U[        5      (       a$  UR                  SS9(       a  URq                  5         S S S 5        S S S 5        S S S 5        [        W[n        5      (       GaK  [        UR                  [        R                  5      (       Ga!  [        UR                  R                  [        R                  5      (       a(  UR                  R                  R                  ST5        GO[        UR                  R                  [        R                  5      (       Ga  UR                  R                  R                  ST5        [        UR                  R                  [        R                  5      (       an  [        UR                  R                  R                  [        R                  5      (       a1  UR                  R                  R                  R                  ST5        O[        UR                  R                  [        R                  5      (       a  UR                  R                  R                  (       ds  [        UR                  R                  R                  S   [        R                  5      (       a3  UR                  R                  R                  S   R                  ST5        T R                  U5        [
        [        R                     " 5       nT R                  T%S   H  nUUR                  5       -  nM     T R                  T&S   H  nUUR                  5       -  nM     [        R                  R                  R                  n TR                  S:X  aG  [        U[        R                  5      (       a(  U R                  U5      (       a  UGR                  U5        S'U%U&U 4S jjn![        R                  GR                  (       a  TR                  S:X  a  U$ G[        [        R                  R                  R                  TR<                  R?                  S 0 5      5      n"U"c   e[        S! U"GR                  5        5       5      n#UU#:  d'   S"U S#U# S$TGR	                  5        S%U!" 5        35       eT GR                  TU5        U$ s  snf ! , (       d  f       GNl= f! , (       d  f       GNv= f! , (       d  f       GN= f)(Nc                Z   > [         R                  S[        TR                  5      U 5        g )Nzlowering %s %s)r  r  r   format_node)msgr  s    ru   r  %GraphLowering.run_node.<locals>.debug  s    II&
1==(A3Grv   r   )CompilerBisectorr   inductorrK   c                    > [        T 5      $ rq   )reprr  s   ru   <lambda>(GraphLowering.run_node.<locals>.<lambda>'  s	    arv   rI   F)add_to_fallback_setflexible_layout-user_defined_triton_kernel_layout_constraintsneeds_fixed_stride_orderrd  r)   z1Unknown triton_kernel_default_layout_constraint: r   r  r   c              3  >   #    U  H  oR                   S :H  v   M     g7f)r   Nr   )r   r  s     ru   r   )GraphLowering.run_node.<locals>.<genexpr>h  s     DGDGGx/Gs   c              3  @   >#    U  H  oR                   T;   v   M     g 7frq   )r   )r   r  as_strided_opss     ru   r   r  j  s      *:A$~-'   inductor_realize_to_strides   )allow_paddingTr   d   )	thresholdorigin_noderL  c                    > TR                   TS   V s/ s H  n SU R                  5        SU  S3PM     nn UR                  S TR                  TS   5       5        SR	                  U5      $ s  sn f )Nunbacked_symbol_defs= in:

c              3  P   #    U  H  nS UR                  5        SU S3v   M     g7f)r   r  r  N)get_unbacked_symbol_defs)r   r   s     ru   r   BGraphLowering.run_node.<locals>.format_new_defs.<locals>.<genexpr>6  s1      ?B ((C(C(E'FfRDPRS?s   $&z***
)r  r  extendr  r   )r  r  buffer_watermarkoperation_watermarkrL  s     ru   format_new_defs/GraphLowering.run_node.<locals>.format_new_defs1  s      <<(8(9::C ((D(D(F'GvcURTU:   HH //*=*>?  <<?"s   !A,unbacked_bindingsc              3     #    U  H8  n[         R                  R                  R                  R	                  X5      v   M:     g 7frq   )rb   r  rN  unbacked_renamingsr   )r   r  s     ru   r   r  g  s5      /
- KK!!4488>>-s   A Azfailed  >= z (inductor >= fx)
fx node is: z
new operations are:

)r  r(  r   r#  r   r(  )!torch._inductor.compiler_bisectorr  r]  r  r  r'   r   fetch_args_kwargs_from_envrV   r+   r  current_originsr  rb   r   rx   r~   r   r   rq  rD  rr  rJ   disable_subsystemrI   r  rT  r  r*   'triton_kernel_default_layout_constraintr   r   rF   rG   r   rs  r   r   r`  SymFloatSymBoolr   ra  r   run_noder   
as_stridedr  as_strided_as_strided_scatterresize	resize_asr   r  r   rD   r)  r[  r  any_is_symbolicmaybe_get_strideget_stride_orderr  require_stride_orderrm  r  r   r   _prims_commonis_non_overlapping_and_denser   ro  r2  FlexibleLayout stride_ordered_for_memory_formatchannels_last_is_viewrequire_exact_stridesrN   realize_hintr   mm_int_mmr   r	  r   r   _has_mkldnnr  _linear_pointwisebinarymkldnn_rnn_layeronednnqlinear_pointwiser\  binary_tensor_convolution_pointwise_convolution_pointwise_ _convolution_transpose_pointwiseqconv_pointwiseqconv2d_pointwisehas_mklmkl_mkl_linearrs   r   r  r@   rA   
mark_reusehas_exceeded_max_readsrC   has_large_inner_fnLoops_post_init_setattrBufferrn  MultiOutputindicesinputsr%  ry   rz   r  r4  r   rN  is_unbacked_symintr  r   r    r  r  create_deferred_runtime_asserts)(rL  r  r  r  originsis_call_functionrs   rt   rh  r}  r~  rd  inp_args
inp_kwargs	is_outputis_user_visibleis_input_for_as_stridedr[  sym_stridesstride_orderr  denseunbacked_symbols_in_stridesr  	num_usersr  need_fixed_layoutneed_fixed_channels_last_layoutcurrnew_unbacked_defsr  r   rN  r	  r  renamed_unbacked_bindingsr  r  r  rQ  s(   ``                                  @@@ru   r  GraphLowering.run_node
  s   	H 	Gt||,!$//2 $.qc?44?2::1=LD~d33GII%%g.!!!$q! 'HHqxx)>)>??NN((33AHH==9!<<'99"K 
 ())!((N
 'HH		 6 6 U UUBBFWWEFBB12  $H!'J+,66::6H+II'I#3A#6%5a%8
'@ ((f (?q'R4'R6'R!//$GF++AxTR&KFLzLzK{|  !** '(FF5MELL%..%--#P  VVE]//44F"W-a0Fb	)!, 		))11		**22		1199		%%--		((00N DAGGDDI4#C#CCO&) *:;''* '# vvzz7??J	E E  &&-..0#oo33CCWM**,7#%#6#6w#?L__AA&,WFvy11v{{BKK88  4*uu||; ; #">>BB1EGffUm224G&3w<!+;**A/.A%655 " "//LLuE 1':;a? 0 8! 12a7!@!@@ / 7"$"3"3"T"T"OO-u/B/B# 73w<< 66%=1133z"KK8 8 &(__%I%I & " 3 3G <.; &J &F *1')0A 0:!U\\/J/JPQ Q)0 $ ' &(__%J%J &} &K &F Jqww/0I1}FI!>!>GGD{{&;;++- "IINN??GG!IINN--55!IINN22::-)
 ;=7#-44UYY^^5O5O5W5WX 88///- %		 0 0 B B J J %		 0 0 B B I I %		 ? ? G G %		 0 0 B B J J %		 0 0 B B I I %		 0 0 B B I I %		 0 0 B B P P2 - < %		 0 0 G G O O %		 0 0 G G N N %		 0 0 H H O O %		 0 0 Q Q Y Y %		 0 0 @ @ H H %		 0 0 B B I I@ ;  %xx// 1eiimm6O6O6W6W5X X 1;;*;;%'__%I%I & " 3 3AFF5M4H4H4J K.2 &J &F !KK+JJ !TYYq\ 1%'__%I%I & " 3 3$B166%=CVCV$W!"&F ww(*%fkk&6&6I8NOO"NN,w $| !!#agg,/ &),,1N1N1P1P ##% &),,FKK1T1T{{''dI.....=(O " % /f fi((ZR]]-S-S&++**BHH55  33M1EFKK,,bii88  33M1Efkk..0A0ABBzKK$$))288H H KK$$))<<]AN v{{//@@"KK,,44!&++"2"2"9"9!"<biiHH((//2EEmUVWv&&u||46<< 0 12C!=!=!?? 3//"5"67B!<!<!>> 8 GG$$..	
 DDM!65<<00,,V44!!&)		# 		# 77144=#8M. 6GG&&

3F(K
 !,,, %/ /
&++-/
 %
!
 !$== 	
'(-F,G H==?+ ,&&5&7%8:	
=
 	,,Q0ABS'[ "! %$ /.ss   !AK.3AK
\AK
'AAK
(TAK
=-AK
/C6AK
%AK-AK.KAK
K

AKKAKK
AK+	K&AK.K.
AK=c                d  ^  S	U 4S jjn[        5       (       a}  UR                  [        R                  R                  R
                  R                  :X  aA  T R                  (       a0  T R                  U5      u  pEUS   S:w  a  U" US   US    S35        g g T =R                  U-  sl	        [        R                  R                  R                  nU GH[  nT R                  R                  U/ 5      nUR                   U   n	UR#                  5       R%                  U	5      (       d{  S
S jn
U
" U	R&                  5      (       a$  U" XyR&                  :  U SU	R&                   35        U
" U	R(                  5      (       a$  U" XyR(                  :*  U SU	R(                   35        U H  n[+        UR,                  5      nUT R                  -
  nU(       a;  [/        U[0        S9nT R                  R3                  U/ 5      R5                  U5        Mi  U" UR,                  UR,                   5        M     GM^     g )Nc                t   > [         R                  " X5      nTR                  USS9  TR                  U5        g )NTr  )r+   AssertScalarr  r
  )ra  r  	assert_oprL  s      ru   make_assertBGraphLowering.create_deferred_runtime_asserts.<locals>.make_assert  s2    2I  T :##I.rv   r   Tz to be Truec                `    U [         [         * 4;   a  g [        U 5        g! [         a     gf = f)NFT)r(   rM  	TypeError)r  s    ru   is_convertibleEGraphLowering.create_deferred_runtime_asserts.<locals>.is_convertible  s5    & 11#()F#'( )#()s     
--r  z <= )r  )ra  r#   r  r(  r   r#  )r  r   r   r  )r   r   r~   r  r   _assert_scalarr  r'  r  r   rb   r4  r   rN  r   popvar_to_range _default_unspecified_value_rangeissubsetlowerupperr   ra  r   r(  
setdefaultr	  )rL  r  rT  r[  	node_argsrd  rN  i0rasvrr_  rafvsmissingi1s   `              ru   rD  -GraphLowering.create_deferred_runtime_assertst  s   D	/ %&&EIINN99AAA::1=LI|t#IaLYq\N+*FG $ ''+<<'((22I (((,,R4++B/ AACLLRPP) &bhh//#B((Nrd$rxxj4IJ%bhh//#B((Nrd$rxxj4IJB/8C!D$?$??G c2**55b"=DDRH#BGGy: ) (rv   c                    [         R                  (       a  [        S5      e[        R                  S;  a  [        S[        R                   35      eg )NzC++ codegen is disabled)linuxdarwinwin32zUnsupported platform )r*   disable_cpp_codegenr5   sysplatformrU  s    ru   !validate_can_generate_cpp_wrapper/GraphLowering.validate_can_generate_cpp_wrapper  s@    %%()BCC<<;;(+@)OPP <rv   c                   U R                   R                  5       nUR                  S5        UR                  S5        [        U5      S::  d%   SR	                  SR                  U5      5      5       e[        U5      S:H  nU(       a  SOUR                  5       U l        U R                  (       a  U R                  5         [        U R                  5      U l        [        U R                  U R                  U R                  5      nUc   SU R                   S35       eUR                  UUUU5      U l        U R                   (       a0  U R                   R                  R"                  U R                  l        g g )	Nr   r   r)   zDoes not support mixing {}+r   zDevice z not supported)r   r   discardr]  formatr   rb  r   r#  ry  r1   r  r2   r$  r?  r  r   _names_iter)rL  is_subgraphr  parent_wrapper_codepartition_signaturesr   only_cpuwrapper_code_gen_clss           ru   init_wrapper_codeGraphLowering.init_wrapper_code  sP    ((--/U#V$< A% 	
'C'J'JHH\"(
 	
% |$)$,5,2B2B2D2241$2B2BC=d.. 
 $/ 	
d&&'~6	
/ 177 	
 ,0,=,=,J,J,V,VD) rv   c                   SS K nUR                  U R                  5      nUR                  U5      n/ nUR                  R                   HY  nUR
                  S:X  d  M  UR                  [        R                  R                  R                  L d  MH  UR                  U5        M[     / n0 n0 n/ n	0 n
U GH  nUR                  S    H\  nU HS  nX;   a  M
  [        U[        R                  R                  5      (       d  M5  [!        U5      X|'   UR                  U5        MU     M^     UR                  S   n[        R"                  R$                  R'                  UR                  S   UR                  S   UR)                  5        VVs0 s H@  u  pU[        U[        R                  R                  5      (       a  UR*                  S   OU_MB     snnUR                  S   5      n0 nUR                  R-                  U5         UR)                  5        H  u  pUU;   aJ  UR                  R/                  [        R0                  U4S	9n[!        U	5      UU'   U	R                  U5        MU  X;   a	  X   UU'   Mc  [!        U	5      X'   U	R                  U5        X   UU'   M     S S S 5        UXR2                  '   GM     X-   nUR                  R                   H&  nUR
                  S
:X  d  M  [5        U5      4Ul          O   UR9                  5         [        R                  R;                  U5      nUR=                  U5      n[!        U5      S:  a  U[!        U	5      S  n0 U l        U H  nSn/ nUR                  S    H|  n/ nU HW  n[        U[        R                  R                  5      (       d  UR                  U5        M?  SnUR                  UX|      5        MY     UR                  [5        U5      5        M~     U(       d  M  UU R>                  UR2                  '   M     US [!        U	5       U l         Xl!        g s  snnf ! , (       d  f       GN= f)Nr   r   gridrt   r  r  r  r  )rs   r   FT)"r   deepcopyr;  r4  r   r   r   r~   r  rT  r  r	  rt   rx   r   r%   r]  r   r   r  r6  r   inserting_beforer   cloner"  r  rs   	recompileInterpreterr  r/  r-  r.  )rL  r   r   	cloned_gmtriton_nodesr   grid_inputsvisited_gridstriton_inputskwargs_inputsvisited_kwargsr  r  rt   rO  rP  r  r  new_nodenew_outputsrunnerreturned_outputsgrid_outputsdynamic_grid	new_gridsnew_grids                             ru   extract_autotune_inputs%GraphLowering.extract_autotune_inputs  s    	MM$,,/	~6OO))D?*KK599#9#9#X#XX##D) * ,.24(*-/)+ DF+C+ !#uxx}}55-0-=*#**3/   , [[*F--@@TTL)/0 !' . 
1ehhmm(D(Dqvve}!K . 56G *,J11$7"LLNDAG|#,??#@#@TUSW#@#X(+M(:
1%,,X6 *(6(9
1 (+M(:N%!((+$2$5JqM + 8 (2M))$M !P $1OO))Dww("";/1	 *
 	%%i0!::n5{a+C,>,@AL$&D!$$.0	 KK/D!H#)#uxx}}==$OOC0$'+ ]5G(HI  $ $$U8_5 0  <7@D))$))4 %  "22FC4F!G"/m 87s   AP8 BP>>
Q	c                  ^  [        U 4S jS 5       5      (       Ga  SU 4S jjn[        R                  R                  (       a  [        R                  R                  (       aW  SnT R
                   H&  n[        U[        R                  5      (       d  M$  Sn  O   U(       a  U" 5       nT R                  U5        T R                  5       $ ST l        T R                  5       R                  nU" 5       n[        R                  R                   R#                  5          U" U5        SSS5        AST l        T R$                  R'                  5         T R(                  R'                  5         T R*                  R'                  5         [,        R.                  R0                  R2                  R'                  5         [,        R.                  R0                  R4                  R'                  5         [6        R8                  " 5         [        R:                  " SS05         T R                  5       sSSS5        $ T R                  5       $ ! , (       d  f       GN = f! , (       d  f       g= f)	zA
For GPU, Triton kernels are autotuned and stored as cubin files
c              3  @   >#    U  H  oTR                   ;   v   M     g 7frq   )r   )r   r   rL  s     ru   r   9GraphLowering.codegen_with_cpp_wrapper.<locals>.<genexpr>Y  s     Iv***r  )cudaxpuc                   >     SS jn [         R                  R                  R                  5       nUb  [	        [
        R                  [        5      (       d  UR                  (       a  UR                  R                  5         UR                   Vs/ s H
  nUc  M  UPM     nn[        R                  " U[
        R                  5       Vs/ s H  nU " U5      PM     nnOU[	        [
        R                  [        5      (       a  TR                  O[
        R                   Vs/ s H  nU " U5      PM     nnTR                  (       a  SSKJn  [#        TR$                  5       VVs/ s H<  u  pxUTR                  ;   d  M  [	        XW   [         R&                  5      (       d  M:  UPM>     n	nnU	 H3  nXW   n
[	        U
[         R&                  5      (       d   eU" U
5      XW'   A
M5     U$ s  snf s  snf s  snf s  snnf )Nc                N   U c  g [        U [        R                  [        R                  45      (       a  U R                  R
                  $ [        U [        5      (       a  [        U 5      $ [        U [        R                  5      (       d   S[        [        U 5      5      -   5       eU $ )Nz&Unknown type when creating real inputs)rx   r~   r`  r  r   hintr   r   r   r(  r  )r  s    ru   materializeXGraphLowering.codegen_with_cpp_wrapper.<locals>.extract_real_inputs.<locals>.materialize\  s     y##Aenn'EFF vv{{*#Az22%ay()!U\\:: Ds4PQ7|S:  !rv   r)   )clone_preserve_strides)r  z1Union[torch.SymInt, torch.SymFloat, torch.Tensor]r   zUnion[int, float, torch.Tensor])r~   _guardsTracingContexttry_getrx   rb   real_inputsra   output_stridesclearparams_flatrF  chainr   r  
compile_fxr  r   r   r   )r  tracing_contextparamr  r  r  r  r   r"  r  mutated_inprL  s              ru   extract_real_inputsCGraphLowering.codegen_with_cpp_wrapper.<locals>.extract_real_inputs[  s   !H!4!  #(--">">"F"F"H".zMM;8 8 '55'66<<> &5%@%@#%@E  %@   # "+amm!L#!LA $A!L   #K  *!--EE !//!"/#/  $A/   # &&B *343D3D)E*)EIC4#6#66  '{'7F )E ' *  2 '2&6)+u||DDDD+A++N('  2 #"W#
##*s*   G&G&G+"G0%G5?G5"G5FTNztriton.autotune_at_compile_time)r   %list[Union[int, float, torch.Tensor]])r   r*   tritonautotune_at_compile_timeautotune_with_sample_inputsr  rx   r+   UserDefinedTritonKernelr  codegenr#  compile_to_modulecallr~   rD  rE  rF  r  r  r  r  rb   r4  r   precomputed_replacementsinv_precomputed_replacementsr,   resetpatch)rL  r  user_defined_kernelsr   r  compileds   `     ru   codegen_with_cpp_wrapper&GraphLowering.codegen_with_cpp_wrapperS  s    IIIIC#J }}55 ==<<+0("oo%b"*D*DEE370! . ,&9&;44[A||~% $) 1138813[[11HHJ[) K $( $$**,''--/''--/  99??A  ==CCE\\#De"LM<<> NM <<>!! KJ NMs   	H<I<
I
Ic                    SSK Jn  [        R                  " SS5         U" U R                  5      U l         SSS5        g! , (       d  f       g= f)z
(Re)initializes the scheduler member.  When initializing the scheduler, no CUBIN
files should be generated (to avoid biasing any benchmarks and pessimizing
fusion decisions).
r)   )	Schedulerztriton.store_cubinFN)r,  r  r*   r  r  )rL  r  s     ru   _update_schedulerGraphLowering._update_scheduler  s2     	)\\.6&t7DN 766s	   ?
Ac                P   [        SSS9   U R                  5         U R                  5         [        R                  R                  U R                  U R                  R                  5        U R                  R                  U 5        U R                  R                  5         [        R	                  S[        R                  R                  5        U R                  R                  U R                   5      nU R                  R#                  5         UsS S S 5        $ ! , (       d  f       g = f)NzGraphLowering.codegenTlog_pt2_compile_eventzFFinished codegen for all nodes. The list of kernel names available: %s)r   r  r  rb   r  draw_orig_fx_graphr;  r,  r   r  push_codegened_graphr  r  r4  rE  generater   pop_codegened_graph)rL  rh  s     ru   r  GraphLowering.codegen  s    1N""$""$GG&&t||T^^5I5IJ2248NN""$IIX00
 &&//0A0ABF113! ONNs   DD
D%c                   [        SSS9   UR                  U l        UR                  U l        UR                  U l        U R	                  5         U R
                  R                  5         SSS5        g! , (       d  f       g= f)a  
This is a more compact version of the `codegen()` above
where we codegen this graph as a subgraph of some parent
graph. The parent graph is passed as an argument: the
intention is to inline codegening of the subgraph in
the parent graph's wrapper code (including the generated
kernels). The wrapper code is not finalized (via `.generate()`
call), as this will be done in the parent graph's `codegen()`.
zGraphLowering.codegen_subgraphTr  N)r   r  r  r#  r  r,  r  )rL  parent_graphs     ru   codegen_subgraphGraphLowering.codegen_subgraph  sb     :RVW , 9 9D*55DO+77D""$NN""$ XWWs   AA22
B c                    Sn/ n/ nU R                   R                   HL  nUR                  5       nX-  nUR                  XES-  45        UR                  XDR	                  5       45        MN     XU4$ )Nr   r  )r,  r   get_read_write_buffers_sizesr	  get_estimated_runtime)rL  total_bytesnode_countsnode_runtimesr   	num_bytess         ru   count_bytesGraphLowering.count_bytes  sy    
 NN((D99;I$K1n56  $(B(B(D!EF	 ) 66rv   zOptional[Callable[[str], None]]save_output_codec                p    [        SSSSS9   U R                  5       sS S S 5        $ ! , (       d  f       g = f)NzGraphLowering.compile_to_modulecode_genT,inductor_code_gen_cumulative_compile_time_us)
phase_namer  dynamo_compile_column_us)r   _compile_to_modulerU  s    ru   r  GraphLowering.compile_to_module	  s4    -!"&%S	
 **,
 
 
s   '
5c                @   U R                   (       a  U R                  5       OU R                  5       u  p[        U[        5      (       a  U R                  U5      nO/[        U[        5      (       a  UnO[        S[        U5       35      eUR                  c   e[        UR                  5        [        R                  SUR                  5        [        R                  " SUR                  5        [        R                   (       a%  [#        SUR                   3[$        R&                  S9  [(        R                  R+                  UR                  5        [(        R                  R-                  [.        R0                  R3                  UR                  5      S   S-   5        U$ )Nz Unrecognized wrapper code type: Output code written to: %szCompiled module path: )filer   z.debug)r#  r  r  rx   r`   _compile_to_module_linesr/   NotImplementedErrorr  __file__ro   r  r  rm   r  r*   benchmark_kernelprintrw  stderrrb   output_coder   ospathsplitext)rL  r  rd  mods       ru   r   GraphLowering._compile_to_module	  s(    04/?/?D))+T\\^ 	 l$455//=C&;<<C%243E2FG  ||'''%		.=93<<H""*3<<.9

K	CLL)	RWW%%cll3A6AB
rv   c                  ^^ SSK Jn  [        R                  R                  (       aw  U R
                  R                  R                  5       nUR                  SS5      nSU-   U R
                  R                  R                  5       -   S-   nUTR                  -   Tl
        [        R                  b  [        R                  TR                  5        [        R                  " STR                  5        [        R                   " 5       n["        R$                  " UTR                  S9   TR&                   VVs/ s H  u  pgXgR(                  4PM     nnnUR+                  TR                  5      u  n	m[        R                  " S	T5        [-        S
U4S jU4S jS9  [1        SSS9   UR3                  U	TU0 U R4                  EU R6                  ES9n
S S S 5        Xl        TU l        Xl        [        R>                  (       a%  [        R@                  (       a  W
RC                  SSS9  W
$ s  snnf ! [.         a    [-        S
U4S jS9  e f = f! , (       d  f       N= f)Nr)   )PyCodeCachez"""z\"\"\"z%"""
Compile-time auto-tuning block: 
z"""
zOutput code: 
%s)coder  inductor_output_codec                    > ST 0$ )Nfilenamerr   )r  s   ru   r  8GraphLowering._compile_to_module_lines.<locals>.<lambda>Y	  s
    T*rv   c                    > T R                   $ rq   r  r  s   ru   r  r  Z	      <#5#5rv   )
payload_fnc                    > T R                   $ rq   r   r  s   ru   r  r  S	  r  rv   zPyCodeCache.load_by_key_pathTr  )linemapattrs)timesrepeat)"	codecacher  r*   r  r  r  kernel_autotune_defsgetvaluereplacekernel_autotune_callsr  r   r  rm   r  rR   inductor_meta_from_configrS   begin_compileline_mapstack_tracewriter   rt  r   load_by_key_pathr  r
  r5  r6  r7  benchmark_harnessprofile_bandwidth_outputbenchmark_compiled_module)rL  r  r  r	  tuning_codeinductor_metaline_nor   r  r  r  r  s    `         @ru   r  &GraphLowering._compile_to_module_lines/	  s!    	+==11#'#4#4#I#I#R#R#T #7#?#?{#S 7&' ##99BBDE 	  "-|/A/A!AL))5**<+=+=>1<3E3EF&@@B**=|?Q?QR	 &2%:%:%:MG **+%:   $)),*<*<=IC!!">E &*5
 8PTU..DD4+C+CD	 / C V $##(G(G))!)<
E  	&5
 	 VUs*   %H* 4H$8H* "+I$H* *I
Ic                   / n[         R                  " S5      n[         R                  " S5      nU H  n[        U[        R                  5      (       a+  UR                  U R                   S[        U5       35        MM  [        U[        R                  5      (       a+  UR                  U R                   S[        U5       35        M  UR                  UR                  5       5        M     U$ )Nr   _none_shape)
rF  rG  rx   r+   NoneAsConstantBufferr	  r"  r)  rB   r  )rL  r   namesshape_counternone_counterr   s         ru   _get_output_namesGraphLowering._get_output_namesm	  s    !* q)!D$ 7 788		{%\0B/CDED"":":;;		{&m1D0EFGT]]_- " rv   c                8    U R                  U R                  5      $ rq   )r!  r   rU  s    ru   get_output_namesGraphLowering.get_output_namesz	  s    %%d&8&899rv   c                V   XR                   R                  5       ;   =(       ar    U R                   U   R                  5       S:H  =(       aK    [        U R                   U   R	                  5       5      S:H  =(       a    [        U R                   U   5      S:H  =(       d    XR                  ;   $ )Nr)   r   r   )r   r  r  r]  ro  r=   r   r  s     ru   is_unspec_argGraphLowering.is_unspec_arg}	  s     %%**,, B!!$'113q8BD%%d+44671<B   1 1$ 78EA	3
 222	3rv   )Xr   r3  rE  r=  r'  r/  r-  r.  r   r  r  rJ  r5  r7  r6  r   r   r  r   r  r  r#  r!  r0  r  rK  r   r9  r  r   r   r8  r   r@  r   r  r   r  r$  r0   r(  r   r   r   r   r  r   rD  r   r   r   r   r  rC  r&  r  r  r  r"  r  r  r  r  r	  r  rB  r2  r   r  r;  r   rI  r+  r   r%  r  r  r  r   r,  r  r   r
  rA  r   rH  r  r   )NNNFFNNFFFNNNNNNF)&rM  torch.fx.GraphModuler   zOptional[Sequence[object]]rN  zOptional[ShapeEnv]r(  zOptional[int]r#  r  r'  r  r   zOptional[bool]r  z4Optional[Callable[[list[ir.ExternKernelNode]], Any]]r   r  r   r  r   r  r  zOptional[dict[str, int]]r   Optional[str]r   r*  r   zOptional[GraphLowering]r"  r*  r   zOptional[Sequence[int]]r$  r  r   r#  )r   r#  )rb  torch.Tensorr   z=tuple[Sequence[Union[int, Expr]], Sequence[Union[int, Expr]]])rb  r+  r   z)tuple[list[sympy.Expr], list[sympy.Expr]])r   zOUnion[ir.TensorBox, ir.StorageBox, ir.Buffer, WorkspaceArg, ir.TorchBindObject]r   zSequence[Expr])r   z2Union[ir.Buffer, WorkspaceArg, ir.TorchBindObject]r   r   )r   z.Union[torch._inductor.ir.IRNode, device, None]ry  r-   r   r  )r  rk   r   rM  )r   torch.device)r   r,  r   Iterator[None]r  )rM  rh   r   r  r   r  )r"  r(  r   r(  )rM  r)  r   zlist[torch.Tensor]r  r(  r   r  )r   zOrderedSet[Node])r"  r(  r   r#  )r   r,  r   r#  )r   z,torch._subclasses.fake_tensor.FakeTensorMode)r  r(  r   z<Optional[Union[ir.TensorBox, ir.Buffer, ir.TorchBindObject]])r  
sympy.Exprr   r#  )r  r(  r   z2Union[ir.TensorBox, ir.Buffer, ir.TorchBindObject])r  r(  r   ztorch.dtype)r  r(  r   zUnion[int, Expr])rs   r   r   r   )r   zir.Operationr   r(  )r  z	ir.Bufferr  r  r   r(  )r  	list[str]r   r(  )r$  r"  r   r#  )r"  r(  r   r+  )r"  r*  rm  zUnion[Tensor]r   r(  rq   )rm  r   r"  r*  r   z*Union[TensorBox, ir.ShapeAsConstantBuffer])r"  r(  rH  zOptional[torch.device]r   r(  )r   r(  rs   tuple[object]rt   dict[str, object]r   zUnion[Expr, TensorBox, None])r   r   rs   r   rt   dict[str, Any]r   r   )r  r+  r   r  )r   r(  rs   z	tuple[()]rt   r1  r   zOUnion[Constant, TensorBox, ShapeAsConstantBuffer, ir.Subgraph, TorchBindObject])r   r   rs   r   rt   r   r   r   )r   r(  rs   r0  rt   r1  r   r#  )r   r   )r   r-  )r  r   r}  
tuple[Any]r~  r2  r  r3  r  r2  r   r#  )r  r   r   object)r  r   rT  zOrderedSet[sympy.Symbol]r   r#  FNNN
r  r  r  r*  r  zOptional[PythonWrapperCodegen]r  z!Optional[GraphPartitionSignature]r   r#  )r   r  r   r#  )r   z)tuple[ValueWithLineMap, ValueWithLineMap])r  r   r   r#  )r   zVtuple[int, list[tuple[BaseSchedulerNode, int]], list[tuple[BaseSchedulerNode, float]]])r   CompiledModule)r  r`   r   r7  )r   r   r   r/  )r   r/  )r"  r(  r   r  )H__name__
__module____qualname____firstlineno____annotations__r   rT  rg  rj  rp  rv  rz  r  r  
contextlibr   r  r  staticmethodr   r  r  r1  r  r  propertyr  r  r  r  r  r  r  r
  r  r  r%  r*  r.  r<  rA  r9  rL  r   r  r  r  r  r   r  r  r  rs  r  rD  ry  r  r  r  r  r  r  r  r  r  r  r  r!  r$  r'  __static_attributes____classcell__rQ  s   @ru   r   r   #  s   ""
 6:(,"&!%) "!$7;,0+/04"37 +E6 E6 3E6 &	E6
  E6 E6 E6 #E6!
E6 E6 E6 E6 5E6  *!E6" )#E6$ .%E6& 'E6( 1)E6* +E6, 
-E6 E6N1# # 	F# J	2#
#
 
#$EFE	EM>M  M 
	M-"4 ( ( Y Yv
 
 +
 	

 

60dA
D  	E$FQQ	;Q8,
8& FK (
@	
 !)6	@ 37

"/
	3
 ff f "	f
 
&fPob 5 51717 17 "	17

17f[
[
 [
 "	[

 
[
z  $ $ $ $I:I: I: #	I:
 I: #I: 
I:VhTS;S;3KS;	S;jQ "'+>BBF#W#W %#W <	#W
 @#W 
#WJ\0C\0	\0|p"	2p"d	8&%$7
7" 9=5<-<<,<	<|:3 3rv   r   c                  ^   ^  \ rS rSrSrSU 4S jjr    S         SU 4S jjjrSrU =r$ )	r  i	  z
Mostly a helper class for the subgraph lowering. The main goal is to call
init_wrapper_code with the subgraph related arguments.
c                2   > Xl         [        TU ]  " U0 UD6  g rq   )r  r   r   )rL  r  rs   rt   rQ  s       ru   r   SubgraphLowering.__init__	  s    $)&)rv   c                `   > [         TU ]  SU R                  U R                  R                  S9  g )NT)r  r  r  )r   r  r"  r  r  )rL  r  r  r  r  rQ  s        ru   r  "SubgraphLowering.init_wrapper_code	  s.     	!)) $ 8 8 	" 	
rv   )r  )r  r   rs   r   rt   r   r   r#  r5  r6  )	r8  r9  r:  r;  __doc__r   r  r@  rA  rB  s   @ru   r  r  	  sV    
* "'+>BBF

 %
 <	

 @
 

 
rv   r  )rs   r   rt   r   r   r#  )r   r.  r   zOptional[torch.dtype])r   r   r   r  )r   rh   r   r(  r   z1Union[Tensor, torch._C.ScriptObject, GraphModule])r   ri   r   dict[Node, tuple[int, ...]])r   ri   r   rI  r   r#  )
__future__r   r=  r>  rF  loggingrk  r  r  rw  r   collectionsr   r   typingr   r   r   r	   r
   r   ry   r   r~   torch._loggingtorch.fxr   r   torch._decompr   torch._dynamo.utilsr   r   "torch._library.fake_class_registryr   torch._library.utilsr   r   r   torch._prims_commonr   r   torch._subclasses.fake_tensorr   torch._utils_internalr   %torch.fx.experimental._backward_stater   torch.fx.experimental.sym_noder   r   %torch.fx.experimental.symbolic_shapesr   r   r   r    r!   r"   r#   r$   torch.fx.noder%   torch.utils._mode_utilsr&   torch.utils._ordered_setr'   torch.utils._sympy.numbersr(   r   r*   r+   r,   codegen.commonr-   r.   r/   r0   r1   r2   r3   r4   excr5   r6   r7   r8   fx_utilsr9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   r  rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   runtimerR   runtime.autotune_cacherS   r   rT   rD  rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   virtualizedra   rb   collections.abcrc   rd   re   typesrf   torch._higher_order_ops.effectsrg   rh   torch.fx.graphri   codegen.wrapperrj   dependenciesrk   r,  rl   r7  torch._inductor.codecacherm   	getLoggerr8  r  _logginggetArtifactLoggerr  r  r   rG  r*  r  torch._inductor.fb.utilsro   r   r   r   r   r   r   r  r   r  rr   rv   ru   <module>ro     s   "      	 	 
  # % J J        , 4 ? : 7 5 : ? L	 	 	  / / - ! !	 	 	  %       $ 8 &    ( << ;$$5!,:'<<=N 5 !00<Hyy~~$??, 	8(
	!6$R/R/+FR/	R/jb!3EHH(( b!3JC
} 
rv   