
    `i}7                     L   S SK r S SKrS SKrS SKrS SKrS SKrS SKJr  S SKrS SKrS SK	r	S SK
r
S SKrS SKJr  S SKJr  \R                   " \5      r " S S\5      rS\R(                  0r    SS jrSS	 jrS
 rS rS rS rS rS r\R<                  4S jrS r SS jr!S r"g)    N)datetime)Enumversionc                   &    \ rS rSrSrSrSrS rSrg)	Precision   fp32fp16int8c                     U R                   $ N)value)selfs    n/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/onnxruntime_tools/transformers/benchmark_helper.py__str__Precision.__str__   s    zz     N)	__name__
__module____qualname____firstlineno__FLOAT32FLOAT16INT8r   __static_attributes__r   r   r   r   r      s    GGDr   r   float32c                    S n SSK JnJnJn	Jn
  U" 5       nU(       a  U	R
                  Ul        OU	R                  Ul        U(       a  SUl        US:  a(  X;l	        [        R                  SUR                   35        U(       a  SUl        OSUl        [        R                  SU  35        U(       d  S/OSS/nU" XUS	9nU$ !   [        R                  S
SS9   U$ = f)Nr   )SessionOptionsInferenceSessionGraphOptimizationLevel__version__Tz%Session option: intra_op_num_threads=   zCreate session for onnx model: CPUExecutionProviderCUDAExecutionProvider)	providers	Exception)exc_info)onnxruntimer    r!   r"   r#   ORT_ENABLE_ALLgraph_optimization_levelORT_ENABLE_BASICenable_profilingintra_op_num_threadsloggerdebuglog_severity_levelerror)onnx_model_pathuse_gpuenable_all_optimizationnum_threadsr.   verbosesessionr    r!   r"   onnxruntime_versionsess_optionsexecution_providerss                r   create_onnxruntime_sessionr=   %   s     G2||%'"4J4Y4YL14J4[4[L1,0L)?0;-LL@AbAb@cde./L+./L+66GHI(/  6 !6MOe5f 	"?L_` N2yD1Ns   B<C Cc                     U (       a  [         R                  " SSS9  g [         R                  " SS9  [        R                  " S5      R	                  [        R
                  5        g )NDEBUGz8[%(filename)s:%(lineno)s - %(funcName)20s()] %(message)s)levelfmtz%(message)s)rA   transformers)coloredlogsinstalllogging	getLoggersetLevelWARNING)r8   s    r   setup_loggerrI   K   sA    '/ij..)227??Cr   c                 p   U (       a:  [         R                  R                  U 5      (       d  [         R                  " U 5        U(       a:  [         R                  R                  U5      (       d  [         R                  " U5        SS KnU(       a  SUR                  5       ;   d   S5       eSS Kn[        R                  S[        R                   35        [        R                  SUR                   35        [        R                  SUR                   35        SSKJn  UR                  " [        R                  5      UR                  " S5      :  d   eUR                  " UR                  5      UR                  " S	5      :  d   eUR                  " UR                  5      UR                  " S
5      :  d   eg )Nr   r&   z=Please install onnxruntime-gpu package to test GPU inference.zPyTorch Version:zTransformers Version:zOnnxruntime Version:r   z1.5.0z3.0.0z1.4.0)ospathexistsmakedirsr*   get_available_providersrB   r0   infotorchr#   	packagingr   parse)	cache_dir
output_dirr5   r*   rB   r   s         r   prepare_environmentrV   S   s@   	22
I"''..44
J&+*M*M +
 
 	KJ	K 
 
KK"5#4#4"567
KK'(@(@'ABC
KK&{'>'>&?@A "==**+w}}W/EEEE==112gmmG6LLLL==001W]]75KKKKr   c           
         [        U 5      [        [        U 5      5      -  S-  n[        R                  " U [        R
                  S9S-  nUSU-  -  n[        U 5      SR                  U5      SR                  [        R                  " U S5      S-  5      SR                  [        R                  " U S5      S-  5      SR                  [        R                  " U S5      S-  5      SR                  U5      SR                  U5      S.$ )Ng     @@)dtypez{:.2f}Z   _   c   )
test_timeslatency_variancelatency_90_percentilelatency_95_percentilelatency_99_percentileaverage_latency_msQPS)sumfloatlennumpyvarfloat64format
percentile)runtimes
batch_size
latency_msr]   
throughputs        r   get_latency_resultro   k   s    Xs8}!55>Jyy?&Hv
23J (m$OO,<=!)1A1A(B1ORX1X!Y!)1A1A(B1ORX1X!Y!)1A1A(B1ORX1X!Y&ooj9z* r   c                    [        USSS9 n/ SQn[        R                  " X#S9nUR                  5         U  H  nUR	                  U5        M     S S S 5        [
        R                  SU 35        g ! , (       d  f       N'= f)Na modenewline)enginer   device	precision	optimizer
io_binding
model_nameinputsthreadsrl   sequence_lengthr   r\   rb   ra   r]   r^   r_   r`   
fieldnamesz&Detail results are saved to csv file: )opencsv
DictWriterwriteheaderwriterowr0   rP   )resultscsv_filenamecsv_filecolumn_names
csv_writerresults         r   output_detailsr   {   so    	lb	1X
 ^^HF
 F'  
2 KK8GH 
2	1s   AA00
A>c                    [        USSS9 n/ SQn/ nUR                   H-  nUR                   H  nUR                  SU SU 35        M     M/     [        R
                  " X4U-   S9nUR                  5         UR                   GH7  n	S GH,  n
UR                   GH  nS	 GH  nUR                   H  n0 nU  H  nUS
   U	:X  d  M  US   U
:X  d  M  US   U:X  d  M$  US   U:X  d  M/  US   U:X  d  M:  UR                  5        VVs0 s H  u  nnUU;   d  M  UU_M     nnnU(       d5  UR                  U5        UR                  U Vs0 s H  nUS_M     sn5        OU H  nUU   UU   :X  a  M   e   US   nUS   nUS   USU SU 3'   M     U(       d  M  UR                  U5        M     GM     GM     GM/     GM:     S S S 5        [        R                  SU 35        g s  snnf s  snf ! , (       d  f       N2= f)Nrq   rr   rs   )	r{   r|   rv   r   rw   rx   ry   rz   r}   b_sr   )         )TFrr   r{   r|   rv   rz   r}   rl   r~   ra   z'Summary results are saved to csv file: )r   batch_sizessequence_lengthsappendr   r   r   modelsenginesr7   itemsupdater   r0   rP   )r   r   argsr   header_names
data_namesrl   r~   r   r{   input_countengine_namerz   r}   rowr   kvheadersr   ss                        r   output_summaryr      s   	lb	1X
 
**J#'#8#8!!Aj\O3D"EF $9 + ^^H
9RS
 ++J(#'<<K&7
'+'7'7G"$C*1#),#7:#E&QYJZ^iJint(0o25@oAEKLEY]gEglr,5m7:AmB@F.d1RSWcRctq!tG.d+.(+

7(;(+

:3N:aArE:3N(O1=A36q6WQZ3G,G3G 2>(.|(<A(./@(AA8>?S8TC!A3b$5 +2  #s * 3 3C 8# (8 '8 $0  ) & 
2D KK9,HI /e 4O1 
2	1sZ   B4G+G+G+G+%G+0G+G G *G+G&G+-%G+)G+ G++
G9c           
      .   SSK Jn  [        USSS9 n/ SQ[        [	        [        U R                  5       5      5      R                  5       5      -   n[        R                  " X4S9nUR                  5         U R                  5        H\  n[        [        R                  " 5       5      X   S'   X U   S	'   [        R                  X   S
'   X`U   S'   UR                  X   5        M^     S S S 5        [         R#                  SU 35        g ! , (       d  f       N'= f)Nr   )r#   rq   rr   rs   )model_filenamer   rB   rQ   r   r   rB   rQ   r   z(Fusion statistics is saved to csv file: )rB   r#   r   listnextitervalueskeysr   r   r   strr   nowrQ   r   r0   rP   )model_fusion_statisticsr   transformers_versionr   r   r   keys          r   output_fusion_statisticsr      s    @	lb	1XNQU-44678==?RA A^^HF
 *//1C7:8<<>7J#(4;OC(8494E4E#(1=@C()9: 7 <= 2 
2 KK:<.IJ 
2	1s   CD
Dc                    ^ ^ 0 n[         R                  " UU 4S jSUS9nUR                  U5        UR                  SS05        UR                  [        Xd5      5        U$ )Nc                  (   > TR                  S T 5      $ r   )run)
ort_inputsort_sessions   r   <lambda>inference_ort.<locals>.<lambda>   s    [__T:%Fr   r   numberrepeatrz   F)timeitr   r   ro   )r   r   result_templaterepeat_timesrl   r   rk   s   ``     r   inference_ortr      sP    F}}FqYefH
MM/"
MM<'(
MM$X:;Mr   c           
      x  ^ ^ 0 nT R                  5       mUR                  5        H  n[        R                  " X   5      R	                  U	5      n[        X   R                  5      [        ;   a  [        [        X   R                  5         OU
nTR                  XR                  R                  SXR                  UR                  5       5        M     [        U5      S:X  a  [        XgU	5        [        [        U5      5       H[  nTR!                  XO   Xo   R                  R                  S["        R$                  X_   R                  Xo   R                  5       5        M]     [&        R(                  " UU 4S jSUS9nUR+                  U5        UR+                  SS05        UR+                  [-        UU5      5        U$ )Nr   c                  &   > TR                  T 5      $ r   )run_with_iobinding)rz   r   s   r   r   /inference_ort_with_io_binding.<locals>.<lambda>   s    [%C%CJ%Or   r   r   rz   T)rz   r   rQ   
from_numpytor   rX   IO_BINDING_DATA_TYPE_MAP
bind_inputrw   typeshapedata_ptrre   allocateOutputBuffersrangebind_outputrf   r   r   r   r   ro   )r   r   r   r   ort_output_namesort_outputsoutput_buffersoutput_buffer_max_sizesrl   rw   	data_typer   namenp_input
input_typeirk   rz   s   `                @r   inference_ort_with_io_bindingr      sz    F '')J!##J$4588@NQR\RbRhRhNi  nF  OF-c*2B2H2H.IJ  LU
dOO$8$8!ZYaYjYjYlm "
 >anvN3'()/2N4E4L4L4Q4QSTV[VcVc*~33^5F5O5O5Q	S * }}OXYbnoH
MM/"
MM<&'
MM$Xz:;Mr   c           	      |    U H6  nU R                  [        R                  " U[        R                  US95        M8     g )N)rX   rw   )r   rQ   emptyr   )r   r   rw   r   s       r   r   r      s-     %ekk!5==PQ %r   c                    SSK nUR                  U 5        [        R                   R                  U 5        [        R                  " U 5        [        R
                  R	                  U 5        [        R
                  R                  U 5        g)z4Set random seed manully to get deterministic resultsr   N)randomseedrf   rQ   manual_seedcudamanual_seed_all)r   r   s     r   set_random_seedr      sU    
KK	LLd	d	JJ4 	JJt$r   c                   ^^^ SS K mSS KmSSKJm   " UUU4S jS5      nU" S5      nU (       a  UR	                  5       OUR                  5       nSSKJn  U" 5        nU" 5       nUR                  U (       a  UR                  OUR
                  5      n UR                  U5      nUR                  5       n	SUl
        UR                  5       n
U (       aV  [        SU SU
 35        [        U5      S	:  a,  [        U
5      S	:  a  US   S
   nU
S   S
   nX-
  sS S S 5        $  S S S 5        g [        SUS SU
S S35        X-
  sS S S 5        $ ! SUl
        UR                  5       n
f = f! , (       d  f       g = f)Nr   )sleepc                   <   > \ rS rSrSS jrU UU4S jrU4S jrSrg)%measure_memory.<locals>.MemoryMonitori  c                     Xl         g r   keep_measuring)r   r   s     r   __init__.measure_memory.<locals>.MemoryMonitor.__init__  s    "0r   c                    > Sn [        UTR                  TR                  " 5       5      R                  5       R                  S-  5      nT" S5        U R
                  (       d   U$ Mc  )Nr      {Gzt?)maxProcessgetpidmemory_inforssr   )r   	max_usagerK   psutilr   s     r   measure_cpu_usage7measure_memory.<locals>.MemoryMonitor.measure_cpu_usage
  sY    I	6>>"))++F+R+R+T+X+X[b+bc	e** r   c                 x  > SSK JnJnJnJnJnJnJn  / n/ n	 U" 5         U" 5       n
[        U
5       Vs/ s H  nSPM     nn[        U
5       Vs/ s H  o" U" U5      5      PM     n	n [        U
5       H.  nU" U" U5      5      n[        X   UR                  S-  5      X'   M0     T" S5        U R                  (       d  OMY  U" 5         [        U
5       Vs/ s H  nUX   X   S.PM     sn$ s  snf s  snf s  snf ! U a7  nU R                  (       d  U R                  R                  SU5         S nAg S nAff = f)Nr   )nvmlInitnvmlDeviceGetCountnvmlDeviceGetHandleByIndexnvmlDeviceGetMemoryInfonvmlDeviceGetNamenvmlShutdown	NVMLErrorr   r   )	device_idr   max_used_MBz-Error fetching GPU information using nvml: %s)py3nvml.py3nvmlr   r   r   r   r   r   r   r   r   usedr   silentr0   r3   )r   r   r   r   r   r   r   r   max_gpu_usagegpu_namedeviceCountr   rP   r3   r   s                 r   measure_gpu_usage7measure_memory.<locals>.MemoryMonitor.measure_gpu_usage  sO   e e eMH
02,1+,> ?,>q,> ?V[\gVhiVhQR-.H.KLVhi";/67QRS7TU+.}/?WAT+U( 0 %L..  
 !-	/ .a "#$K#0#3 .	/ / !@i/
  {{KK%%&UW\]sA   C< C- C< C2&A0C< C7*C< -C< <D9-D44D9r   NT)r   r   r   r   r   r   r  r   )rK   r   r   s   r   MemoryMonitorr     s    	1		 	r   r	  F)ThreadPoolExecutorzGPU memory usage: before=z  peak=r   r   zCPU memory usage: before=z.1fz
 MB, peak=z MB)rK   r   timer   r  r   concurrent.futuresr
  submitr   r   printre   )is_gpufuncr	  monitormemory_before_testr
  executor
mem_thread	fn_threadr   r   beforeafterrK   r   r   s                @@@r   measure_memoryr    se   ' 'R E"G8>224GD]D]D_5		/__&W%>%>gNgNgh
	, -I%%'F%*G""))+I-.@-ATU%&!+I!0C+A.}=!!]3~ 
	" # 
	& -.@-EZPYZ]^abc1) 
	 &+G""))+I 
	s1   6E&!E
5A E&E&)E&
E##E&&
E4)TFFr  ){   )#rK   sysr   rf   r  r   r   argparserE   rC   rQ   onnxenumr   rR   r   rF   r   r0   r   r   r   r=   rI   rV   ro   r   r   r   r   longlongr   r   r   r  r   r   r   <module>r      s    
 
 
           			8	$  u}}  8<+-05',#LDL0 I #JLK & -2NNDR%G2r   