+
    9ifU                     V   ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt	^ RI
t
^ RIt^ RIt^ RIHtHt ^ RIHt ^ RIHt ^ RIHtHtHt ^ RIHt ^ RIHtHtHt ^ RIt]P@                  ! ]!4      t"R R	 lt#R
 R lt$R t%R t&R t'R t(R t)R t*R t+R t,]!R8X  d
   ],! 4        R# R# )    N)measure_memorysetup_logger)get_library_path)ORTModelForSpeechSeq2Seq)ProfilerActivityprofilerecord_function)trange)AutoModelForSpeechSeq2SeqWhisperConfigWhisperProcessorc                8    V ^8  d   QhR\         P                  /#    argsargparse	Namespace)formats   "o/var/www/html/photoedit/myenv/lib/python3.14/site-packages/onnxruntime/transformers/models/whisper/benchmark.py__annotate__r       s     E EX'' E    c                   a aa	a
 S P                   R9  d   \        R4      hV 3R loV 3R lo	RS P                  RS P                  RS P                  RS P
                  R	S P                  R
S P                  /pS P                   R8X  Ed,   VP                  4        FE  w  r#\        P                  ! V.RV9   d   \        P                  M\        P                  R7      W&   KG  	  S P                  '       d5   \        P                  ! S P                  .\        P                  R7      VR&   S P                  '       d5   \        P                  ! S P                   .\        P                  R7      VR&   S P"                  '       d5   \        P                  ! S P$                  .\        P                  R7      VR&   \&        P)                  RS P*                   24       VV	3R lp\-        S VS P.                  4       V! S P.                  4      pS P.                  '       d   WQR&   V# \&        P)                  R4       S P                   R8X  d   RMRo
V V
3R lp\-        S We4       V! V4      pS P                   R8X  d   WqR&   V# TP1                  S P2                  '       d   \4        P6                  M\4        P                  S P8                  R7      VR&   S P:                  VR&   RVR&   RVR&   S P                  '       d   S P                  VR&   V# ) hf-pt-eagerortz/Unable to auto-detect inputs for provided modelc                  t   < \         P                  ! SP                  4      p \         P                  ! V 4      p V # N)whisper
load_audio
audio_pathpad_or_trim)audior   s    r   load_via_ffmpeg#get_inputs.<locals>.load_via_ffmpeg$   s,    ""4??3##E*r   c                  &  < \        SP                  R 4      ;_uu_ 4       p \        P                  ! \	        V P                  4       4      \        P                  R7      p\        P                  ! V.4      pRRR4       V#   + '       g   i     X# ; i)rbdtypeN)openr    npasarraylistreaduint8array)fr"   r   s     r   load_via_numpy"get_inputs.<locals>.load_via_numpy)   s`    $//4((AJJtAFFH~RXX>EHHeW%E )  )( s   AA??B	
max_length
min_length	num_beamsnum_return_sequenceslength_penaltyrepetition_penaltypenaltyr'   decoder_input_idslogits_processortemperaturezLoad audio: c                 0   < V '       d   S! 4       # S! 4       # r    )onnx_e2er#   r1   s   &r   <lambda>get_inputs.<locals>.<lambda>D   s    ^%5%XGX%Xr   audio_streamzFeature extraction: r*   ptc                 j   < SP                   P                  V .SSP                  R 7      P                  # ))return_tensorssampling_rate)	processorfeature_extractorrF   input_features)r"   r   return_types   &r   r@   rA   P   s3    !A!A	4;M;M "B "n"r   rI   )r(   deviceinputsno_repeat_ngram_sizeTearly_stopping	use_cacheforced_decoder_ids>   r   hf-ortr   hf-pt-compile)benchmark_type	Exceptionr3   r4   r5   r6   r7   r8   itemsr*   r/   float32int32has_decoder_input_idsr:   has_logits_processorr;   has_temperaturer<   loggerinfor    time_fnhas_audio_streamtouse_fp16torchfloat16target_devicerM   )r   rL   kvload_audio_fn
audio_dataprocessor_fnrI   r#   r1   rJ   s   f       @@@r   
get_inputsri       s|   "SSIJJ
 	doodooT^^ 9 9$--d55F e#LLNDA!)q.BJJbhhWFI #%%%*,((D4J4J3KSUS[S[*\F&'$$$)+43H3H2IQSQYQY)ZF%&$&HHd.>.>-?rzz$RF=! KK,t/01XMD-!6!67t445J!+~ KK&'--6$DKL D,+!*-Ne##1 %((#}}}emm%--HZHZ ) F8 &*%>%>F!"#FF;'+'='=#$Mr   c                8    V ^8  d   QhR\         P                  /# r   r   )r   s   "r   r   r   h   s     = =H&& =r   c                    R R r!R R rCV P                   R9   d   V P                  '       d   V P                  MV P                  p\        P                  ! 4       p\        P
                  ! TV P                  '       d   \        P                  M\        P                  RR7      P                  V P                  4      p\        P                  ! 4       pV P                   R8X  d   \        P                  ! V4      pMV P                   R9   da   \        P                  ! 4       pV P                  Vn        VP#                  \%        4       4       V P&                  '       d   ^Vn        ^Vn        M\-        RV P                    24      hV P                   R8X  d   \/        V P0                  4      \2        J d   V P0                  ^ ,          MV P0                  p\/        V P0                  4      \2        J d   V P0                  ^,          MR p\        P                  ! 4       p\4        P
                  ! V P6                  VVVRR7      p\        P                  ! 4       pV P                   R8X  d{   \8        P;                  RV P<                   24       \        P                  ! 4       p\        P>                  ! V P<                  VV P0                  .R	7      p\        P                  ! 4       p\8        P;                  R
WC,
           R24       V# )NrR   T)torch_dtyperO   rQ   r   Cannot recognize )providerprovider_optionssession_optionsuse_io_bindingzLoading model from )	providerszLoaded model in  s   r   rR   >   r   rQ   ) rS   hf_pt_model_path
model_nametimer   from_pretrainedr`   ra   rb   rV   r_   rc   compiler   SessionOptionsr   enable_profilingregister_custom_ops_libraryr   verboselog_verbosity_levellog_severity_levelrT   typeexecution_providertupler   hf_ort_dir_pathr[   r\   ort_model_pathInferenceSession)r   modelsess_options
start_timeend_timesourcern   ro   s   &       r   	get_modelr   h   sI   < >>*.*?*?*?&&T__YY[
)99)-EMM
 "T
 	 	
 99;/1MM%(E			 1	1))+(,%001A1CD<<</0L,./L+ +D,?,?+@ABBh&15d6M6M1NRW1W4**1-]a]t]t9=d>U>U9VZ_9_42215eiYY[
(88  -(
 99;e#)$*=*=)>?@YY[
$$../

 99;
KK"8#8"9<=Lr   c                    \        V4      \        J d
   V^ ,          MTp\        V4      \        J d
   V^,          MTp\        P                  ! V P                  4      pV P
                  R8X  d   \        V P                  4      M%\        V P                  \        P                  RR7      pV P                  '       d   V! V4      p\        P                  V4       V F  pV! V4       K  	  V P                  R8w  d    \        P                  P                  V4       \         P                   ! 4       p	V P
                  R8X  d   \        V P"                  4      M%\        V P"                  \        P                  RR7      p
V
 F  pV! V4       K  	  V P                  R8w  d    \        P                  P                  V4       \         P                   ! 4       pV P
                  R8w  d   \        P                  R4       ^pW,
          V P"                  ,          pW,          p\        P                  RV R24       \        P                  R	V R
24       R# )r   r   zWarm up)filedesccpu	Benchmark z	Latency: rs   zThroughput: z qpsN)r   r   ra   rK   rc   rS   rangewarmup_runsr
   sysstdoutr}   r[   r\   cudasynchronizerw   num_runs)r   fnrL   warmup_inputsbenchmark_inputstorch_devicewarmup_rangeoutputs_r   bench_ranger   
batch_sizelatency
throughputs   &&&            r   r]   r]      s   !%f!6F1IFM$(LE$9vayv<< 2 23L
 %' 	dD$$3::IF  |||]#G
=  {{e

|,J %' 	dmmDMM

E 
 
  {{e

|,yy{H e#BJ$5G%J
KK)G9B'(
KK,zl$/0
r   c                    V P                   P                  4        R V P                   R V P                   RVP                  P                  RR 4       RV R\        P                  P                  4       R 2pRpV P                   R9   d   \        \        P                  \        P                  .RRR7      ;_uu_ 4       p\        R4      ;_uu_ 4        V! V4       RRR4       RRR4       XP                  ^R7      P                  V P                  V P                   R7      p\"        P$                  P'                  V P(                  V R	24      p\+        VR
4      ;_uu_ 4       pVP-                  V4       RRR4       V# V! V4       V R2pV#   + '       g   i     L; i  + '       g   i     L; i  + '       g   i     T# ; i)-r   z%Y-%m-%d_%H:%M:%SNT)
activitiesrecord_shapesprofile_memorymodel_inference)group_by_stack_n)sort_by	row_limitz.logw.jsonrt   )rS   lower	precisionrK   __name__replacedatetimenowr   r   CPUCUDAr	   key_averagestablept_filter_bypt_num_rowsospathjoin
log_folderr)   write)	r   r   rL   inputs_typeprefixfilenameprof	prof_datar0   s	   &&&&     r   
profile_fnr      s    ##))+,Adnn-=Qt{{m1R[[M`M`adfiMjLkklmxlyyz  |D  |M  |M  |Q  |Q  |S  Te  {f  gFH>>(,,.>.C.CDTXim
 
 !2336
 4

 %%q%9??HYHYeieueu?v	77<<F84A(C  AGGI ! O 	6
 XU#O 43
 
 !  Os0   >F!	F	F!#F4FF!!F1	4G	c                   aa \         P                  ! 4       p\        P                  ! V4      pVP	                  R R7       S! S4       \
        P                  RVP	                  RR7       R24       \        P                  ! 4        \        P                  P                  4        \        V P                  R8g  VV3R lV P                  R7       \        P                   P#                  4        R# )g?)intervalzCPU usage: N%r   c                     < S ! S4      # r   r>   )r   rL   s   r   r@   measure_fn.<locals>.<lambda>  s	    r&zr   )is_gpufuncmonitor_type)r   getpidpsutilProcesscpu_percentr[   r\   gccollectra   r   empty_cacher   rK   r   r   r   flush)r   r   rL   pidprocesss   &ff  r   
measure_fnr      s    
))+CnnS!G%vJ
KK+g1141@ACD JJL	JJ4;;%/7IX\XiXij JJr   c                   a aa
 V3R  lo
V V
3R lpTpS P                   R8X  d	   V! V4       S P                  '       Ed5   \        S WAR4      pS P                   R8X  Ed   VR\        R4      )  pSP                  P
                  P                  4       pVR,           p\        P                  P                  V4      '       d[   \        P                  RV R	V 24       \        P                  ! V\        P                  P                  S P                  V4      4       SP                  P
                  P                  4       pVR
,           p\        P                  P                  V4      '       d[   \        P                  RV R	V 24       \        P                  ! V\        P                  P                  S P                  V4      4       SP                   P
                  P                  4       pVR,           p\        P                  P                  V4      '       d[   \        P                  RV R	V 24       \        P                  ! V\        P                  P                  S P                  V4      4       R# \        P#                  R4       \%        S WA4       V! V4      w  r\        P#                  R\        V^ ,          4       R24       \        P#                  RV	^ ,           24       \'        S WA4       R# )c                 ,   < SP                   ! R/ V B pV# )Nr>   )generate)rL   predicted_idsr   s   & r   get_pred_ids&run_hf_inference.<locals>.get_pred_ids
  s    00r   c                    < S! V 4      p. p\        SP                  4       F6  pVP                  SP                  P	                  VR R7      ^ ,          4       K8  	  W3# )Tskip_special_tokens)r   r6   appendrG   batch_decode)rL   r   transcriptionr   r   r   s   &   r   gen_and_dec%run_hf_inference.<locals>.gen_and_dec  sY    $V,t001A  !<!<]`d!<!efg!hi 2++r   rR   zgen-and-decrQ   Nr   z-encoder.json	Renaming  to z-decoder.jsonz-decoder-with-past.jsonz
Evaluating PyTorch...Generated token length:  tokensTranscription: )rS   r   r   lenencodersessionend_profilingr   r   isfiler[   warningrenamer   r   decoderdecoder_with_pastr\   r]   r   )r   rL   r   r   generate_fnnew_logname
new_prefixold_lognamer   r   r   s   f&f       @r   run_hf_inferencer     s   
,  Ko-F||| {MJ(*$_G}5J--//==?K$6Kww~~k**;-tK=IJ		+rww||DOO['QR--//==?K$6Kww~~k**;-tK=IJ		+rww||DOO['QR1199GGIK$'@@Kww~~k**;-tK=IJ		+rww||DOO['QR KK)*D+&#.v#6 M
KK*3}Q/?+@*AIJ
KK/-"2!345t[)r   c                   a a RV V3R llpV3R lpV3R lpV 3R lpS P                   R8w  d   TMTpV! V4      pS P                  '       dz   \        S WxR4      p	SP                  4       p
\        P                  RV
 RV	 24       \        P                  ! V
\        P                  P                  S P                  V	4      4       R	# \        P                  R
4       Tp\        S W{4       V! V4      pS P                   R8w  d   VP                  4       pV^ ,          pS P                  '       d(   \        P                  RV^ ,          ^ ,           24       MoV! V^ ,          ^ ,          4      p\        P                  R\        V4       R24       S P                   P#                  V^ ,          RR7      ^ ,          p\%        V 4       \'        S Wx4       R	# )Fc                   < SP                  4        Uu0 uF  q"P                  kK  	  pp\        V P                  4       4      pW4,
          p\	        V4      '       d$   \
        P                  R V 24       \        R4      hWC,
          p\	        V4      '       d%   V F  p\
        P                  RV R24       W K   	  SP                  R8w  d   SP                  4       pV P                  4        F  w  rVP                  W4       K  	  SP                  4        F5  pVP                  VP                  SP                  SP                  R7       K7  	  V# V # u upi )z(The following model inputs are missing: zEThere are missing inputs to the model. Please add them and try again.zRemoving unnecessary input 'z' from user provided inputsr   )device_type	device_id)ri   namesetkeysr   r[   errorrT   r\   rK   
io_bindingrU   bind_cpu_inputget_outputsbind_outputr   )rL   warmupmodel_inputmodel_inputsuser_inputsmissing_inputsunnecessary_inputsunnecessary_inputr  rd   re   outputr   r   s   &&          r   prepare_ort_inputs-run_ort_inference.<locals>.prepare_ort_inputsI  s/   <A<L<L<NO<N[((<NO&++-(%3~LLCNCSTUcdd )7!""%7!:;L:MMhij- &8
 ;;%))+J))!/ '++-&&v{{W[WeWe&f ./ Ps   Ec                 *   < SP                  V 4       V # r   )run_with_iobinding)r  r   s   &r   with_io_binding*run_ort_inference.<locals>.with_io_bindingd  s      ,r   c                 ,   < SP                  R V 4      pV# r   )run)rL   r   r   s   & r   without_io_binding-run_ort_inference.<locals>.without_io_bindingi  s    ))D&)r   c                    < SP                   V 9   d>   \        P                  ! V SP                   8H  4      ^ ,          ^ ,          pV RV^,            # V # )r   N)eos_token_idr*   where)r  	first_endr   s   & r   handle_output(run_ort_inference.<locals>.handle_outputn  sJ    &4+<+<!<=a@CI/IM**r   r   e2er   r   Nz
Evaluating ONNX Runtime...r   r   r   Tr   )F)rK   r   r   r   r[   r   r   r   r   r   r   r\   r]   copy_outputs_to_cpur^   r   rG   r   printr   )r   rL   r   r  r  r  r  r   
ort_inputsr   r   ort_evaluate_inputsort_outputsactual_outputr   s   f&f            r   run_ort_inferencer%  H  s}    6

 &*[[E%9/?QK#F+J||| {F ))+;-tK=AB
		+rww||DOO[IJ KK./$D+3j)K{{e!557a.Kok!nQ&7%89: &k!nQ&78.s=/A.B'JK33KNX\3]^_`!t[-r   c                     V P                   R9   d   \        WV4       R# V P                   R8X  d   \        WV4       R# \        RV P                    24      h)r   r   rm   N>   rQ   r   rR   )rS   r   r%  rT   )r   rL   r   s   &&&r   run_inferencer'    sM    HHu-				%$.+D,?,?+@ABBr   c            
         \         P                  ! 4       p V P                  R R\        R. RMOR7       V P                  RR\        RRR	7       V P                  R
R\        RR. RNORR7       V P                  R\        RRR7       V P                  R\        RRR7       V P                  R\        RRR7       V P                  RR\        RRR	7       T P                  RR\        \        P
                  P                  4       '       d   RMRRR.R7       V P                  RR \        ^ R!7       V P                  R"R#\        ^R!7       V P                  R$R%\        ^
R!7       V P                  R&\        ^R!7       V P                  R'\        R(R)R7       V P                  R*\        R+R!7       V P                  R,\        ^ R!7       V P                  R-\        ^R!7       V P                  R.\        ^R!7       V P                  R/\        R0R!7       V P                  R1\        R0R!7       V P                  R2\        ^R!7       V P                  R3\        R4R5R7       V P                  R6\        ^R7R7       V P                  R8\        R0R9R7       V P                  R:R;R<R=7       V P                  R>\        R?R@R7       V P                  RA\        RBRCR7       V P                  RDR;R<R=7       V P                  RE\        \        P                  P                  RF4      RGR7       V P                  4       p\        P                  P                  VP                  4       \        P                   ! VP                  4       VP"                  Vn        RVP&                  9   dS   VP"                  P)                  4        RH2Vn        VP*                  RI8X  d    VP*                  RJVP,                  /3Vn        VP&                  R8X  d   VP.                  '       g   Q RK4       hVP&                  R8X  d   VP0                  '       g   Q RL4       h\2        P4                  ! VP6                  4      Vn        V# )Oz-btz--benchmark-typeTrQ   r   )r   requiredchoicesz-mz--model-namez;Hugging Face name of model (e.g. 'openai/whisper-large-v2'))r   r)  helpz-pz--precisionfp32zePrecision for model. For ONNX models, the model's precision should be set before running this script.)r   r)  defaultr*  r+  z--hf-pt-model-pathr   zNPath to directory containing all PyTorch files (e.g. tokenizer, PyTorch model))r   r-  r+  z--hf-ort-dir-pathzaPath to directory containing all ONNX files (e.g. tokenizer, encoder, decoder, decoder_with_past)z--ort-model-pathzPath to ONNX modelz-az--audio-pathz%Path to audio file for E2E evaluationz-dz--devicer   r   )r   r-  r*  z-idz--device-id)r   r-  z-wz--warmup-runsz-nz
--num-runsz--seedz--sampling-ratei>  zSampling rate for audio (in Hz)z--max-lengthi  z--min-lengthz--num-beamsz--num-return-sequencesz--length-penaltyg      ?z--repetition-penaltyz--no-repeat-ngram-sizez--decoder-input-idsz[]zThe forced decoder ids for generation. Format is [start token, timestamp token, language token, task token]. Default is [start token]. See `decoder_input_ids` in https://github.com/microsoft/Olive/tree/main/examples/whisper for details.z--logits-processorzLWhether to use timestamps logits processor or not (0 for false, 1 for true).z--temperaturez!Temperature value for generation.z	--profileF
store_true)r-  actionz--pt-filter-byself_cpu_time_totalz"What to filter PyTorch profiler byz--pt-num-rowsi  z.Number of rows for PyTorch profiler to displayz	--verbosez--log-folder.zFolder to cache log filesExecutionProviderCUDAExecutionProviderr   z,Please specify a path to `--hf-ort-dir-path`z+Please specify a path to `--ort-model-path`)r   rR   rQ   r   )int4int8fp16r,  )r   ArgumentParseradd_argumentstrra   r   is_availableintfloatr   r   r   
parse_argsr*   randomseedmanual_seedrK   r   rS   upperr   r   r   r   astliteral_evalr:   )parserr   s     r   r=  r=    s?   $$&F
A   J   0t   ]	   p	   !	   n3Lst
**1133   }3B
oCC
lbA
sA6 )UIjk S#>
S!<
C;
0sAF
*D
.UCH
0sAF  |	   [	   0	   U<H
s,AHl   c4Fvw
U<H
S"'',,s:KRmnD IINN499	dii D###%)[[%6%6%8$99J"K""&=='+'>'>dnn@]&^D# h&###S%SS#e#"""Q$QQ" !--d.D.DEDKr   c                  f   \        4       p \        V P                  4       \        P	                  V P
                  4       R \        P                  P                  n	        \        P                  ! V P                  4      p\        P                  ! V P                  4      pV P                  R8w  d   RV P                   2MV P                  pV P                   R8H  ;'       g'    V P                   R9   ;'       d    V P                  R8g  p\#        V RV4       \#        V RV4       \#        V RV4       \#        V RR4       \#        V R	VP$                  4       \        P	                  R
V P&                   24       \)        V 4      pV P*                  R8X  d   VP-                  4        Uu0 uF  qfP.                  kK  	  ppRV9   V n        \#        V RRV9   4       \#        V RRV9   4       \#        V RRV9   4       V P&                  . 8X  d   VP2                  .V n        \-        V 4      p\5        WV4       R# u upi )Tr   zcuda:r6  rG   rc   r`   r^   Fr  zForced decoder prompt ids: r   rB   rX   r:   rY   r;   rZ   r<   N>   r4  r5  )r=  r   r}   r[   r\   __dict__ra   backendscudnn	benchmarkr   rx   rv   r   rK   r   r   setattrr  r:   r   rS   ri   r   r^   decoder_start_token_idr'  )	r   configrG   rc   r`   r   r  ort_model_inputsrL   s	            r   mainrN  $  s   <D
KK%)ENN"**4??;F 00AI04u0DeDNN+,$++M~~'hhDNN>N,N,g,gSWS^S^bgSgHD+y)D/=1D*h'D$e,D.&"5"56
KK-d.D.D-EFG dOEe#@E@P@P@RS@R,,@RS .2B B-/BFV/VW,.@DT.TU':J)JK!!R'&,&C&C%DD"F$& Ts   "H.__main__)-r   rB  r   r   loggingr   r   rw   numpyr*   r   ra   r   benchmark_helperr   r   onnxruntime_extensionsr   optimum.onnxruntimer   torch.profilerr   r   r	   tqdmr
   transformersr   r   r   onnxruntimer   	getLoggerr   r[   ri   r   r]   r   r   r   r%  r'  r=  rN  r>   r   r   <module>rZ     s     
  	  	 
      9 3 8 E E  S S 			8	$EP=@.b:$=*@O.dC~B!'H zF r   