+
    9i	J                     "   ^ RI t ^ RIt^ RIHt ^ RIHt ^ RIHt ^ RIt^ RI	t	^ RI
Ht ^ RIHtHt ]]]]],          ,          3,          t]P(                  ! ]4      t ! R R4      t ! R	 R
4      t ! R R4      t ! R R]4      t ! R R4      tR# )    N)OrderedDict)Mapping)Any)TensorProto)InferenceSession
RunOptionsc                   8  a  ] tR t^t o ]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       t]V 3R	 lR
 l4       t	]V 3R lR l4       t
]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       tRtV tR# )
TypeHelperc                ,   < V ^8  d   QhRS[ RS[RS[/# )   ort_sessionnamereturn)r   str)format__classdict__s   "h/var/www/html/photoedit/myenv/lib/python3.14/site-packages/onnxruntime/transformers/io_binding_helper.py__annotate__TypeHelper.__annotate__   s#     9 9$4 9C 9C 9    c                    \        V P                  4       4       F$  w  r#VP                  V8X  g   K  VP                  u # 	  \	        R V R24      h)zinput name 
 not found)	enumerate
get_inputsr   type
ValueError)r   r   _iinputs   &&  r   get_input_typeTypeHelper.get_input_type   sH    ";#9#9#;<IBzzT!zz! = ;tfJ788r   c                &   < V ^8  d   QhRS[ RS[ /# )r   r   r   r   )r   r   s   "r   r   r      s     : :3 :3 :r   c                    \        V P                  4       4       F$  w  r#VP                  V8X  g   K  VP                  u # 	  \	        R V R24      h)zoutput name r   )r   get_outputsr   r   r   )r   r   r   outputs   &&  r   get_output_typeTypeHelper.get_output_type   sH    #K$;$;$=>JB{{d"{{" ? <vZ899r   c                    < V ^8  d   QhRS[ /# r   ort_typer"   )r   r   s   "r   r   r   $   s     4 4 4r   c                
   R \         P                  R\         P                  R\         P                  R\         P                  R\
        R\         P                  R\         P                  /pW9  d   \        V  R24      hW,          # )tensor(int64)tensor(int32)tensor(float)tensor(float16)tensor(bool)tensor(uint8)tensor(int8) not found in map)	numpylonglongintcfloat32float16booluint8int8r   )r*   ort_type_to_numpy_type_maps   & r   ort_type_to_numpy_type!TypeHelper.ort_type_to_numpy_type#   sk     U^^UZZU]]u}}DU[[EJJ&
" 5z):;<<)33r   c                    < V ^8  d   QhRS[ /# r)   r"   )r   r   s   "r   r   r   4   s     4 4 4r   c                >   R \         P                  R\         P                  R\         P                  R\         P                  R\         P
                  R\         P                  R\         P                  R\         P                  /pW9  d   \        V  R24      hW,          # 	r,   r-   r.   r/   ztensor(bfloat16)r0   r1   r2   r3   )
torchint64int32r7   r8   bfloat16r9   r:   r;   r   )r*   ort_type_to_torch_type_maps   & r   ort_type_to_torch_type!TypeHelper.ort_type_to_torch_type3   sx     U[[U[[U]]u}}EJJU[[EJJ	&
" 5z):;<<)33r   c                <   < V ^8  d   QhRS[ RS[S[S[3,          /# r   r   r   )r   dictr   int)r   r   s   "r   r   r   E   s$     ! !*: !tCH~ !r   c                   / pV P                  4        F.  p\        P                  VP                  4      WP                  &   K0  	  V P                  4        F.  p\        P                  VP                  4      WP                  &   K0  	  V# )z9Create a mapping from input/output name to onnx data type)r   r
   ort_type_to_onnx_typer   r   r$   )r   name_to_onnx_typer   r%   s   &   r   get_io_onnx_type_mapTypeHelper.get_io_onnx_type_mapD   sp      ++-E,6,L,LUZZ,Xjj) . "--/F-7-M-Mfkk-Zkk* 0  r   c                    < V ^8  d   QhRS[ /# r)   r"   )r   r   s   "r   r   r   P   s     3 3 3r   c                >   R \         P                  R\         P                  R\         P                  R\         P                  R\         P
                  R\         P                  R\         P                  R\         P                  /pW9  d   \        V  R24      hW,          # rA   )
r   INT64INT32FLOATFLOAT16BFLOAT16BOOLUINT8INT8r   )r*   ort_type_to_onnx_type_maps   & r   rN    TypeHelper.ort_type_to_onnx_typeO   s     [..[..[..{22 4 4K,,[..K,,	%
! 4z):;<<(22r   c                4   < V ^8  d   QhRS[ P                  /# )r   
numpy_type)r4   dtype)r   r   s   "r   r   r   a   s     8 8U[[ 8r   c                
   \         P                  \        P                  \         P                  \        P
                  \         P
                  \        P
                  \         P                  \        P                  \         P                  \        P                  \        \        P                  \         P                  \        P                  \         P                  \        P                  /pW9  d   \        V  R 24      hW,          # r3   )r4   r5   rB   rC   r6   rD   r7   r8   r9   r:   r;   r   )r_   numpy_type_to_torch_type_maps   & r   numpy_type_to_torch_type#TypeHelper.numpy_type_to_torch_type`   s     NNEKKJJKKMM5==MM5==%**KKJJ

	(
$ 9
|+<=>>+77r   c                4   < V ^8  d   QhRS[ P                  /# )r   
torch_type)rB   r`   )r   r   s   "r   r   r   r   s     8 8U[[ 8r   c                   \         P                  \        P                  \         P                  \        P
                  \         P                  \        P                  \         P                  \        P                  \         P                  \        \         P                  \        P                  /pW9  d   \        V  R 24      hW,          # rb   )rB   rC   r4   r5   rD   r6   r7   r8   r9   r:   r   )rg   torch_type_to_numpy_type_maps   & r   torch_type_to_numpy_type#TypeHelper.torch_type_to_numpy_typeq   sz     KKKKMM5==MM5==JJKK(
$ 9
|+<=>>+77r   c                P   < V ^8  d   QhRS[ RS[S[S[P                  3,          /# rJ   )r   rK   r   r4   r`   )r   r   s   "r   r   r      *     " "+; "S%++EU@V "r   c                   / pV P                  4        F.  p\        P                  VP                  4      WP                  &   K0  	  V P                  4        F.  p\        P                  VP                  4      WP                  &   K0  	  V# )z:Create a mapping from input/output name to numpy data type)r   r
   r=   r   r   r$   )r   name_to_numpy_typer   r%   s   &   r   get_io_numpy_type_map TypeHelper.get_io_numpy_type_map   s       ++-E-7-N-Nuzz-Zzz* . "--/F.8.O.OPVP[P[.\{{+ 0!!r   c                P   < V ^8  d   QhRS[ RS[S[S[P                  3,          /# rJ   )r   rK   r   rB   r`   )r   r   s   "r   r   r      rm   r   c                   / pV P                  4        F.  p\        P                  VP                  4      WP                  &   K0  	  V P                  4        F.  p\        P                  VP                  4      WP                  &   K0  	  V# )z:Create a mapping from input/output name to torch data type)r   r
   rG   r   r   r$   )r   name_to_torch_typer   r%   s   &   r   get_io_torch_type_map TypeHelper.get_io_torch_type_map   rr   r    N)__name__
__module____qualname____firstlineno__staticmethodr   r&   r=   rG   rP   rN   rd   rj   rp   rv   __static_attributes____classdictcell__r   s   @r   r
   r
      s     9 9 : : 4 4 4 4  ! ! 3 3  8 8  8 8 " " " "r   r
   c                   l   a  ] tR t^t o ]V 3R lR l4       t]V 3R lR l4       t]RR l4       tRtV t	R# )	IOBindingHelperc                    < V ^8  d   QhRS[ /# )r   r   )r   )r   r   s   "r   r   IOBindingHelper.__annotate__   s      (8 r   c                    / pVP                  4        F]  w  rE\        P                  W4      p\        P                  V4      p\        P
                  ! \        P                  ! V4      WrR7      W4&   K_  	  V# )zpReturns a dictionary of output name as key, and 1D tensor as value. The tensor has enough space for given shape.)r`   device)itemsr
   r&   rG   rB   emptyr4   prod)r   output_shapesr   output_buffersr   shaper*   rg   s   &&&     r   get_output_buffers"IOBindingHelper.get_output_buffers   sb     (..0KD!11+DH#::8DJ#(;;uzz%/@
#bN  1 r   c          	         < V ^8  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[ P                  ,          /# )r   	input_idsposition_idsattention_maskpast)rB   Tensorlist)r   r   s   "r   r   r      sN     R R<<R llR 	R
 5<< Rr   c                   \         P                  V 4      pV P                  4       pVP                  4       '       g   Q hVP	                  RVP
                  P                  ^ VR,          \        VP                  4       4      VP                  4       4       Ve   \        V4       F  w  rV
P                  4       '       g   Q hV
P                  4       pV^ 8X  d   VP                  4       pVP	                  RV	 2V
P
                  P                  ^ VRV	 2,          \        V
P                  4       4      V4       K  	  Veo   VP                  4       '       g   Q hVP	                  RVP
                  P                  ^ VR,          \        VP                  4       4      VP                  4       4       Veo   VP                  4       '       g   Q hVP	                  RVP
                  P                  ^ VR,          \        VP                  4       4      VP                  4       4       V P                  4        F  pVP                  pW],          p\        P                  V RVP
                  P                   R\        VP                  4       4       24       VP                  VVP
                  P                  ^ W},          Wm,          VP                  4       4       K  	  V# )zdIO binding for a session: bind inputs (input_ids, position_ids, attention_mask, past_*) and outputs.r   past_r   r   z device type=z shape=)r
   rP   
io_bindingis_contiguous
bind_inputr   r   r   sizedata_ptrr   r$   r   loggerdebugbind_output)r   r   r   r   r   r   r   rO   r   ipast_ir   r%   output_nameoutput_buffers   &&&&&&&        r   prepare_io_binding"IOBindingHelper.prepare_io_binding   s    ';;KH !++-
 &&((((!!k*!" 	
 &t_	++----!??,q=  )113H%%A3KMM&&%aSk2' -$ %!//1111!! %%**!"23^((*+'') #--////!!##((!.1\&&()%%' "--/F ++K*7MLLK=m6J6J6O6O5PPWX\]j]o]o]qXrWstu""$$))!.*&&(	 0 r   c                   . pV P                  4        F  pVP                  pW,          pW&,          pV^ \        P                  ! V4       P	                  V4      P                  4       P                  4       p	V'       d0   VP                  V	P                  4       P                  4       4       K  VP                  V	4       K  	  V# )z3Copy results to cpu. Returns a list of numpy array.)	r$   r   r4   r   reshapeclonedetachappendcpu)
r   r   r   return_numpyort_outputsr%   r   bufferr   copy_tensors
   &&&&      r   "get_outputs_from_io_binding_buffer2IOBindingHelper.get_outputs_from_io_binding_buffer   s     !--/F ++K#0F!.E UZZ%67??FLLNUUWK"";??#4#:#:#<="";/ 0 r   rx   N)T)
ry   rz   r{   r|   r}   r   r   r   r~   r   r   s   @r   r   r      sB       R Rh  r   r   c                      a  ] tR tRt o RtRV 3R lR lltV 3R lR ltR tV 3R lR	 ltV 3R
 lR lt	RV 3R lR llt
]RV 3R lR ll4       tRtV tR# )CudaSessioni  zLInference Session with IO Binding for ONNX Runtime CUDA or TensorRT providerc                :   < V ^8  d   QhRS[ RS[P                  /# )r   r   r   )r   rB   r   )r   r   s   "r   r   CudaSession.__annotate__
  s      1 1$4 1ell 1r   c                6   Wn         V P                   P                  4        Uu. uF  qDP                  NK  	  upV n        V P                   P	                  4        Uu. uF  qUP                  NK  	  upV n        \        P                  V P                   4      V n        \        P                  V P                   4      V n
        V P                   P                  4       V n        W0n        \        4       V n        \        4       V n        W n        / V n        R # u upi u upi N)r   r   r   input_namesr$   output_namesr
   rP   io_name_to_onnx_typerv   io_name_to_torch_typer   enable_cuda_graphr   input_tensorsoutput_tensorsr   buffer_sharing)selfr   r   r   r   r%   s   &&&&  r   __init__CudaSession.__init__
  s    &484D4D4O4O4QR4Q5JJ4QR7;7G7G7S7S7UV7UV[[7UV$.$C$CDDTDT$U!%/%E%EdFVFV%W"**557!2(])m /1 SVs   DDc                &   < V ^8  d   QhRS[ RS[ /# )r   
input_namer   r"   )r   r   s   "r   r   r     s     6 6S 6s 6r   c                    WP                   9   g   Q hW P                  9   g   Q hW P                  V&   WP                  V&   R # r   )r   r   r   )r   r   r   s   &&&r   set_buffer_sharingCudaSession.set_buffer_sharing  sB    -----/////*5J'+5K(r   c                    V = V =V =R # r   )r   r   r   )r   s   &r   __del__CudaSession.__del__   s    Or   c                :   < V ^8  d   QhRS[ RS[P                  /# )r   r   tensor)r   rB   r   )r   r   s   "r   r   r   %  s#     D D# Du|| Dr   c           	        VP                   P                  e   VP                   P                  M^ p\        VP                  4      ^ 8X  d   ^.M\	        VP                  4      pV P
                  P                  VVP                   P                  VV P                  V,          VVP                  4       4       WP                  9   d   V P
                  P                  V P                  V,          VP                   P                  VV P                  V,          VVP                  4       4       W P                  V P                  V,          &   R # R # r   )r   indexlenr   r   r   r   r   r   r   r   r   r   )r   r   r   	device_idtensor_shapes   &&&  r   bind_input_and_buffer_sharing)CudaSession.bind_input_and_buffer_sharing%  s    +1==+>+>+JFMM''PQ	!&,,/14s$v||:L""MM%%d+OO	
 &&&OO''##D)""))$/! >D 3 3D 9: 'r   c                    < V ^8  d   QhRS[ /# )r   
shape_dict)	ShapeDict)r   r   s   "r   r   r   =  s     $ $9 $r   c                   V P                   '       d   VP                  4        F  w  r#W P                  9   g   K  W P                  9   dB   \	        V P                  V,          P
                  4      \	        V4      8X  d   K]  \        R4      hV P                  V,          p\        P                  ! \	        V4      VR7      P                  V P                  R7      pWPP                  V&   V P                  W%4       K  	  VP                  4        EFd  w  r#W P                  9   g   K  W P                  9   d7   \	        V P                  V,          P
                  4      \	        V4      8X  d   K^  W P                  9   d   Kp  V P                  V,          p\        P                  ! \	        V4      VR7      P                  V P                  R7      pWPP                  V&   V P                   P#                  TVP                  P$                  VP                  P&                  e   VP                  P&                  M^ V P(                  V,          \+        VP-                  4       4      VP/                  4       4       EKg  	  R# )z Allocate tensors for I/O Bindingz(Expect static input shape for cuda graph)r`   )r   N)r   r   r   r   tupler   RuntimeErrorr   rB   r   tor   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   torch_dtyper   s   &&    r   allocate_buffersCudaSession.allocate_buffers=  s   !!!)//1+++111 !3!3D!9!?!?@E%LP$*+UVV"&"<"<T"BK"[[u[ILLTXT_T_L`F/5&&t,66tD  2 &++-KD(((...59L9LT9R9X9X3Y]bch]i3i..."88>U5\EHHPTP[P[H\,2##D)++MM&&+1==+>+>+JFMM''PQ--d3'OO% .r   Nc                V   < V ^8  d   QhRS[ S[S[P                  3,          RS[RS[/# )r   	feed_dictrun_optionssynchronize)rK   r   rB   r   r   r9   )r   r   s   "r   r   r   c  s1     # #tC$56 #Z #ei #r   c                   VP                  4        EF  w  rE\        V\        P                  4      '       d   VP	                  4       '       g   Q hW@P
                  9   g   KP  V P                  '       d   V P                  V,          P                  4       VP                  4       8X  g   Q hV P                  V,          P                  VP                  8X  g   Q hVP                  P                  R8X  g   Q hV P                  V,          P                  V4       EK  V P                  WE4       EK  	  V'       dg   V P                  P                  4        V P                   P#                  V P                  V4       V P                  P%                  4        V P&                  # V P                   P#                  V P                  V4       V P&                  # )z$Bind input tensors and run inferencecuda)r   
isinstancerB   r   r   r   r   r   nelementr`   r   r   copy_r   r   synchronize_inputsr   run_with_iobindingsynchronize_outputsr   )r   r   r   r   r   r   s   &&&&  r   inferCudaSession.inferc  sR   %OO-LDfell338L8L8N8NNN''')))--d3<<>&//BSSSS--d399V\\III!==--777&&t,226:66tD . OO..0//MOO//1 """ //M"""r   c          
      H   < V ^8  d   QhRS[ RS[RS[ RS[S[S[3,          /# )r   r   r   streamr   )rL   r9   rK   r   r   )r   r   s   "r   r   r   z  s7      S T SV _cdgildl_m r   c                @    R V RRRV/pV^ 8w  d   \        V4      VR&   V# )r   arena_extend_strategykSameAsRequestedr   user_compute_streamr"   )r   r   r   optionss   &&& r   get_cuda_provider_options%CudaSession.get_cuda_provider_optionsy  s9     #%7!2
 Q;-0[G)*r   )r   r   r   r   r   r   r   r   r   r   r   F)NT)r   )ry   rz   r{   r|   __doc__r   r   r   r   r   r   r}   r   r~   r   r   s   @r   r   r     s[     V1 1 6 6
D D0$ $L# #,   r   r   c                   t   a a ] tR tRt oR	V3R lV 3R llltR
V3R lR lltR
V3R lV 3R llltRtVtV ;t	# )
GpuBindingi  c                |   < V ^8  d   QhRS[ RS[P                  RS[RS[RS[RS[RS[S[S[3,          R,          /# )	r   r   r   r   enable_gpu_graphgpu_graph_idr   r   N)r   rB   r   r   r9   rL   rK   r   )r   r   s   "r   r   GpuBinding.__annotate__  sb     * *%* * 	*
 * * * S#X-*r   c                  < \         S
V `  WV4       V'       d+   VP                  4        F  w  rV P                  W4       K  	  V P	                  V4       WPn        V'       d   \        P                  ! V4      MR V n        W`n	        R V n
        R # r   )superr   r   r   r   r   copydeepcopyr   r   last_run_gpu_graph_id)r   r   r   r   r   r   r   r   r   r   	__class__s   &&&&&&&&  r   r   GpuBinding.__init__  st     	.>?+9+?+?+A'
''
@ ,B 	j)(7G$--
3T%)"r   c                &   < V ^8  d   QhRS[ RS[/# )r   disable_cuda_graph_in_runr   )r9   r   )r   r   s   "r   r   r     s     	 	 	* 	r   c                    \        4       pV'       d   RMV P                  pVP                  R\        V4      4       W0n        V# )   r   )r   r   add_run_config_entryr   r  )r   r  r   r   s   &&  r   get_run_optionsGpuBinding.get_run_options  s8    ,6rD<M<M$$^S5FG%1"r   c                P   < V ^8  d   QhRS[ S[S[P                  3,          RS[/# )r   r   r  )rK   r   rB   r   r9   )r   r   s   "r   r   r     s*     5 5tC$56 5SW 5r   c                   < V P                  V4      pV P                  '       d   VP                  R R4       \        SV `  W4      # )'disable_synchronize_execution_providers1)r
  r   r	  r   r   )r   r   r  r   r  s   &&& r   r   GpuBinding.infer  s>    **+DE;;;,,-VX[\w}Y44r   )r   r  r   r   )Fr  r   Nr   )
ry   rz   r{   r|   r   r
  r   r~   r   __classcell__)r  r   s   @@r   r   r     s(     * *.	 	5 5 5r   r   c                   P   a  ] tR tRt o RtR	V 3R lR lltR
V 3R lR lltRtV tR# )GpuBindingManageri  zA manager for I/O bindings that support multiple CUDA Graphs.
One cuda graph is reused for same input shape. Automatically add a new cuda graph for new input shape.
c                F   < V ^8  d   QhRS[ RS[P                  RS[RS[/# )r   r   r   r   max_cuda_graphs)r   rB   r   rL   )r   r   s   "r   r   GpuBindingManager.__annotate__  s0     / /$4 /ell /TW /nq /r   c                R    Wn         W n        . V n        R V n        W0n        W@n        R # r   )r   r   graph_bindingsno_graph_bindingr   r  )r   r   r   r   r  s   &&&&&r   r   GpuBindingManager.__init__  s,    & ! !%.r   Nc                V   < V ^8  d   QhRS[ RS[RS[S[S[3,          R,          RS[/# )r   r   use_cuda_graphr   Nr   )r   r9   rK   r   r   )r   r   s   "r   r   r    s@      !  ! !  ! S#X-	 !
 
 !r   c           
     P   V P                    F  pVP                  V8X  g   K  Vu # 	  \        V P                   4      V P                  8  g	   V'       gt   V P                  f?   \        V P                  V P                  WP                  VR7      V n        V P                  # V P                  P                  V4       V P                  # \        V P                  V P                  VR\        V P                   4      V P                  VR7      pV P                   P                  V4       V# )N)r   r   T)r   r   r   r   )r  r   r   r  r  r   r   r   r   r   r   )r   r   r  r   gpu_graph_bindings   &&&& r   get_bindingGpuBindingManager.get_binding  s     "&!4!4 ++z9(( "5 t""#t';';;N$$,(2$$dkk:kkbp)%
 ((( %%66zB((( 'KK!T001;;)
 	""#45  r   )r   r  r  r  r   r   )r   r  )FN)	ry   rz   r{   r|   r   r   r  r~   r   r   s   @r   r  r    s#     / / !  !  !r   r  )r   loggingcollectionsr   collections.abcr   typingr   r4   rB   onnxr   onnxruntimer   r   r   r   r   rL   r   	getLoggerry   r   r
   r   r   r   r  rx   r   r   <module>r(     s      # #     4 Cc**+				8	$A" A"Hm m`~ ~B)5 )5X3! 3!r   