+
    9iG                        ^ RI t ^ RIHt ^ RIHt ^ RIHtHtHtH	t	H
t
 ^ RIHt ] P                  ! ]4      t ! R R]4      t ! R R	]4      tR# )
    N)FusionAttention)Fusion)FunctionProto	NodeProtoTensorProtohelpernumpy_helper	OnnxModelc                   j   a a ] tR t^t oRtV3R lV 3R lltR
V3R lR lltR tR tR t	R	t
VtV ;t# )FusionRotaryAttentionz]
Fuse Attention subgraph with rotary positional embeddings into one MultiHeadAttention node.
c                ,   < V ^8  d   QhRS[ RS[RS[/# )   modelhidden_size	num_heads)r   int)format__classdict__s   "n/var/www/html/photoedit/myenv/lib/python3.14/site-packages/onnxruntime/transformers/fusion_rotary_attention.py__annotate__"FusionRotaryAttention.__annotate__   s)     
 

 
 	
    c                4   < \         SV `  VVVR . ROR7       R# )T)use_multi_head_attentionsearch_op_typesN)SimplifiedLayerNormalization SkipSimplifiedLayerNormalizationLayerNormalizationSkipLayerNormalizationAdd)super__init__)selfr   r   r   	__class__s   &&&&r   r#   FusionRotaryAttention.__init__   s(     	%) 	 	
r   c                   < V ^8  d   QhRS[ RS[ RS[RS[RS[RS[ RS[ RS[ R	S[ R
S[ RS[ RS[R,          RS[R,          /# )r   inputoutputq_rotaryk_rotaryv_matmul	attn_maskadd_qkpast_kpast_v	present_k	present_vscaleNreturn)strr   float)r   r   s   "r   r   r   (   s     6 66 6 	6
 6 6 6 6 6 6 6 6 t|6 
T	6r   c                   V P                   ^ 8  g   Q hV P                  ^ 8  dT   V P                  V P                   ,          ^ 8w  d2   \        P                  RV P                   RV P                    24       R# V P                  P                  R4      pVP                  ^ ,          VP                  ^ ,          VP                  ^ ,          RVVVV	.pV.pV
'       d   V'       d   VP                  W.4       \        P                  ! RVVVR7      pRVn
        VP                  P                  \        P                  ! RV P                   4      .4       Ve2   VP                  P                  \        P                  ! R	V4      .4       V P                  eE   VP                  P                  \        P                  ! R
\        V P                  4      4      .4       V P                  R4       V# )r   z)fuse_rotary_attention: input hidden size z# is not a multiple of num of heads NMultiHeadAttention inputsoutputsnamecom.microsoftr   r3   mask_filter_value)r   r   loggerdebugr   create_node_namer)   extendr   	make_nodedomain	attributemake_attributer?   r6   increase_counter)r$   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   mha_node_name
mha_inputsmha_outputsmha_nodes   &&&&&&&&&&&&&    r   create_mha_node%FusionRotaryAttention.create_mha_node(   s    ~~!!!aT%5%5%F1$LLL;D<L<L;MMpquqq  qA  B 

334HIOOAOOAOOA	

 h	56## 	
 *!!6#8#8dnn#U"VW%%v'<'<We'L&MN!!-%%v'<'<=PRWX\XnXnRo'p&qr23r   c	                   V P                   P                  VR .^.4      p	V P                   P                  VR .^.4      p
V	e   V
f   R# V	^ ,          V
^ ,          rV P                   P                  V. RO. RO4      pV P                   P                  V. RO. RO4      pV P                   P                  V. RO. RO4      pV P                   P                  V. RO. RO4      pVe   Ve	   Ve   Vf   R# Vw  pppVw  pppVP                  ^ ,          V8w  g   VP                  ^ ,          V8w  d   R# V^,          P                  VP                  8w  g#   V^,          P                  VP                  8w  d   R# V P                   P                  VR .^.4      pV P                   P                  VR .^.4      pVe   Vf   R# V^ ,          V^ ,          ppV P                   P                  V. RO. R	O4      pV P                   P                  V. R
O. RO4      pV P                   P                  V. RO. RO4      pV P                   P                  V. RO. RO4      pVe   Ve	   Ve   Vf   R# V^,          P                  VP                  8w  gg   V^,          P                  VP                  8w  gE   V^,          P                  VP                  8w  g#   V^,          P                  VP                  8w  d   R# V P                   P                  VR .^.4      pVf   R# V^ ,          pV P                   P                  V. RO. R	O4      p V P                   P                  V. R
O. RO4      p!V e   V!f   R# V ^,          P                  VP                  8w  g#   V!^,          P                  VP                  8w  d   R# V P                   P                  VR .^.4      p"V"f   R# V"^ ,          p#V P                   P                  V#. RO. R	O4      p$V P                   P                  V#. RO. RO4      p%V$e   V%f   R# V$^,          P                  VP                  8w  g#   V%^,          P                  VP                  8w  d   R# V$^,          p&V ^,          p'V^,          p(VP                  ^ ,          p)V&P                  ^ ,          V)8w  g1   V'P                  ^ ,          V)8w  g   V(P                  ^ ,          V)8w  d   R# V P                   P                  V. RO. RO4      p*V P                   P                  V. RO. RO4      p+V*e   V*w  pp,p-MV+e	   V+w   pp,p-MR# V-P                  ^ ,          R9  d   R# V P                   P                  V,. R
O. RO4      p.V P                   P                  V-. R
O. RO4      p/V P                   P                  V-R.^.4      p0V.e	   V/e   V0f   R# V.^,          P                  V/^,          P                  8w  g*   V.^,          P                  V/^,          P                  8w  d   R# V/^,          P                  ^ ,          V0^ ,          P                  ^ ,          8w  d   R# R# )ConcatF	UnsqueezeTrQ   GatherShaper   r   r      r   r   )r   r   r   )rQ   MulrS   rT   r   r   r   r   )rQ   r!   rS   rT   rW   r   r   r   )r   r   r   r   rP   Slicer\   CastrP   r\   r\   >   r-   attention_mask)r   r   rW   r   )r   match_parent_pathr(   r=   r)   )1r$   reshape_qkv_2reshape_qkv_1reshape_q_2reshape_k_2reshape_v_2reshape_v_1r.   
root_inputconcat_qkv_2_pathconcat_qkv_1_pathconcat_qkv_2concat_qkv_1reshape_qkv_2_path_1reshape_qkv_2_path_2reshape_qkv_1_path_1reshape_qkv_1_path_2_gather_1shape_1gather_2shape_2concat_v_2_pathconcat_v_1_path
concat_v_2
concat_v_1reshape_v_2_path_1reshape_v_2_path_2reshape_v_1_path_1reshape_v_1_path_2concat_k_2_path
concat_k_2reshape_k_2_path_1reshape_k_2_path_2concat_q_2_path
concat_q_2reshape_q_2_path_1reshape_q_2_path_2mul_qmul_kmul_vgather_1_outattn_mask_path_1attn_mask_path_2
slice_qk_2
slice_qk_1slice_qk_2_pathslice_qk_1_path_1slice_qk_1_path_2s1   &&&&&&&&&                                        r   &check_runtime_shape_paths_for_function<FusionRotaryAttention.check_runtime_shape_paths_for_function`   s    !JJ88
UVTWX JJ88
UVTWX$(9(A%6q%9;LQ;Ol#zz;;LJjluv#zz;;LJjluv#zz;;LJjluv#zz;;LJjluv (#+#+#+38W38W ==z)W]]1-=-K  "''8==8<PQR<S<X<X\d\i\i<i **66{XJQRPST**66{XJQRPST"o&=!0!3_Q5GJ
!ZZ99?
 "ZZ99?
 "ZZ99*Ffhqr!ZZ99*Ffhqr&!)!)!) q!&&(--7!!$))X]]:!!$))X]]:!!$))X]]: **66{XJQRPST"$Q'
!ZZ99?
 "ZZ99?
 %);)C a %%6:LQ:O:T:TX`XeXe:e **66{XJQRPST"$Q'
!ZZ99?
 "ZZ99*Ffhqr%);)C a %%6:LQ:O:T:TX`XeXe:e #1%"1%"1%q);;q>\)U[[^|-Ku{{[\~amOm  ::77@\^gh::77@dfrs'(8%Az:)+;(Aq*jA&EE**66?
 !JJ88?
 !JJ88k]UVTWX"&7&?CTC\ 1""&7&:&?&???STCUCZCZ^opq^r^w^wCw Q%%a(,=a,@,F,Fq,IIr   c                   V P                   P                  VR .^.4      pVf   R# V^ ,          pV P                   P                  V. RO. RO4      pV P                   P                  V. RO. RO4      p	Ve   V	f   R# Vw  rpV	w  rpVP                  ^ ,          V8w  g   VP                  ^ ,          V8w  d   R# V P                   P                  VR .^.4      pVf   R# V^ ,          pV P                   P                  V. RO. RO4      pV P                   P                  V. RO. RO4      pVe   Vf   R# V^,          P                  VP                  8w  g#   V^,          P                  VP                  8w  d   R# V P                   P                  VR .^.4      pVf   R# V^ ,          pV P                   P                  V. RO. RO4      pV P                   P                  V. RO. RO4      pVe   Vf   R# V^,          P                  VP                  8w  g#   V^,          P                  VP                  8w  d   R# V P                   P                  VR .^.4      pVf   R# V^ ,          pV P                   P                  V. RO. RO4      pV P                   P                  V. RO. RO4      pVe   Vf   R# V^,          P                  VP                  8w  g#   V^,          P                  VP                  8w  d   R# R# )rP   FTrR   rU   rV   )r   r`   r(   r=   )r$   reshape_qkv	reshape_q	reshape_k	reshape_vrg   concat_qkv_path
concat_qkvreshape_qkv_path_1reshape_qkv_path_2rp   rq   rr   rs   rt   concat_v_pathconcat_vreshape_v_path_1reshape_v_path_2concat_k_pathconcat_kreshape_k_path_1reshape_k_path_2concat_q_pathconcat_qreshape_q_path_1reshape_q_path_2s   &&&&&&                     r   #check_runtime_shape_paths_for_nodes9FusionRotaryAttention.check_runtime_shape_paths_for_nodes   s    **66{XJQRPST"$Q'
!ZZ99*Ffhqr!ZZ99*Ffhqr%);)C1W1W ==z)W]]1-=-K 

44Y
QCP  #::77Bbdmn::77Bbdmn#'7'? A##x}}48H8K8P8PT\TaTa8a 

44Y
QCP  #::77Bbdmn::77Bbdmn#'7'? A##x}}48H8K8P8PT\TaTa8a 

44Y
QCP  #::77Bbdmn::77Bbdmn#'7'? A##x}}48H8K8P8PT\TaTa8ar   c                *"  a  VP                   R9  d   R# RpS P                  P                  V. RO. RO4      pS P                  P                  V. R O. R!O4      pS P                  P                  V. R"O. RO4      pVe
   Vw  rrpTpM0Ve	   Vw  rrTpM$Ve
   Vw   rrTpM\        P	                  R4       R# RRRrpRpRpS P                  P                  V. R#O. R$O4      pS P                  P                  V. R%O. R&O4      pS P                  P                  V. R'O. R(O4      pS P                  P                  V. R)O. R*O3. R+O. R,O3. R-O. R.O3. R/O. R0O3. R1O. R2O3. R3O. R4O3. R5O. R6O3. R3O. R7O3. R3O. R8O3.	RR7      w  pppS P                  P                  V. R9O. R:O4      pVe   Vw  ppppppTpS P                  P                  VRR.^ ^.4      pVf   \        P	                  R	4       R# V^ ,          P                  ^ ,          pVR;,          P                  ^ ,          pVP                  ^ ,          pMVe1   Vw  ppppTpVP                  ^ ,          pVP                  ^ ,          pMVe   Vw  pppTpVP                  ^ ,          pMVeK   \        V4      ^	8X  d;   V^ ,          R<R w  ppppTpVP                  ^ ,          pVP                  ^ ,          pMNVe4   Vw  pppppTpTpVP                  ^ ,          pVP                  ^ ,          pM\        P	                  R
4       R# S P                  P                  V. R=O. R>O4      pRRp pVe	   Vw  pppp M\        P	                  R4       R# RRp"p!S P                  P                  V. R?O. R(O4      p#S P                  P                  V. R@O. R!O4      p$S P                  P                  V. RAO. RBO4      p%S P                  P                  V. RCO. RDO4      p&S P                  P                  V. REO. RFO4      p'S P                  P                  V. RGO. RBO4      p(S P                  P                  V. RHO. RIO4      p)V#e   V#w  pp*p+V*P                  ^ ,          p!MV$e   V$w   pp*p+V*P                  ^ ,          p!MV%e+   S P                  V%^ ,          P                  ^ ,          4      p"MV&e+   S P                  V&^ ,          P                  ^ ,          4      p"MV'e   V'^ ,          P                  ^ ,          p"MdV(e   V(^ ,          P                  ^ ,          p"MEV)e+   S P                  V)^ ,          P                  ^ ,          4      p"M\        P	                  R4       R# RRp-p,Rp.Rp/Rp0S P                  P                  V . RJO. R$O4      p1S P                  P                  V . RKO. RO4      p2S P                  P                  V . RLO. RMO4      p3S P                  P                  V . RNO. RIO3. ROO. RPO3. RQO. RRO3. RSO. RTO3. RUO. RVO3. RWO. RXO3. RYO. RZO3. RWO. R[O3. RWO. R\O3.	RR7      w  pp4pS P                  P                  V . R]O. R^O4      p5V1e   V1w  p6pp7pp8p9T1p.S P                  P                  V7RR.^ ^.4      p:V:f   \        P	                  R4       R# V:^ ,          P                  ^ ,          p,V:R;,          P                  ^ ,          p;V7P                  ^ ,          p-VV;8X  g   Q hEMV2e   V2w  pp8pp<p9T2p.V8P                  ^ ,          p-MV3e3   V3w  pp7p8pp<p9T3p.V7P                  ^ ,          p,V7P                  ^ ,          p-MV4eX   \        V44      ^	8X  dH   V4^ ,          R_R w  p<p9V4^ ,          R`Ra w  p7p8T4p.V7P                  ^ ,          p,V7P                  ^ ,          p-MPV5e6   V5w	  pp7p0p8p/pp<pp9T5p.V7P                  ^ ,          p,V7P                  ^ ,          p-M\        P	                  R4       R# Rp=Rp>Rp?S P                  P                  V . RbO. R>O4      p@S P                  P                  V . RcO. R>O4      pAS P                  P                  V . RdO. ReO4      pBV@e   X@w  pCppDpET@p=M6XAe   XAw  pDppFpETAp=M(XBe   XBw  p?pDp>ppFppETBp=M\        P	                  R4       R# XEP                  ^ ,          V9P                  ^ ,          8w  dA   V9P                  ^ ,          VP                  ^ ,          8w  d   \        P	                  R4       R# RpGWE8X  d\   S P                  X	X
XCX6XXVXEP                  ^ ,          4      '       g   \        P	                  R4       R# V	P                  ^ ,          pGEMWFV39   Ed    S P                  XXFX<XXEP                  ^ ,          4      '       g   \        P	                  R4       R# VP                  ^ ,          pGV>'       d   V>P                  ^ ,          MXEP                  ^ ,          XDP                  ^ &   V/'       d   V/P                  ^ ,          MV9P                  ^ ,          V8P                  ^ &   V?f!   V8P                  R,           V8P                  ^ &   WG8X  d
   VR,          pV 3R lpHV?'       Ed5   V0'       Ed,   S P                  P                  R4      pIVIR,           pJ\        P                  ! RV0P                  ^ ,          .VJ.VIR7      pKVKP                   P#                  \        P$                  ! R. RfO4      .4       S P                  P                  R4      pLVLR,           pM\        P                  ! RV?P                  ^ ,          .VM.VLR7      pNVNP                   P#                  \        P$                  ! R. RfO4      .4       XH! X<4      pOVOf   \        P	                  R4       R# S P                  P                  RRR7      pP\        P                  ! RXKP                  ^ ,          XOP                  ^ ,          .VPR,           .VPR7      pQS P                  P                  RRR7      pR\        P                  ! RXNP                  ^ ,          VOP                  ^ ,          .VRR,           .VRR7      pSTQp8TSpDS P&                  P)                  VO4       S P&                  P)                  VK4       S P&                  P)                  VN4       S P&                  P)                  VQ4       S P&                  P)                  VS4       S P*                  S P,                  VOP                  &   S P*                  S P,                  VKP                  &   S P*                  S P,                  VNP                  &   S P*                  S P,                  VQP                  &   S P*                  S P,                  VSP                  &   S P/                  XEP                  ^ ,          XGXDV8VV!V"V,VV-V4      pTVTf   \        P	                  R4       R# S P&                  P)                  XT4       S P*                  S P,                  VTP                  &   S P0                  P#                  VR,          4       VV8w  d)   S P0                  P#                  Vf   VRR; MVRR_ 4       M,V^ ,          R;,          .pUV F  pVS P3                  VVXU4       K  	  S P0                  P#                  V4       V.V18X  d!   S P0                  P#                  V.RR_ 4       EMV.V28X  di   S P0                  P)                  V.^ ,          4       S P0                  P)                  V.^,          4       S P0                  P)                  V.^,          4       EMV.V38X  d   S P0                  P)                  V.^ ,          4       S P0                  P)                  V.^,          4       S P0                  P)                  V.^,          4       S P0                  P)                  V.^,          4       MV.V58X  dF   S P0                  P)                  V.^ ,          4       S P0                  P)                  V.^,          4       MBV.V48X  d<   V.^ ,          R;,          V.^ ,          R<,          .pUV. F  pVS P3                  VVXU4       K  	  V=X@8X  d    S P0                  P#                  V=RR_ 4       MKV=XA8X  dE   S P0                  P)                  V=^,          4       S P0                  P)                  V=^,          4       RS n        R# )gr   NReshape	Transposez0fuse_rotary_attention: failed to match qkv nodesr9   rQ   )output_name_to_noder\   zDfuse_rotary_attention: failed to match past/present concat in v pathz-fuse_rotary_attention: failed to match v pathz/fuse_rotary_attention: failed to match qk nodesz;fuse_rotary_attention: failed to match attention mask nodeszDfuse_rotary_attention: failed to match past/present concat in k pathz.fuse_rotary_attention: failed to match k nodesz.fuse_rotary_attention: failed to match q nodeszKfuse_rotary_attention: failed to find the same root_input for q, k, v pathsz;fuse_rotary_attention: failed to verify runtime shape paths	_output_0:rW   NNc                  < S
P                   P                  V R^4      pVf   \        P                  R4       R# S
P                   P	                  VP
                  ^,          4      pS
P                   P	                  VP
                  ^,          4      pVe   Vf   \        P                  R4       R# V^ ,          pV^ ,          pWE,          pS
P                   P                  RRR7      pS
P                   P                  V4      f'   S
P                  V\        P                  ^.V.RR	7       S
P                   P                  RR
R7      p\        P                  ! RVP
                  ^ ,          VP
                  ^,          V.VR,           .VR7      p	V	P                  P                  \        P                  ! R^ 4      .4       V	# )zDetect num_heads and hidden_size for ONNX model from phi-2
Args:
    reshape_q (NodeProto): reshape node for q
Returns:
    hidden_size_concat_node(NodeProto): Concat node to be used by reshape
rP   NzEfuse_rotary_attention: failed to trace the concat node from reshape_qzMfuse_rotary_attention: failed to get constant nodes of num_heads or head_sizeInitializerr   name_prefixF)r=   	data_typedimsvalsrawhidden_size_concatoutput_0r:   axis)r   match_parentr@   rA   get_constant_valuer(   rB   get_initializeradd_initializerr   INT64r   rD   rF   rC   rG   )r   concatnum_head_constant_nodehead_size_constant_nodenum_head_valuehead_size_valuer   hidden_size_initilizerhidden_size_reshape_node_namehidden_size_concat_noder$   s   &         r   create_hidden_size_concat_nodeBFusionRotaryAttention.fuse.<locals>.create_hidden_size_concat_node  s    ZZ,,Y!DF~de &*ZZ%B%B6<<PQ?%S"&*jj&C&CFLLQRO&T#%-1H1Plm3A6N5a8O(:K%)ZZ%@%@\i%@%j"zz))*@AI$$/)//% %  -1JJ,G,G^r,G,s)&,&6&6LLOLLO*
 7CD2	'# $--44f6K6KFTU6V5WX**r   r:   permz?fuse_rotary_attention: failed to create hidden_size_concat_nodeconcat_k_halfr   concat_q_halfzSfuse_rotary_attention: failed to create multi-head attention with rotary embeddingsT>   r!   r    r   )MatMulr   r   r   r   rW   r   r   r   r   )r   r   r   r   rZ   )	AllReducer   r   r   r   )r   r   rP   r   r   r   )rW   r   r   rW   r   r   )rP   r   r   r   )rW   rW   r   r   )r   r   r   rV   )r   ExpandrQ   rP   r   r   r   )rW   r   r   r   rW   r   r   )r   r   WhereEqualr   rP   rQ   rS   rT   rP   r   r   r   )rW   r   rW   r   r   r   r   r   r   r   rW   r   r   )r   r   r   r   rX   ConstantOfShaperT   r   rP   rQ   rS   rT   rP   r   r   r   )rW   r   rW   r   rW   r   r   r   r   rW   r   r   r   rW   r   r   )r   r   r   r   rT   r   rP   rQ   rS   rT   rP   r   r   r   )rW   r   rW   rW   r   r   r      r   r   r   rW   r   r   )r   r   r   r   rP   rQ   rS   rT   rP   r   r   r   )rW   r   rW   r   r      r   r   r   rW   r   r   )	r   rP   rQ   rS   rT   rP   r   r   r   )	rW   rW   r   r   r   r   rW   r   r   )
r   rP   rQ   rX   rS   rT   rP   r   r   r   )
rW   rW   rW   r   r   r   r   rW   r   r   )	rW   rW   r   r   r   r   rW   r   r   )	rW   rW   r   r   r   r   rW   r   r   )rP   r   r   r!   r   )rW   rW   r   r   rW   )Softmaxr!   Divr   rY   r[   r]   )r!   r   Subr^   r   rQ   rQ   )rW   r   r   rW   r   r   r   )r   r   r^   r   rQ   rQ   )rW   r   rW   r   r   r   )r   r!   r   r   r^   r   rQ   rQ   )rW   r   r   r   rW   r   r   r   )r   r   r   r^   r   rQ   rQ   )	r   r^   r   r^   r   r^   r   rQ   rQ   )	rW   r   r   r   r   rW   r   r   r   )r   r   rP   r   RotaryEmbeddingr   )r   r   r   r   r   )r   rP   r   r   r   r   )rW   r   rW   r   r   r   )	r   r   r   rQ   rP   r   r   r   r   )r   r   r   r   r   r   rP   rQ   rS   rT   rP   r   r   r   r   )rW   r   r   rW   r   r   r   r   r   r   r   rW   r   r   r   )r   r   r   r   r   rX   r   rT   r   rP   rQ   rS   rT   rP   r   r   r   r   )rW   r   r   rW   r   rW   r   r   r   r   rW   r   r   r   rW   r   r   r   )r   r   r   r   r   rT   r   rP   rQ   rS   rT   rP   r   r   r   r   )rW   r   r   rW   rW   r   r   r   r   r   r   r   rW   r   r   r   )r   r   r   r   r   rP   rQ   rS   rT   rP   r   r   r   r   )rW   r   r   rW   r   r   r   r   r   r   rW   r   r   r   )r   r   rP   rQ   rS   rT   rP   r   r   r   r   )rW   r   rW   r   r   r   r   rW   r   r   r   )r   r   rP   rQ   rX   rS   rT   rP   r   r   r   r   )rW   r   rW   rW   r   r   r   r   rW   r   r   r   )rW   r   rW   r   r   r   r   rW   r   r   r   )rW   r   rW   r   r   r   r   rW   r   r   r   )	r   rP   rP   r   r\   r   r   r!   r   )	rW   r   rW   r   r   r   r   r   rW   )r   r   r   r   )r   r   r   r   )rP   r   r\   r   r   r!   r   )r   r   r   r   r   r   rW   )r   r   rW   r   )op_typer   r`   r@   rA   match_parent_paths_allr(   r)   lenreshape_add_qkr   r   r=   rB   r   rD   rF   rC   rG   nodes_to_addappendthis_graph_namenode_name_to_graph_namerM   nodes_to_remove&add_nodes_to_remove_with_nodes_to_keepprune_graph)Wr$   normalize_nodeinput_name_to_nodesr   	qkv_nodesqkv_nodes_1qkv_nodes_2qkv_nodes_3rp   ra   rb   
matmul_qkvr   r0   r2   past_seq_lenv_nodesadd_v	v_nodes_1	v_nodes_2	v_nodes_3	v_nodes_4	v_nodes_5re   r   rf   matmul_vr   transpose_vr   qk_nodesr.   	matmul_qkr-   
add_qk_strattn_mask_nodes_1attn_mask_nodes_2attn_mask_nodes_3attn_mask_nodes_4attn_mask_nodes_5attn_mask_nodes_6attn_mask_nodes_7slice_mask_1slice_mask_2r/   r1   k_nodesslice_kr   	k_nodes_1	k_nodes_2	k_nodes_3	k_nodes_4	k_nodes_5rd   r   rotary_kmatmul_kr   shared_past_seq_lenr   q_nodesslice_qr   	q_nodes_1	q_nodes_2	q_nodes_3rc   rotary_qmatmul_qr   root_outputr   k_transpose_node_namek_tranpose_output_namek_transpose_nodeq_transpose_node_nameq_tranpose_output_nameq_transpose_noder   concat_k_reshape_node_nameconcat_k_reshape_nodeconcat_q_reshape_node_nameconcat_q_reshape_nodenew_nodenodes_to_keep	temp_pathsW   f&&&                                                                                   r   fuseFusionRotaryAttention.fuseE  s   !!)nn
 	jj22C

 jj228

 jj22E

 "=H:Aa
#I$,7)AA#I$/:,Aqq#ILLKL +-b"<JJ00P
	
 JJ008
	
 JJ00.
	
 **;; c)
 <!&$ E',  ?#( 9$ v/
 3  v/
 v/OkX !%] < o
9a` JJ00?
	
  AJ>KHahG JJ88+&AM
 $cd"1%++A.F(,2215L *I"9B6Hk9hG^^A&F *I"/8,KHG#**1-I"s9~':9B1bc9J6Hk9hG^^A&F *I"@I=Hk9eXHG^^A&F *ILLHI::///

 !$	&.#Avq)LLJK !#B:	 JJ88(

 !JJ880

 !JJ88O!

 !JJ88H

 !JJ88Y$

 !JJ88R!

 !JJ88a'

 (,=)A|\$++A.I*/@,Aq,$++A.I*,,->q-A-H-H-KLJ*,,->q-A-H-H-KLJ**1-44Q7J**1-44Q7J*,,->q-A-H-H-KLJLLVW
 	JJ00X
	
 JJ00N
	
 JJ00X
	
 **;;
 0" B%*( K+0$ E',  ?#( 6" 9$ 6" 6gcH !%M < g
9aP JJ00r'
	
  >G;KHa8G JJ88+&AM
 $cd"1%++A.F"/"3"9"9!"< *I#6666"2;/AxIxG *I"<E9Ax1iG^^A&F *I"s9~':"+A,rs"3Ix!*1b!4HhG^^A&F *I"W`TAx'1iHG^^A&F *ILLIJ
 JJ00A
	
 JJ00A
	
 JJ00[!
	
  1:.KHhG"/8,HaHG"JSGM8WaAxGLLIJ>>!q 11hnnQ6G8>>Z[K\6\LLfg#>>q!	 	 Z['..q1K44;;q!  Z[%,,Q/K
 6=q 1(//RSBTHNN15<q 1(//RSBTHNN1 $%-]][%@"'%bM	2	+j =]]$(JJ$?$?$L!%:[%H"%//%,,Q/0/0*	  &&--v/D/DV\/Z.[\ %)JJ$?$?$L!%:[%H"%//%,,Q/0/0*	  &&--v/D/DV\/Z.[\&DY&O#&.^_ *.)D)DY\k)D)l&$*$4$4(//24K4R4RST4UV3kAB/	%! *.)D)DY\k)D)l&$*$4$4(//24K4R4RST4UV3kAB/	%! -H,H$$%<=$$%56$$%56$$%:;$$%:;IMI]I]D(()@)E)EFBFBVBVD(()9)>)>?BFBVBVD(()9)>)>?GKG[G[D(()>)C)CDGKG[G[D(()>)C)CD''NN1
 LLno  *6:6J6J$$X]]3##IbM2i  ''7SVTV<X$QZ^,M$	;;I}U % 	##H-i  ''5	!  ''
3  ''
3  ''
3	!  ''
3  ''
3  ''
3  ''
3	!  ''
3  ''
3	!$QZ^WQZ^<M$	;;I}U % i  ''5	!  ''
3  ''
3r   )r   )r9   r9   r9   r9   r9   r9   N)__name__
__module____qualname____firstlineno____doc__r#   rM   r   r   r$  __static_attributes____classdictcell____classcell__r%   r   s   @@r   r   r      s9     
 
(6 6pZxGRK  K r   r   c                   z   a a ] tR tRt oV3R lV 3R lltV3R lR ltV3R lR ltV3R lR	 ltR
 tRt	Vt
V ;t# )FusionRotaryEmbeddingsiS  c                    < V ^8  d   QhRS[ /# )r   r   r
   )r   r   s   "r   r   #FusionRotaryEmbeddings.__annotate__T  s     ` `i `r   c                   < R V n         \        SV `	  WP                   V P                   V P                   R,           R.4       R# )r   z.1r!   N)	base_namer"   r#   )r$   r   r%   s   &&r   r#   FusionRotaryEmbeddings.__init__T  s3    *RVAVX]0^_r   c                &   < V ^8  d   QhRS[ RS[/# )r   rot_emb_nodefunction)r   r   )r   r   s   "r   r   r2  [  s      9  r   c                  a . . rCVP                    F  pVP                  R 8X  g   K  VP                  . 8X  g   K)  VP                  ^ ,          VP                  9   g   KM  VP	                  V4       \        VP                  4      P                  VP                  ^ ,          4      pVP	                  VP                  V,          4       K  	  . pV Fv  pVP                  ^ ,          P                  p	V P                  P                  R 4      V	n        V P                  P                  V	4       VP	                  V	P                  4       Kx  	  \        WGRR7       Fh  w  op
\        \        V3R lV P                  P                  P                  P                   4      4      pV F  p\         P"                  ! VSV
4       K  	  Kj  	  V# )ConstantF)strictc                 "   < SV P                   9   # N)r(   )entryextra_outputs   &r   <lambda>?FusionRotaryEmbeddings.reassign_extra_outputs.<locals>.<lambda>n  s    8Sr   )noder   r(   r)   r   listindexrF   tr   rB   r=   r   zipfiltergraphr   replace_node_input)r$   r7  r8  extra_constantsextra_outputsfn_nodeoutput_indexextra_initializersextra_constantconstant_tensorprotoextra_initializernodes_to_updatenode_to_updater?  s   &&&          @r   reassign_extra_outputs-FusionRotaryEmbeddings.reassign_extra_outputs[  sd   )+R}}G*,"1DXYIZ^f^m^mIm&&w/#HOO4::7>>!;LM$$\%8%8%FG	 %  -N#1#;#;A#>#@#@ (,

(C(CJ(O %JJ&&';<%%&:&?&?@	 . 03=]b/c+L+"6*SUYU_U_UeUeUkUkUpUp#qrO"1,,^\K\] #2 0d
 r   c                    < V ^8  d   QhRS[ /# )r   rB  )r   )r   r   s   "r   r   r2  t  s     H H9 Hr   c                J  aa V P                   P                  V P                  4      pV P                   P                  SR R.^ ^ .4      pVe   Vw  rEM\        P                  R4       R# VP                  ^ ,          SP                  ^,          .p\        \        V3R lV P                   P                   P                  P                  4      4      p\        \        V3R lV P                   P                   P                  P                  4      4      pRRr\        V4      ^8X  Ed   \        V4      ^8X  Ed   V P                   P                  V	4      Ef   V P                   P                  V
4      Ef   \        P                  ! V^ ,          P                   ^ ,          P"                  4      P%                  4       p\        P                  ! V^ ,          P                   ^ ,          P"                  4      P%                  4       p\&        P(                  ! V	\*        P,                  \        VP.                  4      VP1                  4       P3                  4       R7      pV P                   P5                  WP6                  4       \&        P(                  ! V
\*        P,                  \        VP.                  4      VP1                  4       P3                  4       R7      pV P                   P5                  WP6                  4       V P8                  P;                  V^ ,          V^ ,          .4       VP;                  W.4       SP                  p\        V4      ^8  d   \        \        V3R	 lV P                   P                   P<                  4      4      p\        V4      ^8X  g   Q hV P?                  SV^ ,          4      o\        \        V3R
 lV4      4      p\        V4      ^8X  g   Q h\&        P@                  ! V P                  VVV^R7      pRVn!        V P8                  PE                  V4       V# )r   r   Nz.fuse_rotary_embeddings: failed to match MatMulc                 R   < V P                   ^ ,          SP                  ^,          8H  # r   r)   r(   constantrB  s   &r   r@  OFusionRotaryEmbeddings.create_rotary_embeddings_from_function.<locals>.<lambda>      hooa6HDJJWXM6Yr   c                 R   < V P                   ^ ,          SP                  ^,          8H  # rY  rZ  r[  s   &r   r@  r]    r^  r   	cos_cache	sin_cacher=   r   r   r   c                 6   < V P                   SP                  8H  # r=  )r=   r   )fnrB  s   &r   r@  r]    s    "''T\\*Ar   c                    < V S9  # r=   )output_namerK  s   &r   r@  r]    s    TaAar   r;   r<   r=   interleavedr>   )#r   rB   r4  r`   r@   rA   r)   r(   rC  rG  rH  rB  r   r   r	   to_arrayrF   rE  squeezer   make_tensorr   FLOATshapeflattentolistr   r   r   rC   	functionsrT  rD   rE   r   )r$   rB  rotary_emb_node_namematmul_pathreshape_nodematmul_noderotary_emb_inputscos_cache_nodesin_cache_nodecos_cache_namesin_cache_namer`  ra  cos_cache_tensorsin_cache_tensorrotary_emb_outputsfuncrotary_emb_noderK  s   &f                @r   &create_rotary_embeddings_from_function=FusionRotaryEmbeddings.create_rotary_embeddings_from_functiont  s?   #zz::4>>Jjj22!F

 "(3%L+LLIJ q!JJqM
 f%Y[_[e[e[k[k[q[q[v[vwxf%Y[_[e[e[k[k[q[q[v[vwx)4k 1$N#q(

**>:B

**>:B$--nQ.?.I.I!.L.N.NOWWYI$--nQ.?.I.I!.L.N.NOWWYI%11#%++)//*&&(//1	  JJ&&'79M9MN%11#%++)//*&&(//1	  JJ&&'79M9MN  ''):N1<M(NO  .!AB![[!"Q&A4::CSCSC]C]^_Dt9>!> 77d1gFM!%f-acu&v!w)*a/// **NN$&%
 "1##L1r   c          
      8   < V ^8  d   QhRS[ RS[ RS[ RS[ RS[ /# )r   rg   position_ids	cos_slice	sin_slicer)   )r5   )r   r   s   "r   r   r2    s=     6 66 6 	6
 6 6r   c                L  aa V P                   P                  V P                  4      p\        \	        V3R  lV P                   P                   P
                  P                  4      4      p\        \	        V3R lV P                   P                   P
                  P                  4      4      pRRr\        V4      ^8X  Ed8   \        V4      ^8X  Ed'   V P                   P                  V	4      Ef	   V P                   P                  V
4      Ef   \        P                  ! V^ ,          P                  ^ ,          P                  4      P                  4       p\        P                  ! V^ ,          P                  ^ ,          P                  4      P                  4       pVP                  ^,          pVRRV^,          13,          pVRRV^,          13,          p\        P                   ! V	\"        P$                  \        VP                  4      VP'                  4       P)                  4       R7      pV P                   P+                  WP,                  4       \        P                   ! V
\"        P$                  \        VP                  4      VP'                  4       P)                  4       R7      pV P                   P+                  WP,                  4       V P.                  P1                  V^ ,          V^ ,          .4       \        P2                  ! V P                  WW.V.V^ R7      pRVn        V# )	c                 0   < V P                   ^ ,          S8H  # rY  r)   )r\  r  s   &r   r@  LFusionRotaryEmbeddings.create_rotary_embeddings_from_nodes.<locals>.<lambda>      hooa6HI6Ur   c                 0   < V P                   ^ ,          S8H  # rY  r  )r\  r  s   &r   r@  r    r  r   r`  ra  N:NNNrb  rh  r>   )r   rB   r4  rC  rG  rH  rB  r   r   r	   rj  rF   rE  rk  rn  r   rl  r   rm  ro  rp  r   r   r   rC   rD   rE   )r$   rg   r  r  r  r)   rr  rw  rx  ry  rz  r`  ra  	head_sizer{  r|  r  s   &&&ff&           r   #create_rotary_embeddings_from_nodes:FusionRotaryEmbeddings.create_rotary_embeddings_from_nodes  sr     $zz::4>>J f%UW[WaWaWgWgWmWmWrWrstf%UW[WaWaWgWgWmWmWrWrst)4k 1$N#q(

**>:B

**>:B$--nQ.?.I.I!.L.N.NOWWYI$--nQ.?.I.I!.L.N.NOWWYI "*I!!%7	Q%7"78I!!%7	Q%7"78I%11#%++)//*&&(//1	  JJ&&'79M9MN%11#%++)//*&&(//1	  JJ&&'79M9MN  ''):N1<M(NO **NNnMH%
 "1r   c                  a% V P                   VP                  9  d   VP                  R 8w  d   R# Ro%VP                  R 8w  Ed&   \        VP                  4      R9  g   VP                  ^,          R9  d   \        P                  R4       R# V P                  V4      o%S%f   \        P                  R4       R# V P                  P                  V4       \        \        V%3R lV P                  P                  P                  P                  4      4      p\        V4      ^8X  g   Q hV P                  P                  P                  P                  P                  V^ ,          4       E	M4V P                  P                  V. RO. RO4      pV P                  P                  V. RO. RO4      pT;'       g    TpV P                  P                  V. RO. RO4      pV P                  P                  V. RO. RO4      p	T;'       g    T	p
Ve   V
f   \        P                  R
4       R# V P                  P                  V. RO. R O4      pV P                  P                  V. R!O. R O4      pT;'       g    TpV P                  P                  V. R"O. R#O4      pV P                  P                  V. R$O. R#O4      pT;'       g    TpVe   Vf   \        P                  R4       R# VR%,          P                   VR%,          P                   8w  g|   VR%,          P                   V
R%,          P                   8w  gS   VR%,          P                   VR%,          P                   8w  g*   VR%,          P                   V
R%,          P                   8w  d   \        P                  R4       R# V P                  P                  VRR.^ ^ .4      pV P                  P                  VRR.^ ^ .4      pT;'       g    TpVf   \        P                  R4       R# RRRpppV P                  P                  V. R&O. R'O4      pV P                  P                  V. R(O. R)O4      pV P                  P                  V. R*O. R+O4      pV P                  P                  V. R,O. R-O4      pVe   TpVR.,          P                  ^ ,          pMVe   TpVR/,          P                  ^ ,          pMVe8   TpVR.,          P                  ^ ,          pV^,          P                  ^,          pMRVe8   TpVR/,          P                  ^ ,          pV^,          P                  ^,          pM\        P                  R4       R# RRppV P                  P                  V. R&O. R0O4      pV P                  P                  V. R(O. R1O4      pV P                  P                  V. R*O. R2O4      pV P                  P                  V. R,O. R3O4      p Ve   TpVR.,          P                  ^ ,          pMVe   TpVR/,          P                  ^ ,          pMVe8   TpVR.,          P                  ^ ,          pV^,          P                  ^,          pMRV e8   T pVR/,          P                  ^ ,          pV^,          P                  ^,          pM\        P                  R4       R# VR8X  d   V P                  P                  V^,          R.^.4      p!V P                  P                  V^,          R.^.4      p"V!e.   V"e*   V!^ ,          P                   V"^ ,          P                   8w  d   \        P                  R4       R# V"^ ,          P                  ^ ,          pM. p!. p"RRp$p#VV8X  d   VV8X  g   VV8X  ds   VV8X  dl   VR4,          P                   VR4,          P                   8w  g*   VR%,          P                   VR%,          P                   8w  d   \        P                  R4       R# EM/VV8X  d   VV8X  g   VV8X  Ed   VV 8X  d   VR%,          P                   VR%,          P                   8w  d   \        P                  R4       R# V P                  P                  VR%,          RR	.^^ .4      p#V P                  P                  VR%,          . R5O. R6O4      p$V#eR   V$eN   V P                  P#                  V#R%,          P                  ^ ,          4      e   V$R%,          P                  R8w  d   \        P                  R4       R# M\        P                  R4       V P%                  VR%,          P&                  ^ ,          VVVVP&                  ^ ,          4      o%S%f   \        P                  R4       R# V P)                  V.4       V P)                  VRR% 4       V P)                  VRR% 4       V P)                  VRR% 4       V P)                  V
RR% 4       V P)                  VRR% 4       V P)                  V4       V P)                  V4       V P)                  V!RR% 4       V P)                  V"RR% 4       V#eB   \        V P                  P+                  V#^ ,          4      4      ^8X  d   V P)                  V#4       V$e   V P)                  V$RR% 4       V P-                  V P                   4       V P.                  V P0                  S%P                   &   V P2                  P                  S%4       RV n        R# )7r!   NzLfuse_rotary_embeddings: failed to verify inputs for RotaryEmbedding functionz=fuse_rotary_embeddings: failed to create RotaryEmbedding nodec                 D   < V P                   SP                  ^ ,          8H  # rY  )r=   r)   )rB  r  s   &r   r@  -FusionRotaryEmbeddings.fuse.<locals>.<lambda>  s    DII1G1G1J$Jr   rX   r\   r   rS   rT   z9fuse_rotary_embeddings: failed to match x2 in rotate_halfz9fuse_rotary_embeddings: failed to match x1 in rotate_halfzCfuse_rotary_embeddings: failed to match common input in rotate_halfz8fuse_rotary_embeddings: failed to match x in rotate_halfr9   z>fuse_rotary_embeddings: failed to match sin path in apply_roper   zGfuse_rotary_embeddings: failed to match position ids path in apply_ropezdfuse_rotary_embeddings: failed to match common Gather node and Shape node in sin cache and cos cachezRfuse_rotary_embeddings: failed to match common Add node in sin cache and cos cachezKfuse_rotary_embeddings: failed to match past_seq_len and curr_seq_len pathsz:fuse_rotary_embeddings: failed to match common cache pathsT>   r      >   pospos_idpos_idsposition_idr  )rX   rP   Negr\   r   r   )rX   rP   r  r\   r\   )	rX   rP   r  r\   rQ   r   rS   rT   r   )	rW   r   r   r   rW   r   r   r   r   )	rX   rP   r  r\   rQ   r   rS   rT   r\   )rX   rP   r\   r   )rW   r   rW   r   )rX   rP   r\   r\   )rX   rP   r\   rQ   r   rS   rT   r   )rW   r   rW   r   r   r   r   r   )rX   rP   r\   rQ   r   rS   rT   r\   r   )	rX   rQ   rS   Squeezer  r\   rQ   rS   rT   )	rW   rW   r   r   r   r   r   r   r   )rX   rQ   rS   r  r  r\   rQ   r!   )rW   rW   r   r   r   r   r   r   )rX   rQ   rS   r\   rQ   rS   rT   )rW   rW   r   r   r   r   r   )rX   rQ   rS   r\   rQ   r!   )rW   rW   r   r   r   r   r   r   )	r   rW   r   r   r   r   r   r   r   )r   rW   r   r   r   r   r   r   )r   rW   r   r   r   r   r   )r   rW   r   r   r   r   r   )rS   rT   r   rU   )r4  r   r   r(   r@   rA   r  r   r   rC  rG  r   rH  
value_inforemover`   r=   find_graph_inputr  r)   add_nodes_to_removeget_childrenrH   r   r   r   r   )&r$   rB  r   r   old_shape_inferrotate_half_x2_path_1_1rotate_half_x2_path_1_2rotate_half_x2_path_1rotate_half_x2_path_2_1rotate_half_x2_path_2_2rotate_half_x2_path_2rotate_half_x1_path_1_1rotate_half_x1_path_1_2rotate_half_x1_path_1rotate_half_x1_path_2_1rotate_half_x1_path_2_2rotate_half_x1_path_2x_path_1x_path_2x_pathsin_pathra  r  
sin_path_1
sin_path_2
sin_path_3
sin_path_4cos_pathr`  
cos_path_1
cos_path_2
cos_path_3
cos_path_4position_ids_from_sin_pathposition_ids_from_cos_pathpast_seq_len_pathcurr_seq_len_pathr  s&   &&&&                                 @r   r$  FusionRotaryEmbeddings.fuse  s	   >>-$,,%2G <<5 4::f,

1 F 1 kl"II$OO&\]   ''- #JDJJL\L\LbLbLmLmnO '1,,,JJ""--44_Q5GH( '+jj&B&B>'# '+jj&B&B:'# %<$V$V?V!&*jj&B&Be+'# '+jj&B&Ba+'# %<$V$V?V!$,0E0MXY&*jj&B&B7'# '+jj&B&B3'# %<$V$V?V!&*jj&B&B^('# '+jj&B&BZ('# %<$V$V?V!$,0E0MXY &b)..2G2K2P2PP(,115J25N5S5SS(,115J25N5S5SS(,115J25N5S5SSbc zz33$AH zz33 AH ))F~WX 15b"iH55m+J
 55a(J
 55W%J
 55K"J
 %%$RL..q1	'%$RL..q1	'%$RL..q1	'{003'%$RL..q1	'{003]^ #'iH55m+J
 55a(J
 55W%J
 55K"J
 %%$RL..q1	'%$RL..q1	'%$RL..q1	'{003'%$RL..q1	'{003]^ r!-1ZZ-I-IQKKC.*
 .2ZZ-I-IQKKC.* /6191!499=WXY=Z=_=__LL!jk9!<BB1E-/*-/*370J&8z+AJ&8z+AB<$$(9(99Xb\=N=NRZ[]R^RcRc=cLL~ 	 >d
 j(X-CJ&8z+AB<$$(9(99LL!uv$(JJ$@$@RLw'F%! %)JJ$@$@RL4%! &-(0zz223DR3H3N3Nq3QRZ(,44CLL!no D
 YZ"FF%b)003AO &\] $$dV,$$%:3B%?@$$%:3B%?@$$%:3B%?@$$%:3B%?@$$VCR[1$$X.$$X.$$%?%DE$$%?%DE ,TZZ5L5LM^_`Ma5b1cgh1h (():; ,(():3B)?@dnn-=A=Q=Q$$_%9%9:  1r   )r4  r   )r&  r'  r(  r)  r#   rT  r  r  r$  r+  r,  r-  r.  s   @@r   r0  r0  S  s?     ` ` 2H HT6 6pA  A r   r0  )loggingfusion_attentionr   fusion_baser   onnxr   r   r   r   r	   
onnx_modelr   	getLoggerr&  r@   r   r0  rf  r   r   <module>r     sF   
  ,  L L  			8	$A O A H"d V d r   