+
    9iO                         ^ RI Ht ^ RIHt ^ RIHtHt ^ RIHt ]! ]	4      t
 ! R R]4      t ! R R]4      t ! R	 R
]4      tR# )    )	getLogger)Fusion)TensorProtohelper	OnnxModelc                   T   a a ] tR t^t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )FusionLayerNormalizationc                ,   < V ^8  d   QhRS[ RS[RS[/# )   modelcheck_constant_and_dimensionforce)r   bool)format__classdict__s   "g/var/www/html/photoedit/myenv/lib/python3.14/site-packages/onnxruntime/transformers/fusion_layernorm.py__annotate__%FusionLayerNormalization.__annotate__   s#      i t [_     c                B   < \         SV `  VR R4       W n        W0n        R# LayerNormalization
ReduceMeanN)super__init__r   r   )selfr   r   r   	__class__s   &&&&r   r   !FusionLayerNormalization.__init__   s      4lC,H)
r   c                &   < V ^8  d   QhRS[ RS[ /# r   input_name_to_nodesoutput_name_to_nodedict)r   r   s   "r   r   r      s#     SU SUd SU SUr   c                P   . pV P                   P                  W4      p\        V4      ^ 8X  g   \        V4      ^8  d   R# VP                  ^ ,          pV^ ,          P                  R8w  g    V^ ,          P                  ^ ,          V8w  d   R# \        V4      ^8X  d:   V^,          P                  R8w  g    V^,          P                  ^ ,          V8w  d   R# RpV FW  pV P                   P                  VRVRR7      p	V	e   T	p M1V P                   P                  VRR.4      p
V
f   KN  V
R,          p M	  Vf   R# V P                   P                  V. RO. RO3. RO. RO3.V4      w  rpVf   R# VR,          pW9  d   R# V^,          pV P                   P                  V4      w  ppVe   V^ 8:  g   VR8  d   \        P                  R	V 24       R# V^,          pV P                   P                  VR
4      ^8w  d   R# VP                  ^ ,          V9  d   R# W'P                  ^ ,          ,          pV EFn  pVP                  R8X  dN   VP                  V4       VP                  ^ ,          V9  d   K@  VVP                  ^ ,          ,          ^ ,          pMTpVP                  R8w  d   Kw  VP                  ^ ,          V9  d   K  VVP                  ^ ,          ,          ^ ,          pVP                  R8w  d   K  VP                  V4       VP                  V4       VP                  VRR 4       VP                  VVV.4       VP                  R8w  d   TMTpVP                  ^V P                   P                  VP                  ^ ,          V4      ,
          ,          pV P                   '       d'   V P                   P#                  V^R4      '       g   EK  VP                  ^V P                   P                  VP                  ^ ,          V4      ,
          ,          pV P                   '       d'   V P                   P#                  V^R4      '       g   EK  VP                  ^ ,          pV P                   P%                  VVP                  VV4      '       g3   V P&                  '       d	   RV n        M3\        P                  R4       EK  V P*                  P                  V4       \,        P.                  ! RVP                  ^ ,          VV.V.V P                   P1                  RRR7      R7      pVP2                  P                  \,        P4                  ! R\7        V4      4      .4       V P8                  P                  V4       V P:                  V P<                  VP>                  &   EKq  	  R# )a=  
Fuse Layer Normalization subgraph into one node LayerNormalization:
      +----------------------+
      |                      |
      |                      v
  [Root] --> ReduceMean -->  Sub  --> Pow --> ReduceMean --> Add --> Sqrt --> Div --> Mul --> Add
             (axis=2 or -1)  |      (Y=2)   (axis=2 or -1)  (B=E-6 or E-12)    ^
                             |                                                 |
                             +-------------------------------------------------+

 It also handles cases of duplicated sub nodes exported from older version of PyTorch:
      +----------------------+
      |                      v
      |           +-------> Sub-----------------------------------------------+
      |           |                                                           |
      |           |                                                           v
  [Root] --> ReduceMean -->  Sub  --> Pow --> ReduceMean --> Add --> Sqrt --> Div  --> Mul --> Add
      |                      ^
      |                      |
      +----------------------+
NSubDivF	recursiveCastAdd-C6?Hskip SkipLayerNormalization fusion since epsilon value is not expected:        @Mullayernorm weightlayernorm biasT4It is not safe to fuse LayerNormalization node. Skipr   	LayerNormname_prefixinputsoutputsnameepsilonSqrtr,   r   Powr'      r   r   r   r   )r>   r,   r   r?   r+   r'   )rA   r   r   r   r   r   ) r   get_childrenleninputop_typefind_first_child_by_typematch_child_pathmatch_parent_pathsget_constant_inputloggerdebugfind_constant_inputoutputappendextendinput_indexr   $is_constant_with_specified_dimensionis_safe_to_fuse_nodesr   prune_graphnodes_to_remover   	make_nodecreate_node_name	attributemake_attributefloatnodes_to_addthis_graph_namenode_name_to_graph_namer:   )r   noder"   r#   subgraph_nodeschildren
root_inputdiv_nodechild
div_node_1
div_node_2_path_idparent_nodes_sub_nodeadd_eps_nodeir;   pow_nodediv_children	temp_nodemul_nodelast_add_nodenode_before_weightweight_input
bias_inputlayer_norm_outputnormalize_nodes   &&&&                        r   fuseFusionLayerNormalization.fuse   s   , ::**4Ex=AX!2ZZ]
A;%'8A;+<+<Q+?:+Mx=A{""e+x{/@/@/Cz/QE<<UEK^jo<pJ%% "ZZ88P
))"~H  $(JJ$A$A<oNDFXY  %
! ###AZZ22<@
7?glg.>LLcdkclmn?::))(C8A=??1%88 +??1+=>%I  F*%%i0##A&.AA.y/?/?/BCAF %5(q!)<</0BCAFM$$-!!$'!!(+!!,s"34!!=(H"EF-6->->&-Hi#>>!djj.D.DEWE^E^_`Eack.l*lmL0009h9ha!3: : &,,Q1G1GXYHZ\i1j-jkJ0009h9hA/: :  - 4 4Q 7::33$$##	  :::'+D$LL!WX$$++N;#--$

1|Z@*+ZZ001ES^0_	N $$++V-B-B9eT[n-],^_$$^4@D@T@TD(()<)<=A &r   )r   r   rS   )TF	__name__
__module____qualname____firstlineno__r   ru   __static_attributes____classdictcell____classcell__r   r   s   @@r   r
   r
      s$      
SU SU SUr   r
   c                   l   a a ] tR t^t oV3R lV 3R lltR tR	V3R lR lltV3R lR ltRtVt	V ;t
# )
FusionLayerNormalizationNCHWc                    < V ^8  d   QhRS[ /# r   r   r   )r   r   s   "r   r   )FusionLayerNormalizationNCHW.__annotate__   s     D Di Dr   c                *   < \         SV `  VR R4       R# r   r   r   r   r   r   s   &&r   r   %FusionLayerNormalizationNCHW.__init__   s     4lCr   c                   V P                   P                  V4      pVf   \        P                  V RV R24       R # \	        VP
                  4      ^8w  g1   VP
                  ^,          ^8w  g   VP
                  ^,          ^8w  d*   \        P                  V RV RVP
                   24       R # VP                  VP
                  ^ ,          .4      # )N z is not initializer.z* shall have 3 dimensions Cx1x1. Got shape )r   get_constant_valuerJ   rK   rC   shapereshape)r   output_namedescriptionvalues   &&& r   get_weight_or_bias/FusionLayerNormalizationNCHW.get_weight_or_bias   s    

--k:=LLK=+6JKLu{{q EKKNa$75;;q>Q;NLLK=+6`afalal`mno}}ekk!n-..r   c                6   < V ^8  d   QhRS[ RS[S[,          /# )r   
input_nameperm)strlistint)r   r   s   "r   r   r      s     
 
 
49 
r   c                   V P                   P                  R4      pVf   VR,           R,           V,           p\        P                  ! RV.V.VR7      pVP                  P                  \        P                  ! RV4      .4       V# )z&Append a Transpose node after an input	Transpose_out-r7   r   )r   rV   r   rU   rW   rO   rX   )r   r   r   r   	node_nametranspose_nodes   &&&&  r   create_transpose_node2FusionLayerNormalizationNCHW.create_transpose_node   sw    JJ//<	#f,s2Z?K))+zlU`Tahqr  '')>)>vt)L(MNr   c                &   < V ^8  d   QhRS[ RS[ /# r!   r$   )r   r   s   "r   r   r      s     }, },d }, },r   c                   \         P                  ! VR4      p\        V\        4      '       d	   V^.8w  d   R# . pV P                  P                  W4      p\        V4      ^8w  d   R# VP                  ^ ,          pV^ ,          P                  R8w  g    V^ ,          P                  ^ ,          V8w  d   R# V^ ,          pV P                  P                  VRVRR7      p	V	f   R# V P                  P                  V	. RO. ROV4      p
V
f   R# V
w  rrpW8w  d   R# V P                  P                  V4      w  ppVe   V^ 8:  g   VR8  d   \        P                  R	V 24       R# \         P                  ! VR4      p\        V\        4      '       g   Q hV^.8w  d   R# V P                  P                  VR
4      ^8w  d   R# W)P                  ^ ,          ,          ^ ,          pTpVP                  R8w  d   R# VVP                  ^ ,          ,          ^ ,          pVP                  R8w  d   R# VP!                  V4       VP#                  V
4       VP#                  VVV	.4       V P                  P%                  VVP                  VV4      '       g   \        P                  R4       R# VP                  R8w  d   T	MTpVP                  ^V P                  P'                  VP                  ^ ,          V4      ,
          ,          pV P)                  VR4      pVf   R# VP                  ^V P                  P'                  VP                  ^ ,          V4      ,
          ,          pV P)                  VR4      pVf   R# \*        P,                  ! VR,           \.        P0                  VP2                  V4      p\*        P,                  ! VR,           \.        P0                  VP2                  V4      pV P                  P5                  VV P6                  4       V P                  P5                  VV P6                  4       V P8                  P#                  V4       V P;                  VP                  ^ ,          . RO4      pV P                  P=                  RRR7      pV P;                  VR,           . ROVP                  ^ ,          4      p\*        P>                  ! RVP                  ^ ,          VR,           VR,           .VR,           .VR7      pVP@                  P#                  \*        PB                  ! R\E        V4      4      .4       V PF                  P!                  V4       V PF                  P!                  V4       V PF                  P!                  V4       V P6                  V PH                  VPJ                  &   V P6                  V PH                  VPJ                  &   V P6                  V PH                  VPJ                  &   Rp V PM                  V 4       R# )a  
Fuse Layer Normalization subgraph into one node LayerNormalization:
      +----------------------+
      | NxCxHxW              |
      |                      v                                                     (Cx1x1)  (Cx1x1)
  [Root] --> ReduceMean -->  Sub --> Pow --> ReduceMean --> Add --> Sqrt --> Div --> Mul --> Add -->
             (axes=1)        |      (Y=2)     (axes=1)     (E-6)             ^
                             |                                               |
                             +-----------------------------------------------+

Fused subgraph:
               (0,2,3,1)                            (0,3,1,2)
    [Root] --> Transpose --> LayerNormalization --> Transpose -->
axesNr'   r(   Fr)   r,   r-   r.   r/   r0   r3   r+   r1   r2   _NHWCr   r4   r5   	_out_nhwcr7   r;   zLayerNormalization(NHWC)r=   r@   )r   r      rA   )r   r   rA   r   )'r   get_node_attribute
isinstancer   r   rB   rC   rD   rE   rF   match_parent_pathrI   rJ   rK   rL   rM   rN   rO   rR   rP   r   r   make_tensorr   FLOATr   add_initializerr[   rT   r   rV   rU   rW   rX   rY   rZ   r\   r:   increase_counter)!r   r]   r"   r#   r   r^   r_   r`   subra   rf   
_sqrt_nodesecond_add_nodereduce_mean_noderk   rh   rj   r;   rm   rn   ro   rp   rq   weightrr   biasweight_nhwc	bias_nhwctranspose_inputlayernorm_node_nametranspose_outputrt   counter_names!   &&&&                             r   ru   !FusionLayerNormalizationNCHW.fuse   s    ++D&94&&4A3;::**4Ex=AZZ]
A;%'8A;+<+<Q+?:+Mqk::66sECVbg6hzz337	
 LXI
%5?ZZ22?C
7?glg.>LLcdkclmn++,<fE$%%%%A3;::))(C8A='(:;A>	u$+HOOA,>?B  E)d#l+}hABzz//  	
 
 LLOP)2):):f)DX)~~a$***@*@ASAZAZ[\A]_g*h&hi((7IJ>"((TZZ-C-CHOOTUDVXe-f)fg
&&z3CD<(()?ARARTZT`T`bhi&&zG';[=N=NPVP\P\^de	

"";0D0DE

""9d.B.BC##N344TZZ]LQ"jj99:N\g9h55+-|]=Q=QRS=T
  )) #**1-|g/EzT[G[\(;67$	
 	  '')>)>y%PW.)Y(Z[  1  0  !12=A=Q=Q$$_%9%9:<@<P<P$$^%8%89>B>R>R$$%5%:%:;1l+r    )N)rx   ry   rz   r{   r   r   r   ru   r|   r}   r~   r   s   @@r   r   r      s2     D D
/
 
}, }, },r   r   c                   P   a a ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )FusionLayerNormalizationTFiF  c                    < V ^8  d   QhRS[ /# r   r   )r   r   s   "r   r   'FusionLayerNormalizationTF.__annotate__G  s     C Ci Cr   c                ,   < \         SV `  VR RR4       R# )r   r,   TFNr   r   s   &&r   r   #FusionLayerNormalizationTF.__init__G  s     4eTBr   c                &   < V ^8  d   QhRS[ RS[ /# r!   r$   )r   r   s   "r   r   r   J  s#     _M _Md _M _Mr   c                0	   . pV P                   P                  V. RO. RO3. RO. RO3.V4      w  rVpVf   R# \        V4      ^8X  g   Q hV^ ,          R9   d   V^,          R9   d   V^,          R9   g   \        P	                  R4       R# VR,          w  ppp	p
ppVRR w  rppRp\        V4      ^8X  d   V^,          pVP
                  R8X  g   Q hV P                   P                  VR^ V4      pVf   \        P	                  R4       R# V P                   P                  V^ V4      pVf   TMV P                   P                  V^ V4      pVf   \        P	                  R4       R# V P                   P                  V4      w  ppVe   V^ 8:  g   VR8  d   Vf   \        P	                  R	4       R# Vf\   VP                  ^ ,          VP                  9  g#   VP                  ^ ,          VP                  9  d   \        P	                  R
4       R# Ve\   VP                  ^ ,          VP                  9  g#   VP                  ^ ,          VP                  9  d   \        P	                  R
4       R# VP                  ^ ,          VP                  ^,          8w  d   \        P	                  R4       R# VVVV	V
VVVVVVV.pVeN   V P                   P                  VR^ V4      pVf   \        P	                  R4       R# VP                  VVV.4       V P                   P                  VVP                  V P                   P                  4       V P                   P                  4       4      '       g   \        P	                  R4       R# V P                  P                  V4       V	P                  ^,          pVP                  ^ ,          p\         P"                  ! RVP                  ^ ,          VV.VP                  ^ ,          .V P                   P%                  RRR7      R7      pVP&                  P                  \         P(                  ! R\+        V4      4      .4       V P,                  P/                  V4       V P0                  V P2                  VP4                  &   R# )a  
 Layer Norm from Tensorflow model(using keras2onnx or tf2onnx):
  +------------------------------------+
  |                                    |
  |                                    |
(Cast_1)                               |
  |                                    |
  |                                    v                                           (B)                             (B)             (A)
 Add --> (Cast_1) --> ReduceMean -->  Sub  --> Mul --> ReduceMean --> (Cast_3) --> Add --> Sqrt --> Reciprocol --> Mul --> Mul --> Sub --> Add
  |                       |                                                                                         |       ^              ^
  |                       |                                                                                         |       |              |
  |                       +--------------------------------------------------(Cast_2)-------------------------------|-------+              |
  |                                                                                                                 v                      |
  +---------------------------------------------------------------------------------------------------------------> Mul--------------------+
r0   Nr+   z=return indice is exepected in [0, 1], but got {return_indice}:N   Nzmul_node_3 not foundzroot node is nonegh㈵>zepsilon is not matchedz;reduce_mean_node_1 and mul_node_3 shall link from root nodez%mul_node_2 shall have two same inputszcast_node_2 not foundz$not safe to fuse layer normalizationr   r4   r5   r7   r;   )
r'   r0   r0   
Reciprocalr>   r,   r   r0   r'   r   )
rA   rA   Nr   r   r   Nr   r   N)r'   r0   r0   r   r>   r,   r+   r   r0   r'   r   )rA   rA   Nr   r   r   r   Nr   r   N)r   rA   )r   rH   rC   rJ   rK   rE   match_parent
get_parentrI   rD   rO   rR   rM   r"   r#   rT   r   rU   rV   rW   rX   rY   rZ   rN   r[   r\   r:   )r   r]   r"   r#   return_indicerg   rf   
sub_node_0
mul_node_0
mul_node_1reciprocol_node	sqrt_node
add_node_0reduce_mean_node_0
mul_node_2
sub_node_1reduce_mean_node_1cast_node_3
mul_node_3node_before_reduce	root_noderj   r;   r^   cast_node_2rq   rr   
fused_nodes   &&&&                        r   ru   FusionLayerNormalizationTF.fuseJ  sF     )-)F)F <  ?! B  G$*
&L =!Q&&&a F*}Q/?6/Im\]N^bhNhLLXY 	
IUVXVYIZF
4F|"&q/K&&&000ZZ,,T5!=PQ
LL/0!ZZ223EqJ]^ " &&'91>QR 	
 LL,-ZZ22:>
7?glw/?KDWLL12$$Q'z/?/??CUC[C[\]C^fpfvfvCvLLVW"$$Q'z/?/??CUC[C[\]C^fpfvfvCvLLVWA*"2"21"55LL@A 
 "**11*faI\]K"45!!#5{K"PQzz//KKJJ**,JJ**,	
 
 LL?@##N3!''*%%a(
 %% $$Q'zB[[^$,,-A{,[	

 	##V%:%:9eGn%U$VW  ,8<8L8L$$Z__5r   r   rw   r   s   @@r   r   r   F  s&     C C_M _M _Mr   r   N)loggingr   fusion_baser   onnxr   r   
onnx_modelr   rx   rJ   r
   r   r   r   r   r   <module>r      sM   
   $  	8	YUv YUxY,6 Y,xcM cMr   