+
    9iE                     \    ^ RI Ht ^ RIHt ^ RIHt ^ RIHt ]! ]4      t	 ! R R]4      t
R# )    )	getLogger)Fusion)helper	OnnxModelc                      a a ] tR t^t oV3R lV 3R lltV3R lR ltV3R lR ltV3R lR ltV3R	 lR
 ltV3R lR lt	Rt
VtV ;t# )FusionFastGeluc                    < V ^8  d   QhRS[ /# )   modelr   )format__classdict__s   "f/var/www/html/photoedit/myenv/lib/python3.14/site-packages/onnxruntime/transformers/fusion_fastgelu.py__annotate__FusionFastGelu.__annotate__   s     4 4i 4    c                *   < \         SV `  VR R4       R# )FastGeluTanhN)super__init__)selfr   	__class__s   &&r   r   FusionFastGelu.__init__   s    
F3r   c                &   < V ^8  d   QhRS[ RS[ /# )r   input_name_to_nodesoutput_name_to_node)dict)r   r   s   "r   r   r      s      4 d r   c                    V P                  WV4      '       d   R # V P                  WV4      '       d   R # V P                  WV4      '       d   R # V P                  WV4      '       d   R # R # )N)fuse_1fuse_2fuse_3fuse_4)r   	tanh_noder   r   s   &&&&r   fuseFusionFastGelu.fuse   s\    ;;y7JKK;;y7JKK;;y7JKK;;y7JKK Lr   c                .   < V ^8  d   QhRS[ R,          /# )r   returnN)bool)r   r   s   "r   r   r      s     j jTTX[ jr   c                0   VP                   ^ ,          V9  d   R# W!P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P	                  VR4      '       g   R# VP                   ^ ,          V9  d   R# W%P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P                  VRRV4      pVf   R# V P                  P                  VR4      pV^ 8  d   R# VP                  V^8X  d   ^ M^,          p	V P                  P                  Yx^8X  d   ^ M^V4      p
V P                  P                  VR^ V4      pVf   R# V P                  P                  VRRR7      pV^ 8  d   R# V P                  P                  TRV^8X  d   ^ M^V4      pVf   R# V P                  P                  TRRTV
'       d   V
.M. R	7      pVf   R# V P                  P                  VR
RR7      pV^ 8  d   R# V P                  P                  TRV^8X  d   ^ M^V4      pVf   R# V P                  P	                  VR4      '       g   R# VP                  ^ ,          V	8w  d   R# VVVVVVVV.pV P                  P                  VVP                   ^ ,          .VV4      '       g   R# V P                  P                  V4       \        P                  ! RV	.VP                   V P                  P                  R4      R7      pRVn        V P                   P#                  V4       V P$                  V P&                  VP(                  &   R# )a  
Fuse Gelu with tanh into one node:
      +---------------------------+
      |                           |
      |                           v
    [root] --> Pow --> Mul -----> Add  --> Mul --> Tanh --> Add --> Mul
      |       (Y=3)   (B=0.0447...)       (B=0.7978...)    (B=1)     ^
      |                                                              |
      +------> Mul(B=0.5)--------------------------------------------+
Note that constant input for Add and Mul could be first or second input: like either A=0.5 or B=0.5 is fine.
NAdd      ?Mul      ?+ݓ?-C6?deltaexclude,C?Pow      @r   inputsoutputsnamecom.microsoftT)outputlenop_typer   has_constant_inputmatch_parentfind_constant_inputinput
get_parentis_safe_to_fuse_nodesnodes_to_removeextendr   	make_nodecreate_node_namedomainnodes_to_addappendthis_graph_namenode_name_to_graph_namer;   )r   r$   r   r   childrenadd_after_tanhmul_after_tanhmul_halfi
root_input	root_nodemul_before_tanhadd_before_tanhmul_after_powpowsubgraph_nodes
fused_nodes   &&&&             r   r    FusionFastGelu.fuse_1   sG    A&99&'7'7':;x=A!!4!4!=!!zz,,^SAA  #+>>&'<'<Q'?@x=A!!4!4!=!!::**>5$H[\JJ**8S9q5^^aAQ7
 JJ))(aAQH[\	**11)UAGZ["JJ**?F&*Qq5**11/5qTUv![\^qr"

//#,YK" 0 
  JJ**=&*Oq5jj%%mUaAQPcd;zz,,S#6699Q<:% 	
 zz//""1%&	
 
 ##N3%%<")),,Z8	

 ,
  ,8<8L8L$$Z__5r   c                :   < V ^8  d   QhRS[ RS[ RS[R,          /# r   r   r   r(   Nr   r)   )r   r   s   "r   r   r      /     r rT rPT rY]`dYd rr   c                >   VP                   ^ ,          V9  d   R# W!P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P	                  VR4      '       g   R# VP                   ^ ,          V9  d   R# W%P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P                  VR4      pV^ 8  d   R# VP                   ^ ,          V9  d   R# W&P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P                  TVP                  ^,          VP                   ^ ,          8X  d   ^ M^V4      p	V P                  P                  VR^ V4      p
V
f   R# V P                  P                  V
RRR7      pV^ 8  d   R# V P                  P                  T
RV^8X  d   ^ M^V4      pVf   R# V P                  P                  TRRTV	'       d   V	.M. R	7      pVf   R# V P                  P                  VR
RR7      pV^ 8  d   R# V P                  P                  TRV^8X  d   ^ M^V4      pVf   R# V P                  P	                  VR4      '       g   R# VP                  VP                  ^,          VP                   ^ ,          8X  d   ^ M^,          pVP                  ^ ,          V8w  d   R# VVVVV
VVV.pV P                  P                  VVP                   ^ ,          .VV4      '       g   R# V P                  P                  V4       \        P                  ! RV.VP                   V P                  P                  R4      R7      pRVn        V P                   P#                  V4       V P$                  V P&                  VP(                  &   R# )ax  
This pattern is from Tensorflow model.
Fuse Gelu with tanh into one node:
      +---------------------------+
      |                           |
      |                           v
    [root] --> Pow --> Mul -----> Add  --> Mul --> Tanh --> Add --> Mul(B=0.5)-->Mul-->
      |       (Y=3)   (B=0.0447...)       (B=0.7978...)    (B=1)                  ^
      |                                                                           |
      +---------------------------------------------------------------------------+
Note that constant input for Add and Mul could be first or second input: like either A=0.5 or B=0.5 is fine.
Nr+   r,   r-   r.   r/   r0   r1   r3   r5   r6   r7   r   r8   r<   T)r=   r>   r?   r   r@   rB   rD   rC   rA   rE   rF   rG   r   rH   rI   rJ   rK   rL   rM   rN   r;   )r   r$   r   r   rO   rP   rR   rS   mul_after_mul_halfrU   rV   rW   rX   rY   rT   rZ   r[   s   &&&&             r   r!   FusionFastGelu.fuse_2   s    A&99&'7'7':;x=A!!4!4!=!!zz,,^SAA  #+>>&'<'<Q'?@x=A!!4!4!=A;JJ**8S9q5??1%88&q'9:x=A!!4!4!=%a[ JJ))#))!,0BBA
	 **11)UAGZ["JJ**?F&*Qq5**11/5qTUv![\^qr"

//#,YK" 0 
  JJ**=&*Oq5jj%%mUaAQPcd;zz,,S#66'--3E3K3KA3NRZRaRabcRd3dajkl
99Q<:% 	
 zz//&&q)*	
 
 ##N3%%<&--,,Z8	

 ,
  ,8<8L8L$$Z__5r   c                :   < V ^8  d   QhRS[ RS[ RS[R,          /# r^   r_   )r   r   s   "r   r   r      r`   r   c           	        VP                   ^ ,          V9  d   R# W!P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P	                  VR4      '       g   R# VP                   ^ ,          V9  d   R# W%P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P                  VRRV4      pVf   R# V P                  P                  VR4      pV^ 8  d   R# VP                  V^8X  d   ^ M^,          p	V P                  P                  VR^ V4      p
V
f   R# V P                  P                  V
RRV4      pVf   R# V P                  P                  VR4      pV^ 8  d   R# V P                  P                  V
RRV4      pVf   R# V P                  P                  VRRR7      pV^ 8  d   R# VP                  V^8X  d   ^ M^,          V	8w  d   R# V P                  P                  TRV^8X  d   ^ M^V4      pVf   R# VP                  ^ ,          V	8X  d   ^pMVP                  ^,          V	8X  d   ^ pMR# V P                  P                  VRVV4      pVf   R# V P                  P                  VR	RR7      pV^ 8  d   R# VP                  V^8X  d   ^ M^,          V	8w  d   R# VVVV
VVVVV.	pV P                  P                  VVP                   ^ ,          .VV4      '       g   R# V P                  P                  V4       \        P                  ! R
V	.VP                   V P                  P                  R
4      R7      pRVn        V P                  P!                  V4       V P"                  V P$                  VP&                  &   R# )a6  
OpenAI's gelu implementation, also used in Megatron:
   Gelu(x) = x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1.0 + 0.044715 * x * x)))

Fuse subgraph into a FastGelu node:
    +------------ Mul (B=0.79788456) -------------------+
    |                                                   |
    +-------------------------------+                   |
    |                               |                   |
    |                               v                   v
  [root] --> Mul (B=0.044715) --> Mul --> Add(B=1) --> Mul --> Tanh --> Add(B=1) --> Mul-->
    |                                                                                 ^
    |                                                                                 |
    +-----------> Mul (B=0.5) --------------------------------------------------------+
Nr+   r,   r-   r.   r/   r0   r1   r5   r   r8   r<   T)r=   r>   r?   r   r@   rA   rB   rC   rE   rF   rG   r   rH   rI   rJ   rK   rL   rM   rN   r;   )r   r$   r   r   rO   rP   mul_lastrR   rS   rT   rV   add_1jmul_7978kmul_before_add_1anothermul_0447mrZ   r[   s   &&&&                 r   r"   FusionFastGelu.fuse_3   s     A&99&'7'7':;x=A!!4!4!=!!zz,,^SAA  #+>>&'<'<Q'?@x=A!!4!4!=A;::**8UDBUVJJ**8S9q5^^aAQ7
**11)UAGZ["

''FYZ=JJ**5#6q5::**?E4I\]JJ**8V6*Jq5>>qAv!1-;::225%a1fRSUhi#!!!$
2G##A&*4G::**+;UGM`aJJ**8V6*Jq5>>qAv!1-; 

 zz//__Q 	
 
 ##N3%%<OO,,Z8	

 ,
  ,8<8L8L$$Z__5r   c                :   < V ^8  d   QhRS[ RS[ RS[R,          /# r^   r_   )r   r   s   "r   r   r   s  s/     y yT yPT yY]`dYd yr   c           	        VP                   ^ ,          V9  d   R# W!P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P	                  VR4      '       g   R# VP                   ^ ,          V9  d   R# W%P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pVP                   ^ ,          V9  d   R# W&P                   ^ ,          ,          p\        V4      ^8w  g   V^ ,          P                  R8w  d   R# V^ ,          pV P                  P	                  VR4      '       g   R# VP
                  VP
                  ^,          VP                   ^ ,          8X  d   ^ M^,          pV P                  P                  VR^ V4      p	V	f   R# V P                  P                  V	RRR7      p
V
^ 8  d   R# V P                  P                  T	RV
^8X  d   ^ M^V4      pVf   R# VP
                  ^ ,          V8X  d   ^pMVP
                  ^,          V8X  d   ^ pMR# V P                  P                  VRW4      pVf   R# V P                  P                  VR	RR7      pV^ 8  d   R# V P                  P                  TRV^8X  d   ^ M^V4      pVf   R# VP
                  ^ ,          V8X  d   ^pMVP
                  ^,          V8X  d   ^ pMR# V P                  P                  VRW4      pVf   R# VP
                  ^ ,          V8w  g   VP
                  ^,          V8w  d   R# VVVVV	VVVV.	pV P                  P                  VVP                   ^ ,          .VV4      '       g   R# V P                  P                  V4       \        P                  ! R
V.VP                   V P                  P                  R
4      R7      pRVn        V P                  P!                  V4       V P"                  V P$                  VP&                  &   V P)                  R
4       R# )a  
PyTorch's gelu implementation with tanh approximation:
   Gelu(x) = 0.5 * x * (1 + torch.tanh(0.7978845834732056 * (x + 0.044714998453855515 * x * x * x)))

Fuse Gelu with tanh into one node:
      +-----------------+------------------+
      |                 |                  |
      |                 v                  v
    [root] ==> Mul --> Mul --> Mul -----> Add  --> Mul --> Tanh --> Add -----> Mul --> Mul -->
      |                       (A=0.0447)          (A=0.7978)        (A=1)       ^     (A=0.5)
      |                                                                         |
      +-------------------------------------------------------------------------+
Note that constant input for Add and Mul could be first or second input.
Nr+   r,   r-   r.   r/   g{Gz?r1   r5   r   r8   r<   T)r=   r>   r?   r   r@   rC   rA   rB   rE   rF   rG   r   rH   rI   rJ   rK   rL   rM   rN   r;   increase_counter)r   r$   r   r   rO   rP   rQ   rR   rT   rV   rj   rW   rl   rX   rn   	mul_cubedmul_squaredrZ   r[   s   &&&&               r   r#   FusionFastGelu.fuse_4s  s    A&99&'7'7':;x=A!!4!4!=!!zz,,^SAA  #+>>&'<'<Q'?@x=A!!4!4!=!!  #+>>&'<'<Q'?@x=A!!4!4!=A;zz,,Xs;;#))~/C/CA/F.J_J_`aJb/b!hij
**11)UAGZ["JJ**?F$*Oq5**11/5qTUv![\^qr"  #z1G""1%3G

//e JJ**=&*Mq5JJ++M5qAv!STVij	??1+G__Q:-Gjj--i]Q:-1B1B11E1S 

 zz//__Q 	
 
 ##N3%%<OO,,Z8	

 ,
  ,8<8L8L$$Z__5j)r    )__name__
__module____qualname____firstlineno__r   r%   r    r!   r"   r#   __static_attributes____classdictcell____classcell__)r   r   s   @@r   r	   r	      sR     4 4 j jXr rhr rhy y yr   r	   N)loggingr   fusion_baser   onnxr   
onnx_modelr   rw   loggerr	   rv   r   r   <module>r      s+   
     	8	^V ^r   