+
    9i                         ^ RI Ht ^ RIHt ^ RIHt ^ RIHtHtH	t	 ^ RI
Ht ]! ]4      t ! R R]4      t ! R R	]4      tR
# )    )	getLogger)Fusion)FusionUtils)	NodeProtoTensorProtohelper	OnnxModelc                      a a ] tR t^t oRtRV3R lV 3R llltV3R lR ltV3R lR ltR tR	 t	R
 t
R tR tV3R lR ltRV3R lR lltR tR tRR ltR tR tR tRtVtV ;t# )FusionEmbedLayerNoMaskz
Fuse embedding layer into one node (EmbedLayerNormalization).
It supports the following model types: BERT, DistilBert, ALBert.
c                &   < V ^8  d   QhRS[ RS[/# )   modeldescription)r
   str)format__classdict__s   "h/var/www/html/photoedit/myenv/lib/python3.14/site-packages/onnxruntime/transformers/fusion_embedlayer.py__annotate__#FusionEmbedLayerNoMask.__annotate__   s      i c     c                   < \         SV `  VR RR.V4       \        V4      V n        RV n        RV n        RV n        RV n        R# )EmbedLayerNormalizationLayerNormalizationSkipLayerNormalizationNF)super__init__r   utilsshape_infershape_infer_done	attention
embed_node)selfr   r   	__class__s   &&&r   r   FusionEmbedLayerNoMask.__init__   sP    %!#;<		
 !'
 % r   c                J   < V ^8  d   QhRS[ RRS[S[ S[ 3,          ,          /# )r   addreturnN)r   tuple)r   r   s   "r   r   r   %   s*     	2 	2I 	2$y)?S9T2T 	2r   c                    V P                   P                  VR .^ .4      pVf   R# V P                   P                  VR .^.4      pVf   R# V^ ,          V^ ,          3# )GatherN)r   match_parent_path)r#   r'   gather_0_pathgather_1_paths   &&  r   match_two_gather'FusionEmbedLayerNoMask.match_two_gather%   sa    

44S8*qcJ 

44S8*qcJ Qq!111r   c                X   < V ^8  d   QhRS[ RS[S[S[S[ ,          3,          RS[RS[/# )r   	layernorminput_name_to_nodesis_distil_bertr(   )r   dictr   listbool)r   r   s   "r   r   r   0   sG     R RR "#tI"67R 	R
 
Rr   c                @   V P                   P                  VRVRR7      V n        V P                  e   R# VP                  ^ ,          V9  d   R# W!P                  ^ ,          ,          p\	        V Uu. uF  qUP
                  NK  	  up4      pV. R	O8X  d   V F~  pVP
                  R8X  g   K  V P                   P                  V. R
O. RO4      pVf   K=  VR,          P                  ^ ,          VP                  ^ ,          8X  g   Ko  V^,          V n         R# 	  \        V4      ^8X  Ed   V^ ,          P
                  R8X  d   V^ ,          P                  ^ ,          V9   d   W$^ ,          P                  ^ ,          ,          p	\        V	4      ^8X  d   V	^ ,          P
                  R8X  d   V	^ ,          P                  ^ ,          V9   dh   W)^ ,          P                  ^ ,          ,          p
V
 F  pVP
                  R8X  g   K  Wpn         R# 	  \	        V
 Uu. uF  qUP
                  NK  	  up4      pV'       d5   V. RO8w  d*   V. RO8w  d!   V. RO8w  d   \        P                  R4       R# R# V. RO8w  d!   V. R	O8w  d   \        P                  R4       R# R# u upi u upi )ao  Check that LayerNormalization has a child of Attention node or subgraph like Attention.

Args:
    layernorm (NodeProto): LayerNormalization node
    input_name_to_nodes (Dict[str, List[NodeProto]]): map from input name to nodes
    is_distil_bert (bool): whether it is DistilBert or not

Returns:
    bool: whether there is Attention node or subgraph like Attention
	AttentionF)	recursiveTMatMulr   Addz<No Attention like subgraph in children of LayerNormalization)r;   r;   r;   r   )r<   r;   MultiHeadAttentionr;   )NNr   r   )r;   r;   r;   Shaper   )r<   r;   r;   r;   r?   r?   )r<   r;   r;   r;   r?   )r<   r;   r;   r;   )r   find_first_child_by_typer!   outputsortedop_typer,   inputcross_attentionlenloggerdebug)r#   r2   r3   r4   childrenchildchildren_typesnodepath1grandchildrennodess   &&&&       r   check_attention_subgraph/FusionEmbedLayerNoMask.check_attention_subgraph0   sD     <<{$75 = 
 >>%A&99&'7'7':;H EH5H EF UU <<#;; JJ88I*E
 (U2Y__Q-?9CSCSTUCV-V/4Qx,# ! x=A(1+"5"5"AhqkFXFXYZF[_rFr/0B0B10EFMM"a'!!$,,5!!$++A.2EE+!,<,C,CA,FG!D||{2)-# " "(E(JE5E(J!K  "cc"&]]"&TT[\    " 
 ! %  [\q !F: )Ks   1JJc                   V P                   P                  VRR.^^.4      pVf(   V P                   P                  V. RO. RO4      pVf   R# V^ ,          VR,          reVP                  ^ ,          V8w  d   R# V P                   P                  V. RO. R	O3. R
O. RO3.V4      w  rxpVf   R# V^,          p	V P                  P                  V	^ ^ 4      '       d$   V P                  P                  V	^^4      '       g   R# VR,          p
V P                  P                  V
^^4      '       g   R# VR,          pVP                  ^ ,          V8w  d   R# R# )a  Match position embedding path from input_ids to Gather for DistilBert.

Pattern is like the following:
         (input_ids)
              |
             Shape
               |                          |    Gather (indices=1)
               |       |
               |      Cast (optional)
               |       |
               |      Range (start=0, end=*, delta=1)
               |       |
               |    Unsqueeze
               |    /
              Expand
                |
              Gather
Expandr?   FT)rS   WhereReshaper?   )   rV   r   r   r>   )	UnsqueezeRangeCastr+   r?   )r   r   rV   r   r   )rW   rX   r+   r?   )r   r   rV   r   )r   r,   rD   match_parent_pathsr   check_node_input_value)r#   position_embedding_gather	input_idsoutput_name_to_noderM   expandshape_path2
range_nodegather_node
shape_nodes   &&&&        r   #match_position_embedding_distilbert:FusionEmbedLayerNoMask.match_position_embedding_distilbert   s>   * 

,,-FSZH[^_ab]cd=JJ00)7E
 }a%);;q>Y&jj33BOT:LI  
! =1X
JJ--j!Q??DJJDeDefprsuvDwDwBi

11+q!DD2Y
A)+r   c                    R# )aI  Match position embedding path from input_ids to Gather for Roberta.

Roberta Embedding Layer Pattern (* is optional since it might be removed by ORT, ? is the padding word id):
  (input_ids) --> Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Mul -- Cast(to=7) -- Add(B=1) -- Cast(to=7)* --> Gather
                                        |                              ^
                                        V                              |
                                        +------------------------------+

Roberta new pattern from transformers v4.9:
   (input_ids) --> Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Add(B=0) -- Mul -- Cast(to=7) -- Add(B=1) --> Gather
                                        |                                           ^
                                        V                                           |
                                        +-------------------------------------------+

start_node = position_embedding_gather
start_index = 1

# match optional Cast node.
parent = self.model.get_parent(start_node, start_index, output_name_to_node)
if parent is None:
    return
if parent.op_type == "Cast":
    if OnnxModel.get_node_attribute(parent, "to") != 7:
        return
    start_node = parent
    start_index = 0

i, path, return_indices = self.model.match_parent_paths(
    start_node,
    [ (['Add', 'Cast', 'Mul', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0]),
      (['Add', 'Cast', 'Mul', 'Add', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0, 0])],
    output_name_to_node)

if path is not None:
    # constant input of Add shall be 1.
    i, value = self.model.get_constant_input(path[0])
    if value != 1:
        return False

    _, self.padding_word_id = self.model.get_constant_input(path[-1])

    return input_ids == path[-1].input[0]
F r#   r]   r^   r_   s   &&&&r    match_position_embedding_roberta7FusionEmbedLayerNoMask.match_position_embedding_roberta   s    Z r   c                   V P                   P                  VRR.^^.V4      pVf   R# Vw  rVV P                   P                  VP                  ^ ,          4      pVe   \	        VP
                  4      ^8X  d   VP
                  ^ ,          ^8X  d   V P                  P                  V^^ .4      '       dc   V P                  P                  V^^.4      '       d?   \	        VP                  4      ^8X  g'   V P                  P                  V^^.4      '       g   R# V P                   P                  4       pV^8  d#   \        P                  ! VR^ .4      '       g   R# M&V P                  P                  V^^ .4      '       g   R# V P                   P                  V^ V4      p	V	f   R# V	P                  R8X  dD   V P                  P                  V	^^ 4      '       g   R# V P                   P                  V	^ V4      p
MT	p
V
e   V
P                  R8w  d   R# V P                  P                  V
^^4      '       g   R# V P                   P                  V
^ V4      pVe   VP                  R8w  d   R# W+P                  ^ ,          8H  # )aw  Match position embedding path from input_ids to Gather for BERT.

BERT Embedding Layer Pattern:
                            (input_ids)
                           /                                          /          Shape
                        /              |
                      /              Gather (indices=1)
                     /                  |
                    /                  Add (optional, B=0)
                   /                    |
                Gather (segment_ids) Unsqueeze (axes=0)
                   \        |           |
                    \     Gather      Slice (data[1,512], starts=0, ends=*, axes=1, steps=1)
                      \    /            |
                        Add          Gather
                           \       /
                              Add
                               |
                        LayerNormalization
SlicerW   Faxesr<   r+   r?   )r   r,   get_constant_valuerD   rF   ra   r   r\   get_opset_versionr   check_node_attribute
get_parentrC   )r#   r]   r^   r_   pathslice	unsqueezeslice_weightopset_versionrL   gatherra   s   &&&&        r   match_position_embedding_bert4FusionEmbedLayerNoMask.match_position_embedding_bert   s   , zz++%k"F	
 <zz44U[[^D$L&&'1,""1%*

11%QC@@

11%QC@@U[[!Q&$***K*KESTWXVY*Z*Z

446233IvsKK L ::44YA3GGzz$$Y3FG<<<5 ::44T1a@@ZZ**44GHFF>V^^x7

11&!Q??

%%fa1DE=EMMW4KKN**r   c                n    V P                  WV4      '       d   R # V P                  WV4      '       d   R # R# )TF)r{   rg   rk   s   &&&&r   match_position_embedding/FusionEmbedLayerNoMask.match_position_embedding9  s7    --.GTghh 334MZmnnr   c                n	   VP                   ^,          pV'       d   VP                   ^,          MRpVP                   ^,          pV P                  '       g)   V P                  P                  RR7      V n        RV n        V P                  e   V P                  P                  V4      pV P                  P                  V4      pV'       d	   V'       g   Q h\        V4      ^8X  d&   \        V4      ^8X  d   V^,          V^,          8X  g   \        P                  RV RV 24       R# V'       dX   V P                  P                  WE4      '       g7   \        P                  RV RV P                  P                  V4       24       R# V P                  P                  VP                   ^ ,          4      p	V	e   \        V	P                  4      ^8w  d   \        P                  R	4       R# V P                  P                  VP                   ^ ,          4      p
V
eD   \        V
P                  4      ^8w  g*   V	P                  ^,          V
P                  ^,          8w  d   \        P                  R
4       R# V'       d   V P                  P                  VP                   ^ ,          4      pVeD   \        VP                  4      ^8w  g*   V	P                  ^,          VP                  ^,          8w  d   \        P                  R4       R# V	P                  ^ ,          V
P                  ^ ,          8:  df   \        P                  RVP                   ^ ,           RV	P                  ^ ,           RVP                   ^ ,           RV
P                  ^ ,           24       V'       Ed   V	P                  ^ ,          XP                  ^ ,          8:  df   \        P                  RVP                   ^ ,           RV	P                  ^ ,           RVP                   ^ ,           RVP                  ^ ,           24       V
P                  ^ ,          VP                  ^ ,          8:  df   \        P                  RVP                   ^ ,           RV
P                  ^ ,           RVP                   ^ ,           RVP                  ^ ,           24       R# )zXSanity check of embedding weights, and match hidden_size of weights and shape of inputs.NT)updatez^Cannot fuse EmbedLayerNormalization: input_ids and position_ids not matched in 2nd dimension: z vs FzYCannot fuse EmbedLayerNormalization: input_ids and segment_ids does not have same shape: z != zICannot fuse EmbedLayerNormalization: word embedding table is not expectedzMCannot fuse EmbedLayerNormalization: position embedding table is not expectedzLCannot fuse EmbedLayerNormalization: segment embedding table is not expectedzword_embedding_table (z) size z <= position_embedding_table (z <= segment_embedding_table (zposition_embedding_table ()rD   r    r   infer_runtime_shaper   get_edge_shaperF   rG   infocompare_shaperq   ra   warning)r#   word_embedding_gathersegment_embedding_gatherr]   r^   segment_idsposition_idsinput_ids_shapeposition_ids_shapeword_embedding_tableposition_embedding_tablesegment_embedding_tables   &&&&        r   check_embedding&FusionEmbedLayerNoMask.check_embeddingG  s   )//2	;S.44Q7Y]066q9$$$#zz==T=JD$(D!'"..==iHO!%!1!1!@!@!N"'999O$)*+q0#A&*<Q*??t  vE  uF  FJ  K]  J^  _ 4#3#3#A#A)#Y#Yop  pA  AE  FJ  FV  FV  Fe  Fe  fq  Fr  Es  t #zz<<=R=X=XYZ=[\'3/C/I/I+Ja+OKKcd#'::#@#@AZA`A`abAc#d $,+112a7$**1-1I1O1OPQ1RRKKgh&*jj&C&CD\DbDbcdDe&f#'/.445:(..q15L5R5RST5UUjk  %%a(,D,J,J1,MMNN()>)D)DQ)G(HPdPjPjklPmOn  oM  Ng  Nm  Nm  no  Np  Mq  qx  yQ  yW  yW  XY  yZ  x[  \ ;#))!,0G0M0Ma0PP,-B-H-H-K,LGThTnTnopTqSr  sP  Qi  Qo  Qo  pq  Qr  Ps  sz  {R  {X  {X  YZ  {[  z\  ] (--a04K4Q4QRS4TT01J1P1PQR1S0TT[\t\z\z{|\}[~  \  ]u  ]{  ]{  |}  ]~  \  F  G^  Gd  Gd  ef  Gg  Fh  i r   c                J   < V ^8  d   QhRS[ RS[S[ RS[,          3,          /# )r   
input_namer(   N)r   r)   r   )r   r   s   "r   r   r     s*     - - -c4);K6K0L -r   c                6   RpV P                   P                  V4      pVeZ   VP                  P                  P                  \
        P                  8w  d!   V P                  P                  V4      w  rBWB3# Tp WB3# V P                  P                  V4      w  rBWB3# )ap  Cast a graph input or node input to int32.

Args:
    input_name (str): name of graph input or node input

Returns:
    A tuple of casted input name and the cast node.
    int32_output (str): If input is int32, it is the input name, Otherwise it is output name of Cast node.
    input_cast_node (Union[None, NodeProto]): Cast node. It could be None if input is int32.
N)	r   find_graph_inputtypetensor_type	elem_typer   INT32r   cast_input_to_int32)r#   r   input_cast_nodegraph_inputint32_outputs   &&   r   cast_to_int32$FusionEmbedLayerNoMask.cast_to_int32  s     jj11*="++559J9JJ04

0N0Nz0Z- ,,	  * ,, -1JJ,J,J:,V)L,,r   c                Z   < V ^8  d   QhRS[ RS[RS[RS[RRS[,          RS[ R,          /# )r   r^   r2   r   r]   r   Nr   )r   r   )r   r   s   "r   r   r     sX     ` `` `  )	`
 $-` #'"2` Dj`r   c	                F   . p	V P                  V4      w  rV P                  P                  R4      pVP                  R8X  d(   VP                  ^,          pVP                  ^,          pM&VP                  ^,          pVP                  ^,          pRpVeb   V P                  VP                  ^,          4      w  rVVVP                  ^ ,          VP                  ^ ,          VP                  ^ ,          VV.pM+VRVP                  ^ ,          VP                  ^ ,          RVV.pVe6   VP                  R4       V P                  V4      w  rjVP                  V4       VR,           VR,           .pV'       d!   Ve   TMVR,           pVP                  V4       \        P                  ! RVVVR7      pR	Vn        VP                   F2  pVP                  R
8X  g   K  VP                  P                  V.4       K4  	  \        VP                  4      ^ 8X  d2   VP                  P                  \        P                  ! R
R4      .4       V	P                  V4       V	 F&  pV P                  V P                  VP                  &   K(  	  V P                   P                  V	4       VV n        V# )a  Create an EmbedLayerNormalization node. Note that segment embedding is optional.

Args:
    input_ids (str): input_ids for word embeddings
    layernorm (NodeProto): LayerNormalization or SkipLayerNormalization node.
    word_embedding_gather (NodeProto): the Gather node for word embedding
    position_embedding_gather (NodeProto): the Gather node for position embedding
    segment_embedding_gather (Union[None, NodeProto]): the Gather node for segment embedding, or None.

Returns:
    NodeProto: the EmbedLayerNormalization node created.
r   r   N _output_dummy_mask_index_embedding_sum)outputsnamezcom.microsoftepsilong-q=)r   r   create_node_namerC   rD   appendr   	make_nodedomain	attributer   extendrF   make_attributethis_graph_namenode_name_to_graph_namenodes_to_addr"   )r#   r^   r2   r   r]   r   r   embedding_sum_outputembedding_sum_namer   rb   	node_namegammabetaembed_node_inputsr   embed_node_outputsr   r"   attrL   s   &&&&&&&&&            r   create_fused_node(FusionEmbedLayerNoMask.create_fused_node  sn   . )))4	JJ//0IJ	 44OOA&E??1%DOOA&E??1%D #/!//0H0N0Nq0QRNK %++A.)//2(..q1! %++A.)//2! #$$R("00>OL$$\2')3YAT5TU);)G%YYiMiD%%d+%%%&	

 ,
 &&Cxx9$$$++SE2 ' z##$)  '')>)>y')R(ST 	J' D6:6J6JD((3 !  .$r   c                    V P                   P                  VP                  ^ ,          VP                  ^ ,          4       RV n        R# )r   TN)r   replace_input_of_all_nodesrA   prune_graph)r#   r2   r"   s   &&&r   finish_fusion$FusionEmbedLayerNoMask.finish_fusion
  s5    

--i.>.>q.A:CTCTUVCWXr   c                    VP                   R 8H  ;'       d@    \        VP                  4      ^8  ;'       d     \        VP                  ^,          4      ^ 8  # )r   )rC   rF   rA   )r#   rL   s   &&r   "is_skip_layer_norm_with_sum_output9FusionEmbedLayerNoMask.is_skip_layer_norm_with_sum_output  sJ     88nnc$++>NQR>RnnWZ[_[f[fgh[iWjmnWnnr   c                `   V P                  V4      pVf   R# Vw  rxVP                  ^,          p	VP                  ^,          p
V P                  WRR7      '       g   R# V P                  VR V4      '       g   R# VP                  R8X  d]   V P                  V4      p^pTpV'       d   VP                  ^,          MR pVR J;'       d    V P                  P                  V4      R JpMTpVP                  R8X  d   ^ M^p\        VP                  4      V8  d   VP                  V,          MR pVR J;'       d    V P                  P                  V4      R JpT;'       d!    W9   ;'       d    \        W>,          4      ^8  pVR J;'       d$    VP                  R8g  ;'       g    T;'       g    TpT P                  T	TTTTT
TV'       d   TMR R7      pV'       dE   RVP                  V&   V'       g.   V P                  P                  VVP                  ^,          4       V P                  VV4       R# )NFr4   r   r<   )r   r   _no_use__to_be_removed_T)r/   rD   rP   r   rC   r   rA   r   find_graph_outputrF   r   r   r   )r#   r2   add_before_layernormr3   r_   optional_segment_gather
two_gatherr   r]   r^   r   need_embedding_sum_outputsum_output_indexnode_with_sum_output
sum_outputis_sum_graph_outputis_sum_used_by_multiple_nodesr"   s   &&&&&&            r   	fuse_gpt2 FusionEmbedLayerNoMask.fuse_gpt2  s@   ( **+?@
;E8)//2	066q9,,Y\a,bb##$94AZ[[  88(,(O(OPY(Z% #, 0I))!,tJ#-T#9"u"u

@\@\]g@hpt@t#7 $8$@$@E$Iqq +2236FF %++,<= 
 $.T#9"u"u

@\@\]g@hpt@too
 AoosK^KjGknoGo * *44)? ) )$,,5mm9LmmPm &
 ++!%#!:-@zd , 	

 %<U ''(89&

55j*BSBSTUBVW9j1r   c                J   V P                  V4      pVf   R# Vw  rgVP                  ^,          pV P                  WRR7      '       g   R# V P                  WxV4      '       g   R# V P	                  VRV4      '       g   R# V P                  WWgR4      p	V P                  W4       R# )a  Fuse embedding layer for DistilBert
Args:
    layernorm (NodeProto): node of LayerNormalization or SkipLayerNormalization
    add_before_layernorm (NodeProto): the Add node before LayerNormalization, or the SkipLayerNormalization itself
    input_name_to_nodes (Dict[str, List[NodeProto]]): map from input name to nodes
    output_name_to_node (Dict[str, List[NodeProto]]): map from output name to nodes
NFTr   )r/   rD   rP   r~   r   r   r   )
r#   r2   r   r3   r_   r   r   r]   r^   r"   s
   &&&&&     r   fuse_distilbert&FusionEmbedLayerNoMask.fuse_distilbertc  s    & **+?@
;E8)//2	,,Y\`,aa,,-FSfgg##$94AZ[[++"7TX

 	91r   c                <   V P                   P                  VR.^ .4      pVf   R# V P                  V^ ,          4      pVf   R# Vw  rxVP                  ^,          p	V P	                  WRR7      '       g   R# V P                   P                  VR.^.4      p
V
f   R# V
^ ,          pV P                  WV4      '       g!   V P                  WV4      '       g   R# TpTpTpV P                  WxV4      '       g   R# V P                  V	VVVV4      pV P                  W4       R# )a  Fuse embedding layer for Bert
Args:
    layernorm (NodeProto): node of LayerNormalization or SkipLayerNormalization
    add_before_layernorm (NodeProto): the Add node before LayerNormalization, or the SkipLayerNormalization itself
    input_name_to_nodes (Dict[str, List[NodeProto]]): map from input name to nodes
    output_name_to_node (Dict[str, List[NodeProto]]): map from output name to nodes
r<   Fr   r+   T)	r   r,   r/   rD   rP   r~   r   r   r   )r#   r2   r   r3   r_   add_2_gatherr   r   r   r^   position_embedding_pathr]   tempr"   s   &&&&&         r   	fuse_bert FusionEmbedLayerNoMask.fuse_bert  s:    zz334H5'TUSVW**<?;
:D7)//2	,,Y\a,bb"&**">">?SV^U_bcad"e"*$;A$>!,,-FSfgg001IVijj+D'@$(,%##$9Unoo++!%$

 	91r   c                n   V P                   P                  VR .^ .4      pVP                  R8X  d   Vf   R# V^ ,          pRpMV P                   P                  VR.^ .4      pV P                   P                  VR.^.4      pVf   Ve   Vf   R# V^ ,          pV^ ,          pMDVe=   Vf9   V P                   P                  VR .^.4      pVf   R# V^ ,          pV^ ,          pMTpRpV P                  WW#V4      '       d   R# V P	                  WW#4      '       d   R# V P                  WW#4      '       d   R# R# )r<   r   Nr+   )r   r,   rC   r   r   r   )	r#   rL   r3   r_   first_add_pathr   r   r-   r.   s	   &&&&     r   fuseFusionEmbedLayerNoMask.fuse  sJ   55dUGaSI<<//%#1!#4 &*# JJ88zA3OM JJ88zA3OM$)B!)'5a'8$*7*:'*}/D!%!=!=dUGaS!Q!)'5a'8$*7*:''+$*.'>>(;Ri
 
 <Oee>>$6I__ `r   )r!   rE   r"   r   r   r    r   )zno mask)NFN)N)__name__
__module____qualname____firstlineno____doc__r   r/   rP   rg   rl   r{   r~   r   r   r   r   r   r   r   r   r   __static_attributes____classdictcell____classcell__r$   r   s   @@r   r   r      s     
 	2 	2R Rh<|-^F+PHT- -.` `D 
oOb'R0d" "r   r   c                   T   a a ] tR tRt oRV3R lV 3R llltR tV 3R ltRtVtV ;t	# )FusionEmbedLayerNormalizationi  c                    < V ^8  d   QhRS[ /# )r   r   r	   )r   r   s   "r   r   *FusionEmbedLayerNormalization.__annotate__  s     - -i -r   c                4   < \         SV `  VR 4       W n        R# )z	with maskN)r   r   use_mask_index)r#   r   r   r$   s   &&&r   r   &FusionEmbedLayerNormalization.__init__  s    ,,r   c                   V P                   p\        VP                  4      ^8X  d=   VP                  P                  V4       \        P                  RVP                  4       M\        VP                  4      ^8  dI   VP                  ^,          '       g0   WP                  ^&   \        P                  RVP                  4       M"\        P                  RVP                  4       R# V F  p\        P                  RVP                  4       VP                  R8X  d#   VP                  ^,          VP                  ^&   KV  VP                  R8X  g   Ki  VP                  ^,          VP                  ^&   K  	  R# )   zappend mask to %szreplace mask in %szskip mask in %sNzupdate mask_index in %sr9   r=   )	r"   rF   rD   r   rG   rH   r   rC   rA   )r#   
mask_int32attention_nodesr"   attention_nodes   &&&  r   replace_mask*FusionEmbedLayerNormalization.replace_mask  s    __
z A%##J/LL,joo>!!"Q&z/?/?/B/B",QLL-z?LL*JOO<-NLL2N4G4GH%%4*4*;*;A*>$$Q'''+??*4*;*;A*>$$Q' .r   c                  < R V n         R V n        R V n        \        SV `  WV4       V P                  f   R # V P
                  '       g)   \        P                  R4       V P                  R4       R # V P                   f7   V P                  f)   \        P                  R4       V P                  R4       R # V P                   '       d   V P                   P                  ^,          pMV P                  P                  ^,          pW$,          pV P                  P                  V4      '       dH   V Uu. uF  qP                  R9   g   K  VNK  	  ppV P                  WF4       V P                  R4       R # WC9  d*   \        P                  RV4       V P                  R4       R # W4,          pVP                  R9   d   V Uu. uF  qP                  R9   g   K  VNK  	  ppXP                  R8X  dH   VP                  ^ ,          p\        V4      \        V4      8X  d   V P                  P!                  V4       V P                  WF4       V P                  R4       R # R # u upi u upi )	NzG--use_mask_index is not set: EmbedLayerNormalization will not have maskz EmbedLayerNormalization(no mask)zLEmbedLayerNormalization will not have mask since attention node is not foundz"EmbedLayerNormalization(with mask)zHEmbedLayerNormalization will not have mask since %s is not a node output	ReduceSum)r9   r=   )r   rY   )r!   rE   r"   r   r   r   rG   rH   increase_counterrD   r   r   rC   r   rF   nodes_to_remover   )r#   rL   r3   r_   r   children_nodesr   r$   s   &&&&   r   r   "FusionEmbedLayerNormalization.fuse  s   #T0CD??""""LLbc!!"DE>>!d&:&:&BLLgh!!"DE>>>--a0J--33A6J,8::&&z220>v,,RuButtOvj:!!"FG0LLceop!!"DE".<<000>v,,RuButtOv||{*!ZZ]
~&#o*>>((//5j:!!"FG 1 w ws   /II>I I )r!   rE   r"   r   )F)
r   r   r   r   r   r   r   r   r   r   r   s   @@r   r   r     s      - -?*-H -Hr   r   N)loggingr   fusion_baser   fusion_utilsr   onnxr   r   r   
onnx_modelr
   r   rG   r   r   rj   r   r   <module>r     sC      $ / /  	8	PV PfGH$: GHr   