+ 9i=b|^RIHt^RIt^RIHt^RIHt^RIH t H t H t H t ^RI Ht]!]4t!RR]4tR#) ) getLoggerN)Fusion) FusionUtils) NodeProto TensorProtohelper numpy_helper OnnxModelcaa]tRt^toRtV3RlV3RlltRV3RlRlltV3RlRltV3RlR ltV3R lR lt V3R lR lt V3RlRlt V3RlRlt V3RlRlt V3RlRltV3RlRltV3RlRltRtRtVtV;t#)FusionMultiHeadAttentionMMDitzG Fuse MultiHeadAttention for Multimodal Diffusion Transformer (MMDiT). c <V^8dQhRS[/#)modelr )format __classdict__s"g/var/www/html/photoedit/myenv/lib/python3.14/site-packages/onnxruntime/transformers/fusion_mha_mmdit.py __annotate__*FusionMultiHeadAttentionMMDit.__annotate__s''i'c<<\SV`VRR.R7/VnR#)MultiHeadAttentionSoftmax) fused_op_typesearch_op_typesN)super__init__unsqueeze_update_map)selfr __class__s&&rr&FusionMultiHeadAttentionMMDit.__init__s$ .BU^T_`$&!rc&<V^8dQhRS[RS[/#)r start_nodereturn)rint)rrs"rrrs Z]rc `VPPV.ROV^^.VR7pVf^#VR,p\VP4^8wd^#VPP VP^,4pVf^#\VP 4^8wd^#\ V^,4#)a8 Detect num_heads from Reshape & Transpose of q/k/v for both Stable Diffusion 3.x and Flux 1.x: MatMul .. [-1] [24] .. | | | / / Add Concat(axis=0) | / Reshape | Transpose(perm=0,1,3,2) | (start_node) output_name_to_node) TransposeReshapeConcat)rmatch_parent_pathleninputget_constant_valueshaper%)rr#r( input_indexnodes concat_shapevalues&&&& r get_num_heads+FusionMultiHeadAttentionMMDit.get_num_headss ,, :[!Q?G'-'9'9-"-"3"3SS ($ JJ../?AUAUV!++%OOA.0@A&--a07:;44[A     $ $X .:>:N:ND ( ( 7'/q'9 $8P % %inn 5''rcB<V^8dQhRS[RS[S[S[3,RS[/#)raddr(r$)rdictrDr;)rrs"rrr(s-333cS\nI]3bf3rc \VP4^8wdR#VPPV.RO.ROV4pVfR#\ VP4pVP V^,4pVe V^.8wdR#VP V^,4pVe V^.8wdR#VPPV.RO.ROV4pVfR#VP V^,4pVe V^.8wdR#VP V^,4pVe V^.8wdR#VP V^,4V^,P^&VP V^,4V^,P^&R#)a. Update axes of Unsqueeze from [1] to [2] in the following pattern: Unsqueeze Unsqueeze (axes=[0]) (axes=[0]) | | Unsqueeze Unsqueeze ... (axes=[1]) ... (axes=[1]) | / | / Mul Mul | / Add Args: add (NodeProto): the Add node output_name_to_node (Dict[str, NodeProto]): mapping from output name to node Returns: bool: True if the pattern is matched and updated successfully, False otherwise. FT)Mulr}r})rgrgr)rrgr)r.r/rr-rget_squeeze_or_unsqueeze_axesr)rrr(nodes_b fusion_utilsaxes_1axes_0nodes_as&&& rupdate_unsqueeze_axes3FusionMultiHeadAttentionMMDit.update_unsqueeze_axes(sL& syy>Q **..s4UW`buv ?"4::. ;;GAJG >Vs];;GAJG >Vs]**..s4UW`buv ?;;GAJG >Vs];;GAJG >Vs]"?? K "?? K rc4<V^8dQhRS[RS[R,/#r_ra)rrs"rrr]s*RIRI RI[^ae[eRIrc VPPV.R O.R O4pVfR#VwrErgp\VP4^8wdR#VPPVRR.^^.4pVfR#Vwr\P !VR.R O4'gR#\P !V R.R O4'gR#\P !VR^4'gR#VP WB4'gR#VP^,VP^&V P^,V P^&\P!RVP^,V P^,.VP^,R,.VPPR4^R7p VPPV 4VPVPV P&VPP!VP^,V P^,4VP#VP^,VP^,R ,4#) a[ Adjust graph to change query format from BNSH to BSD for Flux model. Note that the graph pattern is complex, and we only do a shallow match here. Before: | | Transpose(perm=0,2,1,3) Transpose(perm=0,2,1,3) | | SimplifiedLayerNorm SimplifiedLayerNorm | / Concat(axis=2) | Mul Mul | / Add | Mul After (Transpose nods are removed, and a Reshape is added): | | SimplifiedLayerNorm SimplifiedLayerNorm | / Concat(axis=1) | Mul Mul | / Add | Reshape (shape=[0, 0, -1]) r+r=r)Nrcrrrdrsre)Addrr+r=r))rrrrrrf)rr-r.r/rrirrrSrXrTrUrVrRrWrIreplace_input_of_all_nodesr\) rr`r(rjr_mul_artrkrlrurvrws &&& r"adjust_flux_query_from_bnsh_to_bsd@FusionMultiHeadAttentionMMDit.adjust_flux_query_from_bnsh_to_bsd]sBzz++  Q   <26/VK v||  !zz++  +[ 9 F  <!// V\RR// V\RR//BB))#CC%**1- A$**1- A ** LLOU\\!_5]]1%/0,,X6     1=A=Q=Q$$_%9%9: --fmmA.>@V@VWX@YZ!!#**Q-A1GHHrc4<V^8dQhRS[RS[R,/#r_ra)rrs"rrrs&1I1Iy1Ibehlbl1Irc VPPV.RO.RO4pVfR#VwrErg\P!VR.RO4'gR#VP WB4'gR#VP ^,VP ^&VP ^,R,VP ^&VPVP ^,VP ^,R,4#)ax Adjust graph to change query format from BNSH to BSD for Flux model. Note that the graph pattern is complex, and we only do a shallow match here. Before: | Transpose(perm=0,2,1,3) | SimplifiedLayerNorm | Mul Mul | / Add | Mul After (Transpose is removed, and a Reshape is added): | SimplifiedLayerNorm | Mul Mul | / Add | Reshape (shape=[0, 0, -1]) Nrcrdre)rrr=r))rrrrrf)rr-rrirr/rXr\)rr`r(rjrrrkrls&&& r)adjust_flux_single_query_from_bnsh_to_bsdGFusionMultiHeadAttentionMMDit.adjust_flux_single_query_from_bnsh_to_bsds:zz++  G   <*.'U// V\RR))#CC%**1- A 1 / 1 !!#**Q-A1GHHrc4<V^8dQhRS[RS[R,/#)rqr$NrC)rrs"rrrs ; ;s ;CRVJ ;rc @\P!RV.VR,.VPPRRR7.ROR7pVPP V4VP VPVP&VPVR,VR,4#)r)rdTranspose_BNSH_to_BSNH) name_prefix)rIrcrerf) rrSrrTrUrVrRrWrIr\)rrr( transpose_qs&&& rtranspose_reshape_bnsh_to_bsd;FusionMultiHeadAttentionMMDit.transpose_reshape_bnsh_to_bsds&&  C [M,,[F^,_     -9=9M9M$$[%5%56!!!g+q6z::rc ><V^8dQhRS[RS[RS[RS[RS[RS[/#)rrkvrX num_headsr$)rDr%r)rrs"rrrsG)) ) )  )  )  ) )rc V^8gQhWV.pV.p\P!RVVVPPR4R7pRVnVP P \P!RV4.4V#)a. Create a MultiHeadAttention node. Args: q (str): name of q k (str): name of k v (str): name of v output (str): output name of MHA num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning. Returns: NodeProto: the node created. rrJz com.microsoftr)rrSrrTdomain attributeextendmake_attribute) rrrrrXr mha_inputs mha_outputsmha_nodes &&&&&& rcreate_multihead_attention_node=FusionMultiHeadAttentionMMDit.create_multihead_attention_nodes,1}}AY h ## ,,-AB  *!!6#8#8i#P"QRrcVPR8XgQhTpVPPVP^,4'dR#VPP V.R O.R OV4pVfR#Vwrgp\ P !VR.RO4'gR#VPPV.RO.RO4p V fR#V wrrpppV P^,pVVP^,8wdR#VPPV RR.^^.4pVfR#VwppVP^,p\ P !VR.RO4'gR#VPPVRR.^^.4pVfR#V^,P^,V P^,8wdR#VP^,pVPPVR^VR7pVeVPPVR^VR7pVfR#\ P !VR.RO4'gR#VPPVR^VR7pVfR#\ P !VR.RO4'gR#MGVPPVR^VR7pVfR#\ P !VR.RO4'gR#V'dVPVV4MVPWc^R 7pV^8XdVPVVVRJ4pV^8:dR#VeVPW4pMVPW4pVf=VPW4pVf(VPW4pVfVP!VV4pVP#VVVVP^,VR 7pVP$P'V4VP(VP*VP,&VP.P1WgV.4R VnR#)rNr)rcrSqrtDivr+)r2r()r2)rrrrXrT)MatMulr)r*)rrrrrf)rrrrrCastSliceShape)rrrgrrgrrr)rrgrhr)op_typerfind_graph_outputrXmatch_child_pathrrir-r/ match_parentr6r>rxrnrrrrrUrVrRrWrInodes_to_remover prune_graph)rnodeinput_name_to_nodesr(softmaxr3 matmul_s_v transpose_out reshape_outq_nodes matmul_qkr`sqrt_q_2div_qsqrt_q_shape_qq_bnshk_nodesmul_kr9r k_scale_nodesrconcat_v transpose_1 transpose_2rqueryrs&&&& rfuse"FusionMultiHeadAttentionMMDit.fuses||y((( :: ' 'q(9 : :  ++ 79QSf  = 16. ;// v|TT **..  N $  ? CJ@ (61aQ W]]1% % **..y5+:NQRTUPVW ? ${   a // V\RR  44UVUOaQRVT     ! !! $q(9 9    Q ::**:xQdw*x   **11+1J]2K"33KVV**11+1J]2K"33KVVW **11KQL_2K"33KVV    x)< =##JQR#S  >11+?RT\dhThiIA~  66uRE@@\E =;;EWE}FFub=!>>vGZ[E77%%a( 8    *6:6J6J$$X]]3 ##Z $LM r)rr)r)__name__ __module__ __qualname____firstlineno____doc__rr6r>r\rnrxrrrrrrr__static_attributes____classdictcell__ __classcell__)r rs@@rr r s''B//b##4.H.H`MXMX^"("(H33jRIRIh1I1If ; ;))V  rr )loggingrnumpyrO fusion_baserrronnxrrrr onnx_modelr rloggerr rrrs4 $== 8 K  FK  r