
    wi%                        d dl mZ d dlZd dlZd dlmZ ddd
Zd Zd Z		 	 	 	 	 	 	 	 	 	 	 dddZ
 G d de          ZdS )    )annotationsN)OpRunx
np.ndarrayaxisintreturnc                    t          j        | |d          }t          j        | |z
            }t          j        ||d          }||z  S )NT)r   keepdims)npmaxexpsum)r   r   x_maxtmpss        o/root/.openclaw/workspace/chatterbox_venv_py311/lib/python3.11/site-packages/onnx/reference/ops/op_attention.py_softmaxr      sJ    F14$///E
&U

C
s---A7N    c                N    |dk    r| |z  }t          j        |          }||z  S | S )Nr   )r   tanh)XsoftcapYs      r   _softcapr      s0    {{KGAJJ7{Hr   c                    | j         dd         \  }}t          j        t          j        |||z
  f| j                  d          }t          j         ||dk    <   | ddd|dfxx         |z  cc<   | S )zApplies a causal mask on the input `mask`:
    ``mask[i, j] = -inf if past_sequence_length + i > j else 0``.
    Because a softmax is applied on the mask, -inf becomes 0 and 0 becomes 1.
    The modification is done inplace.
    Ndtype   )k.)shaper   triuonesr    inf)maskpast_sequence_lengthq_sequence_lengthtotal_sequence_lengthr$   s        r   _apply_causalr+      s     04z"##,,7
 58L LM*	
 	
 	
   D vgDOaaa%&&	&'''4/'''Kr   FQKV	attn_masknp.ndarray | Nonepast_key
past_valuenonpad_kv_seqlenc                   t          | j                  t          |j                  cxk    rt          |j                  k    sn J t          | j                  }| j        d         }t          | j                  dk    r| j        d         }|j        d         }|j        d         }|	|
J t          ||	z            }|| j        d         |	|g}t          j        | |          } t          j        | d          } t          ||
z            }||j        d         |
|g}t          j        ||          }t          j        |d          }t          ||
z            }||j        d         |
|g}t          j        ||          }t          j        |d          }t          | j                  dk    r0t          |j                  dk    rt          |j                  dk    sJ |$| j        d         }dt          j        |          z  }t          j        |          }|t          j        ||fd          }n|}|t          j        ||fd          }n|}|}|}| j        d         }|j        d         }t          j        ||f| j	                  }|f||j        d	         z
  }|dk    rPd
g|j
        dz
  z  d|fgz   } |j	        t          j        k    rdnt          j         }!t          j        || d|!          }|r|>t          j        ||f| j	                  }"t          |"||j        d         nd          }n|j	        t          j        k    r+d|z
                      | j	                  t          j         z  }t          |                                ||j        d         nd          }nM|K|j	        t          j        k    r1d|z
                      | j	                  }t          j         ||dk    <   ||z   }||                    dd|j
        z
  z  |j        z             }t          j        |          |d d t          j        f         k     }#|#                    |dd|          }#t          j        |#dt          j                   }#||#z  }|	| j        d         }	|
|j        d         }$|j        d         }%n|
}$|
}%|	|$k    rB|	|$z  dk    r9|$|%k    r3|	|$z  }&t          j        ||&d          }t          j        ||&d          }t          j        |d          }'t          j        | |z  |'|z            }(|(|z   })|dk    r|)                                }(|t-          |)|          })|dk    r|)}(|2|)                    t.          j                            |                    })t5          |)          }*|dk    r|*}(|(                    | j	                  }(t          j        |*|                              | j	                  }+|dk    rCt          j        |+d          }+t          j        |+|+j        d         |+j        d         d	f          }+|+|||(fS )Nr         r!   )r   r6   r!   r5      )r   r   r   )r   r   Fconstant)modeconstant_values)r(   )r!   )repeatsr   )r   r!   r5   r6   )lenr#   r	   r   reshape	transposesqrtconcatenatezerosr    ndimbool_r&   padr+   astypecopyarangenewaxiswhererepeatmatmulr   onnxhelpertensor_dtype_to_np_dtyper   ),r,   r-   r.   r/   r1   r2   r3   scale	is_causalq_num_headskv_num_headssoftmax_precisionr   qk_matmul_output_modeinput_shape_len
batch_sizehidden_size_qhidden_size_khidden_size_vhead_size_qintermediate_shape_qhead_size_kintermediate_shape_khead_size_vintermediate_shape_vq_head_sizepresent_keypresent_valuer)   kv_sequence_length	attn_bias	pad_width	pad_shape	pad_value	temp_maskpadding_maskk_num_headsv_num_headsseq_repsk_transposeqk_matmul_outputqk_with_bias
qk_softmaxoutputs,                                               r   _compute_attentionrr   .   s     qw<<3qw<<77773qw<<777777
 !'llOJ
17||q


&<+C+CC-+566 *AGAJ[QJq.//LL))-,677 *AGAJkRJq.//LL))-,677 *AGAJkRJq.//LL))qw<<1QW!2!2s17||q7H7H7HH }gajBGK(((GENNE nh];;;
AQ???AA 
+-?@PPPI &)<<	q==INQ$67Ay>:JJI!*BH!<!<26'I9:y  I  *"35G!HPQPWXXXI%:B:NX^A%6%6TU  II
 "(**]2217;;wG	%  :B:NX^A%6%6TU  II 
	?bh&&Y..qw77I)+Ii1n%	)	#%%A	&')/9
 
	 y!3447G2:7VV#++J1>PQQxa"&99\!	 gajgajgaj""		#	#;&!++K''+-Ia222Ia222( ,q,//KyUK%,?@@#i/L!!',,.. g66 A%%+$#**K001BCC
 
 ,''J!!%'..qw77Yz1%%,,QW55F!fl33FV\!_fl1or$JKK;/???r   c                  .    e Zd Z	 	 	 	 	 	 	 	 	 	 	 dddZdS )	AttentionNFr,   r   r-   r.   r/   r0   r1   r2   r3   r
   c                <    t          |||||||||	|
||||          S )N)r/   r1   r2   r3   rO   rP   rQ   rR   rS   r   rT   )rr   )selfr,   r-   r.   r/   r1   r2   r3   rO   rP   rQ   rR   rS   r   rT   s                  r   _runzAttention._run   sE    " "!-#%/"7
 
 
 	
r   NNNNNFNNNNNr,   r   r-   r   r.   r   r/   r0   r1   r0   r2   r0   r3   r0   r
   r   )__name__
__module____qualname__rw    r   r   rt   rt      sP         (,&*(,.2" 
  
  
  
  
  
  
r   rt   )r   )r   r   r   r	   r
   r   rx   ry   )
__future__r   numpyr   rL   onnx.reference.op_runr   r   r   r+   rr   rt   r}   r   r   <module>r      s    # " " " " "      ' ' ' ' ' '        . $("&$(*.
v@ v@ v@ v@ v@r!
 !
 !
 !
 !
 !
 !
 !
 !
 !
r   