
    dkj9e                    X   S SK Jr  S SKrS SKJr  S SKJrJrJrJ	r	  S SK
r
S SKJs  Jr  \	(       a  S SK
Jr  SSKJrJrJrJrJr  \R,                  \R.                  \R0                  S.rS	r\R6                  " S
5       " S S\5      5       r\R6                  " S
5       " S S\5      5       rg)    )annotationsN)Path)AnyCallableIterableTYPE_CHECKING)Tensor   )	ModelBaseMmprojModel	TextModelgguflogger)silugelureluz3{% for m in messages %}{{m['content']}}{% endfor %} Qwen3TTSForConditionalGenerationc                     ^  \ rS rSr% \R
                  R                  rSrS\	S'   S\	S'   S\	S'   SU 4S jjr
SS	 jrS
 rU 4S jr\SU 4S jj5       rSS jrSU 4S jjrSrU =r$ )Qwen3TTSTalkerModel%   )model.text_embedding.weight!text_projection.linear_fc1.weighttext_projection.linear_fc1.bias!text_projection.linear_fc2.weighttext_projection.linear_fc2.biaszdict[str, Tensor]_text_proj_bufferzTensor | None_folded_text_embed_codec_embedc                  > UR                  SS 5      nUc  [        R                  " USS9n[        US   5      nXPl        US   U l        [        U5      nUS   US'   XdS'   [        TU ]  " U/UQ7SU0UD6  0 U l        S U l	        S U l
        g )NhparamsFis_mistral_formattalker_config
vocab_sizetext_vocab_sizetext_config)popr   load_hparamsdict_talker_confign_codec_vocabsuper__init__r   r   r   )self	dir_modelargskwargsr    raw_talker_configr#   	__class__s          3/mnt/workspace/git/llama.cpp/conversion/qwen3tts.pyr-   Qwen3TTSTalkerModel.__init__5   s    **Y-?,,Y%PG !9:/.|<./&34E&Fl#!.ETE7EfE!#"&     c                   [        U R                  5       Vs/ s H	  nSU S3PM     nnU R                  R                  5        H  u  p4UR	                  S5      (       d  M  US [        S5      *  n[        U[        5      (       a
  SU S3X$'   MK  [        U[        5      (       d  Mb  UR                  5        H  u  pgSU SU S3X''   M     M     U$ s  snf )Nz<|codec_z|>_idz<|_)	ranger+   r*   itemsendswithlen
isinstanceintr)   )r.   inameskeyvalprefixsubkeysubvals           r4   _codec_token_names&Qwen3TTSTalkerModel._codec_token_namesD   s     ,11C1C+DE+Da8A3b!+DE++113HC<<&&3u:+&F#s##!&_
C&&&)iikNF&(&$<EM '2 4  Fs   Cc                N   U R                  5       n[        R                  R                  /[	        U5      -  n U R                  5       u  p4nU R                  R                  S5        U R                  R                  S5        X1 Vs/ s H  ofR                  S5      PM     sn-  nUS/[	        U5      -  -  nXR-  nU R                  R                  U5        U R                  R                  U5        U R                  R                  U5        [        R                  " U R                  [	        U5      S9nUR                  U R                  5        g s  snf ! [          a     Of = fU R#                  5       u  p5nX1-  nXR-  nU R                  R                  S5        U R                  R                  U5        U R                  R                  U5        U R                  R                  U5        [        R                  " U R                  SS9nUR                  U R                  5        g )	Nllamadefaultutf-8g        )n_vocabgpt2T)load_merges)rG   r   	TokenTypeCONTROLr=   _create_vocab_sentencepiecegguf_writeradd_tokenizer_modeladd_tokenizer_preencodeadd_token_listadd_token_scoresadd_token_typesSpecialVocabr/   add_to_ggufFileNotFoundErrorget_vocab_base)	r.   codec_tokenscodec_toktypestokensscorestoktypestspecial_vocabtokpres	            r4   	set_vocabQwen3TTSTalkerModel.set_vocabT   s   ..0..001C4EE	'+'G'G'I$FH009..y9,?,Qxx(,??Fsec,///F&H++F3--f5,,X6 --dnnc&kRM%%d&6&67 @ ! 		 $(#6#6#8 &",,V4**62''/((2))$..dK!!$"2"23s%   AE E B/E E 
E"!E"c                  > [         TU ]  5         U R                  R                  [        5        U R
                  S   U R                  -   nU R
                  S   U R                  S   -   nU R                  R                  [        US-
  U5       Vs/ s H  nX2:w  d  M
  UPM     sn5        U R                  R                  U5        U R                  R                  S5        g s  snf )Nr$   codec_eos_token_idi   F)r,   set_gguf_parametersrS   add_chat_templateDEFAULT_TEMPLATEr    r+   r*   add_suppress_tokensr:   add_eos_token_idadd_add_eos_token)r.   r$   ri   r@   r3   s       r4   rj   'Qwen3TTSTalkerModel.set_gguf_parameterss   s    #%**+;< \\,/$2D2DD
!\\,7$:M:MNb:cc,,Z$.
;.
;!& ;.
 	 	))*<=**51.
s   	C!C!c                   > Uu  p#UR                  S5      (       a  UR                  S5      (       a  g U[        S5      S  n[        TU ]  X#45      $ )Nztalker.talker.code_predictor.)
startswithr=   r,   filter_tensorsclsitemnamegenr3   s       r4   rt   "Qwen3TTSTalkerModel.filter_tensors   sM    	y))T__=U-V-VC	NO$w%tk22r6   c              #     #    U R                   b  U R                  c  g [        R                  " U R                   U R                  /SS9nU R	                  [
        R                  R                  5      U4v   g 7f)Nr   dim)r   r   torchcatformat_tensor_namer   MODEL_TENSOR
TOKEN_EMBD)r.   combineds     r4   _maybe_emit_token_embd*Qwen3TTSTalkerModel._maybe_emit_token_embd   sb     ""*d.?.?.G99d55t7H7HIqQ&&t'8'8'C'CDhOOs   A5A7c              #    >#    US:X  a  Xl         U R                  5        S h  vN   g US:X  a.  U R                  [        R                  R
                  5      U4v   g X R                  ;   a  XR                  U'   [        U R                  5      [        U R                  5      :  a  g [        U R                  S      nU R                  S   nU" [        R                  " UU R                  S   U R                  S   5      5      n[        R                  " UU R                  S   U R                  S   5      nXpl        U R                  5        S h  vN   g [        TU ]=  XU5       S h  vN   g  GNK N! N
7f)	Nzmodel.codec_embedding.weightzcodec_head.weight
hidden_actr   r   r   r   r   )r   r   r   r   r   OUTPUT_TEXT_PROJ_KEYSr   r=   _ACT2FNr    Flinearr   r,   modify_tensors)	r.   
data_torchrx   bidact_fnembedhiddenfoldedr3   s	           r4   r   "Qwen3TTSTalkerModel.modify_tensors   se    11 *22444 &&**4+<+<+C+CDjQQ'''+5""4(4))*S1E1E-FF T\\,78F**+HIEAHHU&*&<&<=`&a&*&<&<=^&_a bF XXf#556YZ#556WXZF '-#224447)*C@@@7 50 5 	As4    E5E.D+E5E1E5(E3)E51E53E5)r   r   r*   r   r+   r/   r   )returnz	list[str]rw   z tuple[str, Callable[[], Tensor]]r   z'tuple[str, Callable[[], Tensor]] | Noner   Iterable[tuple[str, Tensor]]r   r	   rx   strr   z
int | Noner   r   )__name__
__module____qualname____firstlineno__r   
MODEL_ARCHQWEN3TTS
model_archr   __annotations__r-   rG   rf   rj   classmethodrt   r   r   __static_attributes____classcell__r3   s   @r4   r   r   %   sg    ))JO )(%%! 4>2$ 3 3PA Ar6   r   c                    ^  \ rS rSr% SrSr\R                  R                  \R                  R                  \R                  R                  \R                  R                  \R                  R                  \R                  R                  \R                  R                  \R                  R                  \R                  R                   \R                  R"                  \R                  R$                  S.rSr0 rS\S'   0 rS\S'   S	rS
\S'   SU 4S jjrSS jrS rSS jrU 4S jr\SU 4S jj5       rSU 4S jjr SS jr!SS jr"Sr#U =r$$ )Qwen3TTSSpeakerEncoderModel   FT)input_layernormzself_attn.q_projzself_attn.q_normzself_attn.k_projzself_attn.k_normzself_attn.v_projzself_attn.o_projpost_attention_layernormzmlp.gate_projzmlp.up_projzmlp.down_proj   zdict[int, Tensor]_code_embed_buffer_code_head_bufferNdict[str, Any] | None_wav_config_cachec                   > UR                  SS 5      nUc  [        R                  " USS9nSUS   S   0US'   SUS   S	'   [        TU ]  " U/UQ7SU0UD6  S U l        g )
Nr    Fr!   hidden_sizer#   r&      speaker_encoder_confign_layers)r'   r   r(   r,   r-   r   )r.   r/   r0   r1   r    r3   s        r4   r-   $Qwen3TTSSpeakerEncoderModel.__init__   sy    **Y-?,,Y%PG"/1I-1X!Y 9:()*5ETE7EfE!%r6   c                8    U R                   R                  S5      $ )Nr   )global_configgetr.   s    r4   get_audio_config,Qwen3TTSSpeakerEncoderModel.get_audio_config   s    !!%%&>??r6   c                   U R                   R                  U R                  5        U R                   R                  S5        U R                   R	                  [
        R                  R                  5        U R                   R                  U R                  5        U R                   R                  S5        U R                   R                  S5        U R                   R                  S5        U R                   R                  S5        U R                   R                  S5        U R                   R                  S5        U R                   R!                  S5        U R                   R#                  [
        R                  R$                  5        U R&                  S   S   nU R                   R)                  U R                  5        U R                   R+                  US	   5        U R                   R-                  US
   5        U R                   R/                  US   5        U R                   R1                  US   5        U R                   R3                  US   5        U R                   R5                  US   5        g )NT      i   r
   h㈵>r#   code_predictor_configr   intermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_headsrms_norm_eps)rS   add_file_typeftypeadd_clip_has_audio_encoderadd_clip_audio_projector_typer   VisionProjectorTypeQWEN3TTS_SPKENCadd_audio_projection_dimn_embd_textadd_audio_num_mel_binsadd_audio_block_countadd_audio_embedding_lengthadd_audio_head_countadd_audio_feed_forward_length!add_audio_attention_layernorm_epsadd_clip_has_gen_audio_encoder!add_clip_gen_audio_projector_typeQWEN3TTS_GENr   add_gen_audio_projection_dimadd_gen_audio_embedding_length!add_gen_audio_feed_forward_lengthadd_gen_audio_block_countadd_gen_audio_head_countadd_gen_audio_head_count_kv%add_gen_audio_attention_layernorm_eps)r.   r   s     r4   rj   /Qwen3TTSSpeakerEncoderModel.set_gguf_parameters   s   &&tzz233D966t7O7O7_7_` 	11$2B2BC//4..q133D9--a066t<::4@ 	77=::4;S;S;`;`a $ 2 2? CD[ \55d6F6FG778Mm8\]::;PQd;ef223HI\3]^112GH]2^_445JK`5ab>>?TUc?der6   c                    U R                   cF  U R                  S-  S-  n[        USSS9 n[        R                  " U5      nS S S 5        WS   U l         U R                   $ ! , (       d  f       N$= f)Nspeech_tokenizerzconfig.jsonrrL   )encodingdecoder_config)r   r/   openjsonload)r.   pathfcfgs       r4   _wav_decoder_config/Qwen3TTSSpeakerEncoderModel._wav_decoder_config   sg     !!)>>$66FDdC'2aiil 3%()9%:D"%%% 32s   A  
A.c                  > UR                  S5      (       an  US;   dN  SU;   a  UR                  S5      (       d2  SU;   aF  UR                  S5      (       d  UR                  S5      (       a  [        R                  R                  $ UR                  S5      (       a&  SU;   d  SU;   a  [        R                  R                  $ [
        TU ]  XX45      $ )	N.weight)za.gen.wav.pre_conv.weightza.gen.wav.dac.entry.weightza.gen.wav.dac.post_conv.weightz.up.blk.z.dwconv.weightz	.dac.blk.z.conv1.weightz.conv2.weightz.conv.weight)r<   r   GGMLQuantizationTypeF16F32r,   tensor_force_quant)r.   rx   new_namer   n_dimsr3   s        r4   r   .Qwen3TTSSpeakerEncoderModel.tensor_force_quant  s    
 Y''uuh&8+<+<=M+N+Nx'X->->-O-OS[SdSdetSuSu,,000 ^,,*2HK[cLc,,000w)$#FFr6   c                   > Uu  p#UR                  S5      (       d  UR                  S5      (       d  US:X  d  g [        TU ]	  X#45      $ )Nzspeaker_encoder.rr   #talker.model.codec_embedding.weight)rs   r,   rt   ru   s       r4   rt   *Qwen3TTSSpeakerEncoderModel.filter_tensors  sJ    	 OO.//788<<w%tk22r6   c              #     >#    UR                  S5      (       a  X!4v   g US:X  a.  U R                  [        R                  R                  5      U4v   g US:X  a.  U R                  [        R                  R
                  5      U4v   g UR                  S5      (       aE  SUR                  SS5      S   -   nU R                  [        R                  R                  US9U4v   g UR                  S5      (       a  [        UR                  S	5      S   R                  S5      S
   5      nXR                  U'   [        U R                  5      U R                  :  a  g [        R                  " [        U R                  5       Vs/ s H  o`R                  R!                  U5      PM     snS
S9nU R                  [        R                  R"                  5      U4v   g UR                  S5      (       a  [        UR                  S5      S   R                  S5      S
   5      nXR$                  U'   [        U R$                  5      U R                  :  a  g [        R                  " [        U R                  5       Vs/ s H  o`R$                  R!                  U5      PM     snS
S9nU R                  [        R                  R&                  5      U4v   g UR                  S5      (       aq  UR                  S5      S   nUR                  SS5      u  pU
R                  SS5      S
   nU R(                  R+                  U5      nUb  U R                  X5      U4v   g SU;   a  Uc   e[        UR                  S5      S   R                  S5      S
   5      nSUR                  SS5      S   -   n[        R,                  [        R                  R.                     R1                  X=S9U-   nX4v   g [2        TU ]i  XU5       S h  vN   g s  snf s  snf  N7f)Nz
a.gen.wav.r   z'talker.code_predictor.model.norm.weightz.talker.code_predictor.small_to_mtp_projection..r
   suffixz,talker.code_predictor.model.codec_embedding.zcodec_embedding.r   r|   ztalker.code_predictor.lm_head.zlm_head.z#talker.code_predictor.model.layers.zmodel.layers.zres2net_block.blocks.r   xid)rs   r   r   r   A_GEN_CODE_OUT_EMBDA_GEN_CODE_OUTPUT_NORMrsplitA_GEN_CODE_PROJ_INr?   splitr   r=   _CODE_GEN_N_CODEBOOKSr~   stackr:   r'   A_GEN_CODE_EMBDr   A_GEN_CODE_HEAD_CODE_LAYER_TENSOR_MAPr   TENSOR_NAMESA_ENC_CONV_RES2formatr,   r   )r.   r   rx   r   r   idxr@   stackedrestr9   key_with_suffixrB   tensorr   r   r3   s                  r4   r   *Qwen3TTSSpeakerEncoderModel.modify_tensors&  s    
 ??<(($$ 88**4+<+<+P+PQS]^^<<**4+<+<+S+STV`aa??KLL4;;sA.q11F**4+<+<+O+OX^*_akll??IJJdjj!34Q7==cB1EFC+5##C(4**+d.H.HHkk5QUQkQkKl"mKla#:#:#>#>q#AKl"mstuG**4+<+<+L+LMwWW??;<<djj,Q/55c:1=>C*4""3'4))*T-G-GGkk%PTPjPjJk"lJkQ#9#9#=#=a#@Jk"lrstG**4+<+<+L+LMwWW??@AA::o.q1D!%C!3A!((a03C0044S9F!..v;ZHH"d*?"?djj!89!<BB3GJKC4;;sA.q11F(():):)J)JKRRWZRdgmmH((7)*C@@@= #n #m* 	As,   E>P$P%CP7$PE!P<P=Pc              #  @   #    U R                  5        S h  vN   g  N7fN)_generate_code2wav_tensorsr   s    r4   generate_extra_tensors2Qwen3TTSSpeakerEncoderModel.generate_extra_tensorsc  s     22444s   c              #    ^^#    SSK Jn  U R                  5       nU" U R                  S-  S-  5      mS<U4S jjmS=S jnS>U4S jjn[        R
                  nU R                  UR                  5      T" S5      R                  S	5      4v   U R                  UR                  5      T" S
5      R                  S	5      4v   U R                  UR                  5      U" SS5      4v   U R                  UR                  5      T" S5      R                  S	5      4v   U R                  UR                  5      T" S5      R                  S	5      4v   U R                  UR                  5      U" SU R                  5      4v   U R                  UR                  SS9T" S5      4v   U R                  UR                  SS9T" S5      4v   U R                  UR                   SS9T" S5      4v   U R                  UR                   SS9T" S5      4v   U R                  UR"                  SS9T" S5      4v   U R                  UR"                  SS9T" S5      4v   U R                  UR$                  5      T" S5      4v   UR&                  UR(                  UR*                  UR,                  UR.                  UR0                  UR2                  UR4                  UR6                  UR8                  UR:                  S.n[=        US   5       H=  nUR?                  5        H&  u  pU R                  X5      T" SU SU 35      4v   M(     M?     UR@                  S4UR@                  S4URB                  S4URB                  S4URD                  S4URD                  S4URF                  S4URF                  S4URH                  S4URH                  S4URJ                  S4S.n
[=        [M        US    5      5       H?  nU
R?                  5        H(  u  nu  pU R                  XUS9T" S!U SU 35      4v   M*     MA     U R                  URN                  SS9T" S"5      4v   U R                  URN                  SS9T" S#5      4v   [M        US$   5      n[=        U5       GHE  nUS-   nU" T" S%U S&35      T" S%U S'35      5      u  pU R                  URP                  US(S9U4v   U R                  URP                  US)S9U4v   U R                  URR                  USS9T" S%U S*35      4v   U R                  URR                  USS9T" S%U S+35      4v   [=        S,5       GH}  nUS--   nU" T" S%U S.U S/35      T" S%U S.U S035      5      u  nn[        RT                  URV                     RY                  UUS19nUS(-   U4v   US)-   U4v   [        RT                  URZ                     RY                  UUS19nUS-   T" S%U S.U S235      4v   US-   T" S%U S.U S335      4v   U" T" S%U S.U S435      T" S%U S.U S535      5      u  nn[        RT                  UR\                     RY                  UUS19nUS(-   U4v   US)-   U4v   [        RT                  UR^                     RY                  UUS19nUS-   T" S%U S.U S635      4v   US-   T" S%U S.U S735      4v   GM     GMH     U" T" S85      T" S95      5      u  nnU R                  UR`                  S(S9U4v   U R                  UR`                  S)S9U4v   U R                  URb                  SS9T" S:5      4v   U R                  URb                  SS9T" S;5      4v   g 7f)?Nr   )	load_filer   zmodel.safetensorsc                   > TU    $ r   )rx   
state_dicts    r4   r   CQwen3TTSSpeakerEncoderModel._generate_code2wav_tensors.<locals>.getn  s    d##r6   c                f    [         R                  " U 5      S[         R                  " U5      S-   -  4$ )Ng      ?g&.>)r~   exp)alphabetas     r4   
snake_foldJQwen3TTSSpeakerEncoderModel._generate_code2wav_tensors.<locals>.snake_foldq  s*     99U#SEIIdOd,B%CCCr6   c                  > / n[        U5       HP  nT" U  SU S35      nT" U  SU S35      nUR                  XER                  S5      R                  S5      -  5        MR     US:  a  [        R
                  " USS9$ US   $ )	Nz.vq.layers.z._codebook.embedding_sumz._codebook.cluster_usager   r
   r   r|   )r:   append	clamp_min	unsqueezer~   r  )rD   r   booksr@   embedding_sumcluster_usager   s         r4   rvq_codebookLQwen3TTSSpeakerEncoderModel._generate_code2wav_tensors.<locals>.rvq_codebookv  s     E8_ #vhk!<T$U V #vhk!<T$U V]-D-DT-J-T-TUW-XXY % 1915;;u!,J%(Jr6   z-decoder.quantizer.rvq_first.input_proj.weightr%  z.decoder.quantizer.rvq_first.output_proj.weightzdecoder.quantizer.rvq_firstr
   z,decoder.quantizer.rvq_rest.input_proj.weightz-decoder.quantizer.rvq_rest.output_proj.weightzdecoder.quantizer.rvq_restr   r   zdecoder.pre_conv.conv.weightz.biaszdecoder.pre_conv.conv.biasz)decoder.pre_transformer.input_proj.weightz'decoder.pre_transformer.input_proj.biasz*decoder.pre_transformer.output_proj.weightz(decoder.pre_transformer.output_proj.biasz#decoder.pre_transformer.norm.weight)zinput_layernorm.weightzself_attn.q_proj.weightzself_attn.k_proj.weightzself_attn.v_proj.weightzself_attn.o_proj.weightzself_attn_layer_scale.scalezpost_attention_layernorm.weightzmlp.gate_proj.weightzmlp.up_proj.weightzmlp.down_proj.weightzmlp_layer_scale.scaler   zdecoder.pre_transformer.layers.r    )z0.conv.weightz0.conv.biasz1.dwconv.conv.weightz1.dwconv.conv.biasz1.norm.weightz1.norm.biasz1.pwconv1.weightz1.pwconv1.biasz1.pwconv2.weightz1.pwconv2.biasz1.gammaupsampling_ratioszdecoder.upsample.zdecoder.decoder.0.conv.weightzdecoder.decoder.0.conv.biasupsample_rateszdecoder.decoder.z.block.0.alphaz.block.0.betaz.alphaz.betaz.block.1.conv.weightz.block.1.conv.biasr      z.block.z.act1.alphaz
.act1.betar   z.conv1.conv.weightz.conv1.conv.biasz.act2.alphaz
.act2.betaz.conv2.conv.weightz.conv2.conv.biaszdecoder.decoder.5.alphazdecoder.decoder.5.betazdecoder.decoder.6.conv.weightzdecoder.decoder.6.conv.bias)rx   r   r   r	   )r   r	   r!  r	   r   ztuple[Tensor, Tensor])rD   r   r   r?   r   r	   )2safetensors.torchr  r   r/   r   r   r   A_GEN_WAV_QUANT_FIRST_INsqueezeA_GEN_WAV_QUANT_FIRST_OUTA_GEN_WAV_QUANT_FIRST_CBA_GEN_WAV_QUANT_REST_INA_GEN_WAV_QUANT_REST_OUTA_GEN_WAV_QUANT_REST_CBr  A_GEN_WAV_PRE_CONVA_GEN_WAV_TFM_IN_PROJA_GEN_WAV_TFM_OUT_PROJA_GEN_WAV_TFM_OUTPUT_NORMA_GEN_WAV_TFM_ATTN_NORMA_GEN_WAV_TFM_ATTN_QA_GEN_WAV_TFM_ATTN_KA_GEN_WAV_TFM_ATTN_VA_GEN_WAV_TFM_ATTN_OUTA_GEN_WAV_TFM_ATTN_SCALEA_GEN_WAV_TFM_FFN_NORMA_GEN_WAV_TFM_FFN_GATEA_GEN_WAV_TFM_FFN_UPA_GEN_WAV_TFM_FFN_DOWNA_GEN_WAV_TFM_FFN_SCALEr:   r;   A_GEN_WAV_UP_CONVA_GEN_WAV_UP_DWCONVA_GEN_WAV_UP_NORMA_GEN_WAV_UP_PW1A_GEN_WAV_UP_PW2A_GEN_WAV_UP_GAMMAr=   A_GEN_WAV_DAC_ENTRYA_GEN_WAV_DAC_UP_SNAKEA_GEN_WAV_DAC_UP_CONVr
  A_GEN_WAV_DAC_RES_ACT1r  A_GEN_WAV_DAC_RES_CONV1A_GEN_WAV_DAC_RES_ACT2A_GEN_WAV_DAC_RES_CONV2A_GEN_WAV_DAC_POST_SNAKEA_GEN_WAV_DAC_POST_CONV)r.   r  
wav_configr"  r,  Ttfm_layer_mapr   rB   	tensor_idup_mapr   n_dac_blockspyabr   ridxa1b1name1
name_conv1a2b2name2
name_conv2a5b5r   r  s                               @@r4   r  6Qwen3TTSSpeakerEncoderModel._generate_code2wav_tensorsf  sG	     	0--/
t~~0BBEXXY
	$	D
	K  &&q'A'ABCHwDx  EA  EA  BD  EE  F  	F&&q'B'BCSIyEz  FC  FC  DF  FG  H  	H&&q'A'ABLQnpqDrss&&q'@'@A3GuCvC~C~  @B  DC  D  	D&&q'A'ABCHwDx  EA  EA  BD  EE  F  	F&&q'@'@A<Plnr  oI  oI  DJ  K  	K &&q';';I&NPSTrPstt&&q';';G&LcRnNopp &&q'>'>y&QSV  XC  TD  E  	E&&q'>'>w&OQTU~Q  A  	A&&q'?'?	&RTW  YE  UF  G  	G&&q'?'?&PRU  WA  SB  C  	C&&q'B'BCSInEopp /0.G.G./.D.D./.D.D./.D.D./.F.F./.H.H/0/G/G./.F.F./.D.D./.F.F./.G.G
 $789C"/"5"5"7..y>FefiejjklokpDq@rss #8 : #$"5"5y!A"#"5"5w!?%&%:%:I$F%&%:%:G$D"#"5"5y!A"#"5"5w!?"#"4"4i!@"#"4"4g!>"#"4"4i!@"#"4"4g!>"#"6"6!;
 Z(;<=>C,2LLN((i..yf.MsUfgjfkklmplqSrOstt -; ?
 &&q'<'<Y&OQTUtQuvv&&q'<'<W&MsSpOqrr:&678&CqBc$4RD"GH#P`ac`ddqNrJstDA**1+C+CSQY*Z\]^^**1+C+CSQX*Y[\]]**1+B+BCPY*Z\_brsurv  wK  aL  ]M  N  N**1+B+BCPW*XZ]`pqspt  uG  _H  [I  J  JQxQw#C*:2$gdV;(W$XZ]`pqsptt{  }A  |B  BL  _M  [N  OB))!*B*BCJJsX[J\x',,w++!..q/H/HIPPUX^aPb
!I-s5EbTQUPVVh3i/jkk!G+S3CB4wtfTd1e-fgg#C*:2$gdV;(W$XZ]`pqsptt{  }A  |B  BL  _M  [N  OB))!*B*BCJJsX[J\x',,w++!..q/H/HIPPUX^aPb
!I-s5EbTQUPVVh3i/jkk!G+S3CB4wtfTd1e-fgg'   '< C 9:C@X<YZB&&q'A'A(&SUWXX&&q'A'A'&RTVWW&&q'@'@&SUXYxUyzz&&q'@'@&QSVWtSuvvs   ]6]:)r   r   )r   r   )r   zdict[str, Any]r   r   r   )%r   r   r   r   has_vision_encoderhas_audio_encoderr   r   A_GEN_CODE_ATTN_NORMA_GEN_CODE_ATTN_QA_GEN_CODE_ATTN_Q_NORMA_GEN_CODE_ATTN_KA_GEN_CODE_ATTN_K_NORMA_GEN_CODE_ATTN_VA_GEN_CODE_ATTN_OUTA_GEN_CODE_FFN_NORMA_GEN_CODE_FFN_GATEA_GEN_CODE_FFN_UPA_GEN_CODE_FFN_DOWNr	  r  r   r   r   r   r-   r   rj   r   r   r   rt   r   r  r  r   r   r   s   @r4   r   r      s4   
  ,,AA --?? --DD --?? --DD --?? --AA$($5$5$I$I**>>((::**>> ,.).+-(-/3,3	&@f<&G" 
3 
3;Az5yw ywr6   r   )
__future__r   r   pathlibr   typingr   r   r   r   r~   torch.nn.functionalnn
functionalr   r	   baser   r   r   r   r   r   r   r   r   rl   registerr   r   r  r6   r4   <module>r     s    "   9 9    A A FFFFFF M  67NA) NA 8NAb 67gw+ gw 8gwr6   