
    W	j?                       S SK Jr  S SKrS SKJrJrJrJr  \(       a  S SKJ	r	  SSK
JrJrJrJr  SSKJrJr  SSKJr  \R(                  " S	S
SS5       " S S\5      5       r\R(                  " S5       " S S\\5      5       r\R(                  " S5       " S S\5      5       r\R(                  " SSS5       " S S\5      5       r\R(                  " S	5       " S S\5      5       r\R(                  " S
5       " S S\5      5       r\R(                  " S5       " S S\5      5       r\R(                  " S5       " S  S!\5      5       rg)"    )annotationsN)AnyCallableIterableTYPE_CHECKING)Tensor   )MmprojModel	ModelBasegguflogger)
Qwen3ModelQwen3MoeModel)Qwen25AudioModelQwen3VLForConditionalGeneration"Qwen3VLMoeForConditionalGenerationQwen3_5ForConditionalGeneration"Qwen3_5MoeForConditionalGenerationc                  ^   ^  \ rS rSrU 4S jrU 4S jr\SU 4S jj5       rSU 4S jjrSr	U =r
$ )	Qwen3VLVisionModel   c                  > [         TU ]  " U0 UD6  U R                  c  [        R                  " S5        g SU R                  ;  aW  U R                  R                  SS5      nU R                  R                  SS5      n[        US-  U-  5      nXPR                  S'   U R                  R                  S5      U R                  S	'   U R                  R                  S
5      U R                  S'   S/[        U R                  S   =(       d    S5      -  U l        U R                  R                  S/ 5       H  nSU R                  U'   M     g )Nz9No vision config found, skipping vision tensor processing
image_sizenum_position_embeddingsi 	  
patch_sizer   g      ?	num_headsnum_attention_headsdepthnum_hidden_layersFr   deepstack_visual_indexesT)super__init__hparams_visionr   infogetintis_deepstack_layers)selfargskwargsnum_posr   r   idx	__class__s          ./mnt/workspace/llama.cpp/conversion/qwen3vl.pyr"   Qwen3VLVisionModel.__init__   s0   $)&)&KKST t222))--.GNG,,00rBJ Wc\J67J0:- 6:5H5H5L5L[5Y12373F3F3J3J73S/0$)7S1D1DEX1Y1^]^-_#_ &&**+ErJC,0D$$S) K    c                z  > [         TU ]  5         U R                  (       d3  U R                  R	                  [
        R                  R                  5        U R                  R                  S5        U R                  bB  U R                  R                  S5      nUb$  U R                  R                  [        U5      5        U R                  R                  S0 5      R                  SS5      nU R                  R                  U5        U R                  (       a&  U R                  R!                  U R                  5        g g )NTspatial_merge_sizetext_configrms_norm_epsgư>)r!   set_gguf_parametershas_audio_encodergguf_writeradd_clip_projector_typer   VisionProjectorTypeQWEN3VLadd_vision_use_gelur#   r%   add_vision_spatial_merge_sizer&   global_config"add_vision_attention_layernorm_epsr'   add_vision_is_deepstack_layers)r(   
merge_sizer4   r-   s      r.   r5   &Qwen3VLVisionModel.set_gguf_parameters*   s    #%%%44T5M5M5U5UV,,T2*,,001EFJ%  >>s:O ))--mR@DD^UYZ;;LI##;;D<T<TU $r0   c                  > Uu  p#UR                  S5      (       a  g UR                  S5      (       a  g UR                  S5      (       a  UR                  SSS5      nUR                  S5      (       d  g [        TU ]  X#45      $ )Nlm_head.mtp.model.visual.visual.r	   )
startswithreplacer!   filter_tensorsclsitemnamegenr-   s       r.   rI   !Qwen3VLVisionModel.filter_tensors=   sw    	 ??:&& ??6""???++<<A>Dy))w%tk22r0   c              #  J  >#    U R                   c   eUR                  S5      (       GaK  UR                  SSS9SS  u  pEU R                   R                  S/ 5      [	        U5         nUnUR                  S5      (       a0  [
        R                  R                  nUR                  SS5      S   n	OUR                  S	5      (       a0  [
        R                  R                  nUR                  SS5      S   n	OTUR                  S
5      (       a0  [
        R                  R                  nUR                  SS5      S   n	O[        SU 35      eU R                  XSU	 3S9n
[        TU ]5  XU5       S h  vN   g UR                  S5      (       Ga  UR                  SS5      S   n	U	R                  S5      (       a  U	R                  SS5      u  p[	        UR                  SS5      5      nUS:X  a  SnOUS:X  a  SnO[        SU SU 35      eU R                  [
        R                  R                  USU 3S9n
OcU	R                  S5      (       a?  U R                  [
        R                  R                   SU	R                  SS5      S    3S9n
O[        SU 35      eX4v   g US:X  a  UR"                  u  nnn  nAAUS:w  a  [        S5      e[
        R$                  [
        R                  R&                     S-   US S 2S S 2SS4   4v   [
        R$                  [
        R                  R&                     S-   US S 2S S 2SS4   4v   g US:X  a3  [
        R$                  [
        R                  R&                     S-   U4v   g [(        R                  " XX#5       S h  vN   g  GN/ N7f)Nzvisual.deepstack_merger_list..   )maxsplit   r    znorm.r	   zlinear_fc1.zlinear_fc2.zUnexpected deepstack tensor: )suffixvisual.merger.	linear_fc r   zunexpected fc index z in zUnexpected merger tensor: zvisual.patch_embed.proj.weightz=Current implementation only supports temporal_patch_size of 2.weight.z	.weight.1zvisual.patch_embed.proj.bias.bias)r#   rG   splitr%   r&   r   MODEL_TENSOR	V_DS_NORMV_DS_FC1V_DS_FC2
ValueErrorformat_tensor_namer!   modify_tensorsrH   V_MMPROJV_POST_NORMshapeTENSOR_NAMESV_ENC_EMBD_PATCHr
   )r(   
data_torchrM   bidprefixrestr,   targettensor_typerU   new_name
fc_idx_strtailfc_numfc_idxc1c2kt_r-   s                      r.   rb   !Qwen3VLVisionModel.modify_tensorsQ   s    ""...??:;;::cA:6qr:LF%%))*DbI#f+VCF   ))"//99c1-a0""=11"//88c1-a0""=11"//88c1-a0 #@!GHH..{!F8.UHw-jCHHH??+,,ZZQ'*F  --#)<<Q#7 
Z//R@A Q;Fq[F$';F84v%NOO2243D3D3M3Mv`abfag^h2i""7++2243D3D3P3P[\]c]i]ijmop]qrs]t\uYv2w #=dV!DEE((33)//BB1BQw !`aa$$T%6%6%G%GH9TV`abdeghjmamVnoo$$T%6%6%G%GH;VXbcdfgijlocoXpqq11$$T%6%6%G%GH7RT^__--dJJJO IN 	Ks%   E,N#/N0H(N#N!N#!N#)r'   rL   z tuple[str, Callable[[], Tensor]]returnz'tuple[str, Callable[[], Tensor]] | Nonerh   r   rM   strri   z
int | Nonery   zIterable[tuple[str, Tensor]])__name__
__module____qualname____firstlineno__r"   r5   classmethodrI   rb   __static_attributes____classcell__r-   s   @r.   r   r      s.    10V& 3 3&>K >Kr0   r   $Qwen3OmniMoeForConditionalGenerationc                  T    \ rS rSrSrSrS
S jrS
S jrS r\	SS j5       r
SS jrSrg	)Qwen3OmniMmprojModel   Tc                b    U R                   (       a  U R                  S   R                  S5      $ g )Nthinker_configvision_config)has_vision_encoderr=   r%   r(   s    r.   get_vision_config&Qwen3OmniMmprojModel.get_vision_config   s+    ""%%&67;;OLLr0   c                b    U R                   (       a  U R                  S   R                  S5      $ g )Nr   audio_config)r6   r=   r%   r   s    r.   get_audio_config%Qwen3OmniMmprojModel.get_audio_config   s+    !!%%&67;;NKKr0   c                l   U R                   (       aH  [        R                  U 5        U R                  R	                  [
        R                  R                  5        U R                  (       aJ  [        R                  " U 5        U R                  R                  [
        R                  R                  5        g g )N)r   r   r5   r7   add_clip_vision_projector_typer   r9   r:   r6   r   add_clip_audio_projector_typeQWEN3Ar   s    r.   r5   (Qwen3OmniMmprojModel.set_gguf_parameters   su    ""2248;;D<T<T<\<\]!!006::4;S;S;Z;Z[ "r0   c                R   Uu  p#UR                  S5      (       a  g UR                  S5      (       a  g UR                  S5      (       a  UR                  SSS5      nUR                  S5      (       a  UR                  SSS5      nSU;  a  SU;  a  g [        R                  " X#45      $ )NrC   rD   rE   rF   r	   zthinker.audio_tower.audio_tower.)rG   rH   r
   rI   )rK   rL   rM   rN   s       r.   rI   #Qwen3OmniMmprojModel.filter_tensors   s    	 ??:&& ??6""???++<<A>D??122<< 6JDD ^4%?))4+66r0   c              #    #    SU;   a  U R                   (       d  [        SU 35      eUR                  SS5      nSU;   aI  UR                  SS5      nUR                  SS5      nUR                  S	S
5      nUR                  SS5      nO<SU;   a6  UR                  SS5      nUR                  S	S
5      nUR                  SS5      n[        R	                  XX#5       S h  vN   g SU;   a{  U R
                  (       d  [        SU 35      eSU;   a6  UR                  S5      (       a   UR                  S5      R                  S5      n[        R                  " XX#5       S h  vN   g g  N N7f)NrF   z5Model does not have vision encoder, but found tensor zthinker.visual.rE   z.merger_list.z.deepstack_merger_list.z.ln_qz.normz.mlp.0z.linear_fc1z.mlp.2z.linear_fc2z.merger.r   z4Model does not have audio encoder, but found tensor conv2drZ   )	r   r`   rH   r   rb   r6   endswith	unsqueezer   )r(   rh   rM   ri   s       r.   rb   #Qwen3OmniMmprojModel.modify_tensors   sQ    ** #XY]X^!_``<< 1?CD$&||O5NO||GW5||Hm<||Hm<t#||GW5||Hm<||Hm<)884UUUt#)) #WX\W]!^__4DMM'$:$:'11"5??C
'66tSSS $ V Ts%   CE*E&B E*E( E*(E* N)ry   zdict[str, Any] | Nonerx   rz   )r|   r}   r~   r   r6   r   r   r   r5   r   rI   rb   r   r   r0   r.   r   r      s8    \ 7 7,Tr0   r    Qwen3ASRForConditionalGenerationc                      \ rS rSrSrSrSrg)Qwen3ASRMmprojModel   TFr   N)r|   r}   r~   r   r6   r   r   r   r0   r.   r   r      s    r0   r   Glm4vForConditionalGeneration Glm4vMoeForConditionalGenerationGlmOcrForConditionalGenerationc                  2   ^  \ rS rSrS rSU 4S jjrSrU =r$ )Glm4VVisionModel   c                   [         R                  " U 5        U R                  c   eU R                  R	                  [
        R                  R                  5        [        U R                  R                  SS5      5      R                  5       nUS:X  a  U R                  R                  S5        O!US:X  a  U R                  R                  S5        U R                  R                  SS5      nU R                  R                  U5        g )N
hidden_actrX   geluTsilur4   gh㈵>)r
   r5   r#   r7   r8   r   r9   GLM4Vr{   r%   lowerr;   add_vision_use_silur>   )r(   r   r4   s      r.   r5   $Glm4VVisionModel.set_gguf_parameters   s    ''-""...001I1I1O1OP,,00rBCIIK
0066!006**..~tD;;LIr0   c              #     >#    UR                  S5      (       a   [        R                  " XX#5       S h  vN   g [        TU ]	  XU5       S h  vN   g  N N7f)NrV   )rG   r   rb   r!   )r(   rh   rM   ri   r-   s       r.   rb   Glm4VVisionModel.modify_tensors   sM     ??+,, //$LLL7)*C@@@ M@s!   1AAAAAAr   rz   )r|   r}   r~   r   r5   rb   r   r   r   s   @r.   r   r      s    JA Ar0   r   c                  n   ^  \ rS rSr\R
                  R                  rU 4S jr\	SU 4S jj5       r
SrU =r$ )Qwen3VLTextModel   c                &  > [         TU ]  5         SU R                  ;   a   U R                  S   R                  S0 5      nOU R                  R                  S0 5      n[	        UR                  S/ 5      5      nU R
                  R                  U5        g )Nr   r   r    r!   r5   hparamsr%   lenr7   add_num_deepstack_layersr(   r   deepstack_layer_numr-   s      r.   r5   $Qwen3VLTextModel.set_gguf_parameters   s|    #%t||+ LL)9:>>PRSM LL,,_bAM!-"3"34NPR"ST112EFr0   c                P   > Uu  p#UR                  SS5      n[        TU ]	  X#45      $ Nzthinker.rX   rH   r!   rI   rJ   s       r.   rI   Qwen3VLTextModel.filter_tensors  ,    	||J+w%tk22r0   r   rx   )r|   r}   r~   r   r   
MODEL_ARCHr:   
model_archr5   r   rI   r   r   r   s   @r.   r   r      s,    ((JG 3 3r0   r   c                  ~   ^  \ rS rSr\R
                  R                  rU 4S jr\	SU 4S jj5       r
SU 4S jjrSrU =r$ )Qwen3VLMoeTextModeli  c                   > [         TU ]  5         U R                  R                  S0 5      n[	        UR                  S/ 5      5      nU R
                  R                  U5        g )Nr   r    r   r   s      r.   r5   'Qwen3VLMoeTextModel.set_gguf_parameters  sS    #%(("=!-"3"34NPR"ST112EFr0   c                P   > Uu  p#UR                  SS5      n[        TU ]	  X#45      $ r   r   rJ   s       r.   rI   "Qwen3VLMoeTextModel.filter_tensors  r   r0   c              #  4  >#    UR                  S5      (       d  UR                  S5      (       a^  UR                  S5      (       d  U S3OUnUR                  SSS5      R                  5       n[        R                  " XXC5       S h  vN   g UR                  S5      (       d  UR                  S5      (       Ga;  UR
                  S	:  d  UR                  S
   S-  S:w  a$  [        SU S[        UR                  5       35      eUR                  S
   S-  nUSS U24   R                  5       nUSUS 24   R                  5       nUR                  S5      n	U	R                  SS5      S   n
U
 S3nU
 S3nUR                  SSS5      R                  5       nUR                  SSS5      R                  5       n[        R                  " XX5       S h  vN   [        R                  " XX5       S h  vN   g [        TU ]  XU5       S h  vN   g  GN N@ N# N7f)Nzmlp.experts.down_projzmlp.experts.down_proj.weightrY   r   rT   r	   zmlp.experts.gate_up_projzmlp.experts.gate_up_proj.weightrR   r   z"Unexpected gate_up_proj shape for z: .rQ   z.gate_proj.weightz.up_proj.weight)r   permute
contiguousr   rb   ndimre   r`   tupleremovesuffixrsplitr!   )r(   rh   rM   ri   mappedpermuted	split_dimgateup	base_namebasemapped_gate	mapped_up	perm_gateperm_upr-   s                  r.   rb   "Qwen3VLMoeTextModel.modify_tensors   s    ==011T]]Ca5b5b-1]]9-E-EvW%4F!))!Q2==?H //LLL==344Fg8h8h"j&6&6r&:Q&>!&C #EdV2eT^TdTdNeMf!ghh"((,1Ic:I:o.99;DCO,779B
 )))4I##C+A.D!F"34K&0IQ1-88:IjjAq)446G //RRR //yNNN7)*C@@@/ M& SN 	AsI   BHH	EHHH0H1H	H
HHHHr   rx   rz   )r|   r}   r~   r   r   r   
QWEN3VLMOEr   r5   r   rI   rb   r   r   r   s   @r.   r   r     s8    ++JG 3 3A Ar0   r   c                  `   ^  \ rS rSr\R
                  R                  rU 4S jrU 4S jr	Sr
U =r$ )Qwen3OmniMoeTextModeli?  c                  > [         TU ]  5         [        U R                  S-  SSS9 n[        R
                  " U5      nUR                  S0 5      nUR                  5        Hd  u  pEUR                  S5      S:X  d  M  U R                  R                  [        U5      5        U R                  R                  [        U5      5          O   S S S 5        g ! , (       d  f       g = f)Ntokenizer_config.jsonrutf-8encodingadded_tokens_decodercontent
<|im_end|>)r!   	set_vocabopen	dir_modeljsonloadr%   itemsr7   add_bos_token_idr&   add_eos_token_idr(   ftokenizer_configadded_tokenstoken_iddatar-   s         r.   r   Qwen3OmniMoeTextModel.set_vocabC  s    $..#::C'RVW#yy|+//0FKL"."4"4"688I&,6$$55c(mD$$55c(mD	 #7 SRRs   AC=AC
C c                X   > [         TU ]  5         U R                  R                  S5        g Nr   r!   r5   r7   r   r(   r-   s    r.   r5   )Qwen3OmniMoeTextModel.set_gguf_parametersO  "    #%11!4r0   r   )r|   r}   r~   r   r   r   r   r   r   r5   r   r   r   s   @r.   r   r   ?  s!    ++J
5 5r0   r   c                  `   ^  \ rS rSr\R
                  R                  rU 4S jrU 4S jr	Sr
U =r$ )Qwen3ASRTextModeliT  c                X   > [         TU ]  5         U R                  R                  S5        g r   r   r   s    r.   r5   %Qwen3ASRTextModel.set_gguf_parametersX  r  r0   c                  > [         TU ]  5         U R                  R                  S5        [	        U R
                  S-  SSS9 n[        R                  " U5      nUR                  S0 5      nUR                  5        Hd  u  pEUR                  S5      S:X  d  M  U R                  R                  [        U5      5        U R                  R                  [        U5      5          O   S S S 5        g ! , (       d  f       g = f)	Nz{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}r   r   r   r   r   r   r   )r!   r   r7   add_chat_templater   r   r   r   r%   r   r   r&   r   r   s         r.   r   Qwen3ASRTextModel.set_vocab\  s    **  ,x  	y$..#::C'RVW#yy|+//0FKL"."4"4"688I&,6$$55c(mD$$55c(mD	 #7 SRRs   AC-AC--
C;r   )r|   r}   r~   r   r   r   r:   r   r5   r   r   r   r   s   @r.   r  r  T  s!    ((J5 r0   r  )
__future__r   r   typingr   r   r   r   torchr   r   r
   r   r   r   qwenr   r   qwenvlr   registerr   r   r   r   r   r   r   r  r   r0   r.   <module>r     s   "  9 9 6 6 + $ 57[]~  Ae  f~K ~K f~KB :;ET-/? ET <ETP 67.  8
 35WYyzA) A {A, 563z 3 73* 89-A- -A :-A` :;5/ 5 <5( 67(  8r0   