
     iF?                        S SK r S SKrS SKrS SKrS SKrS SKr\R                  R                  \R                  R                  \
5      5      r\R                  R                  \SS5      r\R                  R                  \S5      rS rS rS rS rS	 rS
 rSSS.rSSSSSSSSSSSS.rS rS rS rS rS rS r\S :X  a  \" 5         gg)!    Ncheckpoints	OmniVoicemodelsc                 F    [        SU < SU< 3[        R                  SS9  g )N[z] T)fileflush)printsysstderr)tagmsgs     '/mnt/workspace/omnivoice.cpp/convert.pylogr      s    	sC
 szz>    c                    [        U S5       n[        R                  " SUR                  S5      5      S   n[        R
                  " UR                  U5      5      nS S S 5        WR                  SS 5        USW-   4$ ! , (       d  f       N'= f)Nrbz<Q   r   __metadata__)openstructunpackreadjsonloadspop)pathfnmetas       r   read_sf_headerr!      sm    	dD	QMM$q	*1-zz!&&)$ 
 	HH^T"Q;	 
	s   AA==
Bc                 :   US   u  p4XC-
  n[        U S5       nUR                  X-   5        UR                  U5      nS S S 5        US   nUS   n	US:X  a@  [        R                  " W[        R
                  S9R                  U	5      R                  5       $ US:X  a  [        R                  " W[        R                  S9R                  U	5      n
U
R                  [        R                  5      S-  R                  [        R
                  5      R                  5       $ US	:X  a]  [        R                  " W[        R                  S9R                  U	5      R                  [        R
                  5      R                  5       $ [        S
U-  5      e! , (       d  f       GNU= f)Ndata_offsetsr   dtypeshapeF32r$   BF16   F16zunsupported dtype %s)r   seekr   np
frombufferfloat32reshapecopyuint16astypeuint32viewfloat16RuntimeError)sf_pathhdr_sizeinfooff0off1nbytesr   raw	dtype_strr%   u16s              r   read_sf_tensorr@   #   s7   n%JD[F	gt		xffVn 
 WIMEE}}S

3;;EBGGIIFmmCryy199%@

299%+11"**=BBDDE}}S

3;;EBII"**UZZ\\
-	9
:: 
	s   %F
Fc                    US   nUS   nUS   u  pxX-
  n	[        US5       n
U
R                  X7-   5        U
R                  U	5      nS S S 5        US:X  aE  [        R                  " W[        R
                  S9R                  U5      nU R                  X5        U	$ US:X  a\  [        R                  " W[        R                  S9R                  U5      nU R                  X[        R                  R                  S9  U	$ US	:X  aE  [        R                  " W[        R                  S9R                  U5      nU R                  X5        U	$ [        S
U< SU< 35      e! , (       d  f       GN= f)Nr$   r%   r#   r   r&   r'   r(   )	raw_dtyper*   zunsupported dtype z for )r   r+   r   r,   r-   r.   r/   
add_tensorr1   ggufGGMLQuantizationTyper(   r5   r6   )wnamer7   r8   r9   r>   r%   r:   r;   r<   r   r=   arrs                r   add_tensor_passthroughrI   8   s&   WIMEn%JD[F	gt		xffVn 
 EmmCrzz2::5A	TFmmCryy199%@	T$*C*C*H*HIEmmCrzz2::5A	T
	4H
II 
	s   %E
E)c           	         [        USSS9 n[        R                  " U5      nS S S 5        WR                  S0 5      nUR                  S5      S:w  a  [	        SUR                  S5      -  5      eUR                  S0 5      nUR                  S	/ 5      n[        [        UR                  5       S
S9[        S U 5       S
S95      nS/US-   -  n	UR                  5        H  u  pSUs=::  a  U::  d  M  O  M  XU'   M     U H  nUS   XS   '   M     / nUR                  S/ 5       HK  n[        U[        5      (       a"  UR                  SR                  U5      5        M:  UR                  U5        MM     U R                  S5        U R                  U	5        U R                  U5        [        US[!        U	5      [!        U5      [!        U5      4-  5        g ! , (       d  f       GN= f)Nrutf-8encodingmodeltypeBPEzexpected BPE tokenizer, got %svocabadded_tokens)defaultc              3   *   #    U  H	  oS    v   M     g7f)idN ).0as     r   	<genexpr>$add_bpe_tokenizer.<locals>.<genexpr>\   s     +ltWls       r   contentrW   merges gpt2z(tokenizer: %d vocab, %d merges, %d added)r   r   loadgetr6   maxvaluesitems
isinstancelistappendjoinadd_tokenizer_modeladd_token_listadd_token_mergesr   len)rF   tokenizer_json_pathr   r   tjrO   
vocab_dictrS   max_idtokenstok_strtok_idrZ   r`   ms                  r   add_bpe_tokenizerrx   O   s   	!3	9QYYq\ 
: FF7BEyyE!;eii>OOPP7B'J66."-LJ,+l+R8F TVaZ F%++- &  $6N . I,w  FYYx$aMM#((1+&MM!	 % &!Vv7	Vc&k3|#456 7A 
:	9s   G
G#c                     U S:X  a  gg)Ncodebook_layer_offsetsTFrX   rG   s    r   is_skip_base_tensorr|   t   s     ''r   ztoken_embd.weightzoutput_norm.weight)zllm.embed_tokens.weightzllm.norm.weightzattn_norm.weightzffn_norm.weightzattn_q.weightzattn_k.weightzattn_v.weightzattn_output.weightzattn_q_norm.weightzattn_k_norm.weightzffn_gate.weightzffn_up.weightzffn_down.weight)zinput_layernorm.weightzpost_attention_layernorm.weightzself_attn.q_proj.weightzself_attn.k_proj.weightzself_attn.v_proj.weightzself_attn.o_proj.weightzself_attn.q_norm.weightzself_attn.k_norm.weightzmlp.gate_proj.weightzmlp.up_proj.weightzmlp.down_proj.weightc                     U [         ;   a	  [         U    $ U R                  S5      (       aT  U [        S5      S  nUR                  S5      nUS:  a/  US U nXS-   S  n[        R                  U5      nUb  SU-   S-   U-   $ U $ )Nzllm.layers..r   r^   zblk.)_LM_TOPLEVEL
startswithro   find_LM_LAYER_SUFFIXrd   )hf_namerestdotidxsuffix
new_suffixs         r   rename_lm_tensorr      s    ,G$$-((s=)*+yy~7$3ZC'(^F)--f5J%|c)J66Nr   c           	      j   Sn[         R                  R                  U S5      n[         R                  R                  U S5      n[         R                  R                  U S5      n[        USSS9 n[        R
                  " U5      nS S S 5        WS   n[        US	[         R                  R                  U5      -  5        [        R                  " US
SS9n	U	R                  S5        U	R                  US   5        U	R                  US   5        U	R                  US   5        U	R                  US   5        U	R                  US   5        U	R!                  US   5        U	R#                  US   5        U	R%                  US   5        U	R'                  US   5        U	R)                  US   5        U	R+                  [-        US   S   5      5        U	R/                  S[1        UR3                  SS5      5      5        U	R5                  SUS   5        U	R5                  SUS   5        U	R5                  S US!   5        U	R7                  S"US#    V
s/ s H  n
[9        U
5      PM     sn
5        [        USSS9 n[        R
                  " U5      nS S S 5        WR3                  S$/ 5       Vs0 s H  oS%   US&   _M     nnS' HB  nX;   d  M
  S(UR;                  S)5      R=                  S*S+5      -   nU	R5                  XU   5        MD     [?        XU5        [A        U5      u  nnS,u  nnS-n[C        URE                  5       5       HA  n[G        U5      (       a  US.-  nM  [I        U5      n[K        U	UUUUU   5      nUS.-  nUU-  nMC     [        US/UUS0-  U4-  5        U	RM                  5         U	RO                  5         U	RQ                  SS19  U	RS                  5         [         R                  RU                  U5      S0-  n[        US2UU4-  5        g ! , (       d  f       GN= fs  sn
f ! , (       d  f       GN= fs  snf )3NBASEconfig.jsonmodel.safetensorsztokenizer.jsonrK   rL   rM   
llm_config
writing %szomnivoice-lmTuse_temp_filer   num_hidden_layershidden_sizeintermediate_sizenum_attention_headsnum_key_value_headshead_dim
vocab_sizemax_position_embeddingsrms_norm_epsrope_parameters
rope_thetazomnivoice.tie_word_embeddingstie_word_embeddingsFzomnivoice.num_audio_codebooknum_audio_codebookzomnivoice.audio_vocab_sizeaudio_vocab_sizezomnivoice.audio_mask_idaudio_mask_idz omnivoice.audio_codebook_weightsaudio_codebook_weightsrS   r_   rW   )z<|denoise|>z<|lang_start|>z<|lang_end|>z<|instruct_start|>z<|instruct_end|>z<|text_start|>z<|text_end|>zomnivoice.special.z<|>|_r   r   r   r^   z'total: %d tensors, %.1f MB (%d skipped)   progresswrote %.0f MB -> %s)+osr   rk   r   r   rc   r   basenamerD   
GGUFWriteradd_nameadd_block_countadd_embedding_lengthadd_feed_forward_lengthadd_head_countadd_head_count_kvadd_key_lengthadd_value_lengthadd_vocab_sizeadd_context_lengthadd_layer_norm_rms_epsadd_rope_freq_basefloatadd_boolboolrd   
add_uint32	add_arrayintstripreplacerx   r!   sortedkeysr|   r   rI   write_header_to_filewrite_kv_data_to_filewrite_tensors_to_fileclosegetsize)ckpt_diroutput_pathr   cfg_pathr7   tok_pathr   cfgllm_cfgrF   xrq   rZ   special_maptag_namekeyr    r8   	n_tensorsn_bytesn_skiprG   out_namenbout_mbs                            r   convert_baser      s   
Cww||Hm4Hggll8%89Gww||H&67H	hg	.!iil 
/,G\BGG,,[99:^4HAJJ{ g1237=12g&9:;W234 567WZ()wz*+W\*+!:;<W^45w'89,GHIJJ.W[[AVX]5^0_` LL/5I1JKLL-s3E/FGLL*C,@AKK2!$%=!>?!>AQ!>?A 
hg	.!YYq\ 
/24&&2LM2LQY<4(2LKM "&)>)F)FsC)PPCLL(34 a3' $G,ND(IwFtyy{#t$$aKF#D)#Ax(DJOQ	2 $ 6	Gw'01 2 T*GGIWW__[)W5F"fk%::;I 
/	.6 @ 
/	.Ms$   .P
P
1P$P0
P
P-c                     U R                  S5      (       d  U R                  S5      (       a  gSU ;   aC  U R                  S5      (       d,  U R                  S5      (       d  U R                  S5      (       a  gSU ;   a  gg	)
Nzdecoder_semantic.zfc1.Tzquantizer.quantizers.z.cluster_sizez
.embed_avgz.initedz parametrizations.weight.originalF)r   endswithr{   s    r   is_skip_tokenizer_tensorr      sh    *++tv/F/F$&==))T]]<-H-HDMMZcLdLd)T1r   c                     SnSnSn[        X!X   5      n[        X!X   5      n[        R                  " [        R                  " Xw-  SSS95      nXv-  U-  n	XYR	                  [        R
                  SS94$ )	NzLsemantic_model.encoder.pos_conv_embed.conv.parametrizations.weight.original0zLsemantic_model.encoder.pos_conv_embed.conv.parametrizations.weight.original1z1semantic_model.encoder.pos_conv_embed.conv.weight)r   r^   T)axiskeepdimsF)r0   )r@   r,   sqrtsumr2   r.   )
r    r8   r7   g_namev_nametargetgvnormweights
             r   fold_pos_conv_weight_normr     so    [F[F@Fw$,7Aw$,7A77266!%ft<=DUT\F==%=888r   c           	      H	   Sn[         R                  R                  U S5      n[         R                  R                  US5      n[         R                  R                  US5      n[        USSS9 n[        R
                  " U5      nS S S 5        [        US[         R                  R                  U5      -  5        [        R                  " US	S
S9nUR                  S5        UR                  SWS   5        UR                  SUS   5        UR                  SUS   5        UR                  SUS   5        UR                  SUS   5        US   n	UR                  SU	S   5        UR                  SU	S   5        UR                  SU	S   5        UR                  SU	S   5        UR                  S U	S!   5        UR                  S"U	S#    V
s/ s H  n
[        U
5      PM     sn
5        UR                  S$U	S%    V
s/ s H  n
[        U
5      PM     sn
5        US&   nUR                  S'US   5        UR                  S(US)   5        UR                  S*US+   5        UR                  S,US-   5        UR                  S.US/   5        UR                  S0US1    V
s/ s H  n
[        U
5      PM     sn
5        UR                  S2US3    V
s/ s H  n
[        U
5      PM     sn
5        UR                  S4US5    V
s/ s H  n
[        U
5      PM     sn
5        UR                  S6US7   5        UR                  S8US9   5        UR                  S:[        US;   5      5        [!        U5      u  p[#        XU5      u  pS<u  nnS=n[%        UR'                  5       5       H5  n[)        U5      (       a  US>-  nM  [+        UUX]UU   5      nUS>-  nUU-  nM7     UR-                  X5        US>-  nUUR.                  -  n[        US?UUS@-  U4-  5        UR1                  5         UR3                  5         UR5                  S
SA9  UR7                  5         [         R                  R9                  U5      S@-  n[        USBUU4-  5        g ! , (       d  f       GN= fs  sn
f s  sn
f s  sn
f s  sn
f s  sn
f )CNTOKaudio_tokenizerr   r   rK   rL   rM   r   zomnivoice-tokenizerTr   zOmniVoice-tokenizerzomnivoice.sample_ratesample_ratezomnivoice.semantic_sample_ratesemantic_sample_ratezomnivoice.downsample_factordownsample_factorzomnivoice.codebook_sizecodebook_sizezomnivoice.codebook_dimcodebook_dimacoustic_model_configz&omnivoice.acoustic.encoder_hidden_sizeencoder_hidden_sizez&omnivoice.acoustic.decoder_hidden_sizedecoder_hidden_sizezomnivoice.acoustic.hidden_sizer   zomnivoice.acoustic.n_codebooksn_codebookszomnivoice.acoustic.hop_length
hop_lengthz$omnivoice.acoustic.upsampling_ratiosupsampling_ratiosz&omnivoice.acoustic.downsampling_ratiosdownsampling_ratiossemantic_model_configzomnivoice.semantic.hidden_sizez$omnivoice.semantic.intermediate_sizer   z&omnivoice.semantic.num_attention_headsr   z$omnivoice.semantic.num_hidden_layersr   z*omnivoice.semantic.num_feat_extract_layersnum_feat_extract_layerszomnivoice.semantic.conv_dimconv_dimzomnivoice.semantic.conv_kernelconv_kernelzomnivoice.semantic.conv_strideconv_stridez*omnivoice.semantic.num_conv_pos_embeddingsnum_conv_pos_embeddingsz0omnivoice.semantic.num_conv_pos_embedding_groupsnum_conv_pos_embedding_groupsz!omnivoice.semantic.layer_norm_epslayer_norm_epsr   r   r^   z6total: %d tensors, %.1f MB (%d skipped: training-only)r   r   r   )r   r   rk   r   r   rc   r   r   rD   r   r   r   r   r   add_float32r   r!   r   r   r   r   rI   rC   r<   r   r   r   r   r   )r   r   r   	audio_dirr   r7   r   r   rF   acr   smr    r8   folded_name
folded_arrr   r   r   rG   r   r   s                         r   convert_tokenizerr    sP   
CX'89Iww||I}5Hggll9&9:G	hg	.!iil 
/ \BGG,,[99:%:$OAJJ$% LL(#m*<=LL137M3NOLL.4G0HILL*C,@ALL)3~+>? 
$	%BLL92>S;TULL92>S;TULL12m3DELL12m3DELL0"\2BCKK6!#$7!89!8AQ!89;KK8!#$9!:;!:AQ!:;= 
$	%BLL12m3DELL7<O9PQLL92>S;TULL7<O9PQLL=rB[?\]KK-:/O1A/OPKK02mCT2UCTa3q6CT2UVKK02mCT2UCTa3q6CT2UVLL=rB[?\]LLCRHgEhiMM5uR@P=Q7RS $G,ND 8PKIwFtyy{##D))aKF#AtWT
KQ	2 $ LL)NIz   GE	Gw'01 2 T*GGIWW__[)W5F"fk%::;O 
/	., :; 0P2U2Us*   .Q9
R
9R
R
R
4R
9
Rc                     [         R                  R                  [        5      (       d"  [	        SS5        [
        R                  " S5        [         R                  " [        SS9  [         R                  R                  [        S5      n [         R                  R                  [        S5      n[         R                  R                  U 5      (       a-  [	        SS	[         R                  R                  U 5      -  5        O[        [        U 5        [         R                  R                  U5      (       a-  [	        S
S	[         R                  R                  U5      -  5        O[        [        U5        [	        SS[        -  5        g )NGGUFz9checkpoints/OmniVoice not found, run checkpoints.sh firstr^   T)exist_okzomnivoice-base-F32.ggufzomnivoice-tokenizer-F32.ggufr   zskip: %s existsr   z
done -> %s)r   r   isdirCHECKPOINT_DIRr   r   exitmakedirs
OUTPUT_DIRrk   existsr   r   r  )	base_pathr   s     r   mainr  f  s    77==((FOPKK
T*Z)BCIww||J(FGH	ww~~i  F%(8(8(CCD^Y/	ww~~hE$rww'7'7'AAB.(3z)*r   __main__)r   r   r   r   numpyr,   rD   r   dirnameabspath__file__
SCRIPT_DIRrk   r
  r  r   r!   r@   rI   rx   r|   r   r   r   r   r   r   r  r  __name__rX   r   r   <module>r     s     	  
  WW__RWW__X67
j-EWW\\*h/
?;*J."7J2  33 );(9(7(7(7(<(<(<(9(7(9 J<Z 	9M<^+, zF r   