uliD UddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlZddlmZmZddlmZmZddlmZddlmZddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&ddl'Z(de j)vr:ej*+d e,ee-j.j.d z ddl/Z/dd l/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5e rdd l6m7Z7m8Z8ej9d Z:e;edr!e;edrej<ej=dZ>de?d<e/j@jAZBdZCdZDdZEedGddZFedGddeFZGeGde(jHe(jIddgZJeGde(jHe(jKddgZLeGd e(jHe(jMgZNeGd!e(jHe(jOgd"ZPedGd#d$eFZQedGd%d&eQZReRde(jHe(jKge/jSjTd'e(jHd(d)g*ZUiZVd+e?d,<ePeJeLeNfD]#ZWeWjHeVvreXd-eWeWeVeWjH<$ePeJeLeNd.ZYd/e?d0<Gd1d2ejZZ[e[j\eLe[j]eJe[j^eUiZ_d3e?d4<eGd5d6Z`dd<ZaGd=d>eZbddAZcGdBdCebZddddJZeedZfeGdKdFZgdLZhde?dM<e%dNZide?dO<eGdPdQZjddTZkddWZlddXZmddZZndd[Zodd^ZpeGd_d`ZqeGdadbZrGdcddejsZtddiZuddkZvddnZwejxdoddpZye&dqZze&drZ{ddd|Z|dddZ}GddZ~ddZddZddZddZddZddZddZGddZddZddZddZdddZedkr edSdS)) annotationsN)ABCabstractmethod)ProcessPoolExecutorThreadPoolExecutor) dataclass)Path) TYPE_CHECKINGAnyCallableIOIterableLiteralTypeVar NO_LOCAL_GGUFzgguf-py) BaseVocabVocabNoVocabBpeVocabSentencePieceVocab LlamaHfVocab)Self TypeAliasconvertregisterSIGUSR1znp.ndarray[Any, Any]rNDArrayzadded_tokens.jsonztokenizer.jsonT)frozenc6eZdZUded<ded<ded<d d Zd S) DataTypestrname np.dtype[Any]dtype list[str]valid_conversions n_elementsintreturnc ||jjzSN)r&itemsizeselfr)s examples/convert_legacy_llama.pyelements_to_byteszDataType.elements_to_bytesBsDJ///Nr)r*r+r*)__name__ __module__ __qualname____annotations__r2r3r1r"r"<sK III    000000r3r"ceZdZdS)UnquantizedDataTypeN)r5r6r7r9r3r1r;r;FsDr3r;F16F32Q8_0)r&r(I32BF16)r=r<r>c>eZdZUded<ded<ded<dd Zdd Zd S)QuantizedDataTyper* block_sizer%quantized_dtypezgguf.GGMLQuantizationType ggml_typearrrr+c2td|jd)NzQuantization for z not implemented)NotImplementedErrorr$)r0rFs r1quantizezQuantizedDataType.quantizeWs!"Qdi"Q"Q"QRRRr3r)c||jzdksJd|d|jd|j|jj||jzzS)NrzInvalid number of elements z for z with block size )rCr$rDr.r/s r1r2z#QuantizedDataType.elements_to_bytesZsyDO+q0003OPZ3O3Oaeaj3O3O~B~M3O3O000#, do0MNNr3NrFrr+rr4)r5r6r7r8rIr2r9r3r1rBrBQsiOOO""""((((SSSSOOOOOOr3rBceZdZddZdS)Q8_0QuantizedDataTyperFrr+cX|j|jzdkr |jdksJd|j|jtjksJd|j|j|jz}|||jf}d d}tj||||j S) NrzBad array size zBad array type blocksrr+Iterable[tuple[Any, Any]]c3TKt|dtjdz }tjd5||dddfz }dddn #1swxYwYd||dk<t ||Ed{VdS)Nraxisignore)divider)absmaxnpfloat32errstateroundzip)rOdqss r1quantize_blocks_q8_0z.quantize_blocks_q8_0isF q))BJsOO;Ah/// 3 3qDz)0022 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3BqAvJ1bzz ! ! ! ! ! ! ! ! !s"A>>BB)countr&)rOrr+rP)sizerCr&rYrZreshapefromiterrD)r0rFn_blocksrOr`s r1rIzQ8_0QuantizedDataType.quantizebsx$/)Q..38q===B^TWT\B^B^==@yBJ&&&(E#)(E(E&&&8t.h899 " " " " {//77SWSghhhhr3NrK)r5r6r7rIr9r3r1rMrM_s. i i i i i ir3rM )r^zz!Params.guessed..:tt;f1;f;f;fns;s;s1;s;s;s;sttr3z&model.layers.0.self_attn.W_pack.weightc3,K|]}d|dv |VdS)r.self_attn.W_pack.weightNr9rs r1rz!Params.guessed..rr3c3,K|]}d|dv |VdS)layers.z.attention.wq.weightNr9rs r1rz!Params.guessed..s:jj;\Q;\;\;\di;i;i1;i;i;i;ijjr3rz failed to guess 'n_layer'. This model is unknown or unsupported. Suggestion: provide 'config.json' of the model in the same directory containing model files.h㈵>r}r~rrrrrr) rtnext itertoolsraKeyErrortextwrapdedentr*r|)rr}r~rmsgrn_multrs` r1guessedzParams.guessedsvGbejFjFj% ;<BBpuwNqOqU 4u < <ttttio&7&7tttttGG 5 > >ttttio&7&7tttttGGjjjjio&7&7jjjjjG Q;;pC8?3//00 031F #a'(($-!+67      r3 config_pathr c6t|5}tj|}dddn #1swxYwYdx}x}x}}|d}||dx} ri|d} | }| dkrtjj}n:| dkr"tjj}|d}|d}ntd| d |vr |d } n0d |vr |d } n#d } ttj | d} d}d |vr|d } |d }t|d|d|d| |d|dx}|d|| ||d|d||||S)N rope_scalingtypefactorlinearyarn original_max_position_embeddings finetunedzUnknown rope scaling type: max_sequence_lengthmax_position_embeddingsz failed to guess 'n_ctx'. This model is unknown or unsupported. Suggestion: provide 'config.json' of the model in the same directory containing model files.num_local_expertsnum_experts_per_tok vocab_size hidden_sizenum_hidden_layersintermediate_sizenum_attention_headsnum_key_value_heads rms_norm_eps rope_theta)r}r~rrrrrrrrrrrrr) openjsonloadrqggufRopeScalingTypeLINEARYARNrHrrrr|)rrfconfigrrrrrtyp rope_factorrrrrrs r1loadHFTransformerJsonzParams.loadHFTransformerJsons, +   "!Yq\\F " " " " " " " " " " " " " " "JNMMLM:zz.11  # 0@0@0H0H)H #&**844K&Lh$($8$?!!$($8$=!)*LM !-k!:)*M*M*MNNN F * *01EE && 0 045EEpC8?3//00 0  & ( (23I#$9:N &| 4 &} 5 &': ; % &': ;+12G+H!H & +@& I I ) . &~ 6 & < 8 8 1 , * .    155ct|5}tj|}dddn #1swxYwYd}d}d}d}|drd}n-|ddkrd}n|ddkrd}nd }d |vr|d jd }|dr1|d jd }|dd }|dd}d}|Jt |djd |d|d|||dx} |d| |||d|d| S)Nmoeiri@Bi@norm_epsrizlayers.0.feed_forward.w1.weightrz)layers.0.feed_forward.experts.0.w1.weight num_expertsrg.Ardimn_layersn_heads n_kv_heads) r}r~rrrrrrrrr)rrrrqrtr|) rrrrrrrrrrs r1loadOriginalParamsJsonzParams.loadOriginalParamsJsons +   "!Yq\\F " " " " " " " " " " " " " " "  ::e   EE ZZ % % 0 0EE J 5 ( (EEE , 5 5:;A!DD ::e   #DEKAND#E]=9I#E]+@AN" $%<=CAF%e}%j1$#*0*; ;%zz,??(-%j1%zz,8HII    r model_plus ModelPlusc|jdjdz }|jdjdz }|r!t|j|}no|r!t|j|}n:|jdkr t|j}ntd|jdj|_ |S)Nrz config.jsonz params.jsonnonez-Cannot guess params when model format is none) pathsparentexistsr|rrrformatrrrr)rhf_config_pathorig_config_pathparamss r1rz Params.loadJs%+A.5 E%+A.5 E  " " N11*2BNSSFF  $ $ & & N22:3CEUVVFF  & ( (^^J$455FFLMM M&,Q/6 r3)rrr+r|)rrrr r+r|)rrr+r|)r5r6r7r8rrrrrrrrrr staticmethodrrrrr9r3r1r|r|s!%I%%%%!%N%%%%#'J''''599999%)))))!%L%%%%!J!!!!"&N&&&&!%E%%%%#J""""" " " \" H4 4 4 \4 p. . . \. `\r3r|weightsrr*rr+c|||kr|}|j|d|jd|zdzg|jddRdd|jS)Nrrr)rcrtswapaxes)rrrs r1permutercsr9!4!4 GOFAw}Q'76'AQ'F [WXWYWYIZ [ [ [ Xa^^ WW] # #%r3ceZdZUded<ded<eddZedd Zedd ZeddZeddZ dS)Tensorrndarrayr" data_typer+rcdSr-r9r0rs r1astypez Tensor.astypeos363r3rr*rcdSr-r9r0rrs r1rzTensor.permuteqsRY & & &(jZbZh(j(j & & &ry))R/H == $ $$r3c>eZdZddZddZdd ZddZddZddZdS)UnquantizedTensorrrczt|tjsJ||_t|j|_dSr-) isinstancerYrrhr&r)r0rs r1__init__zUnquantizedTensor.__init__s3'2:..... 0?r3rr"r+c|j}|jtkrt|j|_t |j|Sr-)r&rDT_BF16rrr r)r0rr&s r1rzUnquantizedTensor.astypesE >W $ $' 55DL !4!4U!;!;<<6QSV6V)WY_aj!k!klllr3cz|jjddz}t|j||z||z|zdfSr)rrtr )r0rrs r1rzUnquantizedTensor.partsA L q !Q & a&j1v:>.I3.N!OPPPr3cHtt|j||Sr-)r rrrs r1rzUnquantizedTensor.permutes vy!I!IJJJr3N)rr)rr"r+r )r+r)rr*rr*rr*r+r )rr*r+r )rr*rr*r+r ) r5r6r7r rrrrrr9r3r1r r s@@@@ ==== mmmmQQQQKKKKKKr3r F lazy_tensorrnexpected_dtyper boolcv|}t|tsJt|jj}||jksJ||jf|Q||jjkrA|r |j||_ntd|d|jj|jS)Nz#expected this tensor to have dtype z, got ) rr r listrrtr&rrr)rrrrm actual_shapes r1load_unquantizedrs     F f/ 0 000 0,--L ;, , , ,|[=N.O , , ,!n8L&L&L  q#^22>BBFNNo>ooY_YgYmoopp p >r3cPeZdZUded<ded<ded<ded<dd Zdd ZddZdS)rnzCallable[[], Tensor]_loadz list[int]rtr"rr# descriptionr+rc|}|j|jks6|jj|jjksJ|j|j|jf|Sr-)rrr&r)r0rets r1rzLazyTensor.loadsWjjll}..4>3G3=K^3^3^3^ ^S]D,< =4_3^_ r3c dfd }t|jddjS)Nr+rcRSr-)rr)rr0sr1rzLazyTensor.astype..loads99;;%%i00 0r3zconvert() r+r)validate_conversion_tornrtr)r0rrs`` r1rzLazyTensor.astypese ##I... 1 1 1 1 1 1 1$ I7a)7a7atO_7a7abbbr3Nonec|||jkr.|j|jjvrtd|jd|ddSdS)Nz Cannot validate conversion from z to .)rr$r(rrrs r1r%z!LazyTensor.validate_conversion_tosS  & &9>Aa+a+a```T]```aa a ' &+a+ar3Nr$)rr"r+rn)rr"r+r&)r5r6r7r8rrr%r9r3r1rnrnsccccbbbbbbr3zdict[str, LazyTensor]r)ggmltorch safetensorsr ModelFormatc8eZdZUded<ded<ded<ded<d S) rrr list[Path]rr,rzBaseVocab | NonevocabNr5r6r7r8r9r3r1rrsBr3rmodelslist[LazyModel]cFdD}dfd fd|DS) Nci|] }|D]}|d Sr-r9rrr$s r1 z!merge_sharded..s' > > >E > >T4 > > > >r3r$r#r+rnc2fdDtdkrdStdjdkrdSds*dsdrdndt dj}t fdD|<dfd }d d d Dzdz}t||dj|S)Nc g|] }| Sr9r9r5s r1 z2merge_sharded..convert..s888d 888r3rrztok_embeddings.z.attention.wo.weightz.feed_forward.w2.weightc32K|]}|jVdSr-)rt)rrmrSs r1rz1merge_sharded..convert..s*&U&Ufv|D'9&U&U&U&U&U&Ur3r+r cfdD}tj|}t|S)Nc,g|]}t|Sr9)rrrms r1r9z@merge_sharded..convert..load..s!LLLV(00LLLr3rR)rY concatenater )ndarrays concatenatedrS lazy_tensorss r1rz,merge_sharded..convert..loads8LL|LLLH>(>>>L$\22 2r3zconcatenated[[z] | [c3$K|] }|jV dSr-rrlts r1rz1merge_sharded..convert..s$5\5\bn5\5\5\5\5\5\r3z]]r+r ) rsrt startswithendswithrsumjoinrnr)r$concatenated_shaperrrSrAr1s` @@r1rzmerge_sharded..convertsZ8888888 |   ! ! ? " |A$ % % * *? " ??, - -  ==/ 0 0  ==2 3 3 DDD!,q/"788#&&U&U&U&U &U&U&U#U#U4  3 3 3 3 3 3 3'5\5\|5\5\5\)\)\\_cc $ 2LO4M{[[[r3c(i|]}||Sr9r9)rr$rs r1r6z!merge_sharded..s# 2 2 2DD''$-- 2 2 2r3)r$r#r+rnr9)r1namesrs` @r1 merge_shardedrNsY ? >V > > >E\\\\\\4 3 2 2 2E 2 2 22r3 models_pluslist[ModelPlus]ctd|D}t|dks Jd|}d|D} td|D}n#t$rd}YnwxYwt d|Dr"i}|D]}||jntd|D}t||||S)Nc3$K|] }|jV dSr-)rrmps r1rz)merge_multifile_models..s$#D#D"BI#D#D#D#D#D#Dr3rzdifferent formats?c&g|]}|jD]}|Sr9)r)rrTpaths r1r9z*merge_multifile_models..s' = = =bBH = =DT = = = =r3c32K|]}|j |jVdSr-)r/rSs r1rz)merge_multifile_models..s+LL"rx7KRX7K7K7K7KLLr3c3(K|] }d|jvVdS)rNrrSs r1rz)merge_multifile_models..s* I Ir &"( 2 I I I I I Ir3cg|] }|j Sr9rYrSs r1r9z*merge_multifile_models.. s>>>Brx>>>r3) setrspopr StopIterationanyupdaterrNr)rOformatsrrr/rrTs r1merge_multifile_modelsras+ ##D#D #D#D#D D DG w<<1   2    [[]]F = =K = = =ELL LLLLL  I I[ I I III@ # #B LL " " " " #>>+>>>?? UE65 1 11sA22 BBc ndfd }t|jjdddjzS)Nr+rcTSr-)rr)rrrsr1rzpermute_lazy..loads%!!))&)<<.loads'!!..vvyIIIr3rrrdrer#r$rtcopyrnrr)rrrrrss```` r1permute_part_lazyrlsJJJJJJJJJ  A Q419AaD dA{46X6X6X96X6X6X[f[r6r s ssr3cdfd }j}|ddz|d<t||jdjzS)Nr+rcRSr-)rr)rrsr1rzpart_lazy..loads#!!&&v...r3rrzpart r$ri)rrrrks`` r1 part_lazyrosk///////  A Q419AaD dA{4g @W6W X XXr3rAlist[LazyTensor]c dfd }dj}|dtt ||djdddDzS) Nr+rcrdD}ttjd|DS)Nc6g|]}|Sr9)r)rrs r1r9z3pack_experts_lazy..load..&s$FFF+;##%%FFFr3cg|] }|j Sr9)rr=s r1r9z3pack_experts_lazy..load..'s*P*P*Pf6>*P*P*Pr3)r rYarray)tensorsrAs r1rzpack_experts_lazy..load%s=FFFFF *P*P*P*P*P!Q!QRRRr3rz pack_experts  | c3$K|] }|jV dSr-rCrDs r1rz$pack_experts_lazy..*s'V}V}jlWYWeV}V}V}V}V}V}r3r$)rtrjinsertrsrnrrJ)rArrks` r1pack_experts_lazyrz$sSSSSSS Q""$$AHHQL!!""" dA|A8/EJJV}V}p|V}V}V}L}L}:} ~ ~~r3ceZdZUded<dS)LazyStorageKindr"rNr0r9r3r1r|r|5sr3r|c.eZdZUded<ded<ded<dS) LazyStoragezCallable[[int, int], NDArray]rr|kindr#rNr0r9r3r1r~r~:s6''''r3r~ceZdZUdfd Zdd Ze d d!dZedZeedeede e e e e e e e edZded<d"fd ZxZS)# LazyUnpicklerfp IO[bytes]data_base_pathr#zip_filezipfile.ZipFilecft|||_||_dSr-)superr rr)r0rrr __class__s r1r zLazyUnpickler.__init__Bs. ,  r3pidr r+c>|ddksJt|dtsJ|dj|d}jd|}j|dfd }d d |djj}t||d|S)Nrstoragerr/offsetr* elm_countr+rcBj}j5}|||jz||jz}||}dddn #1swxYwYt ||ksJtj||Sr-) r&rrseekr.readrsrY frombuffer) rrr&rrbdatarinfor0s r1rz+LazyUnpickler.persistent_load..loadOsOE##D)) %R/000 5>1wwt}} % % % % % % % % % % % % % % %t99$$$$=u-- -s=A,,A03A0zstorage data_type=z path-in-zip= path=)rrr)rr*rr*r+r)r r|rrrgetinfofilenamer~)r0r filename_stemrrrrrs` @@r1persistent_loadzLazyUnpickler.persistent_loadGs1v""""#a&/22222F$ A );;M;;}$$X.. . . . . . . . .l9kk8kkSWS`Sikk 3q6{KKKKr3Nrstorage_offsetrbstride requires_gradbackward_hooksmetadatarncttsJdfd }ddj}t|t jj|S)Nr+r cddz}t|S)Nr)r rrc)rrbrrrs r1rz2LazyUnpickler.lazy_rebuild_tensor_v2..load_s@q DG+I$W\\.)%L%L%T%TUY%Z%Z[[ [r3zpickled storage_offset=z in rF)r r~rrnrrr) rrrbrrrrrrs ```` r1lazy_rebuild_tensor_v2z$LazyUnpickler.lazy_rebuild_tensor_v2Zs';///// \ \ \ \ \ \ \ \ \ZYYGDWYY $T GL,BKPPPr3c ||Sr-r9)funcnew_typeargsstates r1rebuild_from_type_v2z"LazyUnpickler.rebuild_from_type_v2estT{r3__func__))z torch._tensor_rebuild_from_type_v2)z torch._utils_rebuild_tensor_v2)r*BFloat16Storage)r* HalfStorage)r* FloatStorage)r* IntStorage)r*rz9dict[tuple[str, str], type[LazyTensor] | LazyStorageKind]CLASSESmoduler$c|ds"t||S|j||fS)Nr*)rGr find_classr)r0rr$rs r1rzLazyUnpickler.find_classusD  )) 477%%fd33 3|VTN++r3)rrrr#rr)rr r+r r-)rr rr rbr rr rr rr rr r+rn)rr#r$r#r+r )r5r6r7r rrrrgetattrr|rDT_F16ruDT_I32rnrr8r __classcell__)rs@r1rrAs:!!!!!! LLLL&X\QQQQ\Q\ 5&>"1/&"9"9#2?6#:#:!0!8!8' J JG    ,,,,,,,,,,r3router_fprrVr ctj|}d|D}t|dks J|||dd}t ||ddd|}|}d|vr|d}t|}t||gdd S) Nc<g|]}|d|S)z.pkl)rH)rr$s r1r9z(lazy_load_torch_file..}s)LLLTdmmF6K6KLDLLLr3rrr)rrrr*rrrr/) zipfileZipFilenamelistrsrrrdictitemsr)rrVzf pickle_paths pickle_fp unpicklerras_dicts r1lazy_load_torch_filer{s  " "BLLR[[]]LLLL |   ! ! !< ! ! ! Q--Ii-9!_SbS-A')+++I NN  E%w5;;==!!G 74& M M MMr3rctjd|d\}tj||}t t j|dt j}|d|zdd fd fd | D}t|gd d S)N.convert..loads/$R]3k%J%J%J%R%RSX%Y%YZZ Zr3zsafetensors begin=z end= type=rrF)rir&rsmathprodr.rn) rrbeginendrrrrrtbyte_bufrVs @@@r1rz+lazy_load_safetensors_file..converts *4=9 o =.) sE1111S1111CMM111111U{di..1EEEEEEuSy! [ [ [ [ [ [ [ [Y5YYsYY)YYSWYY $y+>>>r3c:i|]\}}|dk ||S) __metadata__r9)rr$rrs r1r6z.lazy_load_safetensors_file..s1 ^ ^ ^\dDt~G]G]T774==G]G]G]r3r+r)rrr+rn) structunpackrrloads memoryviewmmapfileno ACCESS_READrr)rrV header_sizeheadermappedrrrs ` @@r1lazy_load_safetensors_filers=rwwqzz22LK(, 277;3G3G(H(HF  "))++q9IJJJ K KFa+o&&'H ? ? ? ? ? ? ? _ ^ ^ ^V\\^^ ^ ^ ^E 5}D Q Q QQr3lengthbytesct||}t||krtd|S)Nz unexpectedly reached end of file)rrsEOFError)rrr s r1 must_readrs6 ''&//C 3xx&9::: Jr3)maxsizec4t|d}|d}|d|dddkrt||St jd|ddkrt ||Std|) NrbrrrsPKrizunknown format: )rrrrrrrrr)rVrfirst8s r1lazy_load_filers dD  B WWQZZFGGAJJJ bqbzU#B--- tV $ $Q '*: : :)"d3332D22333r3InOutrCallable[[In], Out]iterable Iterable[In] concurrency max_workersruse_processpool_executor Iterable[Out]c #K|dkrt||Ed{Vt|}|rt}nt}||5}g}d}t |D]L} |||t|:#t$rd}YnwxYw|r| d } |srt||kr_ |||t|n#t$rd}YnwxYw|st||k_| V|ddddS#1swxYwYdS)aParallel map, but with backpressure. If the caller doesn't call `next` fast enough, this will stop calling `func` at some point rather than letting results pile up in memory. Specifically, there is a max of one output value buffered per thread.rN)rFTr) mapiterrrrangeappendsubmitrr]r\resultrs) rrrrrexecutor_classexecutorfuturesdone_rs r1bounded_parallel_maprs QtX&&&&&&&&&H~~H,,+ K 0 0 0H8:{##  A xtT(^^DDEEEE      [[^^**,,F s7||k99NN8??4h#H#HIIII$DE s7||k99 LLL s[ E"6BE B)%E(B))AE+6D"!E" D1.E0D11EE #E rr/r pad_vocabr&c|jdkr2tdt|tr d|jdndzt|tsdS|j|jkrt ddS|r|j|jkr|j|jz }t d|d|d d td |d zD]0}d|j d |d d <|j d |d d 1|j|_dSd |jd|j d|jd}|j|jcxkr|jdzkrnn|d|j j dz }|j|jkr|dz }t|)NrzNThe model's vocab size is set to -1 in params.json. Please update it manually.z Maybe ?zEIgnoring added_tokens.json since model matches vocab size without it.zPadding vocab with z' token(s) - through rz Most likely you are missing added_tokens.json (should be in z* Add the --pad-vocab option and try again.)r}rrr rrloggerwarningdebugradded_tokens_dictadded_tokens_listrfname_tokenizerr)rr/r pad_countrrs r1check_vocab_sizers ~ \0:5%0H0HP,),,,,b R    eU # #~)))^___ V^e&666NU%55  c) c cT] c c c c   q)a-(( = =A8:E #$4Q$4$4$4$4 5  # * *+;A+;+;+;+; < < < <!> rFN r r%BW r r^c^n r r rC &.@@@@5+;b+@@@@@@ pPUPePlpppp &.(( ;; S//r3ceZdZejjfd2dZd3d Zd4d Zd5dZ d6dZ d7dZ d8dZ d9dZ d9dZd:d#Zd9d$Zeejjd%d&fd;d)Zedd1Zd&S)? OutputFile fname_outr endianessgguf.GGUFEndianchtj|tjt||_dS)Nr)r GGUFWriterMODEL_ARCH_NAMESARCH)r0rrs r1r zOutputFile.__init__s'OIt/DT/JV_``` r3rr|rgguf.Metadata | Noner+r&c` d}||j|j}n!|j |jj}n |jdkrd}|j||Z|j|j|j|j|j|j|j |j |j |j |j |j |j |j|j |j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j|j |j|j!|j"|j!|j#|j$|j#|j%|j&|j%|j'|j(|j'|j)|j*|j)|j+|j,t[|j+t]|j+D]S\}}d|vr!|j/||dd|vr!|j0||dd|vr!|j1||dd|vr!|j2||dd|vr!|j3||dd |vr!|j4||d d |vr!|j5||d d |vr!|j6||d d |vr!|j7||d U|j8|j9t[|j8t]|j8D]S\}}d|vr!|j:||dd|vr!|j;||dd|vr!|j<||dd|vr!|j=||dd|vr!|j>||dd |vr!|j?||d d |vr!|j@||d d |vr!|jA||d d |vr!|jB||d U|jC|jD|jC|jE#|jF|jEdSdSdS) NLLaMArzLLaMA v2r$authorversion organizationrurldoiuuidrepo_url)Gr$rrradd_namer  add_authorr! add_versionr"add_organizationfinetune add_finetunebasename add_basenameradd_description quantized_byadd_quantized_by size_labeladd_size_labellicense add_license license_nameadd_license_name license_linkadd_license_linkr#add_urlr$add_doir%add_uuidr& add_repo_url source_urladd_source_url source_doiadd_source_doi source_uuidadd_source_uuidsource_repo_urladd_source_repo_url base_modelsadd_base_model_countrs enumerateadd_base_model_nameadd_base_model_authoradd_base_model_versionadd_base_model_organizationadd_base_model_descriptionadd_base_model_urladd_base_model_doiadd_base_model_uuidadd_base_model_repo_urldatasetsadd_dataset_countadd_dataset_nameadd_dataset_authoradd_dataset_versionadd_dataset_organizationadd_dataset_descriptionadd_dataset_urladd_dataset_doiadd_dataset_uuidadd_dataset_repo_urltagsadd_tags languages add_languages)r0rrr$keybase_model_entry dataset_entrys r1add_meta_modelzOutputFile.add_meta_models  HM$==DD   *$)DD \T ! !D 4    * $$X_555+ %%h&6777$0 **8+@AAA , &&x'8999 , &&x'8999#/ ))(*>???$0 **8+@AAA". (()<===+ %%h&6777$0 **8+@AAA$0 **8+@AAA|' !!(,///|' !!(,///}( ""8=111 , &&x'8999". (()<===". (()<===#/ ))(*>???'3 --h.FGGG#/ ..s83G/H/HIII-6x7K-L-L]])C)!111 55c;KF;STTT#333 77=Mh=WXXX $444 88>Ny>YZZZ%)999 ==cCSTbCcddd$(888 < ? ? ? ?122 2  " . I ( ()@ A A A  # C&222 I + +F,D E E E I - -f.A B B B   ( I 3 3F4E F F F  , I 0 01F G G G < # I # #FL 1 1 1 1 1 $ #r3r/r5tuple[list[bytes], list[float], list[gguf.TokenType]]cg}g}g}|D]E\}}}||||||Ft||jksJ|||fSr-) all_tokensrrsr)r0r/tokensscorestoktypestextscoretoktypes r1extract_vocabulary_from_modelz(OutputFile.extract_vocabulary_from_models%*$4$4$6$6 % % D% MM$    MM% OOG $ $ $ $6{{e.....vx''r3c|j|j||\}}}|j||j||j|dSr-)radd_tokenizer_modeltokenizer_modelradd_token_listadd_token_scoresadd_token_types)r0r/r{r|r}s r1add_meta_vocabzOutputFile.add_meta_vocabs %%e&;<<<$(#E#Ee#L#L    ((( ""6*** !!(+++++r3svocabgguf.SpecialVocabc:||jdSr-) add_to_ggufr)r0rs r1add_meta_special_vocabz!OutputFile.add_meta_special_vocabs49%%%%%r3r$r#rmrnc>ttj|j}t |jdd}t |jddp |jj}|j|}|j ||j|||dS)NrEquantized_type) raw_dtype) r*rYrrtrrr&r2radd_tensor_info)r0r$rmr)rr data_nbytess r1rzOutputFile.add_tensor_infos..// F,k4@@ F,.>EE_IYI_ &88DD  !!$ iXa!bbbbbr3cj|j|jdSr-)rwrite_header_to_filewrite_kv_data_to_filers r1 write_metazOutputFile.write_metas0 &&((( '')))))r3c8|jdSr-)rwrite_ti_data_to_filers r1write_tensor_infozOutputFile.write_tensor_infos '')))))r3rrkrrrr*c@ttj||}|tjkrttj|||d}nttj|}tj}tt||D]\}\\}} } tj|z } d d| j D} ttt|} td|dz| dd t|d |d d | d d| jjddt'| d |j| dS)N)rT)rrrz x c3K|]}|dV dS)6dNr9)rrs r1rz/OutputFile.write_tensor_data..s&GGckkGGGGGGr3[rr^rz] Writing tensor 38sz | size 16z | type 4z | T+)rrdo_itemrrkrzmaybe_do_quantizertimerHr]rJrtrsr#r rrr$r*rwrite_tensor_data)r0rrrndarrays_innerr?startrr$rrelapsedrbpadis r1rzOutputFile.write_tensor_datas-j.@%++--]hiii L+ + ++,n+cn)-HH :7HHH 1:3u{{}}h;W;W1X1X 1 1 -A-#{WikkE)G::GG[5FGGGGGDs3u::''D KKXAE4XXXXXCJJXXXXXSWXXXdodyd~XXXHKLSHTHTXXX    I ' ' 0 0 0 0 1 1r3c8|jdSr-)rclosers r1rzOutputFile.closes r3FNrrcFt|||t||}|||||||||||dSN)rr)rrrdrwrrrr)rrr/rrrrofs r1write_vocab_onlyzOutputFile.write_vocab_onlys )<<<<  Y 7 7 7 &(+++     %    !!&)))   r3itemtuple[str, LazyTensor]tuple[DataType, NDArray]ct|\}}|}|j|jfSr-)rrrr)rr$rrms r1rzOutputFile.do_items9 k!!##++--%v~66r3rcd|\}}t|ts|S||Sr-)r rBrI)rrvrFs r1rzOutputFile.maybe_do_quantizes5C"/00 J{{3r3rc ft|||t||} | || | |t |t r+| || |n| j |j | D]\} } | | | | | | |||| dSr)rrrdrwr rrrrrrrrrrrr) rrrrr/rrrrrrr$rs r1 write_allzOutputFile.write_alls. )<<<<  Y 7 7 7 &(+++    eU # # ?   e $ $ $  % %f - - - - G ' '(= > > >"' 2 2 D+   t[ 1 1 1 1   UE;777  r3)rr rr)rr|rrr+r&)rr|r+r&)r/rr+rx)r/rr+r&)rrr+r&)r$r#rmrnr+r&)r+r&)rrkrrrr*r+r&)rr rr|r/rrrrrrrrrr+r&)rrr+r)rrr+r)rr rrkrr|rrr/rrrrr*rrrrrrr+r&)r5r6r7r GGUFEndianLITTLEr rdrwrrrrrrrrrrrrDEFAULT_CONCURRENCYrr9r3r1rrsDHODZaaaaaj<j<j<j.&s"YYY4GT;D+/YYYr3z!Unexpected combination of types: )r TENSOR_NAMES MODEL_TENSORATTN_Qrrrurrkrxrryrzrrr)rrwq_type name_to_types r1pick_output_typersD%d&7&>?FF1FMMPYYZdG%O$;FT[K\@\@\""%O$;6@Q@Q%%&  &&YY5;;==YYYL GGG H HHr3rv Iterable[tuple[str, LazyTensor]]tuple[int, int, int]cd}d}d}|D]D\}}|drd}|jD]}||z}d|vr d|vr||z }n||z }||z }E|||fS)Nr)z.attention.masked_biasz.attention.biasz.rotary_emb.inv_freqrz .experts.z .experts.0.)rHrt)rv total_params shared_params expert_paramsr$rsum_weights_in_tensorrs r1!per_model_weight_count_estimationr+sLMM$..k ==^ _ _  &'$ ) )C !S ( ! ! $  $$!66 2 2M--  55r3 output_typecDfd|DS)Nc ji|]/\}}||||0Sr9)rrw)rr$rmrs r1r6z*convert_to_output_type..HsK 1 1 1v &-- ; ;D& I IJJ 1 1 1r3)r)rrs `r1convert_to_output_typerGs6 1 1 1 1"'++-- 1 1 11r3 skip_unknownc tjt|j}t tjtg}|}|jr|jdkrt|jD]}tddD]}g}t|jD]} d|d| d|d|vr3| |d|d| d|d|d|d| d|d=Cd|d | d|d|vr3| |d|d | d|d|d|d | d|d=td |d| d|dt||d|d |d<όtj D])} d| d |vrvtd | t!|d| d |j|j|d| d <t!|d| d|j|j|d| d<d| d|vrtd| t'|d| dd|j|j|d| d <t'|d| dd|j|j|d| d<t)|d| dd|d| d<|d| d=*i} |D]\} } || dpd\}}|4|rtd| dDtd| d||vrtd|yt| dd|dd| jjdd| j| | |<| S)Nrrrrz.feed_forward.experts.z.wrrz.block_sparse_moe.experts.z Expert tensor not found: layers.z.feed_forward.experts.wrzPermuting layer z.self_attn.k_proj.weightrzUnpacking and permuting layer rz.self_attn.v_proj.weight)rz.bias) try_suffixes)NNzUnexpected tensor name: z - skippingz.. Use --skip-unknown to ignore it (e.g. LLaVA)zskipping tensor 48sz -> 40srw6s)r TensorNameMaprrr[MODEL_TENSOR_SKIPrqrrrrrrzrrar r rfrrrlrorget_type_and_namer rr$rt)rrrtmap should_skiptmpi_lwexpertseroutr$r tensor_typename_news r1convert_model_namesrLs  dFN 3 3Dd,00r::;;K C cF,q00(( c cC1a[[ c cv/00xxAKKKAKKKKKuTTu-as-a-aRS-a-aWX-a-a-a'bccc T# T TQ T T! T T TUUWWWWWQWWW[```u-kS-k-k\]-k-kab-k-k-k'lmmm ^ ^ ^q ^ ^TU ^ ^ ^__()vC)v)vgh)v)vlm)v)v)vwwwHYZaHbHbDcDD!DDDEE c_     61 6 6 6% ? ? LL/A// 0 0 0?KER}bcR}R}R}L~AGANPVP]@^@^C;;;; > >?PQVXCghXCXCXCRDFGIOIVX^Xe@f@fC;;;; .s0DDD$DIIdOODDDDDDDr3)zconsolidated.00.pthzpytorch_model-00001-of-*.binz*.ptzpytorch_model.bincDg|]}|D]}|Sr9rrs r1r9z#load_some_model..s0HHHd $HHTHHHHr3zCan't find model in directory rzFound multiple models in z, not sure which to pick: rzLoading model file ) is_dirFileNotFoundErrorrsrrrr rrrra)rVglobsfilesrrOrs` r1load_some_modelrs, {{}} aaaDDDD%DDD IhhhEHHHHeHHHE M#$KT$K$KLL L u::>>```Y^``aa aQx  & &E#%K11 0$00111>$//0000' 44J r3cDeZdZUeeegZded<ddZdd Z ddZ ddZ dS) VocabFactoryzlist[type[Vocab]]_VOCAB_CLASSESrVr c||_dSr-)rV)r0rVs r1r zVocabFactory.__init__s  r3r/rmodel_parent_pathr+rc|jdk}t|tr|jnd}t j||d|S)Nbpe) load_mergesspecial_token_typesr})r$r rrr SpecialVocab)r0r/rr r}s r1_create_special_vocabz"VocabFactory._create_special_vocabsPjE) &0&>&>H%""D # $     r3 vocab_typesr'rcd|jD}i}|D]/} ||||<#t$rtd|dwxYw|D]'\}} ||j}n"#t $rY$wxYwt d|t d|jd|j |S)Nci|] }|j| Sr9)r$)rclss r1r6z6VocabFactory._create_vocab_by_path..s0^0^0^330^0^0^r3zUnsupported vocabulary type z+Could not find a tokenizer matching any of zLoaded vocab file z, type ) rrrrrrVrr rrr$)r0r vocab_classesselected_vocabsvtyperr/s r1_create_vocab_by_pathz"VocabFactory._create_vocab_by_paths!0^0^$J]0^0^0^ 24  S SE S)6u)=&& S S S !G!G!GHHdR S*//11 a aJE3 DI$    $$_R]$_$_`` ` W)>WWWWXXX s %A A22 A?>A?list[str] | None#tuple[BaseVocab, gguf.SpecialVocab]c|t}n||}|||}||fSr-)rrr)r0rrr/ special_vocabs r1 load_vocabzVocabFactory.load_vocabsN  IIEE..{;;E22    m##r3N)rVr )r/rrr r+r)rr'r+r)rrrr r+r) r5r6r7rrrrr8r rrrr9r3r1rrs~);X|(TNTTTT    * $ $ $ $ $ $r3r file_type expert_countmodel_params_countr gguf.Metadatar#cN|j|jnd}|j|jnd}|j|jnd}|j|jnd}|j|jnt j|d|pdi}t jdt jdt j di|} t j |||||| S)Nrrr=r<r>) r$r-r+r!r2rrkrxryrznaming_convention) rrrrr$r-r+r!r2rs r1default_convention_outfiler"s$M58==4D$,$5$Ax  tH$,$5$Ax  tH"*"2">hDG(0(;(G$$T_^pNR@L@QPQNRNRJ  K  !$(GZQ\ ] ]]r3 model_pathsct||||}|dj|dz }||vr2td|dt jd|S)Nrz.ggufzError: Default output path (zN) would overwrite the input. Please explicitly specify a path using --outfile.r)r"rr errorsysexit)r#rrrrdefault_filenamer s r1default_outfiler) s1)\K]_ghh a. %5"<"<"< d=||j?d>j@}t|jAp|p|}t|}|j(rdn|jCDd?}|E||\}}|j)rt|tsJ|jHstUd@|jH}|t9|jIdddddddAB}tK||||| |j2|Ctz>dD|dS|jL|jA|j(s|jL}|J|j#|j$|j$j#|_#t?|j } tz>dE| dFt/jM| d>d$tz>dG|tz>dH||j} t;| ||j} tC| |j"} t| | } |jHpt|j?| |j'| |I}t/jP| dJ|j'pd>i|_P| |_<tz>dK|dL| tQ|| || |||jR| |j2|M tz>dD|dS)NNrrrOPP>z?O?U?U?W?WXX  55   56;L;X"-2HM +E63CEWYabbdeee }MMKLLL #I.. j!!! ?i$Y// rI4G3HSYcghhh  yj!!!&I (O' F ~*T_*Z(( <2  x/  X_S112228FL < #*#-$/l FL  ((()))"(+2dnF F5FGGJ ,,M-G$$T_-B-B3-G-GK(33KARSSE= %'''''| ECDD D, >"-!   F ##GVUM.74>\d $ f f f &W&&'''#(>t}(>     }!2!>). ::;K;Q;Q;S;STT KKE,>EE$Bjk}~lACBCBEEEFFF KK&u&&''' KK6}66777E!%1BCCEudl33E$UE22Eloj.>vGWYkv~G/+=bFL\La`abbHFL KK4744U44555%}%)%5VZVdowyyy KK"""#####r3__main__)rrrr*rr*r+r)rrr+r)NF)rrnrr rrr+r)r1r2r+r)rOrPr+r)rrnrr*rr*r+rn) rrnrr*rr*rr*r+rn)rrnrr*r+rn)rArpr+rn)rrrVr r+r)rrrVr r+r)rrrr*r+r)rVr r+r) rrrrrr*rrrrr+r)F)rr|r/rrrr+r&)rrrrr+rk)rvrr+r)rrrrkr+r)rrrr|rrr+r)rVr rr*r+r)rVr r+r.) rrkrrrrrrr+r#) r#r.rrkrrrrrrr+r )rrr+r&r-)r-rr+r&) __future__rr?r:concurrent.futures concurrentenum faulthandler functoolsrrrrospicklersignalrr&rrrabcrrrr dataclassesrpathlibr typingr r r r rrrnumpyrYenvironrVryr#__file__rrrrrrrrtyping_extensionsrr getLoggerr hasattrrrrr8 MODEL_ARCHLLAMArrADDED_TOKENS_FILEFAST_TOKENIZER_FILEr"r;r&float16rrZruint16rrrrBrMGGMLQuantizationTyper>DT_Q8_0rhrvrrriIntEnumrkrxryrzrpr|rrrr rrrnrr,rrNrarfrlrorzr|r~ Unpicklerrrrr lru_cacherrrrrrrrrrrrrrr"r)r,rYr5r9r3r1rys] """""""   ########FFFFFFFF!!!!!!OOOOOOOOOOOOOOOOOO"*$$HOOAss44>>07)CDDEEE VVVVVVVVVVVVVVVV211111111  9 % % 7<$$*)C)C*L&.)))+++++ '& $00000000 $     (     ehbhrz.B.BY^`fXg h h h  ehbhrz.B.BY^`fXg h h h  ehbhrx.@.@XZ [ [ [  fhbhry.A.AXnXnXn o o o $ O O O O O O O O $iiiii-iii$ (0(<(