xidZddlZddlZddlZddlZddlZddlZddlmZm Z m Z m Z m Z m Z ddlmZddlZddlZddlmZddlmZddlmZmZddlmZdd lmZmZdd lmZdd lm Z m!Z!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'm(Z(m)Z)m*Z*d Z+dZ,GddZ-y)zk 5Hz LM (Language Model) Handler Handles all LM-related operations including initialization and generation N)OptionalDictAnyTupleListUnion)contextmanager)logger)tqdm) AutoTokenizerAutoModelForCausalLM) BaseStreamer)LogitsProcessorList RepetitionPenaltyLogitsProcessor)"MetadataConstrainedLogitsProcessor)DEFAULT_LM_INSTRUCTION!DEFAULT_LM_UNDERSTAND_INSTRUCTIONDEFAULT_LM_INSPIRED_INSTRUCTIONDEFAULT_LM_REWRITE_INSTRUCTION DURATION_MIN DURATION_MAX)get_lm_gpu_memory_ratioget_gpu_memory_gbget_lm_model_sizeget_global_gpu_configg@c  tj}t|dddk7rntjj drNt j dtjjddt|dddtd yyy) N releaselevelfinallinuxzDetected pre-release Python rz (z). This is known to cause segmentation faults with vLLM/nano-vllm on Linux. Please install a stable Python release (e.g. 3.11.12+), or use --backend pt as a workaround.) stacklevel) sys version_infogetattrplatform startswithwarningswarnversionsplitRuntimeWarning)vs 4/mnt/workspace/ACE-Step-1.5/acestep/llm_inference.py_warn_if_prerelease_pythonr/"s Aq.'*g5#,,:Q:QRY:Z *3;;+<+<+>q+A*B"WQP^`bEcDdek k   ;[5c2 eZdZdZdZej jdduZdyde e fdZ dzdZ dzd Z de fd Zdee fd Zd{d e d edededeeeff dZ dyde ede de edefdZde defdZdedefdZ d|dedede ede ee e e fdedededede d ed!e ed"e ede efd#Z d}d$e d%e d&e de de d ede fd'Zd e d(e deee ffd)Zd*ej@d+e edej@fd,Z!d*ej@d-e edej@fd.Z"d*ej@d/edej@fd0Z#d1ej@d2ed3e edefd4Z$d5e ed1ej@fd6Z%d7e&d8ej@d9ee e&fd:e e&d;ede&f d<Z'd=e(e ee fdeee effd>Z) d~d?e d@e dAe d(e dBedCe ejTdee effdDZ+d}d e dEede fdFZ, dd=e(e ee fd/edGede d+e ed-e edededed!e ed"e ede ede ee e e fdedededede d$e d%e d&e dHe eede(e ee ff.dIZ-dJe d/edGede d+e ed-e edededede ede ee e e fdedededede d$e d%e d&e de f(dKZ. dd=e(e ee fd/edGede d+e ed-e edededede ede ee e e fdedededede d$e d%e d&e dHe eede(e ee ff*dLZ/de ee e e fdefdMZ0dNee e&fde fdOZ1 dd$e d%e dPe d/edGede d+e ed-e edededede ede ee e e fdQedRedSedTe edHe eedee e&ff&dUZ2dd$e d%e dVede de de f dWZ3dd$e d%e d&e dVede de f dXZ4 ddYe dVede de fdZZ5 ddYe d/ed+e ed-e ededededeee e&fe ffd[Z6d\e de fd]Z7 dd^e d_edVede de f d`Z8 dd^e d_edae e d/ed+e ed-e ededededeee e&fe ffdbZ9 dd$e d%e dVede de f dcZ: dd$e d%e de ee e&fd/ed+e ed-e ededededeee e&fe ffddZ; ddJe dee ee e&fdedededee e ff dfZ< dydgej@dhe ej@died/ed+e ed-e eded3edje e=d5e edej@fdkZ> dydlej@dme ej@died/edGed+e ed-e eded3edje e=d5e edej@fdnZ?d\e deee e&fe ffdoZ@eAdefdpZBd e deee ffdqZCdrZD dydJe dTed/edGede d+e ed-e edededede ed$e d%e d&e dHe eedee f dsZEdJe d/edGede d+e ed-e edededede ede ee e e fdedededede d$e d%e d&e de f(dtZFdJe d/edGede d+e ed-e edededede ede ee e e fdedededede d$e d%e d&e de f(duZG dd=e(e ee fd/edGede d+e ed-e edededede ede ee e e fdedededede d$e d%e d&e dHe eede(e ee ff*dvZHeIdwZJdxZKy) LLMHandlerz(5Hz LM Handler for audio code generationSPACE_IDNpersistent_storage_pathcd|_d|_d|_d|_d|_d|_t j|_d|_ tjjddjdvxs;ttj dxrtj j# |_||j&rd }||_d|_d|_d|_d|_y) z)Initialize LLMHandler with default valuesNFcpuACESTEP_DISABLE_TQDMr )1trueyesisattyz/data)llm llm_tokenizerllm_initialized llm_backend max_model_lendevicetorchfloat32dtypeoffload_to_cpuosenvirongetlowerhasattrr#stderrr= disable_tqdmIS_HUGGINGFACE_SPACEr5constrained_processor_hf_model_for_scoring _mlx_model_mlx_model_path)selfr5s r.__init__zLLMHandler.__init__6s!$! ]] #JJNN+A2FLLNRfffovwzxBxBDLpMpeRUR\R\RcRcRekf # *t/H/H&- #'>$TX"&*"#r0returncr |jdk(rA t|jdr|jj|j d|_d|_d|_d|_d|_d|_ d|_ ddl }|jtjjr=tjj!tjj#yttdrtj$j&jrqttj&drtj&j#ttj&drtj&j!yttd r\tj(jr=tj(j!tj(j#yyyy#t$rYwxYw#t$rYwxYw#t$rYywxYw) z=Release LM weights/tokenizer and clear caches to free memory.vllmresetNFrmps synchronize empty_cachexpu)rArLr>rY Exception _cleanup_torch_distributed_stater?rPr@rRrSgccollectrDcuda is_availabler\r[backendsrZr])rTr`s r.unloadzLLMHandler.unloadQs 6)txx1(557DH!%D )-D &#(D #D "DO#'D   zz&&( &&( &&(&5>>+=+=+J+J+L599m4II))+599m4II))+&599+A+A+C %%' %%',D&5+!     shH*0H AH*HAH*3B(H*A*H* HH*HH* H'#H*&H''H** H65H6c ddlm}|jr7|jr&t j d|j yyy#t$r"}t j d|Yd}~yd}~wwxYw)zIDestroy default torch distributed process group when already initialized.rNzM[LLM vLLM] Destroying stale default process group before/after vLLM lifecyclez4[LLM vLLM] Failed to clean torch distributed state: )torch.distributed distributedrcis_initializedr warningdestroy_process_groupr^)rTdistexcs r.r_z+LLMHandler._cleanup_torch_distributed_stateusp Y ,  "t':':'<no**,(=" Y NNQRUQVW X X YsA A A;A66A;cl|jr*tjj|jdStjj t }tjj tjj |}tjj|dS)z9Get checkpoint directory, prioritizing persistent storage checkpoints)r5rHpathjoinabspath__file__dirname)rT current_file project_roots r._get_checkpoint_dirzLLMHandler._get_checkpoint_dirsl  ' '77<< < >. ) >2GGLL> 77==+@Q0RMM$'3   r0 model_path minimal_gpu min_ratio max_ratioc R tjd}tjj|j}|dz }d}|r>t ||\} } t jd|d| d| dd|d d |d krd }| |fStjj|} | dz } || z } ||krd |z}d }| |k\rt|t|||z } nt|t|| dz|z } | |fS#t$r"}t jd|Yd}~yd}~wwxYw)a Get GPU memory utilization ratio based on LM model size and available GPU memory. Args: model_path: LM model path (e.g., "acestep-5Hz-lm-0.6B"). Used to determine target memory. minimal_gpu: Minimum GPU memory requirement in GB (fallback) min_ratio: Minimum memory utilization ratio max_ratio: Maximum memory utilization ratio Returns: Tuple of (gpu_memory_utilization_ratio, low_gpu_memory_mode) zcuda:0@Fz%Adaptive LM memory allocation: model=z , target=z GB, ratio=.3fz , total_gpu=.1fGBT?g?z,Failed to calculate GPU memory utilization: N)?F) rDrCrbget_device_properties total_memoryrr infomemory_reservedminmaxr^rj)rTrrrrrCtotal_gpu_mem_bytes total_gpulow_gpu_memory_moderatiotarget_memory_gbreserved_mem_bytes reserved_gpu available_gpues r.get_gpu_memory_utilizationz%LLMHandler.get_gpu_memory_utilizationsu# \\(+F"'**"B"B6"J"W"W +g5I"' *A*i*X'' CJ'K /1!'">"> M "%\3{O3T!U 1A56L7*".!2".!3'!-!(!E!J 4 ) 1C1Cb1HIN-  s$B(( B43B4negative_promptcT|xr%|jxr|jdk7S)z:Check if negative prompt is meaningful (not default/empty) NO USER INPUT)strip)rTrs r._has_meaningful_negative_promptz*LLMHandler._has_meaningful_negative_prompts*i?#8#8#:i?T?T?VZi?iir0repetition_penaltycZt}|dk7r|jt||S)z=Build logits processor list with repetition penalty if needed?penalty)rr|r)rTrlogits_processors r._build_logits_processorz"LLMHandler._build_logits_processors..0  $  # #$DM_$` ar0use_constrained_decodingconstrained_decoding_debug user_metadatastop_at_reasoning skip_genres skip_caption skip_languageis_batchmetadata_temperaturecodes_temperaturec | xr | duxs| du} |s| sy|jj||j_||j_| r#| |j_| |j_n"d|j_d|j_|jj || r|jjd|jjd|jjd|jjd|jjdn|jj||jj||jj||jj||jj||jj| |jS)z8Setup and configure constrained processor for generationNFT) rPrYenableddebugrrset_target_durationset_user_metadataset_stop_at_reasoningset_skip_genresset_skip_captionset_skip_languageset_generation_phase)rTrrrrrrrrrrrruse_phase_temperaturess r._setup_constrained_processorz'LLMHandler._setup_constrained_processors &.!u3Gt3S3tWhptWt'0F ""((*.F""*+E""( ">RD & & ;;LD & & 8>BD & & ;;?D & & 8 ""66G   & & 8 8 >  & & <   & & 8 8 G  & & < <=N O  & & 6 6{ C  & & 7 7 E  & & 8 8 G ""778HI)))r0captionlyricscot_textch|s|dk(r|j|||d|S|j||dd|S)zKBuild unconditional prompt for CFG based on generation phase and batch moderT)is_negative_promptrcot)rrr)build_formatted_prompt_with_cotbuild_formatted_prompt)rTrrrrrrs r._build_unconditional_promptz&LLMHandler._build_unconditional_promptHsY '7277dTc8  ..D5bq/ r0rCcH tj|d|_|js:|jj |j |j |_n9|jj dj |j |_|jj d|_d|_tjd|d|d|}d|fS#t$r/}dd t|d tjfcYd }~Sd }~wwxYw) zALoad PyTorch model from path and return (success, status_message)T)trust_remote_coder8ptz95Hz LM initialized successfully using PyTorch backend on +✅ 5Hz LM initialized successfully Model: z Backend: PyTorch Device: F❌ Error initializing 5Hz LM: Traceback: N)r from_pretrainedr>rGtorFevalrAr@r rr^str traceback format_exc)rTrrC status_msgrs r._load_pytorch_modelzLLMHandler._load_pytorch_model^s m+;;JZ^_DH&&88;;v.11$**=88;;u-00< HHMMO#D #'D KKSTZS[\ ]G |SopvowxJ# # m;CF8CST]ThThTjSkll l msC&C)) D!2$DD!D!logitstop_kcn|2|dkDr-|tj||ddk}td||<|S)zApply top-k filtering to logitsr).N-inf)rDtopkfloat)rTrrindices_to_removes r._apply_top_k_filterzLLMHandler._apply_top_k_filterosB   &FE)B1)Em)T T (-f F$ % r0top_pcb|d|cxkrdkrn|Stj|d\}}tjtj|j dd}||kD}|dddfj |dd df<d |d <|j d ||}t d ||<|S) z)Apply top-p (nucleus) filtering to logitsNrT) descendingrdim.r).rr)rDr}cumsumsoftmaxrclonescatter)rTrr sorted_logitssorted_indicescumulative_probssorted_indices_to_removers r._apply_top_p_filterzLLMHandler._apply_top_p_filtervs  u!2s!2 -2JJv$,O )M>$||EMM-:M:M:OUW,X^`a '7%'? $0Hcrc0R0X0X0Z $S!"W -/0 $V , 8 @ @NTl m (-f F$ % r0 temperaturec|dkDrP|j|z }tj|d}tj|dj dStj |dS)zSample tokens from logits with temperature. Upcasts to float32 for numerical stability (float16 logits can overflow during softmax, especially after CFG scaling). rrrr) num_samples)rrDr multinomialsqueezeargmax)rTrrprobss r._sample_tokenszLLMHandler._sample_tokenssZ ?\\^k1FMM&b1E$$U:BB1E E<<B/ /r0tokens eos_token_id pad_token_idcvtj||k(ry|||k7rtj||k(ryy)z3Check if any token in the batch is EOS or pad tokenTF)rDany)rTr r r s r._check_eos_tokenzLLMHandler._check_eos_tokens: 99V|+ ,  # (Dyy</0r0rPc|@t|jdD]$}|j||j&yy)z8Update constrained processor state with generated tokensNr)rangeshape update_stater)rTrPr bs r.#_update_constrained_processor_statez.LLMHandler._update_constrained_processor_states= ,6<<?+%226!9>>3CD, -r0model generated_ids model_kwargspast_key_values use_cachecZ||dd|i|d|i}|S|d|ddddf|d|d|i}|S)z*Perform forward pass with KV cache supportN input_idsrr)rrr)rTrrrrroutputss r. _forward_passzLLMHandler._forward_passsr  "'$G '23/ /$ G r0formatted_promptsc8t|t}|r||fS|g|fS)zPNormalize batch input: convert single string to list and return (list, is_batch)) isinstancelist)rTrrs r._normalize_batch_inputz!LLMHandler._normalize_batch_inputs+/6 $h. .%&0 0r0r~ lm_model_pathbackendrGrFc  |dk(rtjjrd}nttjdr,tjj jrd}nttdr"tj jrd}nd}n|dk(rtjjsttjdrAtjj jrtjdd}nttdr7tj jrtjdd}ntjdd}n|dk(rttjdr(tjj jstjjrtjd d}n8ttdr6tj jrtjd d}ntjd d}n|dk(rttdrtj jstjjrtjd d}nqttjdr@tjj jrtjd d}ntjdd}||_ ||_ |0|dvrtj|_ nwtj|_ na||_ |dk(rU|jtjk7r8tjd|jdtj|_ |d}tjd|tj j#||}tj j%|sd|dfS|dk(rZtjjrdu} tA| }|d$k(s |d%k(r|dk(r|jCrtjd&|jE|\}}|r|dfStjd'||d$k(rytjd(|jG||\}}|s|dfSd)|d*}|dfS|d$k(r:tjd+|jG||\}}|s|dfSd,|d*}|dfS|d%k(r |dk7rtjd-|d.d/}|d%k(rtI|dk(r tKnd0}d0}|dk(rtjjr ttjd1r'tjjM\}}|d2z }nNtjjOd3jP}|tjjSd3z d2z }|dk(rP|tVkrGtjd4|dd5|dd6tVd7|jG||\}}|s|dfSd8|d*}n|jY||9}tjd:||j[d;r|j\s|dk(rX|jCrHtjd<|jE|\}}|r|dfStjd=|d>tjd?|jG||\}}|s|dfSd8|d*}n |d$k7r|jG||\}}|s|dfSdfS#tT$rd0}YcwxYw#tT$r/}d@t_|dAtajbdfcYd}~Sd}~wwxYw)Ba Initialize 5Hz LM model Args: checkpoint_dir: Checkpoint directory path lm_model_path: LM model path (relative to checkpoint_dir) backend: Backend type ("vllm" or "pt") device: Device type ("auto", "cuda", "mps", "xpu", or "cpu") offload_to_cpu: Whether to offload to CPU dtype: Data type (if None, auto-detect based on device) Returns: (status_message, success) autorbrZr]r8zA[initialize] CUDA requested but unavailable. Falling back to MPS.zA[initialize] CUDA requested but unavailable. Falling back to XPU.zA[initialize] CUDA requested but unavailable. Falling back to CPU.zA[initialize] MPS requested but unavailable. Falling back to CUDA.z@[initialize] MPS requested but unavailable. Falling back to XPU.z@[initialize] MPS requested but unavailable. Falling back to CPU.zA[initialize] XPU requested but unavailable. Falling back to CUDA.z@[initialize] XPU requested but unavailable. Falling back to MPS.z@[initialize] XPU requested but unavailable. Falling back to CPU.N)rbr]z([initialize] Overriding requested dtype z to float32 for LM on MPS.zacestep-5Hz-lm-1.7Bz3[initialize] lm_model_path is None, using default: u❌ 5Hz LM model not found at Fz.loading 5Hz LM tokenizer... it may take 80~90sT)use_fastz(5Hz LM tokenizer loaded successfully in .2f secondsz.Initializing constrained decoding processor...z-Setting constrained decoding max_duration to zs based on GPU config (tier: )) tokenizerrr max_durationz%Constrained processor initialized in hipmlxrXz8Attempting MLX backend for Apple Silicon acceleration...zMLX backend failed: zDMLX explicitly requested but failed, falling back to PyTorch backendu:✅ 5Hz LM initialized (PyTorch fallback from MLX) Model: z Backend: PyTorchz;MLX not available (requires Apple Silicon + mlx-lm package)uD✅ 5Hz LM initialized (PyTorch fallback, MLX not available) Model: zJ[initialize] vllm backend requires CUDA, using PyTorch backend for device=.rr mem_get_inforrz3vLLM disabled due to insufficient free VRAM (total=z GB, free=z GB, need>=u,GB free) — falling back to PyTorch backendu>✅ 5Hz LM initialized successfully (PyTorch fallback) Model: ) enforce_eagerz5Hz LM status message: u❌z)vllm failed on MPS, trying MLX backend...zMLX also failed: z, falling back to PyTorchzFalling back to PyTorch backendrr)2rDrbrcrLrdrZr]r rjrCrGbfloat16rFrErrHrprqryr\r[timer rr?rrtierrrPr*r.bool_is_mlx_available_load_mlx_modelrr/rr1rrrr^VRAM_SAFE_FREE_GB_initialize_5hz_lm_vllmr'r@rrr)rTr~r$r%rCrGrFfull_lm_model_path start_timer?processor_start gpu_configmax_duration_for_constraintis_rocmenforce_eager_for_vllm mlx_success mlx_statussuccessrtotal_gbfree_gb free_bytes_ total_bytesrs r. initializezLLMHandler.initializesr.~ m::**,#FU^^U38J8J8W8W8Y"FUE*uyy/E/E/G"F"F6!%***A*A*C5>>51enn6H6H6U6U6WNN#fg"FUE*uyy/E/E/GNN#fg"FNN#fg"F5'%..%*HU^^M_M_MlMlMn::**,NN#fg#FUE*uyy/E/E/GNN#ef"FNN#ef"F5'%*?EIIDZDZD\::**,NN#fg#FU^^U38J8J8W8W8YNN#ef"FNN#ef"F DK"0D }_,!&DJ!&DJ" U?tzzU]]'BNNB4::,Nhi"'DJ$ 5  QR_Q`ab!#nm!L 77>>"4578J7KLeSSEJJ$;$;$= &&( &&( KKH IJ)99:LW[\M KKB499;Q[C[\_B``hi j!.D  KKH I"iikO.0J*4*I*I ' KKGHcGdeBCMCRCRBSSTU V)K,,8 *D & KK? o@]^a?bbjk lemmU3U 8I8IQU8UG%)'] "%Gv$5&E/))+KK Z[.2.B.BCU.V+K")4//)=j\'JK"e+"NN+qr262J2JK]_e2f/GZ#*'15'8 8+fgyfz{M*NJ#-t#33%NN#`a*.*B*BCUW]*^'GZ")500#hi{h|}O"PJ%t++& Vv%5 `ag`hhij& *,28F2B,.V# (?(?(A&"5::~>,1JJ,C,C,EMJ&0G&"JKK"9* FG!,,U3#33%43I3I3K &/Z [:>:N:NOa:b 7 Z#.+5t+;$; &1B:,Ng/h i"NN+LM262J2JK]_e2f/GZ#*'15'8 8+jk}j~Q*RJE!&*&>&>?QSY&Z#%u,,t# #E%&"%&H m4SVHrAr^rrr) rTrr2rLrMrV device_namerRrr<rs r.r:z"LLMHandler._initialize_5hz_lm_vllmsDzz&&(#(D LLS TQ @ 4 $ f"ZZ668N**44^DK JJ " " $  1 1 3;?:Y:Y% ;Z; 7 "$7#%)"%)" KK::,FWXeWfgQRVRdRdQee@VWZ[\ ]J +%&"00'=,, DH KK=diikJ>VWZ=[[cd e#'D %D A*ZXcWdd~@VWZ[[rsFrGH HM @#(D LLB C @N f#(D 4SVHgeneraterLrr|rar!dictr)%rTrrrYrrrrrrrrrrrrrrrrrrrZrMformatted_prompt_listr batch_sizereffective_sampler_temprPr]sampling_paramsformatted_unconditional_promptr`r output_textsoutputs% r. _run_vllmzLLMHandler._run_vllms< ,+/*E*EFW*X'x./ &.!u3Gt3S3tWhptWt(>K!% A A%=%WAW'A+'/#%'-!5/!B !  11+-++b02 )!.12Pe*?*L*Lko   s?-1-M-M! /!1! .N. *&D$Dz$Q !hh''%&;(G hh''(=OG Fvy)c&...AA.E##FNN1$5$:$:;(##FKK0FD)f.>##F6N3##CK0'/|A@L@r0formatted_promptcj|j|ddd}|j|| | | | | |||d }|j5|jDcic]!\}}||j |j #}}}|j | |t|jjdd}|j|}|d kDr |j|||||d }||g}|jj}d |j_ |j|ddd}||j_ |jDcic]!\}}||j |j #}}}|d }|jd d}|j|||||||||jjxs|jj d| } | dd} n|rZ|j#|d |jd ||||||jjxs|jj d| } nt%j&5|jj(di|||dkDr|nd |dkDrdnd||dkDr|nd|d|cxkrd krnn|ndt+|dkDr|nd|jjxs|jj dd} ddddddt- t$j.r| j1dk(r| d}!n| }!n| d}!|d kDrd j2d}"n|d j2d}"|!|"d}!|!j j4dk7r|!j7}!|jj9|!d}#|#Scc}}wcc}}w#1swYxYw#1swYxYw)zconfigrr padding_siderJ_generate_with_cfg_customr r #_generate_with_constrained_decodingrDinference_modercrbr!Tensorrrtyper8decode)$rTrmrrYrrrrrrrrrrrrrrrrinputsrPkr-rrri batch_textsoriginal_padding_sidebatch_inputs_tokenizedrvrwrr input_length output_texts$ r._run_pt_singlezLLMHandler._run_pt_single:s.##  $ !% A A%='A+'/#%'-!B !  % % '7=||~F~tq!adkk**~FF"99 /!1$TXX__6FM:N $;;*>Ki&)j#9"E'='A'ABRTX'Y$88$3)=#1 +''9!%!3!3!@!@!cDDVDVDcDc!*?9 "!A,)BB$[1#)::.>#?#1 +'9!%!3!3!@!@!cDDVDVDcDc!*?C ))+/dhh//   '53>?K*5/$u','8UQYeD','8S5=N3=NeTX=@AQ=RUV=V)9\`%)%7%7%D%D%gHZHZHgHg!% G,W(t gu|| ,{{}! ' ' #AJM s?2+>DDQGL!+.44Q7L%lm4     $ $ -)--/M((// SX/Y cGF*kN,+W( 'sEN)&N;CN)&N)CN)B NN) N)N& "N))N2c|j|\}}|r5g}|j5t|D]\}}|r|t|krt j ||tj jr#tj j||ndttjdrJtjjjr"tjj |||j||||||||| | ddddd||||}|j|  ddd|S|d}|j||||||||| | | | | ||||||S#1swY|SxYw)a Unified PyTorch generation function supporting both single and batch modes. Accepts either a single formatted prompt (str) or a list of formatted prompts (List[str]). Returns a single string for single mode, or a list of strings for batch mode. Note: PyTorch backend processes batch items sequentially (doesn't support true batching efficiently). rZNFTrmrrYrrrrrrrrrrrrrrrrr)r#r| enumeraterbrD manual_seedrbrcmanual_seed_allrLrdrZrr|)rTrrrYrrrrrrrrrrrrrrrrrZrerrjirmrs r._run_ptzLLMHandler._run_pts<+/*E*EFW*X'x L))++45J+K'A'SZ))%(3 ::224!JJ66uQx@$U^^U;@R@R@_@_@a!II11%(;#'"5"5)9$/"+(7##+=1I3M(7&**/$(%)&*)9 '%!)'#6#K,!'' 4A,L,F 13""-#+1%='A+'/#%'-'#  Q,F s DE66Fc,|yd|vr d|vr d|vrd|vryy)z+Check if all required metadata are present.Fbpmkeyscale timesignaturedurationTr)rTrs r. has_all_metaszLLMHandler.has_all_metas>s9   M !jM&AoYfFfkuzGlGr0metadatacbi}dD]l}||vs||||}|dk(r%|jdr|jdd}t|tr|j r t |}|||<nt |dkDr'tj|ddj}nd}d |d S) a+ Format parsed metadata as CoT text using YAML format (matching training format). Args: metadata: Dictionary with keys: bpm, caption, duration, keyscale, language, timesignature Returns: Formatted CoT text: " {yaml_content} " )rrrrlanguagerrz/4/rT) allow_unicode sort_keysr z z ) endswithr+r!risdigitrrbyamldumpr)rTr cot_itemskeyvaluecot_yamls r._format_metadata_as_cotz"LLMHandler._format_metadata_as_cotFs ZCh8C=#<  /)ennT.B!KK,Q/EeS)emmoJE!& #[ y>A yy$$OUUWHH8*J//r0 infer_type use_cot_metasuse_cot_captionuse_cot_languagerfc |d}|xsdjj}|dvr"d|d}|r|dkDrgni|r|dkDrgndd|d iid S|xr|dkD}|r|nd}i}d}|j| }d }d }|r|-t|Dcgc]}t j d d }}nYt ||krFt|t|t |z Dcgc]}t j d d c}z}n|d|}|dd|sE|rB|rtjdntjdtj}|j||d}tjd||j||||||| d| | | dd||d| | d\}} tj|z }|s|rgni|rgndd| d d|iid S|j|\}}|r4tjd|ddt|jntjd|ddt|jnS|rtjdntjd| jD!"cic] \}!}"|" |!|" }}!}"|dk(rF|r7t|Dcgc]}|j!}#}|#dg|zddd ||d id S|dddd ||d id S|rtjd!|d"ntjd#tj}$|j#|}%|j%|||%}&tjd$|&|d%d&|d"|rU|&g|z}' |j&d'k(r!|j)|'|||||| | | | d(|||%|)}(nP|j&d*k(r!|j+|'|||||| | | | d(|||%|)}(n |j-|'|||||| | | | d(|||%|)}(g}*g}#|(D]F}+|j|+\}},|*j5|,|#j5|j!Htj|$z }|*D-cgc]#}-|-rt |-j7d-dz nd %}.}-tjd.|dd/|.||z}/|#|*dd|||/d,|.t9|.d0d S|j|&|||||| | dddd(|||%d1| | d\}0} |0s||z}/|dd| d |||/d,id Stj|$z }|j|0\}}|rt |j7d-dz nd }1tjd2|dd3|1d4||z}/||dd|||/d,|1d5d Scc}wcc}wcc}"}!wcc}w#t.$r;})d+t1|)}tj2|ggd|d |d |d,id cYd})~)Sd})~)wwxYwcc}-w)6aTwo-phase LM generation: CoT generation followed by audio codes generation. - infer_type='dit': Phase 1 only - generate CoT and return metas (no audio codes) - infer_type='llm_dit': Phase 1 + Phase 2 - generate CoT then audio codes Args: target_duration: Target duration in seconds for codes generation constraint. 5 codes = 1 second. If specified, blocks EOS until target reached. user_metadata: User-provided metadata fields (e.g. bpm/duration/keyscale/timesignature). If specified, constrained decoding will inject these values directly. use_cot_caption: Whether to generate caption in CoT (default True). use_cot_language: Whether to generate language in CoT (default True). batch_size: Optional batch size for batch generation. If None or 1, returns single result. If > 1, returns batch results (lists). seeds: Optional list of seeds for batch generation (for reproducibility). Only used when batch_size > 1. TODO: not used yet Returns: Dictionary containing: - metadata: Dict or List[Dict] - Generated metadata - audio_codes: str or List[str] - Generated audio codes - success: bool - Whether generation succeeded - error: Optional[str] - Error message if failed - extra_outputs: Dict with time_costs and other info NcyNr)argskwargss r.progressz9LLMHandler.generate_with_stop_condition..progresssr0r >ditllm_ditzinvalid infer_type: z (expected 'dit' or 'llm_dit')rF time_costs)r audio_codesrDrS extra_outputsrrlrOz8Phase 1: Generating CoT metadata (once for all items)...z>Batch Phase 1: Generating CoT metadata (once for all items)...z#Phase 1: Generating CoT metadata...r)rz/generate_with_stop_condition: formatted_prompt=T)rrYrrrrrrrrrrrrrmcfgrrr phase1_timezBatch Phase 1 completed in r)zs. Generated metadata: zPhase 1 completed in zABatch Phase 1: Using user-provided metadata (skipping generation)z;Phase 1: Using user-provided metadata (skipping generation)r)r total_timez*Batch Phase 2: Generating audio codes for z items...z"Phase 2: Generating audio codes...z8generate_with_stop_condition: formatted_prompt_with_cot=rz$Phase 2: Generating audio codes for rXr)rrrYrrrrrrrrrrrrZr/z!Error in batch codes generation: )r phase2_timer <|audio_code_zBatch Phase 2 completed in zs. Generated codes: )r codes_counts total_codes)rrYrrrrrrrrrrrrzPhase 2 completed in z s. Generated z audio codes)r codes_count)rrKrrrandomrandintrbr"r rr4rgenerate_from_formatted_promptparse_lm_outputkeysr}copyrrrArl_run_mlxrr^rrSr|r+sum)2rTrrrrrYrrrrrrrrrrrrfrZr error_msgractual_batch_sizerrrrrrH phase1_startrmcot_output_textstatusrr- metadata_list phase2_startrformatted_prompt_with_cotr codes_outputsraudio_codes_listraudio_codes_itemrrrcodes_output_textrs2 r.generate_with_stop_conditionz'LLMHandler.generate_with_stop_conditioncs^   !&B--/557 / /.znP)>A!-1)>HP  :?@Q:R S:RQ:R S -$&4*;#;#!$+6*5'&  !)#%#!$+6*5'&    KKDEVDWW`a b KK< =yy{ //9%)$H$HRXZb$c! NOhNijk<=NO O F ##v-$(NN*;$/"+(7##+=1I3M(7)0 '%!)#%3%M"%%.$(MM*;$/"+(7##+=1I3M(7)0 '%!)#%2%M$%)LL*;$/"+(7##+=1I3M(7)0 '%!)#%1%MF " M, &*&:&:;&G## ''(89$$X]]_5 - ))+ 4K`pp_oV[UC O <=APQQ_oLp KK5k#5FFZ[gZhi j${2J)/(3'2&0# %1#&|#4"  ")-(K(K!:#.!*'6""*<'6%)$(%)(/&$ ("*B+E"'+)L) % v0%(;6 (#%$#$+6+6*4'&  ))+ 4K"112CDNA{IT#k//@AAEZ[K KK/ C/@ k]Zfg h${2J$*(3'2&0# $/ "  cY&sDQ !TL ?AxH  Y' "#%$&$+6+.*5'&   8qsCVV VV8VBV$1(W+$ W(-0W#W(#W(rc|j td|r,|j|}|dk(r|r d|d|d}nd|d}n |}n d|d|d}|jjddtd d d |d gd d S)aA Build the chat-formatted prompt for 5Hz LM from caption/lyrics. Raises a ValueError if the tokenizer is not initialized. Args: caption: Caption text lyrics: Lyrics text is_negative_prompt: If True, builds unconditional prompt for CFG generation_phase: "cot" or "codes" - affects unconditional prompt format negative_prompt: Negative prompt for CFG (used when is_negative_prompt=True) Example: prompt = handler.build_formatted_prompt("calm piano", "hello world") :LLM tokenizer is not initialized. Call initialize() first.r # Caption # Lyric  z# Lyric system# Instruction  rolecontentuserFTtokenizeadd_generation_prompt)r? ValueErrorrapply_chat_templater)rTrrrrrhas_negative_promptprompts r.rz!LLMHandler.build_formatted_prompts    %YZ Z #'"F"F"W 5(&*?*;=PRSF )3F!#7)=CF!!55!@V?WW[.\]F3 "& 6  r0c|j td|r|j|}d}|r|}n|}n|}|}d|d|d} |jjddtdd d | d d |d gd d } | j ds| dz } | S)a Build the chat-formatted prompt for codes generation phase with pre-generated CoT. Args: caption: Caption text lyrics: Lyrics text cot_text: Pre-generated CoT text (e.g., "\nbpm: 120\n...\n") is_negative_prompt: If True, uses empty CoT for CFG unconditional prompt negative_prompt: Negative prompt for CFG (used when is_negative_prompt=True) Returns: Formatted prompt string Example: cot = "\nbpm: 120\ncaption: calm piano\n...\n" prompt = handler.build_formatted_prompt_with_cot("calm piano", "hello", cot) rz rrrrrrrr assistantFr)r?rrrrr) rTrrrrrrcot_for_promptcaption_for_prompt user_prompt formatteds r.rz*LLMHandler.build_formatted_prompt_with_cots$    %YZ Z #'"F"F"W 1N"%4"&-"&N!( $$6#7}VHBO &&::!@V?WW[.\]K8$@  "'; !!$'  Ir0rc|j td|r|r|jr|nd}n|}|jjddtddd|dgdd S) a Build the chat-formatted prompt for audio understanding from codes. This is the reverse of generation: given audio codes, generate metadata and lyrics. Args: audio_codes: Audio code string (e.g., "<|audio_code_123|><|audio_code_456|>...") is_negative_prompt: If True, builds unconditional prompt for CFG negative_prompt: Negative prompt for CFG (used when is_negative_prompt=True) Returns: Formatted prompt string Example: codes = "<|audio_code_18953|><|audio_code_13833|>..." prompt = handler.build_formatted_prompt_for_understanding(codes) rr rrrrrFTr)r?rrrr)rTrrr user_contents r.(build_formatted_prompt_for_understandingz3LLMHandler.build_formatted_prompt_for_understanding=s.    %YZ Z .=/BWBWBY?_aL&L!!55%!01R0SSWX #+ "&6  r0ct|ddsidfS|r|jsidfStjdt |d|j |}t d||j|||||dddddd d d d ||d \} } | si| fS|j| \} } |j| } | r| | d <tjdt | d|rKtjdt| jtjd| dddddj| j}| |fS)a  Understand audio codes and generate metadata + lyrics. This is the reverse of the normal generation flow: - Input: Audio codes - Output: Metadata (bpm, caption, duration, etc.) + Lyrics Note: cfg_scale and negative_prompt are not supported in understand mode. Args: audio_codes: String of audio code tokens (e.g., "<|audio_code_123|><|audio_code_456|>...") temperature: Sampling temperature for generation top_k: Top-K sampling (None = disabled) top_p: Top-P (nucleus) sampling (None = disabled) repetition_penalty: Repetition penalty (1.0 = no penalty) use_constrained_decoding: Whether to use FSM-based constrained decoding for metadata constrained_decoding_debug: Whether to enable debug logging for constrained decoding Returns: Tuple of (metadata_dict, status_message) metadata_dict contains: - bpm: int or str - caption: str - duration: int or str - keyscale: str - language: str - timesignature: str - lyrics: str (extracted from output after ) Example: codes = "<|audio_code_18953|><|audio_code_13833|>..." metadata, status = handler.understand_audio_from_codes(codes) print(metadata['caption']) # "A cinematic orchestral piece..." print(metadata['lyrics']) # "[Intro: ...]\n..." r@F7❌ 5Hz LM not initialized. Please initialize it first.u<❌ No audio codes provided. Please paste audio codes first.z#Understanding audio codes (length: z chars)zformatted_prompt: N understandr rrrrrrrrrrrrrrz#Understanding completed. Generated z metadata fieldsGenerated metadata: Output text preview: ...u;✅ Understanding completed successfully Generated fields: , )r%rr rrbrprintrr_extract_lyrics_from_outputrr"rrq)rTrrrrrrrrmrrrrHrrs r.understand_audio_from_codesz&LLMHandler.understand_audio_from_codesmsZt.6PP P+"3"3"5UU U 9#k:J9K7ST HHU "#3"456#AA-*&8#'!% %!&$$0&>'A#'B  V,v: **;7 !11+> !'HX  9#h-HXYZ % LL/X]]_0E/FG H LL0Tc1B0C3G HSTXT]T]^f^k^k^mTnSop ##r0rcddl}d}|j||}|sy||jdj}|syd}|j |d||j }|j dd|}|jS)aM Extract lyrics section from LLM output. The lyrics appear after the tag and typically start with "# Lyric" or directly with lyric content. Args: output_text: Full LLM output text Returns: Extracted lyrics string, or empty string if no lyrics found rNr3r z^#\s*Lyri[c|cs]?\s*\n)flagsz<\|im_end\|>\s*$)researchendrsub IGNORECASE)rTrrthink_end_patternmatch after_thinklyric_header_patterns r.rz&LLMHandler._extract_lyrics_from_outputs ( +[9"%))+,/557  8ff12{"--fX ff0"kB   ""r0query instrumentalc|j td|rdnd}|r|r|jr|nd}n|d|}|jjddtdd d |d gd d S)a Build the chat-formatted prompt for inspiration/simple mode. This generates a complete sample (caption, lyrics, metadata) from a user's natural language music description query. Args: query: User's natural language music description instrumental: Whether to generate instrumental music (no vocals) is_negative_prompt: If True, builds unconditional prompt for CFG negative_prompt: Negative prompt for CFG (used when is_negative_prompt=True) Returns: Formatted prompt string Example: query = "a soft Bengali love song for a quiet evening" prompt = handler.build_formatted_prompt_for_inspiration(query, instrumental=False) rr;falser z instrumental: rrrrrFTr)r?rrrr)rTr r rrinstrumental_strrs r.&build_formatted_prompt_for_inspirationz1LLMHandler.build_formatted_prompt_for_inspirations4    %YZ Z&26w .=/BWBWBY?_aL$W$67G6HIL!!55%!01P0QQUV #+ "&6  r0vocal_languagec lt|ddsidfS|r|jsd}tjd|ddd|d |d |j || } tj d | d} d} |ri|jrY|jj d k7r8d|ji} tjd|j|j| ||||d| ddddddd || d\} }| si|fS|j| \}}|j| }|r||d<n|rd|d<||d<tjd|d| rKtj dt|jtj d| dddd|}||fS)a Create a complete music sample from a user's natural language query. This is the "Simple Mode" / "Inspiration Mode" feature that generates: - Metadata (bpm, caption, duration, keyscale, language, timesignature) - Lyrics (unless instrumental=True) Args: query: User's natural language music description instrumental: Whether to generate instrumental music (no vocals) vocal_language: Allowed vocal language for constrained decoding (e.g., "en", "zh"). If provided and not "unknown", it will be used. temperature: Sampling temperature for generation (0.0-2.0) top_k: Top-K sampling (None = disabled) top_p: Top-P (nucleus) sampling (None = disabled) repetition_penalty: Repetition penalty (1.0 = no penalty) use_constrained_decoding: Whether to use FSM-based constrained decoding constrained_decoding_debug: Whether to enable debug logging Returns: Tuple of (metadata_dict, status_message) metadata_dict contains: - bpm: int or str - caption: str - duration: int or str - keyscale: str - language: str - timesignature: str - lyrics: str (extracted from output after ) - instrumental: bool (echoed back) Example: query = "a soft Bengali love song for a quiet evening" metadata, status = handler.create_sample_from_query(query, instrumental=False, vocal_language="bn") print(metadata['caption']) # "A gentle romantic acoustic pop ballad..." print(metadata['lyrics']) # "[Intro: ...]\n..." r@FrrzCreating sample from query: Ndz... (instrumental=z, vocal_language=r+)r r z"Formatted prompt for inspiration: unknownrzUsing user-specified language: rr rrr[Instrumental]r z'Sample created successfully. Generated  fieldsrr,ru2✅ Sample created successfully Generated fields: ) r%rr rrrrKrrrr"r)rTr r rrrrrrrrmrrrrrrHrrs r.create_sample_from_queryz#LLMHandler.create_sample_from_query-s&bt.6PP PEKKM#E 25#;-?QR^Q__pqqAABC D FF%G   9:J9KLM  n2249M9M9O9U9U9W[d9d')=)=)?@M KK9.:N:N:P9QR S #AA-*&8#'!. %!&$$0 &>'A#%B  V*v: **;7 !11+> !'HX  !1HX $0  =hZwOP % LL/X]]_0E/FG H LL0Tc1B0C3G HJ8*U ##r0c|j td|r|r|jr|nd}nd|d|}|jjddtddd |dgd d S) a Build the chat-formatted prompt for format/rewrite mode. This formats user-provided caption and lyrics into a more detailed and specific musical description with metadata. Args: caption: User's caption/description of the music lyrics: User's lyrics is_negative_prompt: If True, builds unconditional prompt for CFG negative_prompt: Negative prompt for CFG (used when is_negative_prompt=True) Returns: Formatted prompt string Example: caption = "Latin pop, reggaeton, flamenco-pop" lyrics = "[Verse 1]\nTengo un nudo..." prompt = handler.build_formatted_prompt_for_format(caption, lyrics) rr rrrrrrrFTr)r?rrrr)rTrrrrrs r.!build_formatted_prompt_for_formatz,LLMHandler.build_formatted_prompt_for_formats6    %YZ Z .=/BWBWBY?_aL) vhGL!!55%!01O0PPTU #+ "&6  r0c t|ddsidfS|r|jsd}|r|jsd}tjd|ddd t ||j || } tj d | d} |ri} |jd  t|d } | d kDr| | d <|jd t|d} | d kDr| | d<|jdr|d| d<|jdr|d| d<|jdr|d| d<| sd} ntjd| |j| ||||d| d| r| jddundddddd || d\}}|si|fS|j|\}}|j|}|r||d<n||d<tjd|d| rKtj dt|jtj d|dddddj!|j}||fS#ttf$rYwxYw#ttf$rYwxYw) as Format user-provided caption and lyrics into structured music metadata. This is the "Format" feature that takes user input and generates: - Enhanced caption with detailed music description - Metadata (bpm, duration, keyscale, language, timesignature) - Formatted lyrics (preserved from input) Note: cfg_scale and negative_prompt are not supported in format mode. Args: caption: User's caption/description (e.g., "Latin pop, reggaeton") lyrics: User's lyrics with structure tags user_metadata: Optional dict with user-provided metadata to constrain decoding. Supported keys: bpm, duration, keyscale, timesignature, language temperature: Sampling temperature for generation (0.0-2.0) top_k: Top-K sampling (None = disabled) top_p: Top-P (nucleus) sampling (None = disabled) repetition_penalty: Repetition penalty (1.0 = no penalty) use_constrained_decoding: Whether to use FSM-based constrained decoding constrained_decoding_debug: Whether to enable debug logging Returns: Tuple of (metadata_dict, status_message) metadata_dict contains: - bpm: int or str - caption: str (enhanced) - duration: int or str - keyscale: str - language: str - timesignature: str - lyrics: str (from input, possibly formatted) Example: caption = "Latin pop, reggaeton, flamenco-pop" lyrics = "[Verse 1]\nTengo un nudo en la garganta..." metadata, status = handler.format_sample_from_input(caption, lyrics) print(metadata['caption']) # "A dramatic and powerful Latin pop track..." print(metadata['bpm']) # 100 r@Frrrz&Formatting sample from input: caption=N2z..., lyrics length=)rrzFormatted prompt for format: rrrrrrz*Using user-provided metadata constraints: rr rrrz)Format completed successfully. Generated rrrrru4✅ Format completed successfully Generated fields: r)r%rr rrbrrrJrr TypeErrorrrrr"rrq)rTrrrrrrrrrrmconstrained_metadatabpm_valdur_valrrrrHformatted_lyricsrs r.format_sample_from_inputz#LLMHandler.format_sample_from_inputs ht.6PP Pgmmo%GV\\^%F 'A#%B  V*v: **;7 ! ;;KH !1HX "(HX  ?zQR % LL/X]]_0E/FG H LL0Tc1B0C3G HLTYYW_WdWdWfMgLhi ##E#I.#I.s$$I I# I I #I65I6rct|ddsy|jdk(r|j |jy|j |jy|xsi}|j dd }|j d d }|j d d }|j d} |j d} |j dd } |j d} |j d} |j dd}|j dd}|j dd}|j dd}|j dd}|j dd}|j dd} |jdk(r4|j ||||| | | ||| | ||||||||}|dt|fS|jdk(r4|j||||| | | ||| | ||||||||}|dt|fS|j||||| | | ||| | ||||||||}|d t|fS#t$r4}d!dl }|j}tjd"t|j d#|d$||jdk(r_ d!d%lm}|n#t&$rYnwxYw t)|jd&r|jj+n#t$rYnwxYwt,j.j1r=t,j.j3t,j.j5nt)t,d'r[t,j6j1r=t,j6j3t,j6j5n~t)t,j8d(rdt,j8j:j1r 1 - top_k (int), top_p (float), repetition_penalty (float) - target_duration (float): Target duration in seconds for codes generation - generation_phase (str): "cot" or "codes" for phase-aware CFG use_constrained_decoding: Whether to use FSM-based constrained decoding constrained_decoding_debug: Whether to enable debug logging for constrained decoding stop_at_reasoning: If True, stop generation immediately after tag (no audio codes) Returns: (output_text, status_message) Example: prompt = handler.build_formatted_prompt(caption, lyric) text, status = handler.generate_from_formatted_prompt(prompt, {"temperature": 0.7}) r@F)r rr/N)r u-❌ 5Hz LM is missing MLX model or tokenizer.)r u)❌ 5Hz LM is missing model or tokenizer.rg333333?rYrrrrrrrrrrrrrrr rrrX)rrrYrrrrrrrrrrrrrrrru+✅ Generated successfully (vllm) | length=u*✅ Generated successfully (mlx) | length=u)✅ Generated successfully (pt) | length=rz)Error in generate_from_formatted_prompt: : r) reset_contextrYr]rZu,❌ Error generating from formatted prompt: r)r%rArRr?r>rJrlrbrrr^rrr rSr__name__nanovllm.utils.contextr$rUrLrYrDrbrcr\r[r]rdrZ splitlines)rTrmrrrrrrYrrrrrrrrrrrrrrrr error_detailr$s r.rz)LLMHandler.generate_from_formatted_promptmsJ>t.6P   u $&$*<*<*DJ XX !3!3!;BiRggmS1 GGK- ''"3_E   WW%93?''"340 ww~u5 7 ggmU3 77#5u='')R(2&77:r*h 6)"nn&6 +'$3'9-E/I$3"/&7 +!-"/%5#!%'- *#&QRUVaRbQc$ddd!!U*"mm&6 +'$3'9-E/I$3"/&7 +!-"/%5#!%', *#&PQTU`QaPb$ccc,,"2'# /#5)A+E /+"3')+!1!''K*"KCP[L\K] ^^ ^  /9//1L LLDT!WEUEUDVVXYZX[[]^j]kl m6)D!O"txx1( zz&&( &&( &&(&599+A+A+C %%' %%'/ENN4F4F4S4S4U %%' %%'Ed1gFVFVEWWYZ[Z|_k_v_v_xy{_|Y}~~ ~= sv!AH$AH'3H Q%AQ; J Q JQJQ0K  Q KQKE8QQQrrurrxc |j} |j} |j} ||j}ntj|}d|i}d}t | dxrt | jdd}|jj}||}|j|}tj5tt|dd|jD]L}|j| | |||}|j ddd ddf}| | | |}|D] }|| |} |j#||}|j%||}|j'||}|j)| ||j+|||}|j-d }tj.| |gd } tj.|tj0|j2d d f| |j4 gd }||d<|rt |dr |j6}| | j9||sMnddd| | j;| S#1swYxYw)z Custom generation loop with constrained decoding support (non-CFG). This allows us to call update_state() after each token generation. Nrugeneration_configrTzLLM Constrained DecodingtokendescunitdisablerrrrrCrFr)r>rCrrD ones_likerLr%r*r?r rrr rrNrrrrrrr unsqueezecatonesrrFrputr)rTrrurrrrrr rxrPrrCr attn_maskrrrr rsteprnext_token_logits processor next_tokens should_stopnext_tokens_unsqueezeds r.rz.LLMHandler._generate_with_constrained_decoding s"")  %&,,.I 2I))4 E#67oGED[D[]hjn29SZakok|k|}},,UM)>q)A& % =:P*QWX Y !IIy%**iooa>PRS=T]cktkzkz2{&|CDE 1: -.2C!D&-&=&=O'LL!78W~$\   LLNc$ #s;E.I +I  Irvrwc |j} |j} |jddz}d}|}|j}||j}nt j |}i}|||d<d}t | dxrt| jdd}|jj}|| }|j|}t j5tt|dd |j D]}|j!| ||||}|j"ddd ddf}||||z}||||z}|j%||j%|j%z zz}| ||||z}| ||}||||z}|D] }|||} |j'||}|j)||}|j+||} |j-| | |j/| || }!| j1d }"t j2||"j5dd gd }t j2|t j6|dzd f| |j8gd }||d<|rt |dr |j:}| | j=|"|!snddd| | j?|S#1swYxYw)a Custom CFG generation loop that: 1. Processes both conditional and unconditional sequences in parallel 2. Applies CFG formula to logits 3. Samples tokens only for conditional sequences 4. Applies the same sampled tokens to both conditional and unconditional sequences 5. Optionally applies constrained decoding via FSM-based logits processor Batch format: [cond_input, uncond_input] rr!Nrur*rTzLLM CFG Generationr+r,rrrr0r) r>rCrrrDr1rLr%r*r?r rrr rrNrrrrrrrrr2r3repeatr4rFrr5r)#rTrvrwrrrYrrrr rxrPrrCrfcond_start_idxuncond_start_idxrrurrrr rr7rr8 cond_logits uncond_logits cfg_logitscurrent_input_idsr9r:r;r<s# r.rz$LLMHandler._generate_with_cfg_customn sA0$**1-2 %(--/  +1779N"___=N  +-;L) *E#67oGED[D[]hjn29MT[eievevww,,UM)>q)A& % =:P:W:WXY[\:]*^de f !&NEJJ ST VWGXago}pDpD=E,FLM"N1? -.2C!D&-&=&=O'LL!78sx$x   LLNC$ #sG K!KK c|jdd}tjd|id}ddl}d}|j ||}|rdj |}gd}d}|D]B} |j | ||j} | s#| jdj}n|s*d |vr|jd dn|} | j}|r |jd } dgfd } | D]}|jjd r#|r|djsud |vrq| |jd d}t|dk(sd|djj|d}|jsj||jds|jdsЉsӉj|| |fS)a Parse LM output to extract metadata and audio codes. Expected format: bpm: 73 caption: A calm piano melody duration: 273 genres: Chinese folk keyscale: G major language: en timesignature: 4 <|audio_code_56535|><|audio_code_62918|>... Returns: Tuple of (metadata_dict, audio_codes_string) r3rzDebug output text: r Nz<\|audio_code_\d+\|>)(.*?)rFz(.*?)rrrc.rډrdj}dk(r t|jd<ndk(rtj|d<ndk(r t|jd<ncdk(r|jd<nJdk(r|jd<n1dk(r|jd<ndk(r|jd<d gy #|jd<YxYw#|jd<Y6xYw) z Save the accumulated field valuerrrrgenresrrrN)rqrrrpostprocess_caption)r current_keycurrent_value_linesrs r.save_current_fieldz6LLMHandler.parse_lm_output..save_current_field# s#6 II&9:E"e+<.1%++-.@HUO% 1.P.d.dej.k+$ 2A36u{{}3EHZ0%0-2[[]*$ 2/4{{},$ 2/4{{},$749KKM1" &(#)<.3kkmHUOA38;;=HZ0sC#C=#C:=D<:r!  )r+r rrfindallrqrDOTALLgrouprr'isspacerbrKr|)rTrdebug_output_textrr code_pattern code_matchesreasoning_patternsreasoning_textpatternrlines_before_codeslinesrLlineparts first_valuerJrKrs @@@r.rzLLMHandler.parse_lm_output s((--j9!< *+<*=>? / zz, < '',/K  )GIIg{BII>E!&Q!5!5!7 *JY]hJh!2!2?!CA!Fny /557N "((.EK"$  )@::<**3/Q 1cTk&(!JJsA.E5zQ&+Ahnn&6&<&<&> &+Ah &,,./66{C__S)T__T-B"+2248+0  $$r0c8 ddlm}ddl}y#t$rYywxYw)z4Check if MLX framework is available (Apple Silicon).rNTF)mlx.corecoremlx_lmrU)mxrcs r.r7zLLMHandler._is_mlx_availablec s"  !   s c ddlm}ddlm}t j d|t j } ||\|_}|j9|jj;||_t j |z }t j d|ddd|_d|_ d|d}d|fS#t$r}t j d|dddl }ddl m }ddlm } m } m} m} ||} | | }|jt!| d z }|st#d ||i}|D]"}|j%|j|$t't)|}|j+d sBt j d |j-Dcic] \}}d || ncc}}w}}}| | \}}|j/|}||}t1|dr|j3|}|j5t7|j-d|j9|j;|j9||_Yd}~(d}~wwxYw#t$rJ}ddl!}|jD}t jFd|d|ddt!|fcYd}~Sd}~wwxYw)z Load the 5Hz LM model using mlx-lm for native Apple Silicon acceleration. Args: model_path: Path to the HuggingFace model directory Returns: Tuple of (success, status_message) rN)loadzLoading MLX model from zStandard MLX load failed (z-), retrying with 'model.' prefix remapping...)Path) load_model load_configload_tokenizer _get_classeszmodel*.safetensorszNo safetensors found in zmodel.z;Adding 'model.' prefix to weight keys for MLX compatibility)r~sanitizeT)strictz!MLX model loaded successfully in r)sr/rz> Backend: MLX (Apple Silicon native) Device: Apple Silicon GPUzFailed to load MLX model: rFu❌ MLX load failed: )$rarb mlx_lm.utilsrfr rr4rRr^globpathlibrgrhrirjrkrFileNotFoundErrorupdatenextiterr'r} from_dictrLrl load_weightsr"r parametersrSrAr@rrrj)rTrrdmlx_loadr<rH first_err_globrgrhrirjrk _model_pathr~ weight_filesweightswf sample_keyrr- model_classmodel_args_class model_argsr load_timerrrr(s r.r8zLLMHandler._load_mlx_modelm sL ; ! 5 KK1*> ?J- (%-j%9"\ GGDOO..0 1#-D  j0I KK;Ic?!L M$D #'D $&,-  # #y+ (  0 = 0.20.0) rarbmlx_lm.models.cacherrRrUAttributeError make_cache RuntimeError)rTrdrs r._make_mlx_cachezLLMHandler._make_mlx_cache sg  =$T__5 5^,  1133! "Q   s  A%A  A% A!!A%c]^ddlm^ddl}ddlm}m]ddlm}|j|ddd}|d }|jd }^j|d}|j| d }|jj}|jjxs|}||dkDr|nd |d |cxkrdkrnn|nd||dkDr|nd}|dk7}t|}|dkD}|rdnd}d}ddlm} d}!d}"d}#|j#| | | dddddd d }$|$t%|$drC|$j&7^j|$j&jj}!t%|$dr!|$jt)|$j}"t%|$dr |$j*}#|$j,| j.k(rd|vr| j0|$_d|$_t5j4}%t7j8d|d]^fd}&|r|j;| | ||d d}'|j|'ddd}(^j|(d d})t=|)}*||j>}+||j>},|}-t=|-d kDr~tA|t=|-d z }.|j?|-d|.d|+^jC|+D/cgc]}/|/j,c}/|-|.d}-^jEt=|-d kDr~|)}0t=|0d kDr~tA|t=|0d z }.|j?|0d|.d|,^jC|,D/cgc]}/|/j,c}/|0|.d}0^jEt=|0d kDr~|j?|-d|+}1|j?|0d|,}2^jC|1|2|+g}3|,g}4tGd |D]0}5|3jI|&|+|4jI|&|,2tGd |D]p}5^jB|3|5D/cgc]}/|/jJ|/jJc}/^jB|4|5D/cgc]}/|/jJ|/jJc}/r|1ddddddfg|z}6|2ddddddfg|z}7t5j4|%z }8||*z}9|8dkDr|9|8z nd}:t7j8d|9d |d!|*d"|8d#d$|:d%d&|d'|d z |9zd(n||j>};|}tGd |D]}5|>jI|&|;tGd |D]9}5^jB|>|5D/cgc]}/|/jJ|/jJc}/;|=ddddddfg|z}?t5j4|%z }8|8dkDr||8z nd}:t7j8d|d)|8d#d$|:d%d&|d* tM|d}@tG|DAcgc] }AtM@}B}AtG|DAcgc]}Ag}C}Adg|z}Ddg|z}Eg}FtG|D]>}5|r#|5t=|krFjI||5(FjId+|5d,zz@t5j4}GtO|d-|d.|d/d01}HtG|D]}ItQErntG|D]}5E|5r ^jRjUF|5Id,zz|r7|5|6|5|7|5z zz}Jn?|5}JJjWd d}J|rSt=B|5dkDrB^jB|5}KJdd|Kf}L^jY|LdkD|L|z |L|z}M|M|Jdd|Kf<|!J|!z}J|#|"D|5|#krF^j[Jddd|"f^jtd2gg|Jdd|"d zdfgd 3}Jn^Jdd|"|"d zf}N^j]|Jjtd2}J^j[|Jddd|"f|N|Jdd|"d zdfgd 3}JJ^j_|Jd4z }O||O}P^jC|P|Pja}QC|5jI|QB|5jI|QD|5xxd z cc<|Q|k(rdE|5<|||k7r Q|k(rdE|5<^jQgg}R|rP|j?R3|5}S|j?|R4|5}T|Sddddddf6|5<|Tddddddf7|5<`|j?R>|5}U|Uddddddf?|5<Hjcd Id5zdk(sIdkDsɉ^jEHjet5j4Gz }Vtgd6CD}W|dkDrW|z nd}XVdkDrWVz nd}Y|8Vz}Zt7j8d7|d8Wd9Xd:d;|Vd#d$|Yd%d<|8d#d=|Vd#d>|Zd#d?g}[tG|D]3}5|jjiC|5d@}\[jI|\5[Scc}/wcc}/wcc}/wcc}/wcc}/wcc}/wcc}Awcc}Aw)Aa Optimized native MLX batch generation for codes phase. Strategy: shared prefill + clone cache + interleaved B=1 decode. On Apple Silicon, LLM decode is memory-bandwidth-bound. Batching the forward pass (B>1) doubles the KV cache reads per step and actually *slows down* throughput for 1.7B-class models. Instead, we: 1. Prefill ONCE with B=1, then clone the KV caches for each item. This saves ~50% of prefill time vs sequential generation. 2. Interleave B=1 forward passes across items within each step. Each item gets its own cache, constrained state, and seed. This achieves ~1.25x speedup over fully sequential generation while maintaining the full ~44 tok/s per-item decode speed. Only used for codes generation phase where all prompts are identical. Raises on failure so the caller can fall back to sequential mode. rN)rKVCache make_samplernpFTrorrrrrrrtemprrCFG r rPFSMStatersnon_audio_code_maskr rr3z&MLX batch native: prefilling once for z items (shared prompt)cg}|D]w}}|jQj|j|_j|j|_|j|_|j |y|S)zNDeep-copy a list of KVCache objects so each batch item gets independent state.)rarrayvaluesoffsetr|) cache_listclonedcnew_crrds r._clone_cache_listz;LLMHandler._run_mlx_batch_native.._clone_cache_listJ siF 66%!#!&&!1EJ#%88AHH#5EL#$88EL e$ Mr0r_cacherzMLX batch native prefill:  tokens (cond= , uncond=) in r)s (rz tok/s) [shared across z items, saved z redundant tokens] tokens in z items]*iCBMLX zBatch Gen (native, n=r+toktotalr-r.raxiskeepdimsc32K|]}t|ywr)rb).0ts r. z3LLMHandler._run_mlx_batch_native..+ s;?a3q6?sz&MLX batch native generation complete: z items, z total tokens (z.0fz avg) in  tok/s) | prefill s + decode s = s totalrz)5rarbnumpyrrrmlx_lm.sample_utilsrr?rrrr r r$acestep.constrained_logits_processorrrrLrrrstate THINK_TAGCODES_GENERATIONrr4r rrrbrRrr clear_cacherr|rr"r allrseedreshapewhere concatenatefull logsumexprrscloserr)_rTrmrfrrYrrrrrrrrrrrZrrrr input_ids_np prompt_lengthrrr r sampleruse_rep_penaltyrep_penalty_valuse_cfg cfg_labelprefill_step_sizer_mlx_non_audio_mask _mlx_eos_id _target_codesrP prefill_startr uncond_text uncond_inputs uncond_prompt uncond_lengthbase_cond_cachebase_uncond_cachecond_remaining chunk_sizeruncond_remainingbase_cond_logitsbase_uncond_logitsitem_cond_cachesitem_uncond_cachesritem_last_conditem_last_uncond prefill_timetotal_prefill_tokens prefill_tps base_cache remaining base_logits item_cachesitem_last_logitsbase_token_idsrHitem_all_token_idsitem_new_tokensitem_codes_count item_finisheditem_seed_bases decode_startpbarr7 step_logits token_indicesselectedmodifiedeos_vallogprobs token_arrtoken_id next_inputrArB logits_out decode_time total_tokens avg_tokens decode_tpsrrjrrrds_ @@r._run_mlx_batch_nativez LLMHandler._run_mlx_batch_native s L B4##  $  k* $**1- ,q/*55+$6  ))66 ))66F,  +aS ,u1Bs1B% ,% -3 23c/%F2   B"  !% A A%='A+#$!B !  ! ,,.CDI^IrIrI~&(hh/D/X/X/^/^/`/f/f/h&i#,n=BWBdBdBp!"7"D"DE ,n= 5 B B %**h.@.@@!112:2K2K)/89)5  K 99;6L#0=#@ AMPQAQ.=WXK KK,-A,BC&y@"3's;s*;<"",^Z\K_<_;``rt +4??;JIi.1$ !2C NQ4FG  +: 6t 2>+"))(3"1%,,X6!#q(#|+'+M!$+ 0LQY]iQi'+M!$ XXzl3 "&//*DTUVDW/"XK$(OOJFXYZF[O$\M(3ArsAI(>N1%*723 *B$Q'!%;q>!RJ*4QQY*?$Q'W'Z KKNczQ4!8 o*r iikL0 ;?;; 2?\K/ !K/  4ZLnOJs+;9[QTDUV31,s1C;{[^N__cdnorcssz |  z"A,,33OA4F\a3bK    ,#_;="UV,6P,O9sBl1l6l; l; 7m m m2m m m- mcddlm}ddl}ddlm}ddlm}|j|ddd}|d }|jd }|j|d}|j|| | | | | |||d }|j| | }|jj}|jjxs|}||dkDr|nd |d |cxkrdkrnn|nd||dkDr|nd} |dk7}!t|}"|dkD}#|#rdnd}$d|$d}%d}&ddlm}'d}(d})d}*d}+|t#|drC|j$7|j|j$jj}(t#|dr!|jt'|j})t#|dr |j(}*|dk(rL|j*|'j,k(r3d|vr/|'j.|_d|_d}+t3j4dt7j6},|#r9|j9|||||d}-|j|-ddd}.|j|.d d}/t;|/}0||j<}1||j<}2|}3t;|3d kDr~t?|&t;|3d z }4|j=|3d|4d|1|jA|1D5cgc]}5|5j*c}5|3|4d}3|jCt;|3d kDr~|/}6t;|6d kDr~t?|&t;|6d z }4|j=|6d|4d|2|jA|2D5cgc]}5|5j*c}5|6|4d}6|jCt;|6d kDr~|j=|3d|1}7|j=|6d|2}8|jA|7|8|7ddddddf}9|8ddddddf}:t7j6|,z };||0z}<|;dkDr|<|;z nd}=t3j4d|<d |d!|0d"|;d#d$|=d%d& n||j<}>|}?t;|?d kDr~t?|&t;|?d z }4|j=|?d|4d|>|jA|>D5cgc]}5|5j*c}5|?|4d}?|jCt;|?d kDr~|j=|?d|>}@|jA|@|@ddddddf}At7j6|,z };|;dkDr||;z nd}=t3j4d|d'|;d#d$|=d%d&tE|d}Bg}Ct7j6}DtG||%d()}EtI|D])}F|#r :|9|:z zz}GnA}GGjKd d}G|!rMt;BdkDr?|jB}HGdd|Hf}I|jM|IdkD|I|"z |I|"z}J|J|Gdd|Hf<|~|j*}K|K|'j.k(r|(G|(z}G|*Y|)V|j0|*krG|jOGddd|)f|jtd*gg|Gdd|)d zdfgd +}GnGdd|)|)d zf}L|jQ|Gjtd*}G|jO|Gddd|)f|L|Gdd|)d zdfgd +}GnK|'jRk(rnGjU|jV}M|j|Md,}NtYjZ|N}OtYj\BgtXj^-}P||P|O}O|j|Oj}GG|ja|Gd.z }Q| |Q}R|jA|R|Rjc}SCje|SBje|SEjgd ||jiSS|k(rn| ||k7rS|k(rn|jSgg}T|#rC|j=T1}7|j=|T2}8|7ddddddf}9|8ddddddf}:n!|j=T>}@|@ddddddf}AFd/zdk(sFdkDs|jC,Ejkt7j6Dz }Ut;C}V|UdkDrVUz nd}W|;Uz}Xt3j4d0Vd'|Ud#d$|Wd%d1|;d#d2|Ud#d3|Xd#d4 |jjmCd5}Y|YScc}5wcc}5wcc}5w)6a Optimized native MLX generation using mlx-lm infrastructure. Key improvements over the hybrid approach: 1. Native MLX sampling (temperature, top-k, top-p) via mlx-lm make_sampler - Eliminates numpy/PyTorch round-trip for EVERY generated token 2. Native MLX repetition penalty (no per-step PyTorch conversion) 3. Chunked prefill for memory-efficient long prompt processing 4. Periodic memory cleanup (mx.clear_cache) matching mlx-lm patterns 5. Bridges to PyTorch ONLY for constrained decoding FSM when active Raises on failure so the caller can fall back to the legacy hybrid method. rNrrrFTrorrrsrrrrrr rz Gen (native)rPrrr rrr3zGMLX native: pre-transitioned FSM to CODES_GENERATION (native fast path)r_rrzMLX native prefill: rrrr)rr tok/s)rrrrrrrFrrz MLX native generation complete: rrrrrz)7rarbrrrrrr?rrrrr r rrrrLrrrrrrrr rr4rrbrRrrrr"r rrrrr COMPLETEDastyperErD from_numpytensorlongrrr|rsrrr)ZrTrmrrYrrrrrrrrrrrrrrrrrdrrrrrrrrPrr r rrrrr tqdm_descrrrrr_use_native_codes_pathrrrrr cond_cache uncond_cacherrrrrArB last_cond last_uncondrrrrrr last_logits all_token_ids new_tokensrrr7rrrr _cp_staterstep_logits_f32 np_logitst_logitst_idsrrrrr num_generatedrrrsZ r._run_mlx_single_nativez!LLMHandler._run_mlx_single_native= ss F 94##  $  k* $**1- ,q/*!% A A%='A+'/#%'-!B ! 55+-6  ))66 ))66F,  +aS ,u1Bs1B% ,% -3 23c/%F2 9+\2   B"  !& ,,.CDI^IrIrI~&(hh/D/X/X/^/^/`/f/f/h&i#,n=BWBdBdBp!"7"D"DE ,n= 5 B B  7*/D/J/JhN`N`/`!112:2K2K)/89)5-1*KK ij ::! /!1 ;K!..# /M HH];%?%BCM .M+4??;J,T__=L$Nn%) !2C4G!4KL { ;D AT*5*Q*56!/ !< n%) - &'!+ !2C8H4IA4MN  0* =d C<X,7,Q,78#3JK#@  &'!+//.*>j/QK OO,>+;"#}<| ||<|;};|dk7rdd$l m!}=|=|%}>|>|<|;};|jE|;|};|jG|;|};|jI|;|}?|?jK}@|4jM|@|3jM|@|6jOd ||jQ@@|k(rn| ||k7r@|k(rn|j@gg}A|rV|j'A'})|j'|A(}*|j)|)|*|)ddddddf}+|*ddddddf},|j'A0}1|j)|1|1ddddddf}2|6jSt!j |5z }BtU|4}C|BdkDrCBz nd}D|-Bz}Etj*d&Cd|Bdd|Ddd'|-dd(|Bdd)|Edd* |jjW|4d+}F|FS),a MLX-accelerated single-item generation. Tries optimized native MLX generation first (using mlx-lm infrastructure for sampling, repetition penalty, and chunked prefill). Falls back to hybrid MLX/PyTorch approach if native generation fails. rzNative MLX generation failed (r#z), falling back to hybrid modeNrrFTrorrrsrrrr r Generationr_rrz MLX prefill: rrrr)rrrrrrrr)rrzMLX generation complete: rrrrrz),rr^r rjrr%rarbrr?rrrrr r r4rrrRrrr"r rrr rErDr r r &transformers.generation.logits_processrrrrrr|rsrrrbr)GrTrmrrYrrrrrrrrrrrrrrrr _native_errrdrrrrrrPrr r rrr rrrrrrrrArBrrrrrrrrrrrrr7rrrrrrrep_proct_tokenrrrrrrrsG r._run_mlx_singlezLLMHandler._run_mlx_single s< ..!1'# /#5)A+E /+"3')+!1!'/ *  NN0k1B1K1K0LB{m\./    ##  $  k* $**1- ,'!% A A%='A+'/#%'-!B ! 55+-6  ))66 ))66F, c/%F2 9+Z0  ::! /!1 ;K!..# /M HH];%?@M)//2M--/J//1L//& /CK OOMONM GGK /#ArsAI.I'23 2K99;6L#0=#@ AMPQAQ.=WXK KK 456&y@"3's;s*;7D  ((*Eu=J GGJ $QQY/K99;6L:F:J-,6PQK KK /"3's;s*;7D \!_-  yy{ .yuE.)D)I[9P,QQ ) %--a4K *00q)A#-$/"+(7##+=1I3M(7 '%!)#60 KKG |STU VL'01F'G##QU^IINN58,"22%5 +'$3'9-E/I$3"&&+ $!%"&%5#!%'3 *##K05(H6 13##-#+1%='A+'/#%'-'$  Q!NN3DG4D4D3ERsK:;s1;E)) F+2/F&&F+c# K|jsdy|jdvrdy|j}|dy t|j j j }t|j jdd}||k(rdytjd|j tj}t|dr4|j|j j|jtj|z }tjd|j d|d d  dtjd tj}t|dr|jd t j"j%rt j"j'ntt d r=t j(j%rt j(j'ntt j*drpt j*j,j%rHtt dr8tt j,drt j,j'tj|z }tjd|d d y#t$rd}YwxYw#tjd tj}t|dr|jd t j"j%rt j"j'ntt d r=t j(j%rt j(j'ntt j*drpt j*j,j%rHtt dr8tt j,drt j,j'tj|z }tjd|d d wxYww)z Context manager to load a model to GPU and offload it back to CPU after use. Only used for PyTorch backend when offload_to_cpu is True. N)rXr/rNrzLoading LLM to rzLoaded LLM to z in z.4frnzOffloading LLM to CPUr8r]rZr\zOffloaded LLM to CPU in )rGrAr>rtrxrCr StopIterationrr+r rr4rLrrFrDrbrcr\r]rdrZ)rTrrV target_devicer<r offload_times r.r|zLLMHandler._load_model_contexts< ""      .   =    "!%"2"2"45<<AANDKK(..s3A6 ] *    odkk]34YY[ 5$  HHT[[ ! $ $TZZ 0IIK*,  nT[[Mi_AFG H  KK/ 1Jud#zz&&( &&(&599+A+A+C %%'/ENN4F4F4S4S4UZabginZot{}B}F}FHUuV %%'99;3L KK2<2DAF G? "!N "& KK/ 1Jud#zz&&( &&(&599+A+A+C %%'/ENN4F4F4S4S4UZabginZot{}B}F}FHUuV %%'99;3L KK2<2DAF GsC8R-K(C%RK0F R K-)R,K--R0F Q==Rc|jdk(r |jS|jdk(r|jqtjd|jj }|j j}ddl}|j}tj|d|j|_|j|z }tjd|d d |jr;|jjtjd |jSt|jjj }|jj#||_|jjtjd ||jS|jd k(r|jxtjdt%|dd}| t'dddl}|j}tj|d|j|_|j|z }tjd|d d |jr;|jjtjd |jSt)t*j,dr*t*j,j.j1rdnd}|jj#||_|jjtjd ||jSt'd|j)aw Get HuggingFace model for perplexity scoring. For vllm backend, loads HuggingFace model from disk (weights are cached by transformers). For pt backend, returns the existing model. For mlx backend, loads HuggingFace model from disk (MLX model can't be used for torch scoring). Returns: HuggingFace model instance rrXNz7Loading HuggingFace model for scoring (from checkpoint)rT)r torch_dtypezHuggingFace model loaded in r)rnz?HuggingFace model for scoring kept on CPU (offload_to_cpu=True)z'HuggingFace model for scoring ready on r/zGLoading HuggingFace model for scoring (MLX backend, need PyTorch model)rSzEMLX model path not stored. Cannot load HuggingFace model for scoring.rZr8zUnknown backend: )rAr>rQr r model_runnerr~rr4r rrFrGrrtrxrCrr%rrLrDrdrZrc)rTr.rr4r<rrCs r.get_hf_model_for_scoringz#LLMHandler.get_hf_model_for_scoring4s   t #88O    '))1 UV $xx44 )0066 &TYY[ -A-Q-Q&* $ .* &DIIK*4  :9S/KL &&..335KK ab-- - ","4"4"?"?"ABIIF151K1K1N1Nv1VD...335KK"I& RS-- -    &))1 ef%T+>*5TA$ $6"HY9S9 T#Y 0S0e0di0}B0MRSXZ^S^M_0l'+ :!%::sm : :xjsjtj % DW $04-18*"&8*%)8*"% 8*  S(3-%7 89 8*  8*8*8*8*8*8*'uo8*$E?8* 4 58*B      ,mcm3m5sCSm"%,,x}QVQ]Q] %,, x SXS_S_  0U\\ 0 0%,, 0u||3V^_bVchlERtIuE@E@L@LE ||38n  "#    01c49n8M1RWX\]`XacgXgRh1$'+UmUmUm Um  Um  Um $Um sDy Umn2f#2fd2fWZ2fz*.+004-1+/<@"' "# %%)/mA d3i0mAmA mA  mA } mAmA"mA#'mA%)mA'uomA$E?mA"%mA S(3-%7 89mA mA !mA"#mA$%mA&'mA()mA*+mA,-mA.S "/mA0 sDI~ 1mA^^^^ ^  ^ } ^^"^#'^%)^"%^ S(3-%7 89^ ^^^ !^"#^$%^&'^()^* +^R*.+0+/<@"' "# %%)+b  d3i0b b  b  b } b b "b #'b %)b "%b  S(3-%7 89b  b b b !b "#b $%b &'b ()b *S "+b , sDI~ -b H8Dhsm9K4L+MRV0S#X030D".#!%$')-+0+/<@" $!%$(%))jjj j  j  jj}jj"j#'j%)j"%j S(3-%7 89jj !j"#j$SM%j&S "'j* c3h+jX . c. 3. Y]. y|. X[. ru. `<s<C<SV<lp<LO<fi<B$). . . !.  . . f!#!%$')-+0b$b$b$} b$  b$ " b$#'b$%)b$ tCH~s" #b$H$#s$#s$#R##(. 4 4 4 ! 4  4 4 r#(,!#!%$')-+0x$x$x$! x$  x$ } x$x$"x$#'x$%)x$ tCH~s" #x$|$). 2 2 2 ! 2  2 2 p37!#!%$')-+0P$P$P$ S#X/ P$  P$ } P$P$"P$#'P$%)P$ tCH~s" #P$j)-)-+0"' bbd38n %b#' b %) b  b sCxb^OS[<<[!.[ [  [ } [["[[<([ ((JK[ [ROSww'u||4w w  w  w}ww"ww<(w ((JKw wrv%3v%5c3h9L3Mv%xtV;#V;%c 2BV;p >&*!ggg g  g  g}gg"g#'g%)g"%gggg S "!g" c#gR AAA A  A } AA"A#'A%)A"%A S(3-%7 89A AAA !A"#A$%A&'A()A* +AF nnn n  n } nn"n#'n%)n"%n S(3-%7 89n nnn !n"#n$%n&'n()n* +nr*.+0+/<@"' "# %%)+D  d3i0D D  D  D } D D "D #'D %)D "%D  S(3-%7 89D  D D D !D "#D $%D &'D ()D *S "+D , sDI~ -D T7H7HrVEr0r2).r3rHr#rr4rr(typingrrrrrr contextlibr rrDlogurur r transformersr r !transformers.generation.streamersrrrrrracestep.constantsrrrrrracestep.gpu_configrrrrr9r/r2rr0r.r=sq  ::% <:Tuuss   \=E\=Er0