
    Oi3                        S r SSKrSSKrSSKrSSKrSSKJrJrJrJ	r	J
r
Jr  SSKJr  SSKrSSKJr  SSKJs  Jr  SSKJr  SSKJr  SSKJrJrJr   SSKJr  SS	KJr  S
r  SSK#r$S
r%SSK&J'r'J(r(J)r)  SSK*J+r+  SSK,J-r-  SSK.J/r/J0r0J1r1  SSK2J3r3J4r4J5r5J6r6  SSK7J8r8  SSK9J:r:  / SQr;S\	S\<4S jr=S\<S\R|                  4S jr?S\<S\<4S jr@S\R                  S\
\B\B4   4S jrCS\\R                   R                     S\
\B\B4   4S jrES \R                  R                  S\
\B\B4   4S! jrH S7S\R                  S"\\R                     S\\B\<4   4S# jjrI S8S\\R                   R                     S$\\B\<4   S%\BS\
\B\B\\<   4   4S& jjrJS\R                  S'\\R                     S\\R                   R                     4S( jrKS)\R                  S\L4S* jrMS\R                  S\\R                     4S+ jrNS,\R                  S\
\L\L\L4   4S- jrOS. rP " S/ S0\R                  5      rQ " S1 S25      rR " S3 S4\R                  5      rS " S5 S65      rTg! \! a    Sr \RD                  " S5         GN
f = f! \! a    Sr%\RD                  " S5         GN$f = f)9z
LoRA Trainer for ACE-Step

Lightning Fabric-based trainer for LoRA fine-tuning of ACE-Step DiT decoder.
Supports training from preprocessed tensor files for optimal performance.
    N)OptionalListDictAnyTuple	Generator)logger)nullcontext)AdamW)CosineAnnealingWarmRestartsLinearLRSequentialLR)Fabric)TensorBoardLoggerTFzFLightning Fabric not installed. Training will use basic training loop.z1bitsandbytes not installed. Using standard AdamW.)
LoRAConfig
LoKRConfigTrainingConfig)inject_lora_into_dit)check_peft_available)save_lora_weightssave_training_checkpointload_training_checkpoint)inject_lokr_into_ditsave_lokr_weightssave_lokr_training_checkpointcheck_lycoris_available)PreprocessedDataModule)	safe_path)      ?g.袋?g?g?g      ?g%I$I?g      ?g333333?devicereturnc                     [        U [        R                  5      (       a  U R                  $ [        U [        5      (       a  U R                  SS5      S   $ [	        U 5      $ )z:Normalize torch device or string to canonical device type.:   r   )
isinstancetorchr    typestrsplitr    s    7/mnt/workspace/ACE-Step-1.5/acestep/training/trainer.py_normalize_device_typer,   I   sJ    &%,,''{{&#||C#A&&v;    device_typec                 z    U S;   a  [         R                  $ U S:X  a  [         R                  $ [         R                  $ )z,Pick the compute dtype for each accelerator.cudaxpumps)r&   bfloat16float16float32r.   s    r+   _select_compute_dtyper8   R   s0    o%~~e}}==r-   c                      U S;   a  gU S:X  a  gg)z:Pick Fabric precision plugin setting for each accelerator.r0   z
bf16-mixedr3   z16-mixedz32-true r7   s    r+   _select_fabric_precisionr;   [   s    o%e r-   modulec                    SnSnU R                  5        H  nUR                  (       d  M  US-  nUR                  5       (       d  M2  UR                  [        R
                  :w  d  MR  [        R                  " 5          UR                  R                  5       Ul        SSS5        US-  nM     X4$ ! , (       d  f       N= f)z2Force trainable floating-point parameters to fp32.r   r$   N)	
parametersrequires_gradis_floating_pointdtyper&   r6   no_graddatafloat)r<   castedtotalps       r+   _ensure_trainable_params_fp32rH   f   s    FE 
  QWW%= !aKF ! = !s   < B00
B>	paramsc                     SnSnU  HI  nUR                   nUc  M  US-  n[        R                  " U5      R                  5       (       a  MD  US-  nMK     X4$ )z>Count non-finite gradient tensors among params with gradients.r   r$   )gradr&   isfiniteall)rI   	nonfinitetotal_with_gradrG   gs        r+   _count_nonfinite_gradsrQ   u   s]    IOFF91~~a $$&&NI  %%r-   	optimizerc                    SnSnU R                    H  nUR                  S/ 5       H  nUc  M  US-  nUR                  5       (       d  M$  UR                  [        R
                  :w  d  MD  [        R                  " 5          UR                  R                  5       Ul        SSS5        US-  nM     M     X4$ ! , (       d  f       N= f)z9Force optimizer parameter tensors to fp32 when trainable.r   rI   Nr$   )	param_groupsgetr@   rA   r&   r6   rB   rC   rD   )rR   rE   rF   grouprG   s        r+   _ensure_optimizer_params_fp32rW      s    FE''8R(AyQJE""$$EMM)A]]_VV\\^AF %! ) ( = %_s   ? B77
Cextra_modulec                     0 nU R                  5        H  u  p4X2[        U5      '   M     Ub7  UR                  5        H#  u  p4UR                  [        U5      SU 35        M%     U$ )z?Build a best-effort id(param) -> name lookup for debug logging.zlycoris_net.)named_parametersid
setdefault)r<   rX   lookupnamerG   s        r+   _build_param_name_lookupr_      sd      F**,r!u -#446GDbe|D6%:; 7Mr-   param_name_lookupdetail_limitc                 h   SnSn/ nU  GH"  nUR                   nUc  M  US-  n[        R                  " U5      R                  5       (       a  ME  US-  n[	        U5      U:  a  M[  UR                  [        U5      S[        U5       S35      nUR                  5       R                  5       n	[        [        R                  " U	5      R                  5       R                  5       5      n
[        [        R                  " U	5      R                  5       R                  5       5      nU	[        R                  " U	5         nUR                  5       (       a5  [        UR                  5       R!                  5       R                  5       5      O
[        S5      nUR                  5       R                  5       n[        [        R                  " U5      ) R                  5       R                  5       5      nUR#                  U S[%        UR&                  5       SUR(                   SU
 S	U S
US SU 35        GM%     X4U4$ )zPCount non-finite grads and return up to `detail_limit` offending tensor details.r   r$   z	<unnamed:>nanz	 | shape=z grad_dtype=z nan=z inf=z max_abs_finite=z.3ez param_nonfinite=)rK   r&   rL   rM   lenrU   r[   detachrD   intisnansumitemisinfnumelabsmaxappendtupleshaperA   )rI   r`   ra   rN   rO   detailsrG   rP   pnameg32	nan_count	inf_countfinite_valsmax_abs_finitep32param_nonfinites                   r+   _count_nonfinite_grads_detailedr{      s    IOGFF91>>!  ""Q	w<<'!%%beyAq-ABhhj C(,,.3356	C(,,.3356	%..-.   "" +//#'')..01u 	 hhj s 3388:??ABgYuQWW~.l177) D+U9+-=nS=Q R./1	
3 > w..r-   lycoris_netc                    U R                  5        Vs/ s H  o"R                  (       d  M  UPM     nnU(       a4  [        U Vs0 s H  n[        U5      U_M     snR	                  5       5      $ / nUc  U$ [        US/ 5      =(       d    /  H>  nUR                  5        H'  nUR                  (       d  M  UR                  U5        M)     M@     U(       d;  UR                  5        H'  nUR                  (       d  M  UR                  U5        M)     [        U Vs0 s H  n[        U5      U_M     snR	                  5       5      $ s  snf s  snf s  snf )z
Collect LoKr trainable params robustly.

Primary path is model parameter traversal. If that returns empty due to
wrapper/registration quirks, fall back to LyCORIS module parameters.
loras)r>   r?   listr[   valuesgetattrro   )r<   r|   rG   rI   fallbackms         r+   _collect_lokr_trainable_paramsr      s     **,@,Aa,F@v.v!RUAXv.55788)+H['2.4"4A"   5 '')A" * 8,8aA8,33566! A. -s   EEEEmodelc                 n   U b  [        U S5      (       d  gU R                  nSn[        US5      (       as  [        [        US5      [        R
                  5      (       aJ  UR                  nSn[        US5      (       a+  [        [        US5      [        R
                  5      (       a  MJ  U(       a  Xl        U$ )z
Unwrap stale Lightning Fabric wrappers from decoder left by previous runs.

Returns:
    True if decoder was unwrapped, else False.
decoderF_forward_moduleT)hasattrr   r%   r   nnModuler   )r   r   	unwrappeds      r+   _unwrap_stale_fabric_decoderr      s     }GE955mmGI
',
-
-**+RYY3 3 ))		 ',
-
-**+RYY3 3
 r-   c                    / nU /n[        5       nU(       a  UR                  5       n[        U[        R                  5      (       d  M8  [        U5      nXS;   a  MJ  UR                  U5        UR                  U5        S HA  n[        XFS5      n[        U[        R                  5      (       d  M0  UR                  U5        MC     U(       a  M  U$ )zHCollect wrapper chain modules (Fabric/PEFT/compile/base-model wrappers).)r   	_orig_mod
base_modelr   r<   N)	setpopr%   r   r   r[   addro   r   )r<   modulesstackvisitedcurrent	module_id	attr_namechilds           r+   _iter_module_wrappersr      s    !GHEeG
))+'299--wK	Iw
I G5E%++U#
 %* Nr-   r   c                 n   SnSnSn[        U 5       H  n[        US5      (       a   UR                  5         SnO[        US5      (       a
   SUl        Sn[        US5      (       aF   UR                  5         [        [        USS5      5      n[        USS5      SLnU(       d  U(       a  Sn[        US	S5      nUc  M  [        US
5      (       d  M   [        US
S5      SLa  SUl        SnM  M     XU4$ ! [         a     Nf = f! [         a     Nf = f! [         a     Nuf = f! [         a     GM!  f = f)z
Enable gradient checkpointing and disable use_cache across wrapped decoder modules.

Returns:
    Tuple[checkpointing_enabled, cache_disabled, input_grads_enabled]
Fgradient_checkpointing_enableTgradient_checkpointingenable_input_require_grads!_acestep_input_grads_hook_enabled_require_grads_hookNconfig	use_cache)	r   r   r   	Exceptionr   r   boolr   r   )r   checkpointing_enabledcache_disabledinput_grads_enabledmodhook_enabledhas_require_hookcfgs           r+   #_configure_training_memory_featuresr     sn    "N$W-3788113(,% S233-1*(,% 3455	..0#C!DeL  $+30Et#LTX#X #3*.' c8T*?wsK883T2%?$)CM%)N @? .J !2EEEA        sH   C5	D'ADD%5
DD
DD
D"!D"%
D43D4c                 r    [         R                  " SUR                  S   U 4UR                  S9nX   nUnX44$ )aM  Sample timesteps from discrete turbo shift=3 schedule.

For each sample in the batch, randomly select one of the 8 discrete timesteps
used by the turbo model with shift=3.0.

Args:
    bsz: Batch size
    device: Device
    dtype: Data type (should be bfloat16)

Returns:
    Tuple of (t, r) where both are the same sampled timestep
r   r*   )r&   randintrq   r    )bsztimesteps_tensorindicestrs        r+   sample_discrete_timestepr   N  sH     mm	!!!$sf5E5L5LG 	!A 	
A4Kr-   c            
          ^  \ rS rSrSrS\R                  S\S\S\	R                  S\	R                  4
U 4S jjr SS	\\\	R                  4   S
\S\	R                  4S jjrSrU =r$ )PreprocessedLoRAModuleih  a{  LoRA Training Module using preprocessed tensors.

This module trains only the DiT decoder with LoRA adapters.
All inputs are pre-computed tensors - no VAE or text encoder needed!

Training flow:
1. Load pre-computed tensors (target_latents, encoder_hidden_states, context_latents)
2. Sample noise and timestep
3. Forward through decoder (with LoRA)
4. Compute flow matching loss
r   lora_configtraining_configr    rA   c                 
  > [         T	U ]  5         X l        X0l        [	        U[
        5      (       a  [        R                  " U5      OUU l        [        U R                  5      U l	        [        U R                  5      U l        U R                  S;   U l        [        R                  " [        U R                  U R                  S9U l        SU l        [#        5       (       a  UR%                  5        Hv  nUR&                  R)                  5       Ul        UR+                  5       (       d  M9  [        R,                  " 5          UR&                  R)                  5       Ul        SSS5        Mx     [/        X5      u  U l        U l        [4        R6                  " SU R2                  S   S S35        O#Xl        0 U l        [4        R8                  " S	5        [;        U R2                  5      n[=        [        S
5      (       a}  U R                  S:X  am  U(       df   [4        R6                  " S5        [        R>                  " U R0                  R@                  SS9U R0                  l         [4        R6                  " S5        O4U(       a  [4        R6                  " S5        O[4        R6                  " S5        URD                  U l"        / U l#        g! , (       d  f       GM  = f! [B         a$  n[4        R8                  " SU S35         SnANUSnAff = f)zInitialize the training module.

Args:
    model: The AceStepConditionGenerationModel
    lora_config: LoRA configuration
    training_config: Training configuration
    device: Device to use
    dtype: Data type to use
r0   r    rA   FNzLoRA injected: trainable_params, trainable paramsz2PEFT not available, training without LoRA adapterscompiler1   zCompiling DiT decoder...default)modeztorch.compile successfulztorch.compile failed (z!), continuing without compilationz=Skipping torch.compile (incompatible with PEFT LoRA adapters)zDtorch.compile not available on this device/PyTorch version, skipping)$super__init__r   r   r%   r(   r&   r    r,   r.   r8   rA   transfer_non_blockingtensorTURBO_SHIFT3_TIMESTEPSr   #force_input_grads_for_checkpointingr   r>   rC   cloneis_inferencerB   r   r   	lora_infor	   infowarningr   r   r   r   r   r   training_losses)
selfr   r   r   r    rA   paramhas_pefte	__class__s
            r+   r   PreprocessedLoRAModule.__init__u  s5   " 	&..8.E.Eell6*61$++>*4+;+;<
%)%5%5%H" %"4;;djj!
 490  !!))+"ZZ--/
%%''%*ZZ%5%5%7
 ) , *>e)Q&DJKK!$..1C"DQ!GGXY JDNNNOP '5)$$)9)9V)CH67%*]]4::3E3EI%V

"67 S Z
 ll  "S ).  ,QC/PQ s%   = KA$K 
K	
LK==Lbatchrecord_lossr!   c                    U R                   S;   a*  [        R                  " U R                   U R                  S9nO
[	        5       nU   US   R                  U R                  U R                  U R                  S9nUS   R                  U R                  U R                  U R                  S9nUS   R                  U R                  U R                  U R                  S9nUS   R                  U R                  U R                  U R                  S9nUS   R                  U R                  U R                  U R                  S9nUR                  S	   n	[        R                  " U5      n
Un[        XR                  5      u  pUR                  S
5      R                  S
5      nX-  SU-
  U-  -   nU R                  (       a  UR                  S5      nU R                  R!                  UUUUUUUS9nX-
  n["        R$                  " US	   U5      nSSS5        WR'                  5       nU(       a)  U R(                  R+                  UR-                  5       5        U$ ! , (       d  f       NP= f)aY  Single training step using preprocessed tensors.

Note: This is a distilled turbo model, NO CFG is used.

Args:
    batch: Dictionary containing pre-computed tensors:
        - target_latents: [B, T, 64] - VAE encoded audio
        - attention_mask: [B, T] - Valid audio mask
        - encoder_hidden_states: [B, L, D] - Condition encoder output
        - encoder_attention_mask: [B, L] - Condition mask
        - context_latents: [B, T, 128] - Source context
    record_loss: If True, append loss to training_losses (set False for validation).

Returns:
    Loss tensor (float32 for stable backward)
r1   r2   r3   r.   rA   target_latentsrA   non_blockingattention_maskencoder_hidden_statesencoder_attention_maskcontext_latentsr   r   Thidden_statestimestep
timestep_rr   r   r   r   Nr.   r&   autocastrA   r
   tor    r   rq   
randn_liker   r   	unsqueezer   requires_grad_r   r   Fmse_lossrD   r   ro   rj   )r   r   r   autocast_ctxr   r   r   r   r   r   x1x0r   _t_xtdecoder_outputsflowdiffusion_losss                      r+   training_step$PreprocessedLoRAModule.training_step  sP   , 55 >> ,,DJJL '=L"#34774::D<V<V 8 N ##34774::D<V<V 8 N %**A$B$E$E4::D<V<V %F %! &++C%D%G%G4::D<V<V &H &" $$56994::D<V<V : O !&&q)C !!.1BB ,C1F1FGDAR**2.B C"H?*B77&&t, #jj00 -&;'= / 1 O 7DZZ(:DAN] b (--/  ''(;(;(=>m \s   GI
I$)r   r    r.   rA   r   r   r   r   r   r   r   r   )T)__name__
__module____qualname____firstlineno____doc__r   r   r   r   r&   r    rA   r   r   r(   Tensorr   r   __static_attributes____classcell__r   s   @r+   r   r   h  s    
P"yyP"  P" (	P"
 P" {{P"j !RC%&R R 
	R Rr-   r   c                       \ rS rSrSrS\S\4S jr  SS\S\	\
   S	\	\   S
\\\\\4   SS4   4S jjr SS\S\	\
   S	\	\   S
\\\\\4   SS4   4S jjrS\S\	\
   S
\\\\\4   SS4   4S jrS rSrg)LoRATraineri  zHigh-level trainer for ACE-Step LoRA fine-tuning.

Uses Lightning Fabric for distributed training and mixed precision.
Supports training from preprocessed tensor directories.
r   r   c                     Xl         X l        [        UR                  5      Ul        X0l        SU l        SU l        SU l        g)zInitialize the trainer.

Args:
    dit_handler: Initialized DiT handler (for model access)
    lora_config: LoRA configuration
    training_config: Training configuration
NF)dit_handlerr   r   
output_dirr   r<   fabricis_training)r   r  r   r   s       r+   r   LoRATrainer.__init__#  s?     '&%./I/I%J". r-   N
tensor_dirtraining_stateresume_fromr!   c              #     #    SU l          [        U R                  SS5      nUb  SSSU S34v    SU l         g [        U5      n[
        R                  R                  U5      (       d  SSS
U 34v    SU l         g[        R                  " U R                  R                  5        [        R                  " U R                  R                  5        [        R                  R                  5       (       a3  [        R                  R                  U R                  R                  5         SSKnUR                  R                  U R                  R                  5        [%        U R                  R&                  U R(                  U R                  U R                  R*                  U R                  R,                  S9U l        [1        U R.                  R&                  R2                  5      u  pgnX`R.                  l        [6        R8                  " SU SU SU 35        [;        UU R                  R<                  U R                  R>                  U R                  R@                  U R                  RB                  U R                  RD                  U R                  RF                  [        U R                  SS5      S9n	U	RI                  S5        [K        U	RL                  5      S:X  a  Sv    SU l         gSSS[K        U	RL                  5       S34v   U(       a  Sv   OSv   U(       d  Sv   [N        (       a  U RQ                  XU5       Sh  vN   OU RS                  X5       Sh  vN    SU l         g! [         a    SSS	U 34v    SU l         gf = f! ["         a     GN@f = f NY NA! ["         a3  n
[6        RT                  " S5        SSS[W        U
5       34v    Sn
A
NvSn
A
ff = f! SU l         f = f7f)am  Train LoRA adapters from preprocessed tensor files.

This is the recommended training method for best performance.

Args:
    tensor_dir: Directory containing preprocessed .pt files
    training_state: Optional state dict for stopping control
    resume_from: Optional path to checkpoint directory to resume from

Yields:
    Tuples of (step, loss, status_message)
TquantizationNr           uL   ❌ LoRA training requires a non-quantized DiT model. Current quantization: M. Re-initialize service with INT8 Quantization disabled, then retry training.F&   ❌ Rejected unsafe tensor directory:     ❌ Tensor directory not found: )r   r   r   r    rA   1Training memory features: gradient_checkpointing=, use_cache_disabled=, input_grads_enabled=	val_splitr	  
batch_sizenum_workers
pin_memoryprefetch_factorpersistent_workerspin_memory_devicer  fitr   r  u.   ❌ No valid samples found in tensor directory   📂 Loaded  preprocessed samplesr   r  u/   🧠 Gradient checkpointing enabled for decoderr   r  uK   ⚠️ Gradient checkpointing not enabled (model wrapper did not expose it)r   r  uW   ℹ️ Input-grad hook not available on this DiT; using explicit checkpointing fallbackzTraining failed   ❌ Training failed: ),r  r   r  r   
ValueErrorospathisdirr&   manual_seedr   seedrandomr1   is_availablemanual_seed_allnumpyr   r   r   r   r    rA   r<   r   r   r   r	   r   r   r  r  r  r  r  r  setupre   train_datasetLIGHTNING_AVAILABLE_train_with_fabric_train_basic	exceptionr(   )r   r	  r
  r  quantization_modenpckpt_enabledr   r   data_moduler   s              r+   train_from_preprocessed#LoRATrainer.train_from_preprocessed:  s    $  k	% !((8(8.$ O ,11B0C Dff	  z  %Du&z2
 77==,, @MMMf  %Da d22778KK,,112zz&&((

**4+?+?+D+DE"		t33889 1&&,, ,, $ 4 4''..&&,,DK 4DKK4E4E4M4MN >L*= ?KKK;KKCL> R&&4%55KL_K`b 1%//:: 00<<//:: $ 4 4 D D#'#7#7#J#J"&"6"6"H"H!$"6"6SI	K e$;,,-2NNB  %D= s;#<#<=>>ST 
 OO 
 '  #"22    ,,[III  %Dq   FzlSSSl  %Dq   | J 	;./S1#a&:::	;  %Ds   O.%N" O.M, .N" 4O.<B*N" '3N FN" #O.+AN" NN" 	O" 
N" N N" #O" $O.,N
 N" O.	N

N" 
NN" NN"  N" "
O,)OO" OO" "	O++O.r8  c              #   (  #    [         R                  " U R                  R                  SS9  U R                  R
                  n[        U5      nUS;   a  UOSnSn [        U R                  R                  SS9nUS	US
.n	Ub  U/U	S'   [        Se0 U	D6U l        U R                  R                  5         SSSU SU S34v   US:X  d  UR                  S5      (       aU  U R                  R                  R                   R#                  [$        R&                  S9U R                  R                  l        OZU R                  R                  R                   R#                  U R                  R(                  S9U R                  R                  l        [+        U R                  R                  R                   5      u  p[        R,                  " SU
 SU S35        UR/                  5       n[1        US5      (       a  UR3                  5       OSnUb  / US'   / US'   / US'   / US'   / US'   SUS'   SnSn[5        S5      nSnU R                  R                  R7                  5        Vs/ s H  nUR8                  (       d  M  UPM     nnU(       d  S v   gSSS![;        S" U 5       5      S# S$34v   U R                  R<                  U R                  R>                  S%.n[@        (       a>  US&:X  a8  [        R,                  " S'5        [B        RD                  RF                  " U40 UD6nO5U R                  RH                  RJ                  S&:X  a  SUS('   [M        U40 UD6n[O        S	[P        RR                  " [U        U5      U R                  RV                  -  5      5      nUU R                  RX                  -  n[[        U R                  R\                  [O        S	US)-  5      5      n[_        USS*US+9n[a        U[O        S	UU-
  5      S	U R                  R<                  S,-  S-9n[c        UUU/U/S.9nU R                  Re                  U R                  R                  R                   U5      u  U R                  R                  l        n[g        U5      u  nn[        R,                  " S/U SU S35        U R                  Ri                  U5      nSnSnSn U(       a   [k        U5      nU(       Ga  [         Rn                  Rq                  U5      (       Ga   SSS2U S334v   [s        UUUU R                  RH                  S49n U S5   (       Ga  U S5   n![         Rn                  Ru                  U!S65      n"[         Rn                  Rq                  U"5      (       d   [         Rn                  Ru                  U!S75      n"[         Rn                  Rq                  U"5      (       a  SS8K;J<n#  U"R                  S95      (       a	  U#" U"5      n$O*[$        Rz                  " U"U R                  RH                  SS:9n$U R                  R                  R                   n%[1        U%S;5      (       a  U%R|                  n%U%R                  U$S<S=9  U S>   nU S?   nS@U SAU 3/n&U SB   (       a  U&R                  SC5        U SD   (       a  U&R                  SE5        SSSFRu                  U&5      4v   O(SSSGU! 34v   OSSSHU 34v   OU(       a  SSSKU S134v   Sn'Sn(UR                  SSL9  U R                  R                  R                   R                  5         [        UU R                  RX                  5       GH	  n)Sn*Sn+[        R                  " 5       n,[        U5       GHw  u  n-n.U(       a.  UR                  SMS<5      (       a  UU([O        U'S	5      -  SN4v       gU R                  R                  U.5      n/U/U R                  RV                  -  n/U R                  R                  U/5        U(U/R                  5       -  n(U'S	-  n'U'U R                  RV                  :  d  M  [        U5      u  n0n1U0S:  a-  UR                  SSL9  U[5        SO5      SPU0 SU1 SQ34v   Sn(Sn'GM   U R                  R                  U R                  R                  R                   UU R                  R                  S<SR9  UR                  5         UR                  5         UR                  SSL9  US	-  nU(U'-  n2UU R                  R                  -  S:X  a  UbP  Uc  U2nOUU2-  S	U-
  U-  -   nUS   R                  U5        US   R                  U25        US   R                  U5        U R                  R                  SSU2UST9  U R                  R                  SUUR                  5       S   UST9  UU2SVU)S	-    SU R                  RX                   SWU SXU2SY 34v   U*U2-  n*U+S	-  n+Sn(Sn'GMz     U'S:  Ga  [        U5      u  n0n1U0S:  a+  UR                  SSL9  U[5        SO5      SPU0 SU1 SZ34v   Sn(Sn'O}U R                  R                  U R                  R                  R                   UU R                  R                  S<SR9  UR                  5         UR                  5         UR                  SSL9  US	-  nU(U'-  n2UU R                  R                  -  S:X  a  UbP  Uc  U2nOUU2-  S	U-
  U-  -   nUS   R                  U5        US   R                  U25        US   R                  U5        U R                  R                  SSU2UST9  U R                  R                  SUUR                  5       S   UST9  UU2SVU)S	-    SU R                  RX                   SWU SXU2SY 34v   U*U2-  n*U+S	-  n+Sn(Sn'[        R                  " 5       U,-
  n3U*[O        U+S	5      -  n4Ube  Uc  U4nOUU4-  S	U-
  U-  -   nUS   n5U5(       a	  U5S[   U:w  a<  US   R                  U5        US   R                  U45        US   R                  U5        U R                  R                  S\U4U)S	-   ST9  UGba  U R                  R                  R                   R                  5         Sn6Sn7[$        R                  " 5          U H5  n8U R                  R                  U8S<S]9n9U6U9R                  5       -  n6U7S	-  n7M7     SSS5        U R                  R                  R                   R                  5         U6[O        U7S	5      -  n:Ub(  US   R                  U5        US   R                  U:5        U:U:  ah  U:nUnUb  UUS'   [         Rn                  Ru                  U R                  R                  S^S_5      n;[        U R                  R                  UUU)S	-   UU;5        U)S	-   U R                  R                  -  S:X  d  GM  [         Rn                  Ru                  U R                  R                  S^S`U)S	-    SaU4SY 35      n<[        U R                  R                  UUU)S	-   UU<5        UU4SbU)S	-    34v   GM     [         Rn                  Ru                  U R                  R                  Sc5      n=[        U R                  R                  U=5        U R                  R                  (       a  U R                  R                  S[   OSn>UU>SdU= 34v   g! [         a$  n[        R                  " SU 35         SnAGNSnAff = fs  snf ! [l         a    SSS0U S134v   Sn G
Nif = f! [         a0  n[        R                  " SI5        SSSJU S134v   SnSn SnAGNSnAff = f! , (       d  f       GN= f7f)fzTrain using Lightning Fabric.Texist_okr1   r2   r3   cpuautoNlogsroot_dirr^   ;TensorBoard logger unavailable, continuing without logger: r$   acceleratordevices	precisionloggersr   r      🚀 Starting training (device: , precision: )...r3   -mixedrA   %Trainable tensor dtype fixup: casted / to fp32val_dataloader
plot_steps	plot_lossplot_emaplot_val_stepsplot_val_lossplot_best_step皙?infr   r  u"   ❌ No trainable parameters found!   🎯 Training c              3   @   #    U  H  oR                  5       v   M     g 7fNrl   .0rG   s     r+   	<genexpr>1LoRATrainer._train_with_fabric.<locals>.<genexpr>        E4Dq4D   r    parameterslrweight_decayr1   z:train_with_fabric using bitsandbytes 8-bit AdamW optimizerfused
   r   start_factor
end_factortotal_iters{Gz?T_0T_multeta_min
schedulers
milestones$Optimizer param dtype fixup: casted u(   ⚠️ Rejected unsafe checkpoint path: z, starting freshu   🔄 Loading checkpoint from z...)rR   	schedulerr    adapter_pathzadapter_model.safetensorszadapter_model.bin)	load_filez.safetensors)map_locationweights_onlyr   F)strictepochglobal_stepu   ✅ Resumed from epoch , step loaded_optimizeru   optimizer ✓loaded_scheduleru   scheduler ✓z, u$   ⚠️ Adapter weights not found in u$   ⚠️ No valid checkpoint found in zFailed to load checkpointu"   ⚠️ Failed to load checkpoint: u"   ⚠️ Checkpoint path not found: set_to_noneshould_stop   ⏹️ Training stopped by userrd      ⚠️ Non-finite gradients (z); skipping optimizer stepmax_normerror_if_nonfinite
train/losssteptrain/lrEpoch , Step , Loss: .4fz$); skipping optimizer remainder stepr   train/epoch_loss)r   checkpointsbestepoch__loss_   💾 Checkpoint saved at epoch final%   ✅ Training complete! LoRA saved to r:   )Yr&  makedirsr   r  r<   r.   r;   r   ModuleNotFoundErrorr	   r   r   r  launchendswithr   r   r   r&   r6   rA   rH   r   train_dataloaderr   rR  rD   r>   r?   ri   learning_rateri  HAS_BNBbnboptim	AdamW8bitr    r'   r   rn   mathceilre   gradient_accumulation_steps
max_epochsminwarmup_stepsr   r   r   r/  rW   setup_dataloadersr   r%  r'  existsr   joinsafetensors.torchr{  loadr   load_state_dictro   r   r4  	zero_gradtrainrangetime	enumeraterU   r   backwardrj   rQ   clip_gradientsmax_grad_normr  log_every_n_stepslogget_last_lrevalrB   r   save_every_n_epochsr   r   )?r   r8  r
  r  r.   rH  rF  	tb_loggerr   fabric_kwargscasted_trainabletotal_trainable_tensorstrain_loader
val_loaderema_loss	ema_alphabest_val_lossbest_val_steprG   r   optimizer_kwargsrR   steps_per_epochtotal_stepsr  warmup_schedulermain_schedulerry  casted_opt_paramstotal_opt_paramsstart_epochr  checkpoint_inforz  adapter_weights_pathr{  
state_dictr   status_partsaccumulation_stepaccumulated_lossr  
epoch_lossnum_updatesepoch_start_time
_batch_idxr   lossnonfinite_gradsgrad_tensorsavg_loss
epoch_timeavg_epoch_lossrS  total_val_lossn_val	val_batchv_lossval_lossbest_dircheckpoint_dir
final_path
final_losss?                                                                  r+   r2  LoRATrainer._train_with_fabric  s     	D((33dCkk--,[9	&*GGKV 	
 		)--88vI '"

  (1{M)$-}- .{m=SWX
 	
 %9#5#5h#?#?(,(9(9(A(A(D(Dmm )E )DKK% )-(9(9(A(A(D(Dkk'' )E )DKK% 5RKK%%5
1 	34D3EQG^F__gh	

 #335 {$455 &&( 	 %+-N<(*,N;')+N:&/1N+,.0N?+/3N+,	e {{((335
5!A5 	 
  >> S E4D EEaHT
 	
 &&44 00==
 7{f,KKTU		++,<Q@PQI{{!!&&&0,0 ).C2BCI IIL!D$8$8$T$TT
 &(<(<(G(GG4//<<c![TVEV>WX $$	
 5A{\12((66=	
 !(.9$~
	 04{{/@/@KK%%y0
,!9 /LI.V++23D2EQGWFXX`a	
 {{44\B #'4 277>>+66=  =k]#NNN #;'';;--	# #>22#2>#BL+-77<<$&A,( 77>>*>??/1ww||(*=0, ww~~&:;;?/88HH)23G)HJ). 4-1[[-?-?-1*J #'++"3"3";";"7,=>>&-&=&=G//
5/I&5g&>&5m&D 6k]'+W( ++=>(//@*+=>(//@dii&===(L\N&[[[S$H"VVV S>{mK[\\\ -!!'');(<(<(G(GHEJK#yy{%.|%<!
E!n&8&8&N&N#(3/@!+DD9 
  {{007d22NNN $$T* DIIK/ !Q&! &++GGH 5K(51O\ '*!+++='!%L"??PPQR^Q_ `: !:	  ,/(,-) KK..))11!!%!5!5!C!C+0	 /  NN$NN$''D'91$K  02CCH"T%9%9%K%KKqP)5'/+3 %.$8A	MX;U$U !) +<8??L*;7>>xH*:6==hGh[Q&	(=(=(?(B (  ($$UQYKq1E1E1P1P0QQXYdXeemnvwzm{|  (*J1$K'*$()%a &=h !1$0FGW0X-"Q&''D'9#e;O;LAl^ \@ @	  (+$()%KK..))11!!%!5!5!C!C+0	 /  NN$NN$''D'9q +.??!5!5!G!GG1L%1#+'/H'08';q9}PX>X'XH&|4;;KH&{3::8D&z299(CKKOOL(OMKKOO"I$9$9$;A$>[ $  $  1T-A-A-L-L,MWU`Taaijrsviwx  (*J1$K'*$()% '77J'#k1*==N)#-H(>9Q]h<VVH+L9
!Z^{%B"<077D";/66~F":.55h?KKOO.UQYOO %!!))..0!$]]_%/	!%!:!:9RW!:!X&&++-7
 &0 %
 !!))//1)CqM9!-"#34;;KH"?3::8Dm+$,M$/M%1;H'78!ww||,,77 H -))!!	#  	T11EEEJ!#((33]fUUVYKW]^lmp]qDr" )KK%%AI"  "5eai[A U Ib WW\\$"6"6"A"A7K
$++++Z8 04{{/J/JDKK''+PS 	 3J<@
 	
c # 	NNMaSQ 	z
Z  #>{mK[\ 
 ##B     !<= B1#EUVVV	 b %_s   A~{/ 8G%~| 6| <I~|% -~F>} ~
} ~
} E~3Q2~%<~ !C<~"D~/
|9|~|~%} <~?}  ~
}=%}82~8}==~ 
~	
~c              #   d  #    Sv   [         R                  " U R                  R                  SS9  UR	                  5       nU R
                  R                  R                  5        Vs/ s H  oDR                  (       d  M  UPM     nnU(       d  Sv   g[        (       ax  U R
                  R                  S:X  a^  [        R                  R                  UU R                  R                  U R                  R                  S9n[         R"                  " S5        O3[%        UU R                  R                  U R                  R                  S9n['        S	[(        R*                  " [-        U5      U R                  R.                  -  5      5      nXpR                  R0                  -  n[3        U R                  R4                  ['        S	US
-  5      5      n	[7        USSU	S9n
[9        U['        S	X-
  5      S	U R                  R                  S-  S9n[;        UX/U	/S9nSnSnSnUR=                  SS9  U R
                  R                  R>                  RA                  5         [C        U R                  R0                  5       GH  nSnSn[D        RD                  " 5       nU GHd  nU(       a.  URG                  SS5      (       a  UU['        US	5      -  S4v       gU R
                  RI                  U5      nUU R                  R.                  -  nURK                  5         UURM                  5       -  nUS	-  nUU R                  R.                  :  d  M  [N        RP                  RR                  RU                  XPR                  RV                  5        URY                  5         URY                  5         UR=                  SS9  US	-  nX-  nXR                  RZ                  -  S:X  a  UUSUS	-    SU SUS 34v   UU-  nUS	-  nSnSnGMg     US:  a  [N        RP                  RR                  RU                  XPR                  RV                  5        URY                  5         URY                  5         UR=                  SS9  US	-  nX-  nXR                  RZ                  -  S:X  a  UUSUS	-    SU SUS 34v   UU-  nUS	-  nSnSn[D        RD                  " 5       U-
  nU['        US	5      -  nUUSUS	-    SU R                  R0                   SUS S34v   US	-   U R                  R\                  -  S:X  d  GM  [         R^                  Ra                  U R                  R                  S S!US	-    S"US 35      n[c        U R
                  R                  U5        UUS#4v   GM     [         R^                  Ra                  U R                  R                  S$5      n[c        U R
                  R                  U5        U R
                  Rd                  (       a  U R
                  Rd                  S%   OSnUUS&U 34v   gs  snf 7f)'z#Basic training loop without Fabric.r   r  u$   🚀 Starting basic training loop...Tr<  r[  Nr1   rg  z4train_basic using bitsandbytes 8-bit AdamW optimizerr$   rk  rY  r   rl  rp  rq  ru  r   r  r  r  F   ⏹️ Training stoppedr  r  r  r  
   ✅ Epoch rP   in .1fsr  r  r     💾 Checkpoint savedr  r   r  )3r&  r  r   r  r  r<   r   r>   r?   r  r.   r  r  r  r  ri  r	   r   r   rn   r  r  re   r  r  r  r  r   r   r   r  r   r  r  r  rU   r   r  rj   r&   r   utilsclip_grad_norm_r  r  r  r  r'  r  r   r   )r   r8  r
  r  rG   r   rR   r  r  r  r  r  ry  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  s                               r+   r3  LoRATrainer._train_basic  s     =<
D((33dC"335 {{((335
5!A5 	 
  >>7t{{..&8		++ ''55!11>> , I
 KKNO ''55!11>>I IIL!D$8$8$T$TT
 &(<(<(G(GG4//<<c![TVEV>WX#CC\
 5A{12((66=	
 !(9$~
	 -!!'')4//::;EJK#yy{%!n&8&8&N&N#(3/@!+DD1 
 {{007d22NNN DIIK/ !Q&! &++GGH HHNN22(*>*>*L*L NN$NN$''D'91$K/CH"%9%9%K%KKqP'$$UQYKw{m8HUX>Z  (*J1$K'*$()%M &P !1$..$&:&:&H&H   ###5q +?!5!5!G!GG1L#  7;-xQT~V  h&
q #& $%!'77J'#k1*==NUQYKq)=)=)H)H(IjY\M]]^_  	T11EEEJ!#((33]fUUVYKW]^lmp]qDr" "$++"3"3^D!>3JJJa <d WW\\$"6"6"A"A7K
$++++Z8/3{{/J/JDKK''+PS 	 3J<@
 	
Y
s'   A$X0&X+>X+J?X0GX0+DX0c                     SU l         gzStop training.FNr  r   s    r+   stopLoRATrainer.stop&  
     r-   )r  r  r  r   r<   r   )NNr^  )r   r   r   r   r   r   r   r   r(   r   r   r   r   rg   rD   r9  r   r2  r3  r  r   r:   r-   r+   r  r    s   !  ! (	!4 *.%)	%% !% c]	%
 
5eS)45	6%J &*	K
+K
 !K
 c]	K

 
5eS)45	6K
Z\
+\
 !\
 
5eS)45	6	\
|!r-   r  c            
          ^  \ rS rSrSrS\R                  S\S\S\	R                  S\	R                  4
U 4S jjrS	\\\	R                  4   S
\	R                  4S jrSrU =r$ )PreprocessedLoKRModulei+  z0LoKr training module using preprocessed tensors.r   lokr_configr   r    rA   c                   > [         TU ]  5         X l        X0l        [	        U[
        5      (       a  [        R                  " U5      OUU l        [        U R                  5      U l	        [        U R                  5      U l        U R                  S;   U l        [        R                  " [        U R                  U R                  S9U l        SU l        S U l        [%        5       (       aG  ['        X5      u  U l        U l        U l        [,        R.                  " SU R*                  S   S S35        O#Xl        0 U l        [,        R0                  " S5        UR2                  U l        / U l        g )	Nr0   r   FzLoKr injected: r   r   r   z5LyCORIS not available, training without LoKr adapters)r   r   r  r   r%   r(   r&   r    r,   r.   r8   rA   r   r   r   r   r   r|   r   r   r   	lokr_infor	   r   r   r   r   )r   r   r  r   r    rA   r   s         r+   r   PreprocessedLoKRModule.__init__.  s    	&..8.E.Eell6*61$++>*4+;+;<
%)%5%5%H" %"4;;djj!
 490"$$;O<8DJ($. KK!$..1C"DQ!GGXY JDNNNRSll!r-   r   r!   c                    U R                   S;   a*  [        R                  " U R                   U R                  S9nO
[	        5       nU   US   R                  U R                  U R                  U R                  S9nUS   R                  U R                  U R                  U R                  S9nUS   R                  U R                  U R                  U R                  S9nUS   R                  U R                  U R                  U R                  S9nUS   R                  U R                  U R                  U R                  S9nUR                  S	   n[        R                  " U5      n	Un
[        XR                  5      u  pUR                  S
5      R                  S
5      nX-  SU-
  U
-  -   nU R                  (       a  UR                  S5      nU R                  R!                  UUUUUUUS9nX-
  n["        R$                  " US	   U5      nSSS5        WR'                  5       nU R(                  R+                  UR-                  5       5        U$ ! , (       d  f       NI= f)zSingle LoKr training step.r   r   r   r   r   r   r   r   r   r   r   Tr   Nr   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   s                     r+   r   $PreprocessedLoKRModule.training_stepS  sC   55 >> ,,DJJL '=L"#34774::D<V<V 8 N ##34774::D<V<V 8 N %**A$B$E$E4::D<V<V %F %! &++C%D%G%G4::D<V<V &H &" $$56994::D<V<V : O !&&q)C!!.1BB+C1F1FGDAR**2.BC"H?*B77&&t,"jj00 -&;'= / 1 O 7DZZ(:DANM P (--/##N$7$7$9:U \s   GI
I)r   r    r.   rA   r   r  r  r|   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r&   r    rA   r   r   r(   r   r   r   r   r   s   @r+   r  r  +  sr    :#"yy#"  #" (	#"
 #" {{#"J34U\\(9#: 3u|| 3 3r-   r  c                       \ rS rSrSrS\S\4S jr SS\S\	\
   S	\\\\\4   SS4   4S
 jjrS\S\	\
   S	\\\\\4   SS4   4S jrS\S\	\
   S	\\\\\4   SS4   4S jrS rSrg)LoKRTraineri  z1High-level trainer for ACE-Step LoKr fine-tuning.r  r   c                     Xl         X l        [        UR                  5      Ul        X0l        S U l        S U l        SU l        0 U l        g )NF)	r  r  r   r  r   r<   r  r  run_metadata)r   r  r  r   s       r+   r   LoKRTrainer.__init__  sG     '&%./I/I%J". ,.r-   Nr	  r
  r!   c              #     #    SU l          [        U R                  R                  5      (       a  [        R
                  " S5        [        U R                  SS5      nUb  SSSU S34v    U R                  b:  [        U R                  S	5      (       a  [        U R                  R                  5        [        U S
S5      b7  [        U R                  S	S5      b  [        U R                  R                  5        SU l         g [        U5      n[        R                  R                  U5      (       d  SSSU 34v    U R                  b:  [        U R                  S	5      (       a  [        U R                  R                  5        [        U S
S5      b7  [        U R                  S	S5      b  [        U R                  R                  5        SU l         g[        5       (       d  Sv    U R                  b:  [        U R                  S	5      (       a  [        U R                  R                  5        [        U S
S5      b7  [        U R                  S	S5      b  [        U R                  R                  5        SU l         g[        R                   " U R"                  R$                  5        [&        R$                  " U R"                  R$                  5        [        R(                  R+                  5       (       a3  [        R(                  R-                  U R"                  R$                  5         SSKnUR&                  R%                  U R"                  R$                  5        [3        U R                  R                  U R4                  U R"                  U R                  R6                  U R                  R8                  S9U l        [;        U R                  R                  R<                  5      u  pVnXPR                  l        [        R
                  " SU SU SU 35        [A        UU R"                  RB                  U R"                  RD                  U R"                  RF                  U R"                  RH                  U R"                  RJ                  U R"                  RL                  U R"                  RN                  S9nURQ                  S5        [S        URT                  5      S:X  a  Sv    U R                  b:  [        U R                  S	5      (       a  [        U R                  R                  5        [        U S
S5      b7  [        U R                  S	S5      b  [        U R                  R                  5        SU l         gU[W        [S        URT                  5      5      U R"                  RY                  5       S.U l-        SSS[S        URT                  5       S34v   U(       a  Sv   OSv   U(       d  Sv   [\        (       a  U R_                  X5       Sh  vN   OU Ra                  X5       Sh  vN    U R                  b:  [        U R                  S	5      (       a  [        U R                  R                  5        [        U S
S5      b7  [        U R                  S	S5      b  [        U R                  R                  5        SU l         g! [         a    SSSU 34v    U R                  b:  [        U R                  S	5      (       a  [        U R                  R                  5        [        U S
S5      b7  [        U R                  S	S5      b  [        U R                  R                  5        SU l         gf = f! [0         a     GN!f = f GNr GN[! [0         a4  n	[        Rb                  " S5        SSS[e        U	5       34v    Sn	A	GNSn	A	ff = f! U R                  b:  [        U R                  S	5      (       a  [        U R                  R                  5        [        U S
S5      b7  [        U R                  S	S5      b  [        U R                  R                  5        SU l         f = f7f)z.Train LoKr adapters from preprocessed tensors.Tz;Unwrapped stale Fabric decoder wrapper before LoKr trainingr  Nr   r  uL   ❌ LoKr training requires a non-quantized DiT model. Current quantization: r  r   r  Fr  r  )r   r  u>   ❌ LyCORIS not installed. Install lycoris-lora to train LoKr.)r   r  r   r    rA   r  r  r  r  r  r  )r	  num_samplesr   r  r   r!  r"  r#  zLoKr training failedr$  )3r  r   r  r   r	   r   r   r<   r   r   r%  r&  r'  r(  r   r&   r)  r   r*  r+  r1   r,  r-  r.  r   r  r  r    rA   r   r   r   r   r  r  r  r  r  r  r  r/  re   r0  rg   to_dictr  r1  r2  r3  r4  r(   )
r   r	  r
  r5  r6  r7  r   r   r8  r   s
             r+   r9  #LoKRTrainer.train_from_preprocessed  s      y	%+D,<,<,B,BCCQ !((8(8.$ O ,11B0C Dff	  B {{&74;;+H+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$DM&z2
 77==,, @MMMp {{&74;;+H+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$D{ +,, 
 ` {{&74;;+H+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$Dk d22778KK,,112zz&&((

**4+?+?+D+DE"		t33889 1&&,, ,, $ 4 4''..&&,,DK 4DKK4E4E4M4MN >L*= ?KKK;KKCL> R&&4%55KL_K`b
 1%//:: 00<<//:: $ 4 4 D D#'#7#7#J#J"&"6"6"H"H..88	K e$;,,-2NNJ {{&74;;+H+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$DS )"3{'@'@#AB#'#7#7#?#?#A!D s;#<#<=>>ST 
 OO 
 '  #"22;OOO,,[III {{&74;;+H+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$DI   FzlSSSv {{&74;;+H+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$DI.  | PI 	;34S1#a&::::	; {{&74;;+H+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$Ds   aA]? +Ba Z; .]? :Ba]? "Ba6B*]? !3]( F]? Ba0B]? ]9]? _  ]? !]<"]? &_  'Ba;]%]? Ba$]%%]? (
]62]? 5]66]? <]? ?
^=	)^82_  8^==_   Baar8  c              #   T  #    [         R                  " U R                  R                  SS9  U R                  R
                  n[        U5      nUS;   a  UOSnUR                  S5      (       + nS n [        U R                  R                  SS9nUS	US
.n	Ub  U/U	S'   [        SR0 U	D6U l        U R                  R                  5         SSSU SU S34v   U(       d  [        R                  " S5        US:X  d  UR                  S5      (       aU  U R                  R                   R"                  R%                  [&        R(                  S9U R                  R                   l        OZU R                  R                   R"                  R%                  U R                  R*                  S9U R                  R                   l        [-        U R                  R                   R"                  5      u  pUS:X  aA  [/        U R                  SS 5      b)  [-        U R                  R0                  5      u  pX-  n
X-  n[        R                  " SU
 SU S35        UR3                  5       n[5        U R                  R                   [/        U R                  SS 5      5      n[7        U R                  R                   [/        U R                  SS 5      5      nU(       d  Sv   g US:X  a  [        R                  " S5        SSS[9        S U 5       5      S S34v   U R                  R:                  U R                  R<                  S.nU R                  R>                  R@                  S:X  a  SUS '   [C        U40 UD6n[E        S	[F        RH                  " [K        U5      U R                  RL                  -  5      5      nUU R                  RN                  -  n[Q        U R                  RR                  [E        S	US!-  5      5      n[U        US"S#US$9n[W        U[E        S	UU-
  5      S	U R                  R:                  S%-  S&9n[Y        UUU/U/S'9nU R                  R[                  U R                  R                   R"                  U5      u  U R                  R                   l        n[]        U5      u  nn[        R                  " S(U SU S35        U R                  R_                  U5      nSnSnSnURa                  SS)9  U R                  R                   R"                  Rc                  5         [e        U R                  RN                  5       GHh  nSnSn [f        Rf                  " 5       n!U GHo  n"U(       a.  URi                  S*S+5      (       a  UU[E        US	5      -  S,4v       g U R                  Rk                  U"5      n#U#U R                  RL                  -  n#U R                  Rm                  U#5        UU#Ro                  5       -  nUS	-  nUU R                  RL                  :  d  M  U(       a  [q        UUS!S-9u  n$n%n&U$S:  as  U&(       a?  [        R                  " S.U$ SU% S/US	-    S0U S13	S2Rs                  S3 U& 5       5      -   5        URa                  SS)9  U[u        S45      S5U$ SU% S634v   SnSnGMK  U R                  Rw                  U R                  R                   R"                  UU R                  Rx                  S+S79  UR{                  5         UR{                  5         URa                  SS)9  US	-  nUU-  n'UU R                  R|                  -  S:X  ar  U R                  R                  S8U'US99  U R                  R                  S:UR                  5       S   US99  UU'S;US	-    SU R                  RN                   S<U S=U'S> 34v   UU'-  nU S	-  n SnSnGMr     US:  Ga  U(       a  [q        UUS!S-9u  n$n%n&U$S:  as  U&(       a?  [        R                  " S?U$ SU% S/US	-    S0U S13	S2Rs                  S@ U& 5       5      -   5        URa                  SS)9  U[u        S45      S5U$ SU% SA34v   SnSnGM)  U R                  Rw                  U R                  R                   R"                  UU R                  Rx                  S+S79  UR{                  5         UR{                  5         URa                  SS)9  US	-  nUU-  n'UU R                  R|                  -  S:X  ar  U R                  R                  S8U'US99  U R                  R                  S:UR                  5       S   US99  UU'S;US	-    SU R                  RN                   S<U S=U'S> 34v   UU'-  nU S	-  n SnSn[f        Rf                  " 5       U!-
  n(U[E        U S	5      -  n)U R                  R                  SBU)US	-   S99  UU)SCUS	-    SU R                  RN                   SDU(SE SFU)S> 34v   US	-   U R                  R                  -  S:X  d  GM  [         R                  Rs                  U R                  R                  SGSHUS	-    SIU)S> 35      n*[        U R                  R0                  UUUS	-   UU*U R                  U R                  SJ9  UU)SKUS	-    34v   GMk     [         R                  Rs                  U R                  R                  SL5      n+SMU R                  R                  5       0n,U R                  (       a  U R                  U,SN'   [        U R                  R0                  U+U,SO9  U R                  R                  (       a  U R                  R                  SP   OSn-UU-SQU+ 34v   g ! [         a$  n[        R                  " SU 35         S nAGNS nAff = f7f)SNTr<  r>  r@  rM  rA  rB  rD  r$   rE  rI  r   r  rJ  rK  rL  zpLoKr mixed precision detected: disabling pre-unscale non-finite grad checks; relying on AMP/GradScaler handling.r3   rN  r|   rO  rP  rQ  r[  zsLoKr trainable params discovered via LyCORIS fallback traversal; decoder parameter traversal returned 0 trainables.r\  c              3   @   #    U  H  oR                  5       v   M     g 7fr^  r_  r`  s     r+   rb  1LoKRTrainer._train_with_fabric.<locals>.<genexpr>{  rd  re  r   rf  rg  r1   rj  rk  rY  r   rl  rp  rq  ru  rx  r  r  Fr  )ra   zLoKr non-finite gradients (z) at epoch r  z. Top offending tensors:

c              3   ,   #    U  H
  nS U 3v   M     g7fz  - Nr:   ra  ds     r+   rb  r    s     /VDUq$qc
DU   rd   r  z6); skipping optimizer step (see logs for tensor names)r  r  r  r  r  r  r  r  z%LoKr non-finite remainder gradients (c              3   ,   #    U  H
  nS U 3v   M     g7fr  r:   r  s     r+   rb  r    s     +R@Q1d1#J@Qr  z@); skipping optimizer remainder step (see logs for tensor names)r  r  r  r  z	s, Loss: r  r  r  r  r  r  r  r  r  metadatar   %   ✅ Training complete! LoKr saved to r:   )Ir&  r  r   r  r<   r.   r;   r  r   r  r	   r   r   r  r  r   r   r   r   r&   r6   rA   rH   r   r|   r  r   r_   ri   r  ri  r    r'   r   rn   r  r  re   r  r  r  r  r   r   r   r/  rW   r  r  r  r  r  rU   r   r  rj   r{   r  rD   r  r  r  r  r  r  r  r'  r   r  r  r  r   r   ).r   r8  r
  r.   rH  rF  manual_nonfinite_checkr  excr  r  r  casted_fallbacktotal_fallbackr  r   r`   r  rR   r  r  r  r  r  ry  r  r  r  r  r  r  r  r  r  r   r  r  r  nonfinite_detailsr  r  r  r  r  final_metadatar  s.                                                 r+   r2  LoKRTrainer._train_with_fabric  s#    
 	D((33dCkk--,[9	&*GGKV 	 &/%7%7%A!A		)--88I '"

  (1{M)$-}- .{m=SWX
 	

 &KK6
 %9#5#5h#?#?(,(9(9(A(A(D(Dmm )E )DKK% )-(9(9(A(A(D(Dkk'' )E )DKK% 5RKK%%5
1 $q(]D9E.K''/+O /#5#34D3EQG^F__gh	
 #3359KKDKK5
 5KKDKK5

  >>"a'NNE S E4D EEaHT
 	
 &&44 00==
 ;;""f,(,W%*?.>?	IIL!D$8$8$T$TT
 &(<(<(G(GG4//<<c![TVEV>WX#$	
 5A{\12((66=	
 !(.9$~
	 04{{/@/@KK%%y0
,!9 /LI.V++23D2EQGWFXX`a	
 {{44\B-!!'')4//::;EJK#yy{%!n&8&8&N&N#(3/@!+DD9 
 {{007d22NNN$$T* DIIK/ !Q&! &++GGH .; 0 1-/ I7H +Q.0 &&A/ARRST`Saal',qykE_%a&*ii/VDU/V&V%W!"
 &//D/A + %e&COCTTUVbUc dZ %Z	#  03,01-$KK..))11!!%!5!5!C!C+0	 /  NN$NN$''D'91$K/2CCH"T%9%9%K%KKqPh[Q&	(=(=(?(B (  ($"(1T5I5I5T5T4U V((3}HXcN!L	  (*J1$K'*$()%] &` !1$)7,-)+ EO\3D '*,"NN"GGXXYZfYggr#(19+W[MA[!]"&))+R@Q+R"R!S
 "+++='!%L"??PPQR^Q_ `` !`	  ,/(,-) **KK%%--!11??',	 +    ###5q +.??!5!5!G!GG1LKKOOL(OMKKOO"I$9$9$;A$>[ $  $ $UQYKq1E1E1P1P0Q R$$/=#H	  h&
q #& $%!'77J'#k1*==NKKOO.UQYOO 1T-A-A-L-L,M N$S)>#2FH	  	T11EEEJ!#((33]fUUVYKW]^lmp]qDr" .KK++AI" $ 0 0!%!2!2	  "5eai[A Y <d WW\\$"6"6"A"A7K
*79I9I9Q9Q9S)T-1->->N>*KK###	
 04{{/J/JDKK''+PS 	 3J<@
 	
w	 # 	NNMcUS 	s?   A-n(0m7 Vn(Pn(En(7
n%n n( n%%n(c              #     #    Sv   [         R                  " U R                  R                  SS9  UR	                  5       n[        U R                  R                  [        U R                  SS 5      5      nU(       d  Sv   g [        UU R                  R                  U R                  R                  S9n[        S[        R                  " [        U5      U R                  R                   -  5      5      nX`R                  R"                  -  n[%        U R                  R&                  [        SUS-  5      5      n[)        US	S
US9n	[+        U[        SXx-
  5      SU R                  R                  S-  S9n
[-        UX/U/S9nSnSnSnUR/                  SS9  U R                  R                  R0                  R3                  5         [5        U R                  R"                  5       GH5  nSnSn[6        R6                  " 5       nU GHd  nU(       a.  UR9                  SS5      (       a  UU[        US5      -  S4v       g U R                  R;                  U5      nUU R                  R                   -  nUR=                  5         UUR?                  5       -  nUS-  nUU R                  R                   :  d  M  [@        RB                  RD                  RG                  X@R                  RH                  5        URK                  5         URK                  5         UR/                  SS9  US-  nX-  nXR                  RL                  -  S:X  a  UUSUS-    SU SUS 34v   UU-  nUS-  nSnSnGMg     US:  a  [@        RB                  RD                  RG                  X@R                  RH                  5        URK                  5         URK                  5         UR/                  SS9  US-  nX-  nXR                  RL                  -  S:X  a  UUSUS-    SU SUS 34v   UU-  nUS-  nSnSn[6        R6                  " 5       U-
  nU[        US5      -  nUUSUS-    SU R                  R"                   SUS S34v   US-   U R                  RN                  -  S:X  d  GM  [         RP                  RS                  U R                  R                  SSUS-    S US 35      n[U        U R                  RV                  UUUS-   UUU RX                  U RZ                  S!9  UUS"4v   GM8     [         RP                  RS                  U R                  R                  S#5      nS$U RX                  R]                  5       0nU RZ                  (       a  U RZ                  US%'   [_        U R                  RV                  UUS&9  U R                  R`                  (       a  U R                  R`                  S'   OSnUUS(U 34v   g 7f))Nr  Tr<  r|   r[  rg  r$   rk  rY  r   rl  rp  rq  ru  r   r  r  r  Fr  r  r  r  r  r  rP  r  r  r  r  r  r  r  r  r  r  r  r  r   r  )1r&  r  r   r  r  r   r<   r   r   r   r  ri  rn   r  r  re   r  r  r  r  r   r   r   r  r   r  r  r  rU   r   r  rj   r&   r   r  r  r  r  r  r  r'  r  r   r|   r  r  r  r   r   )r   r8  r
  r  r   rR   r  r  r  r  r  ry  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r!  r  s                               r+   r3  LoKRTrainer._train_basict  s    
 =<
D((33dC"3359KKDKK5
  >>##11--::
	
 IIL!D$8$8$T$TT
 &(<(<(G(GG4//<<c![TVEV>WX#CC\
 5A{12((66=	
 !(9$~
	 -!!'')4//::;EJK#yy{%!n&8&8&N&N#(3/@!+DD1 
 {{007d22NNN DIIK/ !Q&! &++GGH HHNN22(*>*>*L*L NN$NN$''D'91$K/CH"%9%9%K%KKqP'$$UQYKw{m8HUX>Z  (*J1$K'*$()%M &P !1$..$&:&:&H&H   ###5q +?!5!5!G!GG1L#  7;-xQT~V  h&
q #& $%!'77J'#k1*==NUQYKq)=)=)H)H(IjY\M]]^_  	T11EEEJ!#((33]fUUVYKW]^lmp]qDr" .KK++AI" $ 0 0!%!2!2	 ">3JJJs <v WW\\$"6"6"A"A7K
*79I9I9Q9Q9S)T-1->->N>*KK###	
 04{{/J/JDKK''+PS 	 3J<@
 	
s   J-W-3GW-EW-c                     SU l         gr  r  r  s    r+   r  LoKRTrainer.stop  r  r-   )r  r  r  r  r<   r  r   r^  )r   r   r   r   r   r   r   r   r(   r   r   r   r   rg   rD   r9  r   r2  r3  r  r   r:   r-   r+   r  r    s    ;/  / (	/( *.@%@% !@% 
5eS)45	6	@%DS
+S
 !S
 
5eS)45	6	S
j
_
+_
 !_
 
5eS)45	6	_
B!r-   r  r^  )   )Ur   r&  r  r+  r  typingr   r   r   r   r   r   logurur	   r&   torch.nnr   torch.nn.functional
functionalr   
contextlibr
   torch.optimr   torch.optim.lr_schedulerr   r   r   lightning.fabricr   lightning.fabric.loggersr   r1  ImportErrorr   bitsandbytesr  r  acestep.training.configsr   r   r   acestep.training.lora_injectionr   acestep.training.lora_utilsr    acestep.training.lora_checkpointr   r   r   acestep.training.lokr_utilsr   r   r   r   acestep.training.data_moduler   acestep.training.path_safetyr   r   r(   r,   rA   r8   r;   r   rg   rH   	ParameterrQ   r  	OptimizerrW   r_   r{   r   r   r   r   r   r   r   r  r  r  r:   r-   r+   <module>r=     s   
    > >      "  X X	':HG
 L K @ < 
  @ 2	 3 3 s u{{ # # ")) c3h &4(:(:#; &c3h &U[[-B-B uSRUX " <@
II
%-bii%8
	#s(^
  )/##$)/CH~)/ )/ 3T#Y	)/X7II7$,RYY$77	%((

78		 d *")) RYY <0F 0FuT4QUEU?V 0Ff4qRYY qhL! L!^[RYY [|N
! N
!]+  
NNP  HG
NNFGHs$   I "I/ I,+I,/JJ