
    Oi3                        d Z ddlZddlZddlZddlZddlmZmZmZm	Z	m
Z
mZ ddlmZ ddlZddlmZ ddlmc mZ ddlmZ ddlmZ ddlmZmZmZ 	 ddlmZ dd	lmZ d
Z 	 ddl#Z$d
Z%ddl&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z: g dZ;de	de<fdZ=de<dej|                  fdZ?de<de<fdZ@dej                  de
eBeBf   fdZCdeej                   j                     de
eBeBf   fdZEd ej                  j                  de
eBeBf   fd!ZH	 d7dej                  d"eej                     deeBe<f   fd#ZI	 d8deej                   j                     d$eeBe<f   d%eBde
eBeBee<   f   fd&ZJdej                  d'eej                     deej                   j                     fd(ZKd)ej                  deLfd*ZMdej                  deej                     fd+ZNd,ej                  de
eLeLeLf   fd-ZOd. ZP G d/ d0ej                        ZQ G d1 d2      ZR G d3 d4ej                        ZS G d5 d6      ZTy# e!$ r dZ  ejD                  d       Y w xY w# e!$ r dZ% ejD                  d       Y w xY w)9z
LoRA Trainer for ACE-Step

Lightning Fabric-based trainer for LoRA fine-tuning of ACE-Step DiT decoder.
Supports training from preprocessed tensor files for optimal performance.
    N)OptionalListDictAnyTuple	Generator)logger)nullcontext)AdamW)CosineAnnealingWarmRestartsLinearLRSequentialLR)Fabric)TensorBoardLoggerTFzFLightning Fabric not installed. Training will use basic training loop.z1bitsandbytes not installed. Using standard AdamW.)
LoRAConfig
LoKRConfigTrainingConfig)inject_lora_into_dit)check_peft_available)save_lora_weightssave_training_checkpointload_training_checkpoint)inject_lokr_into_ditsave_lokr_weightssave_lokr_training_checkpointcheck_lycoris_available)PreprocessedDataModule)	safe_path)      ?g.袋?g?g?g      ?g%I$I?g      ?g333333?devicereturnc                     t        | t        j                        r| j                  S t        | t              r| j                  dd      d   S t	        |       S )z:Normalize torch device or string to canonical device type.:   r   )
isinstancetorchr    typestrsplitr    s    O/mnt/workspace/acestep.cpp/tests/../../ACE-Step-1.5/acestep/training/trainer.py_normalize_device_typer,   I   sD    &%,,'{{&#||C#A&&v;    device_typec                 t    | dv rt         j                  S | dk(  rt         j                  S t         j                  S )z,Pick the compute dtype for each accelerator.cudaxpumps)r&   bfloat16float16float32r.   s    r+   _select_compute_dtyper8   R   s0    o%~~e}}==r-   c                     | dv ry| dk(  ryy)z:Pick Fabric precision plugin setting for each accelerator.r0   z
bf16-mixedr3   z16-mixedz32-true r7   s    r+   _select_fabric_precisionr;   [   s    o%e r-   modulec                 X   d}d}| j                         D ]  }|j                  s|dz  }|j                         s&|j                  t        j
                  k7  sDt	        j                         5  |j                  j                         |_        ddd       |dz  } ||fS # 1 sw Y   xY w)z2Force trainable floating-point parameters to fp32.r   r$   N)	
parametersrequires_gradis_floating_pointdtyper&   r6   no_graddatafloat)r<   castedtotalps       r+   _ensure_trainable_params_fp32rH   f   s    FE 
 QWW%= !aKF ! 5= !s   - B  B)	paramsc                     d}d}| D ]?  }|j                   }||dz  }t        j                  |      j                         r;|dz  }A ||fS )z>Count non-finite gradient tensors among params with gradients.r   r$   )gradr&   isfiniteall)rI   	nonfinitetotal_with_gradrG   gs        r+   _count_nonfinite_gradsrQ   u   s\    IOFF91~~a $$&NI  o%%r-   	optimizerc                 j   d}d}| j                   D ]  }|j                  dg       D ]z  }||dz  }|j                         s|j                  t        j
                  k7  s:t	        j                         5  |j                  j                         |_        ddd       |dz  }|  ||fS # 1 sw Y   xY w)z9Force optimizer parameter tensors to fp32 when trainable.r   rI   Nr$   )	param_groupsgetr@   rA   r&   r6   rB   rC   rD   )rR   rE   rF   grouprG   s        r+   _ensure_optimizer_params_fp32rW      s    FE''8R(AyQJE""$EMM)A]]_VV\\^AF %! ) ( 5= %_s   4 B))B2extra_modulec                     i }| j                         D ]  \  }}||t        |      <    |6|j                         D ]#  \  }}|j                  t        |      d|        % |S )z?Build a best-effort id(param) -> name lookup for debug logging.zlycoris_net.)named_parametersid
setdefault)r<   rX   lookupnamerG   s        r+   _build_param_name_lookupr_      sj      F**,ar!u -#446GD!be|D6%:; 7Mr-   param_name_lookupdetail_limitc                 >   d}d}g }| D ]  }|j                   }||dz  }t        j                  |      j                         r<|dz  }t	        |      |k\  rP|j                  t        |      dt        |       d      }|j                         j                         }	t        t        j                  |	      j                         j                               }
t        t        j                  |	      j                         j                               }|	t        j                  |	         }|j                         r5t        |j                         j!                         j                               n
t        d      }|j                         j                         }t        t        j                  |       j                         j                               }|j#                  | dt%        |j&                         d|j(                   d|
 d	| d
|dd|         |||fS )zPCount non-finite grads and return up to `detail_limit` offending tensor details.r   r$   z	<unnamed:>nanz	 | shape=z grad_dtype=z nan=z inf=z max_abs_finite=z.3ez param_nonfinite=)rK   r&   rL   rM   lenrU   r[   detachrD   intisnansumitemisinfnumelabsmaxappendtupleshaperA   )rI   r`   ra   rN   rO   detailsrG   rP   pnameg32	nan_count	inf_countfinite_valsmax_abs_finitep32param_nonfinites                   r+   _count_nonfinite_grads_detailedr{      s    IOGFF91>>!  "Q	w<<'!%%beyAq-ABhhj C(,,.3356	C(,,.3356	%..-.   " +//#'')..01u 	 hhj s 3388:??ABgYuQWW~.l177) D+U9+-=nS=Q R./1	
3 > ow..r-   lycoris_netc                 D   | j                         D cg c]  }|j                  s| }}|r1t        |D ci c]  }t        |      | c}j	                               S g }||S t        |dg       xs g D ]5  }|j                         D ]   }|j                  s|j                  |       " 7 |s3|j                         D ]   }|j                  s|j                  |       " t        |D ci c]  }t        |      | c}j	                               S c c}w c c}w c c}w )z
    Collect LoKr trainable params robustly.

    Primary path is model parameter traversal. If that returns empty due to
    wrapper/registration quirks, fall back to LyCORIS module parameters.
    loras)r>   r?   listr[   valuesgetattrro   )r<   r|   rG   rI   fallbackms         r+   _collect_lokr_trainable_paramsr      s     **,@,Aa,F@v.v!RUAXv.55788)+H['2.4"4A"   5 '')A" * 8,8aA8,33566! A. -s   DDD+Dmodelc                 0   | t        | d      sy| j                  }d}t        |d      rct        t        |d      t        j
                        r?|j                  }d}t        |d      r%t        t        |d      t        j
                        r?|r|| _        |S )z
    Unwrap stale Lightning Fabric wrappers from decoder left by previous runs.

    Returns:
        True if decoder was unwrapped, else False.
    decoderF_forward_moduleT)hasattrr   r%   r   nnModuler   )r   r   	unwrappeds      r+   _unwrap_stale_fabric_decoderr      s     }GE95mmGI
',
-**+RYY3 ))		 ',
-**+RYY3
 r-   c                 h   g }| g}t               }|r|j                         }t        |t        j                        s-t        |      }||v r=|j                  |       |j                  |       dD ];  }t        ||d      }t        |t        j                        s+|j                  |       = |r|S )zHCollect wrapper chain modules (Fabric/PEFT/compile/base-model wrappers).)r   	_orig_mod
base_modelr   r<   N)	setpopr%   r   r   r[   addro   r   )r<   modulesstackvisitedcurrent	module_id	attr_namechilds           r+   _iter_module_wrappersr      s    !GHEeG
))+'299-wK	Iw
I GY5E%+U#
 * Nr-   r   c                    d}d}d}t        |       D ]  }t        |d      r	 |j                          d}nt        |d      r
	 d|_        d}t        |d      r<	 |j                          t        t        |dd            }t        |dd      du}|s|rd}t        |d	d      }|t        |d
      s	 t        |d
d      dur	d|_        d} |||fS # t        $ r Y w xY w# t        $ r Y w xY w# t        $ r Y fw xY w# t        $ r Y w xY w)z
    Enable gradient checkpointing and disable use_cache across wrapped decoder modules.

    Returns:
        Tuple[checkpointing_enabled, cache_disabled, input_grads_enabled]
    Fgradient_checkpointing_enableTgradient_checkpointingenable_input_require_grads!_acestep_input_grads_hook_enabled_require_grads_hookNconfig	use_cache)	r   r   r   	Exceptionr   r   boolr   r   )r   checkpointing_enabledcache_disabledinput_grads_enabledmodhook_enabledhas_require_hookcfgs           r+   #_configure_training_memory_featuresr     s`    "N$W-378113(,% S23-1*(,% 345	..0#C!DeL  $+30Et#LTX#X #3*.' c8T*?wsK83T2%?$)CM%)NC .J !.2EEEA        sG   C	C;C.1C=	CC	C+*C+.	C:9C:=	D	D	c                 z    t        j                  d|j                  d   | f|j                        }||   }|}||fS )aq  Sample timesteps from discrete turbo shift=3 schedule.

    For each sample in the batch, randomly select one of the 8 discrete timesteps
    used by the turbo model with shift=3.0.

    Args:
        bsz: Batch size
        device: Device
        dtype: Data type (should be bfloat16)

    Returns:
        Tuple of (t, r) where both are the same sampled timestep
    r   r*   )r&   randintrq   r    )bsztimesteps_tensorindicestrs        r+   sample_discrete_timestepr   N  sL     mm	!!!$sf5E5L5LG 	!A 	
Aa4Kr-   c            
            e Zd ZdZdej
                  dededej                  dej                  f
 fdZ	 ddeeej                  f   d	ed
ej                  fdZ xZS )PreprocessedLoRAModulea  LoRA Training Module using preprocessed tensors.

    This module trains only the DiT decoder with LoRA adapters.
    All inputs are pre-computed tensors - no VAE or text encoder needed!

    Training flow:
    1. Load pre-computed tensors (target_latents, encoder_hidden_states, context_latents)
    2. Sample noise and timestep
    3. Forward through decoder (with LoRA)
    4. Compute flow matching loss
    r   lora_configtraining_configr    rA   c                    t         	|           || _        || _        t	        |t
              rt        j                  |      n|| _        t        | j                        | _	        t        | j                        | _        | j                  dv | _        t        j                  t        | j                  | j                        | _        d| _        t#               r|j%                         D ]n  }|j&                  j)                         |_        |j+                         s3t        j,                         5  |j&                  j)                         |_        ddd       p t/        ||      \  | _        | _        t5        j6                  d| j2                  d   dd       n#|| _        i | _        t5        j8                  d	       t;        | j2                        }t=        t        d
      rw| j                  dk(  rh|sf	 t5        j6                  d       t        j>                  | j0                  j@                  d      | j0                  _         t5        j6                  d       n-|rt5        j6                  d       nt5        j6                  d       |jD                  | _"        g | _#        y# 1 sw Y   xY w# tB        $ r#}t5        j8                  d| d       Y d}~Md}~ww xY w)a  Initialize the training module.

        Args:
            model: The AceStepConditionGenerationModel
            lora_config: LoRA configuration
            training_config: Training configuration
            device: Device to use
            dtype: Data type to use
        r0   r    rA   FNzLoRA injected: trainable_params, trainable paramsz2PEFT not available, training without LoRA adapterscompiler1   zCompiling DiT decoder...default)modeztorch.compile successfulztorch.compile failed (z!), continuing without compilationz=Skipping torch.compile (incompatible with PEFT LoRA adapters)zDtorch.compile not available on this device/PyTorch version, skipping)$super__init__r   r   r%   r(   r&   r    r,   r.   r8   rA   transfer_non_blockingtensorTURBO_SHIFT3_TIMESTEPSr   #force_input_grads_for_checkpointingr   r>   rC   cloneis_inferencerB   r   r   	lora_infor	   infowarningr   r   r   r   r   r   training_losses)
selfr   r   r   r    rA   paramhas_pefte	__class__s
            r+   r   zPreprocessedLoRAModule.__init__u  s1   " 	&..8.Eell6*61$++>*4+;+;<
%)%5%5%H" %"4;;djj!
 490  !))+"ZZ--/
%%'%*ZZ%5%5%7
 ) , *>e[)Q&DJKK!$..1C"DQ!GGXY DJDNNNOP '5)$)9)9V)CH67%*]]4::3E3EI%V

"67 S Z
 ll  "S ).  ,QC/PQ s%   . J3A$J+ J(	+	K4KKbatchrecord_lossr!   c           
         | j                   dv r,t        j                  | j                   | j                        }n
t	               }|5  |d   j                  | j                  | j                  | j                        }|d   j                  | j                  | j                  | j                        }|d   j                  | j                  | j                  | j                        }|d   j                  | j                  | j                  | j                        }|d   j                  | j                  | j                  | j                        }|j                  d	   }	t        j                  |      }
|}t        |	| j                        \  }}|j                  d
      j                  d
      }||
z  d|z
  |z  z   }| j                  r|j                  d      }| j                  j!                  |||||||      }|
|z
  }t#        j$                  |d	   |      }ddd       j'                         }|r)| j(                  j+                  |j-                                |S # 1 sw Y   FxY w)a  Single training step using preprocessed tensors.

        Note: This is a distilled turbo model, NO CFG is used.

        Args:
            batch: Dictionary containing pre-computed tensors:
                - target_latents: [B, T, 64] - VAE encoded audio
                - attention_mask: [B, T] - Valid audio mask
                - encoder_hidden_states: [B, L, D] - Condition encoder output
                - encoder_attention_mask: [B, L] - Condition mask
                - context_latents: [B, T, 128] - Source context
            record_loss: If True, append loss to training_losses (set False for validation).

        Returns:
            Loss tensor (float32 for stable backward)
        r1   r2   r3   r.   rA   target_latentsrA   non_blockingattention_maskencoder_hidden_statesencoder_attention_maskcontext_latentsr   r   Thidden_statestimestep
timestep_rr   r   r   r   Nr.   r&   autocastrA   r
   tor    r   rq   
randn_liker   r   	unsqueezer   requires_grad_r   r   Fmse_lossrD   r   ro   rj   )r   r   r   autocast_ctxr   r   r   r   r   r   x1x0r   _t_xtdecoder_outputsflowdiffusion_losss                      r+   training_stepz$PreprocessedLoRAModule.training_step  sX   , 55 >> ,,DJJL '=L"#34774::D<V<V 8 N ##34774::D<V<V 8 N %**A$B$E$E4::D<V<V %F %! &++C%D%G%G4::D<V<V &H &" $$56994::D<V<V : O !&&q)C !!.1BB ,C1F1FGDAqR**2.B bC"H?*B77&&t, #jj00 -&;'= / 1 O 7DZZ(:DAN] b (--/  ''(;(;(=>m \s   GI  I))T)__name__
__module____qualname____doc__r   r   r   r   r&   r    rA   r   r   r(   Tensorr   r   __classcell__r   s   @r+   r   r   h  s    
P"yyP"  P" (	P"
 P" {{P"j !RC%&R R 
	Rr-   r   c                       e Zd ZdZdedefdZ	 	 ddedee	   dee   d	e
eeeef   ddf   fd
Z	 ddedee	   dee   d	e
eeeef   ddf   fdZdedee	   d	e
eeeef   ddf   fdZd Zy)LoRATrainerzHigh-level trainer for ACE-Step LoRA fine-tuning.

    Uses Lightning Fabric for distributed training and mixed precision.
    Supports training from preprocessed tensor directories.
    r   r   c                     || _         || _        t        |j                        |_        || _        d| _        d| _        d| _        y)zInitialize the trainer.

        Args:
            dit_handler: Initialized DiT handler (for model access)
            lora_config: LoRA configuration
            training_config: Training configuration
        NF)dit_handlerr   r   
output_dirr   r<   fabricis_training)r   r   r   r   s       r+   r   zLoRATrainer.__init__#  sE     '&%./I/I%J". r-   N
tensor_dirtraining_stateresume_fromr!   c              #     K   d| _         	 t        | j                  dd      }|ddd| df 	 d| _         y	 t        |      }t
        j                  j                  |      sddd
| f 	 d| _         yt        j                  | j                  j                         t        j                  | j                  j                         t        j                  j                         r3t        j                  j                  | j                  j                         	 ddl}|j                  j                  | j                  j                         t%        | j                  j&                  | j(                  | j                  | j                  j*                  | j                  j,                        | _        t1        | j.                  j&                  j2                        \  }}}|| j.                  _        t7        j8                  d| d| d|        t;        || j                  j<                  | j                  j>                  | j                  j@                  | j                  jB                  | j                  jD                  | j                  jF                  t        | j                  dd            }	|	jI                  d       tK        |	jL                        dk(  rd 	 d| _         ydddtK        |	jL                         df |rd nd |sd tN        r| jQ                  |	||      E d{    n| jS                  |	|      E d{    d| _         y# t        $ r ddd	| f Y d| _         yw xY w# t"        $ r Y 6w xY w7 W7 ># t"        $ r2}
t7        jT                  d       dddtW        |
       f Y d}
~
rd}
~
ww xY w# d| _         w xY ww)a  Train LoRA adapters from preprocessed tensor files.

        This is the recommended training method for best performance.

        Args:
            tensor_dir: Directory containing preprocessed .pt files
            training_state: Optional state dict for stopping control
            resume_from: Optional path to checkpoint directory to resume from

        Yields:
            Tuples of (step, loss, status_message)
        TquantizationNr           uL   ❌ LoRA training requires a non-quantized DiT model. Current quantization: M. Re-initialize service with INT8 Quantization disabled, then retry training.F&   ❌ Rejected unsafe tensor directory:     ❌ Tensor directory not found: )r   r   r   r    rA   1Training memory features: gradient_checkpointing=, use_cache_disabled=, input_grads_enabled=	val_splitr  
batch_sizenum_workers
pin_memoryprefetch_factorpersistent_workerspin_memory_devicer  fitr   r	  u.   ❌ No valid samples found in tensor directory   📂 Loaded  preprocessed samplesr   r	  u/   🧠 Gradient checkpointing enabled for decoderr   r	  uK   ⚠️ Gradient checkpointing not enabled (model wrapper did not expose it)r   r	  uW   ℹ️ Input-grad hook not available on this DiT; using explicit checkpointing fallbackzTraining failed   ❌ Training failed: ),r  r   r   r   
ValueErrorospathisdirr&   manual_seedr   seedrandomr1   is_availablemanual_seed_allnumpyr   r   r   r   r    rA   r<   r   r   r   r	   r   r   r  r  r  r  r  r  setupre   train_datasetLIGHTNING_AVAILABLE_train_with_fabric_train_basic	exceptionr(   )r   r  r  r  quantization_modenpckpt_enabledr   r   data_moduler   s              r+   train_from_preprocessedz#LoRATrainer.train_from_preprocessed:  s    $  k	% !((8(8.$ O ,11B0C Dff	  z  %Du&z2
 77==, @MMMf  %Da d22778KK,,112zz&&(

**4+?+?+D+DE"		t33889 1&&,, ,, $ 4 4''..&&,,DK 4DKK4E4E4M4MN >L.*= ?KDKK;KKCL> R&&4%55KL_K`b 1%//:: 00<<//:: $ 4 4 D D#'#7#7#J#J"&"6"6"H"H!$"6"6SI	K e$;,,-2NNB  %D= s;#<#<=>>ST 
 OO 
 '  #22    ,,[.III  %Dq   FzlSSSl  %Dq   | J 	;./S1#a&:::	;  %Ds   O$N OM )N .O6B#N 3M7 FN O$AN /N0N 
N	N OM4*N +O3M44N 7	N N NN 	N 	O(O<O	 OO	 		OOr3  c              #   x  K   t        j                  | j                  j                  d       | j                  j
                  }t        |      }|dv r|nd}d}	 t        | j                  j                  d      }|d	|d
}	||g|	d<   t        dei |	| _        | j                  j                          ddd| d| df |dk(  s|j                  d      rX| j                  j                  j                   j#                  t$        j&                        | j                  j                  _        n]| j                  j                  j                   j#                  | j                  j(                        | j                  j                  _        t+        | j                  j                  j                         \  }
}t        j,                  d|
 d| d       |j/                         }t1        |d      r|j3                         nd}|g |d<   g |d<   g |d<   g |d<   g |d<   d|d<   d}d}t5        d      }d}| j                  j                  j7                         D cg c]  }|j8                  s| }}|sd  yddd!t;        d" |D              d#d$f | j                  j<                  | j                  j>                  d%}t@        r;|d&k(  r6t        j,                  d'       tC        jD                  jF                  |fi |}n4| j                  jH                  jJ                  d&k(  rd|d(<   tM        |fi |}tO        d	tQ        jR                  tU        |      | j                  jV                  z              }|| j                  jX                  z  }t[        | j                  j\                  tO        d	|d)z              }t_        |dd*|+      }ta        |tO        d	||z
        d	| j                  j<                  d,z  -      }tc        |||g|g.      }| j                  je                  | j                  j                  j                   |      \  | j                  j                  _        }tg        |      \  }}t        j,                  d/| d| d       | j                  ji                  |      }d}d}d} |r	 tk        |      }|rt         jn                  jq                  |      r	 ddd2| d3f ts        |||| j                  jH                  4      } | d5   r{| d5   }!t         jn                  ju                  |!d6      }"t         jn                  jq                  |"      s t         jn                  ju                  |!d7      }"t         jn                  jq                  |"      rdd8l;m<}# |"j                  d9      r	 |#|"      }$n,t%        jz                  |"| j                  jH                  d:      }$| j                  j                  j                   }%t1        |%d;      r|%j|                  }%|%j                  |$d<=       | d>   }| d?   }d@| dA| g}&| dB   r|&j                  dC       | dD   r|&j                  dE       dddFju                  |&      f ndddG|! f n
dddH| f n|rdddK| d1f d}'d}(|j                  dL       | j                  j                  j                   j                          t        || j                  jX                        D ]  })d}*d}+t        j                         },t        |      D ]s  \  }-}.|r)|j                  dMd<      r||(tO        |'d	      z  dNf   y| j                  j                  |.      }/|/| j                  jV                  z  }/| j                  j                  |/       |(|/j                         z  }(|'d	z  }'|'| j                  jV                  k\  st        |      \  }0}1|0dkD  r.|j                  dL       |t5        dO      dP|0 d|1 dQf d}(d}'| j                  j                  | j                  j                  j                   || j                  j                  d<R       |j                          |j                          |j                  dL       |d	z  }|(|'z  }2|| j                  j                  z  dk(  r|O||2}n||2z  d	|z
  |z  z   }|d   j                  |       |d   j                  |2       |d   j                  |       | j                  j                  dS|2|T       | j                  j                  dU|j                         d   |T       ||2dV|)d	z    d| j                  jX                   dW| dX|2dYf |*|2z  }*|+d	z  }+d}(d}'v |'dkD  rt        |      \  }0}1|0dkD  r.|j                  dL       |t5        dO      dP|0 d|1 dZf d}(d}'n| j                  j                  | j                  j                  j                   || j                  j                  d<R       |j                          |j                          |j                  dL       |d	z  }|(|'z  }2|| j                  j                  z  dk(  r|O||2}n||2z  d	|z
  |z  z   }|d   j                  |       |d   j                  |2       |d   j                  |       | j                  j                  dS|2|T       | j                  j                  dU|j                         d   |T       ||2dV|)d	z    d| j                  jX                   dW| dX|2dYf |*|2z  }*|+d	z  }+d}(d}'t        j                         |,z
  }3|*tO        |+d	      z  }4|^||4}n||4z  d	|z
  |z  z   }|d   }5|5r|5d[   |k7  r<|d   j                  |       |d   j                  |4       |d   j                  |       | j                  j                  d\|4|)d	z   T       |_| j                  j                  j                   j                          d}6d}7t%        j                         5  |D ]7  }8| j                  j                  |8d<]      }9|6|9j                         z  }6|7d	z  }79 	 ddd       | j                  j                  j                   j                          |6tO        |7d	      z  }:|(|d   j                  |       |d   j                  |:       |:|k  rg|:}|}|||d<   t         jn                  ju                  | j                  j                  d^d_      };t        | j                  j                  |||)d	z   ||;       |)d	z   | j                  j                  z  dk(  st         jn                  ju                  | j                  j                  d^d`|)d	z    da|4dY      }<t        | j                  j                  |||)d	z   ||<       ||4db|)d	z    f  t         jn                  ju                  | j                  j                  dc      }=t        | j                  j                  |=       | j                  j                  r| j                  j                  d[   nd}>||>dd|= f y# t        $ r#}t        j                  d|        Y d}~Md}~ww xY wc c}w # tl        $ r ddd0| d1f d}Y 
1w xY w# t        $ r/}t        j                  dI       dddJ| d1f d}d}Y d}~}d}~ww xY w# 1 sw Y   xY ww)fzTrain using Lightning Fabric.Texist_okr1   r2   r3   cpuautoNlogsroot_dirr^   ;TensorBoard logger unavailable, continuing without logger: r$   acceleratordevices	precisionloggersr   r	      🚀 Starting training (device: , precision: )...r3   -mixedrA   %Trainable tensor dtype fixup: casted / to fp32val_dataloader
plot_steps	plot_lossplot_emaplot_val_stepsplot_val_lossplot_best_step皙?infr   r	  u"   ❌ No trainable parameters found!   🎯 Training c              3   <   K   | ]  }|j                           y wNrl   .0rG   s     r+   	<genexpr>z1LoRATrainer._train_with_fabric.<locals>.<genexpr>        E4Dq4D   r    parameterslrweight_decayr1   z:train_with_fabric using bitsandbytes 8-bit AdamW optimizerfused
   r   start_factor
end_factortotal_iters{Gz?T_0T_multeta_min
schedulers
milestones$Optimizer param dtype fixup: casted u(   ⚠️ Rejected unsafe checkpoint path: z, starting freshu   🔄 Loading checkpoint from z...)rR   	schedulerr    adapter_pathzadapter_model.safetensorszadapter_model.bin)	load_filez.safetensors)map_locationweights_onlyr   F)strictepochglobal_stepu   ✅ Resumed from epoch , step loaded_optimizeru   optimizer ✓loaded_scheduleru   scheduler ✓z, u$   ⚠️ Adapter weights not found in u$   ⚠️ No valid checkpoint found in zFailed to load checkpointu"   ⚠️ Failed to load checkpoint: u"   ⚠️ Checkpoint path not found: set_to_noneshould_stop   ⏹️ Training stopped by userrd      ⚠️ Non-finite gradients (z); skipping optimizer stepmax_normerror_if_nonfinite
train/losssteptrain/lrEpoch , Step , Loss: .4fz$); skipping optimizer remainder stepr   train/epoch_loss)r   checkpointsbestepoch__loss_   💾 Checkpoint saved at epoch final%   ✅ Training complete! LoRA saved to r:   )Yr!  makedirsr   r  r<   r.   r;   r   ModuleNotFoundErrorr	   r   r   r  launchendswithr   r   r   r&   r6   rA   rH   r   train_dataloaderr   rL  rD   r>   r?   ri   learning_raterb  HAS_BNBbnboptim	AdamW8bitr    r'   r   rn   mathceilre   gradient_accumulation_steps
max_epochsminwarmup_stepsr   r   r   r*  rW   setup_dataloadersr   r   r"  existsr   joinsafetensors.torchrt  loadr   load_state_dictro   r   r/  	zero_gradtrainrangetime	enumeraterU   r   backwardrj   rQ   clip_gradientsmax_grad_normr  log_every_n_stepslogget_last_lrevalrB   r   save_every_n_epochsr   r   )?r   r3  r  r  r.   rB  r@  	tb_loggerr   fabric_kwargscasted_trainabletotal_trainable_tensorstrain_loader
val_loaderema_loss	ema_alphabest_val_lossbest_val_steprG   r   optimizer_kwargsrR   steps_per_epochtotal_stepsr  warmup_schedulermain_schedulerrr  casted_opt_paramstotal_opt_paramsstart_epochry  checkpoint_infors  adapter_weights_pathrt  
state_dictr   status_partsaccumulation_stepaccumulated_lossrx  
epoch_lossnum_updatesepoch_start_time
_batch_idxr   lossnonfinite_gradsgrad_tensorsavg_loss
epoch_timeavg_epoch_lossrM  total_val_lossn_val	val_batchv_lossval_lossbest_dircheckpoint_dir
final_path
final_losss?                                                                  r+   r-  zLoRATrainer._train_with_fabric  s      	D((33dCkk--,[9	&*GGKV 	
 		)--88vI '"

  (1{M)$-}- .{m=SWX
 	
 %9#5#5h#?(,(9(9(A(A(D(Dmm )E )DKK% )-(9(9(A(A(D(Dkk'' )E )DKK% 5RKK%%5
11 	34D3EQG^F__gh	

 #335 {$45 &&( 	 %+-N<(*,N;')+N:&/1N+,.0N?+/3N+,	e {{((335
5!A5 	 
  >> S E4D EEaHT
 	
 &&44 00==
 {f,KKTU		++,<Q@PQI{{!!&&&0,0 ).C2BCI IIL!D$8$8$T$TT
 &(<(<(G(GG4//<<c![TVEV>WX $$	
 5A{\12((66=	
 !(.9$~
	 04{{/@/@KK%%y0
,!9 /LI.V++23D2EQGWFXX`a	
 {{44\B #'4 277>>+6=  =k]#NNN #;'';;--	# #>2#2>#BL+-77<<$&A,( 77>>*>?/1ww||(*=0, ww~~&:;?/88H)23G)HJ). 4-1[[-?-?-1*J #'++"3"3";";"7,=>&-&=&=G//
5/I&5g&>&5m&D 6k]'+W( ++=>(//@*+=>(//@dii&===(L\N&[[[S$H"VVV S>{mK[\\\ -!!'');(<(<(G(GHEJK#yy{%.|%<!
E!n&8&8&N#(3/@!+DD9 
  {{007d22NNN $$T* DIIK/ !Q&! &++GGH 5K(51O\ '*!+++='!%L"??PPQR^Q_ `: !:	  ,/(,-) KK..))11!!%!5!5!C!C+0	 /  NN$NN$''D'91$K  02CCH"T%9%9%K%KKqP)5'/+3 %.$8A	MX;U$U !) +<8??L*;7>>xH*:6==hGh[Q&	(=(=(?(B (  ($$UQYKq1E1E1P1P0QQXYdXeemnvwzm{|  (*J1$K'*$()%a &=h !1$0FGW0X-"Q&''D'9#e;O;LAl^ \@ @	  (+$()%KK..))11!!%!5!5!C!C+0	 /  NN$NN$''D'9q +.??!5!5!G!GG1L%1#+'/H'08';q9}PX>X'XH&|4;;KH&{3::8D&z299(CKKOOL(OMKKOO"I$9$9$;A$>[ $  $  1T-A-A-L-L,MWU`Taaijrsviwx  (*J1$K'*$()% '77J'#k1*==N)#-H(>9Q]h<VVH+L9
!Z^{%B"<077D";/66~F":.55h?KKOO.UQYOO %!!))..0!$]]_%/	!%!:!:9RW!:!X&&++-7
 &0 %
 !!))//1)CqM9!-"#34;;KH"?3::8Dm+$,M$/M%1;H'78!ww||,,77 H -))!!	#  	T11EEEJ!#((33]fUUVYKW]^lmp]qDr" )KK%%AI"  "5eai[A U Ib WW\\$"6"6"A"A7K
$++++Z8 04{{/J/JDKK''+PS 	 3J<@
 	
c # 	NNMaSQ 	z
Z  #>{mK[\ 
 ##B     !<= B1#EUVVV	 b %_s   A|:!z! ;G|:{+{/I|:2{ =#|:!F9{2 E|:&Q;|:!=|-C:|:D|:!	{*{|:{|:{/+|:.{//|:2	|*;$|%|:%|**|:-|7	2|:c              #   H  K   d t        j                  | j                  j                  d       |j	                         }| j
                  j                  j                         D cg c]  }|j                  s| }}|sd yt        ry| j
                  j                  dk(  r`t        j                  j                  || j                  j                  | j                  j                        }t!        j"                  d       n6t%        || j                  j                  | j                  j                        }t'        d	t)        j*                  t-        |      | j                  j.                  z              }|| j                  j0                  z  }t3        | j                  j4                  t'        d	|d
z              }	t7        |dd|	      }
t9        |t'        d	||	z
        d	| j                  j                  dz        }t;        ||
|g|	g      }d}d}d}|j=                  d       | j
                  j                  j>                  jA                          tC        | j                  j0                        D ]  }d}d}tE        jD                         }|D ]\  }|r)|jG                  dd      r||t'        |d	      z  df   y| j
                  jI                  |      }|| j                  j.                  z  }|jK                          ||jM                         z  }|d	z  }|| j                  j.                  k\  stN        jP                  jR                  jU                  || j                  jV                         |jY                          |jY                          |j=                  d       |d	z  }||z  }|| j                  jZ                  z  dk(  r||d|d	z    d| d|df ||z  }|d	z  }d}d}_ |dkD  rtN        jP                  jR                  jU                  || j                  jV                         |jY                          |jY                          |j=                  d       |d	z  }||z  }|| j                  jZ                  z  dk(  r||d|d	z    d| d|df ||z  }|d	z  }d}d}tE        jD                         |z
  }|t'        |d	      z  }||d|d	z    d| j                  j0                   d|ddf |d	z   | j                  j\                  z  dk(  st         j^                  ja                  | j                  j                  d d!|d	z    d"|d      }tc        | j
                  j                  |       ||d#f  t         j^                  ja                  | j                  j                  d$      }tc        | j
                  j                  |       | j
                  jd                  r| j
                  jd                  d%   nd}||d&| f yc c}w w)'z#Basic training loop without Fabric.r   r	  u$   🚀 Starting basic training loop...Tr6  rU  Nr1   r`  z4train_basic using bitsandbytes 8-bit AdamW optimizerr$   rd  rS  r   re  ri  rj  rn  r   r	  r}  r  F   ⏹️ Training stoppedr  r  r  r  
   ✅ Epoch rJ   in .1fsr  r  r     💾 Checkpoint savedr  r   r  )3r!  r  r   r  r  r<   r   r>   r?   r  r.   r  r  r  r  rb  r	   r   r   rn   r  r  re   r  r  r  r  r   r   r   r  r   r  r  r  rU   r   r  rj   r&   r   utilsclip_grad_norm_r  r  r  r  r"  r  r   r   )r   r3  r  r  rG   r   rR   r  r  r  r  r  rr  ry  r  r  rx  r  r  r  r   r  r  r  r  r  r  r  s                               r+   r.  zLoRATrainer._train_basic  s     =<
D((33dC"335 {{((335
5!A5 	 
  >>t{{..&8		++ ''55!11>> , I
 KKNO ''55!11>>I IIL!D$8$8$T$TT
 &(<(<(G(GG4//<<c![TVEV>WX#CC\
 5A{\12((66=	
 !(.9$~
	 -!!'')4//::;EJK#yy{%!n&8&8&N#(3/@!+DD1 
 {{007d22NNN DIIK/ !Q&! &++GGH HHNN22($*>*>*L*L NN$NN$''D'91$K/2CCH"T%9%9%K%KKqP'$$UQYKw{m8HUX>Z  (*J1$K'*$()%M &P !1$..$d&:&:&H&H   ###5q +.??!5!5!G!GG1L#  7;-xQT~V  h&
q #& $%!'77J'#k1*==NUQYKq)=)=)H)H(IjY\M]]^_  	T11EEEJ!#((33]fUUVYKW]^lmp]qDr" "$++"3"3^D!>3JJJa <d WW\\$"6"6"A"A7K
$++++Z8/3{{/J/JDKK''+PS 	 3J<@
 	
Y
s'   A&X"(X:X>J=X"<G&X"$C>X"c                     d| _         yzStop training.FNr  r   s    r+   stopzLoRATrainer.stop&  
     r-   )NNrX  )r   r   r   r   r   r   r   r(   r   r   r   r   rg   rD   r4  r   r-  r.  r  r:   r-   r+   r   r     s   !  ! (	!4 *.%)	%% !% c]	%
 
5eS)45	6%J &*	K
+K
 !K
 c]	K

 
5eS)45	6K
Z\
+\
 !\
 
5eS)45	6	\
|!r-   r   c            
            e Zd ZdZdej
                  dededej                  dej                  f
 fdZdeeej                  f   d	ej                  fd
Z xZS )PreprocessedLoKRModulez0LoKr training module using preprocessed tensors.r   lokr_configr   r    rA   c                    t         |           || _        || _        t	        |t
              rt        j                  |      n|| _        t        | j                        | _	        t        | j                        | _        | j                  dv | _        t        j                  t        | j                  | j                        | _        d| _        d | _        t%               rGt'        ||      \  | _        | _        | _        t-        j.                  d| j*                  d   dd       n#|| _        i | _        t-        j0                  d       |j2                  | _        g | _        y )	Nr0   r   FzLoKr injected: r   r   r   z5LyCORIS not available, training without LoKr adapters)r   r   r  r   r%   r(   r&   r    r,   r.   r8   rA   r   r   r   r   r   r|   r   r   r   	lokr_infor	   r   r   r   r   )r   r   r  r   r    rA   r   s         r+   r   zPreprocessedLoKRModule.__init__.  s    	&..8.Eell6*61$++>*4+;+;<
%)%5%5%H" %"4;;djj!
 490"$;O{<8DJ($. KK!$..1C"DQ!GGXY DJDNNNRSll!r-   r   r!   c           
         | j                   dv r,t        j                  | j                   | j                        }n
t	               }|5  |d   j                  | j                  | j                  | j                        }|d   j                  | j                  | j                  | j                        }|d   j                  | j                  | j                  | j                        }|d   j                  | j                  | j                  | j                        }|d   j                  | j                  | j                  | j                        }|j                  d	   }t        j                  |      }	|}
t        || j                        \  }}|j                  d
      j                  d
      }||	z  d|z
  |
z  z   }| j                  r|j                  d      }| j                  j!                  |||||||      }|	|
z
  }t#        j$                  |d	   |      }ddd       j'                         }| j(                  j+                  |j-                                |S # 1 sw Y   DxY w)zSingle LoKr training step.r   r   r   r   r   r   r   r   r   r   r   Tr   Nr   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   s                     r+   r   z$PreprocessedLoKRModule.training_stepS  sK   55 >> ,,DJJL '=L"#34774::D<V<V 8 N ##34774::D<V<V 8 N %**A$B$E$E4::D<V<V %F %! &++C%D%G%G4::D<V<V &H &" $$56994::D<V<V : O !&&q)C!!.1BB+C1F1FGDAqR**2.BbC"H?*B77&&t,"jj00 -&;'= / 1 O 7DZZ(:DANM P (--/##N$7$7$9:U \s   GII')r   r   r   r   r   r   r   r   r&   r    rA   r   r   r(   r   r   r   r   s   @r+   r  r  +  sm    :#"yy#"  #" (	#"
 #" {{#"J34U\\(9#: 3u|| 3r-   r  c                       e Zd ZdZdedefdZ	 ddedee	   de
eeeef   ddf   fd	Zd
edee	   de
eeeef   ddf   fdZd
edee	   de
eeeef   ddf   fdZd Zy)LoKRTrainerz1High-level trainer for ACE-Step LoKr fine-tuning.r  r   c                     || _         || _        t        |j                        |_        || _        d | _        d | _        d| _        i | _        y )NF)	r   r  r   r  r   r<   r  r  run_metadata)r   r   r  r   s       r+   r   zLoKRTrainer.__init__  sM     '&%./I/I%J". ,.r-   Nr  r  r!   c           
   #     K   d| _         	 t        | j                  j                        rt	        j
                  d       t        | j                  dd      }|ddd| df 	 | j                  5t        | j                  d	      rt        | j                  j                         t        | d
d      6t        | j                  d	d      t        | j                  j                         d| _         y	 t        |      }t        j                  j                  |      sddd| f 	 | j                  5t        | j                  d	      rt        | j                  j                         t        | d
d      6t        | j                  d	d      t        | j                  j                         d| _         yt               sd 	 | j                  5t        | j                  d	      rt        | j                  j                         t        | d
d      6t        | j                  d	d      t        | j                  j                         d| _         yt        j                   | j"                  j$                         t'        j$                  | j"                  j$                         t        j(                  j+                         r3t        j(                  j-                  | j"                  j$                         	 ddl}|j&                  j%                  | j"                  j$                         t3        | j                  j                  | j4                  | j"                  | j                  j6                  | j                  j8                        | _        t;        | j                  j                  j<                        \  }}}|| j                  _        t	        j
                  d| d| d|        tA        || j"                  jB                  | j"                  jD                  | j"                  jF                  | j"                  jH                  | j"                  jJ                  | j"                  jL                  | j"                  jN                        }|jQ                  d       tS        |jT                        dk(  rd 	 | j                  5t        | j                  d	      rt        | j                  j                         t        | d
d      6t        | j                  d	d      t        | j                  j                         d| _         y|tW        tS        |jT                              | j"                  jY                         d| _-        dddtS        |jT                         df |rd nd |sd t\        r| j_                  ||      E d{    n| ja                  ||      E d{    | j                  5t        | j                  d	      rt        | j                  j                         t        | d
d      6t        | j                  d	d      t        | j                  j                         d| _         y# t        $ r ddd| f Y | j                  5t        | j                  d	      rt        | j                  j                         t        | d
d      6t        | j                  d	d      t        | j                  j                         d| _         yw xY w# t0        $ r Y w xY w7 `7 H# t0        $ r3}	t	        jb                  d       dddte        |	       f Y d}	~	}d}	~	ww xY w# | j                  5t        | j                  d	      rt        | j                  j                         t        | d
d      6t        | j                  d	d      t        | j                  j                         d| _         w xY ww)z.Train LoKr adapters from preprocessed tensors.Tz;Unwrapped stale Fabric decoder wrapper before LoKr trainingr  Nr   r	  uL   ❌ LoKr training requires a non-quantized DiT model. Current quantization: r
  r   r   Fr  r  )r   r	  u>   ❌ LyCORIS not installed. Install lycoris-lora to train LoKr.)r   r  r   r    rA   r  r  r  r  r  r  )r  num_samplesr   r  r  r  r  r  zLoKr training failedr  )3r  r   r   r   r	   r   r   r<   r   r   r   r!  r"  r#  r   r&   r$  r   r%  r&  r1   r'  r(  r)  r   r  r  r    rA   r   r   r   r   r  r  r  r  r  r  r  r*  re   r+  rg   to_dictr  r,  r-  r.  r/  r(   )
r   r  r  r0  r1  r2  r   r   r3  r   s
             r+   r4  z#LoKRTrainer.train_from_preprocessed  s      y	%+D,<,<,B,BCQ !((8(8.$ O ,11B0C Dff	  B {{&74;;+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$DM&z2
 77==, @MMMp {{&74;;+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$D{ +, 
 ` {{&74;;+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$Dk d22778KK,,112zz&&(

**4+?+?+D+DE"		t33889 1&&,, ,, $ 4 4''..&&,,DK 4DKK4E4E4M4MN >L.*= ?KDKK;KKCL> R&&4%55KL_K`b
 1%//:: 00<<//:: $ 4 4 D D#'#7#7#J#J"&"6"6"H"H..88	K e$;,,-2NNJ {{&74;;+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$DS )"3{'@'@#AB#'#7#7#?#?#A!D s;#<#<=>>ST 
 OO 
 '  #22;OOO,,[.III {{&74;;+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$DI   FzlSSSv {{&74;;+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$DI.  | PI 	;34S1#a&::::	; {{&74;;+H,T[[->->?mT2>D,,gt<H,T-=-=-C-CD$Ds   _<A\- $B_<1Y3 <)\- &B_<2\- B_<B#\- 13\ $F\- 2B_<>B	\- \'\- "\*#\- 'B_<3\\- B_<\\- 	\$ \- #\$$\- *\- -	])6(]$], $]))], ,B_99_<r3  c              #   *  K   t        j                  | j                  j                  d       | j                  j
                  }t        |      }|dv r|nd}|j                  d       }d }	 t        | j                  j                  d      }|d	|d
}	||g|	d<   t        dRi |	| _        | j                  j                          ddd| d| df |st        j                  d       |dk(  s|j                  d      rX| j                  j                   j"                  j%                  t&        j(                        | j                  j                   _        n]| j                  j                   j"                  j%                  | j                  j*                        | j                  j                   _        t-        | j                  j                   j"                        \  }
}|dk(  rCt/        | j                  dd       ,t-        | j                  j0                        \  }}|
|z  }
||z  }t        j                  d|
 d| d       |j3                         }t5        | j                  j                   t/        | j                  dd             }t7        | j                  j                   t/        | j                  dd             }|sd y |dk(  rt        j                  d       dddt9        d |D              ddf | j                  j:                  | j                  j<                  d}| j                  j>                  j@                  dk(  rd|d <   tC        |fi |}tE        d	tG        jH                  tK        |      | j                  jL                  z              }|| j                  jN                  z  }tQ        | j                  jR                  tE        d	|d!z              }tU        |d"d#|$      }tW        |tE        d	||z
        d	| j                  j:                  d%z  &      }tY        |||g|g'      }| j                  j[                  | j                  j                   j"                  |      \  | j                  j                   _        }t]        |      \  }}t        j                  d(| d| d       | j                  j_                  |      }d}d}d}|ja                  d)       | j                  j                   j"                  jc                          te        | j                  jN                        D ]b  }d}d} tg        jf                         }!|D ]f  }"|r)|ji                  d*d+      r||tE        |d	      z  d,f   y | j                  jk                  |"      }#|#| j                  jL                  z  }#| j                  jm                  |#       ||#jo                         z  }|d	z  }|| j                  jL                  k\  s|rtq        ||d!-      \  }$}%}&|$dkD  ro|&r>t        j                  d.|$ d|% d/|d	z    d0| d1	d2js                  d3 |&D              z          |ja                  d)       |tu        d4      d5|$ d|% d6f d}d}8| j                  jw                  | j                  j                   j"                  || j                  jx                  d+7       |j{                          |j{                          |ja                  d)       |d	z  }||z  }'|| j                  j|                  z  dk(  rx| j                  j                  d8|'|9       | j                  j                  d:|j                         d   |9       ||'d;|d	z    d| j                  jN                   d<| d=|'d>f ||'z  }| d	z  } d}d}i |dkD  r|rtq        ||d!-      \  }$}%}&|$dkD  ro|&r>t        j                  d?|$ d|% d/|d	z    d0| d1	d2js                  d@ |&D              z          |ja                  d)       |tu        d4      d5|$ d|% dAf d}d}| j                  jw                  | j                  j                   j"                  || j                  jx                  d+7       |j{                          |j{                          |ja                  d)       |d	z  }||z  }'|| j                  j|                  z  dk(  rx| j                  j                  d8|'|9       | j                  j                  d:|j                         d   |9       ||'d;|d	z    d| j                  jN                   d<| d=|'d>f ||'z  }| d	z  } d}d}tg        jf                         |!z
  }(|tE        | d	      z  })| j                  j                  dB|)|d	z   9       ||)dC|d	z    d| j                  jN                   dD|(dEdF|)d>f |d	z   | j                  j                  z  dk(  st         j                  js                  | j                  j                  dGdH|d	z    dI|)d>      }*t        | j                  j0                  |||d	z   ||*| j                  | j                  J       ||)dK|d	z    f e t         j                  js                  | j                  j                  dL      }+dM| j                  j                         i},| j                  r| j                  |,dN<   t        | j                  j0                  |+|,O       | j                  j                  r| j                  j                  dP   nd}-||-dQ|+ f y # t        $ r#}t        j                  d|        Y d }~
d }~ww xY ww)SNTr6  r8  r:  rG  r;  r<  r>  r$   r?  rC  r   r	  rD  rE  rF  zpLoKr mixed precision detected: disabling pre-unscale non-finite grad checks; relying on AMP/GradScaler handling.r3   rH  r|   rI  rJ  rK  rU  zsLoKr trainable params discovered via LyCORIS fallback traversal; decoder parameter traversal returned 0 trainables.rV  c              3   <   K   | ]  }|j                           y wrX  rY  rZ  s     r+   r\  z1LoKRTrainer._train_with_fabric.<locals>.<genexpr>{  r]  r^  r   r_  r`  r1   rc  rd  rS  r   re  ri  rj  rn  rq  r}  r  Fr  )ra   zLoKr non-finite gradients (z) at epoch rz  z. Top offending tensors:

c              3   &   K   | ]	  }d |   ywz  - Nr:   r[  ds     r+   r\  z1LoKRTrainer._train_with_fabric.<locals>.<genexpr>  s     /VDUq$qc
DU   rd   r  z6); skipping optimizer step (see logs for tensor names)r  r  r  r  r  r  r  r  z%LoKr non-finite remainder gradients (c              3   &   K   | ]	  }d |   ywr  r:   r  s     r+   r\  z1LoKRTrainer._train_with_fabric.<locals>.<genexpr>  s     +R@Q1d1#J@Qr  z@); skipping optimizer remainder step (see logs for tensor names)r  r  r  r  z	s, Loss: r  r  r  r  r  r  r  r  r  metadatar   %   ✅ Training complete! LoKr saved to r:   )Ir!  r  r   r  r<   r.   r;   r  r   r  r	   r   r   r  r  r   r   r   r   r&   r6   rA   rH   r   r|   r  r   r_   ri   r  rb  r    r'   r   rn   r  r  re   r  r  r  r  r   r   r   r*  rW   r  r  r  r  r  rU   r   r  rj   r{   r  rD   r  r  r  r  r  r  r  r"  r   r  r  r  r   r   ).r   r3  r  r.   rB  r@  manual_nonfinite_checkr  excr  r  r  casted_fallbacktotal_fallbackr  r   r`   r  rR   r  r  r  r  r  rr  r  r  r  r  ry  rx  r  r  r  r   r  r  r  nonfinite_detailsr  r  r  r  r  final_metadatar  s.                                                 r+   r-  zLoKRTrainer._train_with_fabric  s'    
 	D((33dCkk--,[9	&*GGKV 	 &/%7%7%A!A		)--88I '"

  (1{M)$-}- .{m=SWX
 	

 &KK6
 %9#5#5h#?(,(9(9(A(A(D(Dmm )E )DKK% )-(9(9(A(A(D(Dkk'' )E )DKK% 5RKK%%5
11 $q(]D9E.K''/+O^ /#~5#34D3EQG^F__gh	
 #3359KKDKK5
 5KKDKK5

  >>"a'NNE S E4D EEaHT
 	
 &&44 00==
 ;;""f,(,W%*?.>?	IIL!D$8$8$T$TT
 &(<(<(G(GG4//<<c![TVEV>WX#$	
 5A{\12((66=	
 !(.9$~
	 04{{/@/@KK%%y0
,!9 /LI.V++23D2EQGWFXX`a	
 {{44\B-!!'')4//::;EJK#yy{%!n&8&8&N#(3/@!+DD9 
 {{007d22NNN$$T* DIIK/ !Q&! &++GGH .; 0 1-/ I7H +Q.0 &&A/ARRST`Saal',qykE_%a&*ii/VDU/V&V%W!"
 &//D/A + %e&COCTTUVbUc dZ %Z	#  03,01-$KK..))11!!%!5!5!C!C+0	 /  NN$NN$''D'91$K/2CCH"T%9%9%K%KKqPh[Q&	(=(=(?(B (  ($"(1T5I5I5T5T4U V((3}HXcN!L	  (*J1$K'*$()%] &` !1$)7,-)+ EO\3D '*,"NN"GGXXYZfYggr#(19+W[MA[!]"&))+R@Q+R"R!S
 "+++='!%L"??PPQR^Q_ `` !`	  ,/(,-) **KK%%--!11??',	 +    ###5q +.??!5!5!G!GG1LKKOOL(OMKKOO"I$9$9$;A$>[ $  $ $UQYKq1E1E1P1P0Q R$$/=#H	  h&
q #& $%!'77J'#k1*==NKKOO.UQYOO 1T-A-A-L-L,M N$S)>#2FH	  	T11EEEJ!#((33]fUUVYKW]^lmp]qDr" .KK++AI" $ 0 0!%!2!2	  "5eai[A Y <d WW\\$"6"6"A"A7K
*79I9I9Q9Q9S)T-1->->N>*KK###	
 04{{/J/JDKK''+PS 	 3J<@
 	
w	 # 	NNMcUS 	s?   A)n,!m$ U5nP	nEn$	n-nnnnc              #     K   d t        j                  | j                  j                  d       |j	                         }t        | j                  j                  t        | j                  dd             }|sd y t        || j                  j                  | j                  j                        }t        dt        j                  t        |      | j                  j                   z              }|| j                  j"                  z  }t%        | j                  j&                  t        d|dz              }t)        |d	d
|      }	t+        |t        d||z
        d| j                  j                  dz        }
t-        ||	|
g|g      }d}d}d}|j/                  d       | j                  j                  j0                  j3                          t5        | j                  j"                        D ].  }d}d}t7        j6                         }|D ]\  }|r)|j9                  dd      r||t        |d      z  df   y | j                  j;                  |      }|| j                  j                   z  }|j=                          ||j?                         z  }|dz  }|| j                  j                   k\  st@        jB                  jD                  jG                  || j                  jH                         |jK                          |jK                          |j/                  d       |dz  }||z  }|| j                  jL                  z  dk(  r||d|dz    d| d|df ||z  }|dz  }d}d}_ |dkD  rt@        jB                  jD                  jG                  || j                  jH                         |jK                          |jK                          |j/                  d       |dz  }||z  }|| j                  jL                  z  dk(  r||d|dz    d| d|df ||z  }|dz  }d}d}t7        j6                         |z
  }|t        |d      z  }||d|dz    d| j                  j"                   d|ddf |dz   | j                  jN                  z  dk(  st         jP                  jS                  | j                  j                  dd|dz    d |d      }tU        | j                  jV                  |||dz   ||| jX                  | jZ                  !       ||d"f 1 t         jP                  jS                  | j                  j                  d#      }d$| jX                  j]                         i}| jZ                  r| jZ                  |d%<   t_        | j                  jV                  ||&       | j                  j`                  r| j                  j`                  d'   nd}||d(| f y w))Nr  Tr6  r|   rU  r`  r$   rd  rS  r   re  ri  rj  rn  r   r	  r}  r  Fr  r  r  r  r  r  rJ  r  r  r  r  r  r  r	  r  r  r  r  r
  r   r  )1r!  r  r   r  r  r   r<   r   r   r   r  rb  rn   r  r  re   r  r  r  r  r   r   r   r  r   r  r  r  rU   r   r  rj   r&   r   r  r  r  r  r  r  r"  r  r   r|   r  r  r  r   r   )r   r3  r  r  r   rR   r  r  r  r  r  rr  ry  r  r  rx  r  r  r  r   r  r  r  r  r  r  r  r  s                               r+   r.  zLoKRTrainer._train_basict  s    
 =<
D((33dC"3359KKDKK5
  >>##11--::
	
 IIL!D$8$8$T$TT
 &(<(<(G(GG4//<<c![TVEV>WX#CC\
 5A{\12((66=	
 !(.9$~
	 -!!'')4//::;EJK#yy{%!n&8&8&N#(3/@!+DD1 
 {{007d22NNN DIIK/ !Q&! &++GGH HHNN22($*>*>*L*L NN$NN$''D'91$K/2CCH"T%9%9%K%KKqP'$$UQYKw{m8HUX>Z  (*J1$K'*$()%M &P !1$..$d&:&:&H&H   ###5q +.??!5!5!G!GG1L#  7;-xQT~V  h&
q #& $%!'77J'#k1*==NUQYKq)=)=)H)H(IjY\M]]^_  	T11EEEJ!#((33]fUUVYKW]^lmp]qDr" .KK++AI" $ 0 0!%!2!2	 ">3JJJs <v WW\\$"6"6"A"A7K
*79I9I9Q9Q9S)T-1->->N>*KK###	
 04{{/J/JDKK''+PS 	 3J<@
 	
s   J1W,4G&W,EW,c                     d| _         yr  r  r  s    r+   r  zLoKRTrainer.stop  r  r-   rX  )r   r   r   r   r   r   r   r(   r   r   r   r   rg   rD   r4  r   r-  r.  r  r:   r-   r+   r  r    s    ;/  / (	/( *.@%@% !@% 
5eS)45	6	@%DS
+S
 !S
 
5eS)45	6	S
j
_
+_
 !_
 
5eS)45	6	_
B!r-   r  rX  )   )Ur   r!  r  r&  r  typingr   r   r   r   r   r   logurur	   r&   torch.nnr   torch.nn.functional
functionalr   
contextlibr
   torch.optimr   torch.optim.lr_schedulerr   r   r   lightning.fabricr   lightning.fabric.loggersr   r,  ImportErrorr   bitsandbytesr  r  acestep.training.configsr   r   r   acestep.training.lora_injectionr   acestep.training.lora_utilsr    acestep.training.lora_checkpointr   r   r   acestep.training.lokr_utilsr   r   r   r   acestep.training.data_moduler   acestep.training.path_safetyr   r   r(   r,   rA   r8   r;   r   rg   rH   	ParameterrQ   r  	OptimizerrW   r_   r{   r   r   r   r   r   r   r   r   r  r  r:   r-   r+   <module>r+     s   
    > >      "  X X	':HG
 L K @ < 
  @ 2	 3 3 s u{{ # # ")) c3h &4(:(:#; &c3h &U[[-B-B uSRUX " <@
II
%-bii%8
	#s(^
  )/##$)/CH~)/ )/ 3T#Y	)/X7II7$,RYY$77	%((

78		 d *")) RYY <0F 0FuT4QUEU?V 0Ff4qRYY qhL! L!^[RYY [|N
! N
!]+  FNNP  HGFNNFGHs$   I  "I   II I=<I=