
    ^ie                     .   d Z ddlmZmZmZmZ ddlmZ ddlZddl	m
Z
 ddlmZ d Z	 ddlmZmZmZmZ dZde
j,                  de
j,                  fdZdee   fdZddeddfdZdedeeeeef   f   fdZy# e$ r d	Z ej*                  d
       Y \w xY w)zs
LoRA Injection Utilities for ACE-Step

Provides functions for injecting LoRA adapters into the DiT decoder model.
    )ListTupleAnyDict)loggerN)
LoRAConfigc                    t        | dd      }	 | |       }nd}	 d| _        |S # t        $ r t        j                  dd       Y |S w xY w# t
        $ r 	 d| _        n&# t        $ r t        j                  dd       Y nw xY wt        | dd      sFt        j                  d       	 d| _        Y y# t        $ r t        j                  d	d       Y Y yw xY wY yw xY w)
a=  Safely call enable_input_require_grads on the decoder.

    This helper wraps the original enable_input_require_grads method,
    handling NotImplementedError gracefully and tracking whether the hook
    was successfully enabled.

    Args:
        self: The decoder module to call enable_input_require_grads on.
    (_acestep_orig_enable_input_require_gradsNTz/Failed to set _acestep_input_grads_hook_enabled)exc_infoF$_acestep_input_grads_warning_emittedzkSkipping enable_input_require_grads for decoder: get_input_embeddings is not implemented (expected for DiT)z2Failed to set _acestep_input_grads_warning_emitted)getattr!_acestep_input_grads_hook_enabled	Exceptionr   debugNotImplementedErrorinfor   )selforig_enable_input_require_gradsresults      V/mnt/workspace/acestep.cpp/tests/../../ACE-Step-1.5/acestep/training/lora_injection.py _safe_enable_input_require_gradsr      s	    '.8$'#*646FF	59D2
 	  	LLAD 		
  	5:D2 	LLAD	 tCUKKKM<@9
 	  HSW 	 %ss   A % A  A	A A		A 	CAC B>C B%C'B00 CCCCC)get_peft_model
LoraConfigTaskType	PeftModelTFz@PEFT library not installed. LoRA training will not be available.modulereturnc                 \   | }t        |      h}	 t        |dd      }|n$t        |      }||v rn|j                  |       |}4t        |dd      }|.t        |dd      }|t        |t        j
                        r|}n|}t        |dd      }|t        |t        j
                        r|}|S )av  Unwrap PEFT/Fabric wrappers from a model/decoder to retrieve the base DiT module.

    This internal helper walks the wrapper chain and returns the underlying
    ``nn.Module`` that can be passed to PEFT for adapter injection.

    Args:
        module: A model or decoder that may have PEFT/Fabric wrappers.

    Returns:
        The unwrapped base DiT decoder module.
    _forward_moduleN
base_modelmodel)idr   add
isinstancennModule)r   decoderseen_idsnext_decodernext_idr    inner_modelfinal_models           r   _unwrap_decoderr-   M   s     G7}H
w(94@\"hW  ,5Jj'48"z+ryy'I!G G'7D1K:k299#EN    c                     g }t        | d      rc| j                  j                         D ]F  \  }t        fddD              st	        |t
        j                        s6|j                         H |S )zGet the list of module names in the DiT decoder that can have LoRA applied.

    Args:
        model: The AceStepConditionGenerationModel

    Returns:
        List of module names suitable for LoRA
    r'   c              3   &   K   | ]  }|v  
 y wN ).0projnames     r   	<genexpr>z)get_dit_target_modules.<locals>.<genexpr>   s     U,TD44<,Ts   )q_projk_projv_projo_proj)hasattrr'   named_modulesanyr$   r%   Linearappend)r!   target_modulesr   r5   s      @r   get_dit_target_modulesrA   t   sb     Nui !MM779LD&U,TUUfbii0"))$/ :
 r.   freeze_encoderc                   	 d}| j                         D ]F  \  	}d	v }	j                  |      xs t        	fd|D              }|rd|_        ;|s|r@d|_        H d}d}| j                         D ]8  \  }}||j	                         z  }|j                  s&||j	                         z  }: t        j                  d||z
  d       t        j                  d	|d       y
)zFreeze all non-LoRA parameters in the model.

    Args:
        model: The model to freeze parameters for
        freeze_encoder: Whether to freeze the encoder (condition encoder)
    )encodertext_encodervision_encoderzmodel.encoderlora_c              3   F   K   | ]  }j                  | d         yw).N)
startswith)r3   prefixr5   s     r   r6   z-freeze_non_lora_parameters.<locals>.<genexpr>   s%      >
8HfDOOvhaL)8Hs   !TFr   zFrozen parameters: ,zTrainable parameters: N)named_parametersrJ   r=   requires_gradnumelr   r   )
r!   rB   encoder_prefixesparamis_lora
is_encodertotal_paramstrainable_params_r5   s
            @r   freeze_non_lora_parametersrW      s     V--/eT/__%56 
# >
8H>
 ;

 "&E:"'E 0 L**,5%- -
 KK%l5E&Ea%HIJ
KK()9!(<=>r.   lora_configc                 .   t         st        d      t        | j                        }|| _        t	        |d      r>t	        |d      s2|j
                  }||_        t        j                  t        |      |_        t	        |d      r	 d|_
        t        |j                  |j                  |j                  |j                   |j"                  t$        j&                        }t)        ||      }|| _        | j+                         D ]  \  }}d|vsd|_         t/        d | j1                         D              }t/        d	 | j1                         D              }	||	|d
kD  r|	|z  nd
|j                  |j                  |j                   d}
t3        j4                  d       t3        j4                  d|d       t3        j4                  d|	dd|
d   dd       t3        j4                  d|j                   d|j                          | |
fS # t        $ r Y w xY w)zInject LoRA adapters into the DiT decoder of the model.

    Args:
        model: The AceStepConditionGenerationModel
        lora_config: LoRA configuration

    Returns:
        Tuple of (peft_model, info_dict)
    zJPEFT library is required for LoRA training. Install with: pip install peftenable_input_require_gradsr
   is_gradient_checkpointingF)r
lora_alphalora_dropoutr@   bias	task_typerG   c              3   <   K   | ]  }|j                           y wr1   )rO   r3   ps     r   r6   z'inject_lora_into_dit.<locals>.<genexpr>   s     =*<Qqwwy*<s   c              3   V   K   | ]!  }|j                   s|j                          # y wr1   )rN   rO   rb   s     r   r6   z'inject_lora_into_dit.<locals>.<genexpr>   s     T.@AOO1779.@s   ))r   )rT   rU   trainable_ratiolora_rr]   r@   zLoRA injected into DiT decoder:z  Total parameters: rL   z  Trainable parameters: z (re   z.2%)z  LoRA rank: z	, alpha: )PEFT_AVAILABLEImportErrorr-   r'   r;   rZ   r
   types
MethodTyper   r[   r   r   r\   alphadropoutr@   r_   r   FEATURE_EXTRACTIONr   rM   rN   sum
parametersr   r   )r!   rX   r'   origpeft_lora_configpeft_decoderr5   rQ   rT   rU   r   s              r   inject_lora_into_ditrt      s    X
 	
 emm,GEMw45g;? 11;?8-2-=-=,g.
* w34	05G- "
--$$ (("11-- "'+;<L EM--/e$"'E 0 =%*:*:*<==LTe.>.>.@TT %,>JQ>N+l:TU--!''%44D KK12
KK&|A&678
KK
"#3A"6b>O9PQT8UUVW KK-i8I8I7JKL$;M  		s   H 	HH)T)__doc__typingr   r   r   r   logurur   rj   torch.nnr%   acestep.training.configsr   r   peftr   r   r   r   rh   ri   warningr&   r-   strrA   boolrW   rt   r2   r.   r   <module>r~      s    * )    /,^W  N$BII $")) $NT#Y (?d ?d ?@GG 3S#XGA  WNFNNUVWs   A8 8BB