hti ddlZddlmcmZGddZdgfdej dej fdZddd dgfd ej d ej d ed ededef dZ dZ dZ dZ ddej dej dej dej d edede de fdZ ddej dej dej ded edef dZdej dej dej ded ef dZy) Nc:eZdZddefdZdej fdZy)MomentumBuffermomentumc ||_d|_y)Nrrrunning_average)selfrs Y/mnt/workspace/acestep.cpp/tests/../../ACE-Step-1.5/acestep/models/common/apg_guidance.py__init__zMomentumBuffer.__init__s    update_valuecJ|j|jz}||z|_yNr)r r new_averages r updatezMomentumBuffer.update s#mmd&:&:: +k9r N)g)__name__ __module__ __qualname__floatr torchTensorrr r rrs!!:5<<:r rv0v1c|j}|jj}|dk(r |j|j}}|j |j }}t j jj||}||zj|d|z}||z }|j|j||j|j|fS)NmpsdimTrkeepdim) dtypedevicetypecpudoublernn functional normalizesumto)rrdimsr" device_type v0_parallel v0_orthogonals r projectr0s HHE))..Ke2668B YY["))+B    & &rt & 4B7--D$-7">% # #K 0-2B2B52I2L2L[2Y YYr gg@ pred_cond pred_uncondguidance_scalemomentum_bufferetanorm_thresholdc,||z }||j||j}|dkDrGtj|}|j d|d} tj ||| z } || z}t |||\} } | || zz} ||dz | zz}|S)NrT)prr!)rrr ones_likenormminimumr0)r1r2r3r4r5r6r,diffones diff_norm scale_factor diff_paralleldiff_orthogonalnormalized_update pred_guideds r apg_forwardrF!s { "D"t$..t$IItTI: }}T>I+EF l"%,T9d%C"M?'# *==~15FFFK r c||||z zzSrr) cond_output uncond_output cfg_strengths r cfg_forwardrK;s <;+FG GGr c|tjj|ddz }|tjj|ddz }tj||zdd}|S)z Calculate cosine similarity between two normalized tensors. Args: tensor1: First tensor [B, ...] tensor2: Second tensor [B, ...] Returns: Cosine similarity value [B, 1] r:Tr )rlinalgr<r*)tensor1tensor2cosvalues r call_cos_tensorrQ?s` ))'q$)GGG ))'q$)GGGyy7*4@H Or c|j\}}}|j||z|j}|j||z|j}|j|jk7r t dt j ||zdd}t j ||zdd}||dzz |z}||z }|j||||j|||fS)a\ Decompose latent_diff into parallel and perpendicular components relative to latent_hat_uncond. Args: latent_diff: Difference tensor [B, C, ...] latent_hat_uncond: Unconditional prediction tensor [B, C, ...] Returns: projection: Parallel component perpendicular_component: Perpendicular component zBlatent_diff and latent_hat_uncond must have the same shape [n, d].r:Tr g:0yE>)shapeviewrsize ValueErrorrr*reshape) latent_difflatent_hat_uncondntc dot_product norm_square projectionperpendicular_components r compute_perpendicular_componentraPsGAq!""1q5!,224K)..q1ua8>>@.3355]^^))K*;;DQK))-0AAqRVWKt!348IIJ)J6 ??1a #%<%D%DQ1%M MMr latentsnoise_pred_condnoise_pred_uncondsigma angle_clip apply_norm apply_clipc8|jd|jdk7r_|jd|jdzdk7r td|jd|jdz}|j|d}|jd} |} | j\} } } t|tt frQt j||j|j}|jdddj| dd}nt j|rz|jdk(r%|jdddj| dd}nY|j| k(r|j| dd}n2td| d|jtdt||dz } | | dkDzd z} ||| zz }|||zz }||z }t!|jd | j#t |j%d | j'j#t j)d d }t j*|j| | d}|rt j,| |z| |n| |z}t/||\}}t j0||z}|t j2|zt j2|z t j2|d kDz|| zt j2|d kzz}||z}|rH|t j4j7|dd zt j4j7|dd z }||z |z }|j%| | | j#|j}|S)a ADG (Angle-based Dynamic Guidance) forward pass for Flow Matching. In flow matching (including SD3), sigma represents the current timestep t_curr. The predictions are velocity fields v(x_t, t). Args: latents: Current state x_t [N, T, d] where d=64 noise_pred_cond: Conditional velocity prediction v_cond [N, T, d] noise_pred_uncond: Unconditional velocity prediction v_uncond [N, T, d] sigma: Current timestep t_curr (not t_prev!) guidance_scale: Guidance strength angle_clip: Maximum angle for clipping (default: pi/6) apply_norm: Whether to normalize the result (ADG_w_norm variant) apply_clip: Whether to clip the angle (ADG_wo_clip when False) Returns: Guided velocity prediction [N, T, d] r:rzYnoise_pred_cond time dimension must be a whole-number multiple of latents time dimension.r)r#r"z6sigma has incompatible shape. Expected scalar or size z, got z&sigma must be a number or tensor, got gMbP?rg !g !?Tr )rSrVrepeat_interleave isinstanceintrrtensorr#r"rTexpand is_tensornumel TypeErrorr$rQr+rW contiguousclampacosclipracossinrMr<)rbrcrdrer3rfrgrhrepeatsrZnoise_pred_textr[r\weightlatent_hat_textrYrX cos_theta latent_thetalatent_theta_newprojperp latent_v_new latent_p_new latent_new noise_preds r adg_forwardrks:}}Q?0033   #gmmA&6 6! ;xy y!''*gmmA.>>++G+; a A%O##GAq!%#u& U7>>O 1a#**1a3   ;;=A JJq!Q'..q!Q7E [[]a JJq!Q'EUVWUXX^_d_j_j^klm m@e NOOa F vz "T )F 77O%*;";;!$55K R#&&u-!!"a(33588? eK$::i(--aA6LU_uzz&<"7*jQeknzez0>OPJD$99-.@L%))$455 ,8OO ,$&(*.-599\;RVZ;Z*[\L ,J%,,"3"3OTX"3"YY\a\h\h\m\m At]n]-- J&%/J##Aq!,// >J r c (t||||||ddS)z ADG with normalization - preserves the magnitude of latent predictions. This variant normalizes the final latent to maintain the same norm as the conditional prediction, which can help preserve image quality. T)rfrgrhr)rbrcrdrer3rfs r adg_w_norm_forwardrs( w&(%","&"& ((r c &t|||||ddS)z ADG without angle clipping - allows unbounded angle adjustments. This variant doesn't clip the angle, which may result in more aggressive guidance but could be less stable. F)rgrhr)rbrcrdrer3s r adg_wo_clip_forwardrs w1BE>fkx} ~~r )gX%?FT)gX%?)rtorch.nn.functionalr'r(Frrr0rrFrKrQraboolrrrrr r rs :: Z Z Z*'+ ||$     4H"NB!Q \\Q\\Q||Q << Q  Q  QQQt! ( \\(\\(||(  (  (  (0  \\ \\ ||     r