
    9i                         S SK r S SKJr  S SKJr   " S S\ R
                  R                  5      r " S S\R                  5      r	S r
 SS jrS	 rS
 rSS jrg)    N)	is_masterc                   4    \ rS rSr\S 5       r\S 5       rSrg)SparseBinarizer	   c                     [        UR                  5       U-  5      n[        R                  " UR	                  S5      5      S U nUR                  5       R                  S5      nSUR	                  S5      U'   U$ )N           )intnumeltorchargsortreshapeclonefill_)ctxmask_scoressparsity	num_pruneprune_indicesmasks         k/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/trainers/hooks/compression/utils.pyforwardSparseBinarizer.forward   sk    ))+h67	k&9&9"&=>z	J  "((+*-R'    c                 
    US 4$ N )r   
gradOutputs     r   backwardSparseBinarizer.backward   s    4r   r   N)__name__
__module____qualname____firstlineno__staticmethodr   r    __static_attributes__r   r   r   r   r   	   s(         r   r   c                   H   ^  \ rS rSrSr      SU 4S jjrS rS rSrU =r	$ )SparseLinear   z6
Fully Connected layer with on the fly adaptive mask.
c                 v  > [         [        U ]  5         Xl        U R                  R                  R
                  S   nU R                  R                  R
                  S   n	U R                  R                  U l        S U R                  l        U R                  R                  R                  S5        X l        SU l	        U R                  S:X  Ga  X0l
        X@l        XPl        X`l        Xpl        [        R                   " ["        R$                  " XR                  5      R'                  U R                  R(                  U R                  R*                  S95      U l        [        R                   " ["        R.                  " U R                  U	5      R'                  U R                  R(                  U R                  R*                  S95      U l        [        R                   " ["        R$                  " XR                  5      R'                  U R                  R(                  U R                  R*                  S95      U l        [        R                   " ["        R.                  " U R                  U	5      R'                  U R                  R(                  U R                  R*                  S95      U l        [        R                   " ["        R.                  " U5      R'                  U R                  R(                  U R                  R*                  S95      U l        [        R                   " ["        R.                  " U	5      R'                  U R                  R(                  U R                  R*                  S95      U l        SU R                  l        U R                  R<                  b  SU R                  R<                  l        g g g )Nr   r	   weightr
   pst)devicedtypeF)superr)   __init__moduler,   shape_parameterspoppruning_methodcur_sparsityweight_rankweight_beta	mask_rankmask_alpha1mask_alpha2nn	Parameterr   randntor.   r/   weight_Uzerosweight_Vmask_scores_Amask_scores_Bmask_scores_Rmask_scores_Crequires_gradbias)selfr2   r6   r8   r9   r:   r;   r<   out_featuresin_features	__class__s             r   r1   SparseLinear.__init__   s    	lD*,{{))//2kk((..q1kk((!##H-,%'**&N** LLL*:*:;>>;;--T[[5F5F ? HIDM LLD,,k:==;;--T[[5F5F > HIDM "$L..9<<;;--T[[5F5F = H"ID "$DNNK8;;;;--T[[5F5F < H"ID "$L),,;;--T[[5F5F - H"ID "$K(++;;--T[[5F5F , H"ID ).DKK%{{+16  . ,; (r   c                 "   U R                   S:X  a  U R                  U R                  U R                  -  U R                  -  -   nUR                  5       U R                  U R                  -  U R                  -  -   U R                  U R                  R                  S5      U R                  R                  S5      -   -  -   n[        R                  X0R                  5      nXB-  nXPR                   l        U R                   " U6 $ U R                   " U6 $ Nr-   r	   r   )r6   r,   r9   rA   rC   absr;   rD   rE   r<   rF   	unsqueezerG   r   applyr7   r2   )rJ   inputsr,   r   r   masked_weights         r   r   SparseLinear.forwardT   s    %'[[4#3#3dmm#Cdmm#SSF

""T%7%77$:L:LLM""d&8&8&B&B1&E(,(:(:(D(DQ(G'H II  #((6G6GHD MM!.KK;;'';;''r   c                 *   U R                   S:X  Ga  U R                  U R                  U R                  -  U R                  -  -   nUR                  5       U R                  U R                  -  U R                  -  -   U R                  U R                  R                  S5      U R                  R                  S5      -   -  -   n[        R                  X R                  5      nX1-  n[         R"                  " UR$                  5      U R&                  l        g g rP   )r6   r,   r9   rA   rC   rQ   r;   rD   rE   r<   rF   rR   rG   r   rS   r7   r=   r>   datar2   )rJ   r,   r   r   rU   s        r   convertSparseLinear.converte   s    %'[[4#3#3dmm#Cdmm#SSF

""T%7%77$:L:LLM""d&8&8&B&B1&E(,(:(:(D(DQ(G'H II  #((6G6GHD MM!#m.@.@!ADKK (r   )r7   r;   r<   r:   rD   rE   rG   rF   r2   r6   r,   rA   rC   r9   r8   )r-            ?r[   r\   r\   )
r"   r#   r$   r%   __doc__r1   r   rY   r'   __classcell__)rM   s   @r   r)   r)      s3     57n("B Br   r)   c                 r    UR                  S5      nUS S  H  n[        X5      n M     [        XS   U5        g )N.r   )splitgetattrsetattr)modelnamer2   	name_lists       r   _setattrrg   t   s7    

3I#2$ ER=&)r   c           
         [         R                  /n SSKJn	  UR	                  U	R
                  U	R                  /5        U R                  5        Hq  u  p[        U5      U;   d  M  [        UUUUUUU5      n[        X
U5        [        5       (       d  MD  U(       a  UR                  SU
 S35        Mb  [        SU
 S35        Ms     g ! [         a     Nf = f)Nr   )mpuconvert z to sparse module.)r=   Linearmegatron_utilri   extendRowParallelLinearColumnParallelLinearImportErrornamed_modulestyper)   rg   r   infoprint)rd   r6   r8   r9   r:   r;   r<   loggercompress_moduleri   re   r2   
new_modules                r   convert_sparse_networkrx   {   s     yykO%""C$<$<=	?
 ++-<?*%J U*-{{KK(4&0B CDHTF*<=>' .  s   -C 
CCc                 p    U R                  5        H"  u  p#[        U[        5      (       d  M  Xl        M$     g r   )rq   
isinstancer)   r7   )rd   r   re   r2   s       r   update_network_sparsityr{      s*    ++-fl++"* .r   c                     XU-  ::  a  UnU$ XXA-  -
  :  a  UnU$ X1-  nXC-   U-  n	X-
  U-  U-  n SXU	-
  -  -
  n
XeU-
  U
S-  -  -   nU$ )Nr	      r   )step
total_step	frequencyinitial_warmupfinal_warmupinitial_sparsityfinal_sparsityr   spars_warmup_stepsspars_schedu_steps	mul_coeffs              r   schedule_sparsity_ratior      s     
**# O 
l77	8! O ,8*;zI)i7)C-? ?@@	!%FqL% Or   c                 *   U R                  5        H  u  p#[        U[        5      (       d  M  UR                  5         [	        XUR
                  5        [        5       (       d  MS  U(       aV  UR                  SU S[        R                  " SUR
                  R                  S:H  -  5      R                  5        S35        M  [        SU S[        R                  " SUR
                  R                  S:H  -  5      R                  5        S35        GM     g )Nrj   zE weight to sparse weight,                             sparsity ratio=r\   r   r`   z> weight to sparse,                             sparsity ratio=)rq   rz   r)   rY   rg   r2   r   rs   r   meanr,   itemrt   )rd   ru   re   r2   s       r   generate_sparse_modelr      s    ++-fl++NNU&--0{{KK(4& 1,,1JJsFMM<P<PRS<S7T,U,Z,Z,\+]]^!` " HTF +,,1JJsFMM<P<PRS<S7T,U,Z,Z,\+]]^`  .r   r   )r   torch.nnr=   modelscope.utils.torch_utilsr   autogradFunctionr   Moduler)   rg   rx   r{   r   r   r   r   r   <module>r      sZ      2 enn--  YB299 YBx* %?P+.r   