
    9i62                        S SK r S SKrS SKrS SKrS SKrS SKJrJrJr  S SK	r
S SKrS SKJr  S SKJr  S SKJr  S SKJrJr  S SKJrJr  S SKJr  S S	KJr  S S
KJr  S SKJrJr  S SKJ r   S SK!J"r"  S SK#J$r$  S SK%J&r&J'r'J(r(J)r)  \$" 5       r*Sr+Sr,Sr-Sr.Sr/Sr0Sr1Sr2Sr3Sr4Sr5\Rl                  " \Rn                  S9 " S S\5      5       r8g)    N)CallableDictOptional)nn)optim)Trainers)Model
TorchModel)KWSDataLoader
KWSDataset)BaseTrainer)TRAINERS)update_conf)DEFAULT_MODEL_REVISION	ModelFile)	to_device)create_device)
get_logger)get_dist_infoget_local_rank	init_dist	is_masterbasetrain_easybasetrain_normalbasetrain_hardfinetune_easyfinetune_normalfinetune_hard
checkpointg?g333333?g333333?g      ?)module_namec                      ^  \ rS rSrSr\\\\\	\
4rSS\S4S\S\S\\   S\\   S\\   S	\\   4U 4S
 jjjrS\R&                  4S jrS rS rS rS rS\S\\\4   4S jrS rSrU =r$ )KWSFarfieldTrainer+   z
./work_dirNmodelwork_dircfg_filearg_parse_fnmodel_revisioncustom_confc                   > [        U[        5      (       aR  U R                  X5      U l        Uc8  [        R
                  R                  U R                  [        R                  5      nO.Uc   S5       e[        R
                  R                  U5      U l        [        TU ]-  X45        UR                  SS 5      nU(       a  XR                  R                  l        U R                  R                  R                  U l        U R#                  5       U l        X l        UR                  SS 5      b  ['        US   5        [)        5       u  pU
S:  U l        UR                  SS5      nU R*                  (       a  [-        5       nSU 3n[/        U5      U l        U R0                  R2                  S:X  a%  U R                  R5                  U R0                  5        S	U;  aR  [7        U R                  R8                  S	5      (       d   S
5       eU R                  R8                  R:                  U l        O
US	   U l        UR                  SS 5      U l        UR                  SS 5      U l         U R>                  c%  U R                  R8                  RB                  U l        U R@                  c%  U R                  RD                  RF                  U l         U R                  R8                  RH                  nUR                  SS 5      U l%        U RJ                  c  URL                  U l%        [N        U l(        SU;   a
  US   U l(        URR                  U l*        UR                  SS5      nUS-
  U l+        SU;   aH  [        R
                  R                  U R                  US   5      n[X        RZ                  " USS9U l        OU RV                  S:  a  [        R
                  R                  U R$                  [\         SU RV                  S S35      n[^        R^                  " U5      n[a        U5      S:X  a7  [b        Re                  SUS   5        [X        RZ                  " US   SS9U l        O,[a        U5      S:X  a  [g        SU S35      e[i        SU 35      eU R                  R8                  Rj                  Rl                  n[n        Rp                  " U R                  Rs                  5       U5      U l5        [t        Rv                  " 5       U l<        S U l=        [        R
                  R                  U R$                  SR}                  U R~                  5      5      U l@        / U lA        U R                   H  n[        R
                  R                  U R                  U5      n[        R
                  R                  U R$                  U S35      n[        UUUU   5        U R                  R                  U5        M     [        R                  " U R<                  [        -  5      [        R                  " U R<                  [        -  5      [        R                  " U R<                  [        -  5      4U lJ        g )Nz?Config file should not be None if model is not from pretrained!num_synlauncher   devicegpuzcuda:cuda
max_epochsz1max_epochs is missing from the configuration filetrain_iters_per_epochval_iters_per_epochworkerssingle_rate
next_epoch	model_binT)weights_onlyr   _04dz*.pthz!Loading model from checkpoint: %sz$Failed to load checkpoint file like z. File not found!z7Expecting one but multiple checkpoint files are found: z{}.log.jsonz.conf)K
isinstancestrget_or_download_model_dir	model_dirospathjoinr   CONFIGURATIONdirnamesuper__init__getcfgr$   r+   _num_classesbuild_modelr%   r   r   _distr   r   r.   typetohasattrtrainr1   _max_epochs_train_iters
_val_itersr2   
evaluationr3   
dataloader_threadsworkers_per_gpuBASETRAIN_RATIO_single_ratebatch_size_per_gpu_batch_size_current_epochtorchloadCKPT_PREFIXgloblenloggerinfoFileNotFoundErrorAssertionError	optimizerlrr   Adam
parametersr   CrossEntropyLossloss_fndata_valformat	timestampjson_log_path
conf_files	conf_keysr   appendmathfloor
EASY_RATIONORMAL_RATIO
HARD_RATIOstages)selfr$   r%   r&   r'   r(   r)   kwargsr+   r9   
world_sizedevice_name
local_rankdataloader_configr6   model_bin_fileckpt_file_pattern
ckpt_filesre   conf_keytemplate_file	conf_file	__class__s                         n/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/trainers/audio/kws_farfield_trainer.pyrE   KWSFarfieldTrainer.__init__2   s    eS!!!;;'DN77<<(1(?(?A 'j)jj'WW__X6DN0 **Y-%,HHNN" HHNN22%%'
 ::j$'3fZ()%!^
jj51::')J!*.K#K0;;v%JJMM$++&v%  CBC   $xx~~88D%l3D"JJ'>E **%:DA$ $ D DD??""hh11EEDO HHNN55

9d3== -==DM+F" &} 5D,??ZZa0
(1n& WW\\$..&:MNNNFDJ  1$ "+a0C0CC/HN!P#45J:!#?AO"ZZ
1DI
ZA%':()):<= = % (::D&G H H XX^^%%((DJJ$9$9$;R@**,WW\\$--*7*>*>t~~*NPHGGLLBMT]]xj4FGIy+h2GHOO""9-	 '
 zz$"2"2Z"?@zz$"2"2\"ABzz$"2"2Z"?@B    returnc                     [         R                  " U R                  U R                  SS9n[	        U[
        5      (       a  [        US5      (       a  UR                  $ [	        U[        R                  5      (       a  U$ g)zInstantiate a pytorch model and return.

By default, we will create a model using config from configuration file. You can
override this method in a subclass.

T)cfg_dicttrainingr$   N)
r	   from_pretrainedr>   rG   r;   r
   rM   r$   r   Module)rw   r$   s     r   rI   KWSFarfieldTrainer.build_model   sa     %%NNTXX>eZ((WUG-D-D;;ryy))L *r   c                    U R                   (       d  U R                  5         [        R                  S5        [        R                  R                  5       nSn[        U R                  5       H(  u  pVXF-  nX@R                  -
  nU R                  XW5        M*     [        R                  R                  5       U-
  n[        R                  SR                  UR                  5       S-  5      5        g )NzStart training...r   zTotal time spent: {:.2f} hours
      @)rj   gen_valr`   ra   datetimenow	enumeraterv   rZ   	run_stagerk   total_seconds)rw   argsrx   	totaltimenext_stage_head_epochstage	num_epochepochs_to_runs           r   rN   KWSFarfieldTrainer.train   s    }}LLN'(%%))+	 ! )$++ 6E!.!14G4GGMNN50 !7 %%))+i7	6==##%.0 	1r   c           
         US::  a  [         R                  SU S35        g[         R                  SU S35        U R                  U R                  US-     U R                  US-  S-      5      u  p4[        U5      n[        U5       GH7  nU =R                  S-  sl        [        R                  R                  5       n[         R                  S	U R                  5        S
nSn	[        U R                  5       GH  n
[        U5      u  p[        R                  " US5      n[        XR                  5      n[        XR                  5      nU R                   R#                  5         U R%                  U5      nU R'                  [        R                  " USU R(                  45      U5      n[*        R,                  " UR/                  5       5      (       dA  UR1                  5         U R                   R3                  5         XR/                  5       -  nU	S-  n	SR5                  U R                  U R6                  U
S-   U R                  UR/                  5       5      n[         R                  U5        U R9                  U5        GM     X-  nU R;                  S5      nSR5                  U R                  UU5      n[         R                  U5        U R9                  U5        SR5                  [<        U R                  UU5      n[>        R@                  RC                  U RD                  U5      n[         R                  SU 35        [        RF                  " U R$                  U5        [        R                  R                  5       U-
  n[         R                  SR5                  U R                  URI                  5       S-  5      5        GM:     URK                  5         URM                  5         [         R                  SU S35        g)z
Run training stages with correspond data

Args:
    stage: id of stage
    epochs_to_run: the number of epoch to run in this stage
r   zInvalid epoch number, stage z exit!NzStarting stage z...   r-   zStart epoch %d...        r   z8Epoch: {:04d}/{:04d}, batch: {:04d}/{:04d}, loss: {:.4f} z<Evaluate epoch: {:04d}, loss_train: {:.4f}, loss_val: {:.4f}z/{}_{:04d}_loss_train_{:.4f}_loss_val_{:.4f}.pthzSave model to z%Epoch {:04d} time spent: {:.2f} hoursr   zStage z is finished.)'r`   warningra   create_dataloaderrn   iterrangerZ   r   r   rP   nextr[   reshaper   r.   rd   	zero_gradr$   ri   rH   npisnanitembackwardsteprk   rO   	_dump_logevaluater]   r?   r@   rA   r%   saver   stoprelease)rw   r   r   datasetrS   itr9   	epochtimeloss_train_epochvalidbatchsbifeatlabelpredictlosstrain_resultloss_val_epoch
val_result	ckpt_name	save_paths                       r   r   KWSFarfieldTrainer.run_stage   s&    ANN9%GHoeWC01"44OOEAI&	A(FH*}%A1$ ))--/IKK+T-@-@A"KD--."2heV4 {{3!%5((***T*||MM'B0A0A+BCULxx		,,MMONN'')$		3$1$KY``'')9)926%%tyy{ 4 L)|,+ /0 +!]]2.NW^^##%5~GJKK
#NN:&IPPT002B I T]]I>IKK.45JJtzz9- ))--/);IKK?FF##'')F24 5[ &` 	fUG=12r   c                    [         R                  R                  U R                  S5      nU R                  S:  a[  [
        R                  S5        [        US5       n[        R                  " U5      U l
        SSS5        [
        R                  S5        g[
        R                  S5        U R                  U R                  S   U R                  S	   5      u  p4[        U5      n/ U l
        [        U R                  5       HY  n[
        R                  S
U5        [!        U5      u  px["        R$                  " US5      nU R                  R'                  Xx/5        M[     UR)                  5         UR+                  5         [        US5       n[        R,                  " U R                  U5        SSS5        [
        R                  S5        g! , (       d  f       GNX= f! , (       d  f       N6= f)z
generate validation set
zval_dataset.binr   zStart loading validation set...rbNzFinish loading validation set!z"Start generating validation set...r      zIterating validation data %dr   wbz!Finish generating validation set!)r?   r@   rA   r%   rZ   r`   ra   openpickler\   rj   r   rn   r   r   rQ   r   r[   r   rp   r   r   dump)	rw   val_dump_filefr   rS   r   r   r   r   s	            r   r   KWSFarfieldTrainer.gen_val   sZ    T]]4EF"KK9:mT*a &A +KK8989"44T__Q5G59__Q5GI*(BKK6;r(KDMM%0EMM  $/	 ) 	-&!KKq) '78- +*( '&s   G		"G	
G
G)c                     [        XU R                  U R                  U R                  5      n[	        X0R
                  U R                  S9nUR                  5         X44$ )N)	batchsize
numworkers)r   rT   rW   rH   r   rY   start)rw   	base_pathfinetune_pathr   rS   s        r   r   $KWSFarfieldTrainer.create_dataloader  sU    Yt}}!..0A0AC"//DMMK
""r   checkpoint_pathc           	         [         R                  S5        Sn[        R                  " 5          U R                   H  u  pV[        XPR                  5      n[        X`R                  5      nU R                  U5      nU R                  [        R                  " USU R                  45      U5      nXHR                  5       -  nM     S S S 5        [         R                  S5        X@R                  -  $ ! , (       d  f       N1= f)NzStart validation...r   r   zFinish validation.)r`   ra   r[   no_gradrj   r   r.   r$   ri   r   rH   r   rQ   )	rw   r   r   rx   r   r   r   r   r   s	            r   r   KWSFarfieldTrainer.evaluate$  s    )*]]_#}} {{3!%5**T*||MM'B0A0A+BCUL))+-  -  	()// _s   BC..
C<c                     [        5       (       aB  [        U R                  S5       nUR                  U5        UR                  S5        S S S 5        g g ! , (       d  f       g = f)Nza+
)r   r   rm   write)rw   msgr   s      r   r   KWSFarfieldTrainer._dump_log6  sE    ;;d(($/1 0/ //s   #A
A!)rY   rZ   rJ   rO   rH   rW   rT   rP   rQ   rn   rj   r.   rm   ri   r$   r>   rd   rv   r%   )__name__
__module____qualname____firstlineno__DEFAULT_WORK_DIRBASETRAIN_CONF_EASYFINETUNE_CONF_EASYBASETRAIN_CONF_NORMALFINETUNE_CONF_NORMALBASETRAIN_CONF_HARDFINETUNE_CONF_HARDro   r   r<   r   r   dictrE   r   r   rI   rN   r   r   r   r   floatr   r   __static_attributes____classcell__)r   s   @r   r"   r"   +   s    #$&8&(<$&8:I ,0481G/3gBgBgB $C=gB  (1	gB
 "*#gB 'tngB gBRRYY 1"A3F9>#0 0"3:.0$ r   r"   )9r   r^   rq   r?   r   typingr   r   r   numpyr   r[   r   r   modelscope.metainfor   modelscope.modelsr	   r
   7modelscope.msdatasets.dataset_cls.custom_datasets.audior   r   modelscope.trainers.baser   modelscope.trainers.builderr   "modelscope.utils.audio.audio_utilsr   modelscope.utils.constantr   r   modelscope.utils.data_utilsr   modelscope.utils.devicer   modelscope.utils.loggerr   modelscope.utils.torch_utilsr   r   r   r   r`   r   r   r   r   r   r   r]   rs   rt   ru   rV   register_modulespeech_dfsmn_kws_char_farfieldr"    r   r   <module>r      s       	  + +      ( / 0 0 : G 1 1 .@ @ 
& * & $ ( $ 

 
h&M&MNN N ONr   