
    9iU                        S SK r S SKrS SKJrJrJr  S SKrS SKr	S SK
Js  Jr  S SKrS SKJs  Jr  S SKrS SKJr  S SKJr  S SKJr  S SKJr  S SKJrJr  S SKJr  S S	KJ r   S S
K!J"r"  S SK#J$r$J%r%  S SK&J'r'  S SK(J)r)  S SK*J+r+J,r,J-r-  Sr.\)" 5       r/\"R`                  " \Rb                  S9 " S S\ 5      5       r2 " S S\	Rf                  5      r4g)    N)DictOptionalUnion)autocast)Dataset)tqdm)Trainers)Model
TorchModel)	MsDataset)BaseTrainer)TRAINERS)DEFAULT_MODEL_REVISION	ModelFile)create_device)
get_logger)get_dist_infoget_local_rank	init_distsi-snr)module_namec                       \ rS rSrSrSSS\4S\S\S\\   S\\\	\
4      S\\\	\
4      S	\\   4S
 jjrS\R                  R                  4S jrS rS\S\\\4   4S jrSrg)SeparationTrainer!   aA  A trainer is used for speech separation.

Args:
    model: id or local path of the model
    work_dir: local path to store all training outputs
    cfg_file: config file of the model
    train_dataset: dataset for training
    eval_dataset: dataset for evaluation
    model_revision: the git version of model on modelhub
Nmodelwork_dircfg_filetrain_dataseteval_datasetmodel_revisionc           	      "	   [        U[        5      (       aR  U R                  X5      U l        Uc8  [        R
                  R                  U R                  [        R                  5      nO.Uc   S5       e[        R
                  R                  U5      U l        [        R                  " X5        U R                  5       U l        X l        UR                  SS 5      b  [!        US   5        [#        5       u  pU	S:  U l        UR                  SS5      n
U R$                  (       a  ['        5       nSU 3n
[)        U
5      U l        SU;  aR  [-        U R.                  R0                  S5      (       d   S5       eU R.                  R0                  R2                  U l        O
US   U l        X@l        XPl        [        R
                  R                  U R                  S	5      nU R                  U R.                  R0                  R:                  U R.                  R0                  R<                  R>                  U R.                  R0                  R<                  R@                  U R.                  R0                  R<                  RB                  U R.                  R0                  RD                  RF                  U R.                  R0                  RD                  RH                  U R.                  R0                  RD                  RJ                  S
.nSSK&J'n  [Q        U5       nU" XS9U l)        S S S 5        [T        RV                  " U R                  UUS9  SSSSSSS.nU R*                  RX                  S:X  a2  U R*                  RX                   SU R*                  RZ                   3US'   [T        R\                  R^                  Ra                  U R4                  5      U l1        U Rb                  U RR                  S'   U RR                  S   Re                  SU Rb                  05        U R                  Rg                  5       nU RR                  S   Re                  U5        [i        UU RR                  S   U RR                  UU RR                  S   S9U l5        g ! , (       d  f       GNn= f)Nz?Config file should not be None if model is not from pretrained!launcher   devicegpuzcuda:
max_epochsz1max_epochs is missing from the configuration filezhparams.yaml)output_folderseedlrweight_decayclip_grad_normfactorpatiencedont_halve_until_epochr   )load_hyperpyyaml)	overrides)experiment_directoryhyperparams_to_saver0   Fcpunccl)debugr$   data_parallel_backenddistributed_launchdistributed_backendfind_unused_parameterscuda:epoch_countercheckpointercounter	optimizer)modules	opt_classhparamsrun_optsr=   )6
isinstancestrget_or_download_model_dir	model_dirospathjoinr   CONFIGURATIONdirnamer   __init__build_modelr   r   getr   r   _distr   r   r$   hasattrcfgtrainr&   _max_epochsr   r   r(   r?   r)   r*   r+   lr_schedulerr,   r-   r.   hyperpyyamlr/   openrB   sbcreate_experiment_directorytypeindexutils
epoch_loopEpochCounterr<   add_recoverablesas_dict
Separation	separator)selfr   r   r   r   r   r    kwargs_
world_sizedevice_name
local_rankhparams_filer0   r/   finrC   r@   s                     l/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/trainers/audio/separation_trainer.pyrM   SeparationTrainer.__init__.   s    eS!!!;;'DN77<<(1(?(?A 'j)jj'WW__X6DNT,%%'
 ::j$'3fZ()%!^
jj51::')J!*.K#K0v%  CBC   $xx~~88D%l3D*(ww||DNNNC MMHHNNHHNN$$''HHNN$$11HHNN$$33HHNN''..HHNN''00HHNN''>>!
	& 	1,3+CEDL   	&&!% ,	
 %*"'#)&+
 ;;v%$(KK$4$4#5Qt{{7H7H6I!JHXXX00==d>N>NO(,(:(:_%^$55**+	-**$$&^$55g>#ll;/LLn5
5  s   Q??
Rreturnc                    [         R                  " U R                  U R                  SS9n[	        U[
        5      (       a  [        US5      (       a  UR                  $ [	        U[        R                  R                  5      (       a  U$ g)z0Instantiate a pytorch model and return.
        T)cfg_dicttrainingr   N)r
   from_pretrainedrG   rR   rD   r   rQ   r   torchnnModule)rc   r   s     rk   rN   SeparationTrainer.build_model   se     %%NNTXX>eZ((WUG-D-D;;uxx//L 0    c                     U R                   R                  U R                  U R                  U R                  U R
                  S   U R
                  S   S9  g )Ndataloader_opts)train_loader_kwargsvalid_loader_kwargs)rb   fitr<   r   r   rB   )rc   argsrd   s      rk   rS   SeparationTrainer.train   sN     $-> ? $-> ? 	 	
rv   checkpoint_pathc                    U(       a  XR                   R                  l        O#U R                  R	                  U R
                  S9  U R                  R                  U R                  U R                   S   [        S9n[        U0$ )N)r$   rx   )test_loader_kwargsmin_key)
rB   r=   checkpoints_dirr   load_check_pointr$   rb   evaluater   EVAL_KEY)rc   r~   r|   rd   values        rk   r   SeparationTrainer.evaluate   sm    8GLL%%5JJ''t{{';''#||,=> (  %  rv   )rP   rT   r$   r<   r   rB   r   rG   rb   r   r   )__name__
__module____qualname____firstlineno____doc__r   rE   r   r   r   r   rM   rr   rs   rt   rN   rS   r   floatr   __static_attributes__ rv   rk   r   r   !   s    	 ,0FJEI1Ga
a
a
 $C=a
 !)y'/A)B C	a

  (i.@(ABa
 "*#a
FUXX__ 

! 
!"3:.
!rv   r   c                   X    \ rS rSrSrSS jrS rS rS rS r	S	 r
S
 rS rS rS rSrg)ra      z:A subclass of speechbrain.Brain implements training steps.Nc           	         Uu  pUR                  U R                  5      UR                  U R                  5      pQ[        R                  " [	        U R
                  R                  5       Vs/ s H  nX&   S   R                  S5      PM     snSS9R                  U R                  5      nU[        R                  R                  :X  a  [        R                  " 5          U R
                  R                  (       d  U R
                  R                  (       a$  U R                  X%5      u  pUR                  S5      nU R
                  R                   (       a  U R
                  R#                  X5      nU R
                  R$                  (       a  U R'                  X5      u  pSSS5        U R(                  S   " U5      nU R(                  S   " U5      n[        R*                  " U/U R
                  R                  -  5      nXx-  n	[        R                  " [	        U R
                  R                  5       Vs/ s H)  nU R(                  S   " X   5      R                  S5      PM+     snSS9n
UR-                  S5      nU
R-                  S5      nX:  a   [.        R0                  " U
SSSX-
  45      n
X4$ U
SS2SU2SS24   n
X4$ s  snf ! , (       d  f       GN/= fs  snf )	z?Forward computations from the mixture to the separated signals.r   dimNencodermasknetdecoderr#   )tor$   rr   catrangerB   num_spks	unsqueezerX   StageTRAINno_graduse_speedperturbuse_rand_shiftadd_speed_perturbsumuse_wavedropwavedroplimit_training_signal_lencut_signalsr@   stacksizeFpad)rc   mixtargetsstagenoisemix_lensimix_west_masksep_h
est_sourceT_originT_ests                rk   compute_forwardSeparation.compute_forward   sN    t{{+X[[-EX )) t||4455A 
1''+5 
 "T[[/ 	 BHHNN"<<00DLL4O4O#'#9#9'#LLC!++b/C<<,,,,//>C<<99#'#3#3C#ALC ! Y',<<	*51UGdll&;&;;<  YY t||4455A Y'1;;B?5 

 88A;"zAq!X5E+FGJ "" $Ayy!O4J""W !(s   + J?B?K10K
Kc                 8    U R                   R                  X!5      $ )zComputes the sinr loss)rB   loss)rc   predictionsr   s      rk   compute_objectivesSeparation.compute_objectives   s    ||  66rv   c                 
   UR                   nUR                  UR                  /nU R                  R                  S:X  a  UR                  UR                  5        U R                  (       Gap  [        5          U R                  X#[        R                  R                  5      u  pCU R                  XC5      nU R                  R                  (       aN  U R                  R                  nXUU:     nUR!                  5       S:  a  UR#                  5       nO[%        S5        OUR#                  5       nSSS5        WU R                  R&                  :  Ga  UR!                  5       S:  a  U R(                  R+                  U5      R-                  5         U R                  R.                  S:  a{  U R(                  R1                  U R2                  5        [4        R6                  R8                  R;                  U R<                  R?                  5       U R                  R.                  5        U R(                  RA                  U R2                  5        U R(                  RC                  5         GObU =RD                  S-  sl"        [F        RI                  SRK                  U RD                  5      5        [4        RL                  " S5      RO                  U RP                  5      Ul)        GOU R                  X#[        R                  R                  5      u  pCU R                  XC5      nU R                  R                  (       aB  U R                  R                  nXUU:     nUR!                  5       S:  a  UR#                  5       nOUR#                  5       nXPR                  R&                  :  a  UR!                  5       S:  a  UR-                  5         U R                  R.                  S:  aV  [4        R6                  R8                  R;                  U R<                  R?                  5       U R                  R.                  5        U R2                  RA                  5         OwU =RD                  S-  sl"        [F        RI                  SRK                  U RD                  5      5        [4        RL                  " S5      RO                  U RP                  5      Ul)        U R2                  RU                  5         URW                  5       RY                  5       $ ! , (       d  f       GN= f)zTrains one batch   r   zloss has zero elements!!Nr#   zNinfinite loss or empty loss! it happened {} times so far - skipping this batch)-mix_sigs1_sigs2_sigrB   r   appends3_sigauto_mix_precr   r   rX   r   r   r   threshold_byloss	thresholdnelementmeanprintloss_upper_limscalerscalebackwardr+   unscale_r?   rr   rs   r\   clip_grad_norm_r@   
parametersstepupdatenonfinite_countloggerinfoformattensorr   r$   data	zero_graddetachr3   )rc   batchmixturer   r   r   thloss_to_keeps           rk   	fit_batchSeparation.fit_batch   sk    --<<.<<  A%NN5<<('+';';bhhnn(6$..{D<<00//B#'r	?L#,,.2+0028999;D   dll111dmmo6I!!$'002<<..!3KK((8HHNN22//133   0""$$$)$dVD0013 "LLO..t{{;	#'#7#7"((..$2 K**;@D||,,\\++#2I((*Q.',,.Dyy{ll111dmmo6I<<..!3HHNN224<<3J3J3L37<<3N3NP##%$$)$dVD0013 "LLO..t{{;	  "{{}  ""s s   6B7S==
Tc                 $   UR                   nUR                  nUR                  UR                  /nU R                  R
                  S:X  a  UR                  UR                  5        [        R                  " 5          U R                  XEU5      u  peU R                  Xe5      nSSS5        U[        R                  R                  :X  a  U R                  R                  (       a  [!        U R                  S5      (       aP  U R                  R"                  S:  a5  U R                  US   XEW5        U R                  =R"                  S-  sl        OU R                  US   XEW5        WR%                  5       R'                  5       $ ! , (       d  f       N= f)z/Computations needed for validation/test batchesr   Nn_audio_to_saver   r   )idr   r   r   rB   r   r   r   rr   r   r   r   rX   r   TEST
save_audiorQ   r   r   r   )rc   r   r   snt_idr   r   r   r   s           rk   evaluate_batchSeparation.evaluate_batch5  s   --<<.<<  A%NN5<<(]]_#'#7#7%$) K**;@D  BHHMM!dll&=&=t||%677<<//!3OOF1IwMLL00B60q	7[Iyy{!!## _s   ;&F
Fc                    SU0nU[         R                  R                  :X  a  X@l        U[         R                  R                  :X  Ga  [        U R                  R                  [        R                  5      (       aK  U R                  R                  U R                  /X25      u  pV[        R                  " U R                  U5        O0U R                  R                  R                  R                  S   S   nU R                  R                  R                  UUS.U R                  US9  U R                   R#                  SUS   0S/S9  gg)z"Gets called at the end of a epoch.r   r   r)   )epochr)   )
stats_metatrain_statsvalid_stats)metamin_keysN)rX   r   r   r   VALIDrD   rB   rU   
schedulersReduceLROnPlateaur?   update_learning_rateoptimparam_groupstrain_logger	log_statsr=   save_and_keep_only)rc   r   
stage_lossr   stage_stats
current_lrnext_lrs          rk   on_stage_endSeparation.on_stage_endM  s     ,BHHNN"* BHHNN"$,,33$668 8&*ll&?&?^^$e'9#
//H "\\3399FFqI$O
LL%%//"$ !,,' 0  00H 56" 1 ' #rv   c                 4   SnSnU R                   R                  (       Ga  / nSn[        UR                  S   5       Hu  nU R                   R	                  USS2SS2U4   U5      nUR                  U5        US:X  a  UR                  S   nMQ  UR                  S   U:  d  Mf  UR                  S   nMw     U R                   R                  (       a  Sn[        UR                  S   5       H  n[        R                  " U R                   R                  U R                   R                  S5      nXV   R                  U R                  5      XV'   [        R                  " XV   US   4SS9XV'   M     U(       a  U R                   R                  (       aJ  [        R                  " UR                  S   UUR                  S   UR                  [        R                  S	9n[!        U5       H  u  pgXV   SS2SU24   USS2SS2U4'   M     UR#                  S5      n	X4$ )
z=Adds speed perturbation and random_shift to the input signalsr   FTNr   r#   r#   )shiftsdims)r$   dtype)rB   r   r   shapespeedperturbr   r   rr   randint	min_shift	max_shiftr   r$   rollzerosr   	enumerater   )
rc   r   	targ_lensmin_len	recombinenew_targetsr   
new_target
rand_shiftr   s
             rk   r   Separation.add_speed_perturbm  s    	<<(((KI7==,-!\\66wq!Qw7G7@B
"":.6(..r2G!''+g5","2"22"6 . ||** 	w}}R01A!&t||/E/E/3||/E/Eu"NJ%0^%6%6t{{%CKN%*ZZ#
1/@q&JKN	 2 <<00#kka(b)&~~#kkG &/{%;MA'2~a7l'CGAq!G$ &< kk"o|rv   c           	      @   [         R                  " SS[        SUR                  S   U R                  R
                  -
  5      -   S5      R                  5       nUSS2X3U R                  R
                  -   2SS24   nUSS2X3U R                  R
                  -   24   nX4$ )zThis function selects a random segment of a given length within the mixture.
The corresponding targets are selected accordinglyr   r#   r  N)rr   r	  maxr  rB   training_signal_lenitem)rc   r   r   	randstarts       rk   r   Separation.cut_signals  s     MMAw}}Q'$,,*J*JJKK
 $&	 	
 !Y LL<<(= =>?@ A!Y LL<<(= = = >rv   c                     [        US5      (       a  UR                  5         UR                  5        H  nX:w  d  M
  U R                  U5        M     g)z3Reinitializes the parameters of the neural networksreset_parametersN)rQ   r  r@   reset_layer_recursively)rc   layerchild_layers      rk   r  "Separation.reset_layer_recursively  s@    5,--""$ ==?K#,,[9 +rv   c           	         SSK Jn  [        R                  R	                  U R
                  R                  S5      n/ n/ n/ n/ n/ SQn[        R                  R                  R                  " U40 U R
                  R                  D6n	[        US5       n
[        R                  " XS9nUR                  5         [!        U	SS9 n[#        U5       GH$  u  pUR$                  u  nnUR&                  nUR(                  UR*                  /nU R
                  R,                  S	:X  a  UR/                  UR0                  5        [2        R4                  " 5          U R7                  UR$                  U[        R8                  R:                  5      u  nnS
S
S
5        U R=                  WU5      n[2        R>                  " U/U R
                  R,                  -  SS9nURA                  URB                  5      nU R=                  UU5      nURE                  5       URE                  5       -
  nU" US   RG                  5       RI                  5       RK                  5       US   RG                  5       RM                  5       RI                  5       RK                  5       5      u  n    nU" US   RG                  5       RI                  5       RK                  5       US   RG                  5       RM                  5       RI                  5       RK                  5       5      u  n    nURE                  5       URE                  5       -
  nUS   URE                  5       UURO                  5       * URO                  5       * S.nURQ                  U5        UR/                  URE                  5       5        UR/                  URE                  5       5        UR/                  URO                  5       * 5        UR/                  URO                  5       * 5        GM'     S[R        RT                  " U5      RE                  5       [R        RT                  " U5      RE                  5       [R        RT                  " U5      RE                  5       [R        RT                  " U5      RE                  5       S.nURQ                  U5        S
S
S
5        S
S
S
5        [V        RY                  SR[                  [R        RT                  " U5      RE                  5       5      5        [V        RY                  SR[                  [R        RT                  " U5      RE                  5       5      5        [V        RY                  SR[                  [R        RT                  " U5      RE                  5       5      5        [V        RY                  SR[                  [R        RT                  " U5      RE                  5       5      5        g
! , (       d  f       GN>= f! , (       d  f       GNB= f! , (       d  f       GNL= f)zNThis script computes the SDR and SI-SNR metrics and saves
them into a csv filer   )bss_eval_sourcesztest_results.csv)r   sdrsdr_ir   zsi-snr_iw)
fieldnamesT)dynamic_ncolsr   Nr   r   avgzMean SISNR is {}zMean SISNRi is {}zMean SDR is {}zMean SDRi is {}).mir_eval.separationr#  rH   rI   rJ   rB   r'   rX   dataio
dataloadermake_dataloaderrx   rW   csv
DictWriterwriteheaderr   r  r   r   r   r   r   r   r   rr   r   r   r   r   r   r   r   r$   r   tr3   numpyr   r  writerownparrayr   r   r   )rc   	test_datar#  	save_fileall_sdrs
all_sdrs_i
all_sisnrsall_sisnrs_icsv_columnstest_loaderresults_csvwriterr1  r   r   r   mix_lenr   r   r   sisnrmixture_signalsisnr_baselinesisnr_ir$  re   sdr_baseliner%  rows                                rk   save_resultsSeparation.save_results  si   
 	9 GGLL!;!;!35	 

Fii**::7557 )S![^^KHF  k6! )!HA (-}}$GW"XXF$||U\\:G||,,1u||4/3/C/C!MM7BHHMM0C,W )
 !33KIE &+[[ 	DLL$9$99r&CN%3%6%6w~~%FN%)%<%<&&1N#jjl^-@-@-BBG $4
**,224#A((*113779??A$LCAq
 -=
**,224&q)++-446::<BBD-)L!Q
  HHJ):):)<<E #))"xxz!&#(::<-%,\\^OC OOC( OOCHHJ/%%ejjl3%%uzz|m4 ''8g !-l $88H-224XXj1668 hhz2779 " 6 ; ; = ${ 7 "H 	&--bhhz.B.G.G.IJK'..rxx/E/J/J/LMN$++BHHX,>,C,C,EFG%,,RXXj-A-F-F-HIJq ) 76 "!s>   /W$BW9W 		LWW$ 
W
W
W!	W$$
W3c           	         [         R                  R                  U R                  R                  S5      n[         R                  R                  U5      (       d  [         R                  " U5        [        U R                  R                  5       GHT  nUSSS2U4   nXwR                  5       R                  5       -  S-  n[         R                  R                  USR                  XS-   5      5      n[        R                  " UUR                  S5      R                  5       U R                  R                   5        USSS2U4   nXwR                  5       R                  5       -  S-  n[         R                  R                  USR                  XS-   5      5      n[        R                  " UUR                  S5      R                  5       U R                  R                   5        GMW     US   SSS24   nXwR                  5       R                  5       -  S-  n[         R                  R                  USR                  U5      5      n[        R                  " UUR                  S5      R                  5       U R                  R                   5        g)	zFsaves the test audio (mixture, targets, and estimated sources) on diskaudio_resultsr   Ng      ?zitem{}_source{}hat.wavr#   zitem{}_source{}.wavzitem{}_mix.wav)rH   rI   rJ   rB   save_folderexistsmkdirr   r   absr  r   
torchaudiosaver   r3   sample_rate)	rc   r   r   r   r   	save_pathnssignalr7  s	            rk   r   Separation.save_audio  s    GGLL!9!9?K	ww~~i((HHY--.B !Ar*Fjjl..0036F3::66JLIOOI",,Q/335 LL446
 Q2X&Fjjl..0036F077QGIIOOI",,Q/335 LL446! /* AqD!**,**,,s2GGLL,<,C,CF,KL		((+//14<<3K3K	Mrv   )r   )N)r   r   r   r   r   r   r   r   r   r   r   r   r  rG  r   r   r   rv   rk   ra   ra      s@    D4#l7
C#L$0@-^ :\K|!Mrv   ra   )5r.  rH   typingr   r   r   r2  r4  speechbrainrX   speechbrain.nnet.schedulersnnetr   rr   torch.nn.functionalrs   
functionalr   rO  torch.cuda.ampr   torch.utils.datar   r   modelscope.metainfor	   modelscope.modelsr
   r   modelscope.msdatasetsr   modelscope.trainers.baser   modelscope.trainers.builderr   modelscope.utils.constantr   r   modelscope.utils.devicer   modelscope.utils.loggerr   modelscope.utils.torch_utilsr   r   r   r   r   register_modulespeech_separationr   Brainra   r   rv   rk   <module>rj     s     	 ( (   0 0     # $  ( / + 0 0 G 1 .5 5 	 
h&@&@AL! L! BL!^@M @Mrv   