
    9i;                        S r SSKJr  SSKrSSKJr  SSKJr  SSK	J
r
JrJrJrJrJrJr  SSKrSSKJr  SSKJr  SSKJr  SS	KJrJr  SS
KJrJrJrJrJ r J!r!  SSK"J#r#  SSK$J$r$  SSK%J&r&  SSK'J(r(J)r)  SSK*J+r+  SSK,J-r-J.r.  SSK/J0r0  SSK1J2r2  SSK3J4r4J5r5  SSK6J7r7  SSK8J9r9  SSK:J;r;  SSK<J=r=  SSK>J?r?  SSK@JArA  SSKBJCrCJDrDJErEJFrFJGrG  SSKHJIrI  SSKJJKrK  \K" 5       rL " S S\5      rMS\\\N\
4      S \9S!\\\R                        4S" jrPS\\\N\
4      S#\QS \9S!\\\N\
4      4S$ jrR\;R                  " \)R                  S%9 " S& S'\?5      5       rUg)(z PyTorch trainer for UniTE model.    N)ceil)mkdir)AnyCallableDictListOptionalTupleUnion)	DataFrame)pad)clip_grad_norm_)AdamW	Optimizer)BatchSampler
DataLoaderDatasetSamplerSequentialSamplerSubsetRandomSampler)SummaryWriter)tqdm)AutoTokenizer)MetricsTrainers)Metric)
MetricKeysbuild_metric)
TorchModel)InputFormat)UniTEForTranslationEvaluationcombine_input_sentences)	MsDataset)Preprocessor)TRAINERS)Hook)EpochBasedTrainer)
ConfigDict)
ConfigKeysFieldsModeKeys	ModelFileTrainerStages)create_device)
get_loggerc                   :    \ rS rSrS\S\4S jrS rS\4S jrSrg	)
$TranslationEvaluationTrainingSampler*   num_of_samples batch_size_for_each_input_formatc                    Xl         X l        U R                   S-  U l        U R                  S-  n[        R	                  SU R                   U R                  U R                   U-
  4-  5        X0l         [
        R                  " U R                   5      R                  5       R                  5       n[        5       U l
        [        5       U l        [        5       U l        [        [        R                  [        R                   [        R"                  45       H  u  pVXPR                  -  nXpR                  -   nXGU U R                  U'   [%        ['        U R                  U   5      U R                  SS9U R                  U'   [)        U R                  U   5      U R                  U'   M     SU l        U R-                  5       S:X  a  [/        SS5      eg)	aq  Build a sampler for model training with translation evaluation trainer.
The trainer should derive samples for each subset of the entire dataset.

Args:
    num_of_samples: The number of samples in total.
    batch_size_for_each_input_format: During training, the batch size for each input format

Returns:
    A data sampler for translation evaluation model training.

   zn%d samples are given for training. Using %d samples for each input format. Leaving the last %d samples unused.T
batch_size	drop_lastr   zCThe dataset doesn't contain enough examples to form a single batch.z?Please reduce the batch_size or use more examples for training.N)r3   r4   $num_of_samples_for_each_input_formatloggerinfotorchrandpermcputolistdictsubset_iteratorssubset_samplersindices_for_each_input_format	enumerater    SRC_REFSRCREFr   r   iternum_of_sampled_batches__len__
ValueError)	selfr3   r4   num_of_samples_to_userandom_permutationsinput_format_indexinput_format	start_idxend_idxs	            v/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/trainers/nlp/translation_evaluation_trainer.py__init__-TranslationEvaluationTrainingSampler.__init__,   s    -0P-484G4G14L1 $ I IA M2   $"K"K  #88::	; 4#nn!!$vvx 	 !%#v-1V*;..QR -*-V-VVI"K"KKG 3g F .. 01S1ST`1ab(,(M(M'+-   . 37$$\234D!!,/ S '(#<<>QVQ 
 	    c              #     #      U R                   U R                  5       :X  aW  [        R                  [        R                  [        R
                  4 H  n  [        U R                  U   5        M     SU l         [        5       n[        [        R                  [        R                  [        R
                  45       H   u  p1U[        U R                  U   5      -  nM"     U =R                   S-  sl         Uv   M  ! [         a)    [        U R                  U   5      U R                  U'    M  f = f! [         a     g f = f7f)Nr      )rJ   rK   r    rF   rG   rH   nextrB   StopIterationrI   rC   listrE   )rM   rQ   outputinput_format_idxs       rT   __iter__-TranslationEvaluationTrainingSampler.__iter__e   s    ..$,,.@)4)<)<koo)4):"& $T%:%:<%H I #): 34D/!;#6#6"YZ 3$d4#8#8#FGGF [ ++q0+-  $1 &$()=)=l)K$L !% 5 5l C %& ! sN   EAD1 C;/B
D1 9E;/D.*D1 -D..D1 1
D>;E=D>>Ereturnc                 4    U R                   U R                  -  $ N)r:   r4   )rM   s    rT   rK   ,TranslationEvaluationTrainingSampler.__len__   s    88D<a<aaarW   )r4   rD   rJ   r3   r:   rB   rC   N)	__name__
__module____qualname____firstlineno__intrU   r_   rK   __static_attributes__ rW   rT   r1   r1   *   s*    7s 7367r6b brW   r1   batchpreprocessorra   c                    ^ [        5       nU S   R                  5        H  m[        U4S jU  5       5      UT'   M     U" U5      nU$ )Nr   c              3   ,   >#    U  H	  oT   v   M     g 7frc   rk   ).0xkeys     rT   	<genexpr>,convert_csv_dict_to_input.<locals>.<genexpr>   s     5u!vus   )rA   keysr\   )rl   rm   
input_dictrr   s      @rT   convert_csv_dict_to_inputrw      sF     JQx}}5u55
3  j)JrW   r8   c                 ,  ^^ [        5       n[        5       US'   TR                  [        R                  :X  a  [        [        R                  [        R                  [        R                  45       H  u  pEXA-  nXa-   nXU nUS==   U/U-  -  ss'   TR                  U5        [        UT5      nUR                  5        H7  u  pXR                  5       ;  a  [        5       X9'   X9   R                  U
5        M9     M     OTR                  [        R                  :X  a|  US==   TR                   /[#        U 5      -  -  ss'   [        U T5      n U R                  5        H7  u  pXR                  5       ;  a  [        5       X9'   X9   R                  U
5        M9     O[%        STR                  -  5      e['        S US    5       5      m[        UU4S jUS    5       5      US'   [(        R*                  " US   SS9US'   [(        R,                  " US   5      R/                  S	5      US'   TR                  [        R                  :X  aN  [1        US
   [        5       5      US
'   [1        US   [        5       5      US'   [1        US   [        5       5      US'   U$ )NrQ   z9During training, %s mode is not allowed for preprocessor.c              3   B   #    U  H  oR                  S 5      v   M     g7f)N)size)rp   rq   s     rT   rs   "data_collate_fn.<locals>.<genexpr>   s     I0H1FF2JJ0Hs   	input_idsc           	   3   v   >#    U  H.  n[        US TUR                  S5      -
  4TR                  S9v   M0     g7f)r   rz   )r   valueN)r   r{   pad_token_id)rp   rq   input_max_lengthsrm   s     rT   rs   r|      sB      $P 7O 	A%r
23++	-6Ns   69r   )dimscorerz   lp	raw_score
segment_id)rA   r\   moder+   TRAINrE   r    rF   rG   rH   change_input_formatrw   itemsru   appendEVALrQ   lenrL   maxr=   catTensorviewsum)rl   r8   rm   output_dictrP   rQ   rR   rS   batch_to_processrr   r   r   s     `        @rT   data_collate_fnr      sb    &K"&&KHNN*;..QR -*7I,G$w7'L>J+FF',,\:8 , 0 /446
..00'+vK$ ''. 7 S 
		hmm	+N#(A(A'BSZ'OO#)%>++-JC**,,#'6 ##E* (
 G ! 	! IK0HII# $P 7B+6N$P  PK
  %yy[)AqIK <<G(<=BB2FKHMM)D 146:D#&{;'?#HK $'L(A46$JL!rW   )module_namec                      ^  \ rS rSr   SS\\\\R                  R                  \
4      S\\
   S\
4U 4S jjjrS\S\4U 4S jjrS\4S	 jrS\4S
 jrS rSrU =r$ )TranslationEvaluationTrainer   modelcfg_filedevicec                    >^  U 4S jnU 4S jn[         R                  U[         R                  U0n[        T	T ]  " U/UQ7UUS.UD6  ST l        ST l        g)zBuild a translation evaluation trainer with a model dir or a model id in the model hub.

Args:
    model: A Model instance.
    cfg_file: The path for the configuration file (configuration.json).
    device: Used device for this trainer.

c                 j   > [        U TR                  R                  R                  TR                  S9$ N)r8   rm   )r   cfgtrainr8   train_preprocessorrq   rM   s    rT   data_collator_for_trainFTranslationEvaluationTrainer.__init__.<locals>.data_collator_for_train   s-    "88>>44!446 6rW   c                 j   > [        U TR                  R                  R                  TR                  S9$ r   )r   r   
evaluationr8   eval_preprocessorr   s    rT   data_collator_for_evalETranslationEvaluationTrainer.__init__.<locals>.data_collator_for_eval   s/    "88..99!335 5rW   )r   data_collatorN)r)   r   valsuperrU   train_dataloadereval_dataloader)
rM   r   r   r   argskwargsr   r   r   	__class__s
   `        rT   rU   %TranslationEvaluationTrainer.__init__   sr     	6	5 5NN2

 		 		'	
 	 !%#rW   r   ra   c                 B  >^  T R                   R                  R                  R                  S:w  a  [        TT ]  U5      $ T R                  R                  R                  R                  5        H
  nSUl
        M     [        R                  S5        [        U 4S j[        ST R                   R                  R                  5       5       5      nUR!                  T R                  R                  R                  R                  5       T R                   R                  R                  R"                  S.5        UR!                  T R                  R$                  R                  5       T R                   R                  R                  R&                  S.5        UR!                  T R                  R(                  R                  5       T R                   R                  R                  R&                  S.5        [+        UT R                   R                  R                  R"                  T R                   R                  R                  R,                  T R                   R                  R                  R.                  T R                   R                  R                  R0                  S9nU$ )z/Sets the optimizers to be used during training.r   FzBuilding AdamW optimizer ...c              3   L  >#    U  H  nTR                   R                  R                  R                  U   R                  5       TR                  R
                  R                  R                  TR                  R
                  R                  R                  U-  -  S .v   M     g7f)paramslrN)	r   encoderlayer
parametersr   r   	optimizerplm_lrplm_lr_layerwise_decay)rp   irM   s     rT   rs   ?TranslationEvaluationTrainer.build_optimizer.<locals>.<genexpr>   s~      -? >a	 JJ&&,,Q/::<HHNN$$++hhnn&&==q@A	.
 >s   B!B$r   r   )r   betasepsweight_decay)r   r   r   typer   build_optimizerr   r   
embeddingsr   requires_gradr;   r<   r\   rangenum_hidden_layersr   r   	estimatormlp_lrlayerwise_attentionr   r   r   r   )rM   r   paramlearning_rates_and_parametersr   r   s   `    rT   r   ,TranslationEvaluationTrainer.build_optimizer   s   88>>##((G37*3// ZZ''22==?E"'E @ 	23(, -? DHHNN<<=-? )?% 	&,,JJ))446HHNN$$++	.
 	 	&,,JJ  ++-HHNN$$++	.
 	 	&,,JJ**557HHNN$$++	.
 	 )xx~~''..((..**00((,,11>>
	 rW   c                    [         R                  S5        U R                  c  [         R                  SU R                  R                  R
                  R                  -  5        [        R                  " [        R                  " U R                  U R                  R                  R
                  R                  5      U R                  R                  R
                  R                  S9U l        [        U R                  [        [        U R                  5      U R                  R
                  R                   S9SU R"                  S S9n[         R                  S[        U R                  5      -  5        U$ )Nz$Building dataloader for training ...z"Reading train csv file from %s ...split)r4      batch_samplernum_workers
collate_fn	generatorReading done, %d items in total)r;   r<   train_datasetr   datasetr   namer#   loadospjoin	model_dirr   r   r1   r   r8   train_data_collator)rM   r   s     rT   get_train_dataloader1TranslationEvaluationTrainer.get_train_dataloader&  s   :;%KK<((**00556 7!*)9)9)?)?)D)DEhh&&,,22"4D &>D&&'151J1JL // 	5$,,-. 	/  rW   c                    [         R                  S5        U R                  c  [         R                  SU R                  R                  R
                  R                  -  5        [        R                  " [        R                  " U R                  U R                  R                  R
                  R                  5      U R                  R                  R
                  R                  S9U l        [        U R                  [        [        [!        S[#        U R                  5      5      5      U R                  R$                  R&                  SS9SU R(                  S S9n[         R                  S	[#        U R                  5      -  5        U$ )
Nz&Building dataloader for evaluating ...z!Reading eval csv file from %s ...r   r   Fr7   r   r   r   )r;   r<   eval_datasetr   r   validr   r#   r   r   r   r   r   r   r   r   r   r   r   r8   eval_data_collator)rM   r   s     rT   get_eval_data_loader1TranslationEvaluationTrainer.get_eval_data_loader>  s   <=$KK;((**00556 7 !*)9)9)?)?)D)DEhh&&,,22!4D %&!%3t/@/@+A"BC88..99! .. 	5D<M<M8NNOrW   c                 &   Sn[        U R                  R                  S5      (       aB  U R                  R                  R                  n[	        U R                  4SU R
                  0UD6nU R                  [        R                  5        [        5       n[        R                  [        R                  [        R                  4 GH+  nU R                  R                  U5        U R                   (       a  SSKJn  UR'                  U" U UU R(                  UUU R                  R                  R+                  SS5      U R                  R                  R+                  SS5      U R,                  S	95        O4SS
KJn  UR'                  U" U UU R(                  UUU R,                  S95        U HB  n	[        U	S5      (       d  M  [1        U	R2                  5      (       d  M2  U	R3                  5         MD     GM.     U R                  [        R4                  5        U$ )a{  Evaluation loop used by `TranslationEvaluationTrainer.evaluate()`.

The evaluation process of UniTE model should be arranged with three loops,
corresponding to the input formats of `InputFormat.SRC_REF`, `InputFormat.REF`,
and `InputFormat.SRC`.

Here we directly copy the codes of `EpochBasedTrainer.evaluation_loop`, and change
the input format during each evaluation subloop.
Nvisualizationr   r   )multi_gpu_test	cache_dirgpu_collectF)r   metric_classesvis_closuretmpdirr   data_loader_iters_per_gpu)single_gpu_test)r   r   r   data_loader_itersclear)hasattrr   r   r   partialr   invoke_hookr-   
before_valrA   r    rF   rG   rH   r   r   _dist#modelscope.trainers.utils.inferencer   updater   get_eval_iters_per_epochr   callabler   	after_val)
rM   data_loaderr   r   vis_cfgmetric_valuesrQ   r   r   ms
             rT   evaluation_loop,TranslationEvaluationTrainer.evaluation_loopW  s    488&&88hh))77G!""J,0,=,=JAHJK 	112(00+//(__.L""66|DzzN$$"##{{'5$/#xx2266{DI$(HH$7$7$;$;)5%2262L2L	N
O P$$###{{'5$/*.*D*DFG $1g&&8AGG+<+<GGI $;.B 	001rW   )r   r   r   r   )NNgpu)re   rf   rg   rh   r	   r   r   r=   nnModulestrrU   r(   r   r   r   r   r   r  rj   __classcell__)r   s   @rT   r   r      s    
 04+/$	+ z588??'*(+ ", -+ $C=+ 	+ +Z/: /) /b j  0j 25 5rW   r   )V__doc__os.pathpathr   randommathr   osr   typingr   r   r   r   r	   r
   r   r=   pandasr   torch.nn.functionalr   torch.nn.utilsr   torch.optimr   r   torch.utils.datar   r   r   r   r   r   torch.utils.tensorboardr   r   transformersr   modelscope.metainfor   r   modelscope.metricsr   modelscope.metrics.builderr   r   modelscope.models.baser   )modelscope.models.nlp.unite.configurationr    2modelscope.models.nlp.unite.translation_evaluationr!   r"   modelscope.msdatasetsr#   modelscope.preprocessorsr$   modelscope.trainers.builderr%   modelscope.trainers.hooksr&   modelscope.trainers.trainerr'   modelscope.utils.configr(   modelscope.utils.constantr)   r*   r+   r,   r-   modelscope.utils.devicer.   modelscope.utils.loggerr/   r;   r1   r
  r   rw   ri   r   register_moduletranslation_evaluation_trainerr   rk   rW   rT   <module>r+     s6   &     D D D   # * (F F 1  & 1 % ? - A< + 1 0 * 9 .6 6 1 .	Wb7 WbtDcN#"',T%,,-?'@04S#X/ 0S 0".037S#X3G0f 
h&M&MNF#4 F OFrW   