
    9i7                     T   S SK Jr  S SKrS SKJrJr  S SKrS SK	J
r
  S SKJr  S SKJr  S SKJr  S SKJr  S SKJr  \R*                  S	:  a'  \R,                  R.                  r\R0                  " 5         \" 5       r\R4                  " S
S9 " S S\5      5       r        SS jrSS jrg)    N)DictOptional)snapshot_download)CsanmtForTranslation)BaseTrainer)TRAINERS)	ModelFile)
get_loggerz2.0zcsanmt-translation)module_namec                   l   ^  \ rS rSrSS\S\4U 4S jjjrS rS r SS\\   S\	\\
4   4S	 jjrS
rU =r$ )CsanmtTranslationTrainer   modelcfg_filec                   > U R                  U5      n[        R                  " 5         Xl        [        R
                  " U[        R                  5      U l        Uc%  [        R
                  " U[        R                  5      n[        T	U ]-  U5        0 U l        U R                  5         [        R                  " SS9nSUR                  l        [        R"                  " US9U l        [        R&                  " [        R(                  S S /SS9U l        [        R&                  " [        R(                  S S /SS9U l        0 U l        [        R0                  R3                  5       U l        [7        U R                  40 U R                  D6U l        U R9                  U R*                  U R,                  S9nU R.                  R;                  U5        [        R0                  R=                  [        R>                  " 5       U R                  S   S	9U l         U R$                  RC                  5        n[D        RG                  S
U R                   35        [I        U R                  5      n[        R0                  RK                  U R                  U5        URM                  [        RN                  " 5       5        S S S 5        g ! , (       d  f       g = f)NT)allow_soft_placement)configsource_wids)dtypeshapenametarget_wids)inputlabelkeep_checkpoint_max)max_to_keepzloading model from )(get_or_download_model_dirtfreset_default_graph	model_dirospjoinr	   TF_CHECKPOINT_FOLDER
model_pathCONFIGURATIONsuper__init__params_override_params_from_fileConfigProtogpu_optionsallow_growthSession_sessionplaceholderint64r   r   outputtraincreate_global_stepglobal_stepr   r   updateSaverglobal_variablesmodel_saver
as_defaultloggerinfoget_pretrained_variables_mapinit_from_checkpointrunglobal_variables_initializer)
selfr   r   argskwargs	tf_configr1   sesspretrained_variables_map	__class__s
            r/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/trainers/nlp/csanmt_translation_trainer.pyr'   !CsanmtTranslationTrainer.__init__   s   ..u5
 ((5)*H*HIxxy'>'>?H"'')NN=	-1	*

)4>>((4,]D>>((4,]D88668)$//IT[[I
$"2"2$:J:JK6"88>>!$9: * < ]]%%'4KK-doo->?@'C(!$ HH))$//*BDHHR4467 (''s   %BJ44
Kc                 P   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S	   U R                  S	'   U R                   S   S
   U R                  S
'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   SU R                  S'   SU R                  S'   SU R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   S   U R                  S'   U R                   S   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S   U R                  S'   U R                   S   S    U R                  S '   U R                   S   S!   U R                  S!'   U R                   S   S"   U R                  S"'   U R                   S   S#   U R                  S#'   U R                   S   S$   U R                  S$'   U R                   S   S%   U R                  S%'   U R                   S   S&   U R                  S&'   U R                   S   S'   U R                  S''   U R                   S   S(   U R                  S('   U R                   S   S)   U R                  S)'   U R                   S   S   U R                  S'   U R                   S   S*   U R                  S*'   U R                   S   S+   U R                  S+'   U R                   S   S,   U R                  S,'   U R                   S   S-   U R                  S-'   U R                   S   S.   U R                  S.'   U R                   S   S/   U R                  S/'   U R                   S   S0   U R                  S0'   U R                   S   S1   U R                  S1'   U R                   S   S2   U R                  S2'   U R                   S3   S4   U R                  S4'   U R                   S3   S5   U R                  S5'   U R                   S3   S6   U R                  S6'   U R                   S   S7   U R                  S7'   g )8Nr   hidden_sizefilter_size	num_headsnum_encoder_layersnum_decoder_layerslayer_preproclayer_postproc$shared_embedding_and_softmax_weightsshared_source_target_embeddinginitializer_scaleposition_info_typemax_relative_disnum_semantic_encoder_layerssrc_vocab_sizetrg_vocab_sizeg        attention_dropoutresidual_dropoutrelu_dropoutdataset	train_src	train_trg	src_vocabfile	vocab_src	trg_vocab	vocab_trgr2   num_gpuswarmup_stepsupdate_cycler   
confidence	optimizer
adam_beta1
adam_beta2adam_epsilongradient_clip_normlearning_rate_decayinitializerlearning_ratetrain_batch_size_wordsscale_l1scale_l2train_max_lennum_of_epochssave_checkpoints_stepsnum_of_sampleseta
evaluation	beam_sizelp_ratemax_decoded_trg_lenseed)cfgr(   )r@   s    rG   r)   3CsanmtTranslationTrainer._override_params_from_fileE   sk   %)XXg%6}%EM"%)XXg%6}%EM"#'88G#4[#AK ,0HHW,= -"(),0HHW,= -"()'+xx'8'IO$(,(9:J(K$%>Bhh?;?=:;8<8I,9.45+/88G+<,!'(,0HHW,= -"()*.((7*;<N*O&'59XXg5F)6+12(,(9:J(K$%(,(9:J(K$%+.'(*-&'&)N##'88I#6{#CK #'88I#6{#CK #'88I#6{#CF#KK #'88I#6{#CF#KK "&((7"3J"?J&*hhw&7&GN#&*hhw&7&GN#-1XXg->!.#)*$(HHW$5l$CL!#'88G#4[#AK $(HHW$5l$CL!$(HHW$5l$CL!&*hhw&7&GN#,0HHW,= -"()-1XXg->!.#)*%)XXg%6}%EM"+/88G+<,!'('+xx'8'IO$040A$1&,-"&((7"3J"?J"&((7"3J"?J'+xx'8'IO$'+xx'8'IO$040A$1&,-(,(9:J(K$%!XXg.u5E#'88L#9+#FK !%,!7	!BI-1XXl-C!.#)* #hhw/7F    c                    [         R                  S5        [        R                  " U R                  U R
                  S   5      n[        R                  " U R                  U R
                  S   5      n[        R                  " U R                  U R
                  S   5      n[        R                  " U R                  U R
                  S   5      nSnSnU R                  R                  5        n	 US-  nXpR
                  S	   :  a  GO[        R                  R                  S
[        U4-  5        [        UUUUU R
                  S   U R
                  S   U R
                  S   S:  a  U R
                  S   OSSU	US9
n
U
u  p  U	R                  X/5      u  pUS-  nU R                  UU R                  U0nU R                  R                  U R                  US9nUS   n[         R                  SR!                  UU5      5        XR
                  S   -  S:X  an  [        R                  R                  S[        U4-  5        U R                  S-   nU R"                  R%                  U	U[        R&                  R)                  5       S9  GM  [        R                  R                  [        < S[.        R0                  " [.        R2                  " [.        R.                  " 5       5      5      < S35        S S S 5        g ! [        R*                  R,                   a%    [        R                  R                  SU-  5         Of = fGMT  ! , (       d  f       g = f)NzBegin csanmt trainingr]   r^   ra   rc   r   T   rt   z%s: Epoch %irp   rs   rd   )batch_size_wordsmax_lenrd   is_trainsessionepoch)	feed_dictlossz Iteration: {}, step loss: {:.6f}ru   z%s: Saving model on step: %d.z
model.ckpt)r4   zepoch %d end!z": NMT training completed at time: .)r:   r;   r!   r"   r    r(   r.   r9   r   logging__name__input_fnr>   r   r   r1   formatr8   saver2   get_global_steperrorsOutOfRangeErrortimeasctime	localtime)r@   rA   rB   r]   r^   ra   rc   r   	iteration
tf_sessiontrain_input_fnfeatureslabelsfeatures_batchlabels_batchr   sess_outputs	loss_stepck_paths                      rG   r2   CsanmtTranslationTrainer.train   s   +,HHT^^T[[-EF	HHT^^T[[-EF	HHT^^T[[-EF	HHT^^T[[-EF		]]%%':
KK88

(E1B BC!)%)[[1I%J KK8{{:.2 "[[489!&"! $2 ?7A~~%.804!Q	 ,,n ,,l%	 (,}}'8'8 KK9 (9 (>$0$8	$F$M$M%y%2 3 %{{ 8(: :=>?JJOO,K-5y,A-B C&*nn|&CG ,,11 * ',.HH,D,D,F 2 H' 6 JJOO4<<tyy{(CDFGc ('\ yy00 ?JJOOOu$=>?[  ('s4   ,BMDK9
A&M9A L<9M;L<<M
Mcheckpoint_pathreturnc                     g)a  evaluate a dataset

evaluate a dataset via a specific model from the `checkpoint_path` path, if the `checkpoint_path`
does not exist, read from the config file.

Args:
    checkpoint_path (Optional[str], optional): the model path. Defaults to None.

Returns:
    Dict[str, float]: the results about the evaluation
    Example:
    {"accuracy": 0.5091743119266054, "f1": 0.673780487804878}
N )r@   r   rA   rB   s       rG   evaluate!CsanmtTranslationTrainer.evaluate   s    " 	r   )
r.   r4   r   r    r$   r8   r1   r(   r   r   N)r   
__module____qualname____firstlineno__strr'   r)   r2   r   r   floatr   __static_attributes____classcell__)rF   s   @rG   r   r      sX    )8c )8S )8 )8VC8J>GB 37"*3- #3:. r   r   c           
        ^^^^^^ [         R                  R                  [         R                  R                  U[         R                  [         R                  R
                  R                  [         R                  [         R                  R
                  R                  S9SS9m[         R                  R                  [         R                  R                  U[         R                  [         R                  R
                  R                  [         R                  [         R                  R
                  R                  S9SS9m[         R                  R                  U 5      n[         R                  R                  U5      n[         R                  R                  R                  X45      nUR                  S SS9R                  S5      nUR                  S SS9R                  S5      nUR                  UU4S	 jSS9R                  S5      nU	(       ad  UU4S
 jnU4S jnUUUU4S jnUR                  U4S j5      nUR!                  [         R                  R"                  R%                  UUUS95      nOUR'                  UT-  S /S /4S9n[         R                  R)                  U5      n[         R*                  " [         R,                  R.                  UR0                  5        UR3                  5       u  nnU	(       aE  U
R5                  UR0                  5        US:X  a$  U
R5                  [         R6                  " 5       5        UU4$ )N)	key_dtype	key_indexvalue_dtypevalue_indexr   )num_oov_bucketsc                 ^    [         R                  " U /5      [         R                  " U/5      4$ r   )r   string_splitsrctrgs     rG   <lambda>input_fn.<locals>.<lambda>   s    "//3%0"//3%2HIr   
   )num_parallel_callsi@B c                 2    U R                   UR                   4$ r   )valuesr   s     rG   r   r     s    #**cjj1r   c                 H   > TR                  U 5      TR                  U5      4$ r   )lookup)r   r   r_   rb   s     rG   r   r     s     )**3/1A1A#1FGr   c                   > TT-   S-
  T-  n[         R                  " [         R                  " U S9U-  [         R                  " US9U-  5      n[         R                  " [         R                  " TU5      [         R
                  S9$ )Nr   r   r   )r   maximumsizecastminimumr0   )src_datatrg_databucket_width	bucket_idr   num_bucketss       rG   key_funcinput_fn.<locals>.key_func	  si    #k1A5+EL

h'<7h'<79I 772::k9=RXXNNr   c                 ,   > UR                  TS /S /4S9$ )Npadded_shapes)padded_batch)
unused_keywindowed_datar   s     rG   reduce_funcinput_fn.<locals>.reduce_func  s+     -- $$0@ . B Br   c                 ~   > TT-   S-
  T-  nU S-  n TT-  X-  -  n[         R                  " U[         R                  S9$ )Nr   r   )r   r   r0   )keyr   r   r   r   r   rd   s      rG   window_size_func"input_fn.<locals>.window_size_func  sH    #k1A5+EL1HC//C4FGD774rxx00r   c                    > [         R                  " [         R                  " U S9T:*  [         R                  " US9T:*  5      $ )Nr   )r   logical_andr   )r   r   r   s     rG   r   r     s0    R^^c"g-c"g-/r   )r   r   r   r   )r   r   StaticVocabularyTableTextFileInitializerstringTextFileIndex
WHOLE_LINEr0   LINE_NUMBERdataTextLineDatasetDatasetzipmapprefetchfilterapplyexperimentalgroup_by_windowr   make_initializable_iteratoradd_to_collection	GraphKeysTABLE_INITIALIZERSrn   get_nextr>   tables_initializer)src_filetrg_filesrc_vocab_filetrg_vocab_filer   r   
batch_sizer   rd   r   r   r   src_datasettrg_datasetsrc_trg_datasetr   r   r   iteratorr   r   r_   rb   s       `` ``            @@rG   r   r      s    		//
		%%iiii--88		//;; 	& 	=  0 I 		//
		%%iiii--88		//;; 	& 	=  0 I ''))(3K''))(3Kggoo));*DEO%))I * 'x0  &))1 * 'x0  &))G * 'x0  	O	B	1 	1 *00/0 *//GG  00!'!1 1 34 *66!4&4&1A 7 C ww22?CH88(:N:NO((*HfH(()A:KK--/0Vr   c           	      n  ^ [         R                  R                  [         R                  R                  U 5      5      nUR	                  5       nUc|  [        [         R                  " 5        Vs/ s HQ  nUR                  R                  S5      S   U;   d  M'  UR                  UR                  R                  S5      S   4PMS     sn5      nO[        [         R                  " 5        V^s/ s Hm  mTR                  R                  S5      S   U;   d  M'  [        U4S jU 5       5      (       d  MC  TR                  TR                  R                  S5      S   4PMo     sn5      n/ n[        [        [        S [         R                  " 5       5      [         R                  " 5       5      5      n0 n[         R                  " SSS9   U HF  u  pXz   nUR                  5       R                  5       nXU
   :X  d  M1  UR!                  U5        XU
'   MH     S S S 5        U$ s  snf s  snf ! , (       d  f       U$ = f)N:r   c              3   B   >#    U  H  nUTR                   ;  v   M     g 7fr   )r   ).0scopevars     rG   	<genexpr>/get_pretrained_variables_map.<locals>.<genexpr>>  s"      N;-9E !&SXX 5-9s   c                 >    U R                   R                  S5      S   $ )Nr   r   )r   split)xs    rG   r   .get_pretrained_variables_map.<locals>.<lambda>D  s    !&&,,s+A.r    T)reuse)r   r2   NewCheckpointReaderlatest_checkpointget_variable_to_shape_mapsortedr7   r   r  alldictr   r   variable_scope	get_shapeas_listappend)checkpoint_file_pathignore_scopereadersaved_shapesr  	var_namesrestore_varsname2varrestore_mapvar_namesaved_var_namecurr_var	var_shapes       `        rG   r<   r<   3  s   XX))
""#78:F335L')':':'<G'<"xx~~c215E ?SXXsxx~~c':1'=>'<G H	 ')':':'<<'<"xx~~c215E ?JM N;-9N; K; ?SXXsxx~~c':1'=>'<< =	
 L.0C0C0EF!	#$H K			2T	*(1$H/H **,446I88##H-.6N+ )2 
+ -G< 
+	* s0   -&H.H*&H H 0.H 0H%8H%%
H4)   d      i   r   TNNr   )os.pathpathr!   r   typingr   r   
tensorflowr    modelscope.hub.snapshot_downloadr   modelscope.models.nlpr   modelscope.trainers.baser   modelscope.trainers.builderr   modelscope.utils.constantr	   modelscope.utils.loggerr
   __version__compatv1disable_eager_executionr:   register_moduler   r   r<   r   r   rG   <module>r2     s      !  > 6 0 0 / .>>U	B 	 
&;<C{ C =CT "Rjr   