9iL|SSKrSSKrSSKrSSKJr SSKrSSKrSSKrSSK J r SSK J r J r Jr SSKJr SSKJr SSKJr SSKJr SS KJr SS KJrJrJr SS KJr SS KJ r "S S5r!\RD"\RFS9"SS\55r#\RD"\RHS9"SS\#55r$g)N)Optional)check_model_is_id)UploadStrategypush_to_hub_in_queue wait_for_done)Hooks)HOOKS)CheckpointProcessor)Hook)Priority)DEFAULT_REPOSITORY_REVISIONLogKeys ModelFile) get_logger) is_masterc \rSrSrSrSrSrSrg)CheckpointStrategyby_epochby_stepnoN)__name__ __module__ __qualname____firstlineno__rrr__static_attributes__rt/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/trainers/hooks/checkpoint/checkpoint_hook.pyrrsHG Brr) module_namecV\rSrSrSr\R rSrSr \ RSSSSSSSSS\ \ RS4 S \\S \\S \\S \\S \\S\\S\\S\\S\\S\\S\\S\\S\4SjjrSrSrSrSrSrSrSrS%SjrSrS rS!rS"rS#r S$r!g)&CheckpointHook aSave checkpoints periodically. Args: save_strategy(str): The strategy to save checkpoint, can be `by_epoch`, `by_step` or `no` interval (int): The frequency to save model. If `by_epoch=True`, it means the number of epochs, else means the number of iterations save_dir (str): The directory to save checkpoints. If is None, use `trainer.work_dir` output_dir (str): The absolute path to save the output files for inference. If it's not specified, the default dir is `{sub_dir}/output`. save_last (bool): Whether to save the last checkpoint. Default: True. max_checkpoint_num (int): The max number of checkpoint files, default None which means never delete anything. If the number exceeding the limit, earlier checkpoints will be deleted first. push_to_hub (bool): Whether push the checkpoint to modelhub. hub_repo_id (str): The hub repo id. hub_token (str): The token of the modelhub. You can also set the environment variable `MODELSCOPE_API_TOKEN`. private_hub (bool): Whether push to a private hub, default True. hub_revision (str): Which branch to push the model to, default is `master`. upload_strategy (str): The action adopted when the previous uploading is not done and the next one is coming, can be `cancel` or `wait`. save_trainer_state (bool): Save the trainer state for continue training, default True. kwargs: by_epoch (bool): Same with `save_strategy`, but has a higher priority, legacy argument. output_sub_dir (str): The folder under the `save_dir` to save the output checkpoint for inference. This argument is kept to fit the existing configs. zeval_result.txtztrain.checkpointrNTF save_strategyintervalsave_dir output_dir save_lastmax_checkpoint_num push_to_hub hub_repo_id hub_token private_hub hub_revisionupload_strategysave_trainer_statec X lX0lSU;a0US(a[RO[RUlOXlSU;aUSUlSUlO SUlX@lXPlSUl Xpl Xl Xl Xl XlXlXlSUlSUlSUlUb[)[+U5S5Ul/Ul[/5Ulg)Nroutput_sub_dir)r%r&rrrr$r2r'r( rng_stater*r+r,r-r.r/r0tag is_model_idr)maxinthistory_checkpointsr processor)selfr$r%r&r'r(r)r*r+r,r-r.r/r0kwargss r__init__CheckpointHook.__init__Bs!   @FA!3! KK  > - -g 6 6{{ $6$?$??KK$$/ 0A/B&IKKK$$/ q0@/AGI  ! !' 2 ;;4++##~'<'<<!__ " !__z9FB OOZtE!   gz F,;rcUR[R:wagURU5(a!UR U[R5 ggN)r$rr _should_savergr<rPs rafter_train_epoch CheckpointHook.after_train_epochsE   !3!#> ? &rcUR[R:wagURU5(a!UR U[R5 ggrj)r$rrrkrgrls rafter_train_iterCheckpointHook.after_train_itersE   !3!;!; ;    W % % MM'#5#=#= > &rcURRS5 [URURSS9 [ UR5 UR (aURRS5 gg)Nz,Train finished. Uploading models, waiting...T)strategydonezUploading models done.)rErOrPUSH_TO_HUB_QUEUE_NAMEr/rr*rls r after_runCheckpointHook.after_runsc GH  ' '))  d112    KK  5 6 rcURc%[URUR5UlU=RS- sl[ UR URURUURURUSUR3URUR(aURUS9 $SUS9 $)Nr4zv1.) rs repo_namer'tokenprivatecommit_messager6revision source_reporf) r7rinput_model_idr,r6rrur/r+r-r.)r<rPrer'rfs rrdCheckpointHook._push_to_hubs    #01G1G15 AD  A #  ' '))&&!..$$!dhhZ &&262B2B..! # #IK! # #rc[[RRURUR 5S5nUR [R"UR55 SSS5 g!,(df  g=f)Nw) openrGrHrIr'EVAL_RESULT_FILEwritejsondumps metric_values)r<rPfs rsave_evaluate_results$CheckpointHook.save_evaluate_resultssV "'',,t0E0EF GGDJJw445 6s 0A88 BcZ[RRURU5nUR U5nUR R XURUUR5 URU5 URRU5 URU5 U$)z7Save checkpoint files and remove obsolete ones ) rGrHrIr&_create_training_stater;save_checkpointsr'r0rr:append_remove_obsolete_checkpointsr<rPrecheckpoint_path_prefixmetas rr`CheckpointHook._save_checkpoints"$dmmV!D**73 ''(,(,(?(? A ""7+   ''(>? ))'2 rcURb[UR5UR:aURVs/sHo"PM nnURR5 [ U5HuupEU[U5UR- :a9UR R SU35 URRXS9 MZURRU5 Mw gggs snf)Ndeleting checkpoint: r) r)lenr:clear enumeraterErOr;remove_checkpointsr)r<rPckptr:irs rr+CheckpointHook._remove_obsolete_checkpointss  " " .D,,-0G0GG484L4L"M4LD44L "M  $ $ * * ,-67J-K)s./$2I2IIIKK$$/0F/GHJNN556P,,334JK.LH /"Ms C-cPUR[R:XaURnURnO8UR[R :XaUR nURnOgU"UUR5(dUR(aU"U5(agg)NFT) r$rr is_last_epochevery_n_epochsr is_last_iter every_n_itersr%r()r<rP check_lastcheck_frequencys rrkCheckpointHook._should_saves   !3!rArQrZrgrmrprvrdrr`rrkrrrrrr"r" sM4||H(/1C0K0K+,+/-1-159/4.2,0/3/J2@2G2G,0+/ ( +/#C=+/$C=+/&c] +/ %TN +/ &.c] +/'tn+/'sm+/%SM+/'tn+/ (}+/#+3-+/&*+/Z# N48 G.@?7#$7  L rr"c^\rSrSrSr\R rSSS.rSS\ S\ \ S\ \ S \ \ S \ \ S \ \ S \ 4U4S jjjr SrSrSrSrSrSrSrSrSrSrSrU=r$)BestCkptSaverHooki!a Save best checkpoints hook. Args: metric_key (str): Metric key to compare rule for best score. save_best(bool): Save the best checkpoint, if set to False, this hook will have no effect. rule (str): Comparison rule for best score. Support "max" and "min". If rule is "max", the checkpoint at the maximum `metric_key` will be saved, If rule is "min", the checkpoint at the minimum `metric_key` will be saved. save_file_name: The manual specified saving file name. restore_best (bool): Whether to restore the best checkpoint after training. max_checkpoint_num (int): The max number of checkpoint files, default None which means never delete anything. If the number exceeding the limit, checkpoints with worse metric will be deleted, which is judged by the `rule` and `metric_key` arguments. save_trainer_state (bool): Save the trainer state for continue training, default True. The `BestCkptSaverHook` class accepts `output_sub_dir` and `output_dir` argument as its super class do. If neither of them are passed, the default value is `{save_dir}/output_best`. This class will not accept the `interval` or `save_strategy` or `by_epoch` argument, because the saving interval will follow the `EvaluationHook`. c X:$rjrxys rBestCkptSaverHook.;sAErc X:$rjrrs rrr;squrr8min metric_key save_bestrulesave_file_name restore_bestr)r0c J>US;dS5e0n SU;aSU;a[RU S'URSS5 URSS5 [T U]"SUUS.UDU D6 X lXlX0lSUlSUl X@l XPl [5Ul g) Nrz%Only support "max" or "min" rule now.r2r'r%r$)r)r0r)rTRAIN_BEST_OUTPUT_DIRpopsuperr>rrr _best_metric_best_ckpt_filerrsetr:) r<rrrrrr)r0r= output_kwargsrs rr>BestCkptSaverHook.__init__=s~%N'NN% 6 )l&.H.7.M.MM* + :t$ ?D)  11    #$  #,(#&5 rcSSKJn URU5n[U5S:XaURR S5 USR SUR4:Xa*URU5(aURUS5 ggg)NrEvaluationHookITrying to save the best checkpoint, but there is no evaluation, skipping.rVr) modelscope.trainers.hooksrget_hookrrEerror last_eval_tagrVrkrgr<rPr eval_hooks rrm#BestCkptSaverHook.after_train_epoch[s<$$^4 y>Q  KK  [  Q< % %*( (,0,=,=g,F,F MM': .-G (rcSSKJn URU5n[U5S:XaURR S5 USR SUR4:Xa*URU5(aURUS5 ggg)NrrrrXr) rrrrrErrrXrkrgrs rrp"BestCkptSaverHook.after_train_itergs<$$^4 y>Q  KK  [  Q< % % *& &*.*;*;G*D*D MM'9 -+E &rc^UR=(a URUR5$rj)r_is_best_metricrrls rrkBestCkptSaverHook._should_savess"~~M$"6"6w7L7L"MMrcFUcgURU;a[SURSU35eURcXRUlgURURnU"XRUR5(aXRUlgg)NFzNot find metric_key: z in T)r ValueErrorrrule_mapr)r<r compare_fns rr!BestCkptSaverHook._is_best_metricvs   ??- /''8]OLN N    $ -oo >D tyy1J-8$:K:KLL$1//$B!rcU[R:Xa;S[RURS-SUR UR 3$S[RURS-SUR UR 3$)Nbest_r4rT) rrrrUrVrrrWrXrYs rrZ!BestCkptSaverHook.generate_prefixs{ .77 77==/'--!*;)[URTR5S5nTRS:XaU*$U$)Nr4r8)floatsplitrr)name1metric1r<s rextract_metric_from_filenameTBestCkptSaverHook._remove_obsolete_checkpoints..extract_metric_from_filenames7EKK8;rmrprkrrZr`rrrrvr __classcell__)rs@rrr!s.||H)2DEH.2',150556,0) )$TN) })"*# ) (~ ) &.c] )&*))< / .N$a $P. NNrr)%rGrrbtypingrrrrrmodelscope.hub.check_modelrmodelscope.hub.push_to_hubrrrmodelscope.metainfor!modelscope.trainers.hooks.builderr 9modelscope.trainers.hooks.checkpoint.checkpoint_processorr modelscope.trainers.hooks.hookr "modelscope.trainers.hooks.priorityr modelscope.utils.constantr rrmodelscope.utils.loggerrmodelscope.utils.torch_utilsrrregister_moduler"rrrrrs   877%3/722.2 5#7#78}T}9}@5#:#:;gNgN<gNr