
    9i<                     ~   S SK r S SKJrJr  S SKJrJrJrJrJ	r	J
r
  S SKJrJrJr  S SKJrJr  S SKJrJrJrJr  S SKJrJr  S SKJr  S	S
KJr  \" 5       r0 \R>                  \R@                  4\RB                  _\RD                  \RF                  4\RF                  _\RH                  \RJ                  4\RL                  _\RH                  \RN                  4\RN                  _\RH                  \RP                  4\RP                  _\RH                  \RR                  4\RR                  _\RH                  \RT                  4\RL                  _\RH                  \RV                  4\RL                  _\RH                  \RX                  4\RL                  _\RH                  \RZ                  4\RL                  _\RH                  \R\                  4\RL                  _\RH                  \R^                  4\RL                  _\RH                  \R`                  4\R`                  _\RH                  \Rb                  4\Rd                  _\RH                  \Rf                  4\Rd                  _\RH                  \Rh                  4\Rd                  _\RH                  \Rj                  4\Rd                  _0 \Rl                  \RJ                  4\Rn                  _\Rp                  \RJ                  4\Rr                  _\Rt                  \RJ                  4\Rn                  _\Rv                  \RJ                  4\Rx                  _\Rv                  \Rz                  4\Rx                  _\R|                  \RJ                  4\RL                  _\R|                  \RP                  4\RP                  _\R~                  \RP                  4\R                  _\R                  \RJ                  4\RL                  _\R                  \RP                  4\RP                  _\R                  \R                  4\R                  _\R                  \RT                  4\RL                  _\R                  \RX                  4\RL                  _\R                  \RZ                  4\RL                  _\R                  \R\                  4\RL                  _\R                  \R^                  4\RL                  _\R                  \Rb                  4\Rd                  _E0 \R                  \R                  4\Rd                  _\R                  \Rf                  4\Rd                  _\R                  \Rj                  4\Rd                  _\R                  \R                  4\R                  _\R                  \R                  4\R                  _\R                  \R                  4\R                  _\R                  \RJ                  4\RL                  _\R                  \RP                  4\RP                  _\R                  \RT                  4\RL                  _\R                  \RX                  4\RL                  _\R                  \RZ                  4\RL                  _\R                  \R\                  4\RL                  _\R                  \R                  4\R                  _\R                  \Rj                  4\R                  _\R                  \Rb                  4\R                  _\R                  \Rj                  4\R                  _\R                  \Rj                  4\R                  _E\R                  \R                  4\R                  \Rf                  \Rf                  4\R                  \Rf                  \Rb                  4\R                  \Rf                  \R                  4\R                  \RT                  \RT                  4\RL                  \RP                  \RP                  4\RP                  \R                  \R                  4\R                  \R                  \R                  4\R                  \R`                  \R`                  4\R`                  \R                  \R                  4\Rn                  \R                  \R                  4\R                  \R                  \R                  4\R                  \R                  \R                  4\R                  \R                  \R                  4\R                  0Er[ " S S\5      r\g)    N)ABCabstractmethod)AnyCallableDictOptionalSequenceUnion)ModelsPreprocessors
TaskModels)Config
ConfigDict)DEFAULT_MODEL_REVISIONInvokeModeKeysTasks)read_configsnapshot_download)
get_logger   )build_preprocessorc                   "   \ rS rSrSr\R                  4S jr\S\	\
\4   S\	\
\4   4S j5       r\S 5       r\R                  S 5       r\\S	\R                  4S
\
S\\
   S\4S jj5       r  SS\\
\R.                  4   S\\   S\4S jjrSrg	)Preprocessor   zBase of preprocessors.
    c                     Xl         U R                   [        R                  [        R                  [        R                  4;   d   eS[
        R                  ;   a  [        [
        R                  S   5      OS U l        g )N
LOCAL_RANK)	_moder   	INFERENCETRAINEVALosenvironintdevice)selfmodeargskwargss       ]/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/preprocessors/base.py__init__Preprocessor.__init__   sh    
zzh00(..&mm- - 	- - *6)C JJ|$&IM 	    datareturnc                     g N )r&   r.   s     r*   __call__Preprocessor.__call__   s    r-   c                     U R                   $ r1   r   )r&   s    r*   r'   Preprocessor.mode   s    zzr-   c                     Xl         g r1   r6   )r&   values     r*   r'   r7      s    
r-   Nmodel_name_or_pathrevisioncfg_dictc                 J   [         R                  R                  U5      (       d-  [        UU[        R
                  [        R                  0/ SQS9nOUnUc  [        U5      nOUnUR                  nSU;   a  UR                  S5      n[        R                  " U5      n	SU;   a  UR                  S5      n	U[        R                  :X  a  SOSn
[        US5      (       a  [        UR                   5      S	:X  a   ["        R%                  S
5        ['        5       nOUR                   nSU;  a-  X;   a  [)        X5      nO["        R%                  SU
 S35        UnOUnUR+                  SU05        UR+                  U5        SU;   a,  [-        U[.        5      (       a  [1        S5      e[3        X5      nO["        R%                  SU SU S35        [)        US['        5       5      n[        US5      (       a  UR4                  O[)        USS5      nUb  Uc  ["        R%                  SU SU S35        gX4[6        ;  a  ["        R9                  SX4 S35        g['        S[6        X4   0UE5      n[3        X5      nXMl        UR                  SS5        [        US5      (       d  X}l        U$ )a/  Instantiate a preprocessor from local directory or remote model repo. Note
that when loading from remote, the model revision can be specified.

Args:
    model_name_or_path(str): A model dir or a model id used to load the preprocessor out.
    revision(str, `optional`): The revision used when the model_name_or_path is
        a model id of the remote hub. default `master`.
    cfg_dict(Config, `optional`): An optional config. If provided, it will replace
        the config read out of the `model_name_or_path`
    preprocessor_mode(str, `optional`): Specify the working mode of the preprocessor, can be `train`, `eval`,
        or `inference`. Default value `inference`.
        The preprocessor field in the config may contain two sub preprocessors:
        >>> {
        >>>     "train": {
        >>>         "type": "some-train-preprocessor"
        >>>     },
        >>>     "val": {
        >>>         "type": "some-eval-preprocessor"
        >>>     }
        >>> }
        In this scenario, the `train` preprocessor will be loaded in the `train` mode, the `val` preprocessor
        will be loaded in the `eval` or `inference` mode. The `mode` field in the preprocessor class
        will be assigned in all the modes.
        Or just one:
        >>> {
        >>>     "type": "some-train-preprocessor"
        >>> }
        In this scenario, the sole preprocessor will be loaded in all the modes,
        and the `mode` field in the preprocessor class will be assigned.

    **kwargs:
        task(str, `optional`): The `Tasks` enumeration value to replace the task value
        read out of config in the `model_name_or_path`.
        This is useful when the preprocessor does not have a `type` field and the task to be used is not
        equal to the task of which the model is saved.
        Other kwargs will be directly fed into the preprocessor, to replace the default configs.

Returns:
    The preprocessor instance.

Examples:
    >>> from modelscope.preprocessors import Preprocessor
    >>> Preprocessor.from_pretrained('damo/nlp_debertav2_fill-mask_chinese-base')

)z.*.binz.*.tsz.*.ptz.*.data-00000-of-00001z.*.onnxz.*.metaz.*.pbz.*.index)r;   
user_agentignore_file_patternNtaskfieldtrainvalpreprocessorr   z#No preprocessor field found in cfg.typezNo zJ key and type key found in preprocessor domain of configuration.json file.	model_dirzNot supported yet!z;Cannot find available config to build preprocessor at mode z, current config: z0. trying to build by task and model information.model
model_typezFind task: z, model type: zL. Insufficient information to build preprocessor, skip building preprocessorzNo preprocessor key zn found in PREPROCESSOR_MAP, skip building preprocessor. If the pipeline runs normally, please ignore this log.cfg)r"   pathexistsr   r   KEYPREPROCESSORr   r@   popr   find_field_by_taskr   r    hasattrlenrD   loggerwarningr   getattrupdate
isinstancer	   NotImplementedErrorr   rE   PREPROCESSOR_MAPinfor'   rI   )clsr:   r;   r<   preprocessor_moder)   rF   rI   r@   
field_namesub_keypreprocessor_cfgsub_cfgrD   	model_cfgrH   s                   r*   from_pretrainedPreprocessor.from_pretrained   s   h ww~~011)"!"JJ(;(;<	%	I +Ii(CCxxV::f%D--d3
fG,J.(..@'esN++s33C3C/D/INN@A)|"//))*!"2<' #F GH +&G 	Y/0vW'8,, **>??-gBLNNMN_M` a##*)+[]  Wjl;I+26,# ,#(/	<(N |z1!$~j\ Ba b !)99*J+=*> ?i j  (*);<"" G .gBL-K&|U++"r-   target_folderconfigsave_config_functionc                     Uc  [        U S5      (       a  U R                  nUbR  SU;   a8  US   b2  SUS   ;   a	  SUS   S'   O SUS   ;   a  SUS   S   ;   a  SUS   S   S'   Uc  SSKJn  UnU" X5        gg)	a#  Save the preprocessor, its configuration and other related files to a directory,
    so that it can be re-loaded

By default, this method will save the preprocessor's config with mode `inference`.

Args:
    target_folder (Union[str, os.PathLike]):
        Directory to which to save. Will be created if it doesn't exist.

    config (Optional[dict], optional):
        The config for the configuration.json

    save_config_function (Callable): The function used to save the configuration, call this function
        after the config is updated.
NrI   rD   r'   	inferencerC   r   )save_configuration)rP   rI   modelscope.utils.checkpointrh   )r&   rc   rd   re   rh   s        r*   save_pretrainedPreprocessor.save_pretrainedZ  s    & >gdE22XXF'F>,B,NVN335@F>*62f^446&D((-D/ :/<GF>*51&9#+J'9$ 7 r-   )r   r%   )NN)__name__
__module____qualname____firstlineno____doc__r   r   r+   r   r   strr   r3   propertyr'   setterclassmethodr   r   r   ra   r
   r"   PathLikedictr   rj   __static_attributes__r2   r-   r*   r   r      s     %..  T#s(^ S#X     
[[   3I+/*2*<*<	J,/J"*3-J #)J J\ 269="8',S"++-='>"8 ("8 /7"8 "8r-   r   )]r"   abcr   r   typingr   r   r   r   r	   r
   modelscope.metainfor   r   r   modelscope.utils.configr   r   modelscope.utils.constantr   r   r   r   modelscope.utils.hubr   r   modelscope.utils.loggerr   builderr   rR   canmtcompetency_aware_translationcanmt_translationbarttext_error_correctionbertbackbonesen_cls_tokenizerdocument_segmentation	fill_masksentence_embeddingtext_classification&speaker_diarization_dialogue_detectionnlisentiment_classificationsentence_similarityzero_shot_classificationtext_rankingpart_of_speechtoken_cls_tokenizertoken_classification3speaker_diarization_semantic_speaker_turn_detectionword_segmentationbloomtext_gen_tokenizergpt_neosentence_piecepalmT5text2text_gen_preprocessortext2text_generation
deberta_v2ponetfill_mask_ponet
structbertfaq_question_answering#faq_question_answering_preprocessortoken_classification_for_nernamed_entity_recognitiondoc2bot!document_grounded_dialog_generatedocument_grounded_dialog_rerank"document_grounded_dialog_retrievalvecolcrfsequence_labeling_tokenizer	lcrf_wseg	tcrf_wsegtcrffeature_extractioninformation_extractionre_tokenizertext_generationtinynas_detectionimage_object_detection%object_detection_tinynas_preprocessortinynas_damoyolo domain_specific_object_detectioncontrollable_image_generation*controllable_image_generation_preprocessorrX   r   r2   r-   r*   <module>r      s.	   	 # A A A A 68 8 ? . '	b\\5556##b
 [[%--.''b [[%..!##b [[%--.''b [[%//"b [[%**+$$!b" [[%++,##%b& [[%>>?##)b* [[%))##-b. [[%001##1b2 [[%++,##5b6 [[%001##9b: [[%$$%=b> [[%&&'%%AbB [[%,,-%%EbF [[%KKL%%IbJ [[%))*%%MbR \\5>>"$$Ub\ ^^U^^$  _bh [[%..!$$kbp YY,,sbt YY**+,,wb| '##b@ (CbH \\5??#!!KbP '##SbT (WbX 44555[b\ 112##_b` 		"##cbd 667##gbh 112##kbl 667##obp ,,-%%sbt ((%*H*HI%%wbx 223%%{b| //0%%bD ^^U<<=33GbH ^^U::;11KbL ^^U==>44ObT [[%..!##WbX [[%//"[b\ [[%++,##_b` [[%))##cbd [[%001##gbh [[%++,##kbp [[%001--sbt [[%))*--wbx [[%&&'--{b| u../--b@ u../--CbD [[%001-- $$e&@&@A--$$e&:&:;--$$e&D&DE--##U%>%>?##5??+""E$<$<=$$&&(D(DEe001!6!67$$ u;;<77e::;77eDDE77))5+N+NO<<Cb JG83 G8r-   