
    9i                       S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKrS SKrS SKJr  S SKJr  S SKJr  S SKJr  S SKJr  S SKJrJrJrJrJrJrJrJrJ r   S SK!J"r"  S SK#r#S SK$r$S S	K$J%r%  S S
K&J'r'J(r(  S SK)J*r*  S SK+J,r,  S SK-J.r.J/r/J0r0J1r1J2r2J3r3J4r4J5r5J6r6J7r7J8r8J9r9J:r:J;r;J<r<J=r=J>r>J?r?J@r@JArAJBrBJCrCJDrDJErEJFrFJGrGJHrHJIrIJJrJJKrKJLrLJMrM  S SKNJOrOJPrPJQrQJRrRJSrSJTrTJUrUJVrVJWrWJXrX  S SKYJZrZ  S SK[J\r\J]r]  S SK^J_r_  S SK`Jara  S SKbJcrcJdrdJereJfrfJgrgJhrhJiri  S SKjJkrkJlrlJmrmJnrnJoroJprpJqrqJrrrJsrsJtrtJuruJvrvJwrwJxrxJyryJzrzJ{r{  S SK|J}r}J~r~  S SKJr  S SKJrJrJrJrJrJrJrJr  S SKJr  \" 5       r " S S5      r " S S5      r " S S5      rg)    N)defaultdict)
HTTPStatus)	CookieJar)
expanduser)Path)	AnyBinaryIODictIterableListLiteralOptionalTupleUnion)	urlencode)Session)HTTPAdapterRetry)	HTTPErrortqdm) API_HTTP_CLIENT_MAX_RETRIESAPI_HTTP_CLIENT_TIMEOUTAPI_RESPONSE_FIELD_DATAAPI_RESPONSE_FIELD_EMAIL#API_RESPONSE_FIELD_GIT_ACCESS_TOKENAPI_RESPONSE_FIELD_MESSAGEAPI_RESPONSE_FIELD_USERNAMEDEFAULT_MAX_WORKERSDEFAULT_MODELSCOPE_INTL_DOMAINMODELSCOPE_CLOUD_ENVIRONMENTMODELSCOPE_CLOUD_USERNAMEMODELSCOPE_CREDENTIALS_PATHMODELSCOPE_DOMAINMODELSCOPE_PREFER_AI_SITEMODELSCOPE_REQUEST_IDMODELSCOPE_URL_SCHEMEONE_YEAR_SECONDSREQUESTS_API_HTTP_METHODTEMPORARY_FOLDER_NAME"UPLOAD_BLOB_TQDM_DISABLE_THRESHOLDUPLOAD_COMMIT_BATCH_SIZEUPLOAD_MAX_FILE_COUNTUPLOAD_MAX_FILE_COUNT_IN_DIRUPLOAD_MAX_FILE_SIZE#UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT$UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFSVALID_SORT_KEYSDatasetVisibilityLicensesModelVisibility
VisibilityVisibilityMap)
InvalidParameterNotExistErrorNotLoginExceptionRequestErrordatahub_raise_on_errorhandle_http_post_errorhandle_http_responseis_okraise_for_http_statusraise_on_error)GitCommandWrapper)DatasetInfo	ModelInfo
Repository)	AigcModel)add_content_to_file
get_domainget_endpointget_readable_folder_sizeget_release_datetimeis_env_truemodel_id_to_group_owner_name)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONDEFAULT_REPOSITORY_REVISIONMASTER_MODEL_BRANCHMETA_FILES_FORMATREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORTConfigFieldsDatasetFormationsDatasetMetaFormatsDownloadChannelDownloadMode
Frameworks	ModelFileTasksVirgoDatasetConfig)get_file_hashget_file_size)
get_logger)DATASET_LFS_SUFFIXDEFAULT_IGNORE_PATTERNSMODEL_LFS_SUFFIXCommitHistoryResponse
CommitInfoCommitOperationCommitOperationAdd	RepoUtils)thread_executorc                      \ rS rSrSrS\\4S\\   4S jjr	S\4S jr
SS	\\   4S
 jjr  SS\\   S\\   4S jjr\R                  \R"                  SSSSS4S\S\\   S\\   S\\   S\\   S\\   S\\   S\S   S\4S jjr   SS\S\S\\   S\\   S\S   S\4S jjrSS\S\\   4S jjrSS\S\\   4S jjr\S4S\S\\   S\\   S\4S jjrSS.S\S\\   S\4S  jjr\SS!.S\S\\   S\\   S\4S" jjrSSS!.S\S\\   S\\   S\4S# jjr\\SS$.S\S\\   S\\   S\\   S\ \\4   4
S% jjr!SSSSS&.S\S\\   S\\   S'\\   S\\   S\4S( jjr"SS\S\S\\   4S) jjr#\$S* 5       r%\R                  \R"                  SS+S\&SSS4	S\S,\S\\   S\\   S\\   S-\\   S.\\   S\\   S\\   S/\\ \'\   \4      S0\\ \\'\   4      4S1 jjr(   SS4\S5\\   S6\\   S\\   S\4
S7 jjr)S2S3SSSS8.S4\S5\\   S6\\   S9\\   S:\\   S\\   S\4S; jjr*SS<\ \\+4   S\+4S= jjr,  SS\S>\\   S<\ \\+4   S\'\   4S? jjr-   SS\S>\\   S<\ \\+4   S\\   S\'\   4
S@ jjr.SA r/   SS\SB\\+   S\\   4SC jjr0   SS\SB\\+   S\\   4SD jjr1  SS\S<\ \\+4   S\\   S\2\'\   \'\   4   4SE jjr3 SS\S<\ \\+4   S\2\'\   \'\   4   4SF jjr4\SSS0 S4S\S\\   SG\\   SH\\   S<\ \\+4   SI\\   S\\   S\'\   4SJ jjr5SSK.S\SL\S\\   4SM jjr6S\R"                  \7R                  SS4SN\SO\S\\   S\\   S\\   SP\\   S\\   S\4SQ jjr8SSR\S\\   4SS jjr9SSN\SO\S\\   4ST jjr:    SSN\SO\S\SW\SH\S5\S6\S\\   4SX jjr;\\&S2SYSSZ.S\S\\   S\\   S5\S6\S\\   4S[ jjr<\&S\SUS2SVSS].S\S\SW\SH\S5\S6\S\\   4S^ jjr=\&S4SR\S\\   S\\   4S_ jjr> SSN\SO\SR\S\S\\   4
S` jjr?\$Sa\Sb\4Sc j5       r@ SSN\SO\S\Sb\Sa\Sd\AS\\   4Se jjrB\$Sf\CR                  4Sg j5       rE\FSSUS4Sh\SN\SO\S\\   Si\\   Sj\\   S\\   4Sk jjrG\FS4Sh\SN\SO\S\\   S\\   4
Sl jjrH\FS4SN\SO\S\\   S\\   4Sm jjrI\FS4SN\SO\Sn\S\\   S\\   4
So jjrJSSR\Sp\S\4Sq jjrK SSN\SO\S\Sr\S\\   4
Ss jjrL SS\\   4St jjrM SSu\SN\SO\S\S\\   S\4Sv jjrN SSu\SN\SO\S\S\\   S\4Sw jjrOSx rP SSN\SO\Sy\S\\   SS4
Sz jjrQS{ rRSS\S\\   S\4S| jjrSS\TR                  \S\R"                  SSSUSS}.	S\S\ \\S4   S\\   S\\   S\\   S\\   S\\   S~\\   S\\   S\\U   S\4S jjrVSS\\&SSSS.S\S\W\X   S-\S\\   S\S\\   S\\   S\\   S\S\S\Y4S jjrZS\SSS2SS\&S.S\ \\[\\\]4   S\S\S\ \S4   S\\   S-\\   S\\   S\\   S\\   S\\   S\\   S\Y4S jjr^SSSS\SS\_\&S.	S\S\ \\[\'\   \'\[   4   S\\   S-\\   S\\   S\ \S4   S\\   S\\ \'\   \4      S\\ \'\   \4      S\S\\   S\ \Y\'\Y   4   4S jjr`SSS2S.S\S\S\S\S\ \\[\\\]4   S\\   S\\   S\\   S\4S jjraSS.S\S\S\'\b\\c4      S\\   S\'\b\\c4      4
S jjrd  SS\ \\[\'\   \'\[   4   S\S\\ \'\   \4      S\\ \'\   \4      S\'\ \e\A4      4
S jjrf\$S\W\X   S-\S\b\\c4   4S j5       rgSS\h4S jjri\SS!.S\S\S\ \\'\   4   S\\   S\\   S\b\\c4   4S jjrj SS\S\kS   S\kS   S\ \S4   S\4
S jjrlSrmg)HubApib   zModel hub api interface.
    Nendpointc                    Ub  UO	[        5       U l        S[        R                  5       0U l        [        5       U l        [        USSSSSS9n[        US9nU R                  R                  S	U5        U R                  R                  S
U5        [         HA  n[        U R                  U[        R                  " [        U R                  U5      US95        MC     [        5       U l        g)uz   The ModelScope HubApi。

Args:
    endpoint (str, optional): The modelscope server http|https address. Defaults to None.
N
user-agent      )  i  i  i  F)totalreadconnectbackoff_factorstatus_forcelistrespect_retry_after_header)max_retrieszhttp://zhttps://timeout)rJ   ro   ModelScopeConfigget_user_agentheadersr   sessionr   r   mountr)   setattr	functoolspartialgetattrUploadingCheckupload_checker)selfro   r}   r{   retryadaptermethods          R/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/hub/api.py__init__HubApi.__init__f   s     %-$8ln$&6&E&E&GHy1',
 %09g.:w/.Ff!!DLL&1#%& / -.    access_tokenc                     SSK Jn  SSKJn  U R                  (       a  U" U R                  5      R
                  O	[        5       nU" 5       nUR                  SUUSS9  U$ )z
Get jar cookies for authentication from access_token.

Args:
    access_token (str): user access token on ModelScope.

Returns:
    jar (CookieJar): cookies for authentication.
r   )RequestsCookieJar)urlparsem_session_id/)domainpath)requests.cookiesr   urllib.parser   ro   netlocrI   set)r   r   r   r   r   jars         r   _get_cookiesHubApi._get_cookies   sS     	7)8<ht}}-44:<! 	 	 
r   Fcookies_requiredc                     U(       a  U R                  US9nO[        R                  5       nUc  U(       a  [        S5      eU$ )am  
Get cookies for authentication from local cache or access_token.

Args:
    access_token (str): user access token on ModelScope
    cookies_required (bool): whether to raise error if no cookies found, defaults to `False`.

Returns:
    cookies (CookieJar): cookies for authentication.

Raises:
    ValueError: If no credentials found and cookies_required is True.
)r   zNo credentials found.You can pass the `--token` argument, or use HubApi().login(access_token=`your_sdk_token`). Your token is available at https://modelscope.cn/my/myaccesstoken)r   r~   get_cookies
ValueError)r   r   r   cookiess       r   r   HubApi.get_cookies   sJ     ''\'BG&224G?/T  r   c                 R   Uc  [         R                  R                  S5      nU(       d  gU(       d  U R                  nU S3nU R                  R                  USU0U R                  U R                  5      S9n[        U5        UR                  5       n[        U5        U[           [           nUR                  n[        R                  U5        [        R!                  U5        [        R#                  U[           [$           U[           [&           5        U[           [           U4$ )a$  Login with your SDK access token, which can be obtained from
   https://www.modelscope.cn user center.

Args:
    access_token (str): user access token on modelscope, set this argument or set `MODELSCOPE_API_TOKEN`.
    If neither of the tokens exist, login will directly return.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Returns:
    cookies: to authenticate yourself to ModelScope open-api
    git_token: token to access your git repository.

Note:
    You only have to login once within 30 days.
MODELSCOPE_API_TOKENNNz/api/v1/loginAccessToken)jsonr   )osenvirongetro   r   postbuilder_headersr   r@   r   rA   r   r   r   r~   
save_tokensave_cookiessave_user_infor   r   )r   r   ro   r   rdtokenr   s           r   loginHubApi.login   s   ( ::>>*@AL}}H=)LL.((6  8 	a FFHq)*+NO)) 	##E*%%g.''%&'BC%&'?@	B ()/129: 	:r    model_id
visibilitylicensechinese_nameoriginal_model_idr   
aigc_modelrG   returnc	                 R   Uc  [        S5      eU R                  USS9n	U(       d  U R                  n[        U5      u  pU
UUUUU[        R
                  R                  SS5      S.nUb  U S3nUR                  XR                  U R                  5      US9  UR                  UR                  UR                  UR                  UR                  UR                  UR                   =(       d    UUR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  S	.5        UR0                  (       a  UR0                  US
'   OU S3nU R                  U R                  5      n[2        R4                  " S5      S   nUR7                  S5      R9                  SU 35      (       a  SUS'   U R:                  R=                  UUU	US9n[?        U5        URA                  5       n[C        U5        U SU 3nUb  URE                  XU5        U$ )a  Create model repo at ModelScope Hub.

Args:
    model_id (str): The model id in format {owner}/{name}
    visibility (int, optional): visibility of the model(1-private, 5-public), default 5.
    license (str, optional): license of the model, default apache-2.0.
    chinese_name (str, optional): chinese name of the model.
    original_model_id (str, optional): the base model id which this model is trained from
    endpoint: the endpoint to use, default to None to use endpoint specified in the class
    token (str, optional): access token for authentication
    aigc_model (AigcModel, optional): AigcModel instance for AIGC model creation.
        If provided, will create an AIGC model with automatic file upload.
        Refer to modelscope.hub.utils.aigc.AigcModel for details.

Returns:
    str: URL of the created model repository

Raises:
    InvalidParameter: If model_id is invalid or required AIGC parameters are missing.
    ValueError: If not login.

Note:
    model_id = {owner}/{name}
model_id is required!Tr   r   MODELSCOPE_TRAIN_IDr   )r   NameChineseNamer6   LicenseOriginalModelIdTrainIdz/api/v1/models/aigcr   r   ro   )TagShowNameCoverImagesAigcTypeTagDescriptionVisionFoundation	BaseModelWeightsNameWeightsSha256WeightsSize	ModelPathTriggerWordsModelSourceSubVisionFoundationOfficialTagsz/api/v1/models.r   en_USzX-Modelscope-Accept-Languager   r   r   /models/)#r8   r   ro   rN   r   r   r   preupload_weightsr   r   updatetagcover_images	aigc_typedescriptionbase_model_typebase_model_idweight_filenameweight_sha256weight_size
model_pathtrigger_wordsmodel_sourcebase_model_sub_typeofficial_tagsr    splitrstripendswithr   r   r@   r   rA   upload_to_repo)r   r   r   r   r   r   ro   r   r   r   owner_or_groupnamebodyr   r   intl_endr   r   model_repo_urls                      r   create_modelHubApi.create_model   s(   B "#:;;"""M}}H;HE #'$0zz~~&;R@
 !Z23D((BVBVW[WcWcBdow(x KK)~~)66&00","8"8$.$>$>'55J9J)99!+!9!9)55'22 * 8 8)66'1'E'E   '''1'?'?^$ Z~.D&&t||4177<R@??3((1XJ886=G23LL	  
 	a FFHq$:XhZ8 !%%de<r   tag_namec                    Uc  [        S5      eUc  [        S5      eUR                  5       S;   a  [        SU S35      eU R                  USS9nU(       d  U R                  n[	        U5      u  pxUb  U S3n	UR                  X`R                  U R                  5      US	9  UR                  UUUUR                  UR                  UR                  UR                  UR                  UR                  S
.
n
OU SU S3n	SnUUS.n
U R                  R!                  U	U
UU R                  U R                  5      S9n[#        U5        UR%                  5       n['        U5        U SU SU 3nU$ )a  Create a tag for a model at ModelScope Hub.

Args:
    model_id (str): The model id in format {owner}/{name}
    tag_name (str): The tag name (e.g., "v1.0.0")
    endpoint: the endpoint to use, default to None to use endpoint specified in the class
    token (str, optional): access token for authentication
    aigc_model (AigcModel, optional): AigcModel instance for AIGC model tag creation.
        If provided, will create an AIGC model tag with automatic parameters.
        Refer to modelscope.hub.utils.aigc.AigcModel for details.

Returns:
    str: URL of the created tag

Raises:
    InvalidParameter: If model_id, tag_name, ref, or description is invalid.
    ValueError: If not login.

Note:
    model_id = {owner}/{name}
r   ztag_name is required!)mainmasterz
tag_name "zp" is not allowed. Please use a different tag name (e.g., "v1.0", "v1.1", "latest"). Reserved names: main, masterTr   z/api/v1/models/aigc/repo/tagr   )
r   r   r   r   r   r   r   r   r   r   /api/v1/models/z	/repo/tagr   )TagNameRefr   r   z/tags/)r8   lowerr   ro   rN   r   r   r   r   r   r   r   r   r   r   r   r   r@   r   rA   )r   r   r   ro   r   r   r   r   r   r   r   revisionr   r   tag_urls                  r   create_model_tagHubApi.create_model_tagS  s   6 "#:;;"#:;;>>11"XJ '/ 0  """M}}H;HE !Z;<D((BVBVW[WcWcBdow(x  *66&')99!+!9!9)55 * 8 8&00$.$>$>D Zxj	BDH#D
 LL((6	  8 	a FFHqJhxjxjAr   c                 ,   [         R                  5       nU(       d  U R                  nUc  [        S5      eU SU 3nU R                  R                  UUU R                  U R                  5      S9n[        U5        [        UR                  5       5        g)zDelete model_id from ModelScope.

Args:
    model_id (str): The model id.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Raises:
    ValueError: If not login.

Note:
    model_id = {owner}/{name}
N)Token does not exist, please login first.r   r   r   r~   r   ro   r   r   deleter   r   r@   rA   r   )r   r   ro   r   r   r   s         r   delete_modelHubApi.delete_model  s     #..0}}H?HII?8*5LL(/(,(<(<T\\(J   L 	a qvvx r   c                 8    U(       d  U R                   nU SU S3$ )Nr   z.gitro   )r   r   ro   s      r   get_model_urlHubApi.get_model_url  s!    }}H?8*D99r   r   c                 .   [         R                  5       n[        U5      u  pVU(       d  U R                  nU(       a  U SU SU SU 3nO
U SU SU 3nU R                  R                  XtU R                  U R                  5      S9n[        U[        XA5        UR                  [        R                  :X  aU  [        UR                  5       5      (       a  UR                  5       [           $ [!        UR                  5       ["           5      e[%        U5        g)ax  Get model information at ModelScope

Args:
    model_id (str): The model id.
    revision (str optional): revision of model.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Returns:
    The model detail information.

Raises:
    NotExistError: If the model is not exist, will throw NotExistError

Note:
    model_id = {owner}/{name}
r   r   
?Revision=r  N)r~   r   rN   ro   r   r   r   r   r>   loggerstatus_coder   OKr?   r   r   r9   r   r@   )	r   r   r   ro   r   r   r   r   r   s	            r   	get_modelHubApi.get_model  s    , #..0;HE}}HZ~.>avZPXzZDZ~.>avFDLLT%)%9%9$,,%G  IQ:==JMM)QVVXvvx 788#AFFH-G$HII!!$r   )	repo_typerepo_idr  c          	      J   [         R                  R                  [        5      nUb1  UR	                  5       S:w  a  [
        U-   n U R                  XUSS9  U$ [        [        5      (       + n[        US9nU R                  XUS9(       dA  [        U(       + S9n[        R                  SU S	U S
U S35         U R                  XUSS9  U$ U$ ! [         a    [        R                  SU SU S35        e f = f! [         a     [        R                  SU SU SU 35        e f = f)a  Get proper endpoint for read operation (such as download, list etc.)
1. If user has set MODELSCOPE_DOMAIN, construct endpoint with user-specified domain.
   If the repo does not exist on that endpoint, throw 404 error, otherwise return the endpoint.
2. If domain is not set, check existence of repo in cn-site and ai-site (intl version) respectively.
   Checking order is determined by MODELSCOPE_PREFER_AI_SITE.
     a. if MODELSCOPE_PREFER_AI_SITE is not set ,check cn-site first before ai-site (intl version)
     b. otherwise check ai-site before cn-site
   return the endpoint with which the given repo_id exists.
   if neither exists, throw 404 error
r   T)r  r  ro   re_raiseRepo z does not exist on r   )cn_site)r  ro   z not exists on z#, will try on alternative endpoint )r  ro   r  z not exists on either z or )r   r   r   r$   stripr'   repo_exists	Exceptionr  errorrM   r%   rJ   warning)r   r  r  sro   check_cn_firstprefer_endpointalternative_endpoints           r   get_endpoint_for_readHubApi.get_endpoint_for_read  sZ    JJNN,-=QWWY"_,q0H  PXcg h O()BCC&~>   H#/^9K#M NNU7)??:K L??S>TTUW X,  ;OZ^ ! ` ,+""+  uWI-@
!LM  uWI-COCTTXYmXnops   C 8C8 (C58*D")r   ro   c                    [        U5      u  pEU R                  XUS9nU R                  U[        X#S9nU R	                  XSUS9n[        S0 UDXtUS.D6n	U	$ )a  Get model information including commit history.

Args:
    repo_id (str): The model id in the format of
        ``namespace/model_name``.
    revision (str, optional): Specific revision of the model.
        Defaults to ``DEFAULT_MODEL_REVISION``.
    endpoint (str, optional): Hub endpoint to use. When ``None``,
        use the endpoint specified when initializing :class:`HubApi`.

Returns:
    ModelInfo: The model detailed information returned by
    ModelScope Hub with commit history.
)r   r   ro   r  r  r   ro   T)r   r   	recursivero   commitsauthorsiblings )rN   r  list_repo_commitsrU   get_model_filesrD   )
r   r  r   ro   r   _
model_datar+  r-  
model_infos
             r   r3  HubApi.model_info  s    & 9A^^( $ D
(( ) ^''4( ( T ggW^fg
r   c                    [        U5      u  pEU R                  XUS9nU R                  U[        X#S9nU R	                  X=(       d    [
        SUS9n[        S0 UDXtUS.D6n	U	$ )a  Get dataset information including commit history.

Args:
    repo_id (str): The dataset id in the format of
        ``namespace/dataset_name``.
    revision (str, optional): Specific revision of the dataset.
        Defaults to ``None``.
    endpoint (str, optional): Hub endpoint to use. When ``None``,
        use the endpoint specified when initializing :class:`HubApi`.

Returns:
    DatasetInfo: The dataset detailed information returned by
    ModelScope Hub with commit history.
)
dataset_idr   ro   r(  T)r  r   r)  ro   r*  r.  )rN   get_datasetr/  rT   get_dataset_filesrO   rC   )
r   r  r   ro   r   r1  dataset_datar+  r-  dataset_infos
             r   r:  HubApi.dataset_info>  s    & 9A''H ( F(('88 ) `))&J2JVZem * o #m\m7dlmr   )r  r   ro   c                    Ub
  U[         :X  a  U R                  XUS9$ U[        :X  a  U R                  XUS9$ [	        SU S[
         S35      e)a  Get repository information for models or datasets.

Args:
    repo_id (str): The repository id in the format of
        ``namespace/repo_name``.
    revision (str, optional): Specific revision of the repository.
        Currently only effective for model repositories. Defaults to
        ``DEFAULT_MODEL_REVISION``.
    repo_type (str, optional): Type of the repository. Supported
        values are ``"model"`` and ``"dataset"``. If not provided,
        ``"model"`` is assumed.
    endpoint (str, optional): Hub endpoint to use. When ``None``,
        use the endpoint specified when initializing :class:`HubApi`.

Returns:
    Union[ModelInfo, DatasetInfo]: The repository detailed information
    returned by ModelScope Hub.
)r  r   ro   Arg repo_type z# not supported. Please choose from r   )rU   r3  rT   r:  r8   rV   )r   r  r  r   ro   s        r   	repo_infoHubApi.repo_info^  sj    4 	_ <??7PX?YY))$$WRZ$[[YK'JK\J]]^_a 	ar   )r  ro   r  r   r  c                p   Uc  U R                   nUb&  UR                  5       [        ;  a  [        SU-  5      eUb  UR	                  S5      S:w  a  [        SU-  5      eU R                  USS9n[        U5      u  pxUb#  UR                  5       [        :X  a  U SU SU 3n	O
U SU SU 3n	U R                  R                  XU R                  U R                  5      S	9n
[        U
[        XaS5      nUS
:X  a  gUS:X  a  U(       a  [        U
5      eg[        R                  SU S35        [        SU-  5      e)a  
Checks if a repository exists on ModelScope

Args:
    repo_id (`str`):
        A namespace (user or an organization) and a repo name separated
        by a `/`.
    repo_type (`str`, *optional*):
        `None` or `"model"` if getting repository info from a model. Default is `None`.
        TODO: support studio
    endpoint(`str`):
        None or specific endpoint to use, when None, use the default endpoint
        set in HubApi class (self.endpoint)
    re_raise(`bool`):
        raise exception when error
    token (`str`, *optional*): access token to use for checking existence.
Returns:
    True if the repository exists, False otherwise.
zNot support repo-type: %sr   rs   z5Invalid repo_id: %s, must be of format namespace/nameFr   /api/v1/datasets/r   r     Ti  z%Check repo_exists return status code r   zOFailed to check existence of repo: %s, make sure you have access authorization.)ro   r   rV   r  countr   rN   rT   r   r   r   r   r>   r  r   warn)r   r  r  ro   r  r   r   r   r   r   r   codes               r   r  HubApi.repo_exists  sS   8 }}H!y'8@Q'Q7)CDDOc 2a 7SV__``"""N;GD!y'8<M'MZ00@$HDZ~.>avFDLLT%)%9%9$,,%G  I#AvwG3;S[l"KK?vQGHa r   c                     U(       d  U R                   nU[        :X  a  U R                  X5        O+U[        :X  a  U R	                  X5        O[        SU S35      e[        R                  SU S35        g)a  
Delete a repository from ModelScope.

Args:
    repo_id (`str`):
        A namespace (user or an organization) and a repo name separated
        by a `/`.
    repo_type (`str`):
        The type of the repository. Supported types are `model` and `dataset`.
    endpoint(`str`):
        The endpoint to use. If not provided, the default endpoint is `https://www.modelscope.cn`
        Could be set to `https://ai.modelscope.ai` for international version.
r=  z not supported.r  z deleted successfully.N)ro   rT   delete_datasetrU   r	  r  r  info)r   r  r  ro   s       r   delete_repoHubApi.delete_repo  sg     }}H))2/)g0nYKGHHeG9$:;<r   c                 T   [         R                  R                  U [        R                  5      n[
        R                  [        R                  [
        R                  [        R                  0n[        US5       n[        R                  " X#5        S S S 5        g ! , (       d  f       g = f)Nw)r   r   joinr]   CONFIGURATIONrW   	frameworkr\   torchtaskr^   otheropenr   dump)	model_dircfg_filecfgfiles       r   _create_default_configHubApi._create_default_config  si    77<<	9+B+BC""J$4$4u{{
 (C DIIc  !  s   9B
B'zupload modelrV  commit_messager   ignore_file_pattern
lfs_suffixc           
      

   [         R                  " S[        SS9   Uc  [        S5      eUc  [        S5      e[        R
                  R                  U5      (       a$  [        R
                  R                  U5      (       a  [        S5      e[        R
                  R                  U[        R                  5      n[        R
                  R                  U5      (       d?  [        R                  S[        R                   SU S	35        [        R                  U5        [        R!                  5       nUc  [#        S
5      e[        R$                  " U5      n['        U5      nU
c  / n
[)        U
[*        5      (       a  U
/n
Ub  Uc  [        S5      eU R-                  U5      (       d+  [        R/                  SU-  5        U R1                  UUUUU	S9  [        R
                  R                  U[2        5      n[5        5       n[        R/                  SU SU S35        [        R/                  SU S35         [7        UUS9nUR9                  U5      nUU;  a*  [        R/                  SU-  5        UR;                  UU5        UR=                  UU5        [        R$                  " U5      nU H  nUS   S:w  d  M  [        R
                  R                  UU5      n[        R
                  R                  U5      (       a  [        R>                  " U5        Mj  [@        RB                  " USS9  M     U H  nUS   S:w  d  M  [E        U
 Vs/ s H  n[F        RH                  " UU5      S LPM     sn5      (       a  MH  [        R
                  R                  UU5      n[        R
                  RK                  U5      (       a7  [@        RL                  " U[        R
                  R                  UU5      5        M  [@        RN                  " UU5        M     U(       d7  [P        RP                  RS                  5       RU                  S5      nSU< SU< 3nUb4  [)        U[*        5      (       a  U/OUnU H  nURW                  U5        M     URY                  UUUS9  Ub  UR[                  Xw5        [        R/                  SU SU S35         [@        RB                  " USS9  g s  snf ! [\         a    e f = f! [@        RB                  " USS9  f = f)NzThis function is deprecated and will be removed in future versions. Please use git command directly or use HubApi().upload_folder insteadrr   )
stacklevelzmodel_id cannot be empty!zmodel_dir cannot be empty!z$model_dir must be a valid directory.zNo z file found in z, creating a default one.zMust login before upload!z5Visibility and License cannot be empty for new model.zCreating new model [%s])r   r   r   r   r   zPushing folder z
 as model r   zTotal folder size z;, this may take a while depending on actual pushing size...)rV  
clone_fromzCreating new branch %sr   T)ignore_errorsz%Y_%m_%d_%H_%M_%Sz[automsg] push model z to hub at )r\  local_branchremote_branchzSuccessfully push folder z to remote repo [z].)/warningsrD  DeprecationWarningr8   r   r   existsisfilerN  r]   rO  r  r   rm   rZ  r~   r   r:   listdirrK   
isinstancestrr  rI  r   r*   rB   rF   get_remote_branches
new_branchcheckoutremoveshutilrmtreeanyresearchisdircopytreecopydatetimenowstrftimeadd_lfs_typepushtag_and_pushr  )r   r   rV  r   r   r   r\  r   r   r   r]  r^  rW  r   files_to_savefolder_sizetmp_dirgit_wrapperrepobranchesfiles_in_repofsrcpatterndatelfs_suffix_listsuffixs                              r   
push_modelHubApi.push_model  s&    	T		
.	^ "#>??"#?@@ww~~i((BGGNN9,E,E"#IJJ77<<	9+B+BCww~~h''NNi--.oi[Habd)))4"..0?#$?@@

9-.y9&"$)3//#6"7"#Z[[))KK1H<=!%)"3  5 '',,y*?@')oi[
8*AFG(5pqr*	7HED"66w?Hx'4x?@&&w9  (3JJw/M"Q43;'',,w2Cww~~c**		#c> # #Q43;M`aM`'BIIgq1=M`abb '',,y!4Cww}}S))RWW\\'1-EFC1 # "((,,.778KLd"$%2<Z2M2M:,S]-F%%f- .II-%&  ( !!#+KK3I;>OPXzY[\] MM'63 b,  		 MM'6s9   2A?S 5BS :	S #S
&ES S S((S+ +Trs   
   r   page_number	page_sizec           
         [         R                  5       nU(       d  U R                  nU S3nU R                  R	                  USU< SU< SU< S3UU R                  U R                  5      S9n[        U[        XQ5        UR                  [        R                  :X  aW  [        UR                  5       5      (       a  UR                  5       [           nU$ [        UR                  5       [            5      e[#        U5        g)a  List models in owner or group.

Args:
    owner_or_group(str): owner or group.
    page_number(int, optional): The page number, default: 1
    page_size(int, optional): The page size, default: 10
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Raises:
    RequestError: The request error.

Returns:
    dict: {"models": "list of models", "TotalCount": total_number_of_models_in_owner_or_group}
r   z	{"Path":"z", "PageNumber":z, "PageSize": })datar   r   N)r~   r   ro   r   putr   r   r>   r  r  r   r  r?   r   r   r;   r   r@   )	r   r   r  r  ro   r   r   r   r  s	            r   list_modelsHubApi.list_modelsp  s    & #..0}}H?+LL +y:((6  8 	Q@==JMM)QVVXvvx 78"1668,F#GHH!!$r   )r  r  sortrt  ro   r  rt  c                "   U(       d  U R                   nU S3nUUS.nU(       a,  U[        ;  a  [        SU S[        [        5       35      eXHS'   U(       a  XXS'   U(       a  XS'   [        R                  5       n	U R                  U R                  5      n
U R                  R                  UUU	U
S9n[        U5        UR                  5       nUR                  S	5      S
L a  SU;   a  US   $ UR                  S5      =(       d    Sn[        U5      e)a  List datasets via OpenAPI with pagination, filtering and sorting.

Args:
    owner_or_group (str): Search by dataset authors (including organizations and individuals).
    page_number (int, optional): The page number. Defaults to 1.
    page_size (int, optional): The page size. Defaults to 10.
    sort (str, optional): Sort key. If not provided, the server's default sorting is used.
        choose from ['default', 'downloads', 'likes', 'last_modified'].
    search (str, optional): Search by substring keywords in the dataset's Chinese name,
        English name, and authors (including organizations and individuals).
    endpoint (str, optional): Hub endpoint to use. When None, use the endpoint specified in the class.

Returns:
    dict: The OpenAPI data payload, e.g.
          {
            "datasets": [...],
            "total_count": int,
            "page_number": int,
            "page_size": int
          }
z/openapi/v1/datasets)r  r  zInvalid sort key: z. Supported sort keys: r  rt  r,  paramsr   r   successTr  messagezFailed to list datasets)ro   r2   r8   listr~   r   r   r   r   r   r@   r   r;   )r   r   r  r  r  rt  ro   r   r  r   r   r   respmsgs                 r   list_datasetsHubApi.list_datasets  s   < }}H/0 '""
 ?*&(.Ed?F[E\]_ _!6N%8-8"..0&&t||4LL	  
 	a vvx 88I$&6T>< ((9%B)BCs##r   use_cookiesc                     S n[        U[        5      (       a  UnU$ [        U[        5      (       a)  [        R	                  5       nU(       a  Uc  [        S5      eU$ )Nr  )rj  r   boolr~   r   r   )r   r  r   s      r   _check_cookieHubApi._check_cookie  sU    k9--!G
 	 T**&224Gw !LMMr   cutoff_timestampc                 p    U R                  UUUS9nU(       a  U Vs/ s H  oUS   PM	     nnU$ / nU$ s  snf )a  Get model branch and tags.

Args:
    model_id (str): The model id
    cutoff_timestamp (int): Tags created before the cutoff will be included.
                            The timestamp is represented by the seconds elapsed from the epoch time.
    use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                will load cookie from local. Defaults to False.

Returns:
    Tuple[List[str], List[str]]: Return list of branch name and tags
)r   r  r  Revision)list_model_revisions_detail)r   r   r  r  tags_detailsxtagss          r   list_model_revisionsHubApi.list_model_revisions  s\    " 77IYDO 8 Q " (4 |!*|  (* 	s   3c                 Z   U R                  U5      nUc
  [        5       nU(       d  U R                  nU SU S3U-  nU R                  R	                  XeU R                  U R                  5      S9n[        U[        XQ5        UR                  5       n[        U5        U[           n	U	S   S   $ )aE  Get model branch and tags.

Args:
    model_id (str): The model id
    cutoff_timestamp (int): Tags created before the cutoff will be included.
                            The timestamp is represented by the seconds elapsed from the epoch time.
    use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                will load cookie from local. Defaults to False.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Returns:
    Tuple[List[str], List[str]]: Return list of branch name and tags
r   z/revisions?EndTime=%sr  RevisionMapTags)r  rL   ro   r   r   r   r   r>   r  r   rA   r   )
r   r   r  r  ro   r   r   r   r   rI  s
             r   r  "HubApi.list_model_revisions_detail  s    & $$[1#35}}H?8*4IJM]]LLT%)%9%9$,,%G  IQ:FFHq()M"6**r   c                 0    U H  nUS   U:X  d  M  Us  $    g )Nr  r.  )r   detailsr   items       r   get_branch_tag_detailHubApi.get_branch_tag_detail!  s#    DJ4'  r   r   c           	         U(       d  U R                   n[        5       n[        [        [        R                  R                  5       R                  5       5      5      nU R                  Xc  SOUUS9u  pxU(       a  U V	s/ s H  oS   PM	     sn	O/ n
U(       a  U V	s/ s H  oS   PM	     sn	O/ nXV[        -   :  a~  Uc  [        n[        R                  SU-  5        X*;  a  X+;  a  [        SU< SU< S35      eU R                  X5      nUc  U R                  Xr5      n[        R                  SU-  5        U$ Ub  X*;   a  U R                  Xr5      nU$ [        U5      S	:X  a;  Ub
  U[        :X  a  [        nO[        SU< S
U< S35      eU R                  Xr5      nU$ Uc  U(       a  U V	s/ s H  n	U	S   U::  d  M  U	PM     sn	O/ n[        U5      S	:  a  US	   S   nUS	   nOC[        nU R                  Xr5      nSSR!                  U5      -  n[        R#                  SU-  5        [        R#                  SU-  5        U$ X+;  a]  U[        :X  a'  [        R#                  S5        U R                  Xr5      nO=SSR!                  U5      -  n[        SU< S
U< SU< S35      eU R                  X5      n[        R                  SU-  5        U$ s  sn	f s  sn	f s  sn	f )NF)r  ro   r  z9Model revision not specified, using default [%s] version.zThe model: z has no revision : z .z!Development mode use revision: %sr   z has no revision:  !	CreatedAtz[%s],z5Model revision should be specified from revisions: %sz.Model revision not specified, use revision: %sz?Using the master branch is fragile, please use it with caution!z valid are: !z%Use user-specified model revision: %s)ro   rL   introundrx  ry  	timestamp#get_model_branches_and_tags_detailsr(   rR   r  rI  r9   r  debuglenrN  r   )r   r   r   r   ro   release_timestampcurrent_timestampall_branches_detailall_tags_detailr  all_branchesall_tagsrevision_detailrevisions_detailvls                  r   get_valid_revision_detail HubApi.get_valid_revision_detail'  s   
 }}H02h&7&7&;&;&=&G&G&I JK 04/W/W?%RZ 0X 0\,GZ/BC/B!*/BC`b?N?;?ajM?;TV3CCC.O  +0H#hX`$abb"88SO&"&"<"<=P"[LL<xGHN K #(@"&"<"<=P"[&&?#q(#x3F'F2H'QY[c(dee"&"<"<=P"[8 5 # Q` )8(M(7 /0()+:K(K )*(7(Meg % +,q0#3A#6z#B*:1*=#6*.*D*DEX*c#chhx&88'^bd'efNN#SV^#^_   /#'::"NN+lm.2.H.HI\.gO!'#((8*<!<B"/198R1I #J J +/*D*D_*_KK G( RSk D;6(Ms   >KK,K=Kc                 ,    U R                  UUUUS9S   $ )N)r   r   r   ro   r  )r  )r   r   r   r   ro   s        r   get_valid_revisionHubApi.get_valid_revisionl  s6    
 --x7?6=7? . A BLM 	Mr   c                 J   U R                  U5      nU(       d  U R                  nU SU S3nU R                  R                  XTU R	                  U R
                  5      S9n[        U[        XA5        UR                  5       n[        U5        U[           nUS   S   US   S   4$ )a  Get model branch and tags.

Args:
    model_id (str): The model id
    use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                will load cookie from local. Defaults to False.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Returns:
    Tuple[List[str], List[str]]: Return list of branch name and tags
r   z
/revisionsr  r  Branchesr  )r  ro   r   r   r   r   r>   r  r   rA   r   )	r   r   r  ro   r   r   r   r   rI  s	            r   r  *HubApi.get_model_branches_and_tags_detailsv  s    " $$[1}}H?8*J?LLT%)%9%9$,,%G  IQ:FFHq()M":.]0CF0KKKr   c                     U R                  UUS9u  p4U(       a  U Vs/ s H  oUS   PM	     snO/ nU(       a  U Vs/ s H  oUS   PM	     snO/ nXg4$ s  snf s  snf )a9  Get model branch and tags.

Args:
    model_id (str): The model id
    use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                will load cookie from local. Defaults to False.

Returns:
    Tuple[List[str], List[str]]: Return list of branch name and tags
)r   r  r  )r  )r   r   r  branches_detailtags_detailr  r  r  s           r   get_model_branches_and_tags"HubApi.get_model_branches_and_tags  s     (,'O'OYa\g (P (i$ ) ,; ?ajM? .0 	 ! (3 {!*{ &( 	~	s
   AArootr)  r   c                    U(       d  U R                   nU(       a  U< SU< SU< SU< 3nOU< SU< SU< 3nU R                  U5      n	Ub  USU 3-   nUc  U R                  OUn[        [        R
                  " 5       R                  5      US'   U R                  R                  XUS9n
[        U
[        X5        U
R                  5       n[        U5        / nU[           S   (       d  [        R                  S	U S
U S35        U$ U[           S    H(  nUS   S:X  d	  US   S:X  a  M  UR                  U5        M*     U$ )ac  List the models files.

Args:
    model_id (str): The model id
    revision (Optional[str], optional): The branch or tag name.
    root (Optional[str], optional): The root path. Defaults to None.
    recursive (Optional[bool], optional): Is recursive list files. Defaults to False.
    use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                will load cookie from local. Defaults to False.
    headers: request headers
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Returns:
    List[dict]: Model file list.
r   z/repo/files?Revision=&Recursive=z/repo/files?Recursive=z&Root=zX-Request-IDr  FileszNo files found in model z at revision r   r   z
.gitignorez.gitattributes)ro   r  r   rk  uuiduuid4hexr   r   r>   r  r   rA   r   r   append)r   r   r   r  r)  r  r   ro   r   r   r   r   filesrY  s                 r   r0  HubApi.get_model_files  s:   . }}H(Hi9D (I/D$$[1F4&/)D")/$,,w"%djjl&6&6"7LL7  4 	Q:FFHq()'2NN5hZ}XJVWXYL-.w7DF||+tF|?O/OLL	 8
 r   )r   filenamec                    [         R                  5       nU R                  USUUc  SOUS9nU Vs/ s H  ofS   PM	     nnX%;   $ s  snf )a0  Get if the specified file exists

Args:
    repo_id (`str`): The repo id to use
    filename (`str`): The queried filename, if the file exists in a sub folder,
        please pass <sub-folder-name>/<file-name>
    revision (`Optional[str]`): The repo revision
Returns:
    The query result in bool value
TF)r)  r   r  r   )r~   r   r0  )r   r  r  r   r   r  rY  s          r   file_existsHubApi.file_exists  s_    " #..0$$!(g	 % 
 +00%$f%0   1s   Adataset_name	namespacer   c                    Ub  Uc  [        S5      e[        R                  5       nUc  [        S5      eU(       d  U R                  nU S3n	S U4S U4S U4S U4S U4S U4S.n
U R
                  R                  U	U
UU R                  U R                  5      S9n[        XU
5        [        UR                  5       5        U SU SU 3n[        R                  SU 35        U$ )	Nz(dataset_name and namespace are required!r  z/api/v1/datasets)r   r   Ownerr   r6   Description)r  r   r   z
/datasets/r   zCreate dataset success: )r8   r~   r   r   ro   r   r   r   r   r=   rA   r   r  rI  )r   r  r  r   r   r   r   ro   r   r   r  r   dataset_repo_urls                r   create_datasetHubApi.create_dataset  s    9#4"#MNN"..0?HII}}H+,<( ,/I&g, +.
 LL((6	  
 	q.qvvx &Zz)Al^L./?.@ABr   r6  c                 ,   [         R                  5       nU(       d  U R                  nUc  [        S5      eU SU 3nU R                  R                  UUU R                  U R                  5      S9n[        U5        [        UR                  5       5        g )Nr  rA  r  r  )r   r6  ro   r   r   r   s         r   rH  HubApi.delete_dataset#  s    "..0}}H?HII,ZL9LL(/(,(<(<T\\(J   L 	a qvvx r   c                     U(       d  U R                   nU SU SU 3n[        R                  5       nU R                  R	                  XES9nUR                  5       n[        XGU5        US   S   nUS   S   n	X4$ )zGet the dataset id and type. rA  r   r   DataIdType)ro   r~   r   r   r   r   r<   )
r   r  r  ro   datahub_urlr   r   r  r6  dataset_types
             r   get_dataset_id_and_typeHubApi.get_dataset_id_and_type2  s    }}H!
"3I;a~N"..0LL[:vvx{!4&\$'
F|F+''r   Td   	root_pathc	                 r   [         R                  " S[        5        U R                  XUS9u  pU(       a  SOSnU(       d  U R                  nU SU	 S3nU(       a  UOSU(       a  UOSUXgS	.n[
        R                  5       nU R                  R                  XUS
9nUR                  5       n[        XU5        U$ )z/
@deprecated: Use `get_dataset_files` instead.
zMThe function `list_repo_tree` is deprecated, use `get_dataset_files` instead.r  r  ro   TrueFalserA  
/repo/treer   r   r  Root	Recursive
PageNumberPageSizer  r   )re  rD  rf  r  ro   r~   r   r   r   r   r<   )r   r  r  r   r  r)  r  r  ro   dataset_hub_idr  r  r  r   r   r  s                   r   list_repo_treeHubApi.list_repo_tree?  s     	e(	* (,'C'C%X (D (O$ (FW	}}H!
"3N3C:N*2h'0)c	 +D #..0LL[Ivvx{!4r   2   )r  r   r  r  ro   c          	         SSK Jn  U" U5      (       a)  UR                  S5      S:X  a  UR                  S5      u  pO[	        SU S35      eU(       d  U R
                  nU(       a  U SU SU S	3OU S
U S	3n
U=(       d    [        UUS.n[        R                  5       n U R                  R                  XXR                  U R                  5      S9n[        U5        UR                  5       n[        U5        UR                  S5      [         R"                  :X  a  [$        R&                  " U5      $ g! [(        R*                  R,                   a  n[/        SU S[1        U5       35      eSnAff = f)a  
Get the commit history for a repository.

Args:
    repo_id (str): The repository id, in the format of `namespace/repo_name`.
    repo_type (Optional[str]): The type of the repository. Supported types are `model` and `dataset`.
    revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
    page_number (int): The page number for pagination. Defaults to 1.
    page_size (int): The number of commits per page. Defaults to 50.
    endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.

Returns:
    CommitHistoryResponse: The commit history response.

Examples:
    >>> from modelscope.hub.api import HubApi
    >>> api = HubApi()
    >>> commit_history = api.list_repo_commits('meituan/Meeseeks')
    >>> print(f"Total commits: {commit_history.total_count}")
    >>> for commit in commit_history.commits:
    ...     print(f"{commit.short_id}: {commit.title}")
r   is_relative_pathr   rs   Invalid repo_id: r  z/api/v1/s/z/commitsr   )r   r  r  r  Codez%Failed to get repository commits for : N)datasets.utils.file_utilsr  rC  r   r   ro   rQ   r~   r   r   r   r   r   r@   r   rA   r   r  rf   from_api_responserequests
exceptionsRequestExceptionr  rk  )r   r  r  r   r  r  ro   r  _owner_dataset_namecommits_urlr  r   r   r  es                   r   r/  HubApi.list_repo_commits`  sc   < 	?G$$s);q)@$+MM#$6!FM0	<==}}HMV
(9+RyIjy9 	 ::%!

 #..0	Y  )0:N:Nt||:\ ! ^A!!$668D4 xx:==0,>>tDD 1 ""33 	YCG9BsSTvhWXX	Ys   !BD4 4E1E,,E1r   )r   r  r)  r  r  ro   c                   SSK Jn  U" U5      (       a)  UR                  S5      S:X  a  UR                  S5      u  pO[	        SU S35      eU R                  XUS9u  pU(       d  U R                  nU SU S	3nUUU(       a  S
OSUUS.n[        R                  5       nU R                  R                  XUS9nUR                  5       n[        UUU5        US   S   $ )a  
Get the dataset files.

Args:
    repo_id (str): The repository id, in the format of `namespace/dataset_name`.
    revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
    root_path (str): The root path to list. Defaults to '/'.
    recursive (bool): Whether to list recursively. Defaults to True.
    page_number (int): The page number for pagination. Defaults to 1.
    page_size (int): The number of items per page. Defaults to 100.
    endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.

Returns:
    List: The response containing the dataset repository tree information.
        e.g. [{'CommitId': None, 'CommitMessage': '...', 'Size': 0, 'Type': 'tree'}, ...]
r   r  r   rs   r  r  r  rA  r   r  r  r  r  r  r  )r  r  rC  r   r   r  ro   r~   r   r   r   r   r<   )r   r  r   r  r)  r  r  ro   r  r  r  r  r  r  r  r   r   r  s                     r   r8  HubApi.get_dataset_files  s    2 	?G$$s);q)@$+MM#$6!FM0	<=='+'C'C&8 (D (M$ }}H!
"3N3C:N #,'%!
 #..0LL[Ivvx{D!4F|G$$r   c                 H   [         R                  5       nU(       d  U R                  nU(       a  U SU SU 3nOU SU 3nU R                  R	                  XTU R                  U R                  5      S9n[        U5        UR                  5       n[        XWU5        U[           $ )a#  
Get the dataset information.

Args:
    dataset_id (str): The dataset id.
    revision (Optional[str]): The revision of the dataset.
    endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.

Returns:
    dict: The dataset information.
rA  r  r  )r~   r   ro   r   r   r   r   r@   r   r<   r   )r   r6  r   ro   r   r   r   r  s           r   r7  HubApi.get_dataset  s    " #..0}}HZ0JxjQDZ0=DLL4+?+?+M  Oa vvxt1-+,,r   c           	      L   U(       d  U R                   nU SU SU 3n[        R                  5       nU R                  R	                  UUU R                  U R                  5      S9nUR                  5       n	[        XiU5        U	S   n
U
c  [        SU SU SU S35      eU
S	   n
U
$ )
z'Get the meta file-list of the dataset. rA  z/repo/tree?Revision=r  r  z'The modelscope dataset [dataset_name = z, namespace = z, version = z] dose not existr  )
ro   r~   r   r   r   r   r   r   r<   r9   )r   r  r  r6  r   ro   r  r   r   r  	file_lists              r   get_dataset_meta_file_list!HubApi.get_dataset_meta_file_list  s     }}H!
"3J<?ST\S]^"..0LL[%,%)%9%9$,,%G  I vvx{!4L	9,~V_U` a%J&689 9 g&	r   r  meta_cache_dirc                     [         R                  R                  U[        U 5       [        R
                  R                   35      n[        US5       nUR                  S5        SSS5        g! , (       d  f       g= f)z
Dump the data_type as a local file, in order to get the dataset
 formation without calling the datahub.
More details, please refer to the class
`modelscope.utils.constant.DatasetFormations`.
rM  z3*** Automatically-generated file, do not modify ***N)	r   r   rN  rk  rX   formation_mark_extvaluerT  write)r  r$  dataset_type_file_pathfps       r   dump_datatype_fileHubApi.dump_datatype_file	  s`     "$n14\1B0CDUDhDhDnDnCo.p"r(#."HHJK /..s   A--
A;r!  c           
      ,   [        [        5      n[        U5      n	[        U	   n
[        R                  5       n[        R                  XTS9  U(       d  U R                  nU GH  nUS   n[        R                  R                  U5      S   nX;   d  M2  U SU SU SU SU 3	nU R                  R                  XS9n[        U5        [        R                  R                  XM5      n[        R                  R!                  U5      (       a1  ["        R%                  S	U S
U S35        X   R'                  U5        M  [)        US5       nUR+                  UR,                  5        S S S 5        X   R'                  U5        GM      X4$ ! , (       d  f       N)= f)N)r  r$  r   r   rA  r   /repo?Revision=
&FilePath=r  zReusing dataset z's python file ()wb)r   r  rX   rY   r~   r   rm   r+  ro   r   r   splitextr   r   r@   rN  rg  r  r   r  rT  r(  content)r   r  r  r   r$  r  r!  ro   local_pathsdataset_formationdataset_meta_formatr   	file_info	file_path	extensionr  r   
local_pathr  s                      r   "get_dataset_meta_files_local_paths)HubApi.get_dataset_meta_files_local_paths  sp   
 "$'-l;01BC"..0 	!!|![}}H"I!&)I((3B7I/!)
*;I;a~ V**2:i[JLL$$[$B%a(WW\\.D
77>>*--NN*<.8HTUV  *11*=*d+qGGAII& ,&--j9# #& --	 ,+s   F
F	   c           	        ^ SSK nSSKJn  SSKn[        R
                  R                  XR                  U R                  SS95      R                  5       5      nU[        R                  :X  a:  [        R
                  R                  U5      (       a  [        R                  " U5        [        R
                  R                  U5      (       a  [        R                  SU 35        U$ [         R#                  5       n[        R                  S5        [$        R&                  " XSS	9n[)        UR*                  R'                  S
S5      5      n	U" U	SS9n
U4S jnSn[-        US5       nU" U5       H  nU
R/                  [1        U5      5        U R3                  S5      (       a  U Vs/ s H/  oR5                  5       (       d  M  [6        R8                  " U5      PM1     nn[1        U5      S:X  a  M  US:X  a  SnOSnUR;                  U5      nUR=                  USUSS9  US-  nM  U H  nUR?                  US-   5        M     M     SSS5        U
RA                  5         U$ s  snf ! , (       d  f       N%= f)z?
Fetch the meta-data files from the url, e.g. csv/jsonl files.
r   Nr   zUTF-8encodingzReusing cached meta-data file: zLoading meta-data file ...T)r   streamzcontent-length)ru   dynamic_ncolsc              3      >#    / nU R                  5        H<  nUR                  S5      nUR                  U5        [        U5      T:  d  M6  Uv   / nM>     Uv   g 7f)Nutf-8)
iter_linesdecoder  r  )r  
chunk_datar  
chunk_sizes      r   	get_chunk3HubApi.fetch_meta_files_from_url.<locals>.get_chunkP  sW     J){{7+!!$'z?j0$$!#J * s   AAAajsonlF\)indexheader
escapecharrs   
)!hashlib	tqdm.autor   pandasr   r   rN  md5encode	hexdigestr[   FORCE_REDOWNLOADrg  ro  r  rI  r~   r   r  r   r  r   rT  r   r  r   r  r   loads	DataFrameto_csvr(  close)urlout_pathrH  moderR  r   pdr   response
total_sizeprogressrI  iter_numr  chunklinewith_headerchunk_dfs     `               r   fetch_meta_files_from_url HubApi.fetch_meta_files_from_url9  s   
 	"77<<++cjj'j6R*S*]*]*_`<000RWW^^H5M5MIIh77>>(##KK9(DEO"..0 	01<<TB))--.>BC
j=	 (C A"8,E
+<<((:?P%$::<-TZZ-%EP5zQ 1}&*&+!||E2HOOAU;SWOXMH !&t, !& - !$ 	! Q	 ! s&   #A I!#I?IA)I!I!!
I/	file_nameviewextension_filterc                     U(       a  U(       a  U(       d  [        S5      eSXAUS.n[        U5      nU(       d  U R                  nU SU SU SU 3n	U	$ )Nz:Args (file_name, dataset_name, namespace) cannot be empty!SDK)Sourcer  FilePathViewrA  r   /repo?)r   r   ro   )
r   rk  r  r  r   rl  rm  ro   r  file_urls
             r   get_dataset_file_urlHubApi.get_dataset_file_urlq  s`     IYZZ #(X^bc'}}HZ01\N&QWPXYr   c           	          U(       d  U R                   nU(       a:  [        R                  R                  U5      S   [        ;   a  U SU SU SU SU 3	nU$ )Nr   rA  r   r.  r/  )ro   r   r   r2  rS   )r   rk  r  r  r   ro   s         r   get_dataset_file_url_origin"HubApi.get_dataset_file_url_origin  s`     }}H)))4R8<MM#*$5i[, P$$,:Z	{DIr   c                 d    U(       d  U R                   nU SU SU SU 3nU R                  U5      $ )NrA  r   /ststoken?Revision=)ro   datahub_remote_call)r   r  r  r   ro   r  s         r   get_dataset_access_config HubApi.get_dataset_access_config  sE     }}H!
"3I;a~ N++3*6''44r   check_cookiec                 @   U(       d  U R                   nU SU SU SU 3nU(       a  U R                  SS9nO[        R                  5       nU R                  R                  UUU R                  U R                  5      S9nUR                  5       n	[        U	5        U	S   $ )NrA  r   r{  T)r  r]  r   r   r  )
ro   r  r~   r   r   r   r   r   r   rA   )
r   r  r  r  r   ro   r  r   r   r  s
             r   !get_dataset_access_config_session(HubApi.get_dataset_access_config_session  s     }}H!
"3I;a~ N++3*6((T(:G&224GLL((6  8 vvxtF|r   versionc                    [         R                  R                  [        R                  S5      nU(       d  [        S[        R                   35      eU S3n[        R                  R                  [        R                  5       5      n[        UUS9n[        US9nU R                  R                  UUUU R                  U R                  5      SS9nUR!                  5       n	U	S   S	:w  a  [        S
U	 35      eU	S   $ )z
Get virgo dataset meta info.
r   z"Virgo endpoint is not set in env: z/data/set/download)	dataSetIddataSetVersion)r  i  )r]  r   r   r   r}   rE  r   zFailed to get virgo dataset: r  )r   r   r   r_   env_virgo_endpointRuntimeErrorr  utilsdict_from_cookiejarr~   r   dictr   r   r   r   r   )
r   r6  r  virgo_endpointvirgo_dataset_urlr   r:  r  r   r  s
             r   get_virgo_metaHubApi.get_virgo_meta  s     (:(M(MrR!CDVDiDiCjkll-..@A..445E5Q5Q5ST "
 
 LL"3#'&-&*&:&:4<<&H&)	  +
 vvx<1!>tfEFFF|r   zip_file_namec                 ,   U(       d  U R                   nU SU SU 3n[        R                  5       nU R                  R	                  XgU R                  U R                  5      S9nUR                  5       n	[        U	5        U	S   n
[        R                  " U
S   5      nU SU 3nU R                  R	                  XU R                  U R                  5      S9nUR                  5       n[        U5        US   nUS-   S-   U-   S-   U-   S-   U-   nUUS	'   U$ )
NrA  r   r  r  r6   r{  z	-unzippedr1  Dir)
ro   r~   r   r   r   r   r   r   rA   r7   )r   r  r  r   r  ro   r  r   r   r  r  r   datahub_sts_urlr_stsresp_stsdata_stsfile_dirs                    r   &get_dataset_access_config_for_unzipped-HubApi.get_dataset_access_config_for_unzipped  s(    }}H!
"3I;a~N"..0LL%)%9%9$,,%G  IvvxtF|"&&tL'9:
(M)<XJG  _)-)=)=dll)K ! M::<x F#+c1I=ClRUXX[hh"r   c                     U(       d  U R                   nU SU SU SU SU SU SU 3n[        R                  5       n	U R                  R	                  XSS9n
U
R                  5       n
[        U
5        U
S	   n
U
$ )
NrA  r   z/oss/tree/?MaxLimit=
&Revision=r  z&FilterDir=i  )r]  r   r}   r  )ro   r~   r   r   r   r   rA   )r   r  r  	max_limitis_recursiveis_filter_dirr   ro   r]  r   r  s              r   list_oss_dataset_objectsHubApi.list_oss_dataset_objects  s    }}H
+I;a~ F#JxjL>Q\]j\km #..0||C$Gyy{tF|r   object_namec           	      4   U(       a  U(       a  U(       a  U(       d  [        S5      eU(       d  U R                  nU SU SU SU SU 3	n[        R                  5       nU R                  R                  XgS9nUR                  5       n[        U5        US   nU$ )NArgs cannot be empty!rA  r   z
/oss?Path=r  r]  r   Messager   ro   r~   r   r   r  r   rA   	r   r  r  r  r   ro   r]  r   r  s	            r   delete_oss_dataset_object HubApi.delete_oss_dataset_object  s    ,ix455}}H
+I;a~ZP[}\fgofpq"..0||""s"<yy{tIr   c           	      4   U(       a  U(       a  U(       a  U(       d  [        S5      eU(       d  U R                  nU SU SU SU SU 3	n[        R                  5       nU R                  R                  XgS9nUR                  5       n[        U5        US   nU$ )Nr  rA  r   z/oss/prefix?Prefix=z/&Revision=r  r  r  r  s	            r   delete_oss_dataset_dirHubApi.delete_oss_dataset_dir  s    ,ix455}}H
+I;a~EXYdXe f#*& #..0||""s"<yy{tIr   c                     [         R                  5       nU R                  R                  UUS[         R	                  5       0S9nUR                  5       n[        XU5        US   $ )Nrq   r  r  )r~   r   r   r   r   r   r<   )r   r]  r   r   r  s        r   r|  HubApi.datahub_remote_call*  sc    "..0LL!#3#B#B#DE  G vvxs!,F|r   use_streamingc           	      h   [         R                  " S5      S:H  nU(       d  U R                  nU(       GaR  U(       GaI  U(       Gd@  U(       Gd7   [        R	                  5       nU SU SU S3nU R
                  R                  XvU R                  U R                  5      S9n[        U5        [        R                  R                  n	Sn
[        [         R                  ;   a  [         R                  [           n	[        [         R                  ;   a  [         R                  [           n
U SU SU SU	 S	U
 3	nU R
                  R                  XU R                  U R                  5      S9nUR!                  5       n[#        U5        g g g g g ! [$         a  n[&        R)                  U5         S nAg S nAff = f)
NCI_TESTr  rA  r   z/download/increaser  r   z/download/uv/z?user=)r   getenvro   r~   r   r   r   r   r   r@   rZ   LOCALr'  r!   r   r"   r   rA   r  r  r  )r   r  r  r  ro   
is_ci_testr   download_count_urldownload_count_respchannel	user_namedownload_uv_urldownload_uv_respr  s                 r   dataset_download_statistics"HubApi.dataset_download_statistics4  s   YYy)V3
}}HIj *668 )1z1B9+Q|n\n%o"&*ll&7&78J@D@T@TUYUaUa@b '8 'd#%&9: *//55	/2::= jj)EFG,

: "

+D EI%-J.?	{!L>Yf%,IVI;#@ $(<<#4#4_=A=Q=QRVR^R^=_ $5 $a #3#8#8#: /0/ BOjI<2   Q s   D5F 
F1F,,F1c                 b    [         [        [        R                  " 5       R                  5      0UE$ N)r&   rk  r  r  r  )r   r   s     r   r   HubApi.builder_headersU  s*    %s4::<+;+;'<  	r   c                 d    UR                  S5      u  p4U(       d  U R                  nU SU SU S3$ )Nr   rA  rs  )r   ro   )r   r  ro   
_namespacer  s        r   get_file_base_pathHubApi.get_file_base_pathY  s9    $+MM#$6!
}}H,ZL-OOr   )	r   r   r  r   r   ro   exist_okcreate_default_configr   r  r  c       	         v   U(       d  [        S5      eU(       d  U R                  nU R                  X'S9  U R                  XXrS9nU(       a=  U(       a'  U SU SU 3n[        R                  SU SU 35        U$ [        SU S35      eUR                  S5      n[        U5      S	:w  a  [        S
5      eUu  nnU[        :X  Ga  [        R                  R                  5        VVs0 s H!  u  nnUR                  S5      (       a  M  UU_M#     nnnUR                  UR                  5       5      nUc  [        SU S35      eU R                  UUUUU
S9nU	(       at  [         R"                  " 5        nSSKJn  U" UU5      nSSSS.nUR                  S5      nU(       d  0 n0 UEUEn[)        US[*        R,                  " U5      /SS9  SSS5        [/        SU S3SS9  U$ U[0        :X  a  [2        R                  R                  5        VVs0 s H!  u  nnUR                  S5      (       a  M  UU_M#     nnnUR                  UR                  5       5      nUc  [        SU S35      eU R5                  UUUUUS9n[/        SU S3SS9  U$ [        SU S[6         35      es  snnf ! , (       d  f       N= fs  snnf ) a  
Create a repository on the ModelScope Hub.

Args:
    repo_id (str): The repo id in the format of `owner_name/repo_name`.
    token (Union[str, bool, None]): The access token.
    visibility (Optional[str]): The visibility of the repo,
        could be `public`, `private`, `internal`, default to `public`.
    repo_type (Optional[str]): The repo type, default to `model`.
    chinese_name (Optional[str]): The Chinese name of the repo.
    license (Optional[str]): The license of the repo, default to `apache-2.0`.
    endpoint (Optional[str]): The endpoint to use.
        In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai'
    exist_ok (Optional[bool]): If the repo exists, whether to return the repo url directly.
    create_default_config (Optional[bool]): If True, create a default configuration file in the model repo.
    **kwargs: The additional arguments.

Returns:
    str: The repo url.
Repo id cannot be empty!)r   ro   )r  ro   r   r   r  r  z already exists, got repo url: z already exists!rr   zBInvalid repo id, should be in the format of `owner_name/repo_name`__NInvalid visibility: z9, supported visibilities: `public`, `private`, `internal`)r   r   r   r   r   r   rE   pytorchztext-generationT)rP  rR  allow_remoteconfig_jsonzconfiguration.json)ignore_push_errorz"New model created successfully at r   flush)r  r  r   r   r   z$New dataset created successfully at Invalid repo type: , supported repos: )r   ro   r   r  r  r   r   r  rU   r5   __dict__items
startswithr   upperr   tempfileTemporaryDirectorymodelscope.hub.repositoryrF   rH   r   dumpsprintrT   r3   r  rV   )r   r  r   r   r  r   r   ro   r  r  r   kwargsr  repo_urlrepo_id_listr  	repo_namekvvisibilitiestemp_cache_dirrF   r  default_configr  configs                             r   create_repoHubApi.create_repo`  s
   H 788}}H


9 ,,WT\,j#+*Ai[7) Dwi/NxjYZ 5	1A!BCC}}S)|!abb+	9'-<-E-E-K-K-Mh-MTQUVUaUabfUgDAqD-MLh*..z/?/?/ABJ! #7
| D[ "\ ] ] -- %)% . H %002nD%ng>D%. 1(,&N
 #)**]";K&&(>>+>F',tzz&/A.B*.0 3  6xjB$O( % ++->-G-G-M-M-Oj-OTQWXWcWcdhWiDAqD-OLj*..z/?/?/ABJ! #7
| D[ "\ ] ] //&#)% 0 H 8
!DDQ
  29+=PQbPcdee_ i 32& ks%   %JJ0AJ$
J5+J5$
J2      )commit_descriptionr   r  r   ro   r{   r}   
operationsr  r{   r}   c          
         U(       d  [        S5      eU(       d  U R                  nU[        ;  a  [        SU S[         35      eU SU SU SU 3nU=(       d    SU 3nU=(       d    SnU R                  US	S
9nU R	                  UUS9nSn[        U	5       GHe  n US:  a  [        R                  SUS-    SU S35        [        R                  " UU R                  U R                  5      [        R                  " U5      UU
S9nUR                  S:w  am   UR                  5       nSUR                   SU SU 3nSUR                  s=::  a  S:  a"  O  O[        R#                  SUS-    SU 35        Of[        SU 35      eUR                  5       nUR%                  S0 5      R%                  SS5      n[        R                  SU 35        ['        UUUUS9s  $ XS-
  :  d  GMO  [2        R4                  " S5        GMh     [        R(                  R+                  S!U	 S"U 35      e! [        R                   a    UR                   n GN"f = f! [        R(                  R*                   a3  nUn[        R#                  SUS-    S[-        U5       35         SnANSnAf[.         a=  nUn[        R1                  S US-    S[-        U5       35        XS-
  :X  a  e  SnAGNSnAff = f)#a*  
Create a commit on the ModelScope Hub with retry mechanism.

Args:
    repo_id (str): The repo id in the format of `owner_name/repo_name`.
    operations (Iterable[CommitOperation]): The commit operations.
    commit_message (str): The commit message.
    commit_description (Optional[str]): The commit description.
    token (str): The access token. If None, will use the cookies from the local cache.
        See `https://modelscope.cn/my/myaccesstoken` to get your token.
    repo_type (Optional[str]): The repo type, should be `model` or `dataset`. Defaults to `model`.
    revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
    endpoint (Optional[str]): The endpoint to use.
        In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai'
    max_retries (int): Number of max retry attempts (default: 3).
    timeout (int): Timeout for each request in seconds (default: 180).

Returns:
    CommitInfo: The commit info.

Raises:
    requests.exceptions.RequestException: If all retry attempts fail.
r  r  r  /api/v1/repos/r  z/commit/z
Commit to r   Tr   )r  r\  Nr   zAttempt rs   z to create commit for z...)r   r  r   r}   rB  zHTTP z error from r  rt   iX  zServer error on attempt zClient request failed: r  oidzCommit succeeded: )
commit_urlr\  r  r  zRequest failed on attempt zUnexpected error on attempt zFailed to create commit after z attempts. Last error: )r   ro   rV   r   _prepare_commit_payloadranger  rI  r  r   r   r   r   r  r  JSONDecodeErrortextr   r   rg   r  r  rk  r  r  timesleep)r   r  r  r\  r  r   r  r   ro   r{   r}   r]  r   payloadlast_exceptionattemptra  error_detail	error_msgr  r  r  s                         r   create_commitHubApi.create_commit  s   J 788}}H--29+=PQbPcdee
.2gYhxjQ'AZy+A/52"""M ..!) / 
 [)G1Q;KK(7Q;-7MgYVY Z[#== 00>G,## ''3.5'/}}
   4 45\#b'.*  h228S86w{m2i[Q )+B9+)NOO#==?D((62.225"=CKK"4SE :;%#&'5+=	 " q(

1k *p !!22,[M9PQ_P`a
 	
S  // 5'/}}56 &&77 U!"!;GaK=3q6(STT !";GaK=3q6(STAo- .sQ   A:IH'(AI6A#I' II
IIK$,)JK$'2KK$z[Uploading])r   r  r\  r  buffer_size_mb	tqdm_descdisable_tqdmr   path_or_fileobjpath_in_repor  r  r  c          
         U[         ;  a  [        SU S[          35      eU(       d  [        S5      eU R                  USS9  [        U[        [
        45      (       ae  [        R                  R                  [        R                  R                  U5      5      nU=(       d    [        R                  R                  U5      nOU(       d  [        S5      e[        U[        R                  5      (       a  UR                  5       nU R                  R                  U5        U R                  R!                  U/US9  Ub  UOSU S	3nUS
::  a  [        S5      e[#        UUS9nUS   nUS   nU R%                  UUUU R&                  SSS9  U R)                  UUUUUU
U	S9n[+        UUUS9nU R                  R-                  X5      (       a  SOSUl        US   Ul        U/n[3        SU S3SS9  U R5                  UUUUUUUS9nU$ )a  
Upload a file to the ModelScope Hub.

Args:
    path_or_fileobj (Union[str, Path, bytes, BinaryIO]):
        The local file path or file-like object (BinaryIO) or bytes to upload.
    path_in_repo (str): The path in the repo to upload to.
    repo_id (str): The repo id in the format of `owner_name/repo_name`.
    token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
        See `https://modelscope.cn/my/myaccesstoken` to get your token.
    repo_type (Optional[str]): The repo type, default to `model`.
    commit_message (Optional[str]): The commit message.
    commit_description (Optional[str]): The commit description.
    buffer_size_mb (Optional[int]): The buffer size in MB for reading the file. Default to 1MB.
    tqdm_desc (Optional[str]): The description for the tqdm progress bar. Default to '[Uploading]'.
    disable_tqdm (Optional[bool]): Whether to disable the tqdm progress bar. Default to False.
    revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.

Returns:
    CommitInfo: The commit info.

Examples:
    >>> from modelscope.hub.api import HubApi
    >>> api = HubApi()
    >>> commit_info = api.upload_file(
    ...     path_or_fileobj='/path/to/your/file.txt',
    ...     path_in_repo='optional/path/in/repo/file.txt',
    ...     repo_id='your-namespace/your-repo-name',
    ...     commit_message='Upload file.txt to ModelScope hub'
    ... )
    >>> print(commit_info)
r  r  z$Path or file object cannot be empty!Tr   z#Arg `path_in_repo` cannot be empty!file_path_listr  zUpload z to ModelScope hubr   z4Buffer size: `buffer_size_mb` must be greater than 0)file_path_or_objr  	file_size	file_hashFr  r   r  ro   r  r  r  r  sha256sizer  r  r  r  r   file_hash_infolfsnormalis_uploadedzCommitting file to z ...r  r  r  r\  r  r   r  r   )rV   r   r   rj  rk  r   r   r   abspathr   basenameioBufferedIOBaserv   r   
check_filecheck_normal_filesr`   r  ro   _upload_blobri   is_lfs_upload_mode_is_uploadedr  r  )r   r   r  r  r   r  r\  r  r  r  r  r   hash_info_dr  r  
upload_resadd_operationr  commit_infos                      r   upload_fileHubApi.upload_fileE  s@   ` --29+=PQbPcdeeCDD 	edCoT{33 ggoobgg.@.@.QRO'L277+;+;O+LL   !FGG or'8'899-224O&&7..+, 	/ 	
 -8N~Ug>h 	 QSTT),)
 %[1	$[1	$#,"&--"&/4 	 	6  ,, % - 

 -?%+&-

 /3.A.A.H.H.d.dUjr"%/%>"#_
#G9D1>"&"4"4!)1 #5 #
 r   )	r  r\  r  r   r  allow_patternsignore_patternsmax_workersr   folder_pathr"  r#  r$  c                  ^ ^^ T(       d  [        S5      eUc  [        S5      eT[        ;  a  [        ST S[         35      eT R                  USS9  U(       a  UOSnU	(       a  U	OSn	U	c  / n	O[        U	[        5      (       a  U	/n	U	[
        -  n	Ub!  SU;   a  / n	U	 Vs/ s H  oU;  d  M
  UPM     n	nUb  UOS	T S
3nU=(       d    Sn[        R                  S5        T R                  UUUU	S9n[        U5      S:X  a  [        SU S35      e[        R                  S[        U5       S35        T R                  R                  U VVs/ s H  u  pUPM	     snnTS9  T R                  TUTT R                  SSS9  [        U
SS9UUU 4S j5       nU" UTUTUUSSS9n/ nU Hd  nUS   nUS   nUS   nUS   n[        UUUS9nT R                  R!                  UT5      (       a  SOS Ul        UUl        UR'                  U5        Mf     [        U5      S:X  a  [        SU S35      e[(        S:  a  [(        O
[        U5      n[        U5      S-
  U-  S-   n[+        S![        U5       S"U S#U S$3SS%9  / n[-        [/        U5      S&US'9 HE  nUUU-  US-   U-   nU S(US-    S)U S*3nT R1                  TUUUUTUS+9nUR'                  U5        MG     [        U5      S:X  a  US   $ U$ s  snf s  snnf ),a/  
Upload a folder to the ModelScope Hub.

Args:
    repo_id (str): The repo id in the format of `owner_name/repo_name`.
    folder_path (Union[str, Path, List[str], List[Path]]): The folder path or list of file paths to upload.
    path_in_repo (Optional[str]): The path in the repo to upload to.
    commit_message (Optional[str]): The commit message.
    commit_description (Optional[str]): The commit description.
    token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
        See `https://modelscope.cn/my/myaccesstoken` to get your token.
    repo_type (Optional[str]): The repo type, default to `model`.
    allow_patterns (Optional[Union[List[str], str]]): The patterns to allow.
    ignore_patterns (Optional[Union[List[str], str]]): The patterns to ignore.
    max_workers (int): The maximum number of workers to use for uploading files concurrently.
        Defaults to `DEFAULT_MAX_WORKERS`.
    revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.

Returns:
    Union[CommitInfo, List[CommitInfo]]:
        The commit info or list of commit infos if multiple batches are committed.

Examples:
    >>> from modelscope.hub.api import HubApi
    >>> api = HubApi()
    >>> commit_info = api.upload_folder(
    ...     repo_id='your-namespace/your-repo-name',
    ...     folder_path='/path/to/your/folder',
    ...     path_in_repo='optional/path/in/repo',
    ...     commit_message='Upload my folder',
    ...     token='your-access-token'
    ... )
    >>> print(commit_info.commit_url)
"The arg `repo_id` cannot be empty!Nz%The arg `folder_path` cannot be None!r  r  Tr   z**z
Upload to z on ModelScope hubzUploading fileszPreparing files to upload ...)folder_path_or_filesr  r"  r#  r   z"No files to upload in the folder: r  z	Checking z files to upload ...r  Fr  )r$  r  c           
         > U u  p#[        US9nUS   nUS   nT
R                  TT	UUUU[        :*  SU-   S-   S9nUUUS   US.$ )	N)r  r  r  z[Uploading ]r	  r  )file_path_in_repor8  r  r  )r`   r  r+   )	item_pairr  r+  r8  r  r  r  r  r  r  r   s           r   _upload_items+HubApi.upload_folder.<locals>._upload_items3	  s    +4( -!*!K )5I(5I#00# &*LL'*;;cA  1  J &7&)-8"-	 r   rs   )r  r   r  r\  r  r  r  r+  r8  r  r  r  r  r  zCommitting z
 files in z batch(es) of size r   r  z[Committing batches] )descru   z (batch r   r0  r  )r   rV   r   rj  rk  rd   r  rI  _prepare_upload_folderr  r   r  r  ro   rk   ri   r  r  r  r  r,   r  r   r  r  ) r   r  r%  r  r\  r  r   r  r"  r#  r$  r   pprepared_repo_objectsr1  r  r-  uploaded_items_listr  item_dprepared_path_in_repoprepared_file_pathr  r  optcommit_batch_sizenum_batchescommit_infosibatch_operationsbatch_commit_messager  s    ``     `                        r   upload_folderHubApi.upload_folder  s   b ABBDEE--29+=PQbPcdee 	edC+9t-</$ " O--./O22 %~%"$**a~.E?  
 -8N
7)Se>f 	 0D3D 	34 $ ; ;!,%)+	 !< !
 $%*A+bQRRi$9 :;;OPQ..0EF0EWQD0EF 	/ 	

 	$#,"&--"&/4 	 	6 
[u	E	 
F	4 ,!)1	
 
)F)/0C)D!&,[&9 &} 5K#)*:#;N$2 2-C )-(;(;(B(BCUW`(a(augoC*Cc" *  z?aA+bQRR >VXY=Y!9_bcm_n:*/@@1DC
O,J{mCVWhViijk	)+eK(/FkZA)!.?*?!a%K\A\]&4%5Xa!eWAk]RS#T &*&8&8+3#5#! '9 'K , [ #&l"3q"8|AJlJe, Gs   	K+,K+K0
)r  r  r  r
  r  r  c          
      r   [        S SS S S9n	X4S./n
U R                  UUU
S9n[        U5      S:X  a  US   OS nUc#  [        R	                  SUS S  S	35        S
U	S'   U	$ [
        R                  5       nU(       a  [        U5      OS nUc  [        S5      eU R                  R                  SSUS    305        U R                  U R                  5      nUS-  S-  4S jn[        USS
UUS9 n[        U[        [        45      (       a6  [        US5       n[         R"                  " US   UU" UU5      S9nS S S 5        O[        U[$        5      (       a5  [         R"                  " US   UU" [&        R(                  " U5      U5      S9nOK[        U[&        R*                  5      (       a!  [         R"                  " US   UU" UU5      S9nO[        S5      eS S S 5        [-        WS9  UR/                  5       n[1        US9  US   U	S'   US   U	S'   US   U	S'   U	$ ! , (       d  f       NR= f! , (       d  f       N[= f)NF)r]  r  r  
status_msg)r  r  )r  r  objectsrs   r   zBlob    z  has already uploaded, reuse it.Tr  r  Cookiezm_session_id=r   r=  c              3   ~   #     U R                  U5      nU(       d  gUR                  [        U5      5        Uv   M9  7f)z8Lazy function (generator) to read a file piece by piece.N)rv   r   r  )file_objectpbarrH  cks       r   read_in_chunks+HubApi._upload_blob.<locals>.read_in_chunks	  s8      %%j1CG$ s   ;=B)ru   unit
unit_scaler/  disablerbr]  )r   r  zInvalid data type to uploadrspr  r  r  rA  )r  _validate_blobr  r  r  r~   r   r   r   r   r   r   rj  rk  r   rT  r  r  bytesr  BytesIOr  r@   r   rA   )r   r  r  r
  r  r  r  r  r  res_drB  upload_objectsupload_objectr   r   rI  rG  r  ra  r  s                       r   r  HubApi._upload_blob	  sU    	
 "01,, - 
 .1-@A-Eq)4 LL5,LMN#'E- L"..0#*$w-?HIIXw~7N6O'PQR&&t||49G$9NQU9U 	 $
 $d,,$%'||%e, '+At4 H &% D%((#<<!%(#'

4(8$? D""3"344#<<!%(#'d3 !!>???
B 	(+}}4 $U+e#F|m"9olA &%
 
s%   ,(H(!H5BH(
H%	!H((
H6r  rB  c                   U(       d  U R                   nU SU SU S3nSUS.n[        R                  5       nUc  [        S5      e[        R
                  " UU R                  U R                  5      [        R                  " U5      US9n[        US9  UR                  5       n	[        U	S9  / n
U	S	   S
   nU H#  nU
R                  US   S   S   US   S.5        M%     U
$ )a  
Check the blob has already uploaded.
True -- uploaded; False -- not uploaded.

Args:
    repo_id (str): The repo id ModelScope.
    repo_type (str): The repo type. `dataset`, `model`, etc.
    objects (List[Dict[str, Any]]): The objects to check.
        oid (str): The sha256 hash value.
        size (int): The size of the blob.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Returns:
    List[Dict[str, Any]]: The result of the check.
r  r  z/info/lfs/objects/batchupload)	operationrB  r  )r   r  r   rP  r  rB  actionshrefr  )r]  r  )ro   r~   r   r   r  r   r   r   r   r  r@   rA   r  )r   r  r  rB  ro   r]  r  r   ra  r  rV  resp_objectsobjs                r   rR  HubApi._validate_blob	  s    2 }}H
.2gY>UV "

 #..0?HII==((6G$	
 	(+}}4 F|I.C!!Ix08E
$   r   r(  c                    S nS n[        U[        5      (       a5  [        R                  R	                  US   5      (       a  UnO5[        S5      e[        R                  R	                  U5      (       a  U/nOUnUc  U R                  R                  U5        [        U5      R                  5       R                  5       nUR                  5       (       d  [        SU S35      e[        UR                  S5      5       Vs0 s H:  nUR                  5       (       d  M  UR                  U5      R!                  5       U_M<     nnOm0 nU He  n[        R                  R	                  U5      (       d  M)  U R                  R#                  U5        Xx[        R                  R%                  U5      '   Mg     [        [&        R(                  " UR+                  5       X4S95      n	U(       a  UR-                  S5       S3OSn
U	 Vs/ s H  nX-   [/        X   5      4PM     nn[0        R3                  S	[5        U5       S
35        U$ s  snf s  snf )Nr   z0Uploading multiple folders is not supported now.zProvided path: 'z' is not a directoryz**/*)r"  r#  r   r   z	Prepared z files for upload.)rj  r  r   r   rh  r   r   check_folderr   r   resolveis_dirsortedglobis_filerelative_toas_posixr  r  rj   filter_repo_objectskeysr  rk  r  rI  r  )r   r(  r  r"  r#  r%  
files_pathr   relpath_to_abspathfiltered_repo_objectsprefixrelpathr2  s                r   r0  HubApi._prepare_upload_folder 
  s    
*D11ww~~215661
 !STTww~~23323
2,,[9{+668@@BK%%'' #3K=@T!UVV
 #;#3#3F#;<"<D<<> ?  -668$><  " "$"77>>$''''2248AErww'7'7'=> # !%))"'').!
 3?L&&s+,A.B 1!
0 s#5#>?@0 	 !

 	i$9 :;;MNO$$9"*!
s   >H=$H=<Ic                 t   U/ S.nSnU  GH  n[        U[        5      (       a;  UR                  (       a*  [        R	                  SUR
                   S35        US-  nMT  [        U[        5      (       a}  UR                  S:X  am  UR                  (       a  SOSUR
                  SUR                  R                  S	UR                  5       R                  5       S
S.nUS   R                  U5        M  [        U[        5      (       av  UR                  S:X  af  UR                  (       a  SOSUR
                  SUR                  R                  UR                  R                  S	S	S.nUS   R                  U5        GMq  [        SU S[        USS5       35      e   US:  a  [        R!                  SU S35        U$ )z>
Prepare the commit payload to be sent to the ModelScope hub.
)r\  r\  r   zSkipping file 'z(' in commit (ignored by gitignore file).rs   r  r   creater   base64)actionr   typer  r
  r3  r@  r\  r  z(Unknown operation to commit. Operation: z. Upload mode: r  NzSkipped z/ file(s) in commit (ignored by gitignore file).)rj  ri   _should_ignorer  r  r  r  r  upload_infor  
b64contentrF  r  r
  r   r   rI  )r  r\  r  nb_ignored_filesr[  commit_actions         r   r  HubApi._prepare_commit_payloadY
  s    -

  $I )%788Y=U=Uy/E/E.FFnop A%  )%788Y=S=SW_=_ +4*@*@hh%22$%1166 (335<<> (! 	"))-8 I'9::y?U?UY^?^ +4*@*@hh%22!%1166'33::! "! 	"))-8 !>yk J	>4@AC K $T aKK(#3"44cder   internal_timeoutc                     S[         S[        4S jnU R                   S3nU" X1S9nSnUbK  UR                  5       nSU;   a	  US   S   nOSnU(       a#  U" XaS	9nUb  UR                  R                  5       nU$ )
z
Get the internal acceleration domain.

Args:
    internal_timeout (float): The timeout for the request. Default to 0.2s

Returns:
    str: The internal acceleration domain. e.g. `cn-hangzhou`, `cn-zhangjiakou`
r]  r}   c                      [         R                  " XS9nUR                  5         U$ ! [         R                  R                   a    S n U$ f = f)Nr|   )r  r   raise_for_statusr  r  )r]  r}   ra  s      r   send_request>HubApi._get_internal_acceleration_domain.<locals>.send_request
  sO     #<<=))+ O &&77  O s   $(  AAz&/api/v1/repos/internalAccelerationInfo)r]  r}   r   r  InternalRegionQueryAddressr|   )rk  floatro   r   r  r  )r   r}  r  internal_urlinternal_info_response	region_id
query_addrdomain_responses           r   !_get_internal_acceleration_domain(HubApi._get_internal_acceleration_domain
  s    	c 	E 	 --(NO ".,!Y	!-%;%@%@%B"//3F;<XY
"$
".z"T". / 4 4 : : <Ir   delete_patternsc          
      6   U[         ;  a  [        SU 35      eU(       d  [        S5      e[        U[        5      (       a  U/n[        R                  5       nU(       d  U R                  nUc  [        S5      eU R                  U R                  5      nU[        :X  a5  U R                  UU=(       d    [        SUUS9nU V	s/ s H  oS   PM	     n
n	OU[        :X  ah  / n
SnS	n  U R                  UU=(       d    [        SUUUS
9nU H"  nUS   S:w  d  M  U
R%                  US   5        M$     ['        U5      U:  a  OUS-  nMa  [        SU S[          35      e/ nU
 H<  nU H3  n[(        R(                  " UU5      (       d  M!  UR%                  U5          M:     M>     / / nnU H  n U[        :X  a2  UR+                  S5      u  nnU SU SU S3nU=(       d    [        US.nOCU[        :X  a$  UR+                  S5      u  nnU SU SU S3nSU0nO[        SU S[          35      eU R,                  R/                  UUXgS9n[1        U5        UR3                  5       n[5        U5        UR%                  U5        M     UU['        U5      S.$ s  sn	f ! [         a0  n[         R#                  SU S[        U5       35         SnAGMl  SnAff = f! [         aA  nUR%                  U5        [         R#                  SU S[        U5       35         SnAGMp  SnAff = f)a  
Delete files in batch using glob (wildcard) patterns, e.g. '*.py', 'data/*.csv', 'foo*', etc.

Example:
    # Delete all Python and Markdown files in a model repo
    api.delete_files(
        repo_id='your_username/your_model',
        repo_type=REPO_TYPE_MODEL,
        delete_patterns=['*.py', '*.md']
    )

    # Delete all CSV files in the data/ directory of a dataset repo
    api.delete_files(
        repo_id='your_username/your_dataset',
        repo_type=REPO_TYPE_DATASET,
        delete_patterns='data/*.csv'
    )

Args:
    repo_id (str): 'owner/repo_name' or 'owner/dataset_name', e.g. 'Koko/my_model'
    repo_type (str): REPO_TYPE_MODEL or REPO_TYPE_DATASET
    delete_patterns (str or List[str]): List of glob patterns, e.g. '*.py', 'data/*.csv', 'foo*'
    revision (str, optional): Branch or tag name
    endpoint (str, optional): API endpoint
Returns:
    dict: Deletion result
zUnsupported repo_type: zdelete_patterns cannot be emptyNr  T)r   r)  ro   r  r   rs   r  )r  r   r)  r  r  ro   zGet dataset: z file list failed, message: r  treer  r   r   z/file)r  rq  rA  z/reporq  r  zFailed to delete r  )deleted_filesfailed_filestotal_files)rV   r   rj  rk  r~   r   ro   r   r   rU   r0  rP   rT   r8  rO   r  r  r  r  r  fnmatchr   r   r  r@   r   rA   )r   r  r  r  r   ro   r   r   r  r  
file_pathsr  r  dataset_filesr  file_info_d	to_deleter   delete_patternr  r  ownerr  r]  r  r  r   r  s                               r   delete_filesHubApi.delete_files
  sg   D --6ykBCC>??os++./O"..0}}H?HII&&t||4 '((!;%;!# ) E .33UF)UJ3J++JKI:>:P:P '!)!E-E"&$/"+!) ;Q ;M $1K"6*f4"))+f*=> $1 }%	1q - 0 6ykATUfTghii 	D"1??488$$T* #2  ')"|DC/'.}}S'9$E9%JoeWAi[NC$,$F0F$(F "33*1--*<'E<%J&7wa~USC"DF %'>ykI\]n\o%pqqLL''FG']%a(vvxt$$$T*- 8 +(y>
 	
G 4 ! LL=	9UVYZ[V\U]!^_`  C##D)0bQABBCs7   J$!J 'CK
K
$KK

L5LL)modeldatasetprivatepublicc                    U(       d  [        S5      eUS;  a  [        SU S35      e[        R                  " 5        VVs0 s H  u  pVXe_M	     nnnUR                  US5      nU R	                  USS9n	U[
        :X  Gac  U R                  US9n
U R                   S	U 3nU
R                  S
5      nSn[        U[        5      (       a9  U(       a2  US   n[        U[        5      (       a  U(       a  UR                  S5      nU
R                  SS5      U
R                  SS5      USU
R                  SS5      U
R                  SS5      U
R                  SS5      U
R                  SS5      U
R                  SS5      U
R                  SS5      U
R                  S0 5      R                  SS5      U
R                  S0 5      R                  SS5      UU
R                  SS5      S.nOU[        :X  ao  UR                  S5      n[        U5      S:w  d  [        U5      (       d  [        SU S 35      eU R                  US!   US   S"9u  nnU R                   S#U 3nUSS$.nO[        S%U S&[          35      eU R"                  R%                  UUU	U R'                  U R(                  5      S'9n[+        U5        UR-                  5       n[/        U5        U$ s  snnf )(a  
Set the visibility of a repo.

Args:
    repo_id (str): The repo id in the format of `owner_name/repo_name`.
    repo_type (Literal['model', 'dataset']): The repo type, `model` or `dataset`.
    visibility (Literal['private', 'public']): The visibility to set, `private` or `public`.
    token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
        See `https://modelscope.cn/my/myaccesstoken` to get your token.

Returns:
    dict: The response from the server.
r'  r  r  z-, supported visibilities: `private`, `public`   Tr   )r   r   r^   r   r   r   r   ModelFrameworkPytorchrr   ApprovalModer  r   r   
ModelCoverSubScientificFieldNNEXAScientificFieldrp  r   )r   r  r6   ProtectedModer  r  r   r   r  r  r  rp  	ModelTaskr   r   zInvalid dataset repo_id: z-, should be in format of `owner/dataset_name`rs   )r  r  rA  )r6   r  r  r  r   )r   r7   r  r   r   rU   r  ro   rj  r  r  rT   r   r  allr  rV   r   r  r   r   r@   r   rA   )r   r  r  r   r   r  r  visibility_mapvisibility_coder   r3  r   tasksmodel_tasksfirstr  repo_id_partsdataset_idxr1  r   r  s                        r   set_repo_visibilityHubApi.set_repo_visibilityB  s   & ABB223J<?lmnn;H;N;N;P)Q;P41!$;P)Q-11*a@"""M'9Jmm_OG9=DNN7+EK%&&5aeT**u"'))F"3K)~~mR@",..1A9"M-!" *~q A)~~mR@&NN:r:$.NN3Er$J(nn\2>&0nn5I4&P#->>&"#=#A#ABSUW#X$..4882F(%>>)R8G  ++#MM#.M=!Q&c-.@.@ #<WIEr!stt!99*1-'* : NK
 mm_$5k]CD-!"G
 29+=PQbPcdeeLL((6	  8 	a vvxtu *Rs    K)ro   r   r   r   )Fr   )NNNr  )rs   r  N)NF)NFN)FN)Trs   r  N)rs   )g?)n__name__
__module____qualname____firstlineno____doc__r   r   r   rk  r   r   r  r   r   r5   PUBLICr4   	APACHE_V2r  r   r  r	  r  rP   r  r  r%  rD   r3  rC   r:  rU   r   r>  r  rJ  staticmethodrZ  rQ   r   r  r  r  r   r  r  r  r  r  r  r   r  r  r0  r  r3   r  rH  r  r  r/  r8  r7  r"  r+  r  r;  r[   REUSE_DATASET_IF_EXISTSri  rO   ru  rx  r}  r  r  r  r  r  r  r|  r  r   r  r6   rG   r  r   rh   rg   r  r   rS  r	   r   r   r>  r  r
   r   rR  tupler0  r  r  r  r  r   r  __static_attributes__r.  r   r   rm   rm   b   s    ,008/#C=/B ,(4. > +/&*.:"3-.: sm.:d 2A1G1G.6.@.@378:/3,09=g"g!)#g 'smg $,C=	g
 )1g  (}g %SMg "*+!6g CFgX 4804=AU#&U#&U $,C=U !)	U
 &.k%:U
 GJUn!S !HSM !4:c :Xc] : '=&*	)%)% sm)% sm	)%
 
)%\ :>(#'*(# *2#(# CF(#Z .D-1	 &c] &c]	 7@	F 04/3	!  (}  (}	 9D	H $3"8"&!a!a C=	!a
 3-!a 3-!a 
y+%	&!aN (,&*',#'88  }	8
 sm8 tn8 C=8 
8t=3 =3 =(3- =6 ! ! 0?/E/E,4,>,>153A(,-H6:JNAEN7 N7!N7  (}N7 %SM	N7
 "*#N7 $,C=N7 !N7 &c]N7 '/smN7 )1tCy#~1F(GN7  (c49n(=>N7d 23/1.2	&$'&!)#&  (}& 'sm	& 8<	&V 4513,0.204C$&)C$ $,C=C$ "*#	C$
 %SMC$ 'smC$ !)C$  C$JtY)? I  /327	 'sm tY/	 =AI	6 /327&* + + 'sm + tY/	 +
 sm +
 04Cy +D ,0AE<@	C,/C ,4I+>C -5SM	CN %):>59	M%(M %-Y$7M &.c]	M 38&*	LL tY/L sm	L
 
tCy$s)#	$L@ 38 tY/ 
tCy$s)#	$	2 3I.249>C24264"%4"*3-4 'sm4 $,D>	4
 &+4?%;4 "*$4 #+3-4 <@:4v '+!! !
 sm!< 68080B0B3D3K3K4615& %(& "%&  &.c]&  !)	& 
 $,C=&  %-SM&  "*#&  =@& P! ! !(C (C (S[\_S` ($ *.*+(+15%("% "% #&	
 #' %( #& "*#H 6E4O-.+-48<Y#&<Y &.c]<Y %-SM	<Y
 (+<Y &)<Y %-SM<YB +F+.,0-.+.483%#&3% %(3% &)	3%
 &*3% (+3% &)3% %-SM3%p #>"&	-- 3-- 3-	-D ^bs s /2>AMUVY]* 
L 
Lc 
L 
L  FJ	!.s !.69!.58!. <?!. OR!. _c!. 6>c]	!.F <@|GkGk 5 5x '?#(/3&*  	
 sm 4. 'tn smJ '?&*  	
 sm sm$ '?&*
5
5 
5 sm	
5
 sm
5" '?&*  	
 sm sm2 s 4 D JN=@:= := ?B	
 :B#: cgRZ[^R_ \`S  -0<?KSTW=dg  Y]# S *-9<HPQTad  \`     37 KSTW= dh BP# P# PRU P -1(2(9(9'6*.%-%7%7&*',48.2kk dD)	k
 !k  }k #3-k c]k smk tnk $,D>k !+k 
kf 15'6&A&* v
v
 !1v

  v
 !)v
 v
  }v
 smv
 smv
 v
 v
 
v
| '+'6,004,-'4+0&A@ #3eX#=>@ 	@
 @ d#@  }@ %SM@ !)@ %SM@  }@ #4.@ sm@ 
@N +-,004&*'6>B?C2&A}K }K sD$s)T$Z?@	}K
 #3-}K %SM}K !)}K d#}K  }}K %U49c>%:;}K &eDIsN&;<}K }K sm}K 
z4
++	,}KN ,1'4,-] ] 	]
 ] ] T5(23] #4.]  }] %SM] 
]J '+9 9 	9
 $sCx.)9 sm9 
d38n	9~ ?C?C7%"'T49d4j(H"I7% 7% %U49c>%:;	7%
 &eDIsN&;<7% 
eE4K 	!7%r = 1== 
c3h= =~%% %X 0F/3@
!@
 #@
 ',CcN&;@

  (}@
  (}@
 9=S#X@
L 7;	S%(S'./A'BS )00C(DS $)d#3	S
 "&S Sr   rm   c                       \ rS rSr\" \5      rSrSrSr	Sr
Sr\S 5       r\S\4S j5       r\S	 5       r\S
 5       r\S\4S j5       r\S\S\4S j5       r\S\\\4   4S j5       r\S\\   4S j5       r\SS\\\S4   S\4S jj5       rSrg)r~   i  r   	git_tokenuserr   Fc                  J    [         R                  " [        R                  SS9  g )NT)r  )r   makedirsr~   path_credentialr.  r   r   make_sure_credential_path_exist0ModelScopeConfig.make_sure_credential_path_exist  s    
$44tDr   c                    [         R                  5         [        [        R                  R                  [         R                  [         R                  5      S5       n[        R                  " X5        S S S 5        g ! , (       d  f       g = f)Nzwb+)
r~   r  rT  r   r   rN  r  COOKIES_FILE_NAMEpicklerU  )r   r  s     r   r   ModelScopeConfig.save_cookies  s_    88:-==-??ABGILMKK#I I Is   A;;
B	c                  $   [         R                  R                  [        R                  [        R
                  5      n [         R                  R                  U 5      (       a  [        U S5       n[        R                  " U5      nU Hk  nUR                  S:X  d  M  UR                  5       (       d  M,  [        R                  (       a  MC  S[        l        [        R                  S5          S S S 5        g    UsS S S 5        $ g ! , (       d  f       g = f)NrO  r   TzKNot logged-in, you can login for uploadingor accessing controlled entities.)r   r   rN  r~   r  r  rg  rT  r  loadr   
is_expiredcookie_expired_warningr  rI  )cookies_pathr  r   cookies       r   r   ModelScopeConfig.get_cookies  s    ww||$4$D$D$4$F$FH77>>,''lD)Q ++a.%F{{n49J9J9L9L 0 G G GBF(? %H I# *)%  *)  *) s$   -*DD2D	!D3D
Dc                  ~   [         R                  R                  [        R                  [        R
                  5      n Sn[         R                  R                  U 5      (       a>  [        U S5       n[        UR                  5       R                  5       SS9nUsS S S 5        $ US:X  d  [        U5      S:w  ad  [        [        R                  " 5       R                  5      n[        R                  5         [        U S5       nUR!                  U5        S S S 5        U$ U$ ! , (       d  f       N= f! , (       d  f       U$ = f)Nr   rO  rD  r?      w+)r   r   rN  r~   r  USER_SESSION_ID_FILE_NAMErg  rT  rk  readliner  r  r  r  r  r  r(  )session_path
session_idr  wfs       r   get_user_session_id$ModelScopeConfig.get_user_session_id  s    ww||$4$D$D$4$N$NP
77>>,''lD)Q !3!3!5H
! *) s:"4TZZ\--.J<<>lD)R$ * z *) *) s   /(D>D-
D*-
D<r   c                    [         R                  5         [        [        R                  R                  [         R                  [         R                  5      S5       nUR                  U 5        S S S 5        g ! , (       d  f       g = f)Nr  )	r~   r  rT  r   r   rN  r  GIT_TOKEN_FILE_NAMEr(  r   r  s     r   r   ModelScopeConfig.save_token  s^    88:-==-AACDHJMNGGENJ J Js   A66
Br  
user_emailc                    [         R                  5         [        [        R                  R                  [         R                  [         R                  5      S5       nUR                  U < SU< 35        S S S 5        g ! , (       d  f       g = f)Nr  :)	r~   r  rT  r   r   rN  r  USER_INFO_FILE_NAMEr(  )r  r  r  s      r   r   ModelScopeConfig.save_user_info  sd    88:-==-AACDHJMNGGy*56J J Js   A==
Br   c                  T    [        [        R                  R                  [        R
                  [        R                  5      SSS9 n U R                  5       nUR                  S5      S   UR                  S5      S   4sS S S 5        $ ! , (       d  f       g= f! [         a     gf = f)Nr   rD  r?  r  r   rs   r   )
rT  r   r   rN  r~   r  r  rv   r   FileNotFoundError)r  rI  s     r   get_user_infoModelScopeConfig.get_user_info  s    		GGLL!1!A!A!1!E!EG$	& *+vvxzz#q)4::c?1+==& & &  ! 		s0   AB 8B	?	B 	
BB B 
B'&B'c                     Sn  [        [        R                  R                  [        R
                  [        R                  5      SSS9 nUR                  5       n SSS5        U $ ! , (       d  f       U $ = f! [         a     U $ f = f)zk
Get token or None if not existent.

Returns:
    `str` or `None`: The token, `None` if it doesn't exist.

Nr   rD  r?  )	rT  r   r   rN  r~   r  r  rv   r  r  s     r   	get_tokenModelScopeConfig.get_token  s     	GGLL!1!A!A!1!E!EG$	& *+& & &  ! 		s0   AA6 	A$A6 $
A3.A6 3A6 6
BBN
user_agentc                 h   Sn[         [        R                  ;   a  [        R                  [            nSn[        [        R                  ;   a  [        R                  [           nSSKJn  SU< S[        R                  " 5       < S[        R                  5       < S[        R                  " 5       < S	[        R                  " 5       < S
U< SU< 3n[        U [        5      (       a.  USSR                  S U R                  5        5       5      -   -  nU$ [        U [        5      (       a  USU -   -  nU$ )zFormats a user-agent string with basic info about a request.

Args:
    user_agent (`str`, `dict`, *optional*):
        The user agent info in the form of a dictionary or a single string.

Returns:
    The formatted user-agent string.
customunknownr   )__version__zmodelscope/z	; python/z; session_id/z; platform/z; processor/z; env/z; user/z; c              3   4   #    U  H  u  pU S U 3v   M     g7f)r   Nr.  ).0r  r  s      r   	<genexpr>2ModelScopeConfig.get_user_agent.<locals>.<genexpr>!  s     "M:L$!aS!::Ls   )r!   r   r   r"   
modelscoper  platformpython_versionr~   r  	processorrj  r  rN  r  rk  )r  envr  r  uas        r   r   ModelScopeConfig.get_user_agent  s     '2::5**9:C	$

2

#<=I*##%002 
 j$''$"M*:J:J:L"MMMMB 	 
C(($##B	r   r.  r  )r  r  r  r  r   r#   r  r  r  r  r  r  r  r  r   r   r   r  rk  r   r   r   r  r   r  r   r
   r   r  r.  r   r   r~   r~     s.    !<=O!%  )"E E $i $ $       #   7# 73 7 7 5c?   x}  ( "5sD#9 "s " "r   r~   c                       \ rS rSrSr\\\\\	4S\
S\
S\
S\
S\
4
S jjrSS jrS\\\4   4S jrS\\\\\4   S\S	\4S jrS\\\\4      S\S	S
4S jrSrg
)r   i'  aN  
Check the files and folders to be uploaded.

Args:
    max_file_count (int): The maximum number of files to be uploaded. Default to `UPLOAD_MAX_FILE_COUNT`.
    max_file_count_in_dir (int): The maximum number of files in a directory.
        Default to `UPLOAD_MAX_FILE_COUNT_IN_DIR`.
    max_file_size (int): The maximum size of a single file in bytes. Default to `UPLOAD_MAX_FILE_SIZE`.
    size_threshold_to_enforce_lfs (int): The size threshold to enforce LFS in bytes.
        Files larger than this size will be enforced to be uploaded via LFS.
        Default to `UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS`.
    normal_file_size_total_limit (int): The total size limit of normal files in bytes.
        Default to `UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT`.

Examples:
    >>> from modelscope.hub.api import UploadingCheck
    >>> upload_checker = UploadingCheck()
    >>> upload_checker.check_file('/path/to/your/file.txt')
    >>> upload_checker.check_folder('/path/to/your/folder')
    >>> is_lfs = upload_checker.is_lfs('/path/to/your/file.txt', repo_type='model')
    >>> print(f'Is LFS: {is_lfs}')
max_file_countmax_file_count_in_dirmax_file_sizesize_threshold_to_enforce_lfsnormal_file_size_total_limitc                 @    Xl         X l        X0l        X@l        XPl        g r  )r  r  r   r  r  )r   r  r  r   r  r  s         r   r   UploadingCheck.__init__>  s"     -%:"*-J*,H)r   r   Nc           	      B   [        U[        [        45      (       a3  [        R                  R                  U5      (       d  [        SU S35      e[        U5      nX R                  :  a7  [        R                  SU R                  S-   S[        US-  S5       S35        gg)	z
Check a single file to be uploaded.

Args:
    file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked.

Raises:
    ValueError: If the file does not exist or exceeds the size limit.
File  does not existzFile exceeds size limit:    @z	 GB, got     GBN)rj  rk  r   r   r   rg  r   ra   r   r  r   r  )r   r  r  s      r   r  UploadingCheck.check_fileL  s     &d4477>>"233 5)9(:/!JKK&'78	)))NN6t7I7IY7W6X Y""'	Y(?"C!DCI J *r   r%  c           	         SnSn[        U[        5      (       a  [        U5      nUR                  5        H  nUR	                  5       (       a^  US-  n[        U5      nXPR                  :  a=  [        R                  SU SU R                  S-   S3S[        US-  S5       S35        Mt  Mv  UR                  5       (       d  M  US-  nU R                  U5      u  pgXg-   U R                  :  a   [        S	U S
Xg-    SU R                   35      eX&-  nX7-  nM     X R                  :  a  [        SU SU R                   35      eX#4$ )z
Check a folder to be uploaded.

Args:
    folder_path (Union[str, Path]): The folder path to be checked.

Raises:
    ValueError: If the folder does not exist or exceeds the file count limit.
r   rs   r  z exceeds size limit: r  r
  zgot r	  z
Directory z
 contains z items and exceeds limit: zTotal file count z and exceeds limit: )rj  rk  r   iterdirrg  ra   r   r  r   r  rd  rb  r  r   r  )r   r%  
file_count	dir_countr  	item_sizesub_file_countsub_dir_counts           r   rb  UploadingCheck.check_folder_  sg    
	k3''{+K'')D||~~a
!.t!4	111NNU4&0EdFXFX\eFfEggj#k%)%	Y0G*K)LC#PR 2 Q	040A0A$0G-"2d6P6PP$z$z.B`Aa b;;?;U;U:V&X Y Y,
*	 *" +++0<PQUQdQdPefgg$$r   r  r  c                    Sn[        U[        [        45      (       a  [        U5      nUR                  5       (       d  [	        SU S35      eU[
        :X  a  UR                  [        ;  a  SnO6U[        :X  a  UR                  [        ;  a  SnO[	        SU S[         35      e[        U5      nX@R                  :  =(       d    U$ )a2  
Check if a file should be uploaded via LFS.

Args:
    file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked.
    repo_type (str): The repo type, either `model` or `dataset`.

Returns:
    bool: True if the file should be uploaded via LFS, False otherwise.
Tr  r  Fr  r  )rj  rk  r   rg  r   rU   r  re   rT   rc   rV   ra   r  )r   r  r  hit_lfs_suffixr  s        r   r  UploadingCheck.is_lfs  s     &d44#$45#**,, 5)9(:/!JKKO+#**2BB%*N//#**2DD%*N #6ykATUfTg!hii&'78	===OOr   r  c                    U Vs/ s H  o0R                  X25      (       a  M  UPM     nn[        U Vs/ s H  n[        U5      PM     sn5      nXPR                  :  a"  [	        SUS-   SU R                  S-   S35      egs  snf s  snf )a&  
Check a list of normal files to be uploaded.

Args:
    file_path_list (List[Union[str, Path]]): The list of file paths to be checked.
    repo_type (str): The repo type, either `model` or `dataset`.

Raises:
    ValueError: If the total size of normal files exceeds the limit.

Returns: None
zTotal size of non-lfs files i   zMB and exceeds limit: MBN)r  sumra   r  r   )r   r  r  r  normal_file_listrb  s         r   r  !UploadingCheck.check_normal_files  s     .<`^T;;tC_D^`:JK:J$--:JKL
999;J+<V;W X3373T3TXc3d2eegi j j : aKs   BBB)r  r  r   r  r  )r   N)r  r  r  r  r  r-   r.   r/   r1   r0   r  r   r  r   rk  r   rb  rS  r	   r  r  r   r  r  r.  r   r   r   r   '  s    0 #8)E!51U0SII $'I 	I
 ,/I +.IJ&$%c4i(8 $%LPuS$x-G'H PUX P]a P>jeCI6F1G jTW j\` jr   r   )rx  r  r   r  r   r  r  rs  rp  r  r  r  re  collectionsr   httpr   http.cookiejarr   os.pathr   pathlibr   typingr   r	   r
   r   r   r   r   r   r   r   r   r   r  r   requests.adaptersr   r   requests.exceptionsr   rS  r   modelscope.hub.constantsr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   modelscope.hub.errorsr8   r9   r:   r;   r<   r=   r>   r?   r@   rA   modelscope.hub.gitrB   modelscope.hub.inforC   rD   r  rF   modelscope.hub.utils.aigcrG   modelscope.hub.utils.utilsrH   rI   rJ   rK   rL   rM   rN   modelscope.utils.constantrO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   modelscope.utils.file_utilsr`   ra   modelscope.utils.loggerrb   modelscope.utils.repo_utilsrc   rd   re   rf   rg   rh   ri   rj   modelscope.utils.thread_utilsrk   r  rm   r~   r   r.  r   r   <module>r/     s&      	 	   	      #  $  " " " "    0 ) 5 5 5 5 5 5 5 5 58J J J 1 6 0 /F F F	; 	; 	; 	; 	; E .4 4 4 :	s, s,lYL L^Oj Ojr   