
    9i b                         S r SSKrSSKJr  SSKJr  SSKJrJr  SSK	J
r
  SSKJrJrJrJr  SSKJrJr  SS	KJrJrJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSK J!r!  SSK"J#r#  / SQr$S r%S r& " S S5      r'S r(S r)S r*S r+S r,S r-S r.S r/g)u   
本文件实现了产线任务提交、查询、停止功能

Authors: xiangyiqing(xiangyiqing@baidu.com), suoyi@baidu.com
Date:    2024/3/2
    N)Path)PrettyTable)logconfig)	ErrorEnum)AUTH_DIRAUTH_TOKEN_FILELOG_DIRLOG_LEVEL_FILE)zip_direrr_resp)upload_super_fileMyBosClientupload_file)pipeline)dataset)BceClientConfiguration)BceCredentials)BceHttpClientError)	BosClient)
set_configcreatequerystopc                 .    [         R                   SU  S3$ )u   拼接产线详情链接z
/pipeline/z/detail)r   #STUDIO_MODEL_API_URL_PREFIX_DEFAULT)pipeline_ids    Y/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/aistudio_sdk/sdk/pipeline.pyget_detail_urlr   %   s    889K=PWXX    c                     [        5       n/ SQUl        U  H  nUR                  U5        M     [        R                  " U5        g)u   
表格化打印
tabled_log_info([
    ["pipeline_id", "args.summit_name", "status", "get_detail_url(pipeline_id)", "create_ime"], 
    [], 
    ...
])
)pidnamestatusurl
createTimeN)r   field_namesadd_rowr   info)detail_listtabledetails      r   tabled_log_infor-   *   s4     MEFEf HHUOr    c                   X    \ rS rSrSrSrS rS rS rS r	S r
S	 rS
 rS rS rS rSrg)Pipeline:   u   
pipeline类
r   c                    [         R                  " SU 35        UR                  nU(       a   [        R                  R                  [        5      (       d  [        [        5      R                  SSS9  [        [        S5       nUR                  U5        SSS5        [         R                  " S[         35        UR                   nU(       a   [        R                  R                  [        5      (       d  [        [        5      R                  SSS9  [        [         S5       nUR                  U5        SSS5        [         R                  " S[          35        gg! , (       d  f       N= f! [         a#  n[         R                  " SU 35         SnANSnAff = f! , (       d  f       Nn= f! [         a#  n[         R                  " SU 35         SnAgSnAff = f)u   
配置: token, log_level
u   鉴权配置，参数: T)parentsexist_okwNz[OK] Configuration saved to: z[Error] Configuration faild: )r   debugtokenospathexistsr   r   mkdiropenr	   writer)   	Exceptionerrorr
   r   )selfargsr6   filee	log_levels         r   r   Pipeline.set_config@   s[    			+D623

	?ww~~h//N(((E/3/4JJu% 088IJK HH		?ww~~g..M''t'D.#.$JJy) /88HIJ  0/  ?		9!=>>? /.  ?		9!=>>?sa   AE- E%E- AF. #F5%F. 
E*&E- -
F7FF
F+'F. .
G8GGc                 H   [         R                  R                  [        5      (       d  g [	        [        S5       nUR                  5       R                  5       sSSS5        $ ! , (       d  f       g= f! [         a#  n[        R                  " SU 35         SnAgSnAff = f)u   
获取鉴权token
Nrz"[Error] Read configuration faild: )
r7   r8   r9   r	   r;   readstripr=   r   r>   )r?   rA   rB   s      r   get_authPipeline.get_auth`   so     ww~~o..	os+tyy{((* ,++ 	II:1#>?	s4   A4 A#	A4 #
A1-A4 1A4 4
B!>BB!c                    [         R                  " SU 35        U R                  5       nU(       dT  [         R                  " [	        [
        R                  R                  [
        R                  R                  S-   5      5        g[        UR                  5      [        R                  :  ae  [         R                  " [	        [
        R                  R                  [
        R                  R                   S[        R                   S35      5        gUR                  nU S3n[        R                  R!                  U5      (       dQ  [         R                  " [	        [
        R"                  R                  [
        R"                  R                  5      5        g[        R                  R%                  U5      (       dQ  [         R                  " [	        [
        R&                  R                  [
        R&                  R                  5      5        g [         R                  " SU 35        [)        X45        [         R                  " SU 35        [        R.                  [        R0                  " U5      R2                  :  ak  [         R                  " [	        [
        R4                  R                  [
        R4                  R                   S
[        R.                  S-  S-   S35      5        g [         R                  " S5        / nUR                   H  nUR7                  SU05        M     [8        R:                  " UUR<                  UR>                  UR@                  URB                  URD                  URF                  U5      nUS   [
        RL                  R                  :w  aa  [         R                  " [	        [
        RJ                  R                  [
        RJ                  R                   SUS    3US   US   5      5        g[         R                  " S5        US   S   n	 [         R                  " S5        [8        RN                  " X)5      nUS   [
        RL                  R                  :w  aa  [         R                  " [	        [
        RP                  R                  [
        RP                  R                   SUS    3US   US   5      5        g[         R                  " S5        US   n
U
S   nU
S   nU
S   nU
S   nU
S   nU
S   n [         R                  " S5        [8        RR                  " UUUUUUU5        [         R                  " S!5         [         R                  " S"5        [8        RT                  " UU	S#U[        R                  RY                  U5      5      nUS   [
        RL                  R                  :w  aa  [         R                  " [	        [
        RZ                  R                  [
        RZ                  R                   SUS    3US   US   5      5        g[         R                  " S$5        US   n
U
S%   nU
S&   n[]        U	UR<                  U[_        U	5      U//5        g! [*         a_  n[         R                  " [	        [
        R,                  R                  [
        R,                  R                   S	U 35      5         SnAgSnAff = f! [8        RH                   ab  n[         R                  " [	        [
        RJ                  R                  [
        RJ                  R                   SUSS  35      5         SnAgSnAff = f! [8        RH                   ab  n[         R                  " [	        [
        RP                  R                  [
        RP                  R                   SUSS  35      5         SnAgSnAff = f! [*         az  n[8        RT                  " UU	S 5        [         R                  " [	        [
        RV                  R                  [
        RV                  R                   SUSS  35      5         SnAgSnAff = f! [8        RH                   ab  n[         R                  " [	        [
        RZ                  R                  [
        RZ                  R                   SUSS  35      5         SnAgSnAff = f)'u   
创建产线任务
u   创建产线，参数: /   , 请使用 aistudio config --token {yourToken}Nu   : 单个任务最多挂载u   个数据集z.zipu   step 1: 开始打包代码... u   代码打包完成! u   : 压缩出错
u!   : 代码包总体积不能超过 i   z MBu   step 2: 请求参数校验...	datasetId:   	errorCodeerrorMsglogIdu   参数校验成功!result
pipelineIdu   step 3: 请求申请ak/sk...   申请ak/sk成功!endpoint
bucketNamefileKeyaccessKeyIdsecretAccessKeysessionTokenu   step 4: 代码上传bos...Fu   代码上传成功!u#   step 5: 回调请求创建产线...Tu   创建成功!stager&   )0r   r5   rI   r>   r   r   TOKEN_IS_EMPTYcodemessagelenmount_datasetr   MOUNT_DATASET_LIMITPARAMS_INVALIDr8   r7   r9   FILE_NOT_FOUNDisdirNEED_FOLDERr   r=   INTERNAL_ERRORPIPELINE_CODE_SIZE_LIMITstatst_sizeFILE_TOO_LARGEappend
pp_requestr   summit_namecmdenvdevicegpuspaymentRequestPipelineExceptionREQUEST_CREATE_PIPELINE_FAILEDSUCCESSbosaclREQUEST_BOSACL_FAILED
bos_uploadcreate_callbackBOS_UPLOAD_FAILEDbasename'REQUEST_CREATE_PIPELINE_CALLBACK_FAILEDr-   r   )r?   r@   r6   
input_pathzip_filerB   dataset_list
dataset_idrespr   rS   rV   bucket_namefile_keyaccess_key_idsecret_access_keysession_tokenr\   
create_imes                      r   r   Pipeline.createn   sb    			+D623 IIh((--((003dd  t!!"V%?%??IIh((--++3344NvOiOiNjjvw   YY
 \&ww~~j))IIh((--((00  ww}}Z((IIh%%**%%--  		II6zlCDJ)II,XJ78 **RWWX->-F-FFIIh((--++3344UV\VuVux|V|  @D  WD  VE  EH  I  	II56L"00
##%  1 $$  			D  	 1 1 6 66IIh88==;;CCDBtJGWFXY[!W	  		'(8n\2	II45$$U8D 	 1 1 6 66IIh//4422::;2d:>N=OP[!W	  		&'h*%\*)$}-"#45~.	II23!!!* 			'(	II;<--  *D 	 1 1 6 66IIhAAFFDDLLMRPTU_P`Oab[!W	  		/"hwL)
  {+
 	G  	IIh((--++3344DQCH  	> 22 	IIh88==;;CCDBq#wiP  	* 22 	IIh//4422::;2agYG  	H  	&&
 IIh++00..667r!DS'C  	0 22 	IIhAAFFDDLLMRPQRVSVPWyY  	s   =Y* (B[  ,] 2_ Aa *
[4A[[]*A]]_#A_  _
aA0aac#Ac  cc                    [         R                  " SU 35        U R                  5       nU(       dT  [         R                  " [	        [
        R                  R                  [
        R                  R                  S-   5      5        g [        R                  " UUR                  UR                  UR                  5      nUS   [
        R                   R                  :w  aa  [         R                  " [	        [
        R                  R                  [
        R                  R                   SUS    3US   US   5      5        g[#        5       nUS	    H1  nUR%                  US
   US   US   ['        US
   5      US   /5        M3     [)        U5        g! [        R                   ab  n[         R                  " [	        [
        R                  R                  [
        R                  R                   SUSS  35      5         SnAgSnAff = f)u   
查询产线
u   查询产线，参数: rL   NrN   rO   rP   rQ   rR   rS   rT   pipelineNamer\   r&   )r   r5   rI   r>   r   r   r]   r^   r_   rm   r   query_pipeline_idr#   r$   rt   REQUEST_QUERY_PIPELINE_FAILEDrv   listrl   r   r-   )r?   r@   r6   r   rB   dataress          r   r   Pipeline.query$  s    			+D623 IIh((--((003dd  	##&&			D 	 1 1 6 66IIh77<<::BBC2d:FVEWX[!W	  v>CKK%'L"3|#45% " 	5 22 	IIh77<<::BBC2agYO  	s   7F H#AH  Hc                 p   [         R                  " SU 35        U R                  5       nU(       dT  [         R                  " [	        [
        R                  R                  [
        R                  R                  S-   5      5        g [        R                  " X!R                  5      nUS   [
        R                  R                  :w  aa  [         R                  " [	        [
        R                  R                  [
        R                  R                   SUS    3US   US   5      5        g[         R                  " S	5        g! [        R                   ab  n[         R                  " [	        [
        R                  R                  [
        R                  R                   SUSS  35      5         SnAgSnAff = f)
u   
停止产线
u   停止产线，参数: rL   NrN   rO   rP   rQ   rR   u   [OK] 停止成功.)r   r5   rI   r>   r   r   r]   r^   r_   rm   r   stop_pipeline_idrt   REQUEST_STOP_PIPELINE_FAILEDrv   r)   )r?   r@   r6   r   rB   s        r   r   Pipeline.stopW  s\    			+D623 IIh((--((003dd  	??5*?*?@D 	 1 1 6 66IIh66;;99AAB"T*EUDVW[!W	  %& 22 	IIh66;;99AAB"QtWIN  	s    D? ?F5AF00F5c                    [         R                  " SU 35        U R                  5       nU(       dT  [         R                  " [	        [
        R                  R                  [
        R                  R                  S-   5      5        g [         R                  " S5        [        R                  " X!R                  5      nUS   [
        R                  R                  :w  aa  [         R                  " [	        [
        R                  R                  [
        R                  R                   SUS    3US   US	   5      5        g[         R                  " S
5        US   nUS   nUS   nUS   nUS   n	US   n
US   nUR!                  S5      nUR#                  S5      S-   nUR$                  (       as  UR$                  R!                  S5      Ul        UR$                  R#                  S5      Ul        UR$                  (       a"  UR$                  S-   Ul        XR$                  -   n['        [)        X5      UU
S9n[+        U5      n [         R                  " SU S35        UR-                  X{SS9n[         R.                  " S5        UR0                   H>  nUR2                  R5                  US5      nU(       d  M(  [         R.                  " U5        M@     UR6                   H3  n[         R.                  " UR8                  R5                  US5      5        M5     g! [        R                   ak  n[         R                  " [	        [
        R                  R                  [
        R                  R                   S[        U5      SS  35      5         SnAgSnAff = f! [:         ah  n[         R                  " [	        [
        R<                  R                  [
        R<                  R                   S[        U5       35      5         SnAgSnAff = f)u0   
列出某个 job 的 output 目录下的文件
u"   列出 job 输出文件，参数: rL   N   请求申请ak/sk...rN   rO   rP   rQ   rR   rU   rS   rV   rW   rY   rZ   r[   rX   /credentialsrV   security_tokenu0   列出 output 目录下的文件和文件夹...[])prefix	delimiteru   文件和文件夹列表: )r   r5   rI   r>   r   r   r]   r^   r_   rm   bosacl_ls_cpjob_idrt   rx   strrv   lstriprstrip	directoryr   r   r   list_objectsr)   contentskeyreplacecommon_prefixesr   r   BOS_LIST_FILES_FAILED)r?   r@   r6   r   rB   rS   rV   r   r   r   r   r   bos_conf
bos_clientresposerA   r   ds                     r   list_output_filesPipeline.list_output_filesy  sL    			6tf=> IIh((--((003dd  	II,-**5++>D 	 1 1 6 66IIh//4422::;2d:>N=OP[!W	  		&'h*%\*}-"#45~.)$??3'??3'#->>!^^2237DN!^^2237DN~~!%#!5#nn4)&}H(

 x(
	IIH
RSTU --kVY-ZGHH02((hh&&x43HHSM ) ,,))(B78 -e 22 	IIh//4422::;2c!fTcl^L  	j " 	IIh//4422::;2c!fXF  	s:   6L  A0N 4AN N&A!NN
PAPPc                 P   [         R                  " SU 35        U R                  5       nU(       dT  [         R                  " [	        [
        R                  R                  [
        R                  R                  S-   5      5        g [         R                  " S5        [        R                  " X!R                  5      nUS   [
        R                  R                  :w  aa  [         R                  " [	        [
        R                  R                  [
        R                  R                   SUS    3US   US	   5      5        g[         R                  " S
5        US   nUS   nUS   nUS   nUS   n	US   n
US   nUR!                  S5      nUR#                  S5      S-   n[%        ['        X5      UU
S9n[)        U5      nXR*                  -   n[,        R.                  R1                  UR*                  5      nUR2                  S:X  a8  [,        R.                  R5                  [,        R6                  " 5       U5      Ul        [,        R.                  R9                  UR2                  5      (       a/  [,        R.                  R5                  UR2                  U5      Ul        [         R                  " SU SUR2                   35         UR;                  X{UR*                  -   UR2                  5        [         R<                  " SUR2                   35        g! [        R                   ak  n[         R                  " [	        [
        R                  R                  [
        R                  R                   S[        U5      SS  35      5         SnAgSnAff = f! [>         a  n[         R                  " SUR*                   35        [         R                  " [	        [
        R@                  R                  [
        R@                  R                   S[        U5      SS  35      5         SnAgSnAff = f)u9   
下载某个 job 的 output 目录下的文件到本地
u"   下载 job 输出文件，参数: rL   Nr   rN   rO   rP   rQ   rR   rU   rS   rV   rW   rY   rZ   r[   rX   r   r   .u!   下载 output 目录下的文件[u   ] 到 u   文件下载成功: u-   下载失败：请检查文件是否存在：)!r   r5   rI   r>   r   r   r]   r^   r_   rm   r   r   rt   rx   r   rv   r   r   r   r   r   result_filer7   r8   r|   
local_pathjoingetcwdre   get_object_to_filer)   r   BOS_DOWNLOAD_FAILED)r?   r@   r6   r   rB   rS   rV   r   r   r   r   r   r   r   bos_filelocal_file_names                   r   download_output_filePipeline.download_output_file  sU    			6tf=> IIh((--((003dd  	II,-**5++>D 	 1 1 6 66IIh//4422::;2d:>N=OP[!W	  		&'h*%\*}-"#45~.)$??3'??3'#- *&}H(

 x(
...''**4+;+;<??c! ggll299;HDO77==)) ggll4??OLDO		5hZvdooEVWX
	))+$BRBR7RTXTcTcdHH+DOO+<=>[ 22 	IIh//4422::;2c!fTcl^L  	\ " 	IIEdFVFVEWXYIIh--2200889CF4CL>J  	s3   6L >AN N
A!NN

P%BP  P%c                    / nUnSnU GH  n [         R                  " S5        [        R                  " U5      nUS   [        R                  R                  S   :w  ah  [         R
                  " [        [        R                  R                  S   [        R                  R                  S    SUS    3US   US	   5      5          g[         R                  " S
5        US   n	U	S   n
U	S   nU	S   nU	S   nU	S   nU	S   n[        [        X5      U
US9n[        U5      nUR                  S5      (       a  [         R"                  R%                  U5      n[         R                  " SU SU S35        US-  n[         R"                  R'                  U5      (       dw  [         R
                  " [        [        R(                  R                  S   [        R(                  R                  S    SU 35      5        [         R*                  " SU 35        GM   [-        UXU5        [         R                  " S5        [        R2                  " U[         R"                  R5                  U5      X5      nU(       a-  UR7                  U5        [         R                  " SU 35        GM  [         R
                  " [        [        R0                  R                  S   [        R0                  R                  S    S35      5        [         R*                  " S5        GM     [         R                  " SU 35        U$ ! [        R                   ar  n[         R
                  " [        [        R                  R                  S   [        R                  R                  S    S[        U5      SS  35      5         SnA  gSnAff = f! [.         a  n[         R
                  " [        [        R0                  R                  S   [        R0                  R                  S    S[        U5      SS  35      5        [         R
                  " SU 35         SnAGM=  SnAff = f)u   
上传文件
   r   r   rN   NrO   rP   rQ   rR   rU   rS   rV   rW   rY   rZ   r[   rX   r   r   u	   上传第u
   个文件[u   ]到BOS路径u$   上传失败（文件不存在）：u   BOS文件上传成功u   BOS文件上传失败：z
add file: u   : 上传文件失败   add file 失败u   所有文件上传结束: )r   r5   
ds_requestbos_acl_dataset_fileRequestDatasetExceptionr>   r   r   rx   valuer   rv   r   r   r   
startswithr7   r8   abspathr9   UPLOAD_FILE_NOT_FOUNDr)   r   r   r{   add_filer|   rl   )r?   r6   filesfile_idslocal_filesilocal_file_pathr   rB   rS   rV   r   r   r   r   r   r   r   file_ids                      r   _upload_filesPipeline._upload_files  s    *O		01!66u= K I$5$5$;$;A$>>		(3399!< 66<<Q?@4
CSBTU%M	  II*,(^Fj)H .K"=1M &'8 9">2Mi(H .*=L!,H
 %X.J))#.."$''///"BII	!J.?}MNFA77>>/22		(3399!< 66<<Q?@?BST  ??PQR
!*kHU		13 !))%1A1A/1RT\jG(		Jwi01		(//55a8 2288;<<PQ  *+I  +J 			.xj9:E 55 		(3399!< 66<<Q?@3q6$3<.Q  ` & 		(//55a8 2288;<Bs1vds|nM  		4_4EFGs1   ,L,#N5,N2 A'N--N25
Q?B QQc           	      @   [         R                  " SU 35        [        UR                  5      S:  aZ  [         R                  " [        [        R                  R                  S   [        R                  R                  S    S35      5        gU R                  5       nU(       dZ  [         R                  " [        [        R                  R                  S   [        R                  R                  S   S-   5      5        gU R                  X!R                  5      nU(       a  [         R                  " S5        UR                  (       a  S	OSn[        R                  " X!R                  UUUR                   UR"                  S
9nU(       a'  [         R$                  " SUR                   SU 35        g[         R                  " S5        g[         R                  " [        [        R&                  R                  S   [        R&                  R                  S    S35      5        g)u   
创建数据集
u   创建数据集，参数: (   r   r   u.   : 数据集名称长度不能超过40个字符NrL   u   创建数据集...   )dataset_typedataset_absdataset_licenseu   数据集创建成功: z id: u   数据集创建失败   : 本地文件上传失败)r   r5   r`   r#   r>   r   r   rc   r   rI   r]   r   r   publicr   create_datasetdescriptionlicenser)   DATASET_CREATION_FAILED)r?   r@   r6   r   r   r   s         r   r   Pipeline.create_datasetb  s    			.tf56tyy>BIIh((..q1++11!455cd   IIh((..q1((..q14ee  %%eZZ8II*+ $1L#225))X@L?C?O?OCG<<QJ 2499+U:,OP		12IIh1177:44::1=>>XY r    c                    [         R                  " SU 35        U R                  5       nU(       dZ  [         R                  " [	        [
        R                  R                  S   [
        R                  R                  S   S-   5      5        gU R                  X!R                  5      nU(       a;  [         R                  " SU 35        [        R                  " X!R                  U5      ng[         R                  " S5        [         R                  " [	        [
        R                  R                  S   [
        R                  R                  S    S35      5        g)	u   
上传数据集文件
u    上传数据集文件，参数: r   r   rL   Nu   add file 成功 r   r   )r   r5   rI   r>   r   r   r]   r   r   r   r   add_files_to_datasetidr   )r?   r@   r6   r   r   s        r   r   Pipeline.add_file  s     			4TF;<IIh((..q1((..q14ee  %%eZZ8II(
3411%(KC II)IIh1177:44::1=>>XY r     N)__name__
__module____qualname____firstlineno____doc__OBJECT_NAMEr   rI   r   r   r   r   r   r   r   r   __static_attributes__r   r    r   r/   r/   :   sL     K?@sl1f 'DM^GROb&Rr    r/   c                  0    [        5       R                  " U 6 $ )r   )r/   r   r@   s    r   r   r     s    :  $''r    c                  0    [        5       R                  " U 6 $ )r   )r/   r   r   s    r   r   r     s    :d##r    c                  0    [        5       R                  " U 6 $ )r   )r/   r   r   s    r   r   r     s    :T""r    c                  0    [        5       R                  " U 6 $ r   )r/   r   r   s    r   r   r     s    :??D!!r    c                  0    [        5       R                  " U 6 $ r   )r/   r   r   s    r   r   r     s    :''..r    c                  0    [        5       R                  " U 6 $ r   )r/   r   r   s    r   r   r     s    :**D11r    c                  0    [        5       R                  " U 6 $ r   )r/   r   r   s    r   r   r     s    :$$d++r    c                  0    [        5       R                  " U 6 $ r   )r/   r   r   s    r   r   r     s    :%%r    )0r   r7   pathlibr   prettytabler   aistudio_sdkr   r   aistudio_sdk.constant.err_coder   aistudio_sdk.constant.constr   r	   r
   r   aistudio_sdk.utils.utilr   r   aistudio_sdk.utils.bos_sdkr   r   r   aistudio_sdk.requestsr   rm   r   r   !baidubce.bce_client_configurationr   baidubce.auth.bce_credentialsr   baidubce.exceptionr    baidubce.services.bos.bos_clientr   __all__r   r-   r/   r   r   r   r   r   r   r   r   r   r    r   <module>r     s    
  # $ 4 Z Z 5 R R 8 7 D 8 1 6Y
 g	 g	R($#"/2,&r    