
    9i9                        S SK r S SKrS SKrS SKrS SKrS SKJr  S SKJrJ	r	  S SK
r
S SKrS SKrS SKrS SKJr  S SKJr  S SKJr  S SKJs  Js  Js  Jr  S SKJr  S SKJrJr  S SK J!r!J"r"J#r#  S S	K$J%r%  S S
K&J'r'  S SK(J)r)J*r*  S SK+J,r,  S SK-J.r.  S SK/J0r0  S SK1J2r2  \2" 5       r3\,Rh                  " \.Rj                  \R2                  S9 " S S\*5      5       r6g)    N)partial)AnyDict)VideoCap)Image)	Pipelines)CenterCropV3random_resize)make_irregular_maskmake_rectangle_maskmake_uncrop)	rand_name)
OutputKeys)InputPipeline)	PIPELINES)Tasks)device_placement)
get_logger)module_namec                      ^  \ rS rSrSrS\4U 4S jjrS\S\\\	4   4S jr
S\\\	4   S\\\	4   4S jrS	\\\	4   S\\\	4   4S
 jrS r    SS jrSrU =r$ )VideoComposerPipeline$   a  Video Composer Pipeline.

Examples:

>>> from modelscope.pipelines import pipeline
>>> from modelscope.utils.constant import Tasks
>>> pipe = pipeline(
        task=Tasks.text_to_video_synthesis,
        model='buptwq/videocomposer',
        model_revision='v1.0.1')
>>> inputs = {'Video:FILE': 'path/input_video.mp4',
              'Image:FILE': 'path/input_image.png',
              'text': 'the text description'}
>>> output = pipe(inputs)
modelc           	        > [         TU ]  US9  UR                  SS5      U l        [        R
                  R                  U R                  5      (       d   [        R                  " U R                  5        UR                  SS5      U l        UR                  S/ SQ5      U l	        UR                  SS/5      U l
        UR                  S	S
S
S
S
S.5      U l        [        U R                  5      n[        U R                  5      nU R                  SX4-  -  U-     U l        U R                  [        U R                  5         U l        UR                  SS5      U l        UR                  SS5      U l        UR                  S/ SQ5      U l        UR                  S/ SQ5      U l        UR                  SS5      U l        UR                  S/ SQ5      U l        UR                  S/ SQ5      U l        UR                  SS5      U l        UR                  SS5      U l        UR                  SS5      U l        UR                  SS5      U l        [6        R8                  " [6        R:                  " U R                   S 9[6        R<                  " 5       [6        R>                  " U R$                  U R&                  S!9/5      U l         [6        R8                  " [B        RD                  " [G        [H        U R.                  S 95      [6        R:                  " U R.                  5      [6        R<                  " 5       /5      U l%        [6        R8                  " [B        RL                  " U R                   S 9[B        RN                  " U R                   5      /5      U l(        [B        R8                  " [S        U R(                  5      [B        R<                  " 5       [B        R>                  " U R*                  U R,                  S!9/5      U l*        g")#zl
use `model` to create a videocomposer pipeline for prediction
Args:
    model: model id on modelscope hub.
r   log_dirz./video_outputsfeature_framerate   
frame_lens)   r!   r!   r!   feature_frameratesbatch_sizes   )14816r   
resolution   image_resolutionmean)      ?r-   r-   stdvit_image_size   vit_mean)g3<4'?gwgM?gy{ ?vit_std)gB91?gwt.?g	U?z
kwargs.popi  	visual_mvF	max_wordsi  
mvs_visual)size)r,   r.   N)+super__init__popr   ospathexistsmakedirsr   r    r"   r#   len
max_framesstr
batch_sizer)   r+   r,   r.   r/   r1   r2   	misc_sizer3   r4   r5   dataComposeCenterCropV2ToTensor	Normalizeinfer_transTLambdar   r
   misc_transformsResize
CenterCropmv_transformsr	   vit_transforms)selfr   kwargsl1l2	__class__s        w/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/pipelines/multi_modal/videocomposer_pipeline.pyr8   VideoComposerPipeline.__init__7   s    	u%zz)->?ww~~dll++KK%!',?!C **\ 4
  #)**-AD
 # "::m	6
  !(()//!rw-2*=>**3t+?@ **\37 &

+=s CJJv7	::e_5$jj)93?

:#FHzz)"FHL#6K7K6 **\59<<4??3MMONN		txx8)
   $||HHW]@Adnn-MMO-
   "\\XX4??+\\$//*,-  ii,,-JJLKKT]]=)
     inputreturnc           	         US   nUS   nUS   nS nU R                   n[        R                  R                  U5      (       a.   U R	                  X R                   UU R
                  5      u  pxpnGO[        R                  S	5        [        R                  " SU R                  U R                  5      n[        R                  " SU R                  U R                  5      n[        R                  " U R                  SU R                  U R                  5      n	[        R                  " U R                  SU R                  U R                  5      n
[        R                  " U R                  SU R                  U R                  5      n[         R                   " 5       nUS
:  a  [#        SS5      nOUS:  a  [%        SS5      nO['        SS5      n[        R(                  " [*        R,                  " XR                  U R                  4[*        R.                  S95      R1                  S5      R3                  5       nUR1                  S5      R5                  U R                  SS9nUR1                  S5      UU	R1                  S5      U
R1                  S5      UUR1                  S5      UR1                  S5      US.nU$ ! [         a  n[        R                  SR                  X,5      SS9  [        R                  " SU R                  U R                  5      n[        R                  " U R                  SU R                  U R                  5      n	[        R                  " U R                  SU R                  U R                  5      n
[        R                  " U R                  SU R                  U R                  5      n S nAGN6S nAff = f)Nz
Video:FILEtextz
Image:FILEz&{} get frames failed... with error: {}Tflush      z7The video path does not exist or no video dir provided!gffffff?i   g?)interpolationr   )repeatsdim)	ref_framecap_txt
video_data	misc_datar   maskmv_datastyle_image)r   r:   r;   r<   video_data_preprocessr5   	Exceptionloggerinfoformattorchzerosr/   r?   r+   rB   randomr   r   r   
from_numpycv2resizeINTER_NEAREST	unsqueezefloatrepeat_interleave)rP   rX   	video_keyrd   ri   total_framesr   rc   	vit_imagere   rf   rh   e_prg   video_inputs                    rU   
preprocess VideoComposerPipeline.preprocessy   s   ,'	-L) 2277>>)$$=GKGaGa55|OOH%D	jW* KKIKAt':':$($7$79IAt22D4G4GHAT__a9N9N%)%:%:<JDOOQ$(NN4Ikk$//1d6K6K"&"7"79G MMOs7&sC0DW&sC0DsC(DJJ~~t~~6!//12 3<)A,uuw 	
 ~~a 22OO 3 , #,,Q/$..q1",,Q/!2NN1%((+&	
 o  =<CC!&   
 "KK4+>+>(,(;(;=	"[[!)-)>)>)-)>)>@
 "KKDNN(,8	  ++dooq&*&;&;&*&;&;==s   +J4 4
N=>C4N88N=c                 $    U R                  U5      $ )Nr   )rP   rX   s     rU   forwardVideoComposerPipeline.forward   s    zz%  rW   inputsc                     UR                  SS 5      nSnUb   [        R                  " SS9R                  nSnU(       a  [        R
                  US   0$ [        R
                  US   0$ )Noutput_videoF.gif)suffixT
video_pathvideo)gettempfileNamedTemporaryFilenamer   OUTPUT_VIDEO)rP   r   post_paramsoutput_video_pathtemp_video_files        rU   postprocess!VideoComposerPipeline.postprocess   sh    'OONDA( ( ; ;6 J O O"O++VL-ABB++VG_==rW   c                 H   Un[        S5       H  n U R                  UUUS9u  pxp  O   [        W5      n[        R                  " [        R                  " U5      S:H  U[        R                  " U5      -
  U R                  :  -  5      S   n[        R                  R                  U5      n[        R                  " XU R                  -   5      nU Vs/ s H)  n[        R                  " WU   S S 2S S 2S S S24   5      PM+     nnU Vs/ s H+  n[         R"                  " W	U   R%                  S	5      5      PM-     n	n[         R&                  " U	5      n	U(       a  U Vs/ s H$  nW
U   S S 2S S 2S S S24   R)                  S
5      PM&     nnU R*                  S-   UR-                  S5      S   -   S-   n[.        R0                  R3                  U R*                  S-   5      (       d"  [.        R4                  " U R*                  S-   SS9  [        R7                  SU5        [8        R:                  " UUSS9  [        U5      S:  n[=        [        U5      S-  5      nU(       ac  UU   nU R?                  U5      nU R@                  S S " U5      nU R@                  SS  " U5      nU RC                  U5      nU RE                  U	5      n	O,[         RF                  " SU RH                  U RH                  5      n[         RF                  " U R                  SU RJ                  U RJ                  5      n[         RF                  " U R                  SU RJ                  U RJ                  5      n[         RF                  " U R                  SU RL                  U RL                  5      nU(       a6  UUS [        U5      2S4'   WUS [        U5      2S4'   U	US [        U5      2S4'   UnAAA	UUUUU4$ ! [         a/  n[        R	                  SR                  X5      SS9   S nAGM  S nAff = fs  snf s  snf s  snf )N   )input_videofpsr3   z={} read video frames and motion vectors failed with error: {}Tr\   Ir   )r_   r   r$   uint8z/visual_mv//r   exist_okz&save motion vectors visualization to :   )r   r_   r^   .)'rangeextract_motion_vectorsrk   rl   errorrn   r>   npwherearrayaranger?   rq   choicer   	fromarrayro   rr   	transposestackastyper   splitr:   r;   r<   r=   rm   imageiomimwriteintrO   rK   rH   rN   rp   r/   r+   rB   )rP   ry   r   rz   r3   filenamer}   frame_typesframesmvsr5   r|   start_indexsstart_indexindicesiimagesr;   have_framesmiddle_indixrc   r{   misc_imgs_np	misc_imgsre   rh   rf   s                              rU   rj   +VideoComposerPipeline.video_data_preprocess   s    qA
 7;7R7R ()' 8S 8)4S   ;'xx+!6#!=299\22dooE!G HHIKii&&|4))Kt)FG CJJ'Q%//&)Aq$B$J"78'JFMNguA 0 0 ;<gNkk#&(&! "!}Q4R4Z088A&  (<<-/)//33 !"D77>>$,,">??DLL=84HKK@$GT6q1&kAo3v;?+|,I++I6I//3F;L,,QR0>I%%f-F$$S)CAt':':$($7$79I [[!T5J5J!%!6!68
++dooq$2G2G"335KKDNN $0	,2J|F|S()+4Ils6{lC'(),GLS[L#%&	)ZGCCs   SVI)       KN(s)   O0P2P+P
P##PPc                 R
   U(       ab  [         R                  5       R                  S5      nS H9  n[        R                  " [        R
                  R                  SU 3U5      SS9  M;     [        5       n[        R
                  R                  UR                  S5      S   U UR                  S5      S   -   5      n	[        R                  " U5      n
U
R                  [        R                  5      nU
R                  [        R                  5      nX-  U-  S	:  a  [        US
5      nO[        S	X-  -  5      S
-   nSU SU SU	 3n[        R
                  R!                  U	5      (       a  [        R"                  " U	5        [$        R&                  " USSS9  [)        5       nUR+                  U	5      nU(       d  [-        SU	 35      eSn/ n/ n/ n/ n/ n U(       a  [.        R1                  SUSS9  [2        R4                  " 5       nUR7                  5       u  nnnnn[2        R4                  " 5       nUU-
  nUR9                  U5        U(       d  U(       a  [.        R;                  S5        GO>[<        R>                  " URA                  5       RB                  [<        RD                  S9nU(       a  [G        UU5      nSnURB                  S
   URB                  S   :  a  URB                  S
   URB                  S   -
  S-  nU(       aC  [        RH                  " [        R
                  R                  SSU S35      US S 2UU* 24   5        UR9                  US S 2UU* 24   5        OURB                  S   URB                  S
   -
  S-  nU(       aC  [        RH                  " [        R
                  R                  SSU S35      UUU* 2S S 24   5        UR9                  UUU* 2S S 24   5        URB                  S S u  n n![<        R>                  " U U!S45      n"US S 2SS24   RK                  SU!S
-
  U S
-
  45      n#US S 2SS
24   US S 2SS24   -  US S 2SS 24   -  U"U#S S 2S
4   U#S S 2S4   4'   US
-  nUR9                  U5        UR9                  U5        UR9                  U"5        GM  U(       a*  [.        R1                  S[<        RL                  " U5      5        URO                  5         [        R
                  R!                  U	5      (       a  [        R"                  " U	5        UUUU4$ ) Nz%Y-%m-%dT%H:%M:%S)r   motion_vectorszout-Tr   r   r   r   r!   r$   z%ffmpeg -threads 8 -loglevel error -i z' -filter:v                         fps=z -c:v mpeg4 -f rawvideo x   )argsshelltimeoutzCould not open zFrame:  )endzNo frame read. Stopping.)dtypeFr_   z./mv_visual/zframe-z.jpgr      )r   r   	   zaverage dt: )(datetimenowstrftimer:   r=   r;   joinr   r   rs   VideoCapturer   CAP_PROP_FRAME_COUNTCAP_PROP_FPSmaxr   r<   remove
subprocessrunr   openRuntimeErrorrl   rm   timeperf_counterreadappendwarningr   rp   copyshaper   draw_motion_vectorsimwriteclipr,   release)$rP   r   r   dumpverboser3   r   childtemp	tmp_videovideocapture
frames_num	fps_video
ffmpeg_cmdcapretsteptimesr   r   r   r5   tstartframer   
frame_type	timestamptendtelapsed
frame_savew_halfh_halfhwmvpositions$                                       rU   r   ,VideoComposerPipeline.extract_motion_vectors  s    ,,.))*=>C5BGGLL4uu=M 6{GGLLc"1%$;3D3DS3I"3M'MO	''4!%%c&>&>?
 $$S%5%56	!C'",c1+CbJ234q8C<[M J E!9)F
 77>>)$$IIi JdC@jhhy!<==
It5&&(F AD
=C
I$$&Df}HLL" NN#=>%**,"4"4BHHEJ0^L
 D{{1~Q/++a.5;;q>9a?KK^vdV45HI"1ffWn#457 !!*Qw->"?@++a.5;;q>9a?KK^vdV45HI"66'>1#457 !!*VVG^Q->"?@;;r?DAq1a)$B%a1f-226AE1q5>JH -Q-.1/ ./ 02@AABEC BC 3DDFTUVUVUWVX GYY x1~   AIDz*MM% JJrNk l KK777>>)$$IIi FC33rW   )rA   r#   r   r"   r    r+   rH   r   r?   r4   r,   rB   rK   rN   r5   r)   r.   r3   r/   r1   r2   rO   )r   FFF)__name__
__module____qualname____firstlineno____doc__r@   r8   r   r   r   r   r   r   rj   r   __static_attributes____classcell__)rT   s   @rU   r   r   $   s     @c @DD D$sCx. DL!T#s(^ !S#X !>$sCx. >&*38n>DDP $%$)',).i4 i4rW   r   )7r:   rq   r   r   r   	functoolsr   typingr   r   rs   r   numpyr   ro   torchvision.transforms
transformsrI   mvextractor.videocapr   PILr   0modelscope.models.multi_modal.videocomposer.datamodelsmulti_modalvideocomposerrC   modelscope.metainfor   ;modelscope.models.multi_modal.videocomposer.data.transformsr	   r
   ;modelscope.models.multi_modal.videocomposer.ops.random_maskr   r   r   7modelscope.models.multi_modal.videocomposer.utils.utilsr   modelscope.outputsr   modelscope.pipelines.baser   r   modelscope.pipelines.builderr   modelscope.utils.constantr   modelscope.utils.devicer   modelscope.utils.loggerr   rl   register_moduletext_to_video_synthesisr    rW   rU   <module>r     s    
       
    " )  ? ? )!; ; M ) 5 2 + 4 .	 	!!y/F/FHX4H X4HX4rW   