
    9i                     R    S SK r S SKrS SKrS SKrSS jrSS jrS r " S S5      rg)    Nc                    0 n[         R                  " 5       US'   [        R                  R	                  5       US'   [        R
                  " 5       US'   U b2  U R                  S:X  a"  [         R                  R                  U 5      US'   U$ Nrng_state_torchrng_state_nprng_state_rndcudarng_state_torch_cuda)torchget_rng_statenprandom	get_stategetstatetyper   )devicerandom_statess     d/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/models/nlp/peer/sas_utils.pyget_random_statesr      sy    M',':':'<M#$$&II$7$7$9M.!%+__%6M/"fkkV305

0H0H1,-     c                    [         R                  " U S   5        [        R                  R	                  U S   5        [        R
                  " U S   5        Ub4  UR                  S:X  a#  [         R                  R                  U S   5        g g g r   )r
   set_rng_stater   r   	set_statesetstater   r   )r   r   s     r   set_random_statesr      so    	&789IIn56
OOM/23fkkV3

  /E!FG 4r   c                    U c  g SS/n[         R                  " U 5      (       a]  [         R                  " U 5      R                  5       (       a  SUS'   [         R                  " U 5      R                  5       (       a  SUS'   O[        U 5      [        L a  [        [        U 5      5       H  n[         R                  " X   5      (       d  M"  [         R                  " X   5      R                  5       (       a  US==   S-  ss'   [         R                  " X   5      R                  5       (       d  M  US==   S-  ss'   M     US   S:  a  US==   S-  ss'   US   S:  a  US==   S-  ss'   [        U5      S:  a  U$ S $ )Nr      
   )
r
   	is_tensorisnananyisinfr   tuplerangelensum)dataresultis      r   check_nan_infr)   %   s*   |VFt;;t  ""F1I;;t  ""F1I	du	s4y!Atw'';;tw'++--1INI;;tw'++--1INI " !9q=1IOI!9q=1IOI[1_6.$.r   c                   *    \ rS rSrSS jrS rS rSrg)SequenceSideInfo@   Nc           
         Ub  Xl         OSSKJn  UR                  S5      U l         [        R
                  R                  R                  5       U l        [        U R                   R                  5       Vs/ s H  nU R                   R                  U/5      PM!     nn[        [        [        U5      5       Vs/ s H  o4U   SS S:X  d  M  UPM     sn5      U l        U Vs/ s HP  nUS   S:X  a  US   S:X  a  SO8USS S:X  a  S	[        S
[        U5      S-
  5      -   O[        S	[        U5      5      PMR     nn[         R"                  " U[         R$                  S9U l        g s  snf s  snf s  snf )Nr   )ElectraTokenizerzgoogle/electra-small-generator   z##[]r      dtype)	tokenizertransformersr.   from_pretrainednltktokenizepunktPunktSentenceTokenizersen_tokenizerr#   
vocab_sizedecodesetr$   ind_subtokensminr
   tensorint8
len_tokens)selfr6   r.   r(   tokensttmps          r   __init__SequenceSideInfo.__init__B   s`    &N5-==02DN "]]00GGI 4>>445
5 NN!!1#&5 	 
 !c&k*E*1Qi!n.DQ*EG 	
  12#A%&qVt^ #a!fqj" "9<RQJ 	 	 
  ,,s%**=

 F
s   5&E#8E(E(!AE-c                    U R                   R                  U5      n[        R                  " U R                  R                  U5      S    Vs/ s H  n[        U5      S-
  PM     sn5      nX6SS  R                  5       -
  US'   [        R                  " [        [        U5      5       Vs/ s H+  nU[        R                  " Xg   [        R                  S9-  PM-     sn5      n[        R                  " [        [        U5      5       Vs/ s H(  n[        R                  " Xg   [        R                  S9PM*     sn5      n	[        R                  " X45      $ s  snf s  snf s  snf )N	input_idsr/   r   r   r4   )r=   r:   r   arrayr6   batch_encode_plusr$   r%   concatenater#   onesrD   arange)
rF   sentence_position_embedding
inputs_strseq_len_total	sentencesxsen_lengthsr(   idx_senidx_toks
             r   getSenTokIdxSequenceSideInfo.getSenTokIdxZ   s,   &&//
;	hh^^55i@M 
M FQJM 
 
 'QR)<)<)>>A..3{+,"
, bgg66,"
  ..3{+,"
, IIknBGG4,"
 
 ~~w011! 
"
"
s   E)2E/Ec                 f   Sn[        US   [        [        R                  45      (       a  Sn[        R
                  " U5      n[        U R                  S5      (       a}  U R                  R                  U5      n[        R
                  " [        R                  " U Vs/ s H"  nU R                  XUR                  S   5      PM$     sn5      UR                  S9nO[        R
                  " [        R                  " UR                  5        Vs/ s H<  nU R                  UU R                  R                  U5      UR                  S   5      PM>     sn5      UR                  S9n[        5       nUR                  S   n	US S 2SU	24   US'   US S 2SU	-  SU	-  24   US	'   US:  GaH  [        R                   " USS
9u  pU R"                  R%                  ['        U
5      5      n[)        U5      S:  aE  [        R*                  " U Vs/ s H  oU:H  PM	     sn5      R-                  SS9R/                  5       nO-[        R0                  " UR                  [        R2                  S9nSUS S 2S4'   US S 2SS 24   n[5        SS5       HB  n[        R6                  " UU:H  US S 2SS24   U:H  5      n[)        U5      S:X  a    OUS-   UU'   MD     XS'   U R8                  UR;                  5          nUUS'   U(       a-  UR=                  5        H  nUU   R                  5       UU'   M     U$ s  snf s  snf s  snf )NFr   Tbatch_decoder   )r    ss_sentence_position_in_sequencer/   ss_token_position_in_sentence)return_inverse)axisr4      r1   ss_token_position_in_whole_wordss_token_string_length)
isinstancelistr   ndarrayr
   rC   hasattrr6   r^   rN   r[   shaper   numpyr?   dictuniquerA   intersectionr@   r$   stackr    charzerosrD   r#   logical_andrE   longkeys)rF   rS   	inputs_idis_np_arrayrT   	input_strsen_tok_idx	input_oriside_info_dict
seq_lengthrm   _rA   st
idx_tok_wwidx_tok_ww_1r(   posinputs_str_lenkeys                       r   generate_seq_side_info'SequenceSideInfo.generate_seq_side_infoo   s    ilT2::$677KY/I4>>>2244Y?J,, &0 &0	 %%&A&/ooa&8:%/ 
 !'')K  ,, &/__%6	 &7	 %%&A&*nn&;&;I&F&/ooa&8: &7	  !'')K __Q'
2=a:o2N 	.	0 0;1a*nQ=GNH ?H <H 0I 	+	- '!+		)DAIF ..;;CKHM=!A%"[[.;*.;Om* 3A3;ttv  #[[

K
 Jq!t%ae,L1b\''(9(21ad7(;q(@Bs8q=$%ES! " AK<=!__Y^^-=>N7EN34%**,&4S&9&?&?&As# - k,*s   )L$AL)L.)rA   rE   r=   r6   N)__name__
__module____qualname____firstlineno__rJ   r[   r   __static_attributes__ r   r   r+   r+   @   s    >02*>r   r+   r   )	r   r9   rk   r   r
   r   r   r)   r+   r   r   r   <module>r      s-       
H /6m mr   