9i5zSSKJrJrJrJrJr SSKrSSKJr SSK J r SSK J r SSK Jr SSKJrJr SSKJr SS KJrJr SS KJrJr SS KJr \"5r\ R:"\R<\R>S 9"S S\55r \ R:"\R<\RBS 9"SS\55r"g))AnyDictListTupleUnionN) Preprocessors) Preprocessor) PREPROCESSORS)"TextClassificationPreprocessorBase)NLPTokenizerForLSTM#TokenClassificationPreprocessorBase) NLPTokenizer)FieldsModeKeys)get_model_typeparse_label_mapping) get_logger) module_namec^\rSrSrS SjrSSSSS\R SSS4 S\S\S\\\ 4S\ S\S \ S \ 4U4S jjjr S rU=r$)/SpeakerDiarizationDialogueDetectionPreprocessorNc ~SU;a%UR[R:XaSOSUS'UR"X40UD6$)Nreturn_tensorspt)moder INFERENCE nlp_tokenizer)self sequence1 sequence2kwargss `/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/preprocessors/speaker.py_tokenize_text>SpeakerDiarizationDialogueDetectionPreprocessor._tokenize_textsD 6 ),0II9K9K,KDQU   "!!)A&AAlabelfirst_sequencesecond_sequencelabel2idr max_lengthuse_fastc >U RSS5U S'U RSS5U S'UbUOU RSS5U S'U RSS5 Sn Ub [U5n [XXS9Ul[ T U]XX4XVU 5 g)N truncationTpaddingr*sequence_length)r+tokenize_kwargs)getpoprrrsuper__init__) r model_dirr'r(r&r)rr*r+keep_original_columnsr! model_type __class__s r"r58SpeakerDiarizationDialogueDetectionPreprocessor.__init__!s &zz,=|"JJy,?y*4*@Jfjj!3G(     $d+  ' 2J) HN O!)> @r%r)N)__name__ __module__ __qualname____firstlineno__r#rrstrrrrintboolr5__static_attributes__ __classcell__r9s@r"rrs B '+(,+2"&%//#'"&'+@!$@#&@c4i( @  @  @!@ @@r%rc ^\rSrSrSSSSS\R SSSS4 S\S\S\S \S \S \4 U4S jjjr S\ \\ \44S jr Sr SrSrSrSrSrU=r$):SpeakerDiarizationSemanticSpeakerTurnDetectionPreprocessor;Ntextr&FTr6r'r)label_all_tokensrc *>[T U]XX4XVU U 5 Sn Ub [U5n U RSS5U S'U RSS5U S'UbUOU RSS5U S'U R SS5 U S:gU S'[ UU UU S 9Ulg) Nr-Tr.r*r/r0lstmadd_special_tokens)r6r8r+r1)r4r5rr2r3r r)rr6r'r&r)rJrr*r+r7 return_textr!r8r9s r"r5CSpeakerDiarizationSemanticSpeakerTurnDetectionPreprocessor.__init__@s E)1F$ &  ' 2J%zz,=|"JJy,?y*4*@Jfjj!3G(     $d+'1V';#$0!" $r%c UnUR[R:wa[U[5(dS5eUR R SS5nU(aURS5nUS:XdUR(a [ U5nOQ/nUR[ USU55 URS5 UR[ X5S-S55 UnU(a4UR[R:XaUR"U40UD6upxOPUR RR(aUR"U40UD6upxOUR"U40UD6upxSn[!US5H,upXR RR":XdM*U n O US:wa'[%U['US55H n SUSU 'M UR[R:XaCUR)5H,n [*R,"X|5R/S 5X|'M. Xx4$UR1S S5 Xx4$) NzsInput needs to be lists in training and evaluating,because the length of the words and the labels need to be equal.is_split_into_wordsFz[SEP] input_ids label_maskroffset_mapping)rrr isinstancelistrget_tokenizer_kwargfind is_lstm_modelextendappend_tokenize_text_by_words tokenizeris_fast"_tokenize_text_with_fast_tokenizer"_tokenize_text_with_slow_tokenizer enumerate sep_token_idrangelenkeystorchtensor unsqueezer3) rrIr!tokensrQsep_idx tmp_tokens encodingsword_idsidxtoken_idikeys r"r#ISpeakerDiarizationSemanticSpeakerTurnDetectionPreprocessor._tokenize_text`s/ 99** *fd++ S.S S+"00DD !5* kk'*G"} 2 2f !!$vhw'7"89!!'*!!$vkl';"<=# 4990B0B#B"&">">#" #" Ix    ) ) 1 1"&"I"I#" #" Ix#'"I"I#" #" I&y'=>MC--77DDD? b=7C ,(?$@A-2 ,'*B 99** * ~~'!&in!=!G!G!J ("" MM*D 1""r%c /n/n/n/n[U5HupxURRRUSS9n [ U 5S:Xa!URRR /n UR U 5 UR S/[ U 5-5 UR S/S/[ U 5S- --5 UR XwS-4/5 M URSURRS55n URSURSURRS555n URRS 5(aSOSn [ U5U S U -- :aUSU S U -- nUSU S U -- nUS[U5nU S:XaS/U -U-S/U [ U5- U - --nUS /U [ U5- --nURRR/U -U-URRR/U --URRR/U [ U5- S U -- --nUS/U S ---S/U [ U5- S U -- --nOhS/U -U-S/U --nURRR/U -U-URRR/U --nUS/U S ---nUUUUS .n U S4$) NF)rMrTr.r*r/rMrr)rTattention_maskrUrV) rcrr_encoderf unk_token_idr\r2rYsum cls_token_idrd pad_token_id)rrkr!rTrUrVryoffsettoken subtoken_idsr.r* special_tokenrns r"r^RSpeakerDiarizationSemanticSpeakerTurnDetectionPreprocessor._tokenize_text_by_wordssr  &v.MF--77>>%?1L< A% $ 2 2 < < I IJ   \ *  ! !1#L(9"9 :   tfw#l2Ca2G'HH I  ! !FQJ#7"8 9/**Y!//CCINPZZ  JJ())==lK MN "//CC ""'(  z?Z!m*;; ;#$Ej1}3D&DFJ!"CJ]1B$BDI'(8Z9 l "=0:=:J7-GHIJ+vhS00/22N++55BBCmSV__##--::;mKL##--::;zCPYN?Z]^an]n?nopI,qc!/##&'S^!44q=7HH&JJN =0:=-'(J++55BBCmSV__##--::;mKLI+qc]Q5F.GGN#,$,  $r%c [U[5nUR"U4SUS.UD6n/nUR5n/n[ [ U55HnXhcUR S5 MXhXhS- :Xa2UR S5 U(dUSSUSUS4US'MXMZUR S5 U(aUR XhXhS-45 MUR USU5 M URRS5n U S :XaUS /[ U5[ U5- --nXtS'XTS 'XF4$) NT)return_offsets_mappingrQFrvrRrrVr.r*rxrU)rWrXrrorerfr]rY) rrkr!rQrnrUrorVrrr.s r"ra]SpeakerDiarizationSemanticSpeakerTurnDetectionPreprocessor._tokenize_text_with_fast_tokenizersr(6&& #' 3   %%'s8}%A{"!!%(Q/!!%(**8*, please use a fast tokenizer instead, or try to implement a `z` methodr.r*rMrvrrwrxrVrU)rrrrWr@rget_tokenizer_classhasattr RuntimeErrorgetattrr2rYrfr|) rrkr!rorntokenizer_namemethodrUrVr.r*rs r"rb]SpeakerDiarizationSemanticSpeakerTurnDetectionPreprocessor._tokenize_text_with_slow_tokenizersyyH...:fc3J3J 8 8 8J&& 9(-9179 ++??A5Ft$$vh+,-''-hh89 9&-T%:6%B" **Y!//CCINPZZ $,,@@NP #ZZ    2 2$ &''-.  z?Z!m*;; ;#$Ej1}3D&DFJ'(8Z9 l "=0:=:J7-GHIJ+vhS00/22N =0:=-'(J&4"#",,""r%cx/n/nURRRU5nSnUHnUSSS:gnU(aURS5 OUSSnURS5 XQUSR U5-nU[ U5-n U(aURX45 O USSU 4US'U nM X#4$)Nrrwz##TFrR)rr_tokenizer]indexrf) rrIrUrVrkrris_startstartends r"/get_label_mask_and_offset_mapping_BertTokenizerjSpeakerDiarizationSemanticSpeakerTurnDetectionPreprocessor.get_label_mask_and_offset_mapping_BertTokenizer s ##--66t<Ebq T)H!!$'ab !!%(&']0077E#e*$C%%ul3&4R&8&;S%Ar"F))r%c/n/nURRRU5nSnSnUHnUSS:HnU(a*USSnURS5 [ U5S:XaSnM;OURS5 XQUSR U5-n U [ U5-n U(dU(aURX45 O USSU 4US'U nSnM X#4$)NrF_rvTrR)rr_rr]rfr) rrIrUrVrkr last_is_blankrrrrs r"5get_label_mask_and_offset_mapping_XLMRobertaTokenizerpSpeakerDiarizationSemanticSpeakerTurnDetectionPreprocessor.get_label_mask_and_offset_mapping_XLMRobertaTokenizer"s ##--66t< EaCHab !!$'u:?$(M#!!%(&']0077E#e*$C%%ul3&4R&8&;S%Ar"F!M#$))r%r;)r<r=r>r?rrr@rrBr5rrr#r^rarbrrrCrDrEs@r"rGrG;s #''-%"&*/%// '+!$$!$$$ $ $( $  $$@,#5d3i#8,#\4l#B%#N*,**r%rG)#typingrrrrrrhmodelscope.metainformodelscope.preprocessorsr modelscope.preprocessors.builderr =modelscope.preprocessors.nlp.text_classification_preprocessorr >modelscope.preprocessors.nlp.token_classification_preprocessorr r 3modelscope.preprocessors.nlp.transformers_tokenizerrmodelscope.utils.constantrrmodelscope.utils.hubrrmodelscope.utils.loggerrloggerregister_moduleaudiosen_cls_tokenizerrtoken_cls_tokenizerrGr%r"rs10 -1:'>L6D.  LLm==? @* @? @F LLm??A}*+}*A}*r%