9iSSKJrJrJrJr SSKrSSKrSSKrSSKJ s J r SSK J r SSKJ r SSKJrJr SSKJrJr \R("\5rSrS r\\RRR2\S .r"S S \ R65r"S S\ R65r"SS\ R65r"SS\ R65r"SS\ R65r "SS\ R65r!"SS\ R65r""SS\ R65r#"SS\ R65r$"SS\ R65r%"SS \ R65r&"S!S"\ R65r'"S#S$\ R65r("S%S&\(5r)"S'S(\ R65r*"S)S*\ R65r+"S+S,\(5r,"S-S.\(5r-"S/S0\RR65r.g)1)absolute_importdivisionprint_functionunicode_literalsN)mpu)nn)normal_init_methodscaled_init_method) PlugNLGConfig PlugNLUConfigcnUS-S[R"U[R"S5- 5--$)zImplementation of the gelu activation function. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) g??g@)torcherfmathsqrtxs c/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/models/nlp/plug/backbone.pygelur"s. s7cEIIa$))C.&899 ::c4U[R"U5-$N)rsigmoidrs rswishr*s u}}Q r)rrelurc2^\rSrSrSU4SjjrSrSrU=r$) BertLayerNorm1c>[[U] 5 [R"[ R "U55Ul[R"[ R"U55Ul X l g)zWConstruct a layernorm module in the TF style (epsilon inside the square root). N) superr__init__r Parameterronesweightzerosbiasvariance_epsilon)self hidden_sizeeps __class__s rr#BertLayerNorm.__init__3sJ mT+-ll5::k#:; LL[!9: #rcURSSS9nX- RS5RSSS9nX- [R"X0R-5- nUR U-UR -$)NT)keepdim)meanpowrrr)r&r()r*russ rforwardBertLayerNorm.forward;sh FF2tF $ UKKN  D  1 Uejj%:%:!:; ;{{Q**r)r(r)r&)g-q=__name__ __module__ __qualname____firstlineno__r#r7__static_attributes__ __classcell__r-s@rrr1s$++rrc6^\rSrSrSrU4SjrSSjrSrU=r$)BertEmbeddingsBzLConstruct the embeddings from word, position and token_type embeddings. c b>[[U] 5 [R"UR UR [SURS9S9Ul [R"URUR 5Ul [R"URUR 5UlUR UlUR"UlUR$Ul['UR UR(S9Ul[R,"UR.5Ulg)Nr3std) init_methodr,)r"rBr#rVocabParallelEmbedding vocab_sizer+r initializer_rangeword_embeddingsr Embeddingmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddingsfp32_layernormfp32_embeddingfp32_tokentypesrlayernorm_epsilon LayerNormDropouthidden_dropout_probdropoutr*configr-s rr#BertEmbeddings.__init__Fs nd,."99      *f668 9 $&<<0N0N060B0B$D %'\\&2H2H282D2D&F" %33$33%55&   F$<$<>zz&"<"<= rcURS5nUcN[R"U[RURS9nUR S5R U5nUc[R"U5nURU5nURU5nURU5nUR(dXV-U-nUR(a!UR(dUR5nUR5n UR(aUR!5nUR#U5nUR(a3UR(aUR5nOURU 5nOUR!5UR!5-UR!5-nUR(a!UR(dUR5nUR5n UR(aUR!5nUR#U5nUR(a3UR(aUR5nOURU 5nUR%U5nU$)Nr )dtypedevicer)sizerarangelongr` unsqueeze expand_as zeros_likerMrPrRrUrTrShalftypefloatrWrZ) r* input_idstoken_type_ids position_ids seq_lengthwords_embeddingsrPrR embeddings previous_types rr7BertEmbeddings.forward[s^^A&   <<%**Y5E5EGL'11!4>>yIL  !"--i8N// :"66|D $ : :> J##)?BWWJ""4+>+>'__. &OO-M""'--/  3J""&&!+!2J!+!?J)//14G4M4M5%++-.J##D,?,?'__. &OO-M""'--/  3J""''!+!2J!+!?J\\*- r)rWrZrTrSrUrPrRrM)NN r:r;r<r=__doc__r#r7r>r?r@s@rrBrBBs>*))rrBc.^\rSrSrU4SjrSrSrU=r$)BertSelfOutputc >[[U] 5 [US5(a1UR(a [ SUR URS9nO[SUR S9n[R"URURSSSUS9Ul URUl UR(d$[URUR S9UlOSUl[$R&"UR(5Ulg N deep_initrEr3rG num_layersrFTr ) input_size output_sizer(input_is_parallelstriderHrI)r"rur#hasattrryr rLnum_hidden_layersr rRowParallelLinearr+denserSpre_lnrrVrWrrXrYrZr*r\rHr-s rr#BertSelfOutput.__init__s nd,. 6; ' 'F,<,<,,,!335K -f668K**))**"# % %33}}*""(@(@BDN"DNzz&"<"<= rc<URU5nURU5nX-nURbfUR5nUR(aUR 5nURU5nUR(aURU5nU$UnU$rrrZrWrhrSrir* hidden_states input_tensorln_inputrps rr7BertSelfOutput.forward  =1  ]3  / >> %$MMOM""#>>+ NN84M"" - 2 2= A %MrrWrrZrSr9r@s@rruru>2rruc.^\rSrSrU4SjrSrSrU=r$) BertAttentionc >[[U] 5 URUlUR(a$[ UR URS9UlOSUl[R"UR URURS[SURS9URS9Ul[#U5Ulg)NrITrErF)r+num_attention_heads dropout_proboutput_parallelrHseparate)r"rr#rSrrr+rVrWrBertParallelSelfAttentionrattention_probs_dropout_probr rL attn_separater*ruoutputr[s rr#BertAttention.__init__s mT+-$33 ==*""(@(@BDN"DN11** & : :<< *f668))+ %V, rcZURbyUnUR5nUR(aUR5nURU5nUR(aURU5nUR UU5nOUR UU5nUR UU5nU$r)rWrhrSrir*r)r*rattention_maskrrp ln_output self_outputattention_outputs rr7BertAttention.forwards >> %#H(--/M""'--/x0I""%NN=9 ))K ))K  ;;    r)rWrSrr*r9r@s@rrrs-$  rrc.^\rSrSrU4SjrSrSrU=r$)BertIntermediatec D>[[U] 5 [R"UR UR SSS[SURS9S9Ul [UR[5(a[URUlgURUlg)NTFr rErF)r|r}r( gather_outputrrH)r"rr#rColumnParallelLinearr+intermediate_sizer rLr isinstance hidden_actstrACT2FNintermediate_act_fnr[s rr#BertIntermediate.__init__s .0--))00*f668 9 &++S11$*&*;*;#< 7=7H7H  rcJURU5nURU5nU$rrrr*rs rr7BertIntermediate.forwards( =1 00? rrr9r@s@rrrs Irrc.^\rSrSrU4SjrSrSrU=r$) BertOutputc >[[U] 5 [US5(a1UR(a [ SUR URS9nO[SUR S9n[R"URURSSSUS9Ul URUlUR(d$[!URUR"S9UlOSUl[&R("UR*5Ulgrx)r"rr#rryr rLrr rrrr+rrSrrrVrWrrXrYrZrs rr#BertOutput.__init__s j$(* 6; ' 'F,<,<,,,!335K -f668K**//**"# % %33}}*""(@(@BDN"DNzz&"<"<= rc<URU5nURU5nX-nURbfUR5nUR(aUR 5nURU5nUR(aURU5nU$UnU$rrrs rr7BertOutput.forwardrrrr9r@s@rrrrrrc.^\rSrSrU4SjrSrSrU=r$) BertLayeri/c$>[[U] 5 [U5Ul[ U5Ul[U5UlURUl UR(a$[URURS9UlgSUlgNrI)r"rr#r attentionr intermediaterrrSrrr+rVrWr[s rr#BertLayer.__init__1sq i')&v.,V4 ( $33 ==*""(@(@BDN"DNrcvURX5nURbxUnUR5nUR(aUR 5nURU5nUR(aURU5nUR U5nOUR U5nUR Xs5nU$r)rrWrhrSrirr) r*rrrrrprintermediate_output layer_outputs rr7BertLayer.forward=s>>-H >> %'H,113M""+113x0I""%NN=9 "&"3"3I"> "&"3"34D"E {{#6I r)rWrrSrrr9r@s@rrr/s "rrc8^\rSrSrU4SjrSSjrSrU=r$) BertEncoderiNc^>[[U] 5 [R"[ UR 5Vs/sHn[U5PM sn5UlURUl UR(a$[URURS9UlgSUlgs snfr)r"rr#r ModuleListrangerrlayerrSrrr+rVrWr*r\_r-s rr#BertEncoder.__init__Ps k4)+]](-f.F.F(G H(G1Yv (G HJ $33 ==*""(@(@BDN"DN IsB*c^/nU4SjnU(acSn[TR5n Sn X:aD[R"U"XU -5UUS-5nXX:XaUR 5 X- nX:aMDO[ TR5HupU "X5nX[:XaUR 5 U [TR5S- :XaqTR bdUR5n TR(aUR5nTR U5nTR(aURU 5nU(dMURU5 M U(aU(aTR bdUR5n TR(aUR5nTR U5nTR(aURU 5nURU5 U$)Nc>^^UUU4SjnU$)NcX>TRTTnUSnUHnU"X S5nM U$)Nrr r)inputslayersx_rendr*starts rcustom_forward;BertEncoder.forward..custom..custom_forwardgs9E#.AY#Er!9-B$ r)rrrr*s`` rcustom#BertEncoder.forward..customes " !rrr ) lenrr checkpointdetach_ enumeraterWrhrSriappend)r*rroutput_all_encoded_layerscheckpoint_activations detach_indexall_encoder_layersr layer_idxr{ chunk_lengthi layer_modulerps` rr7BertEncoder.forward[s  " "ITZZJL( #9,&>?"Q&!(  ,!))+) ($-TZZ#8 ,] K $!))+DJJ!++0J$1$6$6$8M**(5(;(;(= $(NN=$AM**(5(:(:=(I ,,&--m<$9),B~~) - 2 2 4 &&$1$7$7$9M $} = &&$1$6$6}$EM  % %m 4!!r)rWrSr)TFr0r9r@s@rrrNs "#'$ 9"9"rrc.^\rSrSrU4SjrSrSrU=r$) BertPooleric>[[U] 5 [R"UR UR 5Ul[R"5Ulgr) r"rr#rLinearr+rTanh activationr[s rr#BertPooler.__init__s; j$(*YYv1163E3EF '')rc\USS2S4nURU5nURU5nU$)Nr)rr)r*rfirst_token_tensor pooled_outputs rr7BertPooler.forwards6+1a40 #56  6 r)rrr9r@s@rrrs$ rrc.^\rSrSrU4SjrSrSrU=r$)BertPredictionHeadTransformicx>[[U] 5 [R"UR UR 5Ul[UR[5(a[URO URUl [UR URS9UlURUlgr)r"rr#rrr+rrrrrtransform_act_fnrrVrWrSr[s rr#$BertPredictionHeadTransform.__init__s )49;YYv1163E3EF &++S11!'v'8'8 97=7H7H &   F$<$<>$33rcURU5nURU5nUR5nUR(aUR 5nUR U5nUR(aURU5nU$r)rrrhrSrirW)r*rrps rr7#BertPredictionHeadTransform.forwardsr =1 --m< %**,   )//1M}5   )..}=Mr)rWrrSrr9r@s@rrrs4  rrc.^\rSrSrU4SjrSrSrU=r$)BertLMPredictionHeadicd>^[[T] 5 [U5TlUTl[ R"[R"URS555Tl STRl URTl URTlU4SjnUTlSTlg)NrTcJ>TR(aUR5$U$r)rTrgtensorr*s rconvert_to_type6BertLMPredictionHead.__init__..convert_to_types""{{}$ rF)r"rr#r transformdecoder_weightrr$rr'rar(model_parallelrTrStype_converter converted)r*r\bert_model_embedding_weightsrr-s` rr#BertLMPredictionHead.__init__s "D244V<;LL KK499!< =? #'  $33$33  .rcUR(dgSUlUR(aOURR5 UR(a$URR R 5 URURU55n[R"U5n[R"URU5URUR5URUR55nU$)NT)r rTrrgrSrWrir rcopy_to_model_parallel_regionFlinearrr(rs rr7BertLMPredictionHead.forwards~~!DN""##%&&NN,,224t':':='IJ 99-H     .    3 3 4    *, r)r(r rrTrSrr r9r@s@rrrs,  rrc.^\rSrSrU4SjrSrSrU=r$)BertPreTrainingHeadsic>[[U] 5 [UU5Ul[ R "URS5Ulg)N) r"rr#r predictionsrrr+seq_relationship)r*r\r r-s rr#BertPreTrainingHeads.__init__s= "D24/0LN " &*<*[[U] 5 [U[5(dX[U[ 5(dC[ SRURRURR55eXl g)NzParameter config in `{}(config)` should be an instance of class `BertConfig`. To create a model from a Google pretrained model use `model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`) r"r"r#rr r ValueErrorformatr-r:r\)r*r\rkwargsr-s rr#PreTrainedBertModel.__init__sp !413&-00 :':'FFLfNN++T^^-D-DGFG G  rc4[U[R[R45(a9URR R SURRS9 O^[U[5(aIURR R5 URR RS5 [U[R5(a3URb%URR R5 ggg)z Initialize the weights. rErFrN) rrrrNr&datanormal_r\rLrr(zero_fill_)r*modules rinit_bert_weights%PreTrainedBertModel.init_bert_weightss fryy",,7 8 8 MM   & &dkk;; ' =  . . KK   " " $ MM   $ $S ) fbii ( (V[[-D KK   " " $.E (r)r\) r:r;r<r=rsr#r.r>r?r@s@rr"r"s  % %rr"c@^\rSrSrSrU4SjrSSjrSrU=r$) BertModelia BERT model ("Bidirectional Embedding Representations from a Transformer"). Params: config: a BertConfig class instance with the configuration to build a new model Inputs: `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts `extract_features.py`, `run_classifier.py` and `run_squad.py`) `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to a `sentence B` token (see BERT paper for more details). `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max input sequence length in the current batch. It's the mask that we typically use for attention when a batch has varying length sentences. `output_all_encoded_layers`: boolean which controls the content of the `encoded_layers` output as described below. Default: `True`. Outputs: Tuple of (encoded_layers, pooled_output) `encoded_layers`: controlled by `output_all_encoded_layers` argument: - `output_all_encoded_layers=True`: outputs a list of the full sequences of encoded-hidden-states at the end of each attention block (i.e. 12 full sequences for BERT-base, 24 for BERT-large), each encoded-hidden-state is a torch.FloatTensor of size [batch_size, sequence_length, hidden_size], - `output_all_encoded_layers=False`: outputs only the full sequence of hidden-states corresponding to the last attention block of shape [batch_size, sequence_length, hidden_size], `pooled_output`: a torch.FloatTensor of size [batch_size, hidden_size] which is the output of a classifier pretrained on top of the hidden state associated to the first character of the input (`CLF`) to train on the Next-Sentence task (see BERT's paper). Examples: >>> # Already been converted into WordPiece token ids >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) >>> config = modeling.BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) >>> model = modeling.BertModel(config=config) >>> all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask) c>[[U] U5 [U5Ul[ U5Ul[U5UlURUR5 gr) r"r1r#rBrorencoderrpoolerapplyr.r[s rr#BertModel.__init__CsH i'/(0"6*  (  4))*rc(Uc[R"U5nUc[R"U5nURS5RS5nUR [ UR R55RS9nSU- S-nURX5nUR UUUUUS9n U Sn URR5Hn U cMU RU 5n O U SS2S4n U(aU(aU Sn X4$) Nr r2r_r)rrrr0r) r ones_likerfrdtonextr3rr_ror4r) r*rjrkrrrrextended_attention_maskembedding_outputencoded_layersrrrs rr7BertModel.forwardJs(  !"__Y7N  !"--i8N#1":":1"="G"G"J#:"<"<t||..0177#=#9#&)@#@H"L??9E  #&?#9% &' ),'')Ay-55a8O  * (1- (,B+B/N,,r)ror3r4)NNTFr0rrr@s@rr1r1s))V+"&$/-/-rr1c4^\rSrSrU4SjrSSjrSrU=r$) DecodeLayeri|c \>^[[T] 5 [SURS9n[ SURUR S9n[R"URURURURUUS9Tl [R"URURURURUSUS9Tl[!URUR"S9Tl[!URUR"S9Tl[!URUR"S9Tl[R*"URUR,SUS9Tl[1UR2[45(a[6UR2O UR2Tl[R:"UR,URS US 9Tl[>R@RCUR5Tl"URFTl#U4S jnUTl$g) NrErFrz)r+rattention_dropout_proboutput_dropout_probrHoutput_layer_init_methodF)r+rrDrErHrrFrI)rrHT)r~rHcJ>TR(aUR5$U$r)rSrirs rr-DecodeLayer.__init__..convert_to_types""||~% r)%r"rBr#r rLr rrGPT2ParallelSelfAttentionr+rrrYrPalmParallelCrossAttentioncross_attentionrrVinput_layernormpost_attention_layernormpost_cross_attention_layernormrrrrrrrrrrrrrXrZrSr )r*r\rHrFrr-s` rr#DecodeLayer.__init__~s k4)+(&224 #5((//$1 66** & : :#)#F#F & : :#%=   #==** & : :#)#F#F & : :#%=  -   F$<$< >(5   F$<$<)>%.;   F$<$</>+ 44     $ $#  &++S11$*&*;*;#<7=7H7H  ++  $ $   "0   xx''(B(BC $33  .rcUnUR5nURURU55nUR(aURU5nUR XUS9nXa-nUnUR URU55nUR(aURU5nUR XU5nXa-nUnURURU55nUR(aURU5nURU5nURU5nURU5nURU5nXa-nU$)Nis_infer) rhrLr rSrrMrKrNrrrrZ)r*renc_hidden_states enc_attn_mask dec_attn_maskrRresidualrps rr7DecodeLayer.forwardsO !%**, ,,    .0   )..}=M 8'= !0  55    .0   )..}=M,,]-:<  0  ;;    .0   )..}=M))-8 00?  M2  ]3  0 r) rrKrZrSrLrrrrMrNr )Fr9r@s@rrBrB|s;.F %%rrBc8^\rSrSrU4SjrSSjrSrU=r$) BertDecoderic,>[[U] 5 [R"[ UR 5Vs/sHn[U5PM sn5Ul[URURS9Ul URUl gs snfr)r"rYr#rrrdec_hidden_layersrBrrr+rVfinal_layernormrSrs rr#BertDecoder.__init__sy k4)+]]*/0H0H*I J*IQ[ *I JL  -   F$<$< >$33 KsBc ^^^UUU4SjnURn U(aTSn [TR5n Sn X:a5[R"U"XU -5UX#S-5nXlX- n X:aM5O([ TR5HupU"UUUTTS9nM UR 5nTR(aUR5nTRU5nTR(aUR U5nU/$)Nc >^^UUUUU4SjnU$)Nc h>TRTTnUSnUHnU"UUSUSTS-TS9nM U$)Nrr r2rQr) rrrrrUrrRr*rs rr;BertDecoder.forward..custom..custom_forwardsSE#.AY#Eq q %)!) +B$ rr)rrrrUrRr*s`` rr#BertDecoder.forward..customs  " !rrr rQ) r)rrrrrrhrSrir\)r*rrSrTrUrrrRrpre_enc_hiddenrr{rrrrps` ` ` rr7BertDecoder.forwards " +// !ITZZJL( #9,&>?%q'8!: *8&) ($-TZZ#8 ,!%!!% !' $9&**,   )//1M,,];   )..}=Mr)r\rSr)FFFr9r@s@rrYrYs4(-*/44rrYc<^\rSrSrU4SjrSSjrSrU=r$) DecodeModeli&c>[[U] U5 [U5UlUR UR 5 gr)r"rfr#rYdecoderr5r.r[s rr#DecodeModel.__init__(s0 k4)&1"6*  4))*rc URS5RS5n U R[URR 55R S9n SU - S-n U"U5n URU UU USUS9nUS$) Nr r2r8rr9FrrRr0)rdr;r<rhrr_) r*rordecode_input_idsrlrTrUrrRr=r>s rr7DecodeModel.forward-s#0"9"9!"<"F"Fq"I"9"<"<t||..0177#=#9#&)@#@H"L%&67,,   # #( 'r""r)rh)NNNFFr9r@s@rrfrf&s#+"""',##rrfcF^\rSrSrU4SjrSSjrSrU=r$)PalmForPreTrainingiFc>[[U] U5 [U5Ul[ XRR RR5Ul [U5Ul URUR5 gr)r"ror#r1bertrrorMr&clsrfrhr5r.r[s rr#PalmForPreTraining.__init__Hsc  $08f% ' II((88??A"6*  4))*rc U c)URUUUSUS9upURX5upO#'99#7#7#G#G#N#NP $o5 5 $c&K&K'!"12 2 #"G"G # r)rqrrrh) NNNNNNFFNTr9r@s@rroroFs2+ $#!%!&*', $ $11rroc\^\rSrSrSrU4SjrSSjrS SjrS SjrSr U=r $) PlugModelia The bare Plug Model transformer outputting raw hidden-states without any specific head on top. This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`PlugNLGConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~DistributedPlug.initialize_model`] method to load the model weights. Examples: >>> # The PLUG model has 27B parameters and usually need to run on multiple GPUs. The example given >>> # here only initializes a slice of the model on a single GPU. >>> # Check out the [`~DistributedPipeline.__init__`] method to initialize entire PLUG model. >>> from modelscope.models.nlp.plug import PlugNLGConfig, PlugModel >>> # Initializing a Plug configuration >>> configuration = PlugNLGConfig() >>> # Initializing a model from the configuration >>> model = PlugModel(configuration) cj>[[U] 5 Xl[ UR5Ulgr)r"r~r#r\romodelr[s rr#PlugModel.__init__s% i') ' 4 rc 2URUUUUUUUUU U S9 $)aM Parameters: input_tokens (`torch.LongTensor` of shape `(batch_size, input_tokens_length)`): `input_tokens_length` = `sequence_length`. Indices of input sequence tokens in the vocabulary. Indices can be obtained using transformers [`BertTokenizer`]. See [`TextGenerationPreprocessor.__call__`] for details. token_type_ids (`torch.LongTensor` of shape `(batch_size, input_tokens_length)`, *optional*, defaults to None): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*, defaults to None): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. target_tokens (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*, defaults to None): Target token ids(labels) for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `target_tokens = input_tokens` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*, defaults to None): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. decode_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*, defaults to None): Mask to avoid performing attention on padding token indices of target tokens. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. checkpoint_activations (`boolean`, *optional*, defaults to `False`): Whether gradient checkpointing is activated for this model or not. is_infer (`boolean`, *optional*, defaults to `False`): Whether or not to perform single inference. sequence_output (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*, defaults to None): Also known as last_hidden_state. Sequence of hidden-states at the output of the last layer of the model. A single forward() call can produce one single token. To generate the current token, the sequence_output generated by the `forward()` of the previous token is required. parallel_output (`boolean`, *optional*, defaults to `True`): To parallel return output, or gather it before return. )rrRrrx)r) r* input_tokensrkr target_tokensrlrvrrRrrxs rr7PlugModel.forwards9~zz      !#9++ - -rc6URRXUS9$)N) destinationprefix keep_vars)r state_dict)r*rrrs rrPlugModel.state_dicts%zz$$#i%I Irc4URRXS9$)N)strict)rload_state_dict)r*rrs rrPlugModel.load_state_dictszz))*)DDr)r\r) NNNNNFFNT)NF)T) r:r;r<r=rsr#r7rrr>r?r@s@rr~r~sC.5 $#"!&*', $ $I-VIEErr~)/ __future__rrrrloggingrrtorch.nn.functionalr functionalr megatron_utilr modelscope.utils.nlp.distributedr r configurationr r getLoggerr:loggerrrrrModulerrBrurrrrrrrrrr"r1rBrYrfror~rrrrs"** B7   8 $;  3 3 8 85 I+BII+"BRYYBJ,RYY,^/ BII/ dryy0,,^ >F""))F"R   ")).%299%P92999$%"))%>b-#b-Je"))eP?"))?D#%#@;,;|mEmEr