9ipSrSSKrSSKJr SSKJrJr SSKrSSKJ r SSK rSSK J r J r SSKJrJr SSKJr SSKJr SS KJrJr SS KJr SS KJr SS KJrJrJ r S SK!J"r" S SK#J$r$ \RJ"5r/SQr&"SS\ RN5r("SS\ RN5r)"SS\ RN5r*"SS\ RN5r+"SS\ RN5r,"SS\ RN5r-"SS\ RN5r."SS \ RN5r/"S!S"\ RN5r0"S#S$\ RN5r1"S%S&\\5r2\"S'S(\55r3\"S)S*\55r4S+r5S,r6\"S-\55"S.S/\255r7"S0S1\25r8"S2S3\ RN5r9g)4zPyTorch PEER model. N) dataclass)OptionalTuple)ACT2FNget_activation) ModelOutputadd_start_docstrings))BaseModelOutputWithPastAndCrossAttentions)PreTrainedModel)Model TorchModel)logger)parse_labels_in_order)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer) PeerConfig)SequenceSideInfo)zgoogle/peer-small-generatorzgoogle/peer-base-generatorzgoogle/peer-large-generatorzgoogle/peer-small-discriminatorzgoogle/peer-base-discriminatorzgoogle/peer-large-discriminatorcL^\rSrSrSrU4SjrSSSSS\"54SjrSrU=r $)PeerEmbeddings4zGConstruct the embeddings from word, position and token_type embeddings.c.>[TU]5 [R"URUR UR S9Ul[R"URUR 5Ul [R"URUR 5Ul [R"UR URS9Ul [R"UR5UlUR#S[$R&"UR5R)S55 [+USS/5UlSUR,;a8SUl[R"UR.UR 5Ulgg) N) padding_idxeps position_ids)rposition_embedding_typeabsolute#absolute_token_position_in_sentence)super__init__nn Embedding vocab_sizeembedding_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandgetattrrside_info_size/position_embeddings__token_position_in_sentenceselfconfig __class__s c/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/models/nlp/peer/backbone.pyr$PeerEmbeddings.__init__7sF !||     ! !++ -$&<<0N0N060E0E$G %'\\&2H2H282G2G&I"   ! !v'<'<>zz&"<"<=    LL77 8 ? ? H J(/v/H0:|(=$ 1D4P4P P"$D CE<<##V%:%:D J": 55 5"&":":<"H   -J 0D4P4P P ;;>?''!+-L?C>r>r? ;  IJ 4#?#? ?>1/3v|,373K3K 4" < (/ 1^^J/ \\*- rA)r/r3rr,r:r9r.r*) __name__ __module__ __qualname____firstlineno____doc__r$rQrZ__static_attributes__ __classcell__r>s@r?rr4s,Q<@#$655rArcP^\rSrSrU4SjrSrSSSSSS\"54SjrSrU=r $)PeerSelfAttentionc>[TU]5 URUR-S:wa5[ US5(d$[ SURUR4-5eURUl[ URUR- 5UlURUR-Ul[R"URUR5Ul [R"URUR5Ul [R"URUR5Ul [R"UR5Ul[#USS/5UlSUR$;aIUR&S-Ul[R*"SUR(-UR5UlGOS UR$;a]UR&S-UlS Ul[R*"SUR(-UR.-UR5UlGOS UR$;aUR&S-UlUR(S-Ul[R*"SUR(-UR5Ul[R*"SUR0-UR5UlOS UR$;anUR&S-UlUR(S-UlSUR(-SUR0--n[R*"UUR5UlOgS UR$;dSUR$;aGUR&S-Ul[R*"SUR(-UR5UlUR4Ulg)Nrr(zLThe hidden size (%d) is not a multiple of the number of attention heads (%d)rr relative_scalar_bias)relative_scalar_bias_with_side_info_tokenr"(relative_scalar_bias_token_plus_sentence,relative_scalar_bias_with_side_info_sentence relative_keyrelative_key_query)r#r$ hidden_sizenum_attention_headshasattr ValueErrorintattention_head_size all_head_sizer%Linearquerykeyvaluer1attention_probs_dropout_probr3r8rr+ max_relative_position_embeddingsr&distance_embeddingr9$max_sen_relative_position_embeddingsdistance_embedding_sentence is_decoder)r<r=vocabr>s r?r$PeerSelfAttention.__init__sY     : : :a ?(I*I*##V%?%?@AB B $*#=#= #&v'9'9)/)C)C(D$E !558P8PPYYv1143E3EF 99V//1C1CDYYv1143E3EF zz&"E"EF '.v/H0:|(=$ "T%A%A A4:4R4RVW4WD 1&(llD999(('*D #9D>>D===?E&(ll5373K3K'MD #t;; ;?SW[WsWs?s4:4R4RVW4WD 1&(llD999(('*D #!++rAcUR5SSURUR4-nUR"U6nUR SSSS5$)Nrrrjr)rMrqruviewpermute)r<x new_x_shapes r?transpose_for_scores&PeerSelfAttention.transpose_for_scoressSffhsmt'?'?'+'?'?'AA FFK yyAq!$$rANFc  URU5n USLn U (aUbUSn USn UnGOU (aCURURU55n URURU55n UnOUbuURURU55n URURU55n [R "USU /SS9n [R "USU /SS9n O@URURU55n URURU55n URU 5n UR (aX4n[R"XRSS55[R"UR5- nSnSUR;a XS- nUS- nS UR;dQS UR;dAS UR;d1S UR;d!S UR;dSUR;GaXUSnURU5nURU RS9nS UR;a@UR!/SQ5R#S5nU[R"U5- U-nGOS UR;dSUR;a1UR!/SQ5nU[R"U5- U-nGOS UR;aUR!/SQ5R#S5nUSnUR%U5nURU RS9nUR!/SQ5nU[R"U5- U-U-nGO S UR;a[[R&"SU U5[R"UR5- nUS- nUU-[R"U5- nOS UR;aw[R&"SU U5n[R&"SU U5nUU-[R"UR5- nUS- nUU-[R"U5- nOU[R"U5- nUbX-n[(R*"SS9"U5nUR-U5nUbUU-n[R"UU 5nUR!SSSS5R/5nUR15SSUR24-nUR4"U6nU(aUU4OU4nUR (aUU4-nU$)NrrrjdimrrJside_info_attention_scoresrnrorhrkrlrm distance_idx)rD)rjrr)rrrrjdistance_idx_sentencezbhld,lrd->bhlrzbhrd,lrd->bhlrr)rxrryrzr5catrmatmul transposemathsqrtrurr}torDr unsqueezereinsumr%Softmaxr3 contiguousrMrvr)r< hidden_statesattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsrVmixed_query_layeris_cross_attention key_layer value_layer query_layerattention_scoresattention_scores_termsrpositional_embeddingrhrpositional_embedding_sentencerelative_scalar_bias_sentencerelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_probs context_layernew_context_layer_shapeoutputss r?rZPeerSelfAttention.forwards!JJ}5 3$> ."<&q)I(+K3N 11./1I33 013K3N  '11$((=2IJI33DJJ}4MNK >!#4i"@aHI))^A%6 $D!LK11$((=2IJI33DJJ}4MNK//0AB ??(5N!<< 5H5H 6yy!9!9:;!" 4#?#? ? ,!. .  "a ' " T99 9=QUYUqUq=q)T-I-II>$B^B^^=A]A]]ATEaEaa).9L#'#:#:<#H #7#:#:!''$;$) &)E)EE';'C'C((y|%#3dii*7,$,.B$C >112C112';'C'C ("$#3dii*7,$,.B$C ,,7  - 9O _kB %--aAq9DDF "/"4"4"6s";   ?"#"%**,CD (9!"$?L>O  ?? 22GrA)rvrur}rr3rryr|r~rqrrxr9rz) r\r]r^r_r$rrQrZrarbrcs@r?reres3>,@% "&#'#6ZZrArec.^\rSrSrU4SjrSrSrU=r$)PeerSelfOutputirc(>[TU]5 [R"URUR5Ul[R "URURS9Ul[R"UR5Ul gNr) r#r$r%rwrpdenser/r0r1r2r3r;s r?r$PeerSelfOutput.__init__tsc YYv1163E3EF    F$9$9;zz&"<"<= rAcpURU5nURU5nURX-5nU$Nrr3r/r<r input_tensors r?rZPeerSelfOutput.forward{5 =1  ]3 }'CD rAr/rr3r\r]r^r_r$rZrarbrcs@r?rrr>rArcP^\rSrSrU4SjrSrSSSSSS\"54SjrSrU=r $) PeerAttentionic>[TU]5 [U5Ul[ U5Ul[ 5Ulgr)r#r$rer<routputset pruned_headsr;s r?r$PeerAttention.__init__s0 %f- $V, ErAc6[U5S:Xag[XRRURRUR 5up[ URRU5URl[ URRU5URl[ URRU5URl [ URRUSS9URl URR[U5- URlURRURR-URl UR RU5Ulg)Nrrr)lenrr<rqrurrrxryrzrrrvunion)r<headsindexs r? prune_headsPeerAttention.prune_headss u:? 7 9900 II ) )4+<+<>  -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(E I) %"&))"?"?$))B_B_"_  --33E:rANFc rURUUUUUUUU5n URU SU5n U 4U SS-n U $)Nrr)r<r) r<rrrrrrrrV self_outputsattention_outputrs r?rZPeerAttention.forwards_yy    ! "      ;;|A F##AB'(rA)rrr<) r\r]r^r_r$rrQrZrarbrcs@r?rrs0" ;, "&#'#6rArc.^\rSrSrU4SjrSrSrU=r$)PeerIntermediateic>[TU]5 [R"URUR 5Ul[UR[5(a[URUl gURUl gr) r#r$r%rwrpintermediate_sizer isinstance hidden_actstrrintermediate_act_fnr;s r?r$PeerIntermediate.__init__s` YYv1163K3KL f'' - -'-f.?.?'@D $'-'8'8D $rAcJURU5nURU5nU$rrr)r<rs r?rZPeerIntermediate.forwards& =1 00? rArrrcs@r?rrs9rArc.^\rSrSrU4SjrSrSrU=r$) PeerOutputic(>[TU]5 [R"URUR 5Ul[R"UR URS9Ul[R"UR5Ul gr) r#r$r%rwrrprr/r0r1r2r3r;s r?r$PeerOutput.__init__sc YYv779K9KL    F$9$9;zz&"<"<= rAcpURU5nURU5nURX-5nU$rrrs r?rZPeerOutput.forwardrrArrrcs@r?rrrrArcP^\rSrSrU4SjrSSSSSS\"54SjrSrSrU=r $) PeerLayericn>[TU]5 URUlSUl[ U5UlUR UlURUlUR(a+UR (d US35e[ U5Ul[U5Ul [U5Ul g)Nrz> should be used as a decoder model if cross attention is added) r#r$chunk_size_feed_forward seq_len_dimr attentionradd_cross_attentioncrossattentionr intermediaterrr;s r?r$PeerLayer.__init__s '-'E'E$&v. ++#)#=#=  # #?? ktf,j$k k?"/"7D ,V4 ( rANFc UbUSSOSn URUUUUU US9n U Sn UR(a U SSn U Sn OU SSn SnUR(aWUbT[US5(d SUS35eUbUS SOSnURU UUUUUU5nUSn U USS-n USnW U-n [ UR UR URU 5nU4U -n UR(aU W 4-n U $) Nrj)rrrVrrrrz'If `encoder_hidden_states` are passed, zp has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r)rrrrrrfeed_forward_chunkrr)r<rrrrrrrrVself_attn_past_key_valueself_attention_outputsrrpresent_key_valuecross_attn_present_key_valuecross_attn_past_key_valuecross_attention_outputs layer_outputs r?rZPeerLayer.forwards9G8R$23423$5X\ !!%   /3) "0" 2!4 ??,Qr2G 6r : ,G(,$ ??4@& Z8?YZ Z'2)7)8< &&*&9&9 %&)!' # 7q9  7"!G,C2+F ( 14P P 01H1H151M1M151A1A1AC  "W, ??!2 55GrAcJURU5nURX!5nU$r)rr)r<rintermediate_outputrs r?rPeerLayer.feed_forward_chunk.s)"//0@A{{#6I rA)rrrrrrrr) r\r]r^r_r$rQrZrrarbrcs@r?rrs2 )  "&#'#6ENrArc V^\rSrSrU4SjrSrSSSSSSSS\"5S4 SjrSrU=r $) PeerEncoderi4cd>[TU]5 Xl[R"[ UR 5Vs/sHn[U5PM sn5Ul[USS/5Ul SUR;Ga*URSRRUl URRUlURRUlURR Ul[R""URR$R&URR$R(5Ul[R""URR,R&URR,R(5Ulggs snf)Nrr rJr)r#r$r=r% ModuleListrangenum_hidden_layersrlayerr8rrr<self_attentionrqrurvrwrx in_features out_features pos_queryrypos_key)r<r=_r>s r?r$PeerEncoder.__init__6sI  ]](-f.F.F(G H(G1Yv (G HJ (/v/H0:|(=$ 4#?#? ?"&**Q-"9"9">">D '+':':'N'ND $'+':':'N'ND $!%!4!4!B!BD YYt':':'@'@'L'L'+':':'@'@'M'MODN99T%8%8%<%<%H%H%)%8%8%<%<%I%IKDL @ IsF-cBURRURU55nURRURU55n[R "UUR SS55nU[R"UR5- nU$)Nrr) rrr r r5rrrrru)r<rrrrs r?get_position_attention_score(PeerEncoder.get_position_attention_scoreLs))>> NN= )+ ''<< LL ') !<< (1(;(;B(CE+dii  $ $/&&rANFTc  ^^"SUR;aURU SSS9n U U S'SUR;dQSUR;dASUR;d1S UR;d!S UR;dS UR;GaURS n URS n[R"U [R UR S9RSS 5n[R"U [R UR S9RS S5nURS RRRn[R"UU- U-S- S SU-S- S9nSU-S- US SS24'SU-S- USS2S 4'SU-S - US'SU-nS UR;a[R"U SS URS RRRS - S9RS5RS S U 5nURS 5RUS S 5nUU-U-nGOFS UR;aU SRUSS 5nU SRUS S5nURS RRR n[R"UU- U-S SU-S - S9nURS 5RUS S 5nUU-U-nOS UR;aU SRUSS 5nU SRUS S5nURS RRR n[R"UU- U-S SU-S - S9nUU S'UU S'U (aSOSnT(aSOSnT(aUR"R$(aSOSnU(aSOSn['UR5HunnU (aUU4-nUbUUOSnUbUUOSm"[)UR"SS5(a=UU"4Sjn [R*R,R-U "U5UUUUUU 5n!OU"UUUUUT"TU 5n!U!S nU(a UU!S4- nT(dMUU!S 4-nUR"R$(dMUU!S4-nM U (aUU4-nU (d[/SUUUUU455$[1UUUUUS9$)NrJrKrL)rrrnrorhrkrlrmrrrCrrjrirGr)rrrF ss_sentence_position_in_sequencerrgradient_checkpointingFc>^UUU4SjnU$)Nc>T"/UQTPTP76$rr)inputsmodulerrs r?custom_forwardJPeerEncoder.forward..create_custom_forward..custom_forwards&% 9v 9~ 9&7 99rAr)rrrrs` r?create_custom_forward2PeerEncoder.forward..create_custom_forwards9*)rAc30# UH nUcMUv M g7frr).0vs r? &PeerEncoder.forward..s" %q  %s )last_hidden_statepast_key_valuesr attentionscross_attentions)rrshaper5r6rOrErrrr<r|rPr9rrepeatr~r=r enumerater8utils checkpointtupler )#r<rrrrrr# use_cacheroutput_hidden_statesrV return_dictrrX batch_sizeposition_ids_lposition_ids_rr|rdistance_idx_maxidx1idx2sen_position_ids_lsen_position_ids_rr~all_hidden_statesall_self_attentionsall_cross_attentionsnext_decoder_cachei layer_modulelayer_head_maskr layer_outputsrs# ` @r?rZPeerEncoder.forwardYs 4#?#? ?)-)J)J,\:02*K*3 &1K , . T99 9=QUYUqUq=q)T-I-II>$B^B^^=A]A]]ATEaEaa&,,Q/J&,,Q/J"\\%**$++--1T"a[ #\\%**$++--1T!R[ 04zz09TT"B"B - ;;/23567881< >L <Z>ZZ{{"#BC 1 //44CC#1ffQ:&>  $--a077 AqI#&66= ?4C_C__%36&88<"B9+#&46&88<"Ar9+#8 *"6BD$5b4 4;;#B#B HL $-R$(4OA|#$58I$I!.7.CilO%1-7; t{{$[TU]5 [R"URUR5Ul[R"URS5UlXlg)Nr)r#r$r%rwrprdense_predictionr=r;s r?r$%PeerDiscriminatorPredictions.__init__sJ YYv1163E3EF " &*<*? &t{{'='=>}M &&}5==bA rA)r=rrC r\r]r^r_r`r$rZrarbrcs@r?rArAsOrArAc2^\rSrSrSrU4SjrSrSrU=r$)PeerGeneratorPredictionsizAPrediction module for the generator, made up of two dense layers.c>[TU]5 [R"UR5Ul[R "UR UR5Ulgr)r#r$r%r/r(rwrprr;s r?r$!PeerGeneratorPredictions.__init__sB f&;&;<YYv1163H3HI rAclURU5n[S5"U5nURU5nU$)Ngelu)rrr/)r<generator_hidden_statesrs r?rZ PeerGeneratorPredictions.forwards4 #:; &v.}= }5 rA)r/rrJrcs@r?rLrLsKJ rArLcR^\rSrSrSr\rSrS/rSS/r Sr \ U4Sj5r S r U=r$) PeerPreTrainedModeli%zz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. teams1_shared_bottomrz peer\.embeddings_project\.weightzpeer\.embeddings_project\.biascH[U[R[R45(a9URR R SURRS9 Oh[U[R5(aIURR R5 URR RS5 [U[R5(a3URb%URR R5 ggg)zInitialize the weights g)meanstdg?N) rr%rwr&weightdatanormal_r=initializer_ranger/biaszero_fill_)r<rs r? _init_weights!PeerPreTrainedModel._init_weights2s fryy",,7 8 8 MM   & &dkk;; ' =  - - KK   " " $ MM   $ $S ) fbii ( (V[[-D KK   " " $.E (rAc >URSS5nURSS5n[X#40UD6nUc[S0UD6nU"U5nU$[[U]"SSU0UD6nU$)a$Instantiate the model. Args: kwargs: Input args. model_dir: The model dir used to load the checkpoint and the label information. num_labels: An optional arg to tell the model how many classes to initialize. Method will call utils.parse_label_mapping if num_labels is not input. label2id: An optional label2id mapping, which will cover the label2id in configuration (if exists). Returns: The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained model_dirNcfgpretrained_model_name_or_pathr)poprrr#r from_pretrained)clskwargsrcrd model_argsr=modelr>s r? _instantiate PeerPreTrainedModel._instantiate?sJJ{D1 jj%*9DVD  -*-FKE %5G.7G;EGE rAr)r\r]r^r_r`r config_classbase_model_prefix_keys_to_ignore_on_load_missing"_keys_to_ignore_on_load_unexpectedr` classmethodrlrarbrcs@r?rTrT%sE L.'6&7#+-N*& %rArTc\rSrSr%SrSr\\R\ S'Sr \R\ S'Sr \\ \R\ S'Sr \\ \R\ S'Srg) PeerForRTDOutputi[a Output type of :class:`~transformers.PeerForRTD`. Args: loss (`optional`, returned when ``labels`` is provided, ``torch.FloatTensor`` of shape :obj:`(1,)`): Total loss of the PEER objective. logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`): Prediction scores of the head (scores for each token before SoftMax). hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``): Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of shape :obj:`(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``): Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape :obj:`(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. NlossrHrr$r)r\r]r^r_r`rurr5 FloatTensor__annotations__rHrrr$rarrAr?rtrt[sb.)-D(5$$ %, $FE  $8`__ subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config (:class:`~transformers.PeerConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. a Args: input_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using :class:`~transformers.PeerTokenizer`. See :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for details. `What are input IDs? <../glossary.html#input-ids>`__ attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`): Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. `What are attention masks? <../glossary.html#attention-mask>`__ token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, 1]``: - 0 corresponds to a `sentence A` token, - 1 corresponds to a `sentence B` token. `What are token type IDs? <../glossary.html#token-type-ids>`_ position_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, config.max_position_embeddings - 1]``. `What are position IDs? <../glossary.html#position-ids>`_ head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`): Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`): Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert :obj:`input_ids` indices into associated vectors than the model's internal embedding lookup matrix. encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. output_attentions (:obj:`bool`, `optional`): Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned tensors for more detail. output_hidden_states (:obj:`bool`, `optional`): Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for more detail. return_dict (:obj:`bool`, `optional`): Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple. aYThe bare Peer Model transformer outputting raw hidden-states without any specific head on top. Identical to the BERT model except that it uses an additional linear layer between the embedding layer and the encoder if the hidden size and embedding size are different.Both the generator and discriminator checkpoints may be loaded into this model.c h^\rSrSrU4SjrSrSrSrSrSSSSSSSS\ "5S4 Sjr S r U=r $) PeerModelic>[TU]U5 [U5UlURUR :wa0[ R"URUR 5Ul[U5Ul Xl UR5 URR(a[5Ul[!5Ulggr)r#r$rrKr(rpr%rwembeddings_projectrencoderr= init_weightsseq_side_info_embeddingsrQinput_sequence_side_inforsequence_side_infor;s r?r$PeerModel.__init__s  (0  F$6$6 6&(ii0E0E060B0B'DD ##6*    ;; / /,0FD )&6&8D # 0rAc.URR$rrKr*)r<s r?get_input_embeddingsPeerModel.get_input_embeddingss...rAc$XRlgrr)r<rzs r?set_input_embeddingsPeerModel.set_input_embeddingss */'rAcUR5H7up#URRURR U5 M9 gz Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel Nitemsrrrrr<heads_to_prunerrs r? _prune_headsPeerModel._prune_heads < +002LE LL  u % / / ; ;E B3rAc6URnSU;d[US5S:Xa1URRURR U5nOUSnUR 5H$nXERUS9R5XE'M& 0UEUEnU$NrrrE rErrgenerate_seq_side_infor=rkeysrrOr<rVrRrErsss r?update_seq_side_infoPeerModel.update_seq_side_info!! %^ ;s9:@'>'U'U 44i(A $ (6*(, $+//1B+C+G+J+J,K,#tv % (2HNG.FGrANc FURR(aURU U5n UbUOURRnUbUOURRnU bU OURR n UbUb [ S5eUbUR5n O"UbUR5SSn O [ S5eUb URO URn Uc[R"XS9nUc$[R"U [RU S9nURX+U 5n URUURR5nUR!UUUUU S9n[#US5(aUR%U5nUR'UU UUUU U S9nU$) NDYou cannot specify both input_ids and inputs_embeds at the same timer5You have to specify either input_ids or inputs_embedsrrC)rRrrSrTrVrrrrr-rVr.)r=rrrr-use_return_dictrsrMrEr5onesrNrOget_extended_attention_mask get_head_maskrrKrrrr)r<rRrrSrrrTrr-rVr.rWrEextended_attention_maskrs r?rZPeerModel.forward&s ;; / /!66~7@BN2C1N-TXT_T_TqTq$8$D KK , , &1%V  "#..*K  &',,.s3KGI I&/%:!! @T@T  !"ZZ CN  !"[[5::f>N#'"B"B #1&&y'+{{'D'DF %)') (  4- . . 33MBM 2/!5)#% rA)r=rKrrrr) r\r]r^r_r$rrrrrQrZrarbrcs@r?rrsK9 /0C,"!%6EErArc \^\rSrSrU4SjrSrSrSSSSSSSS\"5S4 SjrSr U=r $) PeerTopModelinc>[TU]U5 [U5UlXlUR 5 URR (a[5Ul[5Ul ggr) r#r$rrr=rrrQrrrr;s r?r$PeerTopModel.__init__psT  "6*    ;; / /,0FD )&6&8D # 0rAcUR5H7up#URRURR U5 M9 grrrs r?rPeerTopModel._prune_heads{rrAc6URnSU;d[US5S:Xa1URRURR U5nOUSnUR 5H$nXERUS9R5XE'M& 0UEUEnU$rrrs r?r!PeerTopModel.update_seq_side_inforrANc URR(aURU U5n UbUOURRnU bU OURRn U bU OURR n UbUb [ S5eUbUR5n O"UbUR5SSn O [ S5eUb URO URn Uc[R"XS9nUc$[R"U [RU S9nURXV  "#..*K  &',,.s3KGI I&/%:!! @T@T  !"ZZ CN  !"[[5::f>N#'"B"B #1&&y'+{{'D'DF  2/!5)#% rA)r=rrr) r\r]r^r_r$rrrQrZrarbrcs@r?rrns? 9C."!%6<<rArc2^\rSrSrSrU4SjrSrSrU=r$)PeerClassificationHeadiz-Head for sentence-level classification tasks.c,>[TU]5 [R"URUR5Ul[R "UR5Ul[R"URUR5Ul gr) r#r$r%rwrprr1r2r3 num_labelsout_projr;s r?r$PeerClassificationHead.__init__s` YYv1163E3EF zz&"<"<=  &"4"4f6G6GH rAc USS2SSS24nURU5nURU5n[S5"U5nURU5nURU5nU$)NrrP)r3rrr)r<featuresrirs r?rZPeerClassificationHead.forwards` Q1W  LLO JJqM 6 "   LLO MM! rA)rr3rrJrcs@r?rrs7I  rAr):r`r dataclassesrtypingrrr5torch.nnr%torch.utils.checkpointtransformers.activationsrrtransformers.file_utilsrr transformers.modeling_outputsr transformers.modeling_utilsr modelscope.modelsr r modelscope.utilsrloggingmodelscope.utils.nlp.utilsrmodelscope.utils.torch_utilsrrr configurationr sas_utilsr get_logger"PEER_PRETRAINED_MODEL_ARCHIVE_LISTModulerrerrrrrrrArLrTrtryPEER_START_DOCSTRINGPEER_INPUTS_DOCSTRINGrrrrrAr?rs !" ;E.7/.<>>&'    &"VRYYVrb bJ RYY  3BII3l ryy    Y YxK "))K \299$ryy"3*o3l :{: :< :{: :B":zV  y#yyxe&ePRYYrA