9iSrSSKrSSKrSSKrSSKJr SSKJr SSKJr SSK J r SSK J r J r SSKJr SS KJr SS KJr SS KJr SS KJr SS KJrJrJr SSKJr \"5rSr"SS\R@5r!"SS\R@5r""SS\R@5r#"SS\R@5r$"SS\R@5r%"SS\R@5r&"SS\R@5r'"SS \R@5r("S!S"\R@5r)"S#S$\ \5r*\RV"\RX\ RZS%9"S&S'\*55r.g)(zPyTorch MegatronBERT model.N)nn)ACT2FN)PreTrainedModel)Models)Model TorchModel)MODELS)AttentionBackboneModelOutput)Tasks) get_logger)parse_labels_in_order)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)MegatronBertConfigrc\^\rSrSrSrU4SjrSS\R4SjjrSr U=r $)MegatronBertEmbeddings+zGConstruct the embeddings from word, position and token_type embeddings.c@>[TU]5 [R"URUR UR S9Ul[R"URUR 5Ul [R"URUR 5Ul [R"UR5UlURS[ R""UR5R%S55 ['USS5Ulg)N) padding_idx position_ids)rposition_embedding_typeabsolute)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddingsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandgetattrrselfconfig __class__s l/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/models/nlp/megatron_bert/backbone.pyrMegatronBertEmbeddings.__init__.s !||      ++ -$&<<0N0N060B0B$D %'\\&2H2H282D2D&F"zz&"<"<=    LL77 8 ? ? H J(/v/H/9(;$returncUbUR5nOUR5SSnUSnUcURSS2XWU-24nUc8[R"U[RURR S9nUcUR U5nURU5nXH-n URS:XaURU5n X- n URU 5n U $)Nrrdtypedevicer) sizerr+zeroslongr:r"r&rr$r)) r0 input_idstoken_type_idsr inputs_embedspast_key_values_length input_shape seq_lengthr& embeddingsr$s r3forwardMegatronBertEmbeddings.forwardHs  #..*K',,.s3K ^  ,,Q-C/EEF.F.FGL  !"[[5::d6G6G6N6NPN   00;M $ : :> J":  ' ': 5"&":":<"H   -J\\*- r5)r)rr$r&r")NNNNr) __name__ __module__ __qualname____firstlineno____doc__rr+TensorrE__static_attributes__ __classcell__r2s@r3rr+s4Q;8 $ $$r5rc|^\rSrSrSU4SjjrS\R S\R 4SjrS SjrSr U=r $) MegatronBertSelfAttentionpc>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eURUl[ URUR- 5UlURUR-Ul[R"URUR5Ul [R"URUR5Ul [R"URUR5Ul [R"UR5UlU=(d [#USS5UlUR$S:XdUR$S :XaGUR&Ul[R("S UR&-S - UR5UlUR,Ulg) Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()rr relative_keyrelative_key_queryr)rrr num_attention_headshasattr ValueErrorintattention_head_size all_head_sizerLinearquerykeyvaluer'attention_probs_dropout_probr)r.rr#rdistance_embedding is_decoder)r0r1rr2s r3r"MegatronBertSelfAttention.__init__rs     : : :a ?(I*I*#F$6$6#78 445Q89 9$*#=#= #&v'9'9)/)C)C(D$E !558P8PPYYv1143E3EF 99V//1C1CDYYv1143E3EF zz&"E"EF '>(;' -zC;$  ' '> 9T=Y=Y]q=q+1+I+ID (&(llF222Q6(('*D #!++r5xr6cUR5SSURUR4-nURU5nUR SSSS5$)NrrrXr)r;rYr]viewpermute)r0rg new_x_shapes r3transpose_for_scores.MegatronBertSelfAttention.transpose_for_scoressSffhsmt'?'?'+'?'?'AA FF; yyAq!$$r5cURU5nUSLn U (aUbUSn USn UnGOU (aCURURU55n URURU55n UnOUbuURURU55n URURU55n [R "USU /SS9n [R "USU /SS9n O@URURU55n URURU55n URU5n UR (aX4n[R"U U RSS55n URS:XdURS:XGaDUR5Sn[R"U[RURS 9RSS5n[R"U[RURS 9RSS5nUU- nURUUR -S- 5nUR#U R$S 9nURS:Xa[R&"S U U5nU U-n OHURS:Xa8[R&"S U U5n[R&"S U U5nU U-U-n U [(R*"UR,5- n UbX-n [.R0R3U SS9nUR5U5nUbUU-n[R"UU 5nUR7SSSS 5R95nUR5SSUR:4-nURU5nU(aUU4OU4nUR (aUU4-nU$)NrrrXdimrrVrWr8)r9zbhld,lrd->bhlrzbhrd,lrd->bhlrri)r`rmrarbr+catrematmul transposerr;r,r=r:rjrdr#tor9einsummathsqrtr]r functionalsoftmaxr)rk contiguousr^)r0 hidden_statesattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsmixed_query_layeris_cross_attention key_layer value_layer query_layerattention_scoresrCposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_probs context_layernew_context_layer_shapeoutputss r3rE!MegatronBertSelfAttention.forwards!JJ}5 3$> ."<&q)I(+K3N 11./1I33 013K3N  '11$((=2IJI33DJJ}4MNK >!#4i"@aHI))^A%6 $D!LK11$((=2IJI33DJJ}4MNK//0AB ??(5N!<< (1(;(;B(CE  ' '> 9T=Y=Y]q=q&++-a0J"\\%**$++--1T"a[ #\\%**$++--1T!R[ &6H#'#:#:4777!;$= #7#:#:!''$;$) ++~=+0<<$k3G,I(#36N#N --1EE16$k3G2I./4||$i1E0G,#36T#TWs#s +dii  $ $/&&  %/@ --//0@b/I,,7  - 9O _kB %--aAq9DDF "/"4"4"6s";   ?"#"%**+BC (9!"$?L>O  ?? 22Gr5) r^r]rdr)rerar#rYrr`rbNNNNNNF) rGrHrIrJrr+rLrmrErMrNrOs@r3rQrQpsC,8%ell%u||%"#kkr5rQc.^\rSrSrU4SjrSrSrU=r$)MegatronBertSelfOutputic>[TU]5 [R"URUR5Ul[R "UR5Ulgr) rrrr_r denser'r(r)r/s r3rMegatronBertSelfOutput.__init__sB YYv1163E3EF zz&"<"<= r5cNURU5nURU5nX!-$rrr))r0r}residuals r3rEMegatronBertSelfOutput.forward s' =1  ]3 ''r5rrGrHrIrJrrErMrNrOs@r3rrs> ((r5rcD^\rSrSrU4SjrSrSSjrSrU=r$)MegatronBertAttentionic>[TU]5 [R"URUR S9Ul[U5Ul[U5Ul [5Ul g)Neps) rrr LayerNormr layer_norm_epslnrQr0routputset pruned_headsr/s r3rMegatronBertAttention.__init__sN ,,v11v7L7LM-f5 ,V4 Er5c6[U5S:Xag[XRRURRUR 5up[ URRU5URl[ URRU5URl[ URRU5URl [ URRUSS9URl URR[U5- URlURRURR-URl UR RU5Ulg)Nrrrp)lenrr0rYr]rrr`rarbrrr^union)r0headsindexs r3 prune_heads!MegatronBertAttention.prune_headss u:? 7 9900 II ) )4+<+<>  -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(E I) %"&))"?"?$))B_B_"_  --33E:r5c URU5nURUUUUUUU5n URU SU5n U 4U SS-n U $)Nrr)rr0r) r0r}r~rrrrr ln_outputs self_outputsattention_outputrs r3rEMegatronBertAttention.forward-sjWW]+ yy    ! "    ;;|A F##AB'(r5)rrrr0r) rGrHrIrJrrrErMrNrOs@r3rrs+";,"#r5rc.^\rSrSrU4SjrSrSrU=r$)MegatronBertIntermediateiHc>[TU]5 [R"URUR 5Ul[UR[5(a[URUl gURUl gr) rrrr_r intermediate_sizer isinstance hidden_actstrrintermediate_act_fnr/s r3r!MegatronBertIntermediate.__init__Js` YYv1163K3KL f'' - -'-f.?.?'@D $'-'8'8D $r5cJURU5nURU5nU$rrr)r0r}s r3rE MegatronBertIntermediate.forwardRs& =1 00? r5rrrOs@r3rrHs9r5rcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)MegatronBertOutputiYc>[TU]5 [R"URUR 5Ul[R"UR5Ul gr) rrrr_rr rr'r(r)r/s r3rMegatronBertOutput.__init__[sB YYv779K9KL zz&"<"<= r5r} input_tensorr6cNURU5nURU5nX!-$rr)r0r}rs r3rEMegatronBertOutput.forward`s' =1  ]3 ++r5r) rGrHrIrJrr+rLrErMrNrOs@r3rrYs5> ,U\\,#ll,/4||,,r5rcD^\rSrSrU4SjrSSjrSrSrU=r$)MegatronBertLayerihc>[TU]5 URUlSUl[ U5UlUR UlURUlUR(a/UR (d[US35e[ U5Ul [R"URURS9Ul[U5Ul[#U5Ulg)Nrz> should be used as a decoder model if cross attention is addedr)rrchunk_size_feed_forward seq_len_dimr attentionreadd_cross_attention TypeErrorcrossattentionrrr rrr intermediaterrr/s r3rMegatronBertLayer.__init__js '-'E'E$.v6 ++#)#=#=  # #??fZ[#8"?D ,,v11v7L7LM4V<(0 r5c UbUSSOSnURUUUUUS9n U Sn UR(a U SSn U Sn OU SSn Sn UR(aZUbW[US5(d[SUS35eUbUS SOSnUR U UUUUUU5nUSn XSS-n USn W U -n [ UR URURU 5nU4U -n UR(aU W 4-n U $) NrX)rrrrrrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`rr) rrerZAttributeErrorrrfeed_forward_chunkrr)r0r}r~rrrrrself_attn_past_key_valueself_attention_outputsrrpresent_key_valuecross_attn_present_key_valuecross_attn_past_key_valuecross_attention_outputs layer_outputs r3rEMegatronBertLayer.forward{s9G8R$23423$5X\ !!%   /3 "0" 2!4 ??,Qr2G 6r : ,G(,$ ??4@4!122$=dVDDDEE '2)7)8< &&*&9&9 %&)!' # 7q9 "!G,C2+F ( 14P P 01H1H151M1M151A1A1AC  "W, ??!2 55Gr5clURU5nURU5nURX15nU$r)rrr)r0r ln_outputintermediate_outputrs r3r$MegatronBertLayer.feed_forward_chunks7GG,- "// :{{#6I r5) rrrrrrerrrr) rGrHrIrJrrErrMrNrOs@r3rrhs-1("#CJr5rcD^\rSrSrU4SjrSSjrSrU=r$)MegatronBertEncoderic2>[TU]5 Xl[R"[ UR 5Vs/sHn[U5PM sn5Ul[R"URURS9Ul SUl gs snf)NrF)rrr1r ModuleListrangenum_hidden_layersrlayerrr rrgradient_checkpointing)r0r1_r2s r3rMegatronBertEncoder.__init__sy  ]]/4V5M5M/N$ /N! f %/N$   ,,v11v7L7LM&+#$ sBc N^^U (aSOSn T(aSOSn T(aURR(aSOSn U(aSOSn[UR5HunnU (aX4-n UbX?OSnUbUUOSmUR(akUR (aZU(a[ RS5 SnUU4Sjn[RRRU"U5UUUUU5nOU"UUUUUTT5nUSnU(a UUS4- nT(dMU US4-n URR(dMU US4-n M URU5nU (aX4-n U (d[S UUU U U 455$[UUU U U S 9$) NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fc>^UUU4SjnU$)Nc>T"/UQTPTP76$rr)inputsmodulerrs r3custom_forwardRMegatronBertEncoder.forward..create_custom_forward..custom_forwards&% 9v 9~ 9&7 99r5r)rrrrs` r3create_custom_forward:MegatronBertEncoder.forward..create_custom_forwards9*)r5rrrrXc30# UH nUcMUv M g7frr).0vs r3 .MegatronBertEncoder.forward..'s" %q  %s )last_hidden_statepast_key_valuesr} attentionscross_attentions)r1r enumeraterrtrainingloggerwarningr+utils checkpointrtupler )r0r}r~rrrr use_cacheroutput_hidden_states return_dictall_hidden_statesall_self_attentionsall_cross_attentionsnext_decoder_cachei layer_modulelayer_head_maskr layer_outputsrs ` @r3rEMegatronBertEncoder.forwards#7BD$5b4 4;;#B#B HL $-R$(4OA|#$58I$I!.7.CilO%1-7; **t}}NNt!&I*!& 6 6 A A),7!"#)* ! !-!"#)*"%! *!,M"}R'8&;;"  &9!!$=('(#;;222+?%a(C,,,(k 5r .  14E E  "!#$ %   ,+.+*1   r5)r1rrr) NNNNNNFFTrrOs@r3rrs1 ,"#"_ _ r5rc.^\rSrSrU4SjrSrSrU=r$)MegatronBertPooleri8c>[TU]5 [R"URUR5Ul[R "5Ulgr)rrrr_r rTanh activationr/s r3rMegatronBertPooler.__init__:s9 YYv1163E3EF '')r5c\USS2S4nURU5nURU5nU$)Nr)rr)r0r}first_token_tensor pooled_outputs r3rEMegatronBertPooler.forward?s6+1a40 #56  6 r5)rrrrOs@r3rr8s$ r5rcd^\rSrSrSr\rSrSrS/r U4Sjr Sr S Sjr \ U4S j5rS rU=r$) MegatronBertPreTrainedModeliHzz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. bertTrc b>[TU]"UR40UD6 [[U]U5 gr)rr name_or_pathr)r0r1kwargsr2s r3r$MegatronBertPreTrainedModel.__init__Ss* ,,77 eT#F+r5cH[U[R[R45(a9URR R SURRS9 Oh[U[R5(aIURR R5 URR RS5 [U[R5(a3URb%URR R5 ggg)zInitialize the weightsg)meanstdg?N) rrr_rweightdatanormal_r1initializer_rangerbiaszero_fill_)r0rs r3 _init_weights)MegatronBertPreTrainedModel._init_weightsWs fryy",,7 8 8 MM   & &dkk;; ' =  - - KK   " " $ MM   $ $S ) fbii ( (V[[-D KK   " " $.E (r5c<[U[5(aX!lggr)rrr)r0rrbs r3_set_gradient_checkpointing7MegatronBertPreTrainedModel._set_gradient_checkpointingds f1 2 2,1 ) 3r5c >URSS5nURSS5n[X#40UD6nUc[S0UD6nU"U5nO[[U]"SSU0UD6nX&lU$)a Instantiate the model. Args: kwargs: Input args. model_dir: The model dir used to load the checkpoint and the label information. num_labels: An optional arg to tell the model how many classes to initialize. Method will call utils.parse_label_mapping if num_labels not supplied. If num_labels is not found, the model will use the default setting (2 classes). Returns: The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained model_dirNcfgpretrained_model_name_or_pathr)popr rrrfrom_pretrainedr8)clsr&r8r9 model_argsr1modelr2s r3 _instantiate(MegatronBertPreTrainedModel._instantiatehs~JJ{D1 jj%*9DVD  '5*5FKE%5G.7G;EGE# r5r)F)rGrHrIrJrKr config_classbase_model_prefixsupports_gradient_checkpointing_keys_to_ignore_on_load_missingrr2r5 classmethodr@rMrNrOs@r3r"r"HsG &L&*#'6&7#, %2r5r") group_key module_namecn^\rSrSrSrS U4SjjrSrSrSrS S\ 4Sjjr S r U=r $) MegatronBertModeliahThe bare MegatronBert Model transformer outputting raw hidden-states without any specific head on top.", This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`MegatronBertConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. c>[TU]U5 Xl[U5Ul[ U5UlU(a [U5OSUlUR5 gr) rrr1rrDrencoderrpooler post_init)r0r1add_pooling_layerr2s r3rMegatronBertModel.__init__sJ   08*62 4E(04  r5c.URR$rrDr")r0s r3get_input_embeddings&MegatronBertModel.get_input_embeddingss...r5c$XRlgrrR)r0rbs r3set_input_embeddings&MegatronBertModel.set_input_embeddingss */'r5cUR5H7up#URRURR U5 M9 g)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrLrrr)r0heads_to_prunerrs r3 _prune_headsMegatronBertModel._prune_headss< +002LE LL  u % / / ; ;E B3r5r6c U bU OURRn U bU OURRn U bU OURRn URR(aU bU OURR n OSn UbUb [ S5eUbUR5nO"UbUR5SSnO [ S5eUunnUb URO URnU bU SSRSOSnUc[R"UUU-4US9nUc$[R"U[RUS 9nURX/U5nURR(aEUbBUR5unnnUU4nUc[R"UUS9nURU5nOSnUR!UURR"5nUR%UUUUUS 9nUR'UUUUUU U U U U S 9 nUSnUR(bUR)U5OSnU (d UU4US S-$[+UUUR,UR.UR0UR2S 9$)a Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). Others (**kwargs) some additional parameters might passed in from upstream pipeline, which not influence the results. NFzDYou cannot specify both input_ids and inputs_embeds at the same timerz5You have to specify either input_ids or inputs_embedsrrX)r:r8)r>rr?r@rA) r~rrrrr rr r r)r pooler_outputrr}rr)r1rr use_return_dictrer r[r;r:shaper+onesr<r=get_extended_attention_maskinvert_attention_mask get_head_maskrrDrLrMr rr}rr)r0r>r~r?rrr@rrrr rr r r&rB batch_sizerCr:rAextended_attention_maskencoder_batch_sizeencoder_sequence_lengthrencoder_hidden_shapeencoder_extended_attention_maskembedding_outputencoder_outputssequence_outputrs r3rEMegatronBertModel.forwardsH2C1N-TXT_T_TqTq$8$D KK , , &1%V  "#..*K  &',,.s3KGI I"- J%.%:!! @T@T"-"1!3A!6!N 150P0P 11 ;; ! !&;&G=R=W=W> :  7$6$;$= %-).(*9&.2.H.H&/( +/3 +&&y'+{{'D'DF  ??%)'#9 + ,, 2"7#B+/!5#'  *!, $ 7 =A #]3oab6II I+-'+;;)77&11,==   r5)r1rDrLrM)T) NNNNNNNNNNNNN) rGrHrIrJrKrrSrVr[r rErMrNrOs@r3rJrJs]2 /0C##!"&*'+ $"&%) G :G G r5rJ)/rKrxr+torch.utils.checkpointrtransformers.activationsrtransformers.modeling_utilsrmodelscope.metainformodelscope.modelsrrmodelscope.models.builderr modelscope.outputsr modelscope.utils.constantr modelscope.utils.loggerr modelscope.utils.nlp.utilsr modelscope.utils.torch_utilsrrr configurationrr_CONFIG_FOR_DOCModulerrQrrrrrrrr"register_modulebackbone megatron_bertrJrr5r3rs3 # +7&/,;+.<>>. &ARYYAJO Of (RYY (3BII3n ryy " , ,\ \~m "))m b   9*o9xnn&*>*>@{ 3{ @{ r5