
    =pjo                     H   S r SSKrSSKrSSKJr  SSKJr  SSKJr  SSK	J
r
JrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,J-r-J.r.J/r/J0r0J1r1J2r2J3r3  SS	K4J5r5  S
\SS4S jr60 S\/ 4_S\/ 4_S\/ 4_S\/ 4_S\\4_S\\"4_S\\ 4_S\\#4_S\\4_S\\4_S\\4_S\\4_S\\4_S\\4_S\%\$4_S\%\$4_S\'\&4_\'\&4\\4\!/ 4\!/ 4\!/ 4\!/ 4\!/ 4\,\+4\/ 4\/ 4\\4S.Er7\1\1\/\0\0\0S .r8S!r9S"r:/ S#Qr;S$ r<S% r=S& r> " S' S5      r?g)(a  Weight converter: pdparams -> safetensors.

Conversion flow:
  1. paddle.load() -> state dict with OLD PaddleOCR/PaddleDetection key names
  2. Rename BatchNorm keys: _mean -> running_mean, _variance -> running_var
  3. Apply per-architecture regex key mappings (old keys -> HF keys)
  4. Transpose linear weight keys (Paddle [in, out] -> HF [out, in])
  5. Save as safetensors via safetensors.numpy.save_file()
    N)Path   )logging)AttrDict   )*CHART2TABLE_ADDED_TOKENSCHART2TABLE_GENERATION_CONFIGCHART2TABLE_SPECIAL_TOKENS_MAPCHART2TABLE_TOKENIZER_CONFIGMOBILE_DET_DROP_PREFIXESPP_CHART2TABLE_DROP_PREFIXESPP_CHART2TABLE_MAPPINGPP_DOCLAYOUTV2_DROP_PREFIXESPP_DOCLAYOUTV2_MAPPINGPP_FORMULANET_MAPPINGPPLCNET_MAPPINGPPOCRV5_MOBILE_DET_MAPPINGPPOCRV5_MOBILE_REC_MAPPINGPPOCRV5_SERVER_DET_MAPPINGPPOCRV5_SERVER_REC_MAPPINGPPOCRV6_DET_DROP_PREFIXESPPOCRV6_MEDIUM_DET_MAPPINGPPOCRV6_REC_DROP_PREFIXESPPOCRV6_SMALL_DET_MAPPINGPPOCRV6_SMALL_REC_MAPPINGPPOCRV6_TINY_REC_MAPPINGPREPROCESSOR_CONFIGSREC_DROP_PREFIXESRTDETR_MAPPINGSERVER_DET_DROP_PREFIXESSERVER_REC_DROP_PREFIXESSLANET_DROP_PREFIXESSLANET_MAPPINGSLANEXT_DROP_PREFIXESSLANEXT_MAPPINGUNIMERNET_GENERATION_CONFIGUNIMERNET_PROCESSOR_CONFIGUNIMERNET_TOKENIZER_CONFIGUVDOC_DROP_PREFIXESUVDOC_MAPPINGapply_key_mappingbuild_inference_metafuse_v6_medium_det_state_dictfuse_v6_rec_state_dictfuse_v6_small_det_state_dictload_character_dictrename_bn_keys)MODEL_CONFIGSconfigreturnWeightConverterc                     [        U 5      $ )z-Build a weight converter from PaddleX config.)r5   )r3   s    a/var/www/html/pdf-tiff/venv/lib/python3.13/site-packages/paddlex/modules/base/weight_converter.pybuild_weight_converterr8   N   s    6""    zPP-LCNet_x1_0_doc_orizPP-LCNet_x1_0_table_clszPP-LCNet_x0_25_textline_orizPP-LCNet_x1_0_textline_orizPP-OCRv5_mobile_detzPP-OCRv5_server_detPP-OCRv5_mobile_recPP-OCRv5_server_recPP-OCRv6_small_detPP-OCRv6_tiny_detPP-OCRv6_medium_detPP-OCRv6_small_recPP-OCRv6_medium_recPP-OCRv6_tiny_recSLANetSLANet_plusSLANeXt_wired)SLANeXt_wirelesszPP-DocLayoutV2zPP-DocLayoutV3zRT-DETR-L_wired_table_cell_detz!RT-DETR-L_wireless_table_cell_detzPP-DocLayout_plus-LzPP-DocBlockLayoutUVDocPP-FormulaNet-LPP-FormulaNet_plus-LzPP-Chart2Table)r<   r=   r>   r?   r@   rA   )r:   r;   r?   r@   rA   )rG   rH   )Afcchannelwiseout_projq_projk_projv_projo_proj	gate_projup_proj	down_projlm_headlinear_1linear_2linear1linear2zattn.qkvzmlp.lin1zmlp.lin2z	attn.projz	mixer.qkvz
mixer.projzself_attn.qkvzself_attn.projection
mapper_crp
mapper_scaz.mapper.
txt_mappertxt_pooled_mapperclip_img_mapper	kv_mapperclip_mappermm_projector_varyenc_to_dec_proj
score_headenc_score_headdec_score_head	bbox_headenc_bbox_headdec_bbox_headmask_query_head
enc_outputquery_pos_headdec_global_pointerdec_order_headattention_weightssampling_offsets
value_projoutput_projz	head.headctc_headconv_reduce_channelzstructure_attention_cell.scorezstructure_attention_cell.i2hzstructure_attention_cell.h2hzstructure_generator.0.zstructure_generator.1.spatial_projzattention.self.queryzattention.self.keyzattention.self.valuezintermediate.densez.output.denserelative_headlabel_features_projectionpos_projc                 6   ^  [        U 4S j[         5       5      $ )z@Check if a 2D weight tensor should be transposed (linear layer).c              3   ,   >#    U  H	  oT;   v   M     g 7fN ).0subkeys     r7   	<genexpr>$_should_transpose.<locals>.<genexpr>   s     ;%:ccz%:s   )any_TRANSPOSE_SUBSTRINGS)r|   s   `r7   _should_transposer      s    ;%:;;;r9   c                    SSK n0 nU R                  5        GH  u  p4[        US5      (       ad  SSKnUR                  UR
                  UR                  4;   a  UR                  UR                  5      nUR                  5       R                  5       nO.[        XAR                  5      (       a  UnOUR                  U5      nSU;   a  SU;   d  SU;   a  UR                  S5      nSU;   d  S	U;   a  S
U;   an  UR                  5       nUR                  S   S-  nUSU X#R!                  SS5      '   UUSU-   X#R!                  SS5      '   USU-  S X#R!                  SS5      '   OcSU;   a]  UR                  S   S-  nUSU X#R!                  S	S5      '   UUSU-   X#R!                  S	S5      '   USU-  S X#R!                  S	S5      '   GM  UR"                  S:X  a&  SU;  a   [%        U5      (       a  UR                  5       nXbU'   GM     U$ )aD  Preprocess Paddle tensors for safetensors output.

Converts to numpy, transposes linear weight tensors from Paddle [in, out]
to HF [out, in] format, reshapes channelwise parameters, and splits
fused in_proj weights into separate q/k/v.

Applied on OLD key names before regex key mapping.
Returns dict of {key: numpy_array}.
r   NnumpyrJ   gammabeta)r   r   r   in_proj_weightin_proj_biasweightr   zq_proj.weight   zk_proj.weightzv_proj.weightbiaszq_proj.biaszk_proj.biaszv_proj.bias)r   itemshasattrpaddledtypebfloat16float16astypefloat32cpu
isinstancendarrayarrayreshape	transposeshapereplacendimr   )
state_dictnpresultr|   tensorr   	np_weight
split_sizes           r7   _preprocess_tensorsr      s#    F!'')67##||@@v~~6

**,I

++I(ICW^v}!))-8Is"n&;3%//1	&__Q/14
IRZJ{{#3_EF JSZJ{{#3_EF JS
N$J{{#3_EF 3&__Q/14
ENZF{{>=AB FOZF{{>=AB FO
N$F{{>=AB >>Q6#49J39O9O!++-IsY *\ Mr9   c                 l   [        U 5      nUR                  5       (       a9  UR                  R                  S5      (       d  [	        SU 35      e[        U5      $ UR                  5       (       a  / SQnU H)  nX-  nUR                  5       (       d  M  [        U5      s  $    [        UR                  S5      5      n[        U5      S:X  a  [        US   5      $ [        U5      S:  a-  U Vs/ s H  ofR                  PM     nn[	        SU SU S	35      e[        S
U 35      e[        SU 35      es  snf )a  Resolve input_path to a concrete .pdparams file.

Accepts a direct .pdparams file path or a directory containing one.
Directory resolution checks: model_state.pdparams, inference.pdparams,
best_model.pdparams, best_accuracy.pdparams, or the single .pdparams file.
z	.pdparamsz.input_path file must end with .pdparams, got: )zmodel_state.pdparamszinference.pdparamszbest_model.pdparamszbest_accuracy.pdparamsz
*.pdparamsr   r   z"Multiple .pdparams files found in z: z%. Please specify the exact file path.z'No .pdparams files found in directory: zinput_path does not exist: )r   is_filenameendswith
ValueErrorstris_direxistslistgloblenFileNotFoundError)
input_pathp
candidatesr   	candidatepdparams_filesfnamess           r7   _resolve_input_pathr   !  s-    	ZAyy{{vv{++MaSQRR1vxxzz

 DI!!9~% 
 aff\23~!#~a()) 1$%34^VV^E44QCr% A6 6 
 $&MaS$QRR
9!=
>> 5s   ,D1c                   l    \ rS rSrSrS rS rS rS rS r	S r
S	 rS
 rS rS rS rS rS rS rSrg)r5   iK  z8Converts Paddle .pdparams weights to safetensors format.c                   ^ UR                   R                  U l        UR                  m[	        T[
        5      (       a  TR                  OS	U4S jjU l        U R                  S5      U l        U R                  S5      U l	        U R                  c  [        S5      eU R                  c  [        S5      eU R                  [        ;  aG  SR                  [        [        R                  5       5      5      n[        SU R                   SU 35      e[        U R                  5      R!                  5       U l        U R%                  5       U l        g )
Nc                    > [        TX5      $ rx   )getattr)kdconvert_configs     r7   <lambda>*WeightConverter.__init__.<locals>.<lambda>U  s    7>1#@r9   r   
output_dirzdPdparams2safetensors.input_path is required. Specify a .pdparams file or a directory containing one.z,Pdparams2safetensors.output_dir is required.z, zModel 'zJ' is not supported for pdparams2safetensors conversion. Supported models: rx   )Globalmodel
model_namePdparams2safetensorsr   dictget_getr   r   r   _MODEL_REGISTRYjoinsortedkeysr   r   _input_is_dir_load_user_configs_user_configs)selfr3   	supportedr   s      @r7   __init__WeightConverter.__init__N  s    ----44 .$// @ 		 ))L1))L1??"J  ??"KLL??/1		&)=)=)?"@AI$//* +FFO[R 
 "$//299;!446r9   c           	        ^ U R                   (       d#  [        R                  " SU R                  5        0 $ SSKm[        U R                  5      n0 nSS 4SS 4SU4S	 j44 H  u  p4X-  nUR                  5       (       a8  [        US
S9 nU" U5      X#'   SSS5        [        R                  " SU 35        MV  [        R                  " U SU SU R                   S35        M     U$ ! , (       d  f       NX= f)z5Load user-provided config files from input directory.zDInput is a single pdparams file. Using official config files for %s.r   Nconfig.jsonc                 .    [         R                  " U 5      $ rx   jsonloadr   s    r7   r   4WeightConverter._load_user_configs.<locals>.<lambda>|  s    diilr9   preprocessor_config.jsonc                 .    [         R                  " U 5      $ rx   r   r   s    r7   r   r   }  s    499Q<r9   inference.ymlc                 &   > TR                  U 5      $ rx   )	safe_load)r   yamls    r7   r   r   ~  s    q(9r9   utf-8encodingzLoaded user config:  not found in z. Using official default for .)
r   r   infor   r   r   r   r   openwarning)r   	input_diruser_configsfnameloaderfpathr   r   s          @r7   r   "WeightConverter._load_user_configsl  s    !!LL6
 I)	 23')?@9:
ME
 %E||~~%'2a*0)L' 33E7;<g^I; 7226//1B!E
   32s   
C''
C5	c                    SSK Jn  [        U R                     u  p#U R	                  X#5      n[
        R                  " U R                  SS9  U R                  U5        U R                  5         U R                  5         U R                  5         U R                  U;   a  U R                  5         U R                  [        ;   a  U R                  5         [        R                   " SU R                   35        g)z/Execute the pdparams -> safetensors conversion.r   )PP_CHART2TABLE_MODELST)exist_okz&Conversion complete. Output saved to: N)"inference.models.doc_vlm.constantsr   r   r   _convert_weightsosmakedirsr   _save_safetensors_save_model_config_save_preprocessor_config_save_inference_yml_save_llm_configPP_FORMULANET_MODELS_save_pp_formulanet_assetsr   r   )r   r   key_mappingdrop_prefixesnumpy_sds        r7   convertWeightConverter.convert  s    O%4T__%E"((D
DOOd3x(!&&(  "??33!!#??22++-=doo=NOPr9   c                   ^ SSK n[        U R                  5      n[        R                  " SU 35        UR                  U5      nU(       ae  U V^s/ s H!  m[        U4S jU 5       5      (       d  M  TPM#     nnU H  mUT	 M     U(       a#  [        R                  " S[        U5       S35        [        U5      n[        U5      n[        R                  U R                  5      n	U	b?  [        U5      n
U	" U5      n[        R                  " SU R                  U
[        U5      5        U(       a  [        X5      nO$[        R                  " SU R                   S	35        U R                  U5        U$ s  snf )
z@Load pdparams and convert to numpy state dict with HF key names.r   NzLoading weights from: c              3   F   >#    U  H  nTR                  U5      v   M     g 7frx   )
startswith)rz   r   r   s     r7   r}   3WeightConverter._convert_weights.<locals>.<genexpr>  s     ,TmQ\\!__ms   !zDropped z keys not needed for inferencez$Pre-map fusion for %s: %d -> %d keyszNo key mapping defined for z). Keys will be saved as-is from pdparams.)r   r   r   r   r   r   r   r   r1   r   _PRE_MAP_FUSERSr   r   r+   r   _postprocess_weights)r   r   r   r   resolved_pathr   r   droppedr   fuserbefores         `    r7   r    WeightConverter._convert_weights  s=   +DOO<-m_=>[[/
%%a,Tm,T)T:   qM xG~5STU#J/
&z2##DOO4]FXHLL6H	 (?HOO-doo-> ?: :
 	!!(+?s   E23E2c           	         SSK n[        R                  " U R                  0 5      nSnXA;   a  UR                  S5      S:X  a  UR                  S0 5      n[	        U5      S-   nX   R
                  S   nXv:  ae  UR                  Xg-
  X   R
                  S   4X   R                  S9nUR                  X   U/SS	9X'   [        R                  " S
U SU SU S35        Sn	X;   aj  UR                  S5      S:X  aU  UR                  SS5      n
X   R
                  S   U
:w  a.  X   R                  5       X'   [        R                  " SU	 S35        UR                  S5      S:X  a  U Vs/ s HE  nUR                  S5      (       d  M  UR                  S5      (       a  M3  UR                  SS5      PMG     nnSnU H"  nX;  d  M
  UR                  S5      X'   US-  nM$     U(       a  [        R                  " SU S35        gggs  snf )z:Apply model-specific post-processing to converted weights.r   Nz"model.denoising_class_embed.weight
model_typert_detrid2labelr   )r   )axiszPadded z from z to z (added background class)zlm_head.weightpp_chart2table
vocab_sizei4Q zReverted transpose on z (tied embedding, not linear)z.running_meanzmodel.backbone.z.num_batches_trackedzAdded z num_batches_tracked keys)r   r2   r   r   r   r   zerosr   concatenater   r   r   r   r  r   int64)r   r   r   r3   	embed_keyr  expected_sizecurrent_sizepadlm_head_keyr  r   nbt_keysaddeds                 r7   r  $WeightConverter._postprocess_weights  s'   ""4??B78	 VZZ%=%Jzz*b1HMA-M#.44Q7L+hh"183F3L3LQ3OP"-33   ')nn(#. '5 '# i[|nD$o%>@ '"vzz,'?CS'SL&9J$**1-;(0(=(G(G(I%,[M :3 4
 ::l#y0 "!A::o. C78||DU7V C		/+AB!  
 E$"$((1+HKQJE  veW,EFG  1s   'HHHc                     SSK Jn  [        R                  R	                  U R
                  S5      nU" X5        [        R                  " SU 35        g)z+Save numpy state dict as model.safetensors.r   )	save_filezmodel.safetensorszSaved model.safetensors to: N)safetensors.numpyr!  r   pathr   r   r   r   )r   r   r!  out_paths       r7   r   !WeightConverter._save_safetensors  s:    /77<<1DE(%3H:>?r9   c                 v   U R                   R                  S[        R                  " U R                  0 5      5      n[        R
                  R                  U R                  S5      n[        USSS9 n[        R                  " XSSS9  SSS5        [        R                  " S	U 35        g! , (       d  f       N(= f)
u7   Save config.json — user-provided or official default.r   wr   r   r   Findentensure_asciiNzSaved config.json to: )r   r   r2   r   r   r#  r   r   r   r   dumpr   r   r   datar$  r   s       r7   r   "WeightConverter._save_model_config	  s    !!%%door2
 77<<?(C'2aIIdae< 3-hZ89 32s   1B**
B8c                 &   U R                   [        ;   a  gSU R                  ;   a  U R                  S   nO^[        [        R
                  " U R                   0 5      5      nU R                   [        ;   a   S/[        U R                   5      -   S/-   US'   [        R                  R                  U R                  S5      n[        USSS9 n[        R                  " XS	S
S9  SSS5        [        R                   " SU 35        g! , (       d  f       N(= f)uD   Save preprocessor_config.json — user-provided or official default.Nr   blank character_listr'  r   r   r   Fr(  z#Saved preprocessor_config.json to: )r   r   r   r   r   r   _REC_CHARACTER_DICT_MODELSr0   r   r#  r   r   r   r   r+  r   r   r,  s       r7   r   )WeightConverter._save_preprocessor_config  s    ??22 %););;%%&@AD,00"EFD"<<I 3DOO DDuL %& 77<<1KL(C'2aIIdae< 3:8*EF 32s   	D
Dc                    SSK nSU R                  ;   a  U R                  S   nOSSU R                  00nUR                  [	        U R                  5      5        U R                  [
        ;   a)  [        U R                  5      UR                  S0 5      S'   ONU R                  [        ;   a:  [        [        5      nSUS	'   U R                  5       US
.UR                  S0 5      S'   [        R                  R                  U R                  S5      n[!        USSS9 nUR#                  UUSSS9  SSS5        [$        R&                  " SU 35        g! , (       d  f       N(= f)u9   Save inference.yml — user-provided or official default.r   Nr   r   r   PostProcesscharacter_dictVariableDonutProcessorprocessor_class)fast_tokenizer_filetokenizer_config_filer'  r   r   FT)default_flow_styleallow_unicodezSaved inference.yml to: )r   r   r   updater,   r3  r0   
setdefaultr   r   r(   _load_unimernet_fast_tokenizerr   r#  r   r   r   r+  r   r   )r   r   r-  tokenizer_configr$  r   s         r7   r   #WeightConverter._save_inference_yml)  s9   d000%%o6D|T__=>DKK,T__=>"<< (8 r2$ $88 $((B#C 6N !23+/+N+N+P-=Hr23CD
 77<<A(C'2aII#("	   3 	/z:; 32s   E
Ec           	         SSK nU R                  5       n[        R                  R	                  U R
                  S5      nUR                  X#5        [        R                  " SU 35        [        [        [        [        S.nUR                  5        GH$  u  pVU R                  (       a  [        U R                   5      U-  nUR#                  5       (       a8  [$        R&                  " [)        USS95      n[        R                  " SU 35        O8Un[        R*                  " U S	U R                    S
U R,                   S35        OUn[        R                  R	                  U R
                  U5      n	[)        U	SSS9 n
[$        R.                  " XSSS9  SSS5        [        R                  " SU SU	 35        GM'     g! , (       d  f       N0= f)zSave tokenizer and generation config for Chart2Table models.

Outputs: qwen.tiktoken, added_tokens.json, generation_config.json,
special_tokens_map.json, tokenizer_config.json.
r   Nqwen.tiktokenzCopied qwen.tiktoken to: )zadded_tokens.jsongeneration_config.jsonzspecial_tokens_map.jsontokenizer_config.jsonr   r   zLoaded user tokenizer config: r   . Using default for r   r'  r   Fr(  Saved  to: )shutil_resolve_tiktoken_sourcer   r#  r   r   copy2r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r+  )r   rJ  tiktoken_srctiktoken_dst_TOKENIZER_DEFAULTSr   default_datasrcr-  r$  r   s              r7   r    WeightConverter._save_llm_configN  sj    	 446ww||DOO_E\00?@ ":&C'E%A	
 $7#<#<#>E!!4??+e3::<<99T#%@ADLL#A#!GH'DOO '/@ A--1__,=Q@
 $ww||DOOU;Hhg6!		$!%@ 7LL6%hZ89% $?  76s   9F::
G	c                    U R                   (       a\  [        U R                  5      S-  nUR                  5       (       a  [	        U5      $ [
        R                  " SU R                   S35        SSKJn  [        U5      S-  U R                   S3-  S-  nUR                  5       (       a  [	        U5      $ [        SU S	35      e)
z.Find qwen.tiktoken for Chart2Table conversion.rD  zqwen.tiktoken not found in '. Falling back to official model cache.r   	CACHE_DIRofficial_models_safetensorszWqwen.tiktoken not found. For single-file input, ensure the official model is cached at ze (run inference once to download). For directory input, include qwen.tiktoken in the input directory.r   r   r   r   r   r   r   utils.cacherV  r   r   r   rQ  rV  
cache_paths       r7   rK  (WeightConverter._resolve_tiktoken_sourcew  s    t'/9Czz||3xOO-doo-> ?8 8 	- O !./  	 z?""", .QR
 	
r9   c                    U R                   (       a\  [        U R                  5      S-  nUR                  5       (       a  [	        U5      $ [
        R                  " SU R                   S35        SSKJn  [        U5      S-  U R                   S3-  S-  nUR                  5       (       a  [	        U5      $ [        SU S	U R                   S
35      e)u   Resolve filesystem path to tokenizer.json (PP-FormulaNet fast tokenizer).

Resolution chain (mirrors :meth:`_resolve_tiktoken_source`):
    input dir → ~/.paddlex/official_models/{name}_safetensors/tokenizer.json
    → FileNotFoundError
tokenizer.jsonztokenizer.json not found in rT  r   rU  rW  rX  zXtokenizer.json not found. For single-file input, ensure the official model is cached at z (run `create_model('zs', engine='paddle_dynamic')` once to download). For directory input, include tokenizer.json in the input directory.rY  r[  s       r7   #_resolve_unimernet_tokenizer_source3WeightConverter._resolve_unimernet_tokenizer_source  s     t'*::Czz||3xOO.t.? @8 8
 	- O !./  	 z?""", ."oo. / 
 	
r9   c                     U R                  5       n[        R                  " SU 35        [        USS9 n[        R
                  " U5      sSSS5        $ ! , (       d  f       g= f)zKLoad tokenizer.json content as a parsed dict (for inference.yml embedding).zLoaded tokenizer.json: r   r   N)r`  r   r   r   r   r   )r   r#  r   s      r7   r@  .WeightConverter._load_unimernet_fast_tokenizer  sG    779.tf56$)Q99Q< *))s   A
A"c           	         SSK nU R                  5       n[        R                  R	                  U R
                  S5      nUR                  X#5        [        R                  " SU 35        [        [        [        S.nUR                  5        GH0  u  pVU R                  (       a  [        U R                  5      U-  nUR!                  5       (       aC  [#        USS9 n[$        R&                  " U5      n	SSS5        [        R                  " SU 35        O8Un	[        R(                  " U S	U R                   S
U R*                   S35        OUn	[        R                  R	                  U R
                  U5      n
[#        U
SSS9 n[$        R,                  " W	USSS9  SSS5        [        R                  " SU SU
 35        GM3     g! , (       d  f       N= f! , (       d  f       NA= f)u  Save HF-style assets for PP-FormulaNet (transformers-engine compatible).

Outputs ``processor_config.json``, ``generation_config.json``,
``tokenizer_config.json`` (all hardcoded), and copies ``tokenizer.json``
from the input dir or official_models cache. The tokenizer JSON is also
already embedded in inference.yml (read by UniMERNetDecode) — saving the
standalone file lets HF AutoTokenizer load the converted directory.
r   Nr_  zCopied tokenizer.json to: )zprocessor_config.jsonrE  rF  r   r   zLoaded user asset: r   rG  r   r'  r   Fr(  rH  rI  )rJ  r`  r   r#  r   r   rL  r   r   r'   r&   r(   r   r   r   r   r   r   r   r   r   r   r+  )r   rJ  tokenizer_srctokenizer_dst_ASSET_DEFAULTSr   rP  rQ  r   r-  r$  s              r7   r   *WeightConverter._save_pp_formulanet_assets  s|    	 @@BT__6FG]21-AB &@&A%?

 $3#8#8#:E!!4??+e3::<<cG4#yy| 5LL#6se!<='DOO '/@ A--1__,=Q@
 $ww||DOOU;Hhg6!		$!%@ 7LL6%hZ89' $; 54 76s   G?G
G	
G 	)r   r   r   r   r   r   N)__name__
__module____qualname____firstlineno____doc__r   r   r  r   r  r   r   r   r   r   rK  r`  r@  r   __static_attributes__ry   r9   r7   r5   r5   K  sU    B7<BQ,(T2Hh@	:G*#<J':R
:!
F *:r9   )@rm  r   r   pathlibr   utilsr   utils.configr   utils.pdparams2safetensorsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   'utils.pdparams2safetensors.model_configr2   r8   r   r  r3  r   r   r   r   r   r5   ry   r9   r7   <module>rt     s`    	   $+ + + + + + + + + + +X C#8 #0A ##or2#4# "OR#8# !?B"7	#
 68PQ# 68PQ# 68IJ# 68PQ# 46OP# 35NO# 68QR#$ 46OP%#& 57PQ'#( 24MN)#* ~34+#, N$89-#. o'<=/#0 )*?@-/KL%r*'5r&:*8")=*B/("-01-r22B7-/KLE#V 75801/  C D N<
;|&?T]: ]:r9   