
    ip                        S SK r S SKJr  \S:X  a  \ R                  " 5       r\R                  SS\SSS9  \R                  S	S
\SSS9  \R                  SSSSS9  \R                  5       r	\" \	R                  5      r\	R                  (       a  \	R                  O\	R                  R                  SS5      r\R                  \\	R                  S9  gg)    N)Llama2cTokenizer__main__z-tz--tokenizer-modelztokenizer.modelz/path to tokenizer model, given by sentencepiece)typedefaulthelpz-oz--output-pathz,output path of postprocessed tokenizer modelz-pz--prepend-padding
store_truezDwhether to prepend a padding token to the beginning of the tokenizer)actionr   z.modelz.bin)prepend_padding)argparsepytorch_tokenizers.llama2cr   __name__ArgumentParserparseradd_argumentstr
parse_argsargstokenizer_modeltoutput_pathreplaceexportr
        g/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/pytorch_tokenizers/tools/llama2c/convert.py<module>r      s     7 z$$&F
!>   ;   S	   D--.A  	!!))(F; 
 HH[$*>*>H?A r   