happy-llm/docs/chapter6/code/whole.ipynb

整体代码拆分

Tokenizer

1. 训练一个 tokenzier

In [ ]:

import os
from tokenizers import ByteLevelBPETokenizer

dir_path = ""
paths = os.listdir(dir_path)

# 使用字节级的 BPE 分词器
tokenizer = ByteLevelBPETokenizer()

# 进行训练
# vocab_size：词表大小
# min_frequency：最小词频
# special_tokens：特殊 token 列表
tokenizer.train(files=paths, vocab_size=52_000, min_frequency=2, special_tokens=[
    "<s>",
    "<pad>",
    "</s>",
    "<unk>",
    "<mask>",
])

# 训练完成后手动保存
tokenizer.save_model(".", "esperberto")

In [ ]:

# 测试一下
from tokenizers.implementations import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing


tokenizer = ByteLevelBPETokenizer(
    "./models/EsperBERTo-small/vocab.json",
    "./models/EsperBERTo-small/merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
    ("</s>", tokenizer.token_to_id("</s>")),
    ("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)

print(
    tokenizer.encode("Mi estas Julien.")
)
# Encoding(num_tokens=7, ...)
# tokens: ['<s>', 'Mi', 'Ġestas', 'ĠJuli', 'en', '.', '</s>']