happy-llm/docs/chapter6/code/process_dataset.ipynb

In [1]:

import json

fw = open("autodl-tmp/dataset/pretrain_data/mobvoi_seq_monkey_general_open_corpus_small.jsonl", "w")
i = 0
with open("autodl-tmp/dataset/pretrain_data/mobvoi_seq_monkey_general_open_corpus.jsonl", "r") as f:
    while i <= 1000000:
        line = f.readline()
        fw.write(line)
        i += 1
fw.close()