Spaces:
Sleeping
Sleeping
File size: 647 Bytes
17ff0d8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
from datasets import DatasetDict, load_from_disk
tokenized_data_path = (
"/home/lily/jt856/documents/simplex-diffusion/processed_data/openwebtext_50"
)
output_dir = (
"/home/lily/jt856/documents/simplex-diffusion/processed_data/openwebtext_50_split"
)
seed = 42
validation_split_ratio = 0.001
tokenized_datasets = load_from_disk(tokenized_data_path)
train_testvalid = tokenized_datasets["train"].train_test_split(
test_size=validation_split_ratio, shuffle=True, seed=seed
)
tokenized_datasets = DatasetDict(
{"train": train_testvalid["train"], "validation": train_testvalid["test"]}
)
tokenized_datasets.save_to_disk(output_dir)
|