mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-14 23:21:04 +00:00
cceebc631e
- Rename experiencemaker package to reme_ai - Move personal modules to new directory structure - Remove unused classes and imports - Update module initialization files
29 lines
1.2 KiB
Python
29 lines
1.2 KiB
Python
import json
|
|
import random
|
|
import argparse
|
|
|
|
def split_jsonl(input_file, train_file, val_file, ratio=0.8):
|
|
with open(input_file, 'r', encoding='utf-8') as f:
|
|
data = [json.loads(line) for line in f]
|
|
random.shuffle(data)
|
|
|
|
split_idx = int(len(data) * ratio)
|
|
train_data = data[:split_idx]
|
|
val_data = data[split_idx:]
|
|
|
|
with open(train_file, 'w', encoding='utf-8') as f:
|
|
for item in train_data:
|
|
f.write(json.dumps(item, ensure_ascii=False) + '\n')
|
|
with open(val_file, 'w', encoding='utf-8') as f:
|
|
for item in val_data:
|
|
f.write(json.dumps(item, ensure_ascii=False) + '\n')
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description='Split JSONL file into train and validation sets.')
|
|
parser.add_argument('--input', required=True, help='Path to input JSONL file')
|
|
parser.add_argument('--train', required=True, help='Path to output train file')
|
|
parser.add_argument('--val', required=True, help='Path to output validation file')
|
|
parser.add_argument('--ratio', type=float, default=0.5, help='Train ratio (default: 0.8)')
|
|
|
|
args = parser.parse_args()
|
|
split_jsonl(args.input, args.train, args.val, args.ratio)
|