add utils for data process

2025-04-29 18:28:45 +08:00
parent 552776c379
commit 4550697faf
1 changed files with 1 additions and 0 deletions
@@ -11,3 +11,4 @@ def extract_completions(json_file_path, output_file_path):
            output_file.write(completion + '\n')

 extract_completions('corpus/wikipedia-cn-20230720-filtered.json', 'corpus/wikipedia.txt')
+