admin/LLM-from-scratch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add utils for data process

Browse Source
This commit is contained in:
coffee
2025-04-29 18:28:45 +08:00
parent 57a4401c54
commit f6206ce421
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/tans_json_to_txt.py
+1
View File
@@ -11,3 +11,4 @@ def extract_completions(json_file_path, output_file_path):
output_file.write(completion + '\n') output_file.write(completion + '\n')
extract_completions('corpus/wikipedia-cn-20230720-filtered.json', 'corpus/wikipedia.txt') extract_completions('corpus/wikipedia-cn-20230720-filtered.json', 'corpus/wikipedia.txt')