From 51d56faf9fc0868e7b3d57111a400238fb19aeb7 Mon Sep 17 00:00:00 2001 From: coffee Date: Fri, 9 May 2025 18:56:08 +0800 Subject: [PATCH] update trans code --- utils/trans_json_to_txt.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 utils/trans_json_to_txt.py diff --git a/utils/trans_json_to_txt.py b/utils/trans_json_to_txt.py new file mode 100755 index 0000000..f57b5b8 --- /dev/null +++ b/utils/trans_json_to_txt.py @@ -0,0 +1,14 @@ +import json + +def extract_completions(json_file_path, output_file_path): + with open(json_file_path, 'r', encoding='utf-8') as json_file: + data = json.load(json_file) + + completions = [conversion["completion"] for conversion in data] + + with open(output_file_path, 'w', encoding='utf-8') as output_file: + for completion in completions: + output_file.write(completion + '\n') + +extract_completions('corpus/wikipedia-cn-20230720-filtered.json', 'corpus/wikipedia.txt') +