Repository navigation
Expand file tree
/
Copy pathdata_loader.py
More file actions
57 lines (41 loc) · 1.58 KB
/
Copy pathdata_loader.py
File metadata and controls
57 lines (41 loc) · 1.58 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# -*- coding: utf-8 -*-
"""Data loading and saving utilities for corpus and QA pairs."""
import json
import os
from datetime import datetime
from config import CORPUS_PATH, MANUAL_QA_PATH, PROCESSED_DIR, QA_DIR, EVAL_DIR
def load_corpus(path=None):
path = path or CORPUS_PATH
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def load_qa_pairs(path=None):
path = path or MANUAL_QA_PATH
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def save_corpus(corpus, path=None):
path = path or os.path.join(PROCESSED_DIR, f"final_corpus_{datetime.now().strftime('%Y%m%d_%H%M')}.json")
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(corpus, f, ensure_ascii=False, indent=2)
return path
def save_qa_pairs(qa_pairs, path=None):
path = path or os.path.join(QA_DIR, "manual_qa.json")
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(qa_pairs, f, ensure_ascii=False, indent=2)
return path
def save_json(data, path):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
return path
def load_json(path):
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def list_eval_files(prefix=""):
if not os.path.exists(EVAL_DIR):
return []
return sorted(
[f for f in os.listdir(EVAL_DIR) if f.startswith(prefix)],
reverse=True,
)