utils.py

from datasets import Dataset, DatasetDict

def read_txt(input_txt: str):
    with open(input_txt, 'r', encoding='utf-8') as fin:
        for lineno, line in enumerate(fin, 1):
            parts = line.strip().split('=', 2)
            if len(parts) != 3:
                print(f"⚠️ Skipping malformed line {lineno}")
                continue
            _, schema, ontology = [p.strip() for p in parts]
            yield schema, ontology

def build_examples(train_path, test_path, system_prompt) -> DatasetDict:
    train_data = list(read_txt(train_path))
    test_data = list(read_txt(test_path))

    def row(input: str, output: str):
        return {
            "user_text": input,
            "messages": [
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": input},
                {"role": "assistant", "content": output},
            ]
        }
    train = list(map(lambda x: row(x[0], x[1]), train_data))
    eval_ = list(map(lambda x: row(x[0], x[1]), test_data))
    return DatasetDict({"train": Dataset.from_list(train), "eval": Dataset.from_list(eval_)})