Skip to content

Commit 59c012a

Browse files
RyanRanaCopilot
andauthored
Apply suggestions from code review
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
1 parent bdc1d54 commit 59c012a

1 file changed

Lines changed: 19 additions & 1 deletion

File tree

packages/moss-cli/src/moss_cli/documents.py

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,11 +68,26 @@ def _parse_jsonl_docs(raw: str, source: str = "input") -> List[DocumentInfo]:
6868

6969

7070
def _parse_csv_docs(content: str) -> List[DocumentInfo]:
71-
reader = csv.DictReader(io.StringIO(content))
71+
reader = csv.DictReader(io.StringIO(content, newline=""))
7272
if reader.fieldnames is None:
7373
raise typer.BadParameter("CSV is empty: expected a header row")
7474

7575
reader.fieldnames = [(name or "").strip() for name in reader.fieldnames]
76+
77+
seen = set()
78+
dupes = set()
79+
for name in reader.fieldnames:
80+
if not name:
81+
continue
82+
if name in seen:
83+
dupes.add(name)
84+
else:
85+
seen.add(name)
86+
if dupes:
87+
raise typer.BadParameter(
88+
f"CSV header has duplicate column name(s): {', '.join(sorted(dupes))}"
89+
)
90+
7691
missing = [name for name in ("id", "text") if name not in reader.fieldnames]
7792
if missing:
7893
raise typer.BadParameter(
@@ -104,6 +119,9 @@ def _parse_csv_docs(content: str) -> List[DocumentInfo]:
104119

105120

106121
def _parse_csv_json(value: Optional[str], column: str, line_no: int) -> Any:
122+
if value is None:
123+
return None
124+
value = value.strip()
107125
if not value:
108126
return None
109127
try:

0 commit comments

Comments
 (0)