fix: ensure UTF-8 encoding when reading text files in ingestion helper

akshayjalluri6 · akshayjalluri6 · commit b77f90e177da · 2025-01-23T14:42:21.000+05:30
diff --git a/private_gpt/components/ingest/ingest_helper.py b/private_gpt/components/ingest/ingest_helper.py
@@ -89,7 +89,7 @@ def _load_file_to_documents(file_name: str, file_data: Path) -> list[Document]:
             )
             # Read as a plain text
             string_reader = StringIterableReader()
-            return string_reader.load_data([file_data.read_text()])
+            return string_reader.load_data([file_data.read_text(encoding="utf8")])
 
         logger.debug("Specific reader found for extension=%s", extension)
         documents = reader_cls().load_data(file_data)

Original file line number	Diff line number	Diff line change
`@@ -89,7 +89,7 @@ def _load_file_to_documents(file_name: str, file_data: Path) -> list[Document]:`
`89`	`89`	`)`
`90`	`90`	`# Read as a plain text`
`91`	`91`	`string_reader = StringIterableReader()`
`92`		`- return string_reader.load_data([file_data.read_text()])`
	`92`	`+ return string_reader.load_data([file_data.read_text(encoding="utf8")])`
`93`	`93`
`94`	`94`	`logger.debug("Specific reader found for extension=%s", extension)`
`95`	`95`	`documents = reader_cls().load_data(file_data)`