rsj_CRLF_to_LF

only converted the files with actual changes from CRLF to LF
NOTE: github shows Warning: This diff contains a change from 'LF' to 'CRLF' -- ignore

Author: rsjoyner

src/pds_doi_service/core/db/transaction_on_disk.py

@@ -75,7 +75,9 @@ def get_transaction_key(node_id, doi, transaction_time):
 
         prefix, suffix = doi.split("/", maxsplit=1)
 
-        return os.path.join(transaction_dir, node_id, prefix, suffix, transaction_time.isoformat())
+        # 20250720: modify so transaction_time returns a string as YYYY-MM-DDThh:mm:ss.microseconds
+        # return os.path.join(transaction_dir, node_id, prefix, suffix, transaction_time.isoformat())
+        return os.path.join(transaction_dir, node_id, prefix, suffix, transaction_time.strftime("%Y-%m-%dT%H-%M-%S.%f"))
 
     @staticmethod
     def output_label_for_transaction(transaction_record):
@@ -165,7 +167,7 @@ def write(self, transaction_dir, input_ref=None, output_content=None, output_con
                     r = requests.get(input_ref, allow_redirects=True)
 
                     with open(full_input_name, "wb") as outfile:
-                        outfile.write(r.content)
+                        outfile.write(r.content, encoding="utf-8")
 
                     r.close()
 

src/pds_doi_service/core/entities/doi.py

@@ -117,8 +117,17 @@ class Doi:
     identifiers: list[dict] = field(default_factory=list)
     related_identifiers: list[dict] = field(default_factory=list)
     authors: Optional[list[dict]] = field(default_factory=list)  # type: ignore
+    # 20250501 -- add list_authors
+    # -- optional because older XML labels did not implement this field
+    list_authors: Optional[list[dict]] = field(default_factory=list)  # type: ignore
     keywords: set[str] = field(default_factory=get_global_keywords)
     editors: Optional[list[dict]] = field(default_factory=list)  # type: ignore
+    # 20250501 -- add list_editors
+    #   -- add contributors and list_contributors as they didn't exist in the DOI structure
+    # -- optional because older XML labels did not implement this field
+    list_editors: Optional[list[dict]] = field(default_factory=list)  # type: ignore
+    contributors: Optional[list[dict]] = field(default_factory=list)  # type: ignore
+    list_contributors: Optional[list[dict]] = field(default_factory=list)  # type: ignore
     description: Optional[str] = None
     id: Optional[str] = None
     doi: Optional[str] = None

src/pds_doi_service/core/input/input_util.py

@@ -91,6 +91,8 @@ def __init__(self, valid_extensions=None):
         # function pointers
         self._parser_map = {
             ".xml": self.parse_xml_file,
+            # 20250527; add .lblx to be parsed as xml
+            ".lblx": self.parse_xml_file,
             ".xls": self.parse_xls_file,
             ".xlsx": self.parse_xls_file,
             ".csv": self.parse_csv_file,
@@ -100,6 +102,25 @@ def __init__(self, valid_extensions=None):
         if not all([extension in self._parser_map for extension in self._valid_extensions]):
             raise ValueError("One or more the provided extensions are not supported by the DOIInputUtil class.")
 
+    # 20250501: Detect UTF-16/UTF-8-BOM; decode
+    def detect_and_decode_utf(Self, data: bytes) -> str:
+        # Detect and decode UTF-16 (with BOM)
+        if data.startswith(b"\xff\xfe") or data.startswith(b"\xfe\xff"):
+            logger.info(f": Detected UTF-16 BOM.")
+            return data.decode("utf-16")
+
+        try:
+            # Try decoding as UTF-8 with BOM (utf-8-sig handles BOM automatically)
+            logger.info(f": Trying to detect UTF-8 with BOM (utf-8-sig).")
+            decoded_data = data.decode("utf-8-sig")
+        except UnicodeDecodeError:
+            # Fallback
+            logger.info(f":Could not decode as UTF-8-sig. Using fallback UTF-8 with replacement.")
+            decoded_data = data.decode("utf-8", errors="replace")
+
+        dos_line_endings = decoded_data.replace("\r\n", "\n").replace("\r", "\n").replace("\n", "\r\n")
+        return dos_line_endings
+
     def parse_xml_file(self, xml_path):
         """
         Parses DOIs from a file with an .xml extension. The file is expected
@@ -473,12 +494,16 @@ def parse_json_file(self, json_path):
         validator = DOIServiceFactory.get_validator_service()
 
         # First read the contents of the file
-        with open(json_path, "r") as infile:
+        # 20250501: read as binary to avoid encoding issues
+        with open(json_path, "rb") as infile:
             # It's been observed that input files transferred from Windows-based
             # machines can append a UTF-8-BOM hex sequence, which breaks
             # JSON parsing later on. So we perform an encode-decode here to
             # ensure this sequence is stripped before continuing.
-            json_contents = infile.read().encode().decode("utf-8-sig")
+            # 20250501: modify code to call routine to detect and decode UTF-16/UTF-8-BOM
+            # json_contents = infile.read().encode().decode("utf-8-sig")
+            json_contents = infile.read()
+            json_contents = self.detect_and_decode_utf(json_contents)
 
         # Validate and parse the provide JSON label based on the service provider
         # configured within the INI. If there's a mismatch, the validation step
@@ -593,7 +618,7 @@ def _read_from_remote(self, input_url):
             raise InputFormatException(f"Could not read remote file {input_url}, reason: {str(http_err)}")
 
         with tempfile.NamedTemporaryFile(suffix=basename(parsed_url.path)) as temp_file:
-            temp_file.write(response.content)
+            temp_file.write(response.content, encoding="utf-8")
             temp_file.seek(0)
 
             dois = self._read_from_path(temp_file.name)