Implement XTCE validation

- Ruff formatted and ruff lint fixes
- Remove namespace on test_xtce_no_namespace.xml and make namespace handling on definitions.py clearer
- Fix XML structure for XTCE test files
- Fix CTIM XTCE for validity
- Update changelog
- Tighten up warnings for no namespace for XtcePacketDefinition
- Add tests for XSD schema validity
- Add tests for different inputs to validation function
- Add test for different inputs to definition from_xtce
- Add structural validation and tests
- Add validate_xtce to top level module
- Add simple CLI to validation tool

Author: medley56

docs/source/changelog.md

@@ -14,6 +14,7 @@ Release notes for the `space_packet_parser` library
 - *BREAKING*: Removed mid-level abstraction methods `packet_generator()` and `ccsds_packet_generator()`
   from `XtcePacketDefinition`. Use low-level `parse_bytes()` with bytes generators directly, or high-level
   `space_packet_parser.xarr.create_dataset()` for xarray integration.
+- Add validation support for XTCE documents.
 - Add support for creating a packet definition from Python objects and serializing it as XML.
 - BUGFIX: Fix kbps calculation in packet generator for showing progress.
 - Add support for string and float encoded enumerated lookup parameters.

docs/source/users.md

@@ -376,6 +376,56 @@ This equation can be implemented in XTCE by referencing the packet length field
 </xtce:BinaryParameterType>
 ```
 
+## XTCE Document Validation
+
+Space Packet Parser provides comprehensive validation capabilities for XTCE documents to help ensure they are correct and will work properly for parsing packets. The validation system operates in two modes:
+
+- **Schema Validation**: Validates the XML document against the in-document referenced XTCE XSD schema
+- **Structural Validation**: Validates XTCE-specific structure and reference integrity
+
+Schema validation requires correct namespacing declarations at the top of your XTCE document.
+e.g.
+
+```xml
+<xtce:SpaceSystem name="SpacePacketParser"
+                  xmlns:xtce="http://www.omg.org/spec/XTCE/20180204"
+                  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                  xsi:schemaLocation="http://www.omg.org/spec/XTCE/20180204
+                                      https://www.omg.org/spec/XTCE/20180204/SpaceSystem.xsd">
+```
+
+### Basic Validation Usage
+
+```python
+from space_packet_parser import validate_xtce
+
+# Validate an XTCE file against the referenced schema
+result = validate_xtce("my_xtce.xml", level="schema")
+if result.errors:
+    for error in result.errors:
+        print(f"Error: {error}")
+else:
+    print("Document is valid")
+
+# Validate an XTCE document structure to check for
+# unused Parameters ParameterTypes and nonexistent references
+result = validate_xtce("my_xtce.xml", level="structure")
+if result.errors:
+    for error in result.errors:
+        print(f"Error: {error}")
+else:
+    print("Document is valid")
+
+# Comprehensive validation (both schema and structure)
+result = validate_xtce("my_xtce.xml", level="all")
+print(f"Validation completed in {result.validation_time_ms:.1f}ms")
+if result.errors:
+    for error in result.errors:
+        print(f"Error: {error}")
+else:
+    print("Document is valid")
+```
+
 ## Troubleshooting Packet Parsing
 Parsing binary packets is error-prone and getting the XTCE definition correct can be a challenge at first.
 Most flight software teams can export XTCE from their command and telemetry database but these exports usually require

pyproject.toml

@@ -150,4 +150,7 @@ select = [
     "UP"  # pyupgrade syntax upgrader
 ]
 per-file-ignores = { "tests/*" = ["S"] }
-ignore = ["E501"] # Ignore line length errors, as we trust ruff format to handle this.
+ignore = [
+    "E501", # Ignore line length errors, as we trust ruff format to handle this.
+    "F541" # Allow f strings as docstrings
+]

space_packet_parser/__init__.py

@@ -6,12 +6,14 @@
 from space_packet_parser.ccsds import ccsds_generator
 from space_packet_parser.common import SpacePacket
 from space_packet_parser.xtce.definitions import XtcePacketDefinition
+from space_packet_parser.xtce.validation import validate_xtce
 
 __all__ = [
     "ccsds_generator",
     "SpacePacket",
     "XtcePacketDefinition",
     "load_xtce",
+    "validate_xtce",
 ]
 
 

space_packet_parser/cli.py

@@ -26,6 +26,7 @@
 from space_packet_parser import ccsds
 from space_packet_parser.ccsds import ccsds_generator
 from space_packet_parser.xtce.definitions import DEFAULT_ROOT_CONTAINER, XtcePacketDefinition
+from space_packet_parser.xtce.validation import validate_xtce
 
 # Initialize a console instance for rich output
 console = Console()
@@ -208,3 +209,42 @@ def parse(
     # Limit the number of packets and variables printed
     # also limit the length of strings (binary data can be long)
     pretty.pprint(packets, indent_guides=False, max_length=max_items, max_string=max_string)
+
+
+@spp.command()
+@click.argument("file_path", type=click.Path(exists=True, path_type=Path))
+@click.option(
+    "--level",
+    type=click.Choice(["schema", "structure", "all"], case_sensitive=False),
+    default="all",
+    help="Validation level to perform",
+)
+@click.option("--timeout", type=int, default=30, help="Timeout in seconds for schema downloads")
+def validate(file_path: Path, level: str, timeout: int) -> None:
+    """Validate an XTCE document."""
+    logging.debug(f"Validating XTCE file: {file_path}")
+    logging.debug(f"Validation level: {level}")
+
+    result = validate_xtce(file_path, level=level.lower(), timeout=timeout)
+
+    if result.valid:
+        console.print(f"[bold green]✓ VALID[/bold green] ({result.validation_level.value} level)")
+    else:
+        console.print(f"[bold red]✗ INVALID[/bold red] ({result.validation_level.value} level)")
+
+    if result.schema_location:
+        console.print(f"Schema: {result.schema_location}")
+    if result.schema_version:
+        console.print(f"Version: {result.schema_version}")
+
+    if result.validation_time_ms:
+        console.print(f"Validation time: {result.validation_time_ms:.1f}ms")
+
+    if result.errors:
+        console.print(f"\n[bold red]Errors ({len(result.errors)}):[/bold red]")
+        for error in result.errors:
+            console.print(f"  {error}")
+
+    # Exit with error code if validation failed
+    if not result.valid:
+        raise click.ClickException("Validation failed")

space_packet_parser/xtce/__init__.py

@@ -3,7 +3,7 @@
 This module contains Python object representations of XTCE UML/XML models
 """
 
-DEFAULT_XTCE_NS_PREFIX = "xtce"  # Standard XTCE prefix using in xmlns:prefix="url" attribute
+STANDARD_XTCE_NS_PREFIX = "xtce"  # Standard XTCE prefix using in xmlns:prefix="url" attribute
 
 XTCE_1_2_XSD_URL = "https://www.omg.org/spec/XTCE/20180204/SpaceSystem.xsd"
 XTCE_1_2_XMLNS = "https://www.omg.org/spec/XTCE/20180204"
@@ -14,4 +14,4 @@
 # Note: There is no XSD available from omg.org for XTCE 1.0
 
 XTCE_URI = XTCE_1_2_XMLNS
-DEFAULT_XTCE_NSMAP = {DEFAULT_XTCE_NS_PREFIX: XTCE_1_2_XMLNS, "xsi": "http://www.w3.org/2001/XMLSchema-instance"}
+STANDARD_XTCE_NSMAP = {STANDARD_XTCE_NS_PREFIX: XTCE_1_2_XMLNS, "xsi": "http://www.w3.org/2001/XMLSchema-instance"}

space_packet_parser/xtce/definitions.py

@@ -14,8 +14,8 @@
 from space_packet_parser import ccsds, common
 from space_packet_parser.exceptions import InvalidParameterTypeError, UnrecognizedPacketTypeError
 from space_packet_parser.xtce import (
-    DEFAULT_XTCE_NS_PREFIX,
-    DEFAULT_XTCE_NSMAP,
+    STANDARD_XTCE_NS_PREFIX,
+    STANDARD_XTCE_NSMAP,
     containers,
     parameter_types,
     parameters,
@@ -49,8 +49,8 @@ def __init__(
         self,
         container_set: Optional[Iterable[containers.SequenceContainer]] = None,
         *,
-        ns: dict = DEFAULT_XTCE_NSMAP,
-        xtce_ns_prefix: Optional[str] = DEFAULT_XTCE_NS_PREFIX,
+        ns: dict = STANDARD_XTCE_NSMAP,
+        xtce_ns_prefix: Optional[str] = STANDARD_XTCE_NS_PREFIX,
         root_container_name: Optional[str] = DEFAULT_ROOT_CONTAINER,
         space_system_name: Optional[str] = None,
         validation_status: str = "Unknown",
@@ -69,10 +69,10 @@ def __init__(
             e.g. every Parameter object named `MY_PARAM` must be the same class instance.
         ns : dict
             XML namespace mapping, expected as a dictionary with the keys being namespace labels and
-            values being namespace URIs. Default {DEFAULT_XTCE_NSMAP}. An empty dictionary indicates no namespace
+            values being namespace URIs. Default {STANDARD_XTCE_NSMAP}. An empty dictionary indicates no namespace
             awareness, in which case `xtce_ns_prefix` must be None.
         xtce_ns_prefix : str
-            XTCE namespace prefix. Default {DEFAULT_XTCE_NS_PREFIX}. This is the key for the XTCE namespace in the
+            XTCE namespace prefix. Default {STANDARD_XTCE_NS_PREFIX}. This is the key for the XTCE namespace in the
             namespace mapping dictionary, `ns` and is used to write XML output when necessary.
         root_container_name : Optional[str]
             Name of root sequence container (where to start parsing)
@@ -129,8 +129,13 @@ def _update_caches(sc: containers.SequenceContainer) -> None:
                 _update_caches(sequence_container)
 
         self.ns = ns  # Default ns dict used when creating XML elements
-        self.xtce_schema_uri = ns[xtce_ns_prefix] if ns else None  # XTCE schema URI
-        self.xtce_ns_prefix = xtce_ns_prefix
+        # If the ns dict exists but xtce_ns_prefix is not in it
+        # (including the None key representing a default namespace),
+        # we assume the document is using no namespace awareness.
+        self.xtce_ns_uri = ns[xtce_ns_prefix] if ns and xtce_ns_prefix in ns else None  # XTCE namespace URI
+        self.xtce_ns_prefix = (
+            xtce_ns_prefix  # This is basically an alias to the ns URI (not to be confused with the XSD schema URL)
+        )
         self.root_container_name = root_container_name
         self.space_system_name = space_system_name
         self.validation_status = validation_status
@@ -154,11 +159,17 @@ def to_xml_tree(self) -> ElementTree.ElementTree:
         -------
         : ElementTree.ElementTree
         """
+        if self.xtce_ns_uri not in self.ns.values():
+            warnings.warn(
+                "No XTCE namespace defined. This is invalid per XSD, but will be serialized. "
+                "Ensure mydef.xtce_ns_prefix is a key in mydef.ns for valid XTCE output.",
+                UserWarning,
+            )
         # ElementMaker element factory with predefined namespace and namespace mapping
         # The XTCE namespace actually defines the XTCE elements
         # The ns mapping just affects the serialization of XTCE elements
         # Both can be None, resulting in no namespace awareness
-        elmaker = ElementMaker(namespace=self.xtce_schema_uri, nsmap=self.ns)
+        elmaker = ElementMaker(namespace=self.xtce_ns_uri, nsmap=self.ns)
 
         space_system_attrib = {}
         if self.space_system_name:
@@ -196,7 +207,6 @@ def from_xtce(
         cls,
         xtce_document: Union[str, Path, TextIO],
         *,
-        xtce_ns_prefix: Optional[str] = DEFAULT_XTCE_NS_PREFIX,
         root_container_name: Optional[str] = DEFAULT_ROOT_CONTAINER,
     ) -> "XtcePacketDefinition":
         f"""Instantiate an object representation of a CCSDS packet definition,
@@ -217,27 +227,45 @@ def from_xtce(
         ----------
         xtce_document : TextIO
             Path to XTCE XML document containing packet definition.
-        xtce_ns_prefix : Optional[str]
-            The namespace prefix associated with the XTCE xmlns attribute. Default is {DEFAULT_XTCE_NS_PREFIX}.
-            None means XTCE is the default namespace for elements with no prefix. The namespace mapping itself is
-            parsed out of the XML automatically.
         root_container_name : Optional[str]
             Optional override to the root container name. Default is {DEFAULT_ROOT_CONTAINER}.
         """
         # Define a namespace and prefix aware Element subclass so that we don't have to pass the namespace
         # into every from_xml method
         xtce_element_class = common.NamespaceAwareElement
-        xtce_element_lookup = ElementTree.ElementDefaultClassLookup(element=common.NamespaceAwareElement)
+        xtce_element_lookup = ElementTree.ElementDefaultClassLookup(element=xtce_element_class)
         xtce_parser = ElementTree.XMLParser()
         xtce_parser.set_element_class_lookup(xtce_element_lookup)
 
         tree = ElementTree.parse(xtce_document, parser=xtce_parser)  # noqa: S320
 
-        xtce_element_class.set_ns_prefix(xtce_ns_prefix)
-        xtce_element_class.set_nsmap(tree.getroot().nsmap)
-
         space_system = tree.getroot()
-        ns = tree.getroot().nsmap
+        ns = space_system.nsmap
+
+        # Search nsmap dict for the XTCE namespace prefix, if present (may be absent)
+        possible_prefixes = [pre for pre, uri in ns.items() if "xtce" in uri.lower()]
+
+        if len(possible_prefixes) == 1:
+            # Exactly one namespace (possibly prefixed) that looks like XTCE
+            xtce_ns_prefix = possible_prefixes[0]
+        elif len(possible_prefixes) == 0:
+            # This indicates no namespace is present (no xmlns attribute for XTCE)
+            # Some XML documents do not use namespaces at all, which is invalid per the XTCE XSD and will fail XSD validation
+            # We make an effort to parse these documents anyway, but warn the user
+            xtce_ns_prefix = None
+            warnings.warn(
+                "No XTCE namespace found in the document. This is invalid per XSD, but will be parsed. "
+                "Add an `xmlns` attribute to the root XML element to enable namespace awareness.",
+                UserWarning,
+            )
+        else:
+            # If there are multiple namespaces that look like XTCE, we cannot determine which one to use
+            raise ValueError(f"Multiple XTCE namespace prefixes found in the document: {possible_prefixes}. ")
+
+        # These change class attributes on the NamespaceAwareElement class,
+        # which allow the XTCE parser to correctly handle namespaces when parsing (and serializing) elements later on
+        xtce_element_class.set_ns_prefix(xtce_ns_prefix)
+        xtce_element_class.set_nsmap(ns)
 
         header = space_system.find("Header")