Add metadata fields to graph structure (#532)

* Add metadata fields to graph structure for deduplication and traceability

Implements #460 - adds metadata at both graph and node levels:

Graph-level metadata:
- provider: LLM provider used (e.g., openai, ollama)
- model: Model name used (e.g., gpt-4o)
- temperature: Temperature setting for generation
- created_at: ISO 8601 timestamp when graph was created

Node-level metadata (auto-generated in metadata dict):
- uuid: UUID4 for stable node identification across graph modifications
- topic_hash: SHA256 hash of topic string for duplicate detection

Backward compatible - old graph JSON files without metadata still load
correctly, with uuid/topic_hash auto-generated on load.

Signed-off-by: Luke Hinds <lukehinds@gmail.com>

* Remove incorrectly included graph

* Preserve timestamp for generation time of graphs

Signed-off-by: Luke Hinds <lukehinds@gmail.com>

---------

Signed-off-by: Luke Hinds <lukehinds@gmail.com>

Author: lukehinds

deepfabric/config.py

@@ -161,8 +161,7 @@ def validate_configuration(self):
         """Validate that configuration combinations are consistent."""
         if self.reasoning_style is not None and self.type != "cot":
             raise ValueError(
-                f"reasoning_style can only be set when type='cot', "
-                f"got type='{self.type}'"
+                f"reasoning_style can only be set when type='cot', got type='{self.type}'"
             )
 
         if self.type == "cot" and self.reasoning_style is None:

deepfabric/graph.py

@@ -1,7 +1,10 @@
 import asyncio
+import hashlib
 import json
 import textwrap
+import uuid
 
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any
 
 from pydantic import BaseModel, ConfigDict, Field
@@ -69,6 +72,15 @@ class GraphConfig(BaseModel):
     )
 
 
+class GraphMetadata(BaseModel):
+    """Metadata for the entire graph for provenance tracking."""
+
+    provider: str = Field(..., description="LLM provider used (e.g., openai, ollama)")
+    model: str = Field(..., description="Model name used (e.g., gpt-4o)")
+    temperature: float = Field(..., description="Temperature setting used for generation")
+    created_at: str = Field(..., description="ISO 8601 timestamp when graph was created")
+
+
 class NodeModel(BaseModel):
     """Pydantic model for a node in the graph."""
 
@@ -84,6 +96,9 @@ class GraphModel(BaseModel):
 
     nodes: dict[int, NodeModel]
     root_id: int
+    metadata: GraphMetadata | None = Field(
+        default=None, description="Graph-level metadata for provenance tracking"
+    )
 
 
 class Node:
@@ -96,6 +111,14 @@ def __init__(self, topic: str, node_id: int, metadata: dict[str, Any] | None = N
         self.parents: list[Node] = []
         self.metadata: dict[str, Any] = metadata.copy() if metadata is not None else {}
 
+        # Auto-generate uuid if not present (stable node identification)
+        if "uuid" not in self.metadata:
+            self.metadata["uuid"] = str(uuid.uuid4())
+
+        # Auto-generate topic_hash if not present (duplicate detection via SHA256)
+        if "topic_hash" not in self.metadata:
+            self.metadata["topic_hash"] = hashlib.sha256(topic.encode("utf-8")).hexdigest()
+
     def to_pydantic(self) -> NodeModel:
         """Converts the runtime Node to its Pydantic model representation."""
         return NodeModel(
@@ -140,6 +163,9 @@ def __init__(self, **kwargs):
         # Progress reporter for streaming feedback (set by topic_manager)
         self.progress_reporter: ProgressReporter | None = None
 
+        # Store creation timestamp for provenance tracking
+        self.created_at: datetime = datetime.now(timezone.utc)
+
         trace(
             "graph_created",
             {
@@ -181,6 +207,12 @@ def to_pydantic(self) -> GraphModel:
         return GraphModel(
             nodes={node_id: node.to_pydantic() for node_id, node in self.nodes.items()},
             root_id=self.root.id,
+            metadata=GraphMetadata(
+                provider=self.provider,
+                model=self.model_name,
+                temperature=self.temperature,
+                created_at=self.created_at.isoformat(),
+            ),
         )
 
     def to_json(self) -> str:
@@ -203,6 +235,12 @@ def from_json(cls, json_path: str, params: dict) -> "Graph":
         graph = cls(**params)
         graph.nodes = {}
 
+        # Restore original creation timestamp if present in the loaded graph
+        if graph_model.metadata and graph_model.metadata.created_at:
+            # Handle 'Z' suffix for Python < 3.11 compatibility
+            created_at_str = graph_model.metadata.created_at.replace("Z", "+00:00")
+            graph.created_at = datetime.fromisoformat(created_at_str)
+
         # Create nodes
         for node_model in graph_model.nodes.values():
             node = Node(node_model.topic, node_model.id, node_model.metadata)

tests/unit/test_topic_graph.py

@@ -1,6 +1,8 @@
 import asyncio
+import hashlib
 import json
 import tempfile
+import uuid as uuid_module
 
 from pathlib import Path
 from unittest.mock import AsyncMock, patch
@@ -10,6 +12,7 @@
 from deepfabric.graph import (
     Graph,
     GraphConfig,
+    GraphMetadata,
     GraphModel,
     Node,
     NodeModel,
@@ -72,6 +75,51 @@ def test_node_to_pydantic(self):
         assert child_model.children == []
         assert child_model.parents == [1]
 
+    def test_node_uuid_generation(self):
+        """Test that nodes automatically get a UUID4 in metadata."""
+        node = Node("Test topic", 42)
+        assert "uuid" in node.metadata
+        # Verify it's a valid UUID4 format
+        parsed_uuid = uuid_module.UUID(node.metadata["uuid"])
+        assert parsed_uuid.version == 4  # noqa: PLR2004
+
+    def test_node_topic_hash_generation(self):
+        """Test that nodes automatically get a SHA256 topic_hash in metadata."""
+        topic = "Test topic for hashing"
+        node = Node(topic, 42)
+        assert "topic_hash" in node.metadata
+        # Verify it matches the expected SHA256 hash
+        expected_hash = hashlib.sha256(topic.encode("utf-8")).hexdigest()
+        assert node.metadata["topic_hash"] == expected_hash
+        # SHA256 hex digest is 64 characters
+        assert len(node.metadata["topic_hash"]) == 64  # noqa: PLR2004
+
+    def test_node_preserves_existing_metadata(self):
+        """Test that existing metadata is not overwritten during node creation."""
+        existing_uuid = "custom-uuid-value"
+        existing_hash = "custom-hash-value"
+        existing_metadata = {
+            "uuid": existing_uuid,
+            "topic_hash": existing_hash,
+            "custom_field": "custom_value",
+        }
+        node = Node("Test topic", 42, metadata=existing_metadata)
+
+        # Existing values should be preserved
+        assert node.metadata["uuid"] == existing_uuid
+        assert node.metadata["topic_hash"] == existing_hash
+        assert node.metadata["custom_field"] == "custom_value"
+
+    def test_node_metadata_in_pydantic(self):
+        """Test that uuid and topic_hash are included in Pydantic conversion."""
+        node = Node("Test topic", 42)
+        pydantic_model = node.to_pydantic()
+
+        assert "uuid" in pydantic_model.metadata
+        assert "topic_hash" in pydantic_model.metadata
+        assert pydantic_model.metadata["uuid"] == node.metadata["uuid"]
+        assert pydantic_model.metadata["topic_hash"] == node.metadata["topic_hash"]
+
 
 class TestGraphConfig:
     """Tests for GraphConfig model."""
@@ -194,6 +242,49 @@ def test_to_pydantic(self, topic_graph):
         assert pydantic_model.nodes[0].topic == "Test root topic"
         assert pydantic_model.nodes[1].topic == "Child"
 
+    def test_graph_metadata_serialization(self, topic_graph):
+        """Test that graph-level metadata is included in Pydantic conversion."""
+        pydantic_model = topic_graph.to_pydantic()
+
+        # Verify metadata is present
+        assert pydantic_model.metadata is not None
+        assert isinstance(pydantic_model.metadata, GraphMetadata)
+
+        # Verify all required fields are present
+        assert pydantic_model.metadata.provider == "openai"
+        assert pydantic_model.metadata.model == "test-model"
+        assert pydantic_model.metadata.temperature == 0.7  # noqa: PLR2004
+        assert pydantic_model.metadata.created_at is not None
+
+        # Verify created_at is ISO 8601 format (contains 'T' separator)
+        assert "T" in pydantic_model.metadata.created_at
+
+    def test_graph_metadata_in_json(self, topic_graph):
+        """Test that graph-level metadata appears in JSON output."""
+        json_str = topic_graph.to_json()
+        data = json.loads(json_str)
+
+        assert "metadata" in data
+        assert data["metadata"]["provider"] == "openai"
+        assert data["metadata"]["model"] == "test-model"
+        assert data["metadata"]["temperature"] == 0.7  # noqa: PLR2004
+        assert "created_at" in data["metadata"]
+
+    def test_node_metadata_in_json(self, topic_graph):
+        """Test that node-level metadata (uuid, topic_hash) appears in JSON output."""
+        json_str = topic_graph.to_json()
+        data = json.loads(json_str)
+
+        # Check root node metadata
+        root_node = data["nodes"]["0"]
+        assert "metadata" in root_node
+        assert "uuid" in root_node["metadata"]
+        assert "topic_hash" in root_node["metadata"]
+
+        # Verify topic_hash is correct SHA256
+        expected_hash = hashlib.sha256(b"Test root topic").hexdigest()
+        assert root_node["metadata"]["topic_hash"] == expected_hash
+
     def test_to_json(self, topic_graph):
         """Test JSON serialization."""
         node1 = topic_graph.add_node("Child")
@@ -488,3 +579,58 @@ def test_graph_persistence_roundtrip(self):
             assert any(child.topic == "Biology" for child in loaded_chemistry.children)
         finally:
             Path(temp_path).unlink()
+
+    def test_backward_compatibility_no_metadata(self):
+        """Test loading old graph JSON files that don't have metadata fields."""
+        # Simulate an old graph JSON without metadata
+        old_graph_json = {
+            "nodes": {
+                "0": {
+                    "id": 0,
+                    "topic": "Root Topic",
+                    "children": [1],
+                    "parents": [],
+                    "metadata": {},  # Old format: empty metadata
+                },
+                "1": {
+                    "id": 1,
+                    "topic": "Child Topic",
+                    "children": [],
+                    "parents": [0],
+                    "metadata": {},
+                },
+            },
+            "root_id": 0,
+            # No "metadata" field at graph level
+        }
+
+        graph_params = {
+            "topic_prompt": "Root Topic",
+            "model_name": "test-model",
+            "temperature": 0.5,
+            "degree": 2,
+            "depth": 2,
+        }
+
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+            json.dump(old_graph_json, f)
+            temp_path = f.name
+
+        try:
+            # Load should succeed without errors
+            loaded = Graph.from_json(temp_path, graph_params)
+
+            # Verify structure was loaded correctly
+            assert len(loaded.nodes) == 2  # noqa: PLR2004
+            assert loaded.root.topic == "Root Topic"
+            assert len(loaded.root.children) == 1
+
+            # Verify nodes get uuid and topic_hash auto-generated on load
+            assert "uuid" in loaded.root.metadata
+            assert "topic_hash" in loaded.root.metadata
+
+            # Verify topic_hash is correct for the loaded topic
+            expected_hash = hashlib.sha256(b"Root Topic").hexdigest()
+            assert loaded.root.metadata["topic_hash"] == expected_hash
+        finally:
+            Path(temp_path).unlink()