Fix metrics

deepfabric/cli.py

@@ -16,7 +16,7 @@
 from .format_command import format_cli
 from .generator import DataSetGenerator
 from .graph import Graph
-from .metrics import trace
+from .metrics import set_trace_debug, trace
 from .topic_manager import load_or_build_topic_model, save_topic_model
 from .topic_model import TopicModel
 from .tui import get_tui
@@ -343,6 +343,7 @@ def generate(  # noqa: PLR0913
     topic_only: bool = False,
 ) -> None:
     """Generate training data from a YAML configuration file or CLI parameters."""
+    set_trace_debug(debug)
     trace(
         "cli_generate",
         {"mode": mode, "has_config": config_file is not None, "provider": provider, "model": model},

deepfabric/generator.py

@@ -150,8 +150,8 @@ def __init__(self, **kwargs):
         """Initialize DataSetGenerator with parameters."""
         try:
             self.config = DataSetGeneratorConfig.model_validate(kwargs)
-        except Exception as e:
-            raise DataSetGeneratorError(f"Invalid generator configuration: {str(e)}") from e  # noqa: TRY003
+        except Exception as e:  # noqa: TRY003
+            raise DataSetGeneratorError(f"Invalid generator configuration: {str(e)}") from e
 
         # Initialize from config
         self.provider = self.config.provider
@@ -166,7 +166,14 @@ def __init__(self, **kwargs):
             model_name=self.model_name,
             rate_limit_config=self.config.rate_limit,  # Pass rate limit config (can be None)
         )
-        trace("generator_created", {"provider": self.provider})
+        trace(
+            "generator_created",
+            {
+                "provider": self.provider,
+                "model_name": self.model_name,
+                "conversation_type": self.config.conversation_type,
+            },
+        )
 
         # Store dataset system prompt for dataset inclusion (with fallback)
         self.dataset_system_prompt = (
@@ -211,7 +218,7 @@ def _initialize_tool_registry(self):
                 custom_registry=custom_registry,
             )
 
-        except Exception as e:
+        except Exception as e:  # noqa: BLE001
             raise DataSetGeneratorError(f"Failed to initialize tool registry: {str(e)}") from e
 
     def _validate_create_data_params(
@@ -414,18 +421,18 @@ def summarize_failures(self) -> dict:
         }
 
         # Add example failures for each category
-        for category, failures in self.failure_analysis.items():
+        for _category, failures in self.failure_analysis.items():
             if failures:
                 # Get up to 3 examples for each category
                 examples = failures[:3]
-                summary["failure_examples"][category] = [
+                summary["failure_examples"].append(
                     (
                         str(ex)[:200] + "..."
                         if len(str(ex)) > 200  # noqa: PLR2004
-                        else str(ex)  # noqa: PLR2004
-                    )  # noqa: PLR2004
+                        else str(ex)
+                    )
                     for ex in examples
-                ]
+                )
         return summary
 
     def create_data(
@@ -532,6 +539,9 @@ async def create_data_async(
             trace(
                 "dataset_created",
                 {
+                    "provider": self.provider,
+                    "model_name": self.model_name,
+                    "conversation_type": self.config.conversation_type,
                     "samples_count": len(final_result.samples),
                     "failed_samples": len(self.failed_samples),
                     "success": len(final_result.samples) > 0,
@@ -725,7 +735,9 @@ def print_failure_summary(self):
                 print(f"\n{failure_type.replace('_', ' ').title()}: {count}")
                 if failure_type in summary["failure_examples"]:
                     print("Example failures:")
-                    for i, example in enumerate(summary["failure_examples"][failure_type], 1):
+                    for i, example in enumerate(
+                        summary["failure_examples"].get(failure_type, []), 1
+                    ):
                         print(f"  {i}. {example}")
         print("\n=============================")
 

deepfabric/graph.py

@@ -126,7 +126,13 @@ def __init__(self, **kwargs):
         )
 
         trace(
-            "graph_created", {"provider": self.provider, "degree": self.degree, "depth": self.depth}
+            "graph_created",
+            {
+                "provider": self.provider,
+                "model_name": self.model_name,
+                "degree": self.degree,
+                "depth": self.depth,
+            },
         )
 
         self.root: Node = Node(self.config.topic_prompt, 0)
@@ -259,6 +265,8 @@ def _raise_if_build_failed():
             trace(
                 "graph_built",
                 {
+                    "provider": self.provider,
+                    "model_name": self.model_name,
                     "nodes_count": len(self.nodes),
                     "failed_generations": len(self.failed_generations),
                     "success": len(self.nodes) > 1,

deepfabric/metrics.py

@@ -1,73 +1,124 @@
-"""
-Simple analytics for DeepFabric using PostHog.
-
-Provides a single trace() function for anonymous usage analytics.
-All analytics can be disabled by setting ANONYMIZED_TELEMETRY=False.
-
-Privacy-respecting identity:
-- Generates a stable, anonymous user ID based on machine characteristics
-- Uses DEEPFABRIC_DEVELOPER=True to mark developer sessions for filtering
-- Never collects PII (names, emails, IP addresses, etc.)
-"""
-
 import contextlib
-import hashlib
+import logging
 import os
-import platform
 import uuid
 
+from pathlib import Path
+
+from posthog import Posthog, identify_context, new_context
+
+from .tui import get_tui
+
 try:
     import importlib.metadata
 
     VERSION = importlib.metadata.version("deepfabric")
 except (ImportError, importlib.metadata.PackageNotFoundError):
     VERSION = "development"
 
-try:
-    from posthog import Posthog, identify_context, new_context
-
-    POSTHOG_AVAILABLE = True
-except ImportError:
-    POSTHOG_AVAILABLE = False
 
 # Initialize PostHog client
-if POSTHOG_AVAILABLE:
-    posthog = Posthog(
-        project_api_key="phc_Kn8hKQIXHm5OHp5OTxvMvFDUmT7HyOUNlJvWkduB9qO",
-        host="https://us.i.posthog.com",
-    )
-else:
-    posthog = None
+posthog = Posthog(
+    project_api_key="phc_Kn8hKQIXHm5OHp5OTxvMvFDUmT7HyOUNlJvWkduB9qO",
+    host="https://us.i.posthog.com",
+)
 
-# Cache for the generated user ID using a dict to avoid global statement
-_user_id_cache: dict[str, str | None] = {"id": None}
+logger = logging.getLogger(__name__)
 
 
-def _get_user_id() -> str:
-    """
-    Generate a stable, anonymous user ID based on machine characteristics.
+class _TelemetryState:
+    """Holds the state for the telemetry module."""
 
-    Creates a UUID from a hash of platform.node() (hostname) and uuid.getnode()
-    (MAC address). This provides a persistent identifier across sessions without
-    collecting PII.
+    def __init__(self) -> None:
+        self.debug_trace: bool = False
+        self.user_id_announced: bool = False
+        self.user_id_cache: str | None = None
+        self.telemetry_failed_once: bool = False
 
-    Returns:
-        str: A stable UUID string unique to this machine
-    """
-    if _user_id_cache["id"] is not None:
-        return _user_id_cache["id"]
 
-    # Combine hostname and MAC address for machine-specific identifier
-    machine_info = f"{platform.node()}-{uuid.getnode()}"
+_state = _TelemetryState()
+
 
-    # Create SHA256 hash and convert to UUID format
-    hash_digest = hashlib.sha256(machine_info.encode()).hexdigest()
+APP_NAME = "DeepFabric"
+APP_AUTHOR = "DeepFabric"
 
-    # Use first 32 hex chars to create a valid UUID
-    user_uuid = str(uuid.UUID(hash_digest[:32]))
 
-    _user_id_cache["id"] = user_uuid
-    return user_uuid
+try:
+    from platformdirs import user_data_dir
+except ImportError:  # pragma: no cover - optional dependency
+
+    def user_data_dir(appname: str, appauthor: str | None = None) -> str:
+        if os.name == "nt":
+            base = os.environ.get("APPDATA") or os.path.expanduser(r"~\AppData\Roaming")
+        elif os.name == "posix":
+            base = os.environ.get("XDG_DATA_HOME") or os.path.expanduser("~/.local/share")
+        else:
+            base = os.path.expanduser("~")
+        return str(Path(base) / (appauthor or appname) / appname)
+
+
+def _user_id_path() -> Path:
+    candidates: list[Path] = []
+    try:
+        candidates.append(Path(user_data_dir(APP_NAME, APP_AUTHOR)))
+    except Exception:
+        logger.debug("Failed to resolve platform data dir", exc_info=True)
+    candidates.append(Path.home() / f".{APP_NAME.lower()}")
+    candidates.append(Path.cwd())
+
+    for data_dir in candidates:
+        try:
+            data_dir.mkdir(parents=True, exist_ok=True)
+            return data_dir / "telemetry_id"
+        except Exception:
+            logger.debug("Failed to prepare telemetry directory %s", data_dir, exc_info=True)
+
+    return Path("telemetry_id")
+
+
+def _read_user_id(path: Path) -> str | None:
+    try:
+        if path.exists():
+            candidate = path.read_text(encoding="utf-8").strip()
+            if candidate:
+                uuid.UUID(candidate)
+                return candidate
+    except Exception:
+        logger.debug("Failed to read existing telemetry id", exc_info=True)
+    return None
+
+
+def _write_user_id(path: Path) -> str:
+    user_id = str(uuid.uuid4())
+    tmp_path = path.with_suffix(".tmp")
+    try:
+        tmp_path.write_text(user_id, encoding="utf-8")
+        if os.name == "posix":
+            os.chmod(tmp_path, 0o600)
+        tmp_path.replace(path)
+    except Exception:
+        logger.debug("Failed to persist telemetry id", exc_info=True)
+        return user_id
+    else:
+        return user_id
+    finally:
+        with contextlib.suppress(Exception):
+            if tmp_path.exists() and tmp_path != path:
+                tmp_path.unlink()
+
+
+def _get_user_id() -> str:
+    """Generate a stable, anonymous user ID persisted on disk."""
+    if _state.user_id_cache is not None:
+        return _state.user_id_cache
+
+    path = _user_id_path()
+    user_id = _read_user_id(path)
+    if user_id is None:
+        user_id = _write_user_id(path)
+
+    _state.user_id_cache = user_id
+    return user_id
 
 
 def _is_developer() -> bool:
@@ -80,33 +131,43 @@ def _is_developer() -> bool:
     return os.environ.get("DEEPFABRIC_DEVELOPER") == "True"
 
 
+def set_trace_debug(enabled: bool) -> None:
+    """Enable or disable debug output for telemetry events."""
+    _state.debug_trace = enabled
+    if not enabled:
+        _state.user_id_announced = False
+
+
+def _announce_user_id(user_id: str) -> None:
+    if _state.user_id_announced or not _state.debug_trace:
+        return
+    try:
+        get_tui().info(f"metrics user id: {user_id}")
+    except Exception:  # pragma: no cover - fallback to logging
+        logger.debug("metrics user id: %s", user_id)
+
+    _state.user_id_announced = True
+
+
 def trace(event_name, event_properties=None):
     """
-    Send an analytics event if telemetry is enabled.
+    Send an analytics event if metrics is enabled.
 
     Uses privacy-respecting identity tracking with a stable, anonymous user ID
-    generated from machine characteristics. Developer sessions are marked with
+    stored on disk for reuse. Developer sessions are marked with
     the is_developer flag when DEEPFABRIC_DEVELOPER=True.
 
     Args:
         event_name: Name of the event to track
         event_properties: Optional dictionary of event properties
     """
-    # Quick exit if telemetry is disabled
-    if os.environ.get("ANONYMIZED_TELEMETRY") == "False":
+    if not is_enabled():
         return
 
-    # Quick exit if during testing
-    if os.environ.get("DEEPFABRIC_TESTING") == "True":
-        return
-
-    # Quick exit if PostHog not available
-    if not POSTHOG_AVAILABLE or not posthog:
-        return
-
-    with contextlib.suppress(Exception):
+    try:
         # Generate stable user ID
         user_id = _get_user_id()
+        _announce_user_id(user_id)
 
         # Add version and developer flag to all events
         properties = event_properties or {}
@@ -116,13 +177,36 @@ def trace(event_name, event_properties=None):
         # Use identity context to associate events with the user
         with new_context():
             identify_context(user_id)
-            posthog.capture(event=event_name, properties=properties)
+            posthog.capture(
+                distinct_id=user_id,
+                event=event_name,
+                properties=properties,
+            )
+    except Exception:
+        if not _state.telemetry_failed_once:
+            _state.telemetry_failed_once = True
+            logger.warning(
+                "Failed to send telemetry event. Further failures will be logged at DEBUG level.",
+                exc_info=True,
+            )
+        else:
+            logger.debug("Failed to capture metrics event", exc_info=True)
 
 
 def is_enabled():
     """Check if analytics is currently enabled."""
     return (
         os.environ.get("ANONYMIZED_TELEMETRY") != "False"
         and os.environ.get("DEEPFABRIC_TESTING") != "True"
-        and POSTHOG_AVAILABLE
     )
+
+
+def shutdown() -> None:
+    """
+    Shutdown the PostHog client, flushing any buffered events.
+
+    This should be called on application exit to ensure all metrics data is sent.
+    """
+    if is_enabled():
+        logger.debug("Shutting down metrics client.")
+        posthog.shutdown()