diff --git a/deepfabric/cli.py b/deepfabric/cli.py index a771946c..51401bb2 100644 --- a/deepfabric/cli.py +++ b/deepfabric/cli.py @@ -16,7 +16,7 @@ from .format_command import format_cli from .generator import DataSetGenerator from .graph import Graph -from .metrics import trace +from .metrics import set_trace_debug, trace from .topic_manager import load_or_build_topic_model, save_topic_model from .topic_model import TopicModel from .tui import get_tui @@ -343,6 +343,7 @@ def generate( # noqa: PLR0913 topic_only: bool = False, ) -> None: """Generate training data from a YAML configuration file or CLI parameters.""" + set_trace_debug(debug) trace( "cli_generate", {"mode": mode, "has_config": config_file is not None, "provider": provider, "model": model}, diff --git a/deepfabric/generator.py b/deepfabric/generator.py index 10433cc7..3196013f 100644 --- a/deepfabric/generator.py +++ b/deepfabric/generator.py @@ -150,8 +150,8 @@ def __init__(self, **kwargs): """Initialize DataSetGenerator with parameters.""" try: self.config = DataSetGeneratorConfig.model_validate(kwargs) - except Exception as e: - raise DataSetGeneratorError(f"Invalid generator configuration: {str(e)}") from e # noqa: TRY003 + except Exception as e: # noqa: TRY003 + raise DataSetGeneratorError(f"Invalid generator configuration: {str(e)}") from e # Initialize from config self.provider = self.config.provider @@ -166,7 +166,14 @@ def __init__(self, **kwargs): model_name=self.model_name, rate_limit_config=self.config.rate_limit, # Pass rate limit config (can be None) ) - trace("generator_created", {"provider": self.provider}) + trace( + "generator_created", + { + "provider": self.provider, + "model_name": self.model_name, + "conversation_type": self.config.conversation_type, + }, + ) # Store dataset system prompt for dataset inclusion (with fallback) self.dataset_system_prompt = ( @@ -211,7 +218,7 @@ def _initialize_tool_registry(self): custom_registry=custom_registry, ) - except Exception as e: + except Exception as e: # noqa: BLE001 raise DataSetGeneratorError(f"Failed to initialize tool registry: {str(e)}") from e def _validate_create_data_params( @@ -414,18 +421,18 @@ def summarize_failures(self) -> dict: } # Add example failures for each category - for category, failures in self.failure_analysis.items(): + for _category, failures in self.failure_analysis.items(): if failures: # Get up to 3 examples for each category examples = failures[:3] - summary["failure_examples"][category] = [ + summary["failure_examples"].append( ( str(ex)[:200] + "..." if len(str(ex)) > 200 # noqa: PLR2004 - else str(ex) # noqa: PLR2004 - ) # noqa: PLR2004 + else str(ex) + ) for ex in examples - ] + ) return summary def create_data( @@ -532,6 +539,9 @@ async def create_data_async( trace( "dataset_created", { + "provider": self.provider, + "model_name": self.model_name, + "conversation_type": self.config.conversation_type, "samples_count": len(final_result.samples), "failed_samples": len(self.failed_samples), "success": len(final_result.samples) > 0, @@ -725,7 +735,9 @@ def print_failure_summary(self): print(f"\n{failure_type.replace('_', ' ').title()}: {count}") if failure_type in summary["failure_examples"]: print("Example failures:") - for i, example in enumerate(summary["failure_examples"][failure_type], 1): + for i, example in enumerate( + summary["failure_examples"].get(failure_type, []), 1 + ): print(f" {i}. {example}") print("\n=============================") diff --git a/deepfabric/graph.py b/deepfabric/graph.py index c1973e0e..36500e2c 100644 --- a/deepfabric/graph.py +++ b/deepfabric/graph.py @@ -126,7 +126,13 @@ def __init__(self, **kwargs): ) trace( - "graph_created", {"provider": self.provider, "degree": self.degree, "depth": self.depth} + "graph_created", + { + "provider": self.provider, + "model_name": self.model_name, + "degree": self.degree, + "depth": self.depth, + }, ) self.root: Node = Node(self.config.topic_prompt, 0) @@ -259,6 +265,8 @@ def _raise_if_build_failed(): trace( "graph_built", { + "provider": self.provider, + "model_name": self.model_name, "nodes_count": len(self.nodes), "failed_generations": len(self.failed_generations), "success": len(self.nodes) > 1, diff --git a/deepfabric/metrics.py b/deepfabric/metrics.py index 9896739c..8275cc7b 100644 --- a/deepfabric/metrics.py +++ b/deepfabric/metrics.py @@ -1,21 +1,14 @@ -""" -Simple analytics for DeepFabric using PostHog. - -Provides a single trace() function for anonymous usage analytics. -All analytics can be disabled by setting ANONYMIZED_TELEMETRY=False. - -Privacy-respecting identity: -- Generates a stable, anonymous user ID based on machine characteristics -- Uses DEEPFABRIC_DEVELOPER=True to mark developer sessions for filtering -- Never collects PII (names, emails, IP addresses, etc.) -""" - import contextlib -import hashlib +import logging import os -import platform import uuid +from pathlib import Path + +from posthog import Posthog, identify_context, new_context + +from .tui import get_tui + try: import importlib.metadata @@ -23,51 +16,109 @@ except (ImportError, importlib.metadata.PackageNotFoundError): VERSION = "development" -try: - from posthog import Posthog, identify_context, new_context - - POSTHOG_AVAILABLE = True -except ImportError: - POSTHOG_AVAILABLE = False # Initialize PostHog client -if POSTHOG_AVAILABLE: - posthog = Posthog( - project_api_key="phc_Kn8hKQIXHm5OHp5OTxvMvFDUmT7HyOUNlJvWkduB9qO", - host="https://us.i.posthog.com", - ) -else: - posthog = None +posthog = Posthog( + project_api_key="phc_Kn8hKQIXHm5OHp5OTxvMvFDUmT7HyOUNlJvWkduB9qO", + host="https://us.i.posthog.com", +) -# Cache for the generated user ID using a dict to avoid global statement -_user_id_cache: dict[str, str | None] = {"id": None} +logger = logging.getLogger(__name__) -def _get_user_id() -> str: - """ - Generate a stable, anonymous user ID based on machine characteristics. +class _TelemetryState: + """Holds the state for the telemetry module.""" - Creates a UUID from a hash of platform.node() (hostname) and uuid.getnode() - (MAC address). This provides a persistent identifier across sessions without - collecting PII. + def __init__(self) -> None: + self.debug_trace: bool = False + self.user_id_announced: bool = False + self.user_id_cache: str | None = None + self.telemetry_failed_once: bool = False - Returns: - str: A stable UUID string unique to this machine - """ - if _user_id_cache["id"] is not None: - return _user_id_cache["id"] - # Combine hostname and MAC address for machine-specific identifier - machine_info = f"{platform.node()}-{uuid.getnode()}" +_state = _TelemetryState() + - # Create SHA256 hash and convert to UUID format - hash_digest = hashlib.sha256(machine_info.encode()).hexdigest() +APP_NAME = "DeepFabric" +APP_AUTHOR = "DeepFabric" - # Use first 32 hex chars to create a valid UUID - user_uuid = str(uuid.UUID(hash_digest[:32])) - _user_id_cache["id"] = user_uuid - return user_uuid +try: + from platformdirs import user_data_dir +except ImportError: # pragma: no cover - optional dependency + + def user_data_dir(appname: str, appauthor: str | None = None) -> str: + if os.name == "nt": + base = os.environ.get("APPDATA") or os.path.expanduser(r"~\AppData\Roaming") + elif os.name == "posix": + base = os.environ.get("XDG_DATA_HOME") or os.path.expanduser("~/.local/share") + else: + base = os.path.expanduser("~") + return str(Path(base) / (appauthor or appname) / appname) + + +def _user_id_path() -> Path: + candidates: list[Path] = [] + try: + candidates.append(Path(user_data_dir(APP_NAME, APP_AUTHOR))) + except Exception: + logger.debug("Failed to resolve platform data dir", exc_info=True) + candidates.append(Path.home() / f".{APP_NAME.lower()}") + candidates.append(Path.cwd()) + + for data_dir in candidates: + try: + data_dir.mkdir(parents=True, exist_ok=True) + return data_dir / "telemetry_id" + except Exception: + logger.debug("Failed to prepare telemetry directory %s", data_dir, exc_info=True) + + return Path("telemetry_id") + + +def _read_user_id(path: Path) -> str | None: + try: + if path.exists(): + candidate = path.read_text(encoding="utf-8").strip() + if candidate: + uuid.UUID(candidate) + return candidate + except Exception: + logger.debug("Failed to read existing telemetry id", exc_info=True) + return None + + +def _write_user_id(path: Path) -> str: + user_id = str(uuid.uuid4()) + tmp_path = path.with_suffix(".tmp") + try: + tmp_path.write_text(user_id, encoding="utf-8") + if os.name == "posix": + os.chmod(tmp_path, 0o600) + tmp_path.replace(path) + except Exception: + logger.debug("Failed to persist telemetry id", exc_info=True) + return user_id + else: + return user_id + finally: + with contextlib.suppress(Exception): + if tmp_path.exists() and tmp_path != path: + tmp_path.unlink() + + +def _get_user_id() -> str: + """Generate a stable, anonymous user ID persisted on disk.""" + if _state.user_id_cache is not None: + return _state.user_id_cache + + path = _user_id_path() + user_id = _read_user_id(path) + if user_id is None: + user_id = _write_user_id(path) + + _state.user_id_cache = user_id + return user_id def _is_developer() -> bool: @@ -80,33 +131,43 @@ def _is_developer() -> bool: return os.environ.get("DEEPFABRIC_DEVELOPER") == "True" +def set_trace_debug(enabled: bool) -> None: + """Enable or disable debug output for telemetry events.""" + _state.debug_trace = enabled + if not enabled: + _state.user_id_announced = False + + +def _announce_user_id(user_id: str) -> None: + if _state.user_id_announced or not _state.debug_trace: + return + try: + get_tui().info(f"metrics user id: {user_id}") + except Exception: # pragma: no cover - fallback to logging + logger.debug("metrics user id: %s", user_id) + + _state.user_id_announced = True + + def trace(event_name, event_properties=None): """ - Send an analytics event if telemetry is enabled. + Send an analytics event if metrics is enabled. Uses privacy-respecting identity tracking with a stable, anonymous user ID - generated from machine characteristics. Developer sessions are marked with + stored on disk for reuse. Developer sessions are marked with the is_developer flag when DEEPFABRIC_DEVELOPER=True. Args: event_name: Name of the event to track event_properties: Optional dictionary of event properties """ - # Quick exit if telemetry is disabled - if os.environ.get("ANONYMIZED_TELEMETRY") == "False": + if not is_enabled(): return - # Quick exit if during testing - if os.environ.get("DEEPFABRIC_TESTING") == "True": - return - - # Quick exit if PostHog not available - if not POSTHOG_AVAILABLE or not posthog: - return - - with contextlib.suppress(Exception): + try: # Generate stable user ID user_id = _get_user_id() + _announce_user_id(user_id) # Add version and developer flag to all events properties = event_properties or {} @@ -116,7 +177,20 @@ def trace(event_name, event_properties=None): # Use identity context to associate events with the user with new_context(): identify_context(user_id) - posthog.capture(event=event_name, properties=properties) + posthog.capture( + distinct_id=user_id, + event=event_name, + properties=properties, + ) + except Exception: + if not _state.telemetry_failed_once: + _state.telemetry_failed_once = True + logger.warning( + "Failed to send telemetry event. Further failures will be logged at DEBUG level.", + exc_info=True, + ) + else: + logger.debug("Failed to capture metrics event", exc_info=True) def is_enabled(): @@ -124,5 +198,15 @@ def is_enabled(): return ( os.environ.get("ANONYMIZED_TELEMETRY") != "False" and os.environ.get("DEEPFABRIC_TESTING") != "True" - and POSTHOG_AVAILABLE ) + + +def shutdown() -> None: + """ + Shutdown the PostHog client, flushing any buffered events. + + This should be called on application exit to ensure all metrics data is sent. + """ + if is_enabled(): + logger.debug("Shutting down metrics client.") + posthog.shutdown() diff --git a/deepfabric/tree.py b/deepfabric/tree.py index 896e620d..35b4483e 100644 --- a/deepfabric/tree.py +++ b/deepfabric/tree.py @@ -148,7 +148,13 @@ def __init__(self, **kwargs): ) trace( - "tree_created", {"provider": self.provider, "degree": self.degree, "depth": self.depth} + "tree_created", + { + "provider": self.provider, + "model_name": self.model_name, + "degree": self.degree, + "depth": self.depth, + }, ) # Derived attributes @@ -197,6 +203,8 @@ def _raise_if_build_failed(): trace( "tree_built", { + "provider": self.provider, + "model_name": self.model_name, "total_paths": len(self.tree_paths), "failed_generations": len(self.failed_generations), "success": len(self.tree_paths) > 0, diff --git a/uv.lock b/uv.lock index 90ec96e1..64b07fdb 100644 --- a/uv.lock +++ b/uv.lock @@ -397,7 +397,7 @@ wheels = [ [[package]] name = "deepfabric" -version = "2.10.0" +version = "2.11.0" source = { editable = "." } dependencies = [ { name = "anthropic" },