Skip to content

Commit 5883751

Browse files
authored
Fix metrics (#372)
1 parent 8b1447b commit 5883751

6 files changed

Lines changed: 190 additions & 77 deletions

File tree

deepfabric/cli.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from .format_command import format_cli
1717
from .generator import DataSetGenerator
1818
from .graph import Graph
19-
from .metrics import trace
19+
from .metrics import set_trace_debug, trace
2020
from .topic_manager import load_or_build_topic_model, save_topic_model
2121
from .topic_model import TopicModel
2222
from .tui import get_tui
@@ -343,6 +343,7 @@ def generate( # noqa: PLR0913
343343
topic_only: bool = False,
344344
) -> None:
345345
"""Generate training data from a YAML configuration file or CLI parameters."""
346+
set_trace_debug(debug)
346347
trace(
347348
"cli_generate",
348349
{"mode": mode, "has_config": config_file is not None, "provider": provider, "model": model},

deepfabric/generator.py

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -150,8 +150,8 @@ def __init__(self, **kwargs):
150150
"""Initialize DataSetGenerator with parameters."""
151151
try:
152152
self.config = DataSetGeneratorConfig.model_validate(kwargs)
153-
except Exception as e:
154-
raise DataSetGeneratorError(f"Invalid generator configuration: {str(e)}") from e # noqa: TRY003
153+
except Exception as e: # noqa: TRY003
154+
raise DataSetGeneratorError(f"Invalid generator configuration: {str(e)}") from e
155155

156156
# Initialize from config
157157
self.provider = self.config.provider
@@ -166,7 +166,14 @@ def __init__(self, **kwargs):
166166
model_name=self.model_name,
167167
rate_limit_config=self.config.rate_limit, # Pass rate limit config (can be None)
168168
)
169-
trace("generator_created", {"provider": self.provider})
169+
trace(
170+
"generator_created",
171+
{
172+
"provider": self.provider,
173+
"model_name": self.model_name,
174+
"conversation_type": self.config.conversation_type,
175+
},
176+
)
170177

171178
# Store dataset system prompt for dataset inclusion (with fallback)
172179
self.dataset_system_prompt = (
@@ -211,7 +218,7 @@ def _initialize_tool_registry(self):
211218
custom_registry=custom_registry,
212219
)
213220

214-
except Exception as e:
221+
except Exception as e: # noqa: BLE001
215222
raise DataSetGeneratorError(f"Failed to initialize tool registry: {str(e)}") from e
216223

217224
def _validate_create_data_params(
@@ -414,18 +421,18 @@ def summarize_failures(self) -> dict:
414421
}
415422

416423
# Add example failures for each category
417-
for category, failures in self.failure_analysis.items():
424+
for _category, failures in self.failure_analysis.items():
418425
if failures:
419426
# Get up to 3 examples for each category
420427
examples = failures[:3]
421-
summary["failure_examples"][category] = [
428+
summary["failure_examples"].append(
422429
(
423430
str(ex)[:200] + "..."
424431
if len(str(ex)) > 200 # noqa: PLR2004
425-
else str(ex) # noqa: PLR2004
426-
) # noqa: PLR2004
432+
else str(ex)
433+
)
427434
for ex in examples
428-
]
435+
)
429436
return summary
430437

431438
def create_data(
@@ -532,6 +539,9 @@ async def create_data_async(
532539
trace(
533540
"dataset_created",
534541
{
542+
"provider": self.provider,
543+
"model_name": self.model_name,
544+
"conversation_type": self.config.conversation_type,
535545
"samples_count": len(final_result.samples),
536546
"failed_samples": len(self.failed_samples),
537547
"success": len(final_result.samples) > 0,
@@ -725,7 +735,9 @@ def print_failure_summary(self):
725735
print(f"\n{failure_type.replace('_', ' ').title()}: {count}")
726736
if failure_type in summary["failure_examples"]:
727737
print("Example failures:")
728-
for i, example in enumerate(summary["failure_examples"][failure_type], 1):
738+
for i, example in enumerate(
739+
summary["failure_examples"].get(failure_type, []), 1
740+
):
729741
print(f" {i}. {example}")
730742
print("\n=============================")
731743

deepfabric/graph.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,13 @@ def __init__(self, **kwargs):
126126
)
127127

128128
trace(
129-
"graph_created", {"provider": self.provider, "degree": self.degree, "depth": self.depth}
129+
"graph_created",
130+
{
131+
"provider": self.provider,
132+
"model_name": self.model_name,
133+
"degree": self.degree,
134+
"depth": self.depth,
135+
},
130136
)
131137

132138
self.root: Node = Node(self.config.topic_prompt, 0)
@@ -259,6 +265,8 @@ def _raise_if_build_failed():
259265
trace(
260266
"graph_built",
261267
{
268+
"provider": self.provider,
269+
"model_name": self.model_name,
262270
"nodes_count": len(self.nodes),
263271
"failed_generations": len(self.failed_generations),
264272
"success": len(self.nodes) > 1,

deepfabric/metrics.py

Lines changed: 147 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,73 +1,124 @@
1-
"""
2-
Simple analytics for DeepFabric using PostHog.
3-
4-
Provides a single trace() function for anonymous usage analytics.
5-
All analytics can be disabled by setting ANONYMIZED_TELEMETRY=False.
6-
7-
Privacy-respecting identity:
8-
- Generates a stable, anonymous user ID based on machine characteristics
9-
- Uses DEEPFABRIC_DEVELOPER=True to mark developer sessions for filtering
10-
- Never collects PII (names, emails, IP addresses, etc.)
11-
"""
12-
131
import contextlib
14-
import hashlib
2+
import logging
153
import os
16-
import platform
174
import uuid
185

6+
from pathlib import Path
7+
8+
from posthog import Posthog, identify_context, new_context
9+
10+
from .tui import get_tui
11+
1912
try:
2013
import importlib.metadata
2114

2215
VERSION = importlib.metadata.version("deepfabric")
2316
except (ImportError, importlib.metadata.PackageNotFoundError):
2417
VERSION = "development"
2518

26-
try:
27-
from posthog import Posthog, identify_context, new_context
28-
29-
POSTHOG_AVAILABLE = True
30-
except ImportError:
31-
POSTHOG_AVAILABLE = False
3219

3320
# Initialize PostHog client
34-
if POSTHOG_AVAILABLE:
35-
posthog = Posthog(
36-
project_api_key="phc_Kn8hKQIXHm5OHp5OTxvMvFDUmT7HyOUNlJvWkduB9qO",
37-
host="https://us.i.posthog.com",
38-
)
39-
else:
40-
posthog = None
21+
posthog = Posthog(
22+
project_api_key="phc_Kn8hKQIXHm5OHp5OTxvMvFDUmT7HyOUNlJvWkduB9qO",
23+
host="https://us.i.posthog.com",
24+
)
4125

42-
# Cache for the generated user ID using a dict to avoid global statement
43-
_user_id_cache: dict[str, str | None] = {"id": None}
26+
logger = logging.getLogger(__name__)
4427

4528

46-
def _get_user_id() -> str:
47-
"""
48-
Generate a stable, anonymous user ID based on machine characteristics.
29+
class _TelemetryState:
30+
"""Holds the state for the telemetry module."""
4931

50-
Creates a UUID from a hash of platform.node() (hostname) and uuid.getnode()
51-
(MAC address). This provides a persistent identifier across sessions without
52-
collecting PII.
32+
def __init__(self) -> None:
33+
self.debug_trace: bool = False
34+
self.user_id_announced: bool = False
35+
self.user_id_cache: str | None = None
36+
self.telemetry_failed_once: bool = False
5337

54-
Returns:
55-
str: A stable UUID string unique to this machine
56-
"""
57-
if _user_id_cache["id"] is not None:
58-
return _user_id_cache["id"]
5938

60-
# Combine hostname and MAC address for machine-specific identifier
61-
machine_info = f"{platform.node()}-{uuid.getnode()}"
39+
_state = _TelemetryState()
40+
6241

63-
# Create SHA256 hash and convert to UUID format
64-
hash_digest = hashlib.sha256(machine_info.encode()).hexdigest()
42+
APP_NAME = "DeepFabric"
43+
APP_AUTHOR = "DeepFabric"
6544

66-
# Use first 32 hex chars to create a valid UUID
67-
user_uuid = str(uuid.UUID(hash_digest[:32]))
6845

69-
_user_id_cache["id"] = user_uuid
70-
return user_uuid
46+
try:
47+
from platformdirs import user_data_dir
48+
except ImportError: # pragma: no cover - optional dependency
49+
50+
def user_data_dir(appname: str, appauthor: str | None = None) -> str:
51+
if os.name == "nt":
52+
base = os.environ.get("APPDATA") or os.path.expanduser(r"~\AppData\Roaming")
53+
elif os.name == "posix":
54+
base = os.environ.get("XDG_DATA_HOME") or os.path.expanduser("~/.local/share")
55+
else:
56+
base = os.path.expanduser("~")
57+
return str(Path(base) / (appauthor or appname) / appname)
58+
59+
60+
def _user_id_path() -> Path:
61+
candidates: list[Path] = []
62+
try:
63+
candidates.append(Path(user_data_dir(APP_NAME, APP_AUTHOR)))
64+
except Exception:
65+
logger.debug("Failed to resolve platform data dir", exc_info=True)
66+
candidates.append(Path.home() / f".{APP_NAME.lower()}")
67+
candidates.append(Path.cwd())
68+
69+
for data_dir in candidates:
70+
try:
71+
data_dir.mkdir(parents=True, exist_ok=True)
72+
return data_dir / "telemetry_id"
73+
except Exception:
74+
logger.debug("Failed to prepare telemetry directory %s", data_dir, exc_info=True)
75+
76+
return Path("telemetry_id")
77+
78+
79+
def _read_user_id(path: Path) -> str | None:
80+
try:
81+
if path.exists():
82+
candidate = path.read_text(encoding="utf-8").strip()
83+
if candidate:
84+
uuid.UUID(candidate)
85+
return candidate
86+
except Exception:
87+
logger.debug("Failed to read existing telemetry id", exc_info=True)
88+
return None
89+
90+
91+
def _write_user_id(path: Path) -> str:
92+
user_id = str(uuid.uuid4())
93+
tmp_path = path.with_suffix(".tmp")
94+
try:
95+
tmp_path.write_text(user_id, encoding="utf-8")
96+
if os.name == "posix":
97+
os.chmod(tmp_path, 0o600)
98+
tmp_path.replace(path)
99+
except Exception:
100+
logger.debug("Failed to persist telemetry id", exc_info=True)
101+
return user_id
102+
else:
103+
return user_id
104+
finally:
105+
with contextlib.suppress(Exception):
106+
if tmp_path.exists() and tmp_path != path:
107+
tmp_path.unlink()
108+
109+
110+
def _get_user_id() -> str:
111+
"""Generate a stable, anonymous user ID persisted on disk."""
112+
if _state.user_id_cache is not None:
113+
return _state.user_id_cache
114+
115+
path = _user_id_path()
116+
user_id = _read_user_id(path)
117+
if user_id is None:
118+
user_id = _write_user_id(path)
119+
120+
_state.user_id_cache = user_id
121+
return user_id
71122

72123

73124
def _is_developer() -> bool:
@@ -80,33 +131,43 @@ def _is_developer() -> bool:
80131
return os.environ.get("DEEPFABRIC_DEVELOPER") == "True"
81132

82133

134+
def set_trace_debug(enabled: bool) -> None:
135+
"""Enable or disable debug output for telemetry events."""
136+
_state.debug_trace = enabled
137+
if not enabled:
138+
_state.user_id_announced = False
139+
140+
141+
def _announce_user_id(user_id: str) -> None:
142+
if _state.user_id_announced or not _state.debug_trace:
143+
return
144+
try:
145+
get_tui().info(f"metrics user id: {user_id}")
146+
except Exception: # pragma: no cover - fallback to logging
147+
logger.debug("metrics user id: %s", user_id)
148+
149+
_state.user_id_announced = True
150+
151+
83152
def trace(event_name, event_properties=None):
84153
"""
85-
Send an analytics event if telemetry is enabled.
154+
Send an analytics event if metrics is enabled.
86155
87156
Uses privacy-respecting identity tracking with a stable, anonymous user ID
88-
generated from machine characteristics. Developer sessions are marked with
157+
stored on disk for reuse. Developer sessions are marked with
89158
the is_developer flag when DEEPFABRIC_DEVELOPER=True.
90159
91160
Args:
92161
event_name: Name of the event to track
93162
event_properties: Optional dictionary of event properties
94163
"""
95-
# Quick exit if telemetry is disabled
96-
if os.environ.get("ANONYMIZED_TELEMETRY") == "False":
164+
if not is_enabled():
97165
return
98166

99-
# Quick exit if during testing
100-
if os.environ.get("DEEPFABRIC_TESTING") == "True":
101-
return
102-
103-
# Quick exit if PostHog not available
104-
if not POSTHOG_AVAILABLE or not posthog:
105-
return
106-
107-
with contextlib.suppress(Exception):
167+
try:
108168
# Generate stable user ID
109169
user_id = _get_user_id()
170+
_announce_user_id(user_id)
110171

111172
# Add version and developer flag to all events
112173
properties = event_properties or {}
@@ -116,13 +177,36 @@ def trace(event_name, event_properties=None):
116177
# Use identity context to associate events with the user
117178
with new_context():
118179
identify_context(user_id)
119-
posthog.capture(event=event_name, properties=properties)
180+
posthog.capture(
181+
distinct_id=user_id,
182+
event=event_name,
183+
properties=properties,
184+
)
185+
except Exception:
186+
if not _state.telemetry_failed_once:
187+
_state.telemetry_failed_once = True
188+
logger.warning(
189+
"Failed to send telemetry event. Further failures will be logged at DEBUG level.",
190+
exc_info=True,
191+
)
192+
else:
193+
logger.debug("Failed to capture metrics event", exc_info=True)
120194

121195

122196
def is_enabled():
123197
"""Check if analytics is currently enabled."""
124198
return (
125199
os.environ.get("ANONYMIZED_TELEMETRY") != "False"
126200
and os.environ.get("DEEPFABRIC_TESTING") != "True"
127-
and POSTHOG_AVAILABLE
128201
)
202+
203+
204+
def shutdown() -> None:
205+
"""
206+
Shutdown the PostHog client, flushing any buffered events.
207+
208+
This should be called on application exit to ensure all metrics data is sent.
209+
"""
210+
if is_enabled():
211+
logger.debug("Shutting down metrics client.")
212+
posthog.shutdown()

0 commit comments

Comments
 (0)