feat(cli): Add Analytic tracking to CLI commands

Author: blainekasten

src/together/lib/cli/__init__.py

@@ -4,13 +4,15 @@
 from typing import Any
 
 import click
+import httpx
 
 import together
 from together._version import __version__
 from together._constants import DEFAULT_TIMEOUT
 from together.lib.cli.api.beta import beta
 from together.lib.cli.api.evals import evals
 from together.lib.cli.api.files import files
+from together.lib.cli._track_cli import CliTrackingEvents, track_cli
 from together.lib.cli.api.models import models
 from together.lib.cli.api.endpoints import endpoints
 from together.lib.cli.api.fine_tuning import fine_tuning
@@ -57,10 +59,22 @@ def main(
 ) -> None:
     """This is a sample CLI tool."""
     os.environ.setdefault("TOGETHER_LOG", "debug" if debug else "info")
-    ctx.obj = together.Together(
+
+    client = together.Together(
         api_key=api_key, base_url=base_url, timeout=timeout, max_retries=max_retries if max_retries is not None else 0
     )
 
+    # Wrap the client's httpx requests to track the parameters sent on api requests
+    def track_request(request: httpx.Request) -> None:
+        track_cli(
+            CliTrackingEvents.ApiRequest,
+            {"url": str(request.url), "method": request.method, "body": request.content.decode("utf-8")},
+        )
+
+    client._client.event_hooks["request"].append(track_request)
+
+    ctx.obj = client
+
 
 main.add_command(files)
 main.add_command(fine_tuning)

src/together/lib/cli/_track_cli.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import os
+import json
+import time
+import uuid
+import threading
+from enum import Enum
+from typing import Any, TypeVar, Callable
+from functools import wraps
+
+import click
+import httpx
+import machineid
+
+from together import __version__
+from together.lib.utils import log_debug
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+SESSION_ID = int(str(uuid.uuid4().int)[0:13])
+
+
+def is_tracking_enabled() -> bool:
+    # Users can opt-out of tracking with the environment variable.
+    if os.getenv("TOGETHER_TELEMETRY_DISABLED"):
+        log_debug("Analytics tracking disabled by environment variable")
+        return False
+
+    return True
+
+
+class CliTrackingEvents(Enum):
+    CommandStarted = "cli_command_started"
+    CommandCompleted = "cli_commmand_completed"
+    CommandFailed = "cli_command_failed"
+    CommandUserAborted = "cli_command_user_aborted"
+    ApiRequest = "cli_command_api_request"
+
+
+def track_cli(event_name: CliTrackingEvents, args: dict[str, Any]) -> None:
+    """Track a CLI event. Non-Blocking."""
+    if is_tracking_enabled() == False:
+        return
+
+    def send_event() -> None:
+        ANALYTICS_API_ENV_VAR = os.getenv("TOGETHER_TELEMETRY_API")
+        ANALYTICS_API = (
+            ANALYTICS_API_ENV_VAR if ANALYTICS_API_ENV_VAR else "https://api.together.ai/together/gateway/pub/v1/httpRequest"
+        )
+
+        try:
+            client = httpx.Client()
+            client.post(
+                ANALYTICS_API,
+                headers={"content-type": "application/json", "user-agent": f"together-cli:{__version__}"},
+                content=json.dumps(
+                    {
+                        "event_source": "cli",
+                        "event_type": event_name.value,
+                        "event_properties": {
+                            "is_ci": os.getenv("CI") is not None,
+                            **args,
+                        },
+                        "context": {
+                            "session_id": str(SESSION_ID),
+                            "runtime": {
+                                "name": "together-cli",
+                                "version": __version__,
+                            },
+                        },
+                        "identity": {
+                            "device_id": machineid.id().lower(),
+                        },
+                        "event_options": {
+                            "time": int(time.time() * 1000),
+                        },
+                    }
+                ),
+            )
+        except Exception as e:
+            log_debug("Error sending analytics event", error=e)
+            # No-op - this is not critical and we don't want to block the CLI
+            pass
+
+    threading.Thread(target=send_event).start()
+
+
+def auto_track_command(command: str) -> Callable[[F], F]:
+    """Decorator for click commands to automatically track CLI commands start/completion/failure."""
+
+    def decorator(f: F) -> F:
+        @wraps(f)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            track_cli(CliTrackingEvents.CommandStarted, {"command": command, "arguments": kwargs})
+            try:
+                return f(*args, **kwargs)
+            except click.Abort:
+                # Doesn't seem like this is working any more
+                track_cli(
+                    CliTrackingEvents.CommandUserAborted,
+                    {"command": command, "arguments": kwargs},
+                )
+            except Exception as e:
+                track_cli(CliTrackingEvents.CommandFailed, {"command": command, "arguments": kwargs, "error": str(e)})
+                raise e
+            finally:
+                track_cli(CliTrackingEvents.CommandCompleted, {"command": command, "arguments": kwargs})
+
+        return wrapper  # type: ignore
+
+    return decorator  # type: ignore

src/together/lib/cli/api/beta/clusters/create.py

@@ -8,6 +8,7 @@
 from rich import print
 
 from together import Together
+from together.lib.cli._track_cli import auto_track_command
 from together.lib.cli.api._utils import handle_api_errors
 from together.types.beta.cluster_create_params import SharedVolume, ClusterCreateParams
 
@@ -61,6 +62,7 @@
 )
 @click.pass_context
 @handle_api_errors("Clusters")
+@auto_track_command("clusters create")
 def create(
     ctx: click.Context,
     name: str | None = None,

src/together/lib/cli/api/beta/clusters/delete.py

@@ -3,6 +3,7 @@
 import click
 
 from together import Together
+from together.lib.cli._track_cli import auto_track_command
 from together.lib.cli.api._utils import handle_api_errors
 
 
@@ -15,6 +16,7 @@
 )
 @click.pass_context
 @handle_api_errors("Clusters")
+@auto_track_command("clusters delete")
 def delete(ctx: click.Context, cluster_id: str, json: bool) -> None:
     """Delete a cluster by ID"""
     client: Together = ctx.obj

src/together/lib/cli/api/beta/clusters/get_credentials.py

@@ -9,6 +9,7 @@
 
 import click
 
+from together.lib.cli._track_cli import auto_track_command
 from together import Together, TogetherError
 from together.lib.cli.api._utils import handle_api_errors
 
@@ -39,6 +40,7 @@
 )
 @click.pass_context
 @handle_api_errors("Clusters")
+@auto_track_command("clusters get-credentials")
 def get_credentials(
     ctx: click.Context,
     cluster_id: str,

src/together/lib/cli/api/beta/clusters/list.py

@@ -3,6 +3,7 @@
 import click
 
 from together import Together
+from together.lib.cli._track_cli import auto_track_command
 
 
 @click.command()
@@ -12,6 +13,7 @@
     help="Output in JSON format",
 )
 @click.pass_context
+@auto_track_command("clusters list")
 def list(ctx: click.Context, json: bool) -> None:
     """List clusters"""
     client: Together = ctx.obj

src/together/lib/cli/api/beta/clusters/list_regions.py

@@ -6,6 +6,7 @@
 
 from together import Together
 from together.lib.cli.api._utils import handle_api_errors
+from together.lib.cli._track_cli import auto_track_command
 
 
 @click.command()
@@ -16,6 +17,7 @@
 )
 @click.pass_context
 @handle_api_errors("Clusters")
+@auto_track_command("clusters list-regions")
 def list_regions(ctx: click.Context, json: bool) -> None:
     """List regions"""
     client: Together = ctx.obj

src/together/lib/cli/api/beta/clusters/retrieve.py

@@ -5,6 +5,7 @@
 
 from together import Together
 from together.lib.cli.api._utils import handle_api_errors
+from together.lib.cli._track_cli import auto_track_command
 
 
 @click.command()
@@ -16,6 +17,7 @@
 )
 @click.pass_context
 @handle_api_errors("Clusters")
+@auto_track_command("clusters retrieve")
 def retrieve(ctx: click.Context, cluster_id: str, json: bool) -> None:
     """Retrieve a cluster by ID"""
     client: Together = ctx.obj

src/together/lib/cli/api/beta/clusters/storage/create.py

@@ -4,6 +4,7 @@
 
 from together import Together
 from together.lib.cli.api._utils import handle_api_errors
+from together.lib.cli._track_cli import auto_track_command
 
 
 @click.command()
@@ -31,6 +32,7 @@
     help="Output in JSON format",
 )
 @click.pass_context
+@auto_track_command("clusters storage create")
 @handle_api_errors("Clusters Storage")
 def create(ctx: click.Context, region: str, size_tib: int, volume_name: str, json: bool) -> None:
     """Create a storage volume"""

src/together/lib/cli/api/beta/clusters/storage/delete.py

@@ -4,6 +4,7 @@
 
 from together import Together
 from together.lib.cli.api._utils import handle_api_errors
+from together.lib.cli._track_cli import auto_track_command
 
 
 @click.command()
@@ -18,6 +19,7 @@
 )
 @click.pass_context
 @handle_api_errors("Clusters Storage")
+@auto_track_command("clusters storage delete")
 def delete(ctx: click.Context, volume_id: str, json: bool) -> None:
     """Delete a storage volume"""
     client: Together = ctx.obj