feat: Generate metrics documentation (#65)

fix(tests): `clear_lru_caches` fixture conflicts with `saas_mode`/`enterprise_mode` pytest markers

feat(test-tools): Add `snapshot` fixture for snapshot testing

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: khvn26

settings/dev.py

@@ -8,6 +8,14 @@
 
 env = Env()
 
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": ["templates"],
+        "APP_DIRS": True,
+    },
+]
+
 # Settings expected by `mypy_django_plugin`
 AWS_SES_REGION_ENDPOINT: str
 SEGMENT_RULES_CONDITIONS_LIMIT: int
@@ -48,7 +56,8 @@
 TASK_DELETE_RETENTION_DAYS = 15
 TASK_DELETE_RUN_EVERY = timedelta(days=1)
 TASK_DELETE_RUN_TIME = time(5, 0, 0)
-TASK_PROCESSOR_MODE = False
+TASK_PROCESSOR_MODE = env.bool("RUN_BY_PROCESSOR", default=False)
+DOCGEN_MODE = env.bool("DOCGEN_MODE", default=False)
 TASK_RUN_METHOD = TaskRunMethod.TASK_PROCESSOR
 
 # Avoid models.W042 warnings

src/common/core/main.py

@@ -52,6 +52,9 @@ def ensure_cli_env() -> typing.Generator[None, None, None]:
         )
         os.environ["PROMETHEUS_MULTIPROC_DIR"] = prometheus_multiproc_dir_name
 
+    if "docgen" in sys.argv:
+        os.environ["DOCGEN_MODE"] = "true"
+
     if "task-processor" in sys.argv:
         # A hacky way to signal we're not running the API
         os.environ["RUN_BY_PROCESSOR"] = "true"

src/common/core/management/commands/docgen.py

@@ -0,0 +1,63 @@
+from operator import itemgetter
+from typing import Any, Callable
+
+import prometheus_client
+from django.core.management import BaseCommand, CommandParser
+from django.template.loader import get_template
+from django.utils.module_loading import autodiscover_modules
+from prometheus_client.metrics import MetricWrapperBase
+
+
+class Command(BaseCommand):
+    help = "Generate documentation for the Flagsmith codebase."
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        subparsers = parser.add_subparsers(
+            title="sub-commands",
+            required=True,
+        )
+
+        metric_parser = subparsers.add_parser(
+            "metrics",
+            help="Generate metrics documentation.",
+        )
+        metric_parser.set_defaults(handle_method=self.handle_metrics)
+
+    def initialise(self) -> None:
+        from common.gunicorn import metrics  # noqa: F401
+
+        autodiscover_modules(
+            "metrics",
+        )
+
+    def handle(
+        self,
+        *args: Any,
+        handle_method: Callable[..., None],
+        **options: Any,
+    ) -> None:
+        self.initialise()
+        handle_method(*args, **options)
+
+    def handle_metrics(self, *args: Any, **options: Any) -> None:
+        template = get_template("docgen-metrics.md")
+
+        flagsmith_metrics = sorted(
+            (
+                {
+                    "name": collector._name,
+                    "documentation": collector._documentation,
+                    "labels": collector._labelnames,
+                    "type": collector._type,
+                }
+                for collector in prometheus_client.REGISTRY._collector_to_names
+                if isinstance(collector, MetricWrapperBase)
+            ),
+            key=itemgetter("name"),
+        )
+
+        self.stdout.write(
+            template.render(
+                context={"flagsmith_metrics": flagsmith_metrics},
+            )
+        )

src/common/core/metrics.py

@@ -1,10 +1,11 @@
 import prometheus_client
+from django.conf import settings
 
 from common.core.utils import get_version_info
 
 flagsmith_build_info = prometheus_client.Gauge(
     "flagsmith_build_info",
-    "Flagsmith version and build information",
+    "Flagsmith version and build information.",
     ["ci_commit_sha", "version"],
     multiprocess_mode="livemax",
 )
@@ -20,4 +21,5 @@ def advertise() -> None:
     ).set(1)
 
 
-advertise()
+if not settings.DOCGEN_MODE:
+    advertise()

src/common/core/templates/docgen-metrics.md

@@ -0,0 +1,20 @@
+---
+title: Metrics
+---
+
+## Prometheus
+
+To enable the Prometheus `/metrics` endpoint, set the `PROMETHEUS_ENABLED` environment variable to `true`. 
+
+The metrics provided by Flagsmith are described below.
+
+{% for metric in flagsmith_metrics %}
+### `{{ metric.name }}`
+
+{{ metric.type|title }}.
+
+{{ metric.documentation }}
+
+Labels:
+{% for label in metric.labels %} - `{{ label }}`
+{% endfor %}{% endfor %}

src/common/gunicorn/metrics.py

@@ -6,17 +6,17 @@
 
 flagsmith_http_server_requests_total = prometheus_client.Counter(
     "flagsmith_http_server_requests_total",
-    "Total number of HTTP requests",
+    "Total number of HTTP requests.",
     ["route", "method", "response_status"],
 )
 flagsmith_http_server_request_duration_seconds = Histogram(
     "flagsmith_http_server_request_duration_seconds",
-    "HTTP request duration in seconds",
+    "HTTP request duration in seconds.",
     ["route", "method", "response_status"],
 )
 flagsmith_http_server_response_size_bytes = Histogram(
     "flagsmith_http_server_response_size_bytes",
-    "HTTP response size in bytes",
+    "HTTP response size in bytes.",
     ["route", "method", "response_status"],
     buckets=getattr(
         settings,

src/common/test_tools/__init__.py

@@ -1,3 +1,6 @@
-from common.test_tools.types import AssertMetricFixture
+from common.test_tools.types import AssertMetricFixture, SnapshotFixture
 
-__all__ = ("AssertMetricFixture",)
+__all__ = (
+    "AssertMetricFixture",
+    "SnapshotFixture",
+)

src/common/test_tools/plugin.py

@@ -5,7 +5,16 @@
 from prometheus_client.metrics import MetricWrapperBase
 from pyfakefs.fake_filesystem import FakeFilesystem
 
-from common.test_tools.types import AssertMetricFixture
+from common.test_tools.types import AssertMetricFixture, Snapshot, SnapshotFixture
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("snapshot")
+    group.addoption(
+        "--snapshot-update",
+        action="store_true",
+        help="Update snapshot files instead of testing against them.",
+    )
 
 
 def assert_metric_impl() -> Generator[AssertMetricFixture, None, None]:
@@ -68,3 +77,30 @@ def flagsmith_markers_marked(
             request.getfixturevalue("saas_mode")
         if marker.name == "enterprise_mode":
             request.getfixturevalue("enterprise_mode")
+
+
+@pytest.fixture
+def snapshot(request: pytest.FixtureRequest) -> SnapshotFixture:
+    """
+    Retrieve a `Snapshot` object getter for the current test.
+    The snapshot is stored in the `snapshots` directory next to the test file.
+
+    Snapshot files are named after the test function name (+ ".txt") by default.
+    If a name is provided to the getter, the snapshot will be stored in a file with that name.
+    The name is relative to the `snapshots` directory.
+
+    When `--snapshot-update` is provided to `pytest`:
+    - The snapshot will be created if it does not exist.
+    - If the comparison is false, the snapshot will be updated with the string it's being compared to in the test,
+    and the test will be marked as expected to fail.
+    """
+    for_update = request.config.getoption("--snapshot-update")
+    snapshot_dir = request.path.parent / "snapshots"
+    snapshot_dir.mkdir(exist_ok=True)
+
+    def _get_snapshot(name: str = "") -> Snapshot:
+        snapshot_name = name or f"{request.node.name}.txt"
+        snapshot_path = snapshot_dir / snapshot_name
+        return Snapshot(snapshot_path, for_update=for_update)
+
+    return _get_snapshot

src/common/test_tools/types.py

@@ -1,5 +1,8 @@
+from pathlib import Path
 from typing import Protocol
 
+import pytest
+
 
 class AssertMetricFixture(Protocol):
     def __call__(
@@ -9,3 +12,35 @@ def __call__(
         labels: dict[str, str],
         value: float | int,
     ) -> None: ...
+
+
+class SnapshotFixture(Protocol):
+    def __call__(self, name: str = "") -> "Snapshot": ...
+
+
+class Snapshot:
+    """
+    Read contents of `path` and make them available for comparison via the `==` operator.
+    If the contents are different, and `Snapshot` initialised in update mode,
+    (e.g. by running `pytest` with `--snapshot-update`), write the new contents to `path`.
+    """
+
+    def __init__(self, path: Path, for_update: bool) -> None:
+        self.path = path
+        mode = "r" if not for_update else "w+"
+        self.content: str = open(path, encoding="utf-8", mode=mode).read()
+        self.for_update = for_update
+
+    def __eq__(self, other: object) -> bool:
+        if self.content == other:
+            return True
+        if self.for_update and isinstance(other, str):
+            with open(self.path, "w", encoding="utf-8") as f:
+                f.write(other)
+            pytest.xfail(reason=f"Snapshot updated: {self.path}")
+        return False
+
+    def __str__(self) -> str:
+        return self.content
+
+    __repr__ = __str__

src/task_processor/metrics.py

@@ -5,18 +5,18 @@
 
 flagsmith_task_processor_enqueued_tasks_total = prometheus_client.Counter(
     "flagsmith_task_processor_enqueued_tasks_total",
-    "Total number of enqueued tasks",
+    "Total number of enqueued tasks.",
     ["task_identifier"],
 )
 
-if settings.TASK_PROCESSOR_MODE:
+if settings.DOCGEN_MODE or settings.TASK_PROCESSOR_MODE:
     flagsmith_task_processor_finished_tasks_total = prometheus_client.Counter(
         "flagsmith_task_processor_finished_tasks_total",
-        "Total number of finished tasks",
+        "Total number of finished tasks. Only collected by Task Processor. `task_type` label is either `recurring` or `standard`.",
         ["task_identifier", "task_type", "result"],
     )
     flagsmith_task_processor_task_duration_seconds = Histogram(
         "flagsmith_task_processor_task_duration_seconds",
-        "Task processor task duration in seconds",
+        "Task processor task duration in seconds. Only collected by Task Processor. `task_type` label is either `recurring` or `standard`.",
         ["task_identifier", "task_type", "result"],
     )