wipwip

Author: aaxelb

elasticsearch_metrics/apps.py

@@ -1,5 +1,4 @@
 from collections.abc import Iterator
-import importlib
 
 from django.apps import AppConfig
 from django.conf import settings
@@ -21,6 +20,9 @@ def ready(self) -> None:
         autodiscover_modules("metrics")
 
 
+###
+# accessing django settings
+
 def each_timeseries_imp_config() -> Iterator[tuple[str, str, dict[str, str]]]:
     for _imp_name in getattr(settings, _IMPCONFIG_SETTINGS_KEY, ()):
         (_imp_module_path, _imp_config) = get_timeseries_imp_config(_imp_name)
@@ -29,9 +31,11 @@ def each_timeseries_imp_config() -> Iterator[tuple[str, str, dict[str, str]]]:
 
 def get_timeseries_imp_config(imp_name: str) -> tuple[str, dict[str, str]]:
     try:
-        _imp_module_path, _imp_config = getattr(settings, _IMPCONFIG_SETTINGS_KEY)
+        _imps_setting = getattr(settings, _IMPCONFIG_SETTINGS_KEY)
     except AttributeError as _error:
         raise ValueError(f"no `settings.{_IMPCONFIG_SETTINGS_KEY}` found") from _error
+    (_imp_module_path, _imp_config) = _imps_setting.get(imp_name, ('', {}))
+    # TODO: better errors
     assert isinstance(_imp_module_path, str)
     assert isinstance(_imp_config, dict)
     assert all(

elasticsearch_metrics/djelmetrics_imps.py

@@ -0,0 +1,4 @@
+__all__ = (
+    'elastic6',
+    'elastic8',
+)

elasticsearch_metrics/imps/__init__.py

@@ -85,7 +85,7 @@ def __new__(mcls, name, bases, attrs):  # noqa: B902
         # Abstract base metrics can't be instantiated and don't appear in
         # the list of metrics for an app.
         if not abstract:
-            registry.register(app_label, new_cls)
+            registry.register_recordtype(app_label, new_cls)
         return new_cls
 
     # Override IndexMeta.construct_index so that
@@ -224,7 +224,7 @@ def get_timeseries_index_template(cls):
         )
 
     @classmethod
-    def get_index_name(cls, date=None):
+    def get_index_name(cls, date=None) -> str:
         date = date or timezone.now().date()
         dateformat = getattr(
             settings, "ELASTICSEARCH_METRICS_DATE_FORMAT", DEFAULT_DATE_FORMAT
@@ -282,6 +282,7 @@ class DjelmeElastic6Imp(ProtoTimeseriesImp):
 
     imp_name: str
     imp_config: dict[str, str]
+    namespace_prefix: str = ""
 
     @property
     def elastic6_client(self):
@@ -307,7 +308,8 @@ def teardown_timeseries_indexes(self) -> None:
                 pass
 
     def _each_metric_type(self) -> Iterator[type[Metric]]:
-        for _metric in registry.get_metrics(imp_name=self.imp_name):
+        for _metric in registry.each_recordtype(imp_name=self.imp_name):
+            assert issubclass(_metric, Metric)
             yield _metric
 
 

elasticsearch_metrics/imps/elastic6.py

@@ -1,25 +1,32 @@
 """elasticsearch_metrics.elastic8: store events and reports in elasticsearch 8
 
->>> from elasticsearch_metrics.imps import elastic8 as djelme
->>> class MyCountableEvent(djelme.EventLog):
-...     
+>>> from elasticsearch_metrics.imps.elastic8 import EventLog
+>>> class TheThingHappened(EventLog):
+...     intensity: int
+...
+...     class Index:  # elasticsearch index settings
+...         settings = {
+...             "number_of_shards": 2,
+...             "refresh_interval": "5s",
+...         }
 """
 __all__ = (
     'TimeseriesRecord',
     'EventLog',
     'PeriodicReport',
 )
 from collections import ChainMap
+import dataclasses
 import datetime
 import logging
+import typing
 
-from django.apps import apps
 from django.conf import settings
 from django.utils import timezone
 from elasticsearch8.exceptions import NotFoundError
-from elasticsearch8_dsl import Document, connections
-from elasticsearch8_dsl.document import IndexMeta, MetaField
-from elasticsearch8_dsl.index import Index
+from elasticsearch8.dsl import Document, connections
+from elasticsearch8.dsl.document import MetaField
+from elasticsearch8.dsl.index import Index
 
 from elasticsearch_metrics import signals
 from elasticsearch_metrics import exceptions
@@ -34,23 +41,24 @@
 logger = logging.getLogger(__name__)
 
 
-class IndexInheritanceMeta(IndexMeta):
-    # override elasticsearch8.document.IndexMeta.construct_index
-    # so Index attrs are inherited
-    # and each subtype gets their own index (`opts` is never None)
-    @classmethod
-    def construct_index(cls, opts, bases):
-        parent_configs = [
-            base._index.to_dict() for base in bases if hasattr(base, "_index")
-        ]
-        chained_opts = (
-            ChainMap(ReadonlyAttrMap(opts), *parent_configs)
-            if opts
-            else ChainMap(*parent_configs)
-        )
-        # since chained_opts is not None, IndexMeta.construct_index won't reuse the index from a parent type
-        return super().construct_index(chained_opts, bases)
-
+# TODO: do index inheritance without metaclasses
+# class IndexInheritanceMeta(IndexMeta):
+#     # override elasticsearch8.document.IndexMeta.construct_index
+#     # so Index attrs are inherited
+#     # and each subtype gets their own index (`opts` is never None)
+#     @classmethod
+#     def construct_index(cls, opts, bases):
+#         parent_configs = [
+#             base._index.to_dict() for base in bases if hasattr(base, "_index")
+#         ]
+#         chained_opts = (
+#             ChainMap(ReadonlyAttrMap(opts), *parent_configs)
+#             if opts
+#             else ChainMap(*parent_configs)
+#         )
+#         # since chained_opts is not None, IndexMeta.construct_index won't reuse the index from a parent type
+#         return super().construct_index(chained_opts, bases)
+# 
 
 # class TimeseriesIndexMeta(IndexMeta):
 #     """Metaclass for the base `TimeseriesRecord` class."""
@@ -104,7 +112,7 @@ def construct_index(cls, opts, bases):
 #         # Abstract base metrics can't be instantiated and don't appear in
 #         # the list of metrics for an app.
 #         if not abstract:
-#             registry.register(app_label, new_cls)
+#             registry.register_recordtype(app_label, new_cls)
 #         return new_cls
 # 
 #     # Override IndexMeta.construct_index so that
@@ -132,46 +140,78 @@ def construct_index(cls, opts, bases):
 # 
 # 
 
-class TimeseriesRecord(Document, metaclass=IndexInheritanceMeta):
+class TimeseriesRecord(Document):
     """Base document
-
-    Example usage:
-    >>> from elasticsearch_metrics.imps.elastic8 import TimeseriesEvent
-    >>> class PageView(TimeseriesEvent):
-    ...     page_route_key: str
-    ...
-    ...     class Index:
-    ...         settings = {
-    ...             "number_of_shards": 2,
-    ...             "refresh_interval": "5s",
-    ...         }
     """
 
     timestamp = Date(doc_values=True, required=True)
 
     class Meta:
         source = MetaField(enabled=False)
+        abstract = True
 
     def __init_subclass__(cls, **kwargs):
         # (sidenote: simpler? way to register subclasses, compared to metaclasses)
         super().__init_subclass__(**kwargs)
         # TODO: register cls with metrics registry
-#         if not abstract:
-#             registry.register(app_label, cls)
+        if not cls.get_meta_setting('abstract'):
+            registry.register_recordtype(cls.get_app_label(), cls)
 
     @classmethod
     def init(cls, index=None, using=None):
         """Create the index and populate the mappings in elasticsearch."""
         cls.sync_index_template(using=using)
         return super().init(index=index or cls.get_timeseries_index_name(), using=using)
 
+    @classmethod
+    def get_timeseries_index_template(cls):
+        """Return an `IndexTemplate <elasticsearch_dsl.IndexTemplate>` for this metric."""
+        return cls._index.as_template(
+            template_name=cls._template_name, pattern=cls._template_pattern
+        )
+
+    @classmethod
+    def get_template_pattern(cls) -> str:
+        return f"{cls.get_template_name()}_*"
+
+    @classmethod
+    def get_template_pattern(cls) -> str:
+        if not template_name:
+            metric_name = new_cls.__name__.lower()
+            # If template_name not specified in class Meta,
+            # compute it as <app label>_<lowercased class name>
+            template_name = f"{app_label}_{metric_name}"
+
+        new_cls._template_name = template_name
+
+    @classmethod
+    def get_app_label(cls) -> str:
+        _app_label = self.get_meta_setting("app_label")
+        if _app_label is None:
+            # Look for an application configuration to attach the model to.
+            app_config = apps.get_containing_app_config(module)
+            if app_config is None:
+                if not self.get_meta_setting("abstract"):
+                    raise RuntimeError(
+                        "Metric class %s.%s doesn't declare an explicit "
+                        "app_label and isn't in an application in "
+                        "INSTALLED_APPS." % (module, name)
+                    )
+            else:
+                _app_label = app_config.label
+        assert isinstance(_app_label, str)
+        return _app_label
+
+    @classmethod
+    def get_meta_setting(cls, setting_name: str) -> typing.Any:
+        _meta_settings_cls = getattr(cls, 'Meta', None)
+        return getattr(_meta_settings_cls, setting_name, None)
 
     ######
     @classmethod
     def sync_index_template(cls, using=None):
         """Sync the index template for this metric in Elasticsearch."""
         index_template = cls.get_timeseries_index_template()
-        index_template.document(cls)
         signals.pre_index_template_create.send(
             cls, index_template=index_template, using=using
         )
@@ -244,13 +284,6 @@ def check_index_template(cls, using=None):
                 )
             return True
 
-    @classmethod
-    def get_timeseries_index_template(cls):
-        """Return an `IndexTemplate <elasticsearch_dsl.IndexTemplate>` for this metric."""
-        return cls._index.as_template(
-            template_name=cls._template_name, pattern=cls._template_pattern
-        )
-
     @classmethod
     def get_timeseries_index_name(cls, datestamp: datetime.date | None = None) -> str:
         datestamp = datestamp or timezone.now().date()
@@ -306,3 +339,43 @@ class PeriodicReport(TimeseriesRecord):
     timestamp: datetime.datetime
     report_label: str
     report_coverage: ...  # timespan
+
+
+@dataclasses.dataclass
+class DjelmeElastic8Imp(ProtoTimeseriesImp):
+    """DjelmeElastic8Imp: the elastic8 implementation of djelme (for use by generic djelme code)"""
+
+    imp_name: str
+    imp_config: dict[str, str]
+    namespace_prefix: str = ""
+
+    @property
+    def elastic8_client(self):
+        # assumes `configure` was already called
+        return connections.get_connection(self.imp_name)
+
+    def configure(self) -> None:
+        connections.configure(**{self.imp_name: self.imp_config})
+
+    def setup_timeseries_indexes(self) -> None:
+        for _metric_type in self._each_metric_type():
+            # TODO: logger.info
+            _metric_type.sync_index_template(using=self.elastic6_client)
+
+    def teardown_timeseries_indexes(self) -> None:
+        for _metric_type in self._each_metric_type():
+            _indexname_wildcard = _metric_type._template_pattern
+            _templatename = _metric_type._template_name
+            self.elastic6_client.indices.delete(index=_indexname_wildcard)
+            try:
+                self.elastic6_client.indices.delete_template(_templatename)
+            except NotFoundError:
+                pass
+
+    def _each_metric_type(self) -> Iterator[type[Metric]]:
+        for _metric in registry.each_recordtype(imp_name=self.imp_name):
+            assert issubclass(_metric, Metric)
+            yield _metric
+
+
+djelme_imp_from_config = DjelmeElastic8Imp  # for ProtoDjelmetricsImpModule

elasticsearch_metrics/imps/elastic8.py

@@ -39,7 +39,7 @@ def handle(self, *args, **options):
         out_of_sync_count = 0
         self.stdout.write("Checking for outdated index templates...")
         for app_label in app_labels:
-            for metric in registry.get_metrics(app_label=app_label):
+            for metric in registry.each_recordtype(app_label=app_label):
                 try:
                     metric.check_index_template(using=connection)
                 except (

elasticsearch_metrics/management/commands/check_metrics.py

@@ -24,7 +24,7 @@ def handle(self, *args, **options):
             self.stdout.write(
                 "Metrics for '{}':".format(app_label), style.MIGRATE_HEADING
             )
-            for metric in registry.get_metrics(app_label=app_label):
+            for metric in registry.each_recordtype(app_label=app_label):
                 metric_name = style.METRIC(metric.__name__)
                 template_name = metric._template_name
                 template_pattern = style.ES_TEMPLATE(metric._template_pattern)

elasticsearch_metrics/management/commands/show_metrics.py

@@ -38,7 +38,7 @@ def handle(self, *args, **options):
             self.stdout.write(
                 "Syncing metrics for app: '{}'".format(app_label), style.MIGRATE_HEADING
             )
-            for metric in registry.get_metrics(app_label=app_label):
+            for metric in registry.each_recordtype(app_label=app_label):
                 metric_name = style.METRIC(metric.__name__)
                 template_name = metric._template_name
                 template_pattern = style.ES_TEMPLATE(metric._template_pattern)

elasticsearch_metrics/management/commands/sync_metrics.py

@@ -28,4 +28,5 @@ class ProtoTimeseriesImpModule(typing.Protocol):
     def djelme_imp_from_config(
         imp_name: str,
         imp_config: dict[str, str],
+        namespace_prefix: str = "",
     ) -> ProtoTimeseriesImp: ...