wipwip

Author: aaxelb

elasticsearch_metrics/imps/elastic8.py

@@ -161,13 +161,13 @@ def check_djelme_setup(cls, using: str | None = None) -> bool:
         return bool(cls._index.get(using=using))
 
     @classmethod
-    def refresh_all(cls, using: str | None = None) -> None:
+    def refresh_timeseries_indexes(cls, using: str | None = None) -> None:
         cls._get_connection(using).indices.refresh(
             index=cls.format_timeseries_index_pattern()
         )
 
     @classmethod
-    def each_index(
+    def each_timeseries_index(
         cls, using: str | None = None
     ) -> collections.abc.Iterator[tuple[str, dict[str, typing.Any]]]:
         _resp = cls._get_connection(using).indices.get(
@@ -289,11 +289,10 @@ def search_timeseries_range(
         _index_pattern = cls.format_timeseries_index_pattern_for_range(
             from_when, until_when
         )
-        _timedepth = cls.get_timedepth()
         _timestamp_q = esdsl.query.Range(
             timestamp_parts={
-                "gte": timeseries_naming.semverlike_timeparts(from_when, _timedepth),
-                "lt": timeseries_naming.semverlike_timeparts(until_when, _timedepth),
+                "gte": timeseries_naming.full_semverlike_timeparts(from_when),
+                "lt": timeseries_naming.full_semverlike_timeparts(until_when),
             }
         )
         return cls.search(index=_index_pattern).filter(_timestamp_q)

elasticsearch_metrics/registry.py

@@ -255,10 +255,6 @@ def _get_recordtypes_for_app(
             )
         return self.all_recordtypes[app_label]
 
-    def _is_type_downstream_of_module(self, given_type: type, module_name: str) -> bool:
-        _upstream_modules = (_cls.__module__ for _cls in given_type.__mro__)
-        return module_name in _upstream_modules
-
 
 def _import_imp_module(imp_module_name: str) -> ProtoDjelmeImp:
     try:

elasticsearch_metrics/tests/dummy8app/metrics.py

@@ -12,23 +12,23 @@ class Dummy8Event(djelme.EventRecord):
 
     class Index:
         using = "my_elastic8_events"
-        settings = {"refresh_interval": "0"}  # immediate refresh
 
 
-class Dummy8EventWithExplicitNamePrefix(djelme.EventRecord):
+class Monthly8Event(djelme.EventRecord):
     intenzity: int
 
     class Meta:
         timeseries_name_prefix = "dummy8evenz"
         timeseries_recordtype_name = "eventlog"
+        timedepth = 2
 
     class Index:
         using = "my_elastic8_events"
 
 
 class ThingHappened(djelme.EventRecord):
     thing_id: str = ""
-    happen_code: str = ""
+    happen_code: str | None = None
     dot_path: str | None = mapped_field(Text(analyzer=dot_path_analyzer), default=None)
     commentary: str | None = mapped_field(Text(), default=None)
 
@@ -38,17 +38,17 @@ class Index:
 
     class Meta:
         timeseries_recordtype_name = "happen"
-        timedepth = 1
+        timedepth = 1  # yearly timeseries indexes
 
 
 # TODO: tests using ThingHappeningsReport
 class ThingHappeningsReport(djelme.CyclicRecord):
-    item_id: str
-    item_type: str
+    thing_id: str
+    happen_count: int
 
     class Index:
-        settings = {"refresh_interval": "-1"}
         using = "my_elastic8_reports"
 
     class Meta:
         timeseries_name_prefix = "blarg"
+        timedepth = 2  # monthly timeseries indexes

elasticsearch_metrics/tests/test_imps_elastic8.py

@@ -23,7 +23,7 @@
 )
 from elasticsearch_metrics.tests.dummy8app.metrics import (
     Dummy8Event,
-    Dummy8EventWithExplicitNamePrefix,
+    Monthly8Event,
     ThingHappened,
     ThingHappeningsReport,
 )
@@ -37,55 +37,103 @@ def _es8_client(
     return _backend.elastic8_client
 
 
-class TestFormatIndexName(SimpleDjelmeTestCase):
+class TestNamesAndPatterns(SimpleDjelmeTestCase):
     def test_format_timeseries_index_name(self):
         date = dt.date(2020, 2, 14)
         self.assertEqual(
             Dummy8Event.format_timeseries_index_name(date),
             "dummy8app_dummy8event_2020_02_14_",
         )
-
-    def test_format_timeseries_index_name__cls_timedepth(self):
-        date = dt.date(2020, 2, 14)
+        self.assertEqual(
+            Monthly8Event.format_timeseries_index_name(date),
+            "dummy8evenz_eventlog_2020_02_",
+        )
         self.assertEqual(
             ThingHappened.format_timeseries_index_name(date),
             "dummy8app_happen_2020_",
         )
         self.assertEqual(
             ThingHappeningsReport.format_timeseries_index_name(date),
-            "blarg_thinghappeningsreport_2020_02_14_",
+            "blarg_thinghappeningsreport_2020_02_",
         )
 
     def test_format_index_name_respects_date_format_setting(self):
         with self.settings(DJELME_DEFAULT_TIMEDEPTH=4):
             date = dt.date(2020, 2, 14)
+            # no timedepth set; use default setting
             self.assertEqual(
                 Dummy8Event.format_timeseries_index_name(date),
                 "dummy8app_dummy8event_2020_02_14_00_",
             )
+            # timedepth set; ignore default setting
+            self.assertEqual(
+                Monthly8Event.format_timeseries_index_name(date),
+                "dummy8evenz_eventlog_2020_02_",
+            )
             self.assertEqual(
                 ThingHappened.format_timeseries_index_name(date),
-                "dummy8app_happen_2020_",  # ThingHappened.Meta has timedepth=1
+                "dummy8app_happen_2020_",
+            )
+            self.assertEqual(
+                ThingHappeningsReport.format_timeseries_index_name(date),
+                "blarg_thinghappeningsreport_2020_02_",
             )
 
-
-class TestFormatIndexPattern(SimpleDjelmeTestCase):
-    def test_format_timeseries_index_pattern(self):
+    def test_format_timeseries_index_pattern__lotsa_parts(self):
         _timeparts = (2020, 2, 14, 0, 1, 2)
         self.assertEqual(
             Dummy8Event.format_timeseries_index_pattern(_timeparts),
             "dummy8app_dummy8event_2020_02_14_*",
         )
+        self.assertEqual(
+            Monthly8Event.format_timeseries_index_pattern(_timeparts),
+            "dummy8evenz_eventlog_2020_02_*",
+        )
+        self.assertEqual(
+            ThingHappened.format_timeseries_index_pattern(_timeparts),
+            "dummy8app_happen_2020_*",
+        )
+        self.assertEqual(
+            ThingHappeningsReport.format_timeseries_index_pattern(_timeparts),
+            "blarg_thinghappeningsreport_2020_02_*",
+        )
 
-    def test_format_timeseries_index_pattern__cls_timedepth(self):
+    def test_format_timeseries_index_pattern__some_parts(self):
         _timeparts = (2020, 2, 14)
+        self.assertEqual(
+            Dummy8Event.format_timeseries_index_pattern(_timeparts),
+            "dummy8app_dummy8event_2020_02_14_*",
+        )
+        self.assertEqual(
+            Monthly8Event.format_timeseries_index_pattern(_timeparts),
+            "dummy8evenz_eventlog_2020_02_*",
+        )
         self.assertEqual(
             ThingHappened.format_timeseries_index_pattern(_timeparts),
             "dummy8app_happen_2020_*",
         )
         self.assertEqual(
             ThingHappeningsReport.format_timeseries_index_pattern(_timeparts),
-            "blarg_thinghappeningsreport_2020_02_14_*",
+            "blarg_thinghappeningsreport_2020_02_*",
+        )
+
+    def test_format_timeseries_index_pattern__one_part(self):
+        _timeparts = (2020,)
+        self.assertEqual(
+            Dummy8Event.format_timeseries_index_pattern(_timeparts),
+            "dummy8app_dummy8event_2020_*",
+        )
+        self.assertEqual(
+            Monthly8Event.format_timeseries_index_pattern(_timeparts),
+            "dummy8evenz_eventlog_2020_*",
+        )
+        self.assertEqual(
+            ThingHappened.format_timeseries_index_pattern(_timeparts),
+            "dummy8app_happen_2020_*",
+        )
+        self.assertEqual(
+            ThingHappeningsReport.format_timeseries_index_pattern(_timeparts),
+            "blarg_thinghappeningsreport_2020_*",
         )
 
     def test_format_index_pattern_respects_date_format_setting(self):
@@ -95,9 +143,13 @@ def test_format_index_pattern_respects_date_format_setting(self):
                 Dummy8Event.format_timeseries_index_pattern(_timeparts),
                 "dummy8app_dummy8event_2020_02_14_00_*",
             )
+            self.assertEqual(
+                Monthly8Event.format_timeseries_index_pattern(_timeparts),
+                "dummy8evenz_eventlog_2020_02_*",
+            )
             self.assertEqual(
                 ThingHappened.format_timeseries_index_pattern(_timeparts),
-                "dummy8app_happen_2020_*",  # ThingHappened.Meta has timedepth=1
+                "dummy8app_happen_2020_*",
             )
 
 
@@ -138,12 +190,9 @@ def test_get_index_template_creates_template_with_mapping(self):
         assert properties["thing_id"] == {"type": "keyword"}
         assert properties["happen_code"] == {"type": "keyword"}
 
-    # regression test
     def test_mappings_are_not_shared(self):
         template1 = Dummy8Event.get_timeseries_template().to_dict()
-        template2 = (
-            Dummy8EventWithExplicitNamePrefix.get_timeseries_template().to_dict()
-        )
+        template2 = Monthly8Event.get_timeseries_template().to_dict()
         assert "intensity" in template1["template"]["mappings"]["properties"]
         assert "intenzity" not in template1["template"]["mappings"]["properties"]
         assert "intensity" not in template2["template"]["mappings"]["properties"]
@@ -169,7 +218,7 @@ class Meta:
     def test_template_name_defined_with_no_template_falls_back_to_default_template(
         self,
     ):
-        template = Dummy8EventWithExplicitNamePrefix.get_timeseries_template()
+        template = Monthly8Event.get_timeseries_template()
         # prefix and recordtype specified in class Meta
         assert template._template_name == "dummy8evenz_eventlog__template"
         # template pattern generated using same options
@@ -338,7 +387,7 @@ def test_init(self):
         self.assertEqual(_properties["thing_id"], {"type": "keyword"})
         self.assertEqual(_properties["happen_code"], {"type": "keyword"})
         # no indexes; only the template
-        self.assertFalse(list(ThingHappened.each_index()))
+        self.assertFalse(list(ThingHappened.each_timeseries_index()))
 
     def test_check_djelme_setup(self):
         with self.assertRaises(IndexTemplateNotFoundError):
@@ -371,10 +420,10 @@ def setUp(self):
         Dummy8Event.record(timestamp=dt.datetime(1235, 5, 6), intensity=111)
         Dummy8Event.record(timestamp=dt.datetime(2345, 6, 9), intensity=11)
         Dummy8Event.record(timestamp=dt.datetime(2345, 7, 9), intensity=13)
-        Dummy8Event.refresh_all()
+        Dummy8Event.refresh_timeseries_indexes()
 
     def test_indexes(self):
-        _index_names = {_name for _name, _ in Dummy8Event.each_index()}
+        _index_names = {_name for _name, _ in Dummy8Event.each_timeseries_index()}
         self.assertEqual(
             _index_names,
             {
@@ -414,3 +463,121 @@ def _assert_intens(hits, expected_intensities):
             Dummy8Event.search_timeseries_range((1234, 6), dt.date(2345, 7, 1)),
             {7, 111, 11},
         )
+        _assert_intens(
+            Dummy8Event.search_timeseries_range((1234, 5, 6, 2), (1234, 5, 7)),
+            {2},
+        )
+
+
+class TestMonthlyIndexes(RealElasticTestCase, autosetup_djelme_backends=True):
+    def setUp(self):
+        super().setUp()
+        Monthly8Event.record(timestamp=dt.datetime(1234, 5, 6), intenzity=1)
+        Monthly8Event.record(timestamp=dt.datetime(1234, 5, 6, 7), intenzity=2)
+        Monthly8Event.record(timestamp=dt.datetime(1234, 5, 7), intenzity=3)
+        Monthly8Event.record(timestamp=dt.datetime(1234, 6, 1), intenzity=7)
+        Monthly8Event.record(timestamp=dt.datetime(1235, 5, 6), intenzity=111)
+        Monthly8Event.record(timestamp=dt.datetime(2345, 6, 9), intenzity=11)
+        Monthly8Event.record(timestamp=dt.datetime(2345, 7, 9), intenzity=13)
+        Monthly8Event.refresh_timeseries_indexes()
+
+    def test_indexes(self):
+        _index_names = {_name for _name, _ in Monthly8Event.each_timeseries_index()}
+        self.assertEqual(
+            _index_names,
+            {
+                "dummy8evenz_eventlog_1234_05_",
+                "dummy8evenz_eventlog_1234_06_",
+                "dummy8evenz_eventlog_1235_05_",
+                "dummy8evenz_eventlog_2345_06_",
+                "dummy8evenz_eventlog_2345_07_",
+            },
+        )
+
+    def test_search(self):
+        def _assert_intenz(hits, expected_intenzities):
+            _actual = {_hit.intenzity for _hit in hits}
+            self.assertEqual(_actual, expected_intenzities)
+
+        _assert_intenz(
+            Monthly8Event.search(),
+            {1, 2, 3, 7, 11, 13, 111},
+        )
+        _assert_intenz(
+            Monthly8Event.search_timeseries_range((1234,), (1235,)),
+            {1, 2, 3, 7},
+        )
+        _assert_intenz(
+            Monthly8Event.search_timeseries_range((1233,), (1234,)),
+            set(),
+        )
+        _assert_intenz(
+            Monthly8Event.search_timeseries_range(
+                dt.date(1234, 5, 6), dt.date(1234, 5, 7)
+            ),
+            {1, 2},
+        )
+        _assert_intenz(
+            Monthly8Event.search_timeseries_range((1234, 6), dt.date(2345, 7, 1)),
+            {7, 111, 11},
+        )
+        _assert_intenz(
+            Monthly8Event.search_timeseries_range((1234, 5, 6, 2), (1234, 5, 7)),
+            {2},
+        )
+
+
+class TestYearlyIndexes(RealElasticTestCase, autosetup_djelme_backends=True):
+    def setUp(self):
+        super().setUp()
+        ThingHappened.record(timestamp=dt.datetime(1234, 5, 6), thing_id="a")
+        ThingHappened.record(timestamp=dt.datetime(1234, 5, 6, 7), thing_id="b")
+        ThingHappened.record(timestamp=dt.datetime(1234, 5, 7), thing_id="c")
+        ThingHappened.record(timestamp=dt.datetime(1234, 6, 1), thing_id="d")
+        ThingHappened.record(timestamp=dt.datetime(1235, 5, 6), thing_id="e")
+        ThingHappened.record(timestamp=dt.datetime(2345, 6, 9), thing_id="f")
+        ThingHappened.record(timestamp=dt.datetime(2345, 7, 9), thing_id="g")
+        ThingHappened.refresh_timeseries_indexes()
+
+    def test_indexes(self):
+        _index_names = {_name for _name, _ in ThingHappened.each_timeseries_index()}
+        self.assertEqual(
+            _index_names,
+            {
+                "dummy8app_happen_1234_",
+                "dummy8app_happen_1235_",
+                "dummy8app_happen_2345_",
+            },
+        )
+
+    def test_search(self):
+        def _assert_things(hits, expected_thing_ids):
+            _actual = {_hit.thing_id for _hit in hits}
+            self.assertEqual(_actual, expected_thing_ids)
+
+        _assert_things(
+            ThingHappened.search(),
+            {"a", "b", "c", "d", "e", "f", "g"},
+        )
+        _assert_things(
+            ThingHappened.search_timeseries_range((1234,), (1235,)),
+            {"a", "b", "c", "d"},
+        )
+        _assert_things(
+            ThingHappened.search_timeseries_range((1233,), (1234,)),
+            set(),
+        )
+        _assert_things(
+            ThingHappened.search_timeseries_range(
+                dt.date(1234, 5, 6), dt.date(1234, 5, 7)
+            ),
+            {"a", "b"},
+        )
+        _assert_things(
+            ThingHappened.search_timeseries_range((1234, 6), dt.date(2345, 7, 1)),
+            {"d", "e", "f"},
+        )
+        _assert_things(
+            ThingHappened.search_timeseries_range((1234, 5, 6, 2), (1234, 5, 7)),
+            {"b"},
+        )