[ENG-9699] expiring usage events (#11760)

* expire usage events after 90 days

* chore: renames from djelme

Author: aaxelb

api/base/settings/defaults.py

@@ -337,8 +337,7 @@
         },
     },
 }
-# Store yearly indices for time-series metrics
-ELASTICSEARCH_METRICS_DATE_FORMAT = '%Y'
+OSF_USAGEEVENT_EXPIRATION_DAYS = 90
 
 WAFFLE_CACHE_NAME = 'waffle_cache'
 STORAGE_USAGE_CACHE_NAME = 'storage_usage'

osf/metrics/README.md

@@ -15,7 +15,7 @@ for field definitions with comments mapping fields to concepts in the COUNTER sp
 
 each periodic report is a subclass of `osf.metrics.monthly_reports.BaseMonthlyReport`
 or `osf.metrics.daily_reports.BaseDailyReport` (themselves subclasses of
-`elasticsearch_metrics.imps.elastic8.CyclicRecord`) and has a "reporter"
+`elasticsearch_metrics.imps.elastic8.CyclicReport`) and has a "reporter"
 (see `osf.metrics.reporters`) that is invoked periodically to report.
 
 ## api

osf/metrics/daily_reports.py

@@ -22,7 +22,7 @@
 ###
 # base class
 
-class BaseDailyReport(djelme.CyclicRecord):
+class BaseDailyReport(djelme.CyclicReport):
     CYCLE_TIMEDEPTH = DAILY
 
     class Meta:

osf/metrics/events.py

@@ -3,10 +3,13 @@
 import functools
 from urllib.parse import urlsplit
 
+from django.conf import settings as django_settings
+from django.core.management import call_command
 import elasticsearch8.dsl as esdsl
 from elasticsearch_metrics import MONTHLY
 import elasticsearch_metrics.imps.elastic8 as djelme
 
+from framework.celery_tasks import app as celery_app
 from osf.metadata.osfmap_utils import osfid_from_iri
 from osf.metrics.utils import (
     get_database_iri,
@@ -23,6 +26,11 @@
 )
 
 
+@celery_app.task()
+def delete_expired_djelme_indexes():
+    call_command('djelme_indexes', '--delete-expired', '--really-really')
+
+
 ###
 # inner objects for events
 
@@ -94,6 +102,7 @@ class OsfCountedUsageEvent(djelme.CountedUsageRecord):
 
     class Meta:
         timeseries_index_timedepth = MONTHLY
+        timeseries_index_expiration = datetime.timedelta(days=django_settings.OSF_USAGEEVENT_EXPIRATION_DAYS)
 
     class ActionLabel(enum.Enum):
         SEARCH = 'search'  # counter:Search

osf/metrics/monthly_reports.py

@@ -23,7 +23,7 @@
 ###
 # base class
 
-class BaseMonthlyReport(djelme.CyclicRecord):
+class BaseMonthlyReport(djelme.CyclicReport):
     CYCLE_TIMEDEPTH = MONTHLY
 
     class Meta:

osf/metrics/utils.py

@@ -6,7 +6,7 @@
 from hashlib import sha256
 from typing import ClassVar
 
-from elasticsearch_metrics.util.timeparts import format_timeparts
+from elasticsearch_metrics.util.timeparts import serialize_timeparts
 
 from osf.metadata.osfmap_utils import (
     osfmap_type,
@@ -22,7 +22,7 @@ def cycle_coverage_date(given_date: datetime.date) -> str:
     >>> cycle_coverage_date(datetime.datetime(7654, 3, 2, 1))
     '7654.3.2'
     """
-    return format_timeparts(given_date, 3)
+    return serialize_timeparts((given_date.year, given_date.month, given_date.day), 3)
 
 
 def cycle_coverage_yearmonth(given_ym: YearMonth | datetime.date) -> str:
@@ -32,7 +32,7 @@ def cycle_coverage_yearmonth(given_ym: YearMonth | datetime.date) -> str:
     >>> cycle_coverage_yearmonth(datetime.date(1234, 5, 6))
     '1234.5'
     """
-    return format_timeparts((given_ym.year, given_ym.month), 2)
+    return serialize_timeparts((given_ym.year, given_ym.month), 2)
 
 
 def stable_key(*key_parts):

osf_tests/metrics/reporters/_testutils.py

@@ -1,9 +1,9 @@
-from elasticsearch_metrics.imps.elastic8 import CyclicRecord
+from elasticsearch_metrics.imps.elastic8 import CyclicReport
 
 from osf.metrics.reporters._base import MonthlyReporter
 
 
-def list_monthly_reports(reporter: MonthlyReporter) -> list[CyclicRecord]:
+def list_monthly_reports(reporter: MonthlyReporter) -> list[CyclicReport]:
     _each_reports_list = (
         reporter.report(**_kwargs)
         for _kwargs in reporter.iter_report_kwargs()

poetry.lock

@@ -89,7 +89,7 @@ datacite = "1.1.3"
 rdflib = "7.0.0"
 colorlog = "6.8.2"
 # Metrics
-django-elasticsearch-metrics = {git ="https://github.com/CenterForOpenScience/django-elasticsearch-metrics.git", rev = "46890bb61d35459e9793eba92d9ae54d4ce9c6af"}
+django-elasticsearch-metrics = {git ="https://github.com/CenterForOpenScience/django-elasticsearch-metrics.git", rev = "22cea2531783d3f06da3f4407624aae0bbb50c02"}
 # Impact Metrics CSV Export
 djangorestframework-csv = "3.0.2"
 gevent = "24.2.1"

pyproject.toml

@@ -728,6 +728,10 @@ class CeleryConfig:
             'schedule': crontab(minute=0, hour=5),  # Daily 12 a.m
             'kwargs': {'dry_run': False},
         },
+        'delete_expired_djelme_indexes': {
+            'task': 'osf.metrics.events.delete_expired_djelme_indexes',
+            'schedule': crontab(minute=30, hour=7, day_of_month=5),     # Fifth day of month 2:30 a.m. EST
+        },
     }