Merge pull request #90 from aaxelb/10239-djelme-tests

[wip][ENG-10239] djelme tests

Author: aaxelb

.containerignore

@@ -0,0 +1,7 @@
+.git
+.tox
+.venv
+garbage
+**/*.pyc
+testbox.Containerfile
+docker-compose.yml

.github/workflows/test_djelme.yml

@@ -12,43 +12,42 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         id: setup-py
         with:
-          python-version: '3.7'
+          python-version: '3.11'
       - run: pip install -U tox
       - name: alias pythonX.Y for tox
         run: alias python${{ steps.setup-py.outputs.python-version }}=${{ steps.setup-py.outputs.python-path }}
       - run: TOXENV=lint tox
 
 
   test:
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.6', '3.7', '3.8', '3.9', '3.10']
-        django: ['1.11', '2.0', '2.1', '2.2', '3.0', '3.1', '3.2', '4.0', '4.1']
+        python: ['3.10', '3.11', '3.12', '3.13', '3.14']
+        django: ['4.2', '5.1', '5.2']
         # TODO: elasticsearch: ['6', '7', '8', '9']
-        exclude:
-          - {python: '3.6', django: '4.0'}
-          - {python: '3.6', django: '4.1'}
-          - {python: '3.7', django: '4.0'}
-          - {python: '3.7', django: '4.1'}
-          - {python: '3.10', django: '1.11'}
-          - {python: '3.10', django: '2.0'}
-    runs-on: ubuntu-latest
     services:
-      elasticsearch:
+      elasticsearch6:
         image: elasticsearch:6.8.23
         ports:
-          - 9201:9200
+          - 9206:9200
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         id: setup-py
         with:
           python-version: ${{ matrix.python }}
-      - run: pip install -U tox
-      - run: alias python${{ steps.setup-py.outputs.python-version }}=${{ steps.setup-py.outputs.python-path }}
-      - run: TOXENV=`echo 'py${{ matrix.python }}-django${{matrix.django}}' | sed 's/\.//g'` tox
 
+      - name: alias python
+        run: alias python${{ steps.setup-py.outputs.python-version }}=${{ steps.setup-py.outputs.python-path }}
+
+      - name: install despondencies
+        run: pip install -U poetry && poetry install --with=dev --extras=elastic6
+
+      - run: TOXENV=`echo 'py${{ matrix.python }}-django${{matrix.django}}' | sed 's/\.//g'` poetry run tox
+        env:
+          ELASTICSEARCH6_HOST: localhost:9206

.konchrc

@@ -39,7 +39,7 @@ konch.config(
 
 def setup():
     # Set up django and add the default elasticsearch-py client and Metric to the context
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "elasticsearch_metrics.tests.settings")
     django.setup()
     from elasticsearch_metrics.metrics import Metric
 

.pre-commit-config.yaml

@@ -1,16 +1,16 @@
 repos:
 - repo: https://github.com/psf/black
-  rev: 22.8.0
+  rev: 26.1.0
   hooks:
   - id: black
-    language_version: python3.7
+    language_version: python3.11
 - repo: https://github.com/PyCQA/flake8
-  rev: 5.0.4
+  rev: 7.3.0
   hooks:
   - id: flake8
-    additional_dependencies: ["flake8-bugbear==22.8.23"]
+    additional_dependencies: ["flake8-bugbear==25.11.29"]
 - repo: https://github.com/asottile/blacken-docs
   rev: v1.12.1
   hooks:
   - id: blacken-docs
-    additional_dependencies: [black==22.8.0]
+    additional_dependencies: [black==26.1.0]

CONTRIBUTING.md

@@ -1,29 +1,18 @@
 # Contributing
+(TODO: update)
 
 ## Setting up for development
 
 * Create and activate a new virtual environment
 * `pip install -e '.[dev]'`
 * Install django: `pip install django`
 * (Optional but recommended) Run elasticsearch: `docker-compose up`
-* (Optional but recommended) If you're using a Python 3.6 virtual
-    environment, install the pre-commit hooks, which will
-    format and lint your git staged files:
-
-
-```
-# The pre-commit CLI was installed above
-pre-commit install
-```
 
 * Run tests using the following commands:
 
 ```
 # Run all tests (requires elasticsearch to be running)
-pytest
-
-# Skip tests that require elasticsearch to be running
-pytest -m "not es"
+python3 -m elasticsearch_metrics.tests
 
 # Check syntax
 flake8

README.md

@@ -7,8 +7,8 @@ Django app for storing time-series metrics in Elasticsearch.
 
 ## Pre-requisites
 
-* Python 2.7 or >=3.6
-* Django 1.11 or 2.0
+* Python >=3.10
+* Django 4.2, 5.1, or 5.2
 * Elasticsearch 6
 
 ## Install
@@ -43,10 +43,10 @@ A `Metric` is a subclass of [`elasticsearch_dsl.Document`](https://elasticsearch
 ```python
 # myapp/metrics.py
 
-from elasticsearch_metrics import metrics
+from elasticsearch_metrics.imps import elastic8
 
 
-class PageView(metrics.Metric):
+class PageView(elastic8.EventLog):
     user_id = metrics.Integer(index=True, doc_values=True)
 ```
 

docker-compose.yml

@@ -1,15 +1,30 @@
 version: '3'
 services:
-  elasticsearch:
+  elasticsearch6:
     image: docker.elastic.co/elasticsearch/elasticsearch:6.3.1
+    healthcheck:
+      interval: 30s
+      retries: 10
+      test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"'
+    networks:
+      - djelme
   testbox:
     build:
       context: .
-      dockerfile: testbox.Dockerfile
+      dockerfile: testbox.Containerfile
     depends_on:
-      - elasticsearch
+      elasticsearch6:
+        condition: service_healthy
     environment:
-      TOXENV: py39-django40
-      ELASTICSEARCH_HOST: elasticsearch:9200
-    volumes:
-      - ./:/code:cached
+      # TOXENV: py310-django42
+      ELASTICSEARCH6_HOST: elasticsearch6:9200
+    networks:
+      - djelme
+    # volumes:
+    #   - ./:/code:cached
+
+
+networks:
+  djelme:
+    driver: bridge
+

elasticsearch_metrics/apps.py

@@ -1,12 +1,13 @@
 from django.apps import AppConfig
-from django.conf import settings
-from elasticsearch_dsl.connections import connections
 from django.utils.module_loading import autodiscover_modules
 
+from elasticsearch_metrics.djelmetrics_imps import each_djelmetrics_imp
+
 
 class ElasticsearchMetricsConfig(AppConfig):
     name = "elasticsearch_metrics"
 
     def ready(self):
-        connections.configure(**settings.ELASTICSEARCH_DSL)
+        for _imp in each_djelmetrics_imp():
+            _imp.configure()
         autodiscover_modules("metrics")

elasticsearch_metrics/djelmetrics_imps.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+from collections.abc import Iterator
+import importlib
+
+from django.conf import settings
+
+from elasticsearch_metrics.protocols import (
+    ProtoDjelmetricsImp,
+    ProtoDjelmetricsImpModule,
+)
+
+__all__ = (
+    "each_djelmetrics_imp",
+    "get_djelmetrics_imp",
+    "ProtoDjelmetricsImp",
+)
+
+
+def each_djelmetrics_imp() -> Iterator[ProtoDjelmetricsImp]:
+    for _imp_name in getattr(settings, "DJELMETRICS_IMPS", ()):
+        yield get_djelmetrics_imp(_imp_name)
+
+
+def get_djelmetrics_imp(imp_name: str) -> ProtoDjelmetricsImp:
+    try:
+        _raw_imp_config = settings.DJELMETRICS_IMPS[imp_name]
+    except KeyError as _error:
+        raise ValueError(
+            f"{imp_name!r} not found in `settings.DJELMETRICS_IMPS`"
+        ) from _error
+    try:
+        _imp_path = _raw_imp_config["DJELMETRICS_IMP"]
+    except KeyError as _error:
+        raise ValueError(
+            f"missing 'DJELMETRICS_IMP' from {imp_name!r} in `settings.DJELMETRICS_IMPS`"
+            " -- must be a path to a djelme imp module like those under"
+            "`elasticsearch_metrics.imps...`"
+        ) from _error
+    try:
+        _imp_module = importlib.import_module(_imp_path)
+    except ImportError as _error:
+        raise ValueError(
+            f"could not import {_imp_path!r} for `settings.DJELMETRICS_IMPS[{imp_name!r}]`"
+        ) from _error
+    assert isinstance(_imp_module, ProtoDjelmetricsImpModule)
+    _imp_config = {
+        _k: _v for _k, _v in _raw_imp_config.items() if _k != "DJELMETRICS_IMP"
+    }
+    return _imp_module.djelme_imp_from_config(imp_name, _imp_config)

elasticsearch_metrics/exceptions.py

@@ -5,12 +5,14 @@ class ElasticsearchMetricsError(Exception):
 class IndexTemplateNotFoundError(ElasticsearchMetricsError):
     def __init__(self, message, client_error):
         self.client_error = client_error
-        super(IndexTemplateNotFoundError, self).__init__(message)
+        super(IndexTemplateNotFoundError, self).__init__(message, client_error)
 
 
 class IndexTemplateOutOfSyncError(ElasticsearchMetricsError):
     def __init__(self, message, mappings_in_sync, patterns_in_sync, settings_in_sync):
         self.mappings_in_sync = mappings_in_sync
         self.patterns_in_sync = patterns_in_sync
         self.settings_in_sync = settings_in_sync
-        super(IndexTemplateOutOfSyncError, self).__init__(message)
+        super(IndexTemplateOutOfSyncError, self).__init__(
+            message, mappings_in_sync, patterns_in_sync, settings_in_sync
+        )