various fixes

.env

@@ -0,0 +1,42 @@
+# This env file is used by docker compose to inject env variables inside the docker-compose.yml file (not provided to container)
+
+# Project namespace (defaults to the current folder name if not set)
+#COMPOSE_PROJECT_NAME=myproject
+
+# Password for the 'elastic' user (at least 6 characters)
+ELASTIC_PASSWORD=changeme
+
+
+# Password for the 'kibana_system' user (at least 6 characters)
+KIBANA_PASSWORD=changeme
+
+
+# Version of Elastic products
+STACK_VERSION=8.7.1
+
+
+# Set the cluster name
+CLUSTER_NAME=docker-cluster
+
+
+# Set to 'basic' or 'trial' to automatically start the 30-day trial
+LICENSE=basic
+#LICENSE=trial
+
+
+# Port to expose Elasticsearch HTTP API to the host
+ES_PORT=9200
+
+
+# Port to expose Kibana to the host
+KIBANA_PORT=5601
+
+
+# Increase or decrease based on the available host memory (in bytes)
+ES_MEM_LIMIT=1073741824
+KB_MEM_LIMIT=1073741824
+LS_MEM_LIMIT=1073741824
+
+
+# SAMPLE Predefined Key only to be used in POC environments
+ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2

.github/workflows/main.yml

@@ -1,47 +1,14 @@
-name: build ⚙️
+name: test ⚙️
 
 on: [ push, pull_request ]
 
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
-      matrix:
-        python-version: ['3.7', '3.8', '3.9']
+      fail-fast: false
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      name: Setup Python ${{ matrix.python-version }}
-      with:
-        python-version: ${{ matrix.python-version }}
-        cache: pip
-    - name: Install requirements
+    - uses: actions/checkout@v3
+    - name: Run test suite docker container 📦
       run: |
-        sudo apt-get update
-        sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libsqlite3-mod-spatialite spatialite-bin
-        pip install -r requirements-test.txt
-        pip install -r requirements-dev.txt
-        pip install pygdal=="`gdal-config --version`.*"
-        pip install .
-    - name: Configure sysctl limits
-      run: |
-        sudo swapoff -a
-        sudo sysctl -w vm.swappiness=1
-        sudo sysctl -w fs.file-max=262144
-        sudo sysctl -w vm.max_map_count=262144
-    - name: Install and run Elasticsearch 📦
-      uses: getong/[email protected]
-      with:
-        elasticsearch version: '8.2.2'
-        host port: 9200
-        container port: 9200
-        host node port: 9300
-        node port: 9300
-        discovery type: 'single-node'
-    - name: Run unit tests
-      run: |
-        pytest
-    # - name: run pre-commit (code formatting, lint and type checking)
-    #   run: |
-    #     python -m pip install pre-commit
-    #     pre-commit run --all-files
+        docker compose up --build test-gdal-322

.gitignore

@@ -102,7 +102,6 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
 .venv
 env/
 venv/

README.md

@@ -5,6 +5,7 @@ pygeofilter is a pure Python parser implementation of OGC filtering standards
 [![Build Status](https://github.com/geopython/pygeofilter/workflows/build%20%E2%9A%99%EF%B8%8F/badge.svg)](https://github.com/geopython/pygeofilter/actions)
 [![Documentation Status](https://readthedocs.org/projects/pygeofilter/badge/?version=latest)](https://pygeofilter.readthedocs.io/en/latest/?badge=latest)
 
+* Tested on debian 11 with gdal version 3.2.2 with python versions 3.9, 3.10 in combination with django versions 4.0, 4.1, 4.2.
 
 ## Features
 
@@ -173,28 +174,48 @@ class MyAPIEvaluator(Evaluator):
 
 ## Testing
 
+### Proconditions
+
+1. Install [gdal](https://gdal.org/download.html#binaries) on your system. This package and some site-packages will use it to calculate geospatial data.
+2. Provide an elasticsearch instance. You can simply use the provided docker compose setup.
+
+
+### Python dependencies
+
 For testing, several requirements must be satisfied. These can be installed, via pip:
 
 ```bash
+pip install -r requirements-base.txt
 pip install -r requirements-dev.txt
 pip install -r requirements-test.txt
 ```
 
-The functionality can be tested using `pytest`.
+### Docker
+
+Start the elasticsearch instance:
 
 ```bash
-python -m pytest
+docker compose up es01
+
 ```
 
-### Docker
+### Excecuting tests
 
-To execute tests in Docker:
 
+The functionality can be tested using `tox`. To run tests in a specific python and django version run
+
+```bash
+tox -e py311-django41
 ```
-docker build -t pygeofilter/test -f Dockerfile-3.9 .
-docker run --rm pygeofilter/test
+
+
+To execute tests with Docker:
+
+```bash
+docker compose up --build test-gdal-322
 ```
 
+
 ## Backends
 
 The following backends are shipped with `pygeofilter`. Some require additional dependencies, refer to the [installation](#installation) section for further details.

docker-compose.yml

@@ -0,0 +1,154 @@
+version: '3.8'
+services:
+
+  test-gdal-322:
+    build: 
+      context: .
+      dockerfile: docker/test/bullseye.Dockerfile 
+    tty: true # To support colorized log output.
+    networks:
+      - default
+    volumes:
+      - type: bind
+        source: ./pygeofilter
+        target: /app/pygeofilter
+      - type: bind
+        source: ./tests
+        target: /app/tests
+    environment:
+      - ESHOSTNAME=es01
+
+    depends_on:
+      - es01
+
+    # to get debug information run in verbose mode
+    # command: >
+    #  /bin/sh -c "tox run -v -v -v -v"
+
+    # to debug container
+    # command: >
+    #  tail -f /dev/null
+
+    command: >
+      /bin/sh -c "tox run"
+
+
+  es01:
+    depends_on:
+      - setup
+
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
+    ports:
+      - ${ES_PORT}:9200
+
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true 
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+    mem_limit: ${ES_MEM_LIMIT}
+    volumes:
+     - certs:/usr/share/elasticsearch/config/certs
+     - esdata01:/usr/share/elasticsearch/data
+    networks:
+      - default
+    healthcheck:
+     test:
+       [
+         "CMD-SHELL",
+         "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+       ]
+     interval: 10s
+     timeout: 10s
+     retries: 120
+
+
+  setup:
+   image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
+   volumes:
+     - certs:/usr/share/elasticsearch/config/certs
+   user: "0"
+   env_file:
+      - .env
+   command: >
+     bash -c '
+       if [ x${ELASTIC_PASSWORD} == x ]; then
+         echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+         exit 1;
+       elif [ x${KIBANA_PASSWORD} == x ]; then
+         echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+         exit 1;
+       fi;
+       if [ ! -f config/certs/ca.zip ]; then
+         echo "Creating CA";
+         bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+         unzip config/certs/ca.zip -d config/certs;
+       fi;
+       if [ ! -f config/certs/certs.zip ]; then
+         echo "Creating certs";
+         echo -ne \
+         "instances:\n"\
+         "  - name: es01\n"\
+         "    dns:\n"\
+         "      - es01\n"\
+         "      - localhost\n"\
+         "    ip:\n"\
+         "      - 127.0.0.1\n"\
+         "  - name: kibana\n"\
+         "    dns:\n"\
+         "      - kibana\n"\
+         "      - localhost\n"\
+         "    ip:\n"\
+         "      - 127.0.0.1\n"\
+         > config/certs/instances.yml;
+         bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+         unzip config/certs/certs.zip -d config/certs;
+       fi;
+       echo "Setting file permissions"
+       chown -R root:root config/certs;
+       find . -type d -exec chmod 750 \{\} \;;
+       find . -type f -exec chmod 640 \{\} \;;
+       echo "Waiting for Elasticsearch availability";
+       until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+       echo "Setting kibana_system password";
+       until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
+       echo "All done!";
+     '
+   healthcheck:
+     test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+     interval: 1s
+     timeout: 5s
+     retries: 120
+
+
+volumes:
+ certs:
+   driver: local
+ esdata01:
+   driver: local
+ kibanadata:
+   driver: local
+ metricbeatdata01:
+   driver: local
+ filebeatdata01:
+   driver: local
+ logstashdata01:
+   driver: local
+
+
+networks:
+ default:
+   name: elastic
+   external: false

docker/test/bullseye.Dockerfile

@@ -0,0 +1,37 @@
+FROM python:3.10-bullseye
+# finally there will be python version 3.9, 3.10 to run matrix tests with tox
+
+LABEL description="Test executor"
+
+
+RUN set -x \
+    && pythonVersions='python3.9 python3.9-dev' \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends  \ 
+    # diverse python versions
+    software-properties-common gpg-agent\
+    && add-apt-repository -y ppa:deadsnakes/ppa \
+    && apt-get purge -y --autoremove software-properties-common gpg-agent  \
+    && apt-get install -y --no-install-recommends $pythonVersions \
+    # Project os dependencies
+    && apt-get install -y --no-install-recommends \
+    binutils \
+    libproj-dev \
+    gdal-bin \
+    libgdal-dev \
+    libsqlite3-mod-spatialite \
+    spatialite-bin \ 
+    # cleanup
+    && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /app
+WORKDIR /app
+
+COPY requirements-test.txt requirements-base.txt README.md setup.py tox.ini ./
+
+# Code base will binded by docker compose. Otherwise the container needs to rebuild on any code change.
+
+# create dynamic requirements file with the current os gdal version
+RUN echo "gdal==`gdal-config --version`.*" >> requirements-gdal.txt \
+    && pip install tox
+