Support Elasticsearch 8 along with Elasticsearch 7

.github/workflows/tests.yml

@@ -31,13 +31,13 @@ jobs:
       # git checkout
       - uses: actions/checkout@v2
 
-      - name: Setup elasticsearch docker container with ingest attachment plugin
+      - name: Setup elasticsearch 7.17.7 docker container with ingest attachment plugin
         run: |
           docker container create --name elastictest \
           -e "discovery.type=single-node" \
           -e "cluster.name=docker-cluster" \
           -e "http.cors.enabled=true" \
-          -e "http.cors.allow-origin=*" \
+          -e "http.cors.allow-origin='*'" \
           -e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \
           -e "http.cors.allow-credentials=true" \
           -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \
@@ -57,9 +57,32 @@ jobs:
 
       - name: Install package
         run: |
-          pip install -e ".[test, redis]"
+          pip install -e ".[test, redis]" elasticsearch==7.17.7
 
-      # test
-      - name: test
+      # test elasticsearch 7.17.7
+      - name: test elasticsearch 7.17.7
         run: |
+          zope-testrunner --auto-color --auto-progress --test-path src; \
+          docker stop elastictest; \
+          docker rm elastictest
+
+      # test elasticsearch 8.10.2
+      - name: Setup elasticsearch 8.10.2 docker container
+        run: |
+          docker container create --name elastictest8 \
+          -e "discovery.type=single-node" \
+          -e "cluster.name=docker-cluster" \
+          -e "http.cors.enabled=true" \
+          -e "http.cors.allow-origin='*'" \
+          -e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \
+          -e "http.cors.allow-credentials=true" \
+          -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \
+          -p 9200:9200 \
+          -p 9300:9300 \
+          elasticsearch:8.10.2; \
+          docker start elastictest8; \
+
+      - name: test elasticsearch 8.10.2
+        run: |
+          pip install elasticsearch==8.10.0; /
           zope-testrunner --auto-color --auto-progress --test-path src

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 5.1.0 (unreleased)
+
+- Add suport of elasticsearch to 8.10.2 @maethu
+
 ## 5.0.1 (unreleased)
 
 - Update elasticsearch to 7.17.7 (Ready for 8.x and apple silicon images are available) @maethu

Makefile

@@ -20,7 +20,8 @@ PLONE6=6.0-latest
 
 INSTANCE_YAML=instance.yaml
 
-ELASTIC_SEARCH_IMAGE=elasticsearch:7.17.7
+ELASTIC_SEARCH_IMAGE_7=elasticsearch:7.17.7
+ELASTIC_SEARCH_IMAGE_8=elasticsearch:8.10.2
 ELASTIC_SEARCH_CONTAINER=elastictest
 
 REDIS_IMAGE=redis:7.0.5
@@ -138,29 +139,57 @@ lint-pyroma: ## validate using pyroma
 lint-zpretty: ## validate ZCML/XML using zpretty
 	$(LINT) zpretty ${CODEPATH}
 
-.PHONY: elastic
-elastic: ## Create Elastic Search container
+.PHONY: elastic-7
+elastic-7: ## Create Elastic Search container
+	@echo "$(GREEN)==> Create Elastic Search Version 7 Container $(RESET)"
 	@if [ $(ELASTIC_SEARCH_CONTAINERS) -eq 0 ]; then \
 		docker container create --name $(ELASTIC_SEARCH_CONTAINER) \
 		-e "discovery.type=single-node" \
 		-e "cluster.name=docker-cluster" \
 		-e "http.cors.enabled=true" \
-		-e "http.cors.allow-origin=*" \
+		-e "http.cors.allow-origin='*'" \
 		-e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \
 		-e "http.cors.allow-credentials=true" \
+		-e "xpack.security.enabled=false" \
 		-e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \
 		-p 9200:9200 \
 		-p 9300:9300 \
-		$(ELASTIC_SEARCH_IMAGE); \
+		$(ELASTIC_SEARCH_IMAGE_7); \
 		docker start $(ELASTIC_SEARCH_CONTAINER); \
 		docker exec $(ELASTIC_SEARCH_CONTAINER) /bin/sh -c "bin/elasticsearch-plugin install ingest-attachment -b"; \
-		docker stop $(ELASTIC_SEARCH_CONTAINER);fi
+		docker stop $(ELASTIC_SEARCH_CONTAINER); else \
+		echo "$(RED)==> Could not create container: Container already exists $(RESET)";fi
 
-.PHONY: start-elastic
-start-elastic: elastic ## Start Elastic Search
+.PHONY: elastic-8
+elastic-8: ## Create Elastic Search container
+	@echo "$(GREEN)==> Create Elastic Search Version 8 Container $(RESET)"
+	@if [ $(ELASTIC_SEARCH_CONTAINERS) -eq 0 ]; then \
+		pip install elasticsearch==8.10.0; \
+		docker container create --name $(ELASTIC_SEARCH_CONTAINER) \
+		-e "discovery.type=single-node" \
+		-e "cluster.name=docker-cluster" \
+		-e "http.cors.enabled=true" \
+		-e "http.cors.allow-origin='*'" \
+		-e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \
+		-e "http.cors.allow-credentials=true" \
+		-e "xpack.security.enabled=false" \
+		-e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \
+		-p 9200:9200 \
+		-p 9300:9300 \
+		$(ELASTIC_SEARCH_IMAGE_8); else \
+		echo "$(RED)==> Could not create container: Container already exists $(RESET)";fi
+
+.PHONY: start-elastic-7
+start-elastic-7: elastic-7 ## Start Elastic Search
 	@echo "$(GREEN)==> Start Elastic Search$(RESET)"
 	@docker start $(ELASTIC_SEARCH_CONTAINER)
 
+.PHONY: start-elastic-8
+start-elastic-8: elastic-8 ## Start Elastic Search
+	@echo "$(GREEN)==> Start Elastic Search$(RESET)"
+	@docker start $(ELASTIC_SEARCH_CONTAINER)
+
+
 .PHONY: stop-elastic
 stop-elastic: ## Stop Elastic Search
 	@echo "$(GREEN)==> Stop Elastic Search$(RESET)"
@@ -187,12 +216,23 @@ stop-redis: ## Stop redis
 
 .PHONY: test
 test: ## run tests
-	make start-elastic
+	bin/pip install elasticsearch==8.10.0
+	make start-elastic-8
+	make start-redis
+	PYTHONWARNINGS=ignore ./bin/zope-testrunner --auto-color --auto-progress --test-path src/
+	make stop-elastic
+	make stop-redis
+
+.PHONY: test-elastic-7
+test-elastic-7: ## run tests with elasticsearch 7
+	bin/pip install elasticsearch==7.17.7
+	make start-elastic-7
 	make start-redis
 	PYTHONWARNINGS=ignore ./bin/zope-testrunner --auto-color --auto-progress --test-path src/
 	make stop-elastic
 	make stop-redis
 
+
 .PHONY: start
 start: ## Start a Plone instance on localhost:8080
 	PYTHONWARNINGS=ignore ./bin/runwsgi instance/etc/zope.ini
@@ -202,7 +242,7 @@ populate: ## Populate site with wikipedia content
 	PYTHONWARNINGS=ignore ./bin/zconsole run etc/zope.conf scripts/populate.py
 
 .PHONY: start-redis-support
-start-redis-support: ## Start a Plone instance on localhost:8080
+start-redis-support: ## Start a Plone instance on localhost:8080 with redis support
 	@echo "$(GREEN)==> Set env variables, PLONE_REDIS_DSN, PLONE_BACKEND, PLONE_USERNAME and PLONE_PASSWORD before start instance$(RESET)"
 	PYTHONWARNINGS=ignore \
 	$(DEFAULT_ENV_ES_REDIS) \

README.md

@@ -263,6 +263,14 @@ make format
 make lint
 ```
 
+# Upgrade from elasticsearch 7.17.7 to 8.10.x
+
+I you were already on elasticsearch 7.17.7 there is usually not mutch you need to do with elasticsearch itself. I will update itself automatically, once you update to elasticsearch 8 and restart the service.
+
+- If there is a configuration issue creating the new elasticsearch 8 container, make sure you have single or douple quotes around certain values. Like change http.cors.allow-origin=* to http.cors.allow-origin="*".
+- Add port to all hosts in the control panel hosts section. If there is no port defined we add port 9200 by default if you are running elasticsearch 8.
+- Please check the elasticsearch upgrade guide https://www.elastic.co/guide/en/elastic-stack/8.10/upgrading-elastic-stack.html#prepare-to-upgrade
+
 ## License
 
 The project is licensed under the GPLv2.

docker-compose.dev.yaml

@@ -20,9 +20,10 @@ services:
       - discovery.type=single-node
       - cluster.name=docker-cluster
       - http.cors.enabled=true
-      - http.cors.allow-origin=*
+      - http.cors.allow-origin="*"
       - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization
       - http.cors.allow-credentials=true
+      - xpack.security.enabled=false
       - ES_JAVA_OPTS=-Xms512m -Xmx512m
     volumes:
       - elasticsearch_data:/usr/share/elasticsearch/data
@@ -36,6 +37,7 @@ services:
       - PLONE_BACKEND=http://plone:8080/Plone
       - PLONE_USERNAME=admin
       - PLONE_PASSWORD=admin
+      - PLONE_ELASTICSEARCH_HOST=host.docker.internal
 
   plone:
     build:

docker/elasticsearch.Dockerfile

@@ -1,3 +1,3 @@
-FROM elasticsearch:7.17.7
+FROM elasticsearch:8.10.2
 
 RUN bin/elasticsearch-plugin install ingest-attachment -b

docker/plone.Dockerfile

@@ -1,8 +1,8 @@
-FROM plone/plone-backend:6.0.0b3
+FROM plone/plone-backend:6.0.7
 
 WORKDIR /app
 
-RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@mle-redis-rq#egg=collective.elasticsearch[redis]
+RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@main#egg=collective.elasticsearch[redis]
 
 ENV PROFILES="collective.elasticsearch:default collective.elasticsearch:docker-dev"
 ENV TYPE="classic"

docker/worker.Dockerfile

@@ -1,7 +1,7 @@
-FROM plone/plone-backend:6.0.0b3
+FROM plone/plone-backend:6.0.7
 
 WORKDIR /app
 
-RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@mle-redis-rq#egg=collective.elasticsearch[redis]
+RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@main#egg=collective.elasticsearch[redis]
 
 CMD /app/bin/rq worker normal low --with-scheduler --url=$PLONE_REDIS_DSN

setup.py

@@ -56,7 +56,7 @@
     python_requires=">=3.7",
     install_requires=[
         "setuptools",
-        "elasticsearch==7.17.7",
+        "elasticsearch>=7.17.7, <=8.10.0",
         "plone.app.registry",
         "plone.api",
         "setuptools",

src/collective/elasticsearch/browser/controlpanel.py

@@ -1,6 +1,7 @@
 from collective.elasticsearch.interfaces import IElasticSettings
 from collective.elasticsearch.manager import ElasticSearchManager
 from collective.elasticsearch.utils import is_redis_available
+from elastic_transport import ConnectionTimeout
 from elasticsearch.exceptions import ConnectionError as conerror
 from plone import api
 from plone.app.registry.browser.controlpanel import ControlPanelFormWrapper
@@ -52,6 +53,7 @@ def connection_status(self):
             except (
                 conerror,
                 ConnectionError,
+                ConnectionTimeout,
                 NewConnectionError,
                 ConnectionRefusedError,
                 AttributeError,
@@ -60,7 +62,10 @@ def connection_status(self):
 
     @property
     def es_info(self):
-        return self.es.info
+        try:
+            return self.es.info
+        except ConnectionTimeout:
+            return None
 
     @property
     def enabled(self):

src/collective/elasticsearch/interfaces.py

@@ -1,3 +1,4 @@
+from collective.elasticsearch.utils import ELASTIC_SEARCH_VERSION
 from dataclasses import dataclass
 from Products.CMFCore.interfaces import IIndexQueueProcessor
 from typing import Dict
@@ -57,7 +58,7 @@ class IElasticSettings(Interface):
 
     hosts = schema.List(
         title="Hosts",
-        default=["127.0.0.1"],
+        default=ELASTIC_SEARCH_VERSION == 8 and ["http://127.0.0.1:9200"] or ["127.0.0.1"],
         unique=True,
         value_type=schema.TextLine(title="Host"),
     )

src/collective/elasticsearch/manager.py

@@ -4,11 +4,11 @@
 from collective.elasticsearch import utils
 from collective.elasticsearch.result import BrainFactory
 from collective.elasticsearch.result import ElasticResult
+from collective.elasticsearch.utils import ELASTIC_SEARCH_VERSION
 from collective.elasticsearch.utils import use_redis
 from DateTime import DateTime
 from elasticsearch import Elasticsearch
 from elasticsearch import exceptions
-from elasticsearch.exceptions import NotFoundError
 from plone import api
 from Products.CMFCore.indexing import processQueue
 from Products.CMFCore.permissions import AccessInactivePortalContent
@@ -27,6 +27,13 @@
 INDEX_VERSION_ATTR = "_elasticindexversion"
 
 
+try:
+    from elasticsearch.exceptions import BadRequestError
+except ImportError:
+    # Backwards compatibility with ES 7
+    BadRequestError = Exception
+
+
 @implementer(interfaces.IElasticSearchManager)
 class ElasticSearchManager:
 
@@ -155,7 +162,7 @@ def info(self) -> list:
                     ("Elastic Search Version", cluster_version),
                     ("Number of docs (Catalog)", catalog_docs),
                 ]
-        except NotFoundError:
+        except exceptions.NotFoundError:
             logger.warning("Error getting stats", exc_info=True)
             return []
 
@@ -194,14 +201,16 @@ def _recreate_catalog(self):
             conn.indices.delete(index=self.real_index_name)
         except exceptions.NotFoundError:
             pass
-        except exceptions.TransportError as exc:
+        except (BadRequestError, exceptions.TransportError) as exc:
             if exc.error != "illegal_argument_exception":
                 raise
             conn.indices.delete_alias(index="_all", name=self.real_index_name)
 
         if self.index_version:
             try:
-                conn.indices.delete_alias(self.index_name, self.real_index_name)
+                conn.indices.delete_alias(
+                    index=self.index_name, name=self.real_index_name
+                )
             except exceptions.NotFoundError:
                 pass
         self.flush_indices()
@@ -231,7 +240,8 @@ def _setup_attachment_pipeline_and_default_index(self):
         actual binary data is available to extract.
         """
 
-        if "attachment" not in self.connection.cat.plugins():
+        if (ELASTIC_SEARCH_VERSION == 7 and
+                "attachment" not in self.connection.cat.plugins()):
             return
 
         body = {
@@ -245,7 +255,6 @@ def _setup_attachment_pipeline_and_default_index(self):
                             "attachment": {
                                 "target_field": "_ingest._value.attachment",
                                 "field": "_ingest._value.data",
-                                # "remove_binary": "true",  # version 8
                                 "properties": ["content"],
                             }
                         },
@@ -260,7 +269,9 @@ def _setup_attachment_pipeline_and_default_index(self):
                                     if (ctx['attachments'][i]['attachment']['content'].length() > 0) {
                                         ctx['SearchableText'] += ctx['attachments'][i]['attachment']['content'];
                                     }
-                                    ctx['attachments'][i]['data'] = null;
+                                    if (ctx['attachments'][i].containsKey('data')) {
+                                        ctx['attachments'][i]['data'] = null;
+                                    }                                   
                                 }
                             }
                             """,
@@ -278,7 +289,11 @@ def _setup_attachment_pipeline_and_default_index(self):
             #     }
             # ]
         }
-        self.connection.ingest.put_pipeline("cbor-attachments", body=body)
+
+        if ELASTIC_SEARCH_VERSION == 8:
+            body['processors'][0]['foreach']['processor']['attachment']['remove_binary'] = True
+
+        self.connection.ingest.put_pipeline(id="cbor-attachments", body=body)
 
         settings = {"index": {"default_pipeline": "cbor-attachments"}}
         self.connection.indices.put_settings(body=settings, index=self.index_name)