Merge branch 'sunman' of https://github.com/elastic/elasticsearch-labs …

…into sunman

.github/workflows/docker-chatbot-rag-app.yml

@@ -0,0 +1,101 @@
+name: build chatbot-rag-app image
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - example-apps/chatbot-rag-app/**
+      - .github/workflows/docker-chatbot-rag-app.yml
+      - '!**/*.md'
+  pull_request:
+    branches:
+      - main
+    paths:
+      # Verify changes to the Dockerfile on PRs
+      - example-apps/chatbot-rag-app/Dockerfile
+      - .github/workflows/docker-chatbot-rag-app.yml
+      - '!**/*.md'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  IMAGE: ghcr.io/${{ github.repository }}/chatbot-rag-app
+
+jobs:
+  build-image:
+    strategy:
+      matrix:
+        runner:
+          - ubuntu-24.04
+          - ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        if: github.event_name == 'push'
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/build-push-action@v6
+        id: build
+        with:
+          context: example-apps/chatbot-rag-app
+          outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.event_name == 'push' && 'true' || 'false' }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: export digest
+        if: github.event_name == 'push'
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+      - name: upload digest
+        if: github.event_name == 'push'
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.runner }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  push-manifest:
+    runs-on: ubuntu-24.04
+    needs: build-image
+    if: github.event_name == 'push'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=raw,latest
+            type=sha,format=long
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+            path: /tmp/digests
+            pattern: digests-*
+            merge-multiple: true
+      - run: ls /tmp/digests
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.IMAGE }}@sha256:%s ' *)
+      - name: Inspect image to verify
+        run: |
+          docker buildx imagetools inspect ${{ env.IMAGE }}:${{ steps.meta.outputs.version }}

.github/workflows/tests.yml

@@ -14,11 +14,12 @@ on:
 jobs:
   notebook-tests:
     strategy:
+      fail-fast: false
       matrix:
         es_stack:
-          - 8.13.4
-          - 8.14.0
-          - 8.15.0-SNAPSHOT
+          - 8.16.1
+          - 8.17.0
+          - 8.18.0-SNAPSHOT
     runs-on: ubuntu-latest
     services:
       elasticsearch:

README.md

@@ -29,6 +29,13 @@ The [`notebooks`](notebooks/README.md) folder contains a range of executable Pyt
 - [`question-answering.ipynb`](./notebooks/generative-ai/question-answering.ipynb)
 - [`chatbot.ipynb`](./notebooks/generative-ai/chatbot.ipynb)
 
+### Playground RAG Notebooks
+
+Try out Playground in Kibana with the following notebooks:
+
+- [`OpenAI Example`](./notebooks/playground-examples/openai-elasticsearch-client.ipynb)
+- [`Anthropic Claude 3 Example`](./notebooks/playground-examples/bedrock-anthropic-elasticsearch-client.ipynb)
+
 ### LangChain
 
 - [`question-answering.ipynb`](./notebooks/generative-ai/question-answering.ipynb)
@@ -55,6 +62,14 @@ The [`notebooks`](notebooks/README.md) folder contains a range of executable Pyt
 - [`04-multilingual.ipynb`](./notebooks/search/04-multilingual.ipynb)
 - [`05-query-rules.ipynb`](./notebooks/search/05-query-rules.ipynb)
 - [`06-synonyms-api.ipynb`](./notebooks/search/06-synonyms-api.ipynb)
+- [`07-inference.ipynb`](./notebooks/search/07-inference.ipynb)
+- [`08-learning-to-rank.ipynb`](./notebooks/search/08-learning-to-rank.ipynb)
+- [`09-semantic-text.ipynb`](./notebooks/search/09-semantic-text.ipynb)
+
+#### Semantic reranking
+
+- [`10-semantic-reranking-retriever-cohere.ipynb`](./notebooks/search/10-semantic-reranking-retriever-cohere.ipynb)
+- [`11-semantic-reranking-hugging-face.ipynb`](./notebooks/search/11-semantic-reranking-hugging-face.ipynb)
 
 ### Integrations
 

bin/find-notebooks-to-test.sh

@@ -5,6 +5,9 @@ EXEMPT_NOTEBOOKS=(
     "notebooks/esql/esql-getting-started.ipynb"
     "notebooks/search/07-inference.ipynb"
     "notebooks/search/08-learning-to-rank.ipynb"
+    "notebooks/search/10-semantic-reranking-retriever-cohere.ipynb"
+    "notebooks/search/11-semantic-reranking-hugging-face.ipynb"
+    "notebooks/search/12-semantic-reranking-elastic-rerank.ipynb"
     "notebooks/images/image-similarity.ipynb"
     "notebooks/langchain/langchain-vector-store.ipynb"
     "notebooks/langchain/self-query-retriever-examples/chatbot-example.ipynb"
@@ -26,15 +29,21 @@ EXEMPT_NOTEBOOKS=(
     "notebooks/integrations/llama3/rag-elastic-llama3-elser.ipynb"
     "notebooks/integrations/llama3/rag-elastic-llama3.ipynb"
     "notebooks/integrations/azure-openai/vector-search-azure-openai-elastic.ipynb"
-    "notebooks/enterprise-search/app-search-engine-exporter.ipynb"
+    "notebooks/enterprise-search/app-search-engine-exporter.ipynb",
+    "notebooks/playground-examples/bedrock-anthropic-elasticsearch-client.ipynb",
+    "notebooks/playground-examples/openai-elasticsearch-client.ipynb",
+    "notebooks/integrations/hugging-face/huggingface-integration-millions-of-documents-with-cohere-reranking.ipynb",
+    "notebooks/integrations/cohere/updated-cohere-elasticsearch-inference-api.ipynb",
+    "notebooks/integrations/alibabacloud-ai-search/inference-alibabacloud-ai-search.ipynb",
+    "notebooks/integrations/jinaai/inference-jinaai.ipynb"
 )
 
 # Per-version testing exceptions
 # use variables named EXEMPT_NOTEBOOKS__{major}_[minor} to list notebooks that
 # cannot run on that stack version or older
 # Examples:
 # EXEMPT_NOTEBOOKS__8 for notebooks that must be skipped on all versions 8.x and older
-# EXEMPT_NOTEBOOKS__8_12 for notebooks that must skipped on versions 8.12 and older
+# EXEMPT_NOTEBOOKS__8_12 for notebooks that must be skipped on versions 8.12 and older
 
 EXEMPT_NOTEBOOKS__8_12=(
     # Add any notebooks that must be skipped on versions 8.12 or older here
@@ -44,6 +53,14 @@ EXEMPT_NOTEBOOKS__8_12=(
     "notebooks/langchain/langchain-using-own-model.ipynb"
 )
 
+EXEMPT_NOTEBOOKS__8_14=(
+    # Add any notebooks that must be skipped on versions 8.14 or older here
+    "notebooks/search/09-semantic-text.ipynb",
+    # This notebook has the text_expansion deprecation notice for 8.15. 
+    # Only running on 8.15 so includes the deprecation notice and newer so the local output is the same as CI
+    "notebooks/langchain/langchain-vector-store-using-elser.ipynb",
+)
+
 # this function parses a version given as M[.N[.P]] or M[_N[_P]] into a numeric form
 function parse_version { echo "$@" | awk -F'[._]' '{ printf("%02d%02d\n", $1, $2); }'; }
 

docker/README.md

@@ -0,0 +1,26 @@
+# Running your own Elastic Stack with Docker
+
+If you'd like to start Elastic locally, you can use the provided
+[docker-compose-elastic.yml](docker-compose-elastic.yml) file. This starts
+Elasticsearch, Kibana, and APM Server and only requires Docker installed.
+
+Note: If you haven't checked out this repository, all you need is one file:
+```bash
+wget https://raw.githubusercontent.com/elastic/elasticsearch-labs/refs/heads/main/docker/docker-compose-elastic.yml
+```
+
+Use docker compose to run Elastic stack in the background:
+
+```bash
+docker compose -f docker-compose-elastic.yml up --force-recreate -d
+```
+
+Then, you can view Kibana at http://localhost:5601/app/home#/
+
+If asked for a username and password, use username: elastic and password: elastic.
+
+Clean up when finished, like this:
+
+```bash
+docker compose -f docker-compose-elastic.yml down
+```

docker/docker-compose-elastic.yml

@@ -0,0 +1,91 @@
+name: elastic-stack
+
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0
+    container_name: elasticsearch
+    ports:
+      - 9200:9200
+    environment:
+      - node.name=elasticsearch
+      - cluster.name=docker-cluster
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=elastic
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=false
+      - xpack.security.transport.ssl.enabled=false
+      - xpack.license.self_generated.type=trial
+      - ES_JAVA_OPTS=-Xmx8g
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=500ms"]
+      retries: 300
+      interval: 1s
+
+  elasticsearch_settings:
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0
+    container_name: elasticsearch_settings
+    restart: 'no'
+    command: >
+      bash -c '        
+        # gen-ai assistants in kibana save state in a way that requires security to be enabled, so we need to create
+        # a kibana system user before starting it.
+        echo "Setup the kibana_system password";
+        until curl -s -u "elastic:elastic" -X POST http://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"elastic\"}" -H "Content-Type: application/json" | grep -q "^{}"; do sleep 5; done;
+      '
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.17.0
+    container_name: kibana
+    depends_on:
+      elasticsearch_settings:
+        condition: service_completed_successfully
+    ports:
+      - 5601:5601
+    environment:
+      - SERVERNAME=kibana
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD=elastic
+      # Non-default settings from here:
+      # https://github.com/elastic/apm-server/blob/main/testing/docker/kibana/kibana.yml
+      - MONITORING_UI_CONTAINER_ELASTICSEARCH_ENABLED=true
+      - XPACK_SECURITY_ENCRYPTIONKEY=fhjskloppd678ehkdfdlliverpoolfcr
+      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=fhjskloppd678ehkdfdlliverpoolfcr
+      - SERVER_PUBLICBASEURL=http://127.0.0.1:5601
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status | grep -q 'All services are available'"]
+      retries: 300
+      interval: 1s
+
+  apm-server:
+    image: docker.elastic.co/apm/apm-server:8.17.0
+    container_name: apm-server
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+    command: >
+      apm-server
+        -E apm-server.kibana.enabled=true
+        -E apm-server.kibana.host=http://kibana:5601
+        -E apm-server.kibana.username=elastic
+        -E apm-server.kibana.password=elastic
+        -E output.elasticsearch.hosts=["http://elasticsearch:9200"]
+        -E output.elasticsearch.username=elastic
+        -E output.elasticsearch.password=elastic
+    cap_add: ["CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID"]
+    cap_drop: ["ALL"]
+    ports:
+      - 8200:8200
+    healthcheck:
+      test: ["CMD-SHELL", "bash -c 'echo -n > /dev/tcp/127.0.0.1/8200'"]
+      retries: 300
+      interval: 1s
+

example-apps/chatbot-rag-app/.dockerignore

@@ -0,0 +1,2 @@
+.venv
+*/node_modules

example-apps/chatbot-rag-app/.gitignore

@@ -1,7 +1,7 @@
 frontend/build
 frontend/node_modules
 api/__pycache__
-api/.env
 .venv
 venv
-.DS_Store
+.DS_Store
+.env

example-apps/chatbot-rag-app/Dockerfile

@@ -1,18 +1,15 @@
-# app/Dockerfile
-
-FROM node:16-alpine as build-step
+FROM node:22-alpine AS build-step
 WORKDIR /app
-ENV PATH /node_modules/.bin:$PATH
+ENV PATH=/node_modules/.bin:$PATH
 COPY frontend ./frontend
-RUN rm -rf /app/frontend/node_modules
 RUN cd frontend && yarn install
 RUN cd frontend && REACT_APP_API_HOST=/api yarn build
 
-FROM python:3.9-slim
+FROM python:3.12-slim
 
 WORKDIR /app
 RUN mkdir -p ./frontend/build
-COPY --from=build-step ./app/frontend/build ./frontend/build 
+COPY --from=build-step ./app/frontend/build ./frontend/build
 RUN mkdir ./api
 RUN mkdir ./data
 
@@ -24,12 +21,26 @@ RUN apt-get update && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 
-COPY api ./api
-COPY data ./data
 COPY requirements.txt ./requirements.txt
 RUN pip3 install -r ./requirements.txt
-ENV FLASK_ENV production
+COPY api ./api
+COPY data ./data
 
 EXPOSE 4000
-WORKDIR /app/api
-CMD [ "python3", "-m" , "flask", "run", "--host=0.0.0.0", "--port=4000" ]
+
+# Default to disabling instrumentation, can be overridden to false in
+# docker invocations to reenable.
+ENV OTEL_SDK_DISABLED=true
+
+# https://github.com/elastic/genai-instrumentation/issues/255
+# Currently Python SDK has a bug that spams logs when opentelemetry-instrument is used
+# with SDK being disabled. Until it is fixed, we handle it in our own entrypoint by
+# avoiding opentelemetry-instrument when SDK is disabled.
+RUN echo 'if [ "${OTEL_SDK_DISABLED:-true}" == "false" ]; \
+  then \
+    opentelemetry-instrument $@; \
+  else \
+    exec $@; \
+  fi' > entrypoint.sh
+ENTRYPOINT [ "bash", "-eu", "./entrypoint.sh" ]
+CMD [ "python", "api/app.py"]