pagopa · uolter · Aug 8, 2025 · Aug 8, 2025 · Aug 8, 2025 · Aug 8, 2025
diff --git a/.changeset/cruel-rocks-kick.md b/.changeset/cruel-rocks-kick.md
@@ -0,0 +1,5 @@
+---
+"chatbot": patch
+---
+
+Code refactor with new gemini models
diff --git a/.changeset/rich-pears-sin.md b/.changeset/rich-pears-sin.md
@@ -0,0 +1,5 @@
+---
+"chatbot-index": patch
+---
+
+Add package.json to the chatbot-index app to allow versioning
diff --git a/.changeset/wild-hoops-wink.md b/.changeset/wild-hoops-wink.md
@@ -0,0 +1,5 @@
+---
+"chatbot": minor
+---
+
+Add function to add, update, and remove documents and the relative nodes from the vector index.
@@ -90,7 +90,7 @@ jobs:
           IMAGE_TAG: ${{ github.sha }}
         with:
             context: apps/chatbot-index
-            file: apps/chatbot-index/docker/lambda.Dockerfile
+            file: apps/chatbot-index/docker/lambda_refresh_index.Dockerfile
             push: true
             tags: ${{ env.ECR_REGISTRY }}/chatbot-index:${{ env.IMAGE_TAG }}
             provenance: false

@@ -90,7 +90,7 @@ jobs:
           IMAGE_TAG: ${{ github.sha }}
         with:
             context: apps/chatbot-index
-            file: apps/chatbot-index/docker/lambda_refresh_index.Dockerfile
+            file: apps/chatbot-index/docker/lambda_create_index.Dockerfile
             push: true
             tags: ${{ env.ECR_REGISTRY }}/chatbot-reindex:${{ env.IMAGE_TAG }}
             provenance: false

@@ -402,7 +402,8 @@ apps/chatbot/index
 apps/chatbot/*.ipynb
 apps/chatbot/results
 apps/chatbot/test.py
-app/chabot/docker/volume
+!apps/chatbot/docker
+apps/chatbot/docker/volume
 apps/chatbot/.env
 apps/chatbot/empty_htmls.json
 apps/chatbot/.google_service_account.json
@@ -413,6 +414,13 @@ apps/chatbot-evaluate/*.ipynb
 apps/chatbot-evaluate/.env
 apps/chatbot-evaluate/.google_service_account.json
 
+# chatbot-index
+apps/chatbot-index/*.ipynb
+!apps/chatbot-index/docker
+apps/chatbot-index/.env
+!apps/chatbot-index/.env.example
+apps/chatbot-index/.google_service_account.json
+
 /results
 
 # terraform

@@ -0,0 +1,21 @@
+AWS_ENDPOINT_URL=http://localstack:4566
+AWS_ACCESS_KEY_ID=dummy
+AWS_DEFAULT_REGION=eu-south-1
+AWS_SECRET_ACCESS_KEY=dummy
+AWS_LAMBDA_FUNCTION_TIMEOUT=900
+CHB_AWS_GOOGLE_API_KEY=...
+CHB_AWS_S3_BUCKET_NAME_STATIC_CONTENT=devportal-d-website-static-content
+CHB_REDIS_URL=redis://redis:6379
+CHB_WEBSITE_URL=https://dev.developer.pagopa.it
+LOG_LEVEL=DEBUG
+PYTHONPATH=/app
+
+CHB_EMBED_BATCH_SIZE=100
+CHB_EMBEDDING_DIM=768
+CHB_EMBED_MODEL_ID=gemini-embedding-001
+CHB_EMBED_RETRIES=30
+CHB_EMBED_RETRY_MIN_SECONDS=1.5
+CHB_MODEL_MAXTOKENS=2048
+CHB_MODEL_ID=gemini-2.5-flash-lite
+CHB_MODEL_TEMPERATURE=0.3
+CHB_PROVIDER=google
@@ -0,0 +1,17 @@
+receivers:
+  telemetryapi:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+exporters:
+  logging:
+    loglevel: debug
+  awsxray:
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [logging, awsxray]
@@ -0,0 +1,4 @@
+vector_index:
+  index_id: discovery-index
+  chunk_size: 1024
+  chunk_overlap: 200
@@ -0,0 +1,39 @@
+FROM public.ecr.aws/lambda/python:3.12
+ARG DEBIAN_FRONTEND=noninteractive
+
+ENV PYTHONPATH=$LAMBDA_TASK_ROOT
+
+RUN dnf install -y wget unzip jq shadow-utils \
+  && curl -sSL https://dl.google.com/linux/linux_signing_key.pub -o /etc/pki/rpm-gpg/RPM-GPG-KEY-google \
+  && echo '[google-chrome]' > /etc/yum.repos.d/google-chrome.repo \
+  && echo 'name=google-chrome - x86_64' >> /etc/yum.repos.d/google-chrome.repo \
+  && echo 'baseurl=https://dl.google.com/linux/chrome/rpm/stable/x86_64' >> /etc/yum.repos.d/google-chrome.repo \
+  && echo 'enabled=1' >> /etc/yum.repos.d/google-chrome.repo \
+  && echo 'gpgcheck=1' >> /etc/yum.repos.d/google-chrome.repo \
+  && echo 'gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-google' >> /etc/yum.repos.d/google-chrome.repo \
+  && dnf install -y google-chrome-stable \
+  \
+  # Install chromedriver using Chrome for Testing API
+  && DRIVER_VERSION=$(curl -s https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions.json \
+  | jq -r ".channels.Stable.version") \
+  && wget -q "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/${DRIVER_VERSION}/linux64/chromedriver-linux64.zip" \
+  && unzip chromedriver-linux64.zip -d /usr/bin/ \
+  && mv /usr/bin/chromedriver-linux64/chromedriver /usr/bin/chromedriver \
+  && rm -rf chromedriver-linux64.zip /usr/bin/chromedriver-linux64 \
+  \
+  && dnf clean all
+
+RUN pip install --upgrade pip \
+  && pip install poetry
+
+WORKDIR $LAMBDA_TASK_ROOT
+COPY pyproject.toml $LAMBDA_TASK_ROOT
+COPY poetry.lock $LAMBDA_TASK_ROOT
+COPY --from=039804388894.dkr.ecr.eu-south-1.amazonaws.com/lambda-extension/otel-collector:v1 /src/collector /opt/extensions/collector
+COPY config/adot-config.yaml /opt/collector-config/config.yaml
+RUN poetry config virtualenvs.create false
+RUN poetry install
+
+COPY ./ ${LAMBDA_TASK_ROOT}/
+
+CMD ["src.lambda_create_index.lambda_handler"]
@@ -0,0 +1,39 @@
+FROM public.ecr.aws/lambda/python:3.12
+ARG DEBIAN_FRONTEND=noninteractive
+
+ENV PYTHONPATH=$LAMBDA_TASK_ROOT
+
+RUN dnf install -y wget unzip jq shadow-utils \
+  && curl -sSL https://dl.google.com/linux/linux_signing_key.pub -o /etc/pki/rpm-gpg/RPM-GPG-KEY-google \
+  && echo '[google-chrome]' > /etc/yum.repos.d/google-chrome.repo \
+  && echo 'name=google-chrome - x86_64' >> /etc/yum.repos.d/google-chrome.repo \
+  && echo 'baseurl=https://dl.google.com/linux/chrome/rpm/stable/x86_64' >> /etc/yum.repos.d/google-chrome.repo \
+  && echo 'enabled=1' >> /etc/yum.repos.d/google-chrome.repo \
+  && echo 'gpgcheck=1' >> /etc/yum.repos.d/google-chrome.repo \
+  && echo 'gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-google' >> /etc/yum.repos.d/google-chrome.repo \
+  && dnf install -y google-chrome-stable \
+  \
+  # Install chromedriver using Chrome for Testing API
+  && DRIVER_VERSION=$(curl -s https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions.json \
+  | jq -r ".channels.Stable.version") \
+  && wget -q "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/${DRIVER_VERSION}/linux64/chromedriver-linux64.zip" \
+  && unzip chromedriver-linux64.zip -d /usr/bin/ \
+  && mv /usr/bin/chromedriver-linux64/chromedriver /usr/bin/chromedriver \
+  && rm -rf chromedriver-linux64.zip /usr/bin/chromedriver-linux64 \
+  \
+  && dnf clean all
+
+RUN pip install --upgrade pip \
+  && pip install poetry
+
+WORKDIR $LAMBDA_TASK_ROOT
+COPY pyproject.toml $LAMBDA_TASK_ROOT
+COPY poetry.lock $LAMBDA_TASK_ROOT
+COPY --from=039804388894.dkr.ecr.eu-south-1.amazonaws.com/lambda-extension/otel-collector:v1 /src/collector /opt/extensions/collector
+COPY config/adot-config.yaml /opt/collector-config/config.yaml
+RUN poetry config virtualenvs.create false
+RUN poetry install
+
+COPY ./ ${LAMBDA_TASK_ROOT}/
+
+CMD ["src.lambda_refresh_index.lambda_handler"]
@@ -0,0 +1,6 @@
+{
+  "name": "chatbot-index",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {}
+}