[CAI-741][CAI-762] Chatbot-index creates new indexes on Redis

.changeset/eager-colts-smile.md

@@ -0,0 +1,5 @@
+---
+"chatbot-index": minor
+---
+
+Create index now also considers structured data

.github/actions/chatbot/action.yaml

@@ -3,6 +3,10 @@ name: chatbot
 description: Create Chatbot LLamaindex Vector Index
 
 inputs:
+  api:
+    description: 'Whether to create vector index using api data'
+    required: true
+    default: 'true'
   aws_region:
     description: 'The AWS region selected'
     required: false
@@ -11,6 +15,22 @@ inputs:
     description: the name of the chatbot lambda function
     required: true
     default: ''
+  dynamic:
+    description: 'Whether to create vector index using dynamic data'
+    required: true
+    default: 'true'
+  index_id:
+    description: 'The indexID to create'
+    required: true
+    default: 'devportal-index'
+  static:
+    description: 'Whether to create vector index using static data'
+    required: true
+    default: 'true'
+  structured:
+    description: 'Whether to create vector index using structured data'
+    required: true
+    default: 'false'
 
 runs:
   using: "composite"
@@ -42,4 +62,5 @@ runs:
       shell: bash
       run: |
         cd apps/chatbot-index
-        PYTHONPATH=. poetry run python src/modules/create_vector_index.py
+        export CHB_INDEX_ID=${{ inputs.index_id }}
+        PYTHONPATH=. poetry run python src/modules/create_vector_index.py ${{ inputs.static == 'true' && ' --static' || '' }} ${{ inputs.dynamic == 'true' && ' --dynamic' || '' }} ${{ inputs.api == 'true' && ' --api' || '' }} ${{ inputs.structured == 'true' && ' --structured' || '' }} --clean-redis

.github/workflows/chatbot_create_index.yaml

@@ -14,6 +14,34 @@ on:
           - dev
           - uat
           - prod
+      api:
+        description: 'Consider API documentation for index creation'
+        type: boolean
+        required: true
+        default: true
+      dynamic:
+        description: 'Consider dynamic documentation for index creation'
+        type: boolean
+        required: true
+        default: true
+      static:
+        description: 'Consider static documentation for index creation'
+        type: boolean
+        required: true
+        default: true
+      structured:
+        description: 'Consider structured documentation for index creation'
+        type: boolean
+        required: true
+        default: false
+      index_id:
+        type: choice
+        description: 'The indexID to create'
+        required: true
+        default: devportal-index
+        options:
+          - devportal-index
+          - cittadino-index
 
   # Allows external webhook trigger
   repository_dispatch:
@@ -81,4 +109,9 @@ jobs:
       - name: Chatbot Vector Index
         uses: ./.github/actions/chatbot
         with:
+          api: ${{ github.event.inputs.api && 'true' || 'false'}}
           chatbot_lambda_name: ${{ vars.CHATBOT_LAMBDA_NAME }}
+          dynamic: ${{ github.event.inputs.dynamic && 'true' || 'false' }}
+          index_id: ${{ github.event.inputs.index_id }}
+          static: ${{ github.event.inputs.static && 'true' || 'false' }}
+          structured: ${{ github.event.inputs.structured && 'true' || 'false' }}

apps/chatbot-index/config/params.yaml

@@ -1,4 +1,3 @@
 vector_index:
-  index_id: discovery-index
   chunk_size: 1024
   chunk_overlap: 20

apps/chatbot-index/docker/create_index.Dockerfile

@@ -1,7 +1,8 @@
 FROM python:3.12.4-slim-bullseye
 ARG DEBIAN_FRONTEND=noninteractive
 
-ENV PYTHONPATH=/app
+ENV LAMBDA_TASK_ROOT=/app
+ENV PYTHONPATH=$LAMBDA_TASK_ROOT
 
 RUN apt-get update && \
   apt-get install -y \
@@ -12,6 +13,10 @@ RUN apt-get update && \
   unzip \
   jq
 
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+  unzip awscliv2.zip && \
+  ./aws/install
+
 COPY ./docker/chrome-installer.sh ./chrome-installer.sh
 RUN chmod +x ./chrome-installer.sh && \
   ./chrome-installer.sh && \
@@ -25,3 +30,8 @@ COPY pyproject.toml $LAMBDA_TASK_ROOT
 COPY poetry.lock $LAMBDA_TASK_ROOT
 RUN poetry config virtualenvs.create false
 RUN poetry install --only main
+
+COPY ./src ${LAMBDA_TASK_ROOT}/src
+COPY ./config ${LAMBDA_TASK_ROOT}/config
+COPY ./scripts ${LAMBDA_TASK_ROOT}/scripts
+COPY ./docker/files/s3 ${LAMBDA_TASK_ROOT}/files/s3