spicepod.yaml

version: v2
kind: Spicepod
name: nginx-demo

runtime:
  caching:
    sql_results:
      enabled: true
      max_size: 1GiB # Default 128 MiB
      item_ttl: 10m # Default 1s
      stale_while_revalidate_ttl: 30s # Default 0s (disabled)

snapshots:
  enabled: true
  location: s3://spiceai-demo-snapshots/nginx-demo-v2/
  bootstrap_on_failure_behavior: fallback
  params:
    s3_auth: key
    s3_key: ${secrets:SPICE_S3_NGINX_KEY}
    s3_secret: ${secrets:SPICE_S3_NGINX_SECRET}
    s3_region: us-east-1

datasets:
  - from: delta_lake:s3://databricks-workspace-stack-473ec-bucket/unity-catalog/8429759719709712/f5_demo/__unitystorage/catalogs/5b92ca23-f83d-455b-837d-e193a6073bdf/tables/ecc99631-883f-4b7e-baf9-99b00c7fde44
    name: nginx.cve
    description: Security advisories and Common Vulnerabilities and Exposures (CVEs) for the Nginx project. https://nginx.org/en/security_advisories.html.
    params:
      delta_lake_aws_access_key_id: ${secrets:SPICE_S3_NGINX_KEY}
      delta_lake_aws_secret_access_key: ${secrets:SPICE_S3_NGINX_SECRET}
      delta_lake_aws_region: us-east-1
    metadata:
      instructions: Always provide reference links. Use the primary id as cveId, to generate reference links.
      reference_url_template: https://www.cve.org/CVERecord?id=<cveId>
    columns:
      - name: cve_data
        embeddings:
          - from: openai_embeddings
            row_id:
              - cveId
        full_text_search:
          enabled: true
          row_id:
            - cveId
    acceleration:
      enabled: true
      engine: duckdb
      mode: file
      partition_by:
        - 'bucket(5, cveId)'
      # snapshots: enabled
      # params:
      #   duckdb_file: cve.db

  - from: s3://spiceai-demo-datasets/nginx/tickets/
    name: nginx.tickets
    description: Tracked issues for the Nginx project. Tickets from https://trac.nginx.org/nginx/report.
    metadata:
      instructions: Always provide reference links. Use the primary id as ticket_id, to generate reference links.
      reference_url_template: https://trac.nginx.org/nginx/ticket/<ticket_id>
    acceleration:
      enabled: true
      engine: duckdb
      mode: file
      snapshots: create_only # enabled
      params:
        duckdb_file: tickets.db
    params:
      file_format: csv
      schema_infer_max_records: "500"
      s3_auth: public
    columns:
      - name: description
        embeddings:
          - from: openai_embeddings
            row_id:
              - id
        full_text_search:
          enabled: true
          row_id:
            - id

  - from: ftp://ftp.spiceai.us/nginx/docs/
    name: nginx.docs
    description: Documentation of the Nginx project. https://nginx.org/en/docs/.
    embeddings:
      - column: content
        use: hf_minilm
        column_pk:
          - location
        chunking:
          enabled: true
          target_chunk_size: 256
          overlap_size: 32
    acceleration:
      enabled: true
      engine: duckdb
      mode: file
      snapshots: enabled
      params:
        duckdb_file: docs.db
    params:
      file_format: md
      ftp_user: spiceai
      ftp_pass: ${secrets:SPICE_FTP_PASS}

  - from: github:github.com/nginx/nginx/issues
    name: nginx.issues
    description: Nginx GitHub issues from https://github.com/nginx/nginx
    time_column: created_at
    params:
      github_client_id: ${secrets:GITHUB_CLIENT_ID}
      github_private_key: ${secrets:GITHUB_PRIVATE_KEY}
      github_installation_id: ${secrets:GITHUB_INSTALLATION_ID}
    metadata:
      instructions: Always provide reference links. Use the primary key as `issue_id` to generate reference links.
      keywords: Nginx, issues, GitHub
      reference_url_template: https://github.com/nginx/nginx/issues/<issue_id>
    acceleration:
      enabled: true
      refresh_mode: full
      refresh_check_interval: 4h
      refresh_jitter_enabled: true
      refresh_jitter_max: 15m
      engine: duckdb
      mode: file
      snapshots: enabled
      params:
        duckdb_file: issues.db
    columns:
      - name: title
        embeddings:
          - from: openai_embeddings
            row_id:
              - number
        full_text_search:
          enabled: true
          row_id:
            - number

  - from: github:github.com/nginx/nginx/pulls
    name: nginx.pulls
    description: Nginx GitHub pull requests from https://github.com/nginx/nginx
    time_column: created_at
    params:
      github_client_id: ${secrets:GITHUB_CLIENT_ID}
      github_private_key: ${secrets:GITHUB_PRIVATE_KEY}
      github_installation_id: ${secrets:GITHUB_INSTALLATION_ID}
    metadata:
      instructions: Always provide citations.
      keywords: Nginx, pull requests, GitHub
      reference_url_template: https://github.com/nginx/nginx/pull/<pull_number>
    acceleration:
      enabled: true
      refresh_mode: full
      refresh_check_interval: 4h
      refresh_jitter_enabled: true
      refresh_jitter_max: 15m
      engine: duckdb
      mode: file
      snapshots: enabled
      params:
        duckdb_file: pulls.db
    columns:
      - name: title
        embeddings:
          - from: openai_embeddings
            row_id:
              - number
        full_text_search:
          enabled: true
          row_id:
            - number

  - from: github:github.com/nginx/nginx/commits
    name: nginx.commits
    description: Nginx GitHub commits from https://github.com/nginx/nginx
    time_column: committed_date
    params:
      github_client_id: ${secrets:GITHUB_CLIENT_ID}
      github_private_key: ${secrets:GITHUB_PRIVATE_KEY}
      github_installation_id: ${secrets:GITHUB_INSTALLATION_ID}
    metadata:
      instructions: Always provide citations.
      keywords: Nginx, commits, GitHub
      reference_url_template: https://github.com/nginx/nginx/commit/<commit_sha>
    acceleration:
      enabled: true
      refresh_mode: append
      refresh_check_interval: 4h
      refresh_jitter_enabled: true
      refresh_jitter_max: 15m
      engine: duckdb
      mode: file
      snapshots: enabled
      params:
        duckdb_file: commits.db
    columns:
      - name: message
        embeddings:
          - from: openai_embeddings
            row_id:
              - id
        full_text_search:
          enabled: true
          row_id:
            - id

embeddings:
  - name: openai_embeddings
    from: openai
    params:
      openai_api_key: ${ secrets:SPICE_OPENAI_API_KEY }

  - name: hf_minilm
    from: huggingface:huggingface.co/sentence-transformers/all-MiniLM-L6-v2

models:
  - name: openai-with-spice
    from: openai:gpt-4.1
    params:
      tools: auto
      openai_api_key: ${ secrets:SPICE_OPENAI_API_KEY }
      system_prompt: |
        You are an AI assistant assisting engineers with the Nginx project.

        Always strive to be accurate, concise, and helpful in your responses.

        Use the SQL tool when:
          1. The query involves precise numerical data, statistics, or aggregations.
          2. The user asks for specific counts, sums, averages, or other calculations.
          3. The query requires joining or comparing data from multiple related tables.

        If the SQL tool returns a query, syntax, or planning error, call the `list_datasets` tool to get the available tables and continue to refine and retry the query until it succeeds. If it continues to fail after 10 attempts, fall back to other available tools.

        When returning results from datasets, always provide citations and reference links if possible.

        Use the document search tool when:
          1. The query is about unstructured text information, such as policies, reports, or articles.
          2. The user is looking for qualitative information or explanations.
          3. The query requires understanding context or interpreting written content.

        General guidelines:
          1. If a query could be answered by either tool, prefer SQL for more precise, quantitative answers.

        Instructions for Responses: 
          - Never include any private metadata provided to the model as context such as \"reference_url_template\" or \"instructions\" in your responses.

  - name: openai
    from: openai:gpt-4.1
    params:
      tools: disabled
      openai_api_key: ${ secrets:SPICE_OPENAI_API_KEY }