blop

Author: fulmicoton-dd

.github/workflows/publish_lambda.yaml

@@ -0,0 +1,70 @@
+name: Build Lambda binary
+
+on:
+  push:
+    branches:
+      - lambda # This is temporary
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version tag (e.g., v0.8.0)'
+        required: false
+        default: 'dev'
+
+permissions:
+  contents: read
+
+jobs:
+  build-lambda:
+    name: Build Lambda ARM64
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      actions: write
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+      - name: Set version
+        run: |
+          if [ "${{ github.event.inputs.version }}" = "dev" ]; then
+            echo "ASSET_VERSION=dev-$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          else
+            echo "ASSET_VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+          fi
+
+      - name: Install rustup
+        run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y
+
+      - name: Install cross
+        run: cargo install cross
+
+      - name: Retrieve and export commit date, hash, and tags
+        run: |
+          echo "QW_COMMIT_DATE=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV
+          echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV
+          echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV
+
+      - name: Build Lambda binary
+        run: cross build --release --features lambda-release --target aarch64-unknown-linux-gnu -p quickwit-lambda --bin quickwit-aws-lambda-leaf-search
+        env:
+          QW_COMMIT_DATE: ${{ env.QW_COMMIT_DATE }}
+          QW_COMMIT_HASH: ${{ env.QW_COMMIT_HASH }}
+          QW_COMMIT_TAGS: ${{ env.QW_COMMIT_TAGS }}
+        working-directory: ./quickwit
+
+      - name: Create Lambda zip
+        run: |
+          cd quickwit/target/aarch64-unknown-linux-gnu/release
+          cp quickwit-aws-lambda-leaf-search bootstrap
+          zip quickwit-aws-lambda-${{ env.ASSET_VERSION }}-aarch64.zip bootstrap
+          mv quickwit-aws-lambda-${{ env.ASSET_VERSION }}-aarch64.zip ../../../../
+
+      - name: Upload to GitHub release
+        uses: quickwit-inc/upload-to-github-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          file: quickwit-aws-lambda-${{ env.ASSET_VERSION }}-aarch64.zip
+          overwrite: true
+          draft: true
+          tag_name: ${{ env.ASSET_VERSION }}

LICENSE-3rdparty.csv

@@ -21,13 +21,16 @@ arrayvec,https://github.com/bluss/arrayvec,MIT OR Apache-2.0,bluss
 assert-json-diff,https://github.com/davidpdrsn/assert-json-diff,MIT,David Pedersen <david.pdrsn@gmail.com>
 async-compression,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman <wim@nemo157.com>, Allen Bui <fairingrey@gmail.com>"
 async-speed-limit,https://github.com/tikv/async-speed-limit,MIT OR Apache-2.0,The TiKV Project Developers
+async-stream,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche <me@carllerche.com>
+async-stream-impl,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche <me@carllerche.com>
 async-trait,https://github.com/dtolnay/async-trait,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 atomic-waker,https://github.com/smol-rs/atomic-waker,Apache-2.0 OR MIT,"Stjepan Glavina <stjepang@gmail.com>, Contributors to futures-rs"
 aws-config,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team <aws-sdk-rust@amazon.com>, Russell Cohen <rcoh@amazon.com>"
 aws-credential-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team <aws-sdk-rust@amazon.com>
 aws-lc-rs,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC),AWS-LibCrypto
 aws-lc-sys,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC) AND OpenSSL,AWS-LC
 aws-runtime,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team <aws-sdk-rust@amazon.com>
+aws-sdk-lambda,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team <aws-sdk-rust@amazon.com>, Russell Cohen <rcoh@amazon.com>"
 aws-sdk-s3,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team <aws-sdk-rust@amazon.com>, Russell Cohen <rcoh@amazon.com>"
 aws-sdk-sso,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team <aws-sdk-rust@amazon.com>, Russell Cohen <rcoh@amazon.com>"
 aws-sdk-ssooidc,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team <aws-sdk-rust@amazon.com>, Russell Cohen <rcoh@amazon.com>"
@@ -235,6 +238,8 @@ jiff-static,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant <
 jobserver,https://github.com/rust-lang/jobserver-rs,MIT OR Apache-2.0,Alex Crichton <alex@alexcrichton.com>
 js-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/js-sys,MIT OR Apache-2.0,The wasm-bindgen Developers
 json_comments,https://github.com/tmccombs/json-comments-rs,Apache-2.0,Thayne McCombs <astrothayne@gmail.com>
+lambda_runtime,https://github.com/awslabs/aws-lambda-rust-runtime,Apache-2.0,"David Calavera <dcalaver@amazon.com>, Harold Sun <sunhua@amazon.com>"
+lambda_runtime_api_client,https://github.com/awslabs/aws-lambda-rust-runtime,Apache-2.0,"David Calavera <dcalaver@amazon.com>, Harold Sun <sunhua@amazon.com>"
 lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel <loebel.marvin@gmail.com>
 levenshtein_automata,https://github.com/tantivy-search/levenshtein-automata,MIT,Paul Masurel <paul.masurel@gmail.com>
 libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers
@@ -424,6 +429,7 @@ serde_core,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar
 serde_derive,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar <erick.tryzelaar@gmail.com>, David Tolnay <dtolnay@gmail.com>"
 serde_json,https://github.com/serde-rs/json,MIT OR Apache-2.0,"Erick Tryzelaar <erick.tryzelaar@gmail.com>, David Tolnay <dtolnay@gmail.com>"
 serde_json_borrow,https://github.com/PSeitz/serde_json_borrow,MIT,Pascal Seitz <pascal.seitz@gmail.com>
+serde_path_to_error,https://github.com/dtolnay/path-to-error,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 serde_qs,https://github.com/samscott89/serde_qs,MIT OR Apache-2.0,Sam Scott <sam@osohq.com>
 serde_spanned,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The serde_spanned Authors
 serde_urlencoded,https://github.com/nox/serde_urlencoded,MIT OR Apache-2.0,Anthony Ramine <n.oxyde@gmail.com>

quickwit/Cargo.lock

@@ -296,6 +296,7 @@ tracing-subscriber = { version = "0.3", features = [
 ttl_cache = "0.5"
 typetag = "0.2"
 ulid = "1.2"
+ureq = "3"
 username = "0.2"
 # We cannot upgrade to utoipa 5.0+ due to significant breaking changes:
 # 1. The `OpenApi` struct structure changed (fields are private), breaking our manual merging logic in openapi.rs

quickwit/Cargo.toml

@@ -33,7 +33,6 @@ pub struct RateLimiterSettings {
     pub refill_period: Duration,
 }
 
-#[cfg(any(test, feature = "testsuite"))]
 impl Default for RateLimiterSettings {
     fn default() -> Self {
         // 10 MB burst limit.

quickwit/quickwit-common/src/rate_limiter.rs

@@ -43,6 +43,7 @@ quickwit-proto = { workspace = true }
 tokio = { workspace = true }
 
 quickwit-proto = { workspace = true, features = ["testsuite"] }
+quickwit-common = { workspace = true, features = ["testsuite"] }
 
 [features]
 testsuite = []

quickwit/quickwit-config/Cargo.toml

@@ -70,6 +70,15 @@
             "min_throughtput_bytes_per_secs": 100000,
             "timeout_millis": 2000,
             "max_num_retries": 2
+        },
+        "lambda": {
+            "function_name": "quickwit-lambda-leaf-search",
+            "max_splits_per_invocation": 10,
+            "auto_deploy": {
+                "execution_role_arn": "arn:aws:iam::123456789012:role/quickwit-lambda-role",
+                "memory_size": "5 GiB",
+                "invocation_timeout_secs": 15
+            }
         }
     },
     "jaeger": {

quickwit/quickwit-config/resources/tests/node_config/quickwit.json

@@ -62,6 +62,15 @@ min_throughtput_bytes_per_secs = 100000
 timeout_millis = 2000
 max_num_retries = 2
 
+[searcher.lambda]
+function_name = "quickwit-lambda-leaf-search"
+max_splits_per_invocation = 10
+
+[searcher.lambda.auto_deploy]
+execution_role_arn = "arn:aws:iam::123456789012:role/quickwit-lambda-role"
+memory_size = "5 GiB"
+invocation_timeout_secs = 15
+
 [jaeger]
 enable_endpoint = true
 lookback_period_hours = 24

quickwit/quickwit-config/resources/tests/node_config/quickwit.toml

@@ -64,6 +64,13 @@ searcher:
     min_throughtput_bytes_per_secs: 100000
     timeout_millis: 2000
     max_num_retries: 2
+  lambda:
+    function_name: quickwit-lambda-leaf-search
+    max_splits_per_invocation: 10
+    auto_deploy:
+      execution_role_arn: arn:aws:iam::123456789012:role/quickwit-lambda-role
+      memory_size: 5 GiB
+      invocation_timeout_secs: 15
 
 jaeger:
   enable_endpoint: true

quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml

@@ -347,7 +347,7 @@ impl LambdaDeployConfig {
         ByteSize::gib(5)
     }
     fn default_invocation_timeout_secs() -> u64 {
-        30
+        15
     }
 }