Merge branch 'dev' into feat_benchmarks

Author: belveryin

Cargo.toml

@@ -50,16 +50,16 @@ arrow-schema = { version = "56.2.0", features = ["serde"] }
 arrow-select = "56.2.0"
 crc32c = "0.6"
 crossbeam-skiplist = "0.1"
-fusio = { version = "0.5.0-a0", default-features = false, features = [
+fusio = { version = "0.5.0", default-features = false, features = [
     "aws",
     "dyn",
     "executor",
     "fs",
 ] }
-fusio-manifest = { version = "0.5.0-a0", package = "fusio-manifest", default-features = false, features = [
+fusio-manifest = { version = "0.5.0", package = "fusio-manifest", default-features = false, features = [
     "std",
 ] }
-fusio-parquet = { version = "0.5.0-a0", package = "fusio-parquet" }
+fusio-parquet = { version = "0.5.0", package = "fusio-parquet" }
 futures = "0.3"
 lockable = "0.2"
 once_cell = "1"

docs/overview.md

@@ -650,6 +650,10 @@ Tonbo runs as a **manifest-orchestrated** system over **object storage**. The **
 - **Read path:** Resolve manifest snapshot -> parallel, columnar scans with pushdown -> optional local caching.
 - **Compaction path (background):** Select L/L+1 SSTs -> merge -> upload new SSTs -> **CAS manifest update** -> GC old objects.
 
+**Compaction defaults**
+- **Minor compaction:** enabled by default; flushes once ~4 sealed immutables accumulate, emitting L0 SSTs. Opt-out via `DbBuilder::disable_minor_compaction` is intended for tests/bulk-load tooling only.
+- **Major compaction:** not scheduled automatically yet; invoke the admin trigger or an opt-in loop when available. The planner/executor path is wired, but background scheduling is still landing.
+
 ### Why this fits Edge Compute & Shared Storage
 
 - **Immutable by design:** write-new-only + **conditional commits** match object storage semantics—no in-place mutation.

examples/cloudflare-worker/.gitignore

@@ -0,0 +1,12 @@
+# Build artifacts
+/target/
+/build/
+
+# Wrangler
+.wrangler/
+
+# Local development secrets (never commit real credentials!)
+.dev.vars
+
+# Rust lock file (optional, can be committed for reproducibility)
+# Cargo.lock

examples/cloudflare-worker/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "tonbo-cloudflare-worker"
+version = "0.1.0"
+edition = "2024"
+publish = false
+
+# Standalone workspace (not part of parent tonbo workspace)
+[workspace]
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+# Tonbo with web (WASM) features
+tonbo = { path = "../..", default-features = false, features = ["web"] }
+
+# Fusio for WebExecutor and AmazonS3 types
+fusio = { version = "0.5.0", default-features = false, features = [
+    "aws",
+    "executor-web",
+] }
+
+# Arrow for RecordBatch creation
+arrow-array = "56.2.0"
+arrow-schema = "56.2.0"
+
+# Cloudflare Workers runtime
+worker = "0.7"
+console_error_panic_hook = "0.1"
+
+# getrandom needs wasm_js feature for WASM
+getrandom = { version = "0.3", features = ["wasm_js"] }
+
+# For async streams
+futures = "0.3"
+
+# Size optimizations - Cloudflare Workers have a 10MB limit
+[profile.release]
+opt-level = "z"       # Optimize for size
+lto = "fat"           # Full link-time optimization
+strip = "symbols"     # Strip debug symbols
+codegen-units = 1     # Better optimization (slower compile)
+panic = "abort"       # Smaller panic handling

examples/cloudflare-worker/README.md

@@ -0,0 +1,239 @@
+# Tonbo on Cloudflare Workers
+
+This example demonstrates running Tonbo as an embedded database on [Cloudflare Workers](https://workers.cloudflare.com/), using either Cloudflare R2 or any S3-compatible storage backend.
+
+## Prerequisites
+
+- [Rust](https://rustup.rs/) (1.90+ recommended)
+- [Node.js](https://nodejs.org/) (for wrangler CLI)
+- [wrangler](https://developers.cloudflare.com/workers/wrangler/install-and-update/) CLI: `npm install -g wrangler`
+- For local testing: [Docker](https://docker.com/) (to run LocalStack)
+
+## Quick Start
+
+### 1. Local Testing with LocalStack
+
+Start LocalStack (S3-compatible local storage):
+
+```bash
+docker run -d --name localstack -p 4566:4566 -e SERVICES=s3 localstack/localstack:3.0.2
+```
+
+Create a test bucket:
+
+```bash
+AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test \
+  aws --endpoint-url=http://localhost:4566 s3api create-bucket \
+  --bucket tonbo-test --region us-east-1
+```
+
+Set CORS (required for browser/WASM access):
+
+```bash
+AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test \
+  aws --endpoint-url=http://localhost:4566 s3api put-bucket-cors \
+  --bucket tonbo-test \
+  --cors-configuration '{"CORSRules":[{"AllowedOrigins":["*"],"AllowedMethods":["GET","PUT","HEAD","DELETE"],"AllowedHeaders":["*"],"ExposeHeaders":["ETag"],"MaxAgeSeconds":300}]}'
+```
+
+Create `.dev.vars` with test credentials:
+
+```bash
+cat > .dev.vars << 'EOF'
+TONBO_S3_ACCESS_KEY=test
+TONBO_S3_SECRET_KEY=test
+EOF
+```
+
+Run the worker locally:
+
+```bash
+npx wrangler dev
+```
+
+Test it:
+
+```bash
+# Write and read back data
+curl -X POST http://localhost:8787/write
+# Output:
+# Wrote 2 rows (alice=100, bob=200) to Tonbo DB.
+# Read back: alice = 100
+# Note: Cloudflare Workers have subrequest limits...
+```
+
+### 2. Deploy to Cloudflare with R2
+
+#### Create R2 bucket via CLI
+
+```bash
+# Login to Cloudflare (opens browser for authentication)
+npx wrangler login
+
+# Create a new R2 bucket
+npx wrangler r2 bucket create your-bucket-name
+
+# Note your account ID from the output, or get it with:
+npx wrangler whoami
+```
+
+#### Create R2 API Token (Dashboard required)
+
+R2 API tokens must be created in the Cloudflare dashboard:
+
+1. Go to [Cloudflare Dashboard](https://dash.cloudflare.com/) > R2 > Overview
+2. Click "Manage R2 API Tokens" in the right sidebar
+3. Click "Create API token"
+4. Configure the token:
+   - **Token name**: e.g., "tonbo-worker"
+   - **Permissions**: Object Read & Write
+   - **Specify bucket(s)**: Select your bucket (e.g., "your-bucket-name")
+5. Click "Create API Token"
+6. **IMPORTANT**: Copy both the Access Key ID and Secret Access Key immediately (the secret is only shown once!)
+
+#### Configure wrangler.toml
+
+Update `wrangler.toml` with your R2 endpoint:
+
+```toml
+[vars]
+TONBO_S3_ENDPOINT = "https://YOUR_ACCOUNT_ID.r2.cloudflarestorage.com"
+TONBO_S3_BUCKET = "your-bucket-name"
+TONBO_S3_REGION = "auto"
+```
+
+Your Account ID can be found in:
+- R2 overview page URL: `dash.cloudflare.com/ACCOUNT_ID/r2/...`
+- Or run: `npx wrangler whoami`
+
+#### Set secrets
+
+```bash
+npx wrangler secret put TONBO_S3_ACCESS_KEY
+# Paste your R2 Access Key ID when prompted
+
+npx wrangler secret put TONBO_S3_SECRET_KEY
+# Paste your R2 Secret Access Key when prompted
+```
+
+#### Deploy
+
+```bash
+npx wrangler deploy
+```
+
+#### Test the deployment
+
+```bash
+# Write data and read it back
+curl -X POST https://your-worker.your-subdomain.workers.dev/write
+
+# Expected output:
+# Wrote 2 rows (alice=100, bob=200) to Tonbo DB.
+# Read back: alice = 100
+# Note: Cloudflare Workers have subrequest limits...
+```
+
+## How It Works
+
+This example uses Tonbo with:
+
+- **WebExecutor**: Fusio's executor for browser/WASM environments
+- **AmazonS3**: S3-compatible filesystem backend (works with R2, LocalStack, MinIO, etc.)
+- **Arrow RecordBatch**: For efficient columnar data storage
+
+The worker demonstrates:
+1. **POST /write** - Opens database, inserts 2 rows, reads one back (in same request)
+2. **GET /read** - Opens database, queries specific keys
+3. **GET /debug** - Lists files in S3/R2 bucket
+
+## Important Limitations
+
+### Cloudflare Workers Subrequest Limit
+
+Cloudflare Workers limit HTTP subrequests per invocation. Tonbo operations (opening database, reading manifests, WAL replay, scanning data) each require HTTP requests to S3/R2.
+
+| Plan | Subrequests | Tonbo Compatibility |
+|------|-------------|---------------------|
+| Free | 50 | Limited - write+read in same request |
+| Paid | 1,000 | Good - separate operations should work |
+
+**On the free tier (50 subrequests):**
+- Write + single read works in one request ✓
+- Multiple reads may exceed the limit ✗
+- Separate read requests may fail if state accumulated ✗
+
+**On the paid tier (1,000 subrequests):**
+- Most Tonbo operations should work fine
+- Separate read requests after writes should work
+- Consider compaction to reduce files over time for best performance
+
+**Workarounds for free tier:**
+- Do write and immediate verification read in the same request
+- Use Cloudflare Durable Objects for more complex operations
+- Upgrade to paid plan for production use
+
+## Size Optimization
+
+Cloudflare Workers have a 10MB size limit. This example includes optimizations in `Cargo.toml`:
+
+```toml
+[profile.release]
+opt-level = "z"       # Optimize for size
+lto = "fat"           # Full link-time optimization
+strip = "symbols"     # Strip debug symbols
+codegen-units = 1     # Better optimization
+panic = "abort"       # Smaller panic handling
+```
+
+Current build size: ~6MB (well under the limit).
+
+## Troubleshooting
+
+### "SignatureDoesNotMatch" errors
+
+This usually means CORS isn't configured correctly, or the S3 endpoint isn't accessible from the worker. Ensure:
+- CORS is configured on the bucket (see setup steps above)
+- The endpoint URL is correct
+- Credentials are set properly
+
+### Build errors with fusio
+
+This example requires a patched version of fusio with WASM fixes. See [fusio PR #257](https://github.com/tonbo-io/fusio/pull/257). The `[patch.crates-io]` section in `Cargo.toml` handles this.
+
+### WAL persistence limitation
+
+Currently, small files (like WAL segments < 5MB) are only persisted to S3 on `close()`, not on `flush()`. This means:
+- Write + read in the **same request** works (data in memory)
+- Write in one request, read in another may not see the data if `close()` wasn't called
+
+This is a known limitation in fusio's S3Writer. A fix is needed to make `flush()` persist small files, but it requires careful handling of the append code path. See the fusio repository for updates.
+
+### "error: executor-web is only supported on wasm32 targets"
+
+This happens when running `cargo check` without the wasm32 target. Use:
+
+```bash
+cargo check --target wasm32-unknown-unknown
+```
+
+Or just use `npx wrangler build` which handles this automatically.
+
+## Project Structure
+
+```
+examples/cloudflare-worker/
+├── Cargo.toml      # Dependencies and build config
+├── wrangler.toml   # Cloudflare Workers configuration
+├── src/
+│   └── lib.rs      # Worker implementation
+├── .dev.vars       # Local development secrets (git-ignored)
+└── .gitignore
+```
+
+## Learn More
+
+- [Tonbo Documentation](https://docs.rs/tonbo)
+- [Cloudflare Workers Documentation](https://developers.cloudflare.com/workers/)
+- [Cloudflare R2 Documentation](https://developers.cloudflare.com/r2/)
+- [Fusio Documentation](https://docs.rs/fusio)