feat: Explicit mode selection, no silent fallback

Extension registers both VFS names:
- "turbolite" — always registered, local compressed VFS
- "turbolite-s3" — registered only when TURBOLITE_BUCKET is set,
  fails hard if config is invalid

Python API: turbolite.connect(path, mode="s3", bucket=...) validates
config before loading. ValueError if mode="s3" with no bucket.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: russellromney

README.md

@@ -40,13 +40,10 @@ pip install turbolite
 ```
 
 ```python
-import sqlite3
 import turbolite
 
-# convenience wrapper for manually loading the extension
+# Local compressed database
 conn = turbolite.connect("my.db")
-
-# execute queries
 conn.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, email TEXT)")
 conn.execute("INSERT INTO users VALUES (1, 'alice', 'alice@example.com')")
 conn.commit()
@@ -56,15 +53,22 @@ print(alice[1])
 >>> "alice"
 ```
 
-You can also manually load the extension:
+```python
+# S3 tiered database — serve cold queries from S3
+conn = turbolite.connect("my.db", mode="s3",
+    bucket="my-bucket",
+    endpoint="https://t3.storage.dev")
+```
 
 ```python
-# Load the extension (registers the "turbolite" VFS process-wide)
+# Or manually load the extension for full control
+import sqlite3
 conn = sqlite3.connect(":memory:")
 turbolite.load(conn)
 conn.close()
 
-conn = sqlite3.connect("file:my.db?vfs=turbolite", uri=True)
+conn = sqlite3.connect("file:my.db?vfs=turbolite", uri=True)       # local
+conn = sqlite3.connect("file:my.db?vfs=turbolite-s3", uri=True)    # S3 (needs TURBOLITE_BUCKET)
 ```
 
 See (installation details below) for Node, Go, Rust, and using the `.so` loadable extension directly
@@ -95,7 +99,7 @@ SQLite: "read page 4,271"
 Manifest: "page 4,271 is in group 16, sub-chunk 3, bytes 81,920-106,496"
     |
     v
-turbolite check local storage: "page not in cache"
+turbolite: check local storage -> cache miss
     |
     v
 S3: GET for ~256KB compressed sub-chunk → decompress → return page
@@ -151,7 +155,9 @@ let config = TieredConfig {
 
 **Encrypt after compress, decrypt before decompress.** Compression operates on plaintext (compressing ciphertext is useless). On the S3 path: `plaintext → zstd compress → GCM encrypt → S3 PUT`. On read: `S3 GET → GCM decrypt → zstd decompress → plaintext`.
 
-**Security model:** S3 data uses authenticated encryption (GCM) with unique nonces per frame - the strongest path for long-lived data. Local files use CTR with deterministic nonces (page number / byte offset), providing confidentiality against single-snapshot disk-at-rest attackers. CTR's deterministic nonces mean that an attacker who can capture multiple snapshots of the same file (e.g., via filesystem snapshots or NVMe wear leveling) could potentially recover XOR of plaintexts at reused offsets. This matches the trade-off made by SQLite's own SEE extension in OFB mode. For most deployments, the local cache is ephemeral and recreatable from S3, so this is acceptable.
+**Security model:** S3 data uses GCM with unique nonces per frame (authenticated, tamper-detecting). Local files use CTR with deterministic nonces (page number / byte offset), providing confidentiality against disk-at-rest attackers. CTR's deterministic nonces mean multi-snapshot attackers could recover XOR of plaintexts at reused offsets, matching SQLite's own SEE extension tradeoff. The local cache is ephemeral and recreatable from S3.
+
+**Key rotation:** `rotate_encryption_key(config, new_key)` re-encrypts, adds, or removes encryption on all S3 data without decompressing. `Some` to `Some` rotates keys, `Some` to `None` removes encryption, `None` to `Some` adds it. Crash-safe: old objects are never overwritten, the manifest upload is the atomic commit point, and a verification step confirms new data is readable before committing. Orphans from partial runs are cleaned by `gc()`.
 
 ## Strengths and Limitations
 

packages/python/turbolite/__init__.py

@@ -1,20 +1,16 @@
 """
-turbolite — compressed SQLite for Python.
+turbolite — compressed SQLite with S3 tiered storage for Python.
 
-Usage::
+Local mode (default)::
 
-    import sqlite3
     import turbolite
+    conn = turbolite.connect("my.db")
 
-    conn = sqlite3.connect(":memory:")
-    turbolite.load(conn)
-
-    # Open a compressed database via URI
-    conn2 = sqlite3.connect("file:my.db?vfs=turbolite", uri=True)
-
-Or use the convenience wrapper::
+S3 tiered mode::
 
-    conn = turbolite.connect("my.db")
+    conn = turbolite.connect("my.db", mode="s3",
+        bucket="my-bucket",
+        endpoint="https://t3.storage.dev")
 """
 
 from __future__ import annotations
@@ -24,7 +20,7 @@
 import sqlite3
 import sys
 
-__version__ = "0.1.0"
+__version__ = "0.2.19"
 
 
 def _find_ext() -> str:
@@ -41,7 +37,6 @@ def _find_ext() -> str:
 
     path = os.path.join(pkg_dir, name)
     if os.path.isfile(path):
-        # Return without extension — sqlite3.load_extension appends it
         return os.path.splitext(path)[0]
 
     raise FileNotFoundError(
@@ -57,10 +52,9 @@ def load(conn: sqlite3.Connection) -> None:
     """
     Load the turbolite extension into a sqlite3 connection.
 
-    After loading, the "turbolite" VFS is registered process-wide.
-    Open compressed databases with::
-
-        sqlite3.connect("file:path.db?vfs=turbolite", uri=True)
+    After loading, the "turbolite" VFS (local compressed) is always
+    registered. If TURBOLITE_BUCKET is set in the environment,
+    "turbolite-s3" (tiered S3) is also registered.
 
     Args:
         conn: Any open sqlite3.Connection (can be :memory:).
@@ -75,28 +69,71 @@ def load(conn: sqlite3.Connection) -> None:
 def connect(
     path: str,
     *,
-    vfs: str = "turbolite",
+    mode: str = "local",
+    bucket: str | None = None,
+    prefix: str | None = None,
+    endpoint: str | None = None,
+    region: str | None = None,
+    cache_dir: str | None = None,
+    compression_level: int | None = None,
+    prefetch_threads: int | None = None,
+    read_only: bool = False,
 ) -> sqlite3.Connection:
     """
-    Convenience: load the extension and open a compressed database.
-
-    Equivalent to::
-
-        conn = sqlite3.connect(":memory:")
-        turbolite.load(conn)
-        conn.close()
-        return sqlite3.connect(f"file:{path}?vfs={vfs}", uri=True)
+    Open a turbolite database.
 
     Args:
         path: Path to the database file.
-        vfs: VFS name (default "turbolite").
+        mode: "local" for compressed VFS, "s3" for S3 tiered VFS.
+        bucket: S3 bucket (required for mode="s3", or set TURBOLITE_BUCKET).
+        prefix: S3 key prefix (default "turbolite").
+        endpoint: S3 endpoint URL (Tigris, MinIO). Falls back to AWS_ENDPOINT_URL.
+        region: AWS region. Falls back to AWS_REGION.
+        cache_dir: Local cache directory (default /tmp/turbolite).
+        compression_level: Zstd level 1-22 (default 3).
+        prefetch_threads: Prefetch worker threads (default num_cpus + 1).
+        read_only: Open in read-only mode.
 
     Returns:
-        An open sqlite3.Connection using the compressed VFS.
+        An open sqlite3.Connection.
+
+    Raises:
+        ValueError: If mode="s3" but no bucket is configured.
+        RuntimeError: If the tiered VFS fails to initialize.
     """
+    if mode not in ("local", "s3"):
+        raise ValueError(f"mode must be 'local' or 's3', got {mode!r}")
+
+    if mode == "s3":
+        # Set env vars for the extension before loading.
+        # Fail fast if bucket is missing.
+        effective_bucket = bucket or os.environ.get("TURBOLITE_BUCKET")
+        if not effective_bucket:
+            raise ValueError(
+                "mode='s3' requires a bucket. Pass bucket= or set TURBOLITE_BUCKET."
+            )
+        os.environ["TURBOLITE_BUCKET"] = effective_bucket
+        if prefix is not None:
+            os.environ["TURBOLITE_PREFIX"] = prefix
+        if endpoint is not None:
+            os.environ["TURBOLITE_ENDPOINT_URL"] = endpoint
+        if region is not None:
+            os.environ["TURBOLITE_REGION"] = region
+        if cache_dir is not None:
+            os.environ["TURBOLITE_CACHE_DIR"] = cache_dir
+        if read_only:
+            os.environ["TURBOLITE_READ_ONLY"] = "true"
+
+    if compression_level is not None:
+        os.environ["TURBOLITE_COMPRESSION_LEVEL"] = str(compression_level)
+    if prefetch_threads is not None:
+        os.environ["TURBOLITE_PREFETCH_THREADS"] = str(prefetch_threads)
+
     global _loaded
     if not _loaded:
         bootstrap = sqlite3.connect(":memory:")
         load(bootstrap)
         bootstrap.close()
+
+    vfs = "turbolite-s3" if mode == "s3" else "turbolite"
     return sqlite3.connect(f"file:{path}?vfs={vfs}", uri=True)

src/ext.rs

@@ -3,20 +3,18 @@
 //! Exports `turbolite_ext_register_vfs()` which is called from the C entry
 //! point in `ext_entry.c` after `SQLITE_EXTENSION_INIT2` stores the API table.
 //!
-//! The C shim provides symbol shims for `sqlite3_vfs_register` etc. that route
-//! through the extension API table, so `sqlite_vfs::register()` works correctly
-//! inside a loadable extension.
+//! ## VFS registration
 //!
-//! ## VFS selection
+//! Always registers **"turbolite"** — local compressed VFS (zstd).
 //!
-//! If `TURBOLITE_BUCKET` is set, registers a **tiered S3 VFS** (requires the
-//! `tiered` feature). Otherwise, registers a **local compressed VFS**.
+//! If `TURBOLITE_BUCKET` is set, also registers **"turbolite-s3"** — tiered
+//! S3 VFS. Fails hard if bucket is set but configuration is invalid.
 //!
 //! ### Environment variables (tiered mode)
 //!
 //! | Variable | Required | Default | Description |
 //! |---|---|---|---|
-//! | `TURBOLITE_BUCKET` | yes | — | S3 bucket name |
+//! | `TURBOLITE_BUCKET` | yes | — | S3 bucket name (triggers S3 VFS registration) |
 //! | `TURBOLITE_PREFIX` | no | `"turbolite"` | S3 key prefix |
 //! | `TURBOLITE_CACHE_DIR` | no | `"/tmp/turbolite"` | Local cache directory |
 //! | `TURBOLITE_ENDPOINT_URL` | no | — | Custom S3 endpoint (Tigris, MinIO) |
@@ -30,29 +28,38 @@
 
 use std::sync::atomic::{AtomicBool, Ordering};
 
-static VFS_REGISTERED: AtomicBool = AtomicBool::new(false);
+static LOCAL_VFS_REGISTERED: AtomicBool = AtomicBool::new(false);
+static TIERED_VFS_REGISTERED: AtomicBool = AtomicBool::new(false);
 
 /// Called from C entry point (`sqlite3_turbolite_init` in ext_entry.c).
 /// Returns 0 on success, 1 on error. Idempotent: second call is a no-op.
+///
+/// Always registers "turbolite" (local compressed VFS).
+/// If TURBOLITE_BUCKET is set, also registers "turbolite-s3" (tiered VFS).
+/// Panics if TURBOLITE_BUCKET is set but tiered VFS creation fails.
 #[no_mangle]
 pub extern "C" fn turbolite_ext_register_vfs() -> std::os::raw::c_int {
-    if VFS_REGISTERED.swap(true, Ordering::SeqCst) {
-        return 0;
+    // Register local VFS (always)
+    if !LOCAL_VFS_REGISTERED.swap(true, Ordering::SeqCst) {
+        if let Err(e) = register_local() {
+            LOCAL_VFS_REGISTERED.store(false, Ordering::SeqCst);
+            eprintln!("turbolite: failed to register local VFS: {e}");
+            return 1;
+        }
     }
 
-    let result = if std::env::var("TURBOLITE_BUCKET").is_ok() {
-        register_tiered()
-    } else {
-        register_local()
-    };
-
-    match result {
-        Ok(()) => 0,
-        Err(_) => {
-            VFS_REGISTERED.store(false, Ordering::SeqCst);
-            1
+    // Register tiered VFS if TURBOLITE_BUCKET is set
+    if std::env::var("TURBOLITE_BUCKET").is_ok()
+        && !TIERED_VFS_REGISTERED.swap(true, Ordering::SeqCst)
+    {
+        if let Err(e) = register_tiered() {
+            TIERED_VFS_REGISTERED.store(false, Ordering::SeqCst);
+            eprintln!("turbolite: TURBOLITE_BUCKET is set but tiered VFS failed: {e}");
+            return 1;
         }
     }
+
+    0
 }
 
 fn register_local() -> Result<(), std::io::Error> {
@@ -86,7 +93,7 @@ fn register_tiered() -> Result<(), std::io::Error> {
     let prefetch_threads = std::env::var("TURBOLITE_PREFETCH_THREADS")
         .ok()
         .and_then(|s| s.parse().ok())
-        .unwrap_or(0); // 0 = use default (num_cpus + 1)
+        .unwrap_or(0);
     let compression_level = std::env::var("TURBOLITE_COMPRESSION_LEVEL")
         .ok()
         .and_then(|s| s.parse().ok())
@@ -110,13 +117,14 @@ fn register_tiered() -> Result<(), std::io::Error> {
     }
 
     let vfs = TieredVfs::new(config)?;
-    crate::tiered::register("turbolite", vfs)
+    crate::tiered::register("turbolite-s3", vfs)
 }
 
 #[cfg(not(feature = "tiered"))]
 fn register_tiered() -> Result<(), std::io::Error> {
     Err(std::io::Error::new(
         std::io::ErrorKind::Unsupported,
-        "TURBOLITE_BUCKET is set but this extension was built without the 'tiered' feature",
+        "TURBOLITE_BUCKET is set but this extension was built without the 'tiered' feature. \
+         Rebuild with: make ext  (includes tiered by default)",
     ))
 }