fix: improve packaging of the schema and versioning on rust

Author: jorgedanisc

packages/ducpy/.gitignore

@@ -1,8 +1,6 @@
 # Test files
 dist/
 inputs/
-schema/*
-!schema/.gitkeep
 
 
 

packages/ducpy/crate/src/lib.rs

@@ -58,6 +58,45 @@ fn list_external_files(py: Python<'_>, buf: &[u8]) -> PyResult<PyObject> {
         .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))
 }
 
+/// Returns the current DUC schema version as a semver string (e.g. "3.0.0").
+#[pyfunction]
+fn get_schema_version() -> PyResult<String> {
+    Ok(duc::db::bootstrap::CURRENT_SCHEMA_VERSION_SEMVER.into())
+}
+
+/// Returns the raw integer schema version (e.g. 3000000).
+#[pyfunction]
+fn get_schema_version_int() -> PyResult<i64> {
+    Ok(duc::db::bootstrap::current_schema_version_int())
+}
+
+/// Returns the canonical `duc.sql` schema string.
+#[pyfunction]
+fn get_duc_schema_sql() -> PyResult<String> {
+    Ok(duc::db::bootstrap::DUC_SCHEMA_SQL.into())
+}
+
+/// Returns the `version_control.sql` schema string.
+#[pyfunction]
+fn get_version_control_schema_sql() -> PyResult<String> {
+    Ok(duc::db::bootstrap::VERSION_CONTROL_SCHEMA_SQL.into())
+}
+
+/// Returns the `search.sql` schema string.
+#[pyfunction]
+fn get_search_schema_sql() -> PyResult<String> {
+    Ok(duc::db::bootstrap::SEARCH_SCHEMA_SQL.into())
+}
+
+/// Returns all migrations as a list of `(from_version, to_version, sql)` tuples.
+#[pyfunction]
+fn get_migrations(_py: Python<'_>) -> PyResult<Vec<(i64, i64, String)>> {
+    Ok(duc::db::bootstrap::MIGRATIONS
+        .iter()
+        .map(|(f, t, sql)| (*f, *t, sql.to_string()))
+        .collect())
+}
+
 /// Native duc file format operations.
 #[pymodule]
 fn ducpy_native(m: &Bound<'_, PyModule>) -> PyResult<()> {
@@ -66,5 +105,11 @@ fn ducpy_native(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(serialize_duc, m)?)?;
     m.add_function(wrap_pyfunction!(get_external_file, m)?)?;
     m.add_function(wrap_pyfunction!(list_external_files, m)?)?;
+    m.add_function(wrap_pyfunction!(get_schema_version, m)?)?;
+    m.add_function(wrap_pyfunction!(get_schema_version_int, m)?)?;
+    m.add_function(wrap_pyfunction!(get_duc_schema_sql, m)?)?;
+    m.add_function(wrap_pyfunction!(get_version_control_schema_sql, m)?)?;
+    m.add_function(wrap_pyfunction!(get_search_schema_sql, m)?)?;
+    m.add_function(wrap_pyfunction!(get_migrations, m)?)?;
     Ok(())
 }

packages/ducpy/package.json

@@ -3,14 +3,13 @@
   "version": "0.0.0-development",
   "private": true,
   "scripts": {
-    "sync:schema": "uv run python scripts/sync_schema.py",
-    "build": "bash -c 'uv run python scripts/sync_schema.py && SETUPTOOLS_SCM_PRETEND_VERSION=${1} uv build' --",
+    "build": "bash -c 'SETUPTOOLS_SCM_PRETEND_VERSION=${1} uv build' --",
     "gen:docs": "bun clean:docs && cd docs && uv run make html",
     "dev:docs": "uv run --with sphinx-autobuild sphinx-autobuild docs docs/_build/html --port 8240 --open-browser",
     "clean:docs": "cd docs && uv run make clean",
     "test": "uv run -m pytest src/tests/src/test_*.py",
     "test:verbose": "uv run -m pytest -v src/tests/src/test_*.py",
-    "prerelease": "uv sync && uv run python scripts/sync_schema.py && bun run test",
+    "prerelease": "uv sync && bun run test",
     "semantic-release": "semantic-release"
   },
   "devDependencies": {

packages/ducpy/pyproject.toml

@@ -36,8 +36,6 @@ python-packages = ["ducpy", "ducpy_native"]
 module-name = "ducpy_native"
 include = [
     { path = "LICENSE", format = "sdist" },
-    { path = "schema/**/*", format = "sdist" },
-    { path = "schema/**/*", format = "wheel" }
 ]
 
 [tool.setuptools]

packages/ducpy/release.config.cjs

@@ -16,7 +16,7 @@ module.exports = {
       "@semantic-release/exec",
       {
         prepareCmd:
-          "uv sync && uv run python scripts/sync_schema.py && sed -i 's/^version = \".*\"$/version = \"${nextRelease.version}\"/' crate/Cargo.toml && SETUPTOOLS_SCM_PRETEND_VERSION=${nextRelease.version} uv build --sdist",
+          "uv sync && sed -i 's/^version = \".*\"$/version = \"${nextRelease.version}\"/' crate/Cargo.toml && SETUPTOOLS_SCM_PRETEND_VERSION=${nextRelease.version} uv build --sdist",
         publishCmd: "uv publish --token ${process.env.PYPI_TOKEN} dist/*.tar.gz",
       },
     ],

packages/ducpy/scripts/sync_schema.py

@@ -1,79 +1,8 @@
-import os
-import re
-from pathlib import Path
-
 from setuptools import setup
 
-# Path where _version.py will be written
-VERSION_PY_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'src', 'ducpy', '_version.py')
-
-
-def find_schema_file_path():
-    env_path = os.environ.get('DUC_SCHEMA_DIR')
-    if env_path:
-        candidate = Path(env_path) / 'duc.sql'
-        if candidate.exists():
-            return str(candidate)
-
-    current = Path(__file__).resolve()
-    for parent in current.parents:
-        candidate = parent / 'schema' / 'duc.sql'
-        if candidate.exists():
-            return str(candidate)
-
-    return None
-
-def get_schema_version_from_sql(sql_file_path):
-    """Extract schema version from PRAGMA user_version in duc.sql.
-    
-    Format: user_version = 3000000 → version "3.0.0"
-    Encoding: major * 1_000_000 + minor * 1_000 + patch
-    """
-    try:
-        with open(sql_file_path, 'r') as f:
-            content = f.read()
-            match = re.search(r'PRAGMA\s+user_version\s*=\s*(\d+)', content)
-            if match:
-                version_int = int(match.group(1))
-                major = version_int // 1_000_000
-                minor = (version_int % 1_000_000) // 1_000
-                patch = version_int % 1_000
-                return f"{major}.{minor}.{patch}"
-    except FileNotFoundError:
-        return None
-    except Exception as e:
-        print(f"Warning: Could not parse schema version from {sql_file_path}: {e}. Defaulting to 0.0.0.")
-    return "0.0.0"
-
-def generate_version_py():
-    """Generates the _version.py file with DUC_SCHEMA_VERSION."""
-    schema_file_path = find_schema_file_path()
-    schema_version = get_schema_version_from_sql(schema_file_path) if schema_file_path else None
-
-    if schema_version is not None:
-        print(f"Generating DUC schema version file at: {VERSION_PY_PATH} with version: {schema_version}")
-        os.makedirs(os.path.dirname(VERSION_PY_PATH), exist_ok=True)
-        with open(VERSION_PY_PATH, 'w') as f:
-            f.write(f'# This file is auto-generated by setup.py when the package is built.\n')
-            f.write(f'# Do not edit this file manually.\n')
-            f.write(f'DUC_SCHEMA_VERSION = "{schema_version}"\n')
-    elif os.path.exists(VERSION_PY_PATH):
-        print(f"DUC schema file not found. Using existing _version.py: {VERSION_PY_PATH}")
-    else:
-        print(f"DUC schema file not found and _version.py does not exist. Generating with default version 0.0.0 at: {VERSION_PY_PATH}")
-        os.makedirs(os.path.dirname(VERSION_PY_PATH), exist_ok=True)
-        with open(VERSION_PY_PATH, 'w') as f:
-            f.write(f'# This file is auto-generated by setup.py (fallback).\n')
-            f.write(f'DUC_SCHEMA_VERSION = "0.0.0"\n')
-
-# --- Main part of setup.py --- 
-
-# Generate the version file when setup.py is processed by the build system.
-generate_version_py()
-
 # Minimal setup call.
 # Most package metadata (name, version via setuptools-scm, etc.) and 
 # package discovery are handled by pyproject.toml.
 # This setup.py is present for compatibility and can be used for tasks 
 # like C-extension building or build-time file generation if needed.
-setup() 
+setup()

packages/ducpy/setup.py

@@ -29,67 +29,33 @@
 from __future__ import annotations
 
 import os
-import re
 import sqlite3
 import tempfile
 from pathlib import Path
 from typing import Any, List, Optional, Union
 
-__all__ = ["DucSQL"]
-
-
-_MIGRATION_RE = re.compile(r"^(\d+)_to_(\d+)$")
+import ducpy_native
 
-
-def _find_schema_dir() -> Optional[Path]:
-    env_path = os.environ.get("DUC_SCHEMA_DIR")
-    if env_path:
-        candidate = Path(env_path)
-        if (candidate / "duc.sql").exists():
-            return candidate
-
-    current = Path(__file__).resolve()
-    for parent in current.parents:
-        candidate = parent / "schema"
-        if (candidate / "duc.sql").exists():
-            return candidate
-    return None
+__all__ = ["DucSQL"]
 
 
 def _get_current_schema_version() -> int:
-    """Read the target user_version from duc.sql — mirrors Rust's CURRENT_VERSION."""
-    schema_dir = _find_schema_dir()
-    if schema_dir is None:
-        return 0
-    duc_sql = (schema_dir / "duc.sql").read_text(encoding="utf-8")
-    m = re.search(r"PRAGMA\s+user_version\s*=\s*(\d+)", duc_sql, re.IGNORECASE)
-    return int(m.group(1)) if m else 0
+    """Current schema version integer (e.g. 3000000) — from the Rust crate."""
+    return ducpy_native.get_schema_version_int()
 
 
 def _read_migrations() -> list[tuple[int, int, str]]:
-    """Load all migration SQL files from schema/migrations/, sorted by from_version."""
-    schema_dir = _find_schema_dir()
-    if schema_dir is None:
-        return []
-    migrations_dir = schema_dir / "migrations"
-    if not migrations_dir.exists():
-        return []
-    result: list[tuple[int, int, str]] = []
-    for path in sorted(migrations_dir.glob("*.sql")):
-        m = _MIGRATION_RE.match(path.stem)
-        if m:
-            result.append((int(m.group(1)), int(m.group(2)), path.read_text(encoding="utf-8")))
-    result.sort(key=lambda x: x[0])
-    return result
+    """All migrations as (from_version, to_version, sql) — from the Rust crate."""
+    raw = ducpy_native.get_migrations()
+    # get_migrations returns list of tuples with i64 values
+    return [(int(f), int(t), sql) for f, t, sql in raw]
 
 
 def _apply_migrations(conn: sqlite3.Connection) -> None:
     """Walk the migration chain until user_version reaches the current schema version.
 
-    Mirrors the migration logic in Rust's ``bootstrap.rs``:
-    reads ``schema/migrations/{from}_to_{to}.sql`` files in order and executes
-    them until ``PRAGMA user_version`` matches the version declared in ``duc.sql``.
-    Safe to call on already-current or brand-new databases.
+    Mirrors the migration logic in Rust's ``bootstrap.rs``. Safe to call on
+    already-current or brand-new databases.
     """
     user_version: int = conn.execute("PRAGMA user_version").fetchone()[0]
     if user_version == 0:
@@ -112,18 +78,12 @@ def _apply_migrations(conn: sqlite3.Connection) -> None:
 
 
 def _read_schema_sql() -> str:
-    schema_dir = _find_schema_dir()
-    if schema_dir is None:
-        raise FileNotFoundError(
-            "Could not locate schema/duc.sql. "
-            "Ensure you are running from within the DUC repository."
-        )
-    parts: list[str] = []
-    for filename in ("duc.sql", "version_control.sql", "search.sql"):
-        path = schema_dir / filename
-        if path.exists():
-            parts.append(path.read_text(encoding="utf-8"))
-    return "\n".join(parts)
+    """Concatenate the three schema SQL strings shipped by the Rust crate."""
+    return "\n".join([
+        ducpy_native.get_duc_schema_sql(),
+        ducpy_native.get_version_control_schema_sql(),
+        ducpy_native.get_search_schema_sql(),
+    ])
 
 
 def _apply_pragmas(conn: sqlite3.Connection) -> None:

packages/ducpy/src/ducpy/_version.py

@@ -29,61 +29,8 @@ def __init__(self, failures: List[str]):
         super().__init__("Embedded code validation failed:\n" + "\n".join(f"- {failure}" for failure in failures))
 
 
-def _find_schema_file() -> Path | None:
-    env_path = Path(os.environ["DUC_SCHEMA_DIR"]) if "DUC_SCHEMA_DIR" in os.environ else None
-    if env_path is not None:
-        candidate = env_path / "duc.sql"
-        if candidate.exists():
-            return candidate
-
-    current = Path(__file__).resolve()
-    for parent in current.parents:
-        candidate = parent / "schema" / "duc.sql"
-        if candidate.exists():
-            return candidate
-
-    return None
-
-
-def _decode_user_version_to_semver(user_version: int) -> str:
-    """Decode sqlite-style schema user_version to semver.
-
-    Encoding convention:
-        major * 1_000_000 + minor * 1_000 + patch
-    """
-    if user_version < 0:
-        return "0.0.0"
-
-    major = user_version // 1_000_000
-    minor = (user_version % 1_000_000) // 1_000
-    patch = user_version % 1_000
-    return f"{major}.{minor}.{patch}"
-
-
-def _read_schema_version_fallback() -> str:
-    """Resolve schema version directly from repository `schema/duc.sql`.
-
-    This is used when `ducpy._version` isn't available (for example, in clean
-    CI environments before setup-time generation has run).
-    """
-    try:
-        schema_path = _find_schema_file()
-        if schema_path is None:
-            return "0.0.0"
-        content = schema_path.read_text(encoding="utf-8")
-        match = re.search(r"PRAGMA\s+user_version\s*=\s*(\d+)\s*;", content)
-        if match:
-            return _decode_user_version_to_semver(int(match.group(1)))
-    except Exception as exc:  # pragma: no cover - defensive fallback for CI/runtime variance
-        logger.warning("Failed to resolve schema version fallback from duc.sql: %s", exc)
-
-    return "0.0.0"
-
-
-try:
-    from ducpy._version import DUC_SCHEMA_VERSION
-except ModuleNotFoundError:
-    DUC_SCHEMA_VERSION = _read_schema_version_fallback()
+# Single source of truth for the schema version — comes from the Rust crate.
+DUC_SCHEMA_VERSION = ducpy_native.get_schema_version()
 
 # Map Python element class names → Rust serde type tag strings.
 _ELEMENT_CLASS_TO_TYPE: Dict[str, str] = {