Use spacetimedb-standalone to get schema for spacetime generate

Author: coolreader18

Cargo.lock

@@ -71,7 +71,6 @@ toml_edit.workspace = true
 tracing = { workspace = true, features = ["release_max_level_off"] }
 walkdir.workspace = true
 wasmbin.workspace = true
-wasmtime.workspace = true
 webbrowser.workspace = true
 clap-markdown.workspace = true
 

crates/cli/Cargo.toml

@@ -5,6 +5,7 @@ use fs_err as fs;
 use regex::Regex;
 use spacetimedb_cli::generate::{csharp, generate};
 use spacetimedb_lib::{RawModuleDef, RawModuleDefV8};
+use spacetimedb_schema::def::ModuleDef;
 use std::path::Path;
 use std::sync::OnceLock;
 
@@ -30,8 +31,9 @@ fn main() -> anyhow::Result<()> {
     fs::remove_dir_all(dir)?;
     fs::create_dir(dir)?;
 
+    let module: ModuleDef = module.try_into()?;
     generate(
-        RawModuleDef::V8BackCompat(module),
+        &module,
         &csharp::Csharp {
             namespace: "SpacetimeDB.Internal",
         },

crates/cli/examples/regen-csharp-moduledef.rs

@@ -4,20 +4,17 @@ use anyhow::Context;
 use clap::parser::ValueSource;
 use clap::Arg;
 use clap::ArgAction::Set;
-use core::mem;
 use fs_err as fs;
 use spacetimedb_lib::de::serde::DeserializeWrapper;
-use spacetimedb_lib::{bsatn, RawModuleDefV8};
-use spacetimedb_lib::{RawModuleDef, MODULE_ABI_MAJOR_VERSION};
-use spacetimedb_primitives::errno;
+use spacetimedb_lib::{sats, RawModuleDef};
 use spacetimedb_schema;
 use spacetimedb_schema::def::{ModuleDef, ReducerDef, ScopedTypeName, TableDef, TypeDef};
 use spacetimedb_schema::identifier::Identifier;
 use std::path::{Path, PathBuf};
-use wasmtime::{Caller, StoreContextMut};
+use std::process::{Command, Stdio};
 
 use crate::generate::util::iter_reducers;
-use crate::util::y_or_n;
+use crate::util::{resolve_sibling_binary, y_or_n};
 use crate::Config;
 use crate::{build, common_args};
 use clap::builder::PossibleValue;
@@ -127,11 +124,10 @@ pub async fn exec(config: Config, args: &clap::ArgMatches) -> anyhow::Result<()>
         };
         let spinner = indicatif::ProgressBar::new_spinner();
         spinner.enable_steady_tick(std::time::Duration::from_millis(60));
-        spinner.set_message("Compiling wasm...");
-        let module = compile_wasm(&wasm_path)?;
         spinner.set_message("Extracting schema from wasm...");
-        extract_descriptions_from_module(module)?
+        extract_descriptions(&wasm_path).context("could not extract schema")?
     };
+    let module: ModuleDef = module.try_into()?;
 
     fs::create_dir_all(out_dir)?;
 
@@ -147,7 +143,7 @@ pub async fn exec(config: Config, args: &clap::ArgMatches) -> anyhow::Result<()>
         Language::TypeScript => &typescript::TypeScript,
     };
 
-    for (fname, code) in generate(module, lang)? {
+    for (fname, code) in generate(&module, lang)? {
         let fname = Path::new(&fname);
         // If a generator asks for a file in a subdirectory, create the subdirectory first.
         if let Some(parent) = fname.parent().filter(|p| !p.as_os_str().is_empty()) {
@@ -230,8 +226,7 @@ impl clap::ValueEnum for Language {
     }
 }
 
-pub fn generate(module: RawModuleDef, lang: &dyn Lang) -> anyhow::Result<Vec<(String, String)>> {
-    let module = &ModuleDef::try_from(module)?;
+pub fn generate(module: &ModuleDef, lang: &dyn Lang) -> anyhow::Result<Vec<(String, String)>> {
     Ok(itertools::chain!(
         module
             .tables()
@@ -266,113 +261,14 @@ pub trait Lang {
         Self: Sized;
 }
 
-pub fn extract_descriptions(wasm_file: &Path) -> anyhow::Result<RawModuleDef> {
-    let module = compile_wasm(wasm_file)?;
-    extract_descriptions_from_module(module)
-}
-
-fn compile_wasm(wasm_file: &Path) -> anyhow::Result<wasmtime::Module> {
-    wasmtime::Module::from_file(&wasmtime::Engine::default(), wasm_file)
-}
-
-fn extract_descriptions_from_module(module: wasmtime::Module) -> anyhow::Result<RawModuleDef> {
-    let engine = module.engine();
-    let ctx = WasmCtx {
-        mem: None,
-        sink: Vec::new(),
-    };
-    let mut store = wasmtime::Store::new(engine, ctx);
-    let mut linker = wasmtime::Linker::new(engine);
-    linker.allow_shadowing(true).define_unknown_imports_as_traps(&module)?;
-    let module_name = &*format!("spacetime_{MODULE_ABI_MAJOR_VERSION}.0");
-    linker.func_wrap(
-        module_name,
-        "console_log",
-        |mut caller: Caller<'_, WasmCtx>,
-         _level: u32,
-         _target_ptr: u32,
-         _target_len: u32,
-         _filename_ptr: u32,
-         _filename_len: u32,
-         _line_number: u32,
-         message_ptr: u32,
-         message_len: u32| {
-            let (mem, _) = WasmCtx::mem_env(&mut caller);
-            let slice = deref_slice(mem, message_ptr, message_len).unwrap();
-            println!("from wasm: {}", String::from_utf8_lossy(slice));
-        },
-    )?;
-    linker.func_wrap(module_name, "bytes_sink_write", WasmCtx::bytes_sink_write)?;
-    let instance = linker.instantiate(&mut store, &module)?;
-    let memory = instance.get_memory(&mut store, "memory").context("no memory export")?;
-    store.data_mut().mem = Some(memory);
-
-    let mut preinits = instance
-        .exports(&mut store)
-        .filter_map(|exp| Some((exp.name().strip_prefix("__preinit__")?.to_owned(), exp.into_func()?)))
-        .collect::<Vec<_>>();
-    preinits.sort_by(|(a, _), (b, _)| a.cmp(b));
-    for (_, func) in preinits {
-        func.typed(&store)?.call(&mut store, ())?
-    }
-    let module: RawModuleDef = match instance.get_func(&mut store, "__describe_module__") {
-        Some(f) => {
-            store.data_mut().sink = Vec::new();
-            f.typed::<u32, ()>(&store)?.call(&mut store, 1)?;
-            let buf = mem::take(&mut store.data_mut().sink);
-            bsatn::from_slice(&buf)?
-        }
-        // TODO: shouldn't we return an error here?
-        None => RawModuleDef::V8BackCompat(RawModuleDefV8::default()),
-    };
-    Ok(module)
-}
-
-struct WasmCtx {
-    mem: Option<wasmtime::Memory>,
-    sink: Vec<u8>,
-}
-
-fn deref_slice(mem: &[u8], offset: u32, len: u32) -> anyhow::Result<&[u8]> {
-    anyhow::ensure!(offset != 0, "ptr is null");
-    mem.get(offset as usize..)
-        .and_then(|s| s.get(..len as usize))
-        .context("pointer out of bounds")
-}
-
-fn read_u32(mem: &[u8], offset: u32) -> anyhow::Result<u32> {
-    Ok(u32::from_le_bytes(deref_slice(mem, offset, 4)?.try_into().unwrap()))
-}
-
-impl WasmCtx {
-    pub fn get_mem(&self) -> wasmtime::Memory {
-        self.mem.expect("Initialized memory")
-    }
-
-    fn mem_env<'a>(ctx: impl Into<StoreContextMut<'a, Self>>) -> (&'a mut [u8], &'a mut Self) {
-        let ctx = ctx.into();
-        let mem = ctx.data().get_mem();
-        mem.data_and_store_mut(ctx)
-    }
-
-    pub fn bytes_sink_write(
-        mut caller: Caller<'_, Self>,
-        sink_handle: u32,
-        buffer_ptr: u32,
-        buffer_len_ptr: u32,
-    ) -> anyhow::Result<u32> {
-        if sink_handle != 1 {
-            return Ok(errno::NO_SUCH_BYTES.get().into());
-        }
-
-        let (mem, env) = Self::mem_env(&mut caller);
-
-        // Read `buffer_len`, i.e., the capacity of `buffer` pointed to by `buffer_ptr`.
-        let buffer_len = read_u32(mem, buffer_len_ptr)?;
-        // Write `buffer` to `sink`.
-        let buffer = deref_slice(mem, buffer_ptr, buffer_len)?;
-        env.sink.extend(buffer);
-
-        Ok(0)
-    }
+fn extract_descriptions(wasm_file: &Path) -> anyhow::Result<RawModuleDef> {
+    let bin_path = resolve_sibling_binary("spacetimedb-standalone")?;
+    let child = Command::new(&bin_path)
+        .arg("extract-schema")
+        .arg(wasm_file)
+        .stdout(Stdio::piped())
+        .spawn()
+        .with_context(|| format!("failed to spawn {}", bin_path.display()))?;
+    let sats::serde::SerdeWrapper(def) = serde_json::from_reader(child.stdout.unwrap())?;
+    Ok(def)
 }

crates/cli/src/subcommands/generate/mod.rs

@@ -6,6 +6,8 @@ use anyhow::Context;
 use clap::{Arg, ArgMatches};
 use spacetimedb_paths::SpacetimePaths;
 
+use crate::util::resolve_sibling_binary;
+
 pub fn cli() -> clap::Command {
     clap::Command::new("start")
         .about("Start a local SpacetimeDB instance")
@@ -45,12 +47,7 @@ pub async fn exec(paths: &SpacetimePaths, args: &ArgMatches) -> anyhow::Result<E
         Edition::Standalone => "spacetimedb-standalone",
         Edition::Cloud => "spacetimedb-cloud",
     };
-    let resolved_exe = std::env::current_exe().context("could not retrieve current exe")?;
-    let bin_path = resolved_exe
-        .parent()
-        .unwrap()
-        .join(bin_name)
-        .with_extension(std::env::consts::EXE_EXTENSION);
+    let bin_path = resolve_sibling_binary(bin_name)?;
     let mut cmd = Command::new(&bin_path);
     cmd.arg("start")
         .arg("--data-dir")

crates/cli/src/subcommands/start.rs

@@ -5,7 +5,7 @@ use spacetimedb_auth::identity::{IncomingClaims, SpacetimeIdentityClaims};
 use spacetimedb_client_api_messages::name::GetNamesResponse;
 use spacetimedb_lib::Identity;
 use std::io::Write;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use crate::config::Config;
 use crate::login::{spacetimedb_login_force, DEFAULT_AUTH_HOST};
@@ -320,3 +320,13 @@ pub async fn get_login_token_or_log_in(
         spacetimedb_login_force(config, &host, true).await
     }
 }
+
+pub fn resolve_sibling_binary(bin_name: &str) -> anyhow::Result<PathBuf> {
+    let resolved_exe = std::env::current_exe().context("could not retrieve current exe")?;
+    let bin_path = resolved_exe
+        .parent()
+        .unwrap()
+        .join(bin_name)
+        .with_extension(std::env::consts::EXE_EXTENSION);
+    Ok(bin_path)
+}

crates/cli/src/util.rs

@@ -1,21 +1,20 @@
-use spacetimedb_cli::generate::{csharp::Csharp, extract_descriptions, generate, rust::Rust, typescript::TypeScript};
+use spacetimedb_cli::generate::{csharp::Csharp, generate, rust::Rust, typescript::TypeScript};
 use spacetimedb_data_structures::map::HashMap;
+use spacetimedb_schema::def::ModuleDef;
 use spacetimedb_testing::modules::{CompilationMode, CompiledModule};
-use std::path::Path;
 use std::sync::OnceLock;
 
-fn compiled_module() -> &'static Path {
-    static COMPILED_MODULE: OnceLock<CompiledModule> = OnceLock::new();
+fn compiled_module() -> &'static ModuleDef {
+    static COMPILED_MODULE: OnceLock<ModuleDef> = OnceLock::new();
     COMPILED_MODULE
-        .get_or_init(|| CompiledModule::compile("module-test", CompilationMode::Debug))
-        .path()
+        .get_or_init(|| CompiledModule::compile("module-test", CompilationMode::Debug).extract_schema_blocking())
 }
 
 macro_rules! declare_tests {
     ($($name:ident => $lang:expr,)*) => ($(
         #[test]
         fn $name() {
-            let module = extract_descriptions(compiled_module()).unwrap();
+            let module = compiled_module();
             let outfiles: HashMap<_, _> = generate(module, &$lang)
                 .unwrap()
                 .into_iter()

crates/cli/tests/codegen.rs

@@ -21,10 +21,11 @@ use log::{info, trace, warn};
 use parking_lot::{Mutex, RwLock};
 use spacetimedb_data_structures::map::IntMap;
 use spacetimedb_durability::{self as durability, TxOffset};
-use spacetimedb_lib::hash_bytes;
+use spacetimedb_lib::{hash_bytes, Identity};
 use spacetimedb_paths::server::{ReplicaDir, ServerDataDir};
 use spacetimedb_paths::FromPathUnchecked;
 use spacetimedb_sats::hash::Hash;
+use spacetimedb_schema::def::ModuleDef;
 use std::future::Future;
 use std::ops::Deref;
 use std::sync::Arc;
@@ -96,7 +97,7 @@ struct HostRuntimes {
 }
 
 impl HostRuntimes {
-    fn new(data_dir: &ServerDataDir) -> Arc<Self> {
+    fn new(data_dir: Option<&ServerDataDir>) -> Arc<Self> {
         let wasmtime = WasmtimeRuntime::new(data_dir);
         Arc::new(Self { wasmtime })
     }
@@ -170,7 +171,7 @@ impl HostController {
             program_storage,
             energy_monitor,
             durability,
-            runtimes: HostRuntimes::new(&data_dir),
+            runtimes: HostRuntimes::new(Some(&data_dir)),
             data_dir,
         }
     }
@@ -259,8 +260,8 @@ impl HostController {
     ///
     /// This is not necessary during hotswap publishes,
     /// as the automigration planner and executor accomplish the same validity checks.
-    pub async fn check_module_validity(&self, database: Database, program: Program) -> anyhow::Result<()> {
-        Host::try_init_in_memory_to_check(self, database, program).await
+    pub async fn check_module_validity(&self, database: Database, program: Program) -> anyhow::Result<Arc<ModuleInfo>> {
+        Host::try_init_in_memory_to_check(&self.runtimes, database, program).await
     }
 
     /// Run a computation on the [`RelationalDB`] of a [`ModuleHost`] managed by
@@ -815,12 +816,10 @@ impl Host {
     /// This is not necessary during hotswap publishes,
     /// as the automigration planner and executor accomplish the same validity checks.
     async fn try_init_in_memory_to_check(
-        host_controller: &HostController,
+        runtimes: &Arc<HostRuntimes>,
         database: Database,
         program: Program,
-    ) -> anyhow::Result<()> {
-        let HostController { runtimes, .. } = host_controller;
-
+    ) -> anyhow::Result<Arc<ModuleInfo>> {
         // Even in-memory databases acquire a lockfile.
         // Grab a tempdir to put that lockfile in.
         let phony_replica_dir = TempDir::with_prefix("spacetimedb-publish-in-memory-check")
@@ -858,7 +857,7 @@ impl Host {
             Result::from(call_result)?;
         }
 
-        Ok(())
+        Ok(launched.module_host.info)
     }
 
     /// Attempt to replace this [`Host`]'s [`ModuleHost`] with a new one running
@@ -946,3 +945,26 @@ async fn metric_reporter(replica_ctx: Arc<ReplicaContext>) {
         tokio::time::sleep(STORAGE_METERING_INTERVAL).await;
     }
 }
+
+/// Extracts the schema from a given module.
+///
+/// Spins up a dummy host and returns the `ModuleDef` that it extracts.
+pub async fn extract_schema(program_bytes: Box<[u8]>, host_type: HostType) -> anyhow::Result<ModuleDef> {
+    let owner_identity = Identity::from_u256(0xdcba_u32.into());
+    let database_identity = Identity::from_u256(0xabcd_u32.into());
+    let program = Program::from_bytes(program_bytes);
+
+    let database = Database {
+        id: 0,
+        database_identity,
+        owner_identity,
+        host_type,
+        initial_program: program.hash,
+    };
+
+    let runtimes = HostRuntimes::new(None);
+    let module_info = Host::try_init_in_memory_to_check(&runtimes, database, program).await?;
+    let module_info = Arc::into_inner(module_info).unwrap();
+
+    Ok(module_info.module_def)
+}