diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae7d8d29..c51b01d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ env: # Use the local .curlrc CURL_HOME: . # Disable DFX telemetry - DFX_TELEMETRY: 'off' + DFX_TELEMETRY: "off" jobs: discover: @@ -24,6 +24,31 @@ jobs: - id: set-matrix run: echo "matrix=$(python3 .github/scripts/test-matrix.py)" >> $GITHUB_OUTPUT + unit-tests: + name: Unit tests on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + - uses: actions/checkout@v4 + + - name: Show Rust toolchain version + run: rustup show + + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ matrix.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-test-1 + + - name: Run unit tests + run: cargo test --workspace --lib --bins + test: name: ${{ matrix.test }} on ${{ matrix.os }} needs: discover @@ -61,8 +86,8 @@ jobs: name: test:required if: ${{ always() }} runs-on: ubuntu-latest - needs: [test] + needs: [unit-tests, test] steps: - name: check result - if: ${{ needs.test.result != 'success' }} + if: ${{ needs.unit-tests.result != 'success' || needs.test.result != 'success' }} run: exit 1 diff --git a/Cargo.lock b/Cargo.lock index 6a9b3833..3bf87625 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -725,6 +725,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.19" @@ -1380,11 +1386,26 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "icp-adapter" +version = "0.1.0" +dependencies = [ + "async-trait", + "camino", + "camino-tempfile", + "dunce", + "serde", + "shellwords", + "snafu", + "tokio", +] + [[package]] name = "icp-canister" version = "0.1.0" dependencies = [ "camino", + "icp-adapter", "icp-fs", "serde", "snafu", @@ -1403,6 +1424,7 @@ dependencies = [ "elliptic-curve", "hex", "ic-agent", + "icp-adapter", "icp-canister", "icp-dirs", "icp-fs", @@ -2856,6 +2878,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "shellwords" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e515aa4699a88148ed5ef96413ceef0048ce95b43fbc955a33bde0a70fcae6" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "shlex" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index 929632de..475c2ef9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "bin/icp-cli", + "lib/icp-adapter", "lib/icp-canister", "lib/icp-dirs", "lib/icp-fs", @@ -12,18 +13,22 @@ members = [ resolver = "3" [workspace.dependencies] +async-trait = "0.1.88" bip32 = "0.5.0" camino = { version = "1.1.9", features = ["serde1"] } +camino-tempfile = "1" candid = "0.10.14" clap = { version = "4.5.3", features = ["derive"] } dialoguer = "0.11.0" directories = "6.0.0" +dunce = "1.0.5" ed25519-consensus = "2.1.0" elliptic-curve = { version = "0.13.8", features = ["sec1", "std", "pkcs8"] } fd-lock = "4.0.4" glob = "0.3.2" hex = "0.4.3" ic-agent = { version = "0.40.1" } +icp-adapter = { path = "lib/icp-adapter" } icp-canister = { path = "lib/icp-canister" } icp-dirs = { path = "lib/icp-dirs" } icp-fs = { path = "lib/icp-fs" } @@ -37,13 +42,11 @@ pem = "3.0.5" pkcs8 = { version = "0.10.2", features = ["encryption", "std"] } pocket-ic = "9.0.0" rand = "0.9.1" -reqwest = { version = "0.12.15", default-features = false, features = [ - "rustls-tls", -] } sec1 = { version = "0.7.3", features = ["pkcs8"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9.34" +shellwords = "1.1.0" snafu = "0.8.5" strum = { version = "0.27", features = ["derive"] } tempfile = "3" @@ -53,6 +56,11 @@ tokio = { version = "1.45.0", features = ["macros", "rt-multi-thread"] } uuid = { version = "1.16.0", features = ["serde", "v4"] } zeroize = "1.8.1" +[workspace.dependencies.reqwest] +version = "0.12.15" +default-features = false +features = ["rustls-tls"] + [workspace.lints.rust] missing_debug_implementations = "warn" diff --git a/bin/icp-cli/Cargo.toml b/bin/icp-cli/Cargo.toml index 406dc11e..ad4c8ef4 100644 --- a/bin/icp-cli/Cargo.toml +++ b/bin/icp-cli/Cargo.toml @@ -16,6 +16,7 @@ dialoguer.workspace = true elliptic-curve.workspace = true hex.workspace = true ic-agent.workspace = true +icp-adapter.workspace = true icp-canister.workspace = true icp-dirs.workspace = true icp-fs.workspace = true diff --git a/bin/icp-cli/src/commands/build.rs b/bin/icp-cli/src/commands/build.rs index 2eb42f09..64c43c2c 100644 --- a/bin/icp-cli/src/commands/build.rs +++ b/bin/icp-cli/src/commands/build.rs @@ -1,13 +1,16 @@ use clap::Parser; -use icp_canister::model::{CanisterManifest, LoadCanisterManifestError}; -use icp_project::directory::{FindProjectError, ProjectDirectory}; -use icp_project::model::{LoadProjectManifestError, ProjectManifest}; +use icp_adapter::{Adapter as _, AdapterCompileError}; +use icp_canister::model::{Adapter, CanisterManifest, LoadCanisterManifestError}; +use icp_project::{ + directory::{FindProjectError, ProjectDirectory}, + model::{LoadProjectManifestError, ProjectManifest}, +}; use snafu::Snafu; #[derive(Parser, Debug)] pub struct Cmd; -pub async fn exec(_cmd: Cmd) -> Result<(), BuildCommandError> { +pub async fn exec(_: Cmd) -> Result<(), BuildCommandError> { // Project let pd = ProjectDirectory::find()?.ok_or(BuildCommandError::ProjectNotFound)?; @@ -21,12 +24,27 @@ pub async fn exec(_cmd: Cmd) -> Result<(), BuildCommandError> { let mut cs = Vec::new(); for c in pm.canisters { - let cm = CanisterManifest::from_file(pds.canister_yaml_path(&c))?; - cs.push(cm); + let mpath = pds.canister_yaml_path(&c); + let cm = CanisterManifest::from_file(&mpath)?; + cs.push((c, cm)); } // Build canisters - println!("{cs:?}"); + for (path, c) in cs { + match c.build.adapter { + Adapter::Script(adapter) => { + adapter.compile(&path).await?; + } + + Adapter::Motoko(adapter) => { + adapter.compile(&path).await?; + } + + Adapter::Rust(adapter) => { + adapter.compile(&path).await?; + } + } + } Ok(()) } @@ -44,4 +62,7 @@ pub enum BuildCommandError { #[snafu(transparent)] CanisterLoad { source: LoadCanisterManifestError }, + + #[snafu(transparent)] + BuildAdapter { source: AdapterCompileError }, } diff --git a/bin/icp-cli/tests/build_tests.rs b/bin/icp-cli/tests/build_tests.rs new file mode 100644 index 00000000..343f9277 --- /dev/null +++ b/bin/icp-cli/tests/build_tests.rs @@ -0,0 +1,50 @@ +use crate::common::TestEnv; +use icp_fs::fs::{create_dir_all, write}; +use predicates::{ord::eq, str::PredicateStrExt}; + +mod common; + +#[test] +fn build_adapter_script_simple() { + let env = TestEnv::new(); + + // Setup project + let project_dir = env.create_project_dir("icp"); + + // Project manifest + let pm = r#" + canisters: + - my-canister + "#; + + write( + project_dir.join("icp.yaml"), // path + pm, // contents + ) + .expect("failed to write project manifest"); + + // Canister manifest + let cm = r#" + name: my-canister + build: + adapter: + type: script + command: echo hi + "#; + + create_dir_all(project_dir.join("my-canister")).expect("failed to create canister directory"); + + write( + project_dir.join("my-canister/canister.yaml"), // path + cm, // contents + ) + .expect("failed to write project manifest"); + + // Invoke build + env.icp() + .current_dir(project_dir) + .args(["build"]) + .assert() + .success() + .stdout(eq("hi").trim()); +} diff --git a/lib/icp-adapter/Cargo.toml b/lib/icp-adapter/Cargo.toml new file mode 100644 index 00000000..2eab1dc2 --- /dev/null +++ b/lib/icp-adapter/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "icp-adapter" +version = "0.1.0" +edition = "2024" + +[dependencies] +async-trait = { workspace = true } +camino = { workspace = true } +dunce = { workspace = true } +serde = { workspace = true } +shellwords = { workspace = true } +snafu = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true } +camino-tempfile = { workspace = true } diff --git a/lib/icp-adapter/src/lib.rs b/lib/icp-adapter/src/lib.rs new file mode 100644 index 00000000..8cb1617f --- /dev/null +++ b/lib/icp-adapter/src/lib.rs @@ -0,0 +1,27 @@ +use async_trait::async_trait; +use camino::Utf8Path; +use motoko::MotokoAdapterCompileError; +use rust::RustAdapterCompileError; +use script::ScriptAdapterCompileError; +use snafu::Snafu; + +pub mod motoko; +pub mod rust; +pub mod script; + +#[async_trait] +pub trait Adapter { + async fn compile(&self, path: &Utf8Path) -> Result<(), AdapterCompileError>; +} + +#[derive(Debug, Snafu)] +pub enum AdapterCompileError { + #[snafu(transparent)] + Rust { source: RustAdapterCompileError }, + + #[snafu(transparent)] + Motoko { source: MotokoAdapterCompileError }, + + #[snafu(transparent)] + Script { source: ScriptAdapterCompileError }, +} diff --git a/lib/icp-adapter/src/motoko.rs b/lib/icp-adapter/src/motoko.rs new file mode 100644 index 00000000..6d1f8388 --- /dev/null +++ b/lib/icp-adapter/src/motoko.rs @@ -0,0 +1,27 @@ +use crate::{Adapter, AdapterCompileError}; +use async_trait::async_trait; +use camino::Utf8Path; +use serde::Deserialize; +use snafu::Snafu; + +/// Configuration for a Motoko-based canister build adapter. +#[derive(Debug, Deserialize)] +pub struct MotokoAdapter { + /// Optional path to the main Motoko source file. + /// If omitted, a default like `main.mo` may be assumed. + #[serde(default)] + pub main: Option, +} + +#[async_trait] +impl Adapter for MotokoAdapter { + async fn compile(&self, _path: &Utf8Path) -> Result<(), AdapterCompileError> { + Ok(()) + } +} + +#[derive(Debug, Snafu)] +pub enum MotokoAdapterCompileError { + #[snafu(display("an unexpected build error occurred"))] + Unexpected, +} diff --git a/lib/icp-adapter/src/rust.rs b/lib/icp-adapter/src/rust.rs new file mode 100644 index 00000000..f4594d71 --- /dev/null +++ b/lib/icp-adapter/src/rust.rs @@ -0,0 +1,25 @@ +use crate::{Adapter, AdapterCompileError}; +use async_trait::async_trait; +use camino::Utf8Path; +use serde::Deserialize; +use snafu::Snafu; + +/// Configuration for a Rust-based canister build adapter. +#[derive(Debug, Deserialize)] +pub struct RustAdapter { + /// The name of the Cargo package to build. + pub package: String, +} + +#[async_trait] +impl Adapter for RustAdapter { + async fn compile(&self, _path: &Utf8Path) -> Result<(), AdapterCompileError> { + Ok(()) + } +} + +#[derive(Debug, Snafu)] +pub enum RustAdapterCompileError { + #[snafu(display("an unexpected build error occurred"))] + Unexpected, +} diff --git a/lib/icp-adapter/src/script.rs b/lib/icp-adapter/src/script.rs new file mode 100644 index 00000000..702953d8 --- /dev/null +++ b/lib/icp-adapter/src/script.rs @@ -0,0 +1,227 @@ +use crate::{Adapter, AdapterCompileError}; +use async_trait::async_trait; +use camino::Utf8Path; +use serde::Deserialize; +use snafu::{OptionExt, ResultExt, Snafu}; +use std::process::{Command, Stdio}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum CommandField { + /// Command used to build a canister + Command(String), + + /// Set of commands used to build a canister + Commands(Vec), +} + +impl CommandField { + fn as_vec(&self) -> Vec { + match self { + Self::Command(cmd) => vec![cmd.clone()], + Self::Commands(cmds) => cmds.clone(), + } + } +} + +/// Configuration for a custom canister build adapter. +#[derive(Debug, Deserialize)] +pub struct ScriptAdapter { + /// Command used to build a canister + #[serde(flatten)] + pub command: CommandField, +} + +#[async_trait] +impl Adapter for ScriptAdapter { + async fn compile(&self, path: &Utf8Path) -> Result<(), AdapterCompileError> { + for input_cmd in self.command.as_vec() { + // Parse command input + let input = shellwords::split(&input_cmd).context(CommandParseSnafu { + command: &input_cmd, + })?; + + // Separate command and args + let (cmd, args) = input.split_first().context(InvalidCommandSnafu { + command: &input_cmd, + reason: "command must include at least one element".to_string(), + })?; + + // Try resolving the command as a local path (e.g., ./mytool) + let cmd = match dunce::canonicalize(path.join(cmd)) { + // Use the canonicalized local path if it exists + Ok(p) => p, + + // Fall back to assuming it's a command in the system PATH + Err(_) => cmd.into(), + }; + + // Command + let mut cmd = Command::new(cmd); + + // Args + cmd.args(args); + + // Set directory + cmd.current_dir(path); + + // Output + cmd.stdin(Stdio::inherit()); + cmd.stdout(Stdio::inherit()); + cmd.stderr(Stdio::inherit()); + + // Execute + let status = cmd.status().context(CommandInvokeSnafu { + command: &input_cmd, + })?; + + // Status + if !status.success() { + return Err(ScriptAdapterCompileError::CommandStatus { + command: input_cmd, + code: status.code().map_or("N/A".to_string(), |c| c.to_string()), + } + .into()); + } + } + + Ok(()) + } +} + +#[derive(Debug, Snafu)] +pub enum ScriptAdapterCompileError { + #[snafu(display("failed to parse command '{command}'"))] + CommandParse { + command: String, + source: shellwords::MismatchedQuotes, + }, + + #[snafu(display("invalid command '{command}': {reason}"))] + InvalidCommand { command: String, reason: String }, + + #[snafu(display("failed to execute command '{command}'"))] + CommandInvoke { + command: String, + source: std::io::Error, + }, + + #[snafu(display("command '{command}' failed with status code {code}"))] + CommandStatus { command: String, code: String }, +} + +#[cfg(test)] +mod tests { + use super::*; + use camino_tempfile::NamedUtf8TempFile; + use std::io::Read; + + #[tokio::test] + async fn single_command() { + // Create temporary file + let mut f = NamedUtf8TempFile::new().expect("failed to create temporary file"); + + // Define adapter + let v = ScriptAdapter { + command: CommandField::Command(format!("sh -c 'echo test > {}'", f.path())), + }; + + // Invoke adapter + v.compile("/".into()).await.expect("unexpected failure"); + + // Verify command ran + let mut out = String::new(); + + f.read_to_string(&mut out) + .expect("failed to read temporary file"); + + assert_eq!(out, "test\n".to_string()); + } + + #[tokio::test] + async fn multiple_commands() { + // Create temporary file + let mut f = NamedUtf8TempFile::new().expect("failed to create temporary file"); + + // Define adapter + let v = ScriptAdapter { + command: CommandField::Commands(vec![ + format!("sh -c 'echo cmd-1 >> {}'", f.path()), + format!("sh -c 'echo cmd-2 >> {}'", f.path()), + format!("sh -c 'echo cmd-3 >> {}'", f.path()), + ]), + }; + + // Invoke adapter + v.compile("/".into()).await.expect("unexpected failure"); + + // Verify command ran + let mut out = String::new(); + + f.read_to_string(&mut out) + .expect("failed to read temporary file"); + + assert_eq!(out, "cmd-1\ncmd-2\ncmd-3\n".to_string()); + } + + #[tokio::test] + async fn invalid_command() { + // Define adapter + let v = ScriptAdapter { + command: CommandField::Command("".into()), + }; + + // Invoke adapter + let out = v.compile("/".into()).await; + + // Assert failure + assert!(matches!( + out, + Err(AdapterCompileError::Script { + source: ScriptAdapterCompileError::InvalidCommand { .. } + }) + )); + } + + #[tokio::test] + async fn failed_command_not_found() { + // Define adapter + let v = ScriptAdapter { + command: CommandField::Command("invalid-command".into()), + }; + + // Invoke adapter + let out = v.compile("/".into()).await; + + println!("{out:?}"); + + // Assert failure + assert!(matches!( + out, + Err(AdapterCompileError::Script { + source: ScriptAdapterCompileError::CommandInvoke { .. } + }) + )); + } + + #[tokio::test] + async fn failed_command_error_status() { + // Define adapter + let v = ScriptAdapter { + command: CommandField::Command("sh -c 'exit 1'".into()), + }; + + // Invoke adapter + let out = v.compile("/".into()).await; + + println!("{out:?}"); + + // Assert failure + assert!(matches!( + out, + Err(AdapterCompileError::Script { + source: ScriptAdapterCompileError::CommandStatus { .. } + }) + )); + } +} diff --git a/lib/icp-canister/Cargo.toml b/lib/icp-canister/Cargo.toml index ac7dd727..42b0040d 100644 --- a/lib/icp-canister/Cargo.toml +++ b/lib/icp-canister/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] camino = { workspace = true } +icp-adapter = { "path" = "../icp-adapter" } icp-fs = { "path" = "../icp-fs" } serde = { workspace = true } snafu = { workspace = true } diff --git a/lib/icp-canister/src/model.rs b/lib/icp-canister/src/model.rs index 9ca69d3d..111d38ee 100644 --- a/lib/icp-canister/src/model.rs +++ b/lib/icp-canister/src/model.rs @@ -1,31 +1,9 @@ use camino::Utf8Path; +use icp_adapter::{motoko::MotokoAdapter, rust::RustAdapter, script::ScriptAdapter}; use icp_fs::yaml::{LoadYamlFileError, load_yaml_file}; use serde::Deserialize; use snafu::Snafu; -/// Configuration for a Rust-based canister build adapter. -#[derive(Debug, Deserialize)] -pub struct RustAdapter { - /// The name of the Cargo package to build. - pub package: String, -} - -/// Configuration for a Motoko-based canister build adapter. -#[derive(Debug, Deserialize)] -pub struct MotokoAdapter { - /// Optional path to the main Motoko source file. - /// If omitted, a default like `main.mo` may be assumed. - #[serde(default)] - pub main: Option, -} - -/// Configuration for a custom canister build adapter. -#[derive(Debug, Deserialize)] -pub struct CustomAdapter { - /// Path to a script or executable used to build the canister. - pub script: String, -} - /// Identifies the type of adapter used to build the canister, /// along with its configuration. /// @@ -46,7 +24,7 @@ pub enum Adapter { Motoko(MotokoAdapter), /// A canister built using a custom script or command. - Custom(CustomAdapter), + Script(ScriptAdapter), } /// Describes how the canister should be built into WebAssembly, diff --git a/lib/icp-project/src/model.rs b/lib/icp-project/src/model.rs index 5d92ee71..504c1381 100644 --- a/lib/icp-project/src/model.rs +++ b/lib/icp-project/src/model.rs @@ -98,13 +98,13 @@ pub enum LoadProjectManifestError { #[snafu(transparent)] InvalidPathUtf8 { source: camino::FromPathBufError }, - #[snafu(display("failed to glob pattern {pattern}"))] + #[snafu(display("failed to glob pattern '{pattern}'"))] GlobPattern { source: glob::PatternError, pattern: String, }, - #[snafu(display("failed to glob pattern in {path}"))] + #[snafu(display("failed to glob pattern in '{path}'"))] GlobWalk { source: glob::GlobError, path: Utf8PathBuf,