Migrated code generation logic to it's own crate (#890)

* Migrated code generation logic to it's own crate

* Skip javy-codegen during CI test command.

* Skip javy-codegen during CI lint command.

* Correct various typo's and updated CI flows and Makefiles

* Added integration testing for codegen and cleaned up documentation.

* Corrected minor formatting issue in README.md

* Updated feature documentation to no longer include javy_unstable

* Added missing newline in documentation.

* Updated integration tests to use the default plugin.

* Updated test comment

* Support using Cow instead of Vec<u8> for Plugins

* Upgrade codegen to vet approved version of convert_case

* Updated cargo lock and vetting information

* Revert "Updated cargo lock and vetting information"

This reverts commit fc91898.

* Fixed up vetting information.

Author: ianmarmour

.github/workflows/ci.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Test
         env:
           CARGO_TARGET_WASM32_WASIP1_RUNNER: wasmtime --dir=.
-        run: cargo hack test --target=wasm32-wasip1 --workspace --exclude=javy-cli --exclude=javy-runner --exclude=javy-test-plugin --each-feature -- --nocapture
+        run: cargo hack test --target=wasm32-wasip1 --workspace --exclude=javy-cli --exclude=javy-codegen --exclude=javy-runner --exclude=javy-test-plugin --each-feature -- --nocapture
 
       - name: Test Runner
         run: cargo test --package=javy-runner
@@ -66,6 +66,7 @@ jobs:
         run: |
           cargo clippy --workspace \
           --exclude=javy-cli \
+          --exclude=javy-codegen \
           --exclude=javy-runner \
           --exclude=javy-fuzz \
           --target=wasm32-wasip1 --all-targets -- -D warnings
@@ -84,14 +85,24 @@ jobs:
       - name: Test CLI
         run: CARGO_PROFILE_RELEASE_LTO=off cargo test --package=javy-cli --release -- --nocapture
 
+      - name: Test CodeGen
+        run: |
+          target/release/javy emit-plugin -o crates/codegen/default_plugin.wasm
+          CARGO_PROFILE_RELEASE_LTO=off cargo hack test --package=javy-codegen --release --each-feature -- --nocapture
+
       - name: Check benchmarks
         run: CARGO_PROFILE_RELEASE_LTO=off cargo check --package=javy-cli --release --benches
 
-      - name: Lint
+      - name: Lint CLI
         run: |
           cargo fmt -- --check
           CARGO_PROFILE_RELEASE_LTO=off cargo clippy --package=javy-cli --release --all-targets -- -D warnings
 
+      - name: Lint CodeGen
+        run: |
+          cargo fmt -- --check
+          CARGO_PROFILE_RELEASE_LTO=off cargo hack clippy --package=javy-codegen --release --all-targets --each-feature -- -D warnings
+
       - name: WPT
         run: |
           npm install --prefix wpt

Cargo.lock

@@ -1,6 +1,7 @@
 [workspace]
 members = [
   "crates/cli",
+  "crates/codegen",
   "crates/javy",
   "crates/plugin",
   "crates/plugin-api",

Cargo.toml

@@ -34,6 +34,10 @@ test-plugin-api:
 test-plugin:
 	CARGO_TARGET_WASM32_WASIP1_RUNNER="wasmtime" cargo test --package=javy-plugin --target=wasm32-wasip1 -- --nocapture
 
+test-codegen: cli
+	target/release/javy emit-plugin -o crates/codegen/default_plugin.wasm
+	CARGO_PROFILE_RELEASE_LTO=off cargo hack test --package=javy-codegen --release --each-feature -- --nocapture
+
 # Test in release mode to skip some debug assertions
 # Note: to make this faster, the engine should be optimized beforehand (wasm-strip + wasm-opt).
 # Disabling LTO substantially improves compile time
@@ -45,11 +49,11 @@ test-runner:
 
 test-wpt: cli
 	npm install --prefix wpt
-	npm test --prefix wpt 
+	npm test --prefix wpt
 
-tests: test-javy test-plugin-api test-plugin test-runner test-cli test-wpt
+tests: test-javy test-plugin-api test-plugin test-runner test-codegen test-cli test-wpt
 
-fmt: fmt-javy fmt-plugin-api fmt-plugin fmt-cli
+fmt: fmt-javy fmt-plugin-api fmt-plugin fmt-cli fmt-codegen
 
 fmt-javy:
 	cargo fmt --package=javy -- --check
@@ -68,3 +72,7 @@ fmt-plugin:
 fmt-cli:
 	cargo fmt --package=javy-cli -- --check
 	CARGO_PROFILE_RELEASE_LTO=off cargo clippy --package=javy-cli --release --all-targets -- -D warnings
+
+fmt-codegen:
+	cargo fmt --package=javy-codegen -- --check
+	cargo clippy --package=javy-codegen --release --all-targets -- -D warnings

Makefile

@@ -10,30 +10,18 @@ build = "build.rs"
 name = "javy"
 path = "src/main.rs"
 
-[features]
-dump_wat = ["dep:wasmprinter"]
-
 [dependencies]
+wasi-common = { workspace = true }
 wizer = { workspace = true }
 anyhow = { workspace = true }
-brotli = "7.0.0"
-wasmprinter = { version = "0.226.0", optional = true }
 wasmtime = { workspace = true }
-wasmtime-wasi = { workspace = true }
-wasi-common = { workspace = true }
 walrus = "0.23.3"
-swc_core = { version = "10.7.0", features = [
-  "common_sourcemap",
-  "ecma_ast",
-  "ecma_parser",
-] }
-wit-parser = "0.212.0"
-convert_case = "0.8.0"
-wasm-opt = { workspace = true }
+wasm-opt = "0.116.1"
 tempfile = { workspace = true }
 clap = { version = "4.5.31", features = ["derive"] }
 serde = { workspace = true, default-features = false }
 serde_json = { workspace = true }
+javy-codegen = { path = "../codegen/", features = ["plugin_internal"] }
 
 [dev-dependencies]
 criterion = "0.5"

crates/cli/Cargo.toml

@@ -1,4 +1,3 @@
-mod codegen;
 mod commands;
 mod js_config;
 mod option;
@@ -7,9 +6,9 @@ mod plugin;
 use crate::commands::{Cli, Command, EmitPluginCommandOpts};
 use anyhow::Result;
 use clap::Parser;
-use codegen::js::JS;
-use codegen::{plugin::Plugin, wit::WitOptions, Generator, LinkingKind};
+
 use commands::CodegenOptionGroup;
+use javy_codegen::{Generator, LinkingKind, Plugin, WitOptions, JS};
 use js_config::JsConfig;
 use plugin::{
     CliPlugin, PluginKind, UninitializedPlugin, PLUGIN_MODULE, QUICKJS_PROVIDER_V2_MODULE,
@@ -30,33 +29,38 @@ fn main() -> Result<()> {
 
                 Refer to https://github.com/bytecodealliance/javy/issues/702 for
                 details.
-                
+
                 Use the `build` command instead.
             "#
             );
 
             let js = JS::from_file(&opts.input)?;
 
-            let mut generator = Generator::new();
-            generator
-                .wit_opts(WitOptions::from_tuple((
-                    opts.wit.clone(),
-                    opts.wit_world.clone(),
-                ))?)
-                .source_compression(!opts.no_source_compression)
-                .js_runtime_config(JsConfig::default().to_json()?);
+            let plugin_bytes = if opts.dynamic {
+                QUICKJS_PROVIDER_V2_MODULE
+            } else {
+                PLUGIN_MODULE
+            };
+
+            let mut generator = Generator::new(Plugin::new(plugin_bytes.into()));
 
             if opts.dynamic {
                 generator
-                    .plugin(Plugin::new(QUICKJS_PROVIDER_V2_MODULE.into()))
                     .linking(LinkingKind::Dynamic)
                     .linking_v2_plugin(true);
             } else {
                 generator
-                    .plugin(Plugin::new(PLUGIN_MODULE.into()))
                     .linking(LinkingKind::Static)
                     .linking_default_plugin(true);
-            };
+            }
+
+            generator
+                .wit_opts(WitOptions::from_tuple((
+                    opts.wit.clone(),
+                    opts.wit_world.clone(),
+                ))?)
+                .source_compression(!opts.no_source_compression)
+                .js_runtime_config(JsConfig::default().to_json()?);
 
             let wasm = generator.generate(&js)?;
 
@@ -66,7 +70,6 @@ fn main() -> Result<()> {
         Command::Build(opts) => {
             let js = JS::from_file(&opts.input)?;
             let codegen_opts: CodegenOptionGroup = opts.codegen.clone().try_into()?;
-            let mut generator = Generator::new();
 
             // Always assume the default plugin if no plugin is provided.
             let cli_plugin = match &codegen_opts.plugin {
@@ -76,14 +79,15 @@ fn main() -> Result<()> {
 
             let js_opts = JsConfig::from_group_values(&cli_plugin, opts.js.clone())?;
 
+            let mut generator = Generator::new(cli_plugin.into_plugin());
+
             // Always link to the default plugin if no plugin is provided.
             if codegen_opts.plugin.is_none() {
                 generator.linking_default_plugin(true);
             }
 
             // Configure the generator with the provided options.
             generator
-                .plugin(cli_plugin.into_plugin())
                 .wit_opts(codegen_opts.wit)
                 .source_compression(!codegen_opts.source_compression)
                 .js_runtime_config(js_opts.to_json()?);

crates/cli/src/main.rs

@@ -1,5 +1,5 @@
-use crate::codegen::plugin::Plugin;
 use anyhow::{bail, Result};
+use javy_codegen::Plugin;
 use std::{fs, str};
 use walrus::{ExportItem, ValType};
 use wizer::Wizer;

crates/cli/src/plugin.rs

@@ -0,0 +1 @@
+default_plugin.wasm

crates/codegen/.gitignore

@@ -0,0 +1,11 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic
+Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.0.0] - 2025-2-15
+
+Initial release

crates/codegen/CHANGELOG.md

@@ -0,0 +1,33 @@
+[package]
+name = "javy-codegen"
+version = "1.0.0-alpha.1"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "Wasm generation library for use with Javy"
+homepage = "https://github.com/bytecodealliance/javy/tree/main/crates/codegen"
+repository = "https://github.com/bytecodealliance/javy/tree/main/crates/codegen"
+categories = ["wasm"]
+
+[features]
+plugin_internal = []
+dump_wat = ["dep:wasmprinter"]
+
+[dependencies]
+wizer = { workspace = true }
+anyhow = { workspace = true }
+brotli = "7.0.0"
+wasmprinter = { version = "0.224.0", optional = true }
+wasmtime = { workspace = true }
+wasmtime-wasi = { workspace = true }
+wasi-common = { workspace = true }
+walrus = "0.23.3"
+swc_core = { version = "10.7.0", features = [
+  "common_sourcemap",
+  "ecma_ast",
+  "ecma_parser",
+] }
+wit-parser = "0.212.0"
+convert_case = "0.8.0"
+wasm-opt = "0.116.1"
+tempfile = { workspace = true }

crates/codegen/Cargo.toml

@@ -0,0 +1,35 @@
+<div align="center">
+  <h1><code>javy-codegen</code></h1>
+  <p>
+    <strong>A crate for generating Wasm modules using Javy</strong>
+  </p>
+  <p>
+    <a href="https://docs.rs/javy-codegen"><img src="https://docs.rs/javy-codegen/badge.svg" alt="Documentation Status" /></a>
+    <a href="https://crates.io/crates/javy-codegen"><img src="https://img.shields.io/crates/v/javy-codegen" alt="crates.io status" /></a>
+  </p>
+</div>
+
+Refer to the [crate level documentation](https://docs.rs/javy-codegen) to learn more.
+
+Example usage:
+
+```rust
+use std::path::Path;
+use javy_codegen::{Generator, LinkingKind, Plugin, JS};
+
+fn main() {
+  // Load your target Javascript.
+  let js = JS::from_file(Path::new("example.js"));
+
+  // Load existing pre-initialized Javy plugin.
+  let plugin = Plugin::new_from_path(Path::new("example-plugin.wasm"));
+
+  // Configure code generator.
+  let mut generator = Generator::new();
+  generator.plugin(plugin);
+  generator.linking(LinkingKind::Static);
+
+  // Generate your Wasm module.
+  let wasm = generator.generate(&js)?;
+}
+```

crates/codegen/README.md

@@ -2,7 +2,8 @@ use anyhow::{anyhow, Result};
 use convert_case::{Case, Casing};
 use std::{env, path::Path};
 
-use crate::{codegen::js::JS, codegen::wit};
+use crate::js::JS;
+use crate::wit;
 
 pub(crate) type Exports = Vec<Export>;