Workflow stamp: record every declared schema for top-hash parity (#773)

fiskus · web-flow · commit 50cce65317e3 · 2026-07-14T16:42:53.000+02:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/docs/verification.md b/docs/verification.md
@@ -41,6 +41,11 @@ split -l 1 ~/MANIFEST.jsonl \
   tr -d '\n' | sha256sum
 ```
 
+**Caveat**: Use this meta-null form only when rows actually carry
+`"meta": null`. Applied to a header line (which has no `meta`), `jq` reads the
+absent `.meta` as null and injects `meta: {}`, corrupting the header hash — use
+the plain `del(.physical_keys)` form otherwise.
+
 ## CRC64/NVMe Verification
 
 CRC64-NVMe is a whole-file checksum (no chunking). The digest is 8 bytes,
diff --git a/quilt-cli/src/cli/push.rs b/quilt-cli/src/cli/push.rs
@@ -171,6 +171,7 @@ mod tests {
         use quilt_rs::manifest::WorkflowId;
         use quilt_rs::paths::DomainPaths;
         use quilt_uri::S3Uri;
+        use std::collections::BTreeMap;
         use tempfile::TempDir;
 
         let home_dir = TempDir::new()?;
@@ -234,7 +235,7 @@ mod tests {
             config: config_uri.to_string().parse()?,
             id: Some(WorkflowId {
                 id: "gate".to_string(),
-                metadata: None,
+                schemas: BTreeMap::new(),
             }),
         };
         package
diff --git a/quilt-rs/CHANGELOG.md b/quilt-rs/CHANGELOG.md
@@ -9,6 +9,12 @@
 <!-- markdownlint-disable MD013 -->
 # Changelog
 
+## [v0.33.0-alpha10] - 2026-07-14
+
+### Fixed
+
+- The workflow provenance stamp now records every schema a workflow declares, not only its metadata schema, so a package committed under an entries-schema workflow gets the same top-hash as the Python client (<https://github.com/quiltdata/quilt-rs/pull/773>)
+
 ## [v0.33.0-alpha9] - 2026-07-13
 
 ### Added
diff --git a/quilt-rs/Cargo.toml b/quilt-rs/Cargo.toml
@@ -3,7 +3,7 @@ name = "quilt-rs"
 description = "Rust library for accessing Quilt data packages."
 
 # Inherit from workspace
-version = "0.33.0-alpha9"
+version = "0.33.0-alpha10"
 edition.workspace = true
 rust-version.workspace = true
 license.workspace = true
diff --git a/quilt-rs/fixtures/equivalent-packages/b5af547f5ea5f6d7c878cb39592ce1fe0582958f1096b60c1c6c76d19fb5f936-workflow-schemas-canonical.jsonl b/quilt-rs/fixtures/equivalent-packages/b5af547f5ea5f6d7c878cb39592ce1fe0582958f1096b60c1c6c76d19fb5f936-workflow-schemas-canonical.jsonl
@@ -0,0 +1,2 @@
+{"message":"","user_meta":{},"version":"v0","workflow":{"config":"s3://workflow/config","id":"dual-wf","schemas":{"meta-schema":"s3://bucket/workflows/meta.json","entries-schema":"s3://bucket/workflows/entries.json"}}}
+{"logical_key":"test1.txt","physical_keys":["s3://bucket/test1.txt"],"hash":{"type":"sha2-256-chunked","value":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},"size":0,"meta":{}}
diff --git a/quilt-rs/fixtures/equivalent-packages/b5af547f5ea5f6d7c878cb39592ce1fe0582958f1096b60c1c6c76d19fb5f936-workflow-schemas-key-order.jsonl b/quilt-rs/fixtures/equivalent-packages/b5af547f5ea5f6d7c878cb39592ce1fe0582958f1096b60c1c6c76d19fb5f936-workflow-schemas-key-order.jsonl
@@ -0,0 +1,2 @@
+{"version":"v0","message":"","user_meta":{},"workflow":{"id":"dual-wf","config":"s3://workflow/config","schemas":{"entries-schema":"s3://bucket/workflows/entries.json","meta-schema":"s3://bucket/workflows/meta.json"}}}
+{"logical_key":"test1.txt","physical_keys":["s3://bucket/test1.txt"],"hash":{"type":"sha2-256-chunked","value":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},"size":0,"meta":{}}
diff --git a/quilt-rs/src/fixtures/top_hash.rs b/quilt-rs/src/fixtures/top_hash.rs
@@ -84,6 +84,22 @@ pub const EMPTY_EMPTY_SIMPLE_WORKFLOW_TOP_HASH: &str =
 pub const EMPTY_EMPTY_COMPLEX_WORKFLOW_TOP_HASH: &str =
     "714b1c209a98a7b9239076b94305a7852dc60946c5ba0afac64246ea9958ba08";
 
+/// Manifest header: `{"message":"","user_meta":{},"version":"v0","workflow":{"config":"s3://workflow/config","id":"entries-wf","schemas":{"entries-schema":"s3://bucket/workflows/entries.json"}}}`
+///
+/// A workflow that declares only an `entries_schema` — the stamp the old
+/// single-schema model dropped entirely. Corroborated against quilt3's
+/// `_calculate_top_hash` on the identical header.
+pub const EMPTY_EMPTY_ENTRIES_WORKFLOW_TOP_HASH: &str =
+    "30804452ca34818546a752710397e6d2102d2683e12b67b53cfa9eac3fc83439";
+
+/// Manifest header: `{"message":"","user_meta":{},"version":"v0","workflow":{"config":"s3://workflow/config","id":"dual-wf","schemas":{"entries-schema":"s3://bucket/workflows/entries.json","meta-schema":"s3://bucket/workflows/meta.json"}}}`
+///
+/// A workflow declaring both a `metadata_schema` and an `entries_schema`: the
+/// stamp carries both content addresses. Corroborated against quilt3's
+/// `_calculate_top_hash` on the identical header.
+pub const EMPTY_EMPTY_DUAL_WORKFLOW_TOP_HASH: &str =
+    "93add62524fc06c6aa17c072f5370bfc96350257f253010c6bf688f890277178";
+
 /// Manifest header: `{"message":"Initial","user_meta":{},"version":"v0","workflow":{"config":"s3://workflow/config","id":null}}`
 pub const INITIAL_EMPTY_SIMPLE_WORKFLOW_TOP_HASH: &str =
     "c716c54535bd3c896d0813dafd672430456f68b2d407a6a65a558ccab53f4990";
@@ -156,3 +172,11 @@ pub const MIXED_HASH_TYPES_TOP_HASH: &str =
 /// ```
 pub const NORMALIZED_EQUIVALENCE_TOP_HASH: &str =
     "10c3b62176b4fbb25b4988181bb65e3861087403f36f13c8adb66bce52d6471b";
+
+/// Cross-client equivalence for the workflow stamp's `schemas` map: two manifests
+/// carrying a dual-schema stamp with the `schemas` keys (and header field order)
+/// written differently must address identically, because both clients sort keys
+/// before hashing. Variants: `workflow-schemas-canonical`, `workflow-schemas-key-order`.
+/// Corroborated against quilt3's `_calculate_top_hash` on the header + row.
+pub const WORKFLOW_SCHEMAS_EQUIVALENCE_TOP_HASH: &str =
+    "b5af547f5ea5f6d7c878cb39592ce1fe0582958f1096b60c1c6c76d19fb5f936";
diff --git a/quilt-rs/src/flow/commit.rs b/quilt-rs/src/flow/commit.rs
@@ -390,7 +390,7 @@ mod tests {
             config: config_uri.clone(),
             id: Some(WorkflowId {
                 id: "gate".to_string(),
-                metadata: None,
+                schemas: BTreeMap::new(),
             }),
         };
 
diff --git a/quilt-rs/src/flow/publish.rs b/quilt-rs/src/flow/publish.rs
@@ -546,7 +546,6 @@ mod tests {
     /// doc (2×), compiling identical validators twice.
     #[test(tokio::test)]
     async fn test_publish_fetches_config_and_schema_once() -> Res {
-        use crate::manifest::MetadataSchema;
         use crate::manifest::Workflow;
         use crate::manifest::WorkflowId;
 
@@ -581,10 +580,7 @@ mod tests {
             config: config_uri.parse()?,
             id: Some(WorkflowId {
                 id: "gate".to_string(),
-                metadata: Some(MetadataSchema {
-                    id: "meta".to_string(),
-                    url: schema_uri.parse()?,
-                }),
+                schemas: BTreeMap::from([("meta".to_string(), schema_uri.parse()?)]),
             }),
         };
 
diff --git a/quilt-rs/src/flow/push.rs b/quilt-rs/src/flow/push.rs
@@ -328,6 +328,7 @@ mod tests {
     use quilt_uri::S3Uri;
     use serde_json::Value;
     use serde_json::json;
+    use std::collections::BTreeMap;
 
     #[test(tokio::test)]
     async fn test_no_push_if_no_commit() -> Res {
@@ -358,7 +359,7 @@ mod tests {
                 .expect("valid config uri"),
             id: Some(WorkflowId {
                 id: "gate".to_string(),
-                metadata: None,
+                schemas: BTreeMap::new(),
             }),
         }
     }
@@ -820,7 +821,7 @@ mod tests {
                 .expect("valid config uri"),
             id: Some(WorkflowId {
                 id: "gate".to_string(),
-                metadata: None,
+                schemas: BTreeMap::new(),
             }),
         });
 
diff --git a/quilt-rs/src/flow/recommit.rs b/quilt-rs/src/flow/recommit.rs
@@ -426,7 +426,7 @@ mod tests {
             config: "s3://bucket/.quilt/workflows/config.yml".parse()?,
             id: Some(crate::manifest::WorkflowId {
                 id: "test-workflow".to_string(),
-                metadata: None,
+                schemas: BTreeMap::new(),
             }),
         };
 
@@ -572,7 +572,7 @@ mod tests {
             config: config_uri.clone(),
             id: Some(WorkflowId {
                 id: "gate".to_string(),
-                metadata: None,
+                schemas: BTreeMap::new(),
             }),
         };
 
diff --git a/quilt-rs/src/io/manifest.rs b/quilt-rs/src/io/manifest.rs
@@ -19,8 +19,6 @@ use crate::io::storage::Storage;
 use crate::manifest::Manifest;
 use crate::manifest::ManifestHeader;
 use crate::manifest::ManifestRow;
-#[cfg(test)]
-use crate::manifest::MetadataSchema;
 use crate::manifest::TopHasher;
 #[cfg(test)]
 use crate::manifest::Workflow;
diff --git a/quilt-rs/src/io/manifest/top_hash_header_tests.rs b/quilt-rs/src/io/manifest/top_hash_header_tests.rs
@@ -4,6 +4,7 @@
 //! the Python client — these tests must never be updated to new hashes.
 
 use super::*;
+use std::collections::BTreeMap;
 
 use test_log::test;
 
@@ -514,10 +515,10 @@ async fn test_empty_manifest_header_empty_empty_complex_workflow() -> Res {
                 config: "s3://workflow/config".parse()?,
                 id: Some(WorkflowId {
                     id: "test-workflow".to_string(),
-                    metadata: Some(MetadataSchema {
-                        id: "test-schema".to_string(),
-                        url: "s3://bucket/workflows/test.json".parse()?,
-                    }),
+                    schemas: BTreeMap::from([(
+                        "test-schema".to_string(),
+                        "s3://bucket/workflows/test.json".parse()?,
+                    )]),
                 }),
             }),
             ..ManifestHeader::default()
@@ -612,10 +613,10 @@ async fn test_empty_manifest_header_initial_empty_complex_workflow() -> Res {
                 config: "s3://workflow/config".parse()?,
                 id: Some(WorkflowId {
                     id: "test-workflow".to_string(),
-                    metadata: Some(MetadataSchema {
-                        id: "test-schema".to_string(),
-                        url: "s3://bucket/workflows/test.json".parse()?,
-                    }),
+                    schemas: BTreeMap::from([(
+                        "test-schema".to_string(),
+                        "s3://bucket/workflows/test.json".parse()?,
+                    )]),
                 }),
             }),
             ..ManifestHeader::default()
@@ -802,10 +803,10 @@ async fn test_empty_manifest_header_initial_none_complex_workflow() -> Res {
                 config: "s3://workflow/config".parse()?,
                 id: Some(WorkflowId {
                     id: "test-workflow".to_string(),
-                    metadata: Some(MetadataSchema {
-                        id: "test-schema".to_string(),
-                        url: "s3://bucket/workflows/test.json".parse()?,
-                    }),
+                    schemas: BTreeMap::from([(
+                        "test-schema".to_string(),
+                        "s3://bucket/workflows/test.json".parse()?,
+                    )]),
                 }),
             }),
             ..ManifestHeader::default()
diff --git a/quilt-rs/src/io/manifest/top_hash_row_tests.rs b/quilt-rs/src/io/manifest/top_hash_row_tests.rs
@@ -240,3 +240,52 @@ async fn test_hash_normalization_equivalence_manifest() -> Res {
 
     Ok(())
 }
+
+#[test(tokio::test)]
+async fn test_workflow_schemas_equivalence_manifest() -> Res {
+    // Two manifests carrying a dual-schema workflow stamp, written with the
+    // `schemas` keys (and header fields) in different orders, must address
+    // identically — both clients sort object keys before hashing. Guards the
+    // schema-map path added for cross-client stamp parity.
+    let storage = MockStorage::default();
+    let dest_dir = storage.temp_dir.path();
+    let local_storage = LocalStorage::default();
+
+    let canonical = top_hash::load_equivalent_fixture(
+        top_hash::WORKFLOW_SCHEMAS_EQUIVALENCE_TOP_HASH,
+        "workflow-schemas-canonical",
+    )?;
+    let key_order = top_hash::load_equivalent_fixture(
+        top_hash::WORKFLOW_SCHEMAS_EQUIVALENCE_TOP_HASH,
+        "workflow-schemas-key-order",
+    )?;
+
+    let manifest1 = Manifest::from_path(&local_storage, &canonical).await?;
+    let manifest2 = Manifest::from_path(&local_storage, &key_order).await?;
+
+    let (_, calculated_hash1) = build_manifest_from_rows_stream(
+        &storage,
+        dest_dir.to_path_buf(),
+        manifest1.header.clone(),
+        manifest1.records_stream().await,
+    )
+    .await?;
+    let (_, calculated_hash2) = build_manifest_from_rows_stream(
+        &storage,
+        dest_dir.to_path_buf(),
+        manifest2.header.clone(),
+        manifest2.records_stream().await,
+    )
+    .await?;
+
+    assert_eq!(
+        calculated_hash1, calculated_hash2,
+        "schemas key order and header field order must not affect the address"
+    );
+    assert_eq!(
+        calculated_hash1,
+        top_hash::WORKFLOW_SCHEMAS_EQUIVALENCE_TOP_HASH
+    );
+
+    Ok(())
+}
diff --git a/quilt-rs/src/io/remote/workflow.rs b/quilt-rs/src/io/remote/workflow.rs
diff --git a/quilt-rs/src/manifest.rs b/quilt-rs/src/manifest.rs
diff --git a/quilt-rs/src/manifest/top_hasher.rs b/quilt-rs/src/manifest/top_hasher.rs
diff --git a/quilt-rs/src/manifest/workflow.rs b/quilt-rs/src/manifest/workflow.rs
diff --git a/quilt-sync/src-tauri/src/commands/commit_data.rs b/quilt-sync/src-tauri/src/commands/commit_data.rs

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+{"message":"","user_meta":{},"version":"v0","workflow":{"config":"s3://workflow/config","id":"dual-wf","schemas":{"meta-schema":"s3://bucket/workflows/meta.json","entries-schema":"s3://bucket/workflows/entries.json"}}}`
	`2`	`+{"logical_key":"test1.txt","physical_keys":["s3://bucket/test1.txt"],"hash":{"type":"sha2-256-chunked","value":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},"size":0,"meta":{}}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+{"version":"v0","message":"","user_meta":{},"workflow":{"id":"dual-wf","config":"s3://workflow/config","schemas":{"entries-schema":"s3://bucket/workflows/entries.json","meta-schema":"s3://bucket/workflows/meta.json"}}}`
	`2`	`+{"logical_key":"test1.txt","physical_keys":["s3://bucket/test1.txt"],"hash":{"type":"sha2-256-chunked","value":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},"size":0,"meta":{}}`