Merge pull request #364 from AerynOS/feat/stone-ffi

Feat/stone ffi

Author: tarkah

Cargo.lock

@@ -1,5 +1,5 @@
 [workspace]
-members = ["boulder", "moss", "crates/*"]
+members = ["boulder", "moss", "libstone", "crates/*"]
 default-members = ["moss"]
 resolver = "3"
 
@@ -95,6 +95,8 @@ zbus = { version = "5.1.1", default-features = false, features = ["tokio"] }
 infer = "0.19.0"
 tempfile = "3.20.0"
 kdl = "6.5.0"
+libc = "0.2.62"
+cbindgen = "0.29.2"
 
 [workspace.lints.rust]
 rust_2018_idioms = { level = "warn", priority = -1 }

Cargo.toml

@@ -6,9 +6,9 @@ use std::{io, num::NonZeroU64};
 
 use fs_err as fs;
 use itertools::Itertools;
+use stone::StoneDigestWriterHasher;
 use thiserror::Error;
 
-use stone::write::digest;
 use stone_recipe::{Package, script};
 
 use crate::{Macros, Paths, Recipe, Timing, build, container, timing, util};
@@ -60,7 +60,7 @@ impl<'a> Packager<'a> {
 
     pub fn package(&self, timing: &mut Timing) -> Result<(), Error> {
         // Hasher used for calculating file digests
-        let mut hasher = digest::Hasher::new();
+        let mut hasher = StoneDigestWriterHasher::new();
 
         let timer = timing.begin(timing::Kind::Analyze);
 

boulder/src/package.rs

@@ -9,7 +9,7 @@ use std::{
 };
 
 use moss::{Dependency, Provider};
-use stone::write::digest;
+use stone::StoneDigestWriterHasher;
 use tui::{ProgressBar, ProgressStyle, Styled};
 
 use crate::{Paths, Recipe};
@@ -25,12 +25,17 @@ pub struct Chain<'a> {
     recipe: &'a Recipe,
     paths: &'a Paths,
     collector: &'a Collector,
-    hasher: &'a mut digest::Hasher,
+    hasher: &'a mut StoneDigestWriterHasher,
     pub buckets: BTreeMap<String, Bucket>,
 }
 
 impl<'a> Chain<'a> {
-    pub fn new(paths: &'a Paths, recipe: &'a Recipe, collector: &'a Collector, hasher: &'a mut digest::Hasher) -> Self {
+    pub fn new(
+        paths: &'a Paths,
+        recipe: &'a Recipe,
+        collector: &'a Collector,
+        hasher: &'a mut StoneDigestWriterHasher,
+    ) -> Self {
         Self {
             handlers: vec![
                 Box::new(handler::ignore_blocked),
@@ -157,7 +162,7 @@ impl Bucket {
 pub struct BucketMut<'a> {
     pub providers: &'a mut BTreeSet<Provider>,
     pub dependencies: &'a mut BTreeSet<Dependency>,
-    pub hasher: &'a mut digest::Hasher,
+    pub hasher: &'a mut StoneDigestWriterHasher,
     pub recipe: &'a Recipe,
     pub paths: &'a Paths,
 }

boulder/src/package/analysis.rs

@@ -13,8 +13,7 @@ use astr::AStr;
 use fs_err as fs;
 use glob::Pattern;
 use nix::libc::{S_IFDIR, S_IRGRP, S_IROTH, S_IRWXU, S_IXGRP, S_IXOTH};
-use stone::payload::{Layout, layout};
-use stone::write::digest;
+use stone::{StoneDigestWriter, StoneDigestWriterHasher, StonePayloadLayoutFile, StonePayloadLayoutRecord};
 use thiserror::Error;
 
 #[derive(Debug, Clone, Eq, PartialEq)]
@@ -72,7 +71,7 @@ impl Collector {
     }
 
     /// Produce a [`PathInfo`] from the provided [`Path`]
-    pub fn path(&self, path: &Path, hasher: &mut digest::Hasher) -> Result<PathInfo, Error> {
+    pub fn path(&self, path: &Path, hasher: &mut StoneDigestWriterHasher) -> Result<PathInfo, Error> {
         let metadata = fs::metadata(path)?;
         self.path_with_metadata(path.to_path_buf(), &metadata, hasher)
     }
@@ -81,7 +80,7 @@ impl Collector {
         &self,
         path: PathBuf,
         metadata: &Metadata,
-        hasher: &mut digest::Hasher,
+        hasher: &mut StoneDigestWriterHasher,
     ) -> Result<PathInfo, Error> {
         let target_path = Path::new("/").join(path.strip_prefix(&self.root).expect("path is ancestor of root"));
 
@@ -96,7 +95,7 @@ impl Collector {
     pub fn enumerate_paths(
         &self,
         subdir: Option<(PathBuf, Metadata)>,
-        hasher: &mut digest::Hasher,
+        hasher: &mut StoneDigestWriterHasher,
     ) -> Result<Vec<PathInfo>, Error> {
         let mut paths = vec![];
 
@@ -139,7 +138,7 @@ impl Collector {
 pub struct PathInfo {
     pub path: PathBuf,
     pub target_path: PathBuf,
-    pub layout: Layout,
+    pub layout: StonePayloadLayoutRecord,
     pub size: u64,
     pub package: String,
 }
@@ -149,7 +148,7 @@ impl PathInfo {
         path: PathBuf,
         target_path: PathBuf,
         metadata: &Metadata,
-        hasher: &mut digest::Hasher,
+        hasher: &mut StoneDigestWriterHasher,
         package: String,
     ) -> Result<Self, Error> {
         let layout = layout_from_metadata(&path, &target_path, metadata, hasher)?;
@@ -163,19 +162,19 @@ impl PathInfo {
         })
     }
 
-    pub fn restat(&mut self, hasher: &mut digest::Hasher) -> Result<(), Error> {
+    pub fn restat(&mut self, hasher: &mut StoneDigestWriterHasher) -> Result<(), Error> {
         let metadata = fs::metadata(&self.path)?;
         self.layout = layout_from_metadata(&self.path, &self.target_path, &metadata, hasher)?;
         self.size = metadata.size();
         Ok(())
     }
 
     pub fn is_file(&self) -> bool {
-        matches!(self.layout.entry, layout::Entry::Regular(_, _))
+        matches!(self.layout.file, StonePayloadLayoutFile::Regular(_, _))
     }
 
     pub fn file_hash(&self) -> Option<u128> {
-        if let layout::Entry::Regular(hash, _) = &self.layout.entry {
+        if let StonePayloadLayoutFile::Regular(hash, _) = &self.layout.file {
             Some(*hash)
         } else {
             None
@@ -200,8 +199,8 @@ fn layout_from_metadata(
     path: &Path,
     target_path: &Path,
     metadata: &Metadata,
-    hasher: &mut digest::Hasher,
-) -> Result<Layout, Error> {
+    hasher: &mut StoneDigestWriterHasher,
+) -> Result<StonePayloadLayoutRecord, Error> {
     // Strip /usr
     let target: AStr = target_path
         .strip_prefix("/usr")
@@ -211,29 +210,29 @@ fn layout_from_metadata(
 
     let file_type = metadata.file_type();
 
-    Ok(Layout {
+    Ok(StonePayloadLayoutRecord {
         uid: metadata.uid(),
         gid: metadata.gid(),
         mode: metadata.mode(),
         tag: 0,
-        entry: if file_type.is_symlink() {
+        file: if file_type.is_symlink() {
             let source = fs::read_link(path)?;
 
-            layout::Entry::Symlink(source.to_string_lossy().into(), target)
+            StonePayloadLayoutFile::Symlink(source.to_string_lossy().into(), target)
         } else if file_type.is_dir() {
-            layout::Entry::Directory(target)
+            StonePayloadLayoutFile::Directory(target)
         } else if file_type.is_char_device() {
-            layout::Entry::CharacterDevice(target)
+            StonePayloadLayoutFile::CharacterDevice(target)
         } else if file_type.is_block_device() {
-            layout::Entry::BlockDevice(target)
+            StonePayloadLayoutFile::BlockDevice(target)
         } else if file_type.is_fifo() {
-            layout::Entry::Fifo(target)
+            StonePayloadLayoutFile::Fifo(target)
         } else if file_type.is_socket() {
-            layout::Entry::Socket(target)
+            StonePayloadLayoutFile::Socket(target)
         } else {
             hasher.reset();
 
-            let mut digest_writer = digest::Writer::new(io::sink(), hasher);
+            let mut digest_writer = StoneDigestWriter::new(io::sink(), hasher);
             let mut file = fs::File::open(path)?;
 
             // Copy bytes to null sink so we don't
@@ -242,7 +241,7 @@ fn layout_from_metadata(
 
             let hash = hasher.digest128();
 
-            layout::Entry::Regular(hash, target)
+            StonePayloadLayoutFile::Regular(hash, target)
         },
     })
 }

boulder/src/package/collect.rs

@@ -11,6 +11,7 @@ use fs_err::{self as fs, File};
 use itertools::Itertools;
 use moss::{Dependency, Provider, package::Meta};
 use regex::Regex;
+use stone::{StoneHeaderV1FileType, StoneWriteError, StoneWriter};
 use thiserror::Error;
 use tui::{ProgressBar, ProgressStyle, Styled};
 
@@ -183,7 +184,7 @@ fn emit_package(paths: &Paths, package: &Package<'_>) -> Result<(), Error> {
     let mut out_file = File::create(out_path)?;
 
     // Create stone binary writer
-    let mut writer = stone::Writer::new(&mut out_file, stone::header::v1::FileType::Binary)?;
+    let mut writer = StoneWriter::new(&mut out_file, StoneHeaderV1FileType::Binary)?;
 
     // Add metadata
     {
@@ -244,7 +245,7 @@ fn emit_package(paths: &Paths, package: &Package<'_>) -> Result<(), Error> {
 #[derive(Debug, Error)]
 pub enum Error {
     #[error("stone binary writer")]
-    StoneBinaryWriter(#[from] stone::write::Error),
+    StoneBinaryWriter(#[from] StoneWriteError),
     #[error("manifest")]
     Manifest(#[from] manifest::Error),
     #[error("io")]

boulder/src/package/emit.rs

@@ -4,6 +4,7 @@
 
 use std::{collections::BTreeSet, io, path::PathBuf};
 
+use stone::StoneWriteError;
 use thiserror::Error;
 
 use crate::{Architecture, Paths, Recipe};
@@ -69,7 +70,7 @@ impl<'a> Manifest<'a> {
 #[derive(Debug, Error)]
 pub enum Error {
     #[error("stone binary writer")]
-    StoneWriter(#[from] stone::write::Error),
+    StoneWriter(#[from] StoneWriteError),
     #[error("encode json")]
     Json(#[from] serde_json::Error),
     #[error("io")]

boulder/src/package/emit/manifest.rs

@@ -7,8 +7,7 @@ use std::{collections::BTreeSet, path::Path};
 use fs_err::File;
 use moss::Dependency;
 use stone::{
-    header::v1::FileType,
-    payload::{self, meta},
+    StoneHeaderV1FileType, StonePayloadMetaPrimitive, StonePayloadMetaRecord, StonePayloadMetaTag, StoneWriter,
 };
 
 use super::Error;
@@ -17,7 +16,7 @@ use crate::package::emit::Package;
 pub fn write(path: &Path, packages: &BTreeSet<&Package<'_>>, build_deps: &BTreeSet<String>) -> Result<(), Error> {
     let mut output = File::create(path)?;
 
-    let mut writer = stone::Writer::new(&mut output, FileType::BuildManifest)?;
+    let mut writer = StoneWriter::new(&mut output, StoneHeaderV1FileType::BuildManifest)?;
 
     // Add each package
     for package in packages {
@@ -29,9 +28,9 @@ pub fn write(path: &Path, packages: &BTreeSet<&Package<'_>>, build_deps: &BTreeS
         // Add build deps
         for name in build_deps {
             if let Ok(dep) = Dependency::from_name(name) {
-                payload.push(payload::Meta {
-                    tag: meta::Tag::BuildDepends,
-                    kind: meta::Kind::Dependency(dep.kind.into(), dep.name),
+                payload.push(StonePayloadMetaRecord {
+                    tag: StonePayloadMetaTag::BuildDepends,
+                    primitive: StonePayloadMetaPrimitive::Dependency(dep.kind.into(), dep.name),
                 });
             }
         }

boulder/src/package/emit/manifest/binary.rs

@@ -53,7 +53,7 @@ pub fn write(
                 .analysis
                 .paths
                 .iter()
-                .map(|p| format!("/usr/{}", p.layout.entry.target()))
+                .map(|p| format!("/usr/{}", p.layout.file.target()))
                 .sorted()
                 .collect();
 

boulder/src/package/emit/manifest/json.rs

@@ -4,10 +4,15 @@ version.workspace = true
 edition.workspace = true
 rust-version.workspace = true
 
+[features]
+default = []
+diesel = ["dep:diesel"]
+
 [dependencies]
-diesel.workspace = true
 stable_deref_trait = "1.2.1"
 triomphe.workspace = true
 
+diesel = { workspace = true, optional = true }
+
 [lints]
 workspace = true