Skip to content

Install with fsverity enabled + required #935

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 21, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,16 +65,31 @@ jobs:
# For a not-ancient podman
runs-on: ubuntu-24.04
steps:
- name: Get a newer podman for heredoc support (from debian testing)
run: |
set -eux
echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list
sudo apt update
sudo apt install -y crun/testing podman/testing skopeo/testing
- name: Checkout repository
uses: actions/checkout@v4
- name: Free up disk space on runner
run: sudo ./ci/clean-gha-runner.sh
- name: Enable fsverity for /
run: sudo tune2fs -O verity $(findmnt -vno SOURCE /)
- name: Install utils
run: sudo apt -y install fsverity
- name: Integration tests
run: |
set -xeu
# Build images to test; TODO investigate doing single container builds
# via GHA and pushing to a temporary registry to share among workflows?
sudo podman build -t localhost/bootc -f hack/Containerfile .
sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity

export CARGO_INCREMENTAL=0 # because we aren't caching the test runner bits
cargo build --release -p tests-integration

df -h /
sudo install -m 0755 target/release/tests-integration /usr/bin/bootc-integration-tests
rm target -rf
Expand All @@ -84,8 +99,16 @@ jobs:
-v /run/dbus:/run/dbus -v /run/systemd:/run/systemd localhost/bootc /src/ostree-ext/ci/priv-integration.sh
# Nondestructive but privileged tests
sudo bootc-integration-tests host-privileged localhost/bootc
# Finally the install-alongside suite
# Install tests
sudo bootc-integration-tests install-alongside localhost/bootc

# And the fsverity case
sudo podman run --privileged --pid=host localhost/bootc-fsverity bootc install to-existing-root --stateroot=other \
--acknowledge-destructive --skip-fetch-check
# Crude cross check
sudo find /ostree/repo/objects -name '*.file' -type f | while read f; do
sudo fsverity measure $f >/dev/null
done
docs:
if: ${{ contains(github.event.pull_request.labels.*.name, 'documentation') }}
runs-on: ubuntu-latest
Expand Down
18 changes: 9 additions & 9 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 14 additions & 0 deletions ci/Containerfile.install-fsverity
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Enable fsverity at install time
FROM localhost/bootc
RUN <<EORUN
set -xeuo pipefail
cat > /usr/lib/ostree/prepare-root.conf <<EOF
[composefs]
enabled = verity
EOF
cat > /usr/lib/bootc/install/90-ext4.toml <<EOF
[install.filesystem.root]
type = "ext4"
EOF
bootc container lint
EORUN
1 change: 0 additions & 1 deletion lib/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ chrono = { workspace = true, features = ["serde"] }
clap = { workspace = true, features = ["derive","cargo"] }
clap_mangen = { workspace = true, optional = true }
#composefs = "0.2.0"
composefs = { git = "https://github.com/containers/composefs-rs", rev = "55ae2e9ba72f6afda4887d746e6b98f0a1875ac4" }
cap-std-ext = { workspace = true, features = ["fs_utf8"] }
hex = { workspace = true }
fn-error-context = { workspace = true }
Expand Down
2 changes: 1 addition & 1 deletion lib/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ use cap_std_ext::cap_std;
use cap_std_ext::cap_std::fs::Dir;
use clap::Parser;
use clap::ValueEnum;
use composefs::fsverity;
use fn_error_context::context;
use ostree::gio;
use ostree_container::store::PrepareResult;
use ostree_ext::composefs::fsverity;
use ostree_ext::container as ostree_container;
use ostree_ext::container_utils::ostree_booted;
use ostree_ext::keyfileext::KeyFileExt;
Expand Down
156 changes: 156 additions & 0 deletions lib/src/fsck.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,26 @@
// Unfortunately needed here to work with linkme
#![allow(unsafe_code)]

use std::fmt::Write as _;
use std::future::Future;
use std::pin::Pin;
use std::process::Command;

use bootc_utils::iterator_split_nonempty_rest_count;
use camino::Utf8PathBuf;
use cap_std::fs::{Dir, MetadataExt as _};
use cap_std_ext::cap_std;
use cap_std_ext::dirext::CapStdExtDirExt;
use fn_error_context::context;
use linkme::distributed_slice;
use ostree_ext::ostree_prepareroot::Tristate;
use ostree_ext::{composefs, ostree};
use serde::{Deserialize, Serialize};

use crate::store::Storage;

use std::os::fd::AsFd;

/// A lint check has failed.
#[derive(thiserror::Error, Debug)]
struct FsckError(String);
Expand Down Expand Up @@ -112,6 +121,153 @@ fn check_resolvconf(storage: &Storage) -> FsckResult {
fsck_ok()
}

#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
pub(crate) enum VerityState {
Enabled,
Disabled,
Inconsistent((u64, u64)),
}

#[derive(Debug, Default)]
struct ObjectsVerityState {
/// Count of objects with fsverity
enabled: u64,
/// Count of objects without fsverity
disabled: u64,
/// Objects which should have fsverity but do not
missing: Vec<String>,
}

/// Check the fsverity state of all regular files in this object directory.
#[context("Computing verity state")]
fn verity_state_of_objects(
d: &Dir,
prefix: &str,
expected: bool,
) -> anyhow::Result<ObjectsVerityState> {
let mut enabled = 0;
let mut disabled = 0;
let mut missing = Vec::new();
for ent in d.entries()? {
let ent = ent?;
if !ent.file_type()?.is_file() {
continue;
}
let name = ent.file_name();
let name = name
.into_string()
.map(Utf8PathBuf::from)
.map_err(|_| anyhow::anyhow!("Invalid UTF-8"))?;
let Some("file") = name.extension() else {
continue;
};
let f = d.open(&name)?;
let r: Option<composefs::fsverity::Sha256HashValue> =
composefs::fsverity::ioctl::fs_ioc_measure_verity(f.as_fd())?;
drop(f);
if r.is_some() {
enabled += 1;
} else {
disabled += 1;
if expected {
missing.push(format!("{prefix}{name}"));
}
}
}
let r = ObjectsVerityState {
enabled,
disabled,
missing,
};
Ok(r)
}

async fn verity_state_of_all_objects(
repo: &ostree::Repo,
expected: bool,
) -> anyhow::Result<ObjectsVerityState> {
// Limit concurrency here
const MAX_CONCURRENT: usize = 3;

let repodir = Dir::reopen_dir(&repo.dfd_borrow())?;

// It's convenient here to reuse tokio's spawn_blocking as a threadpool basically.
let mut joinset = tokio::task::JoinSet::new();
let mut results = Vec::new();

for ent in repodir.read_dir("objects")? {
// Block here if the queue is full
while joinset.len() >= MAX_CONCURRENT {
results.push(joinset.join_next().await.unwrap()??);
}
let ent = ent?;
if !ent.file_type()?.is_dir() {
continue;
}
let name = ent.file_name();
let name = name
.into_string()
.map(Utf8PathBuf::from)
.map_err(|_| anyhow::anyhow!("Invalid UTF-8"))?;

let objdir = ent.open_dir()?;
let expected = expected.clone();
joinset.spawn_blocking(move || verity_state_of_objects(&objdir, name.as_str(), expected));
}

// Drain the remaining tasks.
while let Some(output) = joinset.join_next().await {
results.push(output??);
}
// Fold the results.
let r = results
.into_iter()
.fold(ObjectsVerityState::default(), |mut acc, v| {
acc.enabled += v.enabled;
acc.disabled += v.disabled;
acc.missing.extend(v.missing);
acc
});
Ok(r)
}

#[distributed_slice(FSCK_CHECKS)]
static CHECK_FSVERITY: FsckCheck =
FsckCheck::new("fsverity", 10, FsckFnImpl::Async(check_fsverity));
fn check_fsverity(storage: &Storage) -> Pin<Box<dyn Future<Output = FsckResult> + '_>> {
Box::pin(check_fsverity_inner(storage))
}

async fn check_fsverity_inner(storage: &Storage) -> FsckResult {
let repo = &storage.repo();
let verity_state = ostree_ext::fsverity::is_verity_enabled(repo)?;
tracing::debug!(
"verity: expected={:?} found={:?}",
verity_state.desired,
verity_state.enabled
);

let verity_found_state =
verity_state_of_all_objects(&storage.repo(), verity_state.desired == Tristate::Enabled)
.await?;
let Some((missing, rest)) =
iterator_split_nonempty_rest_count(verity_found_state.missing.iter(), 5)
else {
return fsck_ok();
};
let mut err = String::from("fsverity enabled, but objects without fsverity:\n");
for obj in missing {
// SAFETY: Writing into a String
writeln!(err, " {obj}").unwrap();
}
if rest > 0 {
// SAFETY: Writing into a String
writeln!(err, " ...and {rest} more").unwrap();
}
fsck_err(err)
}

pub(crate) async fn fsck(storage: &Storage, mut output: impl std::io::Write) -> anyhow::Result<()> {
let mut checks = FSCK_CHECKS.static_slice().iter().collect::<Vec<_>>();
checks.sort_by(|a, b| a.ordering.cmp(&b.ordering));
Expand Down
32 changes: 24 additions & 8 deletions lib/src/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ use fn_error_context::context;
use ostree::gio;
use ostree_ext::oci_spec;
use ostree_ext::ostree;
use ostree_ext::ostree_prepareroot::{ComposefsState, Tristate};
use ostree_ext::prelude::Cast;
use ostree_ext::sysroot::SysrootLock;
use ostree_ext::{container as ostree_container, ostree_prepareroot};
Expand Down Expand Up @@ -77,6 +78,15 @@ const SELINUXFS: &str = "/sys/fs/selinux";
const EFIVARFS: &str = "/sys/firmware/efi/efivars";
pub(crate) const ARCH_USES_EFI: bool = cfg!(any(target_arch = "x86_64", target_arch = "aarch64"));

const DEFAULT_REPO_CONFIG: &[(&str, &str)] = &[
// Default to avoiding grub2-mkconfig etc.
("sysroot.bootloader", "none"),
// Always flip this one on because we need to support alongside installs
// to systems without a separate boot partition.
("sysroot.bootprefix", "true"),
("sysroot.readonly", "true"),
];

/// Kernel argument used to specify we want the rootfs mounted read-write by default
const RW_KARG: &str = "rw";

Expand Down Expand Up @@ -638,14 +648,7 @@ async fn initialize_ostree_root(state: &State, root_setup: &RootSetup) -> Result
crate::lsm::ensure_dir_labeled(rootfs_dir, "boot", None, 0o755.into(), sepolicy)?;
}

for (k, v) in [
// Default to avoiding grub2-mkconfig etc.
("sysroot.bootloader", "none"),
// Always flip this one on because we need to support alongside installs
// to systems without a separate boot partition.
("sysroot.bootprefix", "true"),
("sysroot.readonly", "true"),
] {
for (k, v) in DEFAULT_REPO_CONFIG.iter() {
Command::new("ostree")
.args(["config", "--repo", "ostree/repo", "set", k, v])
.cwd_dir(rootfs_dir.try_clone()?)
Expand All @@ -657,6 +660,19 @@ async fn initialize_ostree_root(state: &State, root_setup: &RootSetup) -> Result
ostree::Sysroot::new(Some(&gio::File::for_path(path)))
};
sysroot.load(cancellable)?;
let repo = &sysroot.repo();

let repo_verity_state = ostree_ext::fsverity::is_verity_enabled(&repo)?;
let prepare_root_composefs = state
.prepareroot_config
.get("composefs.enabled")
.map(|v| ComposefsState::from_str(&v))
.transpose()?
.unwrap_or(ComposefsState::default());
if prepare_root_composefs.requires_fsverity() || repo_verity_state.desired == Tristate::Enabled
{
ostree_ext::fsverity::ensure_verity(repo).await?;
}

let stateroot_exists = rootfs_dir.try_exists(format!("ostree/deploy/{stateroot}"))?;
ensure!(
Expand Down
Loading