Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 38 additions & 24 deletions crates/dlin/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1370,6 +1370,40 @@ fn find_deleted_manifest_files(
deleted
}

/// Collect files whose mtimes can make a manifest stale.
///
/// The project file and the optional project-level vars file affect dbt's
/// interpretation of the discovered resources, so they are freshness inputs
/// even though they do not live under one of the configured resource paths.
fn collect_manifest_freshness_inputs(
project_dir: &Path,
project: &parser::project::DbtProject,
) -> Result<Vec<PathBuf>> {
let paths = project.resolve_paths(project_dir);
let files = parser::discovery::discover_files(&paths)?;
let mut inputs = files
.model_sql_files
.into_iter()
.chain(files.macro_sql_files)
.chain(files.seed_files)
.chain(files.snapshot_sql_files)
.chain(files.test_sql_files)
.chain(files.yaml_files)
.collect::<Vec<_>>();

for name in ["dbt_project.yml", "vars.yml"] {
let path = project_dir.join(name);
if path.is_file() {
inputs.push(path);
}
}

inputs.sort();
inputs.dedup();

Ok(inputs)
}

/// Check manifest.json freshness, returning None if manifest is irrelevant.
#[cfg(not(tarpaulin_include))]
fn check_manifest_freshness(
Expand Down Expand Up @@ -1397,23 +1431,13 @@ fn check_manifest_freshness(
Err(_) => return Some(not_found),
};

let paths = project.resolve_paths(project_dir);
let files = match parser::discovery::discover_files(&paths) {
let files = match collect_manifest_freshness_inputs(project_dir, project) {
Ok(f) => f,
Err(_) => return None,
};

let mut stale_files: Vec<String> = Vec::new();
let all_files = files
.model_sql_files
.iter()
.chain(files.macro_sql_files.iter())
.chain(files.seed_files.iter())
.chain(files.snapshot_sql_files.iter())
.chain(files.test_sql_files.iter())
.chain(files.yaml_files.iter());

for file in all_files {
for file in &files {
if let Ok(meta) = std::fs::metadata(file)
&& let Ok(mtime) = meta.modified()
&& mtime > manifest_mtime
Expand Down Expand Up @@ -1472,21 +1496,11 @@ fn run_check_manifest_command(args: CheckManifestArgs) -> Result<()> {

// Discover project files
let project = parser::project::DbtProject::load(&project_dir)?;
let paths = project.resolve_paths(&project_dir);
let files = parser::discovery::discover_files(&paths)?;
let files = collect_manifest_freshness_inputs(&project_dir, &project)?;

// Collect all SQL/YAML files and compare mtimes
let mut stale_files: Vec<PathBuf> = Vec::new();
let all_files = files
.model_sql_files
.iter()
.chain(files.macro_sql_files.iter())
.chain(files.seed_files.iter())
.chain(files.snapshot_sql_files.iter())
.chain(files.test_sql_files.iter())
.chain(files.yaml_files.iter());

for file in all_files {
for file in &files {
match std::fs::metadata(file) {
Ok(meta) => {
if let Ok(mtime) = meta.modified()
Expand Down
129 changes: 129 additions & 0 deletions crates/dlin/tests/integration_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,49 @@ mod freshness {
set_mtime_newer_than(manifest, &latest);
}

fn run_check_manifest(tmp: &tempfile::TempDir) -> serde_json::Value {
let output = Command::new(binary_path())
.args([
"check-manifest",
"--project-dir",
tmp.path().to_str().unwrap(),
"-o",
"json",
])
.output()
.expect("Failed to run check-manifest");
serde_json::from_slice(&output.stdout).expect("check-manifest should emit JSON")
}

fn run_manifest_summary(tmp: &tempfile::TempDir) -> serde_json::Value {
let output = Command::new(binary_path())
.args([
"summary",
"--source",
"manifest",
"--project-dir",
tmp.path().to_str().unwrap(),
"-o",
"json",
])
.output()
.expect("Failed to run summary");
assert!(
output.status.success(),
"summary should succeed: {}",
String::from_utf8_lossy(&output.stderr)
);
let report: serde_json::Value =
serde_json::from_slice(&output.stdout).expect("summary should emit JSON");
report["manifest_status"].clone()
}

fn stale_files(report: &serde_json::Value) -> &Vec<serde_json::Value> {
report["stale_files"]
.as_array()
.expect("stale_files should be an array")
}

#[test]
fn test_check_manifest_up_to_date() {
let tmp = copy_fixture_to_temp();
Expand Down Expand Up @@ -417,6 +460,92 @@ mod freshness {
assert_eq!(parsed["deleted_file_count"], 0);
assert_eq!(parsed["deleted_files"].as_array().unwrap().len(), 0);
}

#[test]
fn test_root_project_inputs_make_both_freshness_paths_stale_when_newer() {
for root_file in ["dbt_project.yml", "vars.yml"] {
let tmp = copy_fixture_to_temp();
if root_file == "vars.yml" {
fs::write(tmp.path().join(root_file), "vars: {}\n").unwrap();
}

let manifest_path = tmp.path().join("target/manifest.json");
set_mtime_newer_than_fixture(&manifest_path, tmp.path());
set_mtime_newer_than(&tmp.path().join(root_file), &manifest_path);

let check_report = run_check_manifest(&tmp);
assert_eq!(check_report["is_stale"], true);
assert!(
stale_files(&check_report)
.iter()
.any(|file| file.as_str() == Some(root_file)),
"check-manifest should report newer {root_file}: {check_report}"
);

let summary_status = run_manifest_summary(&tmp);
assert_eq!(summary_status["is_stale"], true);
assert!(
stale_files(&summary_status)
.iter()
.any(|file| file.as_str() == Some(root_file)),
"summary should report newer {root_file}: {summary_status}"
);
}
}

#[test]
fn test_optional_vars_file_older_than_manifest_does_not_make_manifest_stale() {
let tmp = copy_fixture_to_temp();
let vars_path = tmp.path().join("vars.yml");
fs::write(&vars_path, "vars: {}\n").unwrap();
let manifest_path = tmp.path().join("target/manifest.json");
set_mtime_newer_than_fixture(&manifest_path, tmp.path());

let check_report = run_check_manifest(&tmp);
assert_eq!(check_report["is_stale"], false);
assert!(stale_files(&check_report).is_empty());

let summary_status = run_manifest_summary(&tmp);
assert_eq!(summary_status["is_stale"], false);
assert!(stale_files(&summary_status).is_empty());
}

#[test]
fn test_missing_optional_vars_file_does_not_make_manifest_stale() {
let tmp = copy_fixture_to_temp();
assert!(!tmp.path().join("vars.yml").exists());
let manifest_path = tmp.path().join("target/manifest.json");
set_mtime_newer_than_fixture(&manifest_path, tmp.path());

let check_report = run_check_manifest(&tmp);
assert_eq!(check_report["is_stale"], false);
assert!(stale_files(&check_report).is_empty());

let summary_status = run_manifest_summary(&tmp);
assert_eq!(summary_status["is_stale"], false);
assert!(stale_files(&summary_status).is_empty());
}

#[test]
fn test_root_freshness_input_is_not_duplicated_when_discovered() {
let tmp = copy_fixture_to_temp();
fs::write(
tmp.path().join("dbt_project.yml"),
"name: simple_project\nmodel-paths: [.]\n",
)
.unwrap();
let manifest_path = tmp.path().join("target/manifest.json");
set_mtime_newer_than_fixture(&manifest_path, tmp.path());
set_mtime_newer_than(&tmp.path().join("dbt_project.yml"), &manifest_path);

let check_report = run_check_manifest(&tmp);
assert_eq!(check_report["stale_file_count"], 1);
assert_eq!(stale_files(&check_report)[0], "dbt_project.yml");

let summary_status = run_manifest_summary(&tmp);
assert_eq!(summary_status["stale_file_count"], 1);
assert_eq!(stale_files(&summary_status)[0], "dbt_project.yml");
}
}

mod cli {
Expand Down