use content hash

Author: gtfierro

Cargo.lock

@@ -38,6 +38,7 @@ oxigraph = "0.5"
 memmap2 = "0.9"
 zstd = "0.13"
 rand = "0.9"
+blake3 = "1.5"
 
 rdf5d = { version = "0.4.1-a1", path = "rdf5d", features = ["oxigraph", "zstd"] }
 

Cargo.toml

@@ -33,3 +33,4 @@ tempdir = "0.3.7"
 pretty-bytes = "0.2.2"
 fs2 = "0.4"
 url = "2.5"
+blake3.workspace = true

lib/Cargo.toml

@@ -15,13 +15,15 @@ use chrono::prelude::*;
 use oxigraph::model::{Dataset, Graph, NamedNode, NamedNodeRef, NamedOrBlankNodeRef, TripleRef};
 use oxigraph::store::Store;
 use petgraph::visit::EdgeRef;
-use std::io::{BufReader, Write};
+use std::fs::File;
+use std::io::{BufReader, Read, Write};
 use std::path::Path;
 use std::path::PathBuf;
 
 use crate::io::GraphIO;
 use crate::ontology::{GraphIdentifier, Ontology, OntologyLocation};
 use anyhow::{anyhow, Result};
+use blake3;
 use log::{debug, error, info, warn};
 use petgraph::graph::{Graph as DiGraph, NodeIndex};
 use std::collections::{HashMap, HashSet, VecDeque};
@@ -867,48 +869,47 @@ impl OntoEnv {
                     }
                 };
 
-                // if the source modified is missing, then we assume it has been updated
-                let source_modified = self
-                    .io
-                    .source_last_modified(ontology.id())
-                    .unwrap_or(Utc::now());
-                // if the ontology has no modified time, then we assume it has never been updated
                 let last_updated = ontology
                     .last_updated
                     .unwrap_or(Utc.timestamp_opt(0, 0).unwrap());
 
-                if source_modified > last_updated {
-                    if let OntologyLocation::File(path) = location {
-                        // Mtime is newer, so now check if content is different
-                        let new_graph = match self.io.read_file(path) {
-                            Ok(g) => g,
+                match location {
+                    OntologyLocation::File(path) => {
+                        // Prefer a fast content hash comparison to avoid mtime granularity issues.
+                        let current_hash = match hash_file(path) {
+                            Ok(h) => h,
                             Err(e) => {
                                 warn!(
-                                    "Could not read file for update check {}: {}",
+                                    "Could not hash file for update check {}: {}",
                                     path.display(),
                                     e
                                 );
-                                return true; // If we can't read it, assume it's updated
+                                return true; // assume updated if we cannot hash
                             }
                         };
-                        let old_graph = match self.io.get_graph(ontology.id()) {
-                            Ok(g) => g,
-                            Err(e) => {
-                                warn!(
-                                    "Could not get graph from store for update check {}: {}",
-                                    ontology.id(),
-                                    e
-                                );
-                                return true; // If we can't get the old one, assume updated
+
+                        if let Some(stored_hash) = ontology.content_hash() {
+                            if stored_hash == current_hash {
+                                return false;
                             }
-                        };
-                        return new_graph != old_graph;
+                            return true;
+                        }
+
+                        // Fallback to mtime when legacy records lack a stored hash.
+                        let source_modified = self
+                            .io
+                            .source_last_modified(ontology.id())
+                            .unwrap_or(Utc::now());
+                        source_modified > last_updated
+                    }
+                    _ => {
+                        let source_modified = self
+                            .io
+                            .source_last_modified(ontology.id())
+                            .unwrap_or(Utc::now());
+                        source_modified > last_updated
                     }
-                    // For non-file locations, we can't easily check content, so stick with mtime.
-                    return true;
                 }
-
-                false
             })
             .map(|(graphid, _)| graphid.clone())
             .collect()
@@ -1792,6 +1793,21 @@ impl OntoEnv {
     }
 }
 
+fn hash_file(path: &Path) -> Result<String> {
+    let file = File::open(path)?;
+    let mut reader = BufReader::new(file);
+    let mut hasher = blake3::Hasher::new();
+    let mut buf = [0u8; 8192];
+    loop {
+        let n = reader.read(&mut buf)?;
+        if n == 0 {
+            break;
+        }
+        hasher.update(&buf[..n]);
+    }
+    Ok(hasher.finalize().to_hex().to_string())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

lib/src/api.rs

@@ -6,6 +6,7 @@ use crate::ontology::{GraphIdentifier, Ontology, OntologyLocation};
 use crate::options::Overwrite;
 use crate::util::get_file_contents;
 use anyhow::{anyhow, Error, Result};
+use blake3;
 use chrono::prelude::*;
 use fs2::FileExt;
 use log::{error, info};
@@ -63,6 +64,8 @@ fn add_ontology_bytes(
     let tmp_store = load_staging_store_from_bytes(bytes, format)?;
     let staging_id = GraphIdentifier::new_with_location(staging_graph.as_ref(), location.clone());
     let mut ontology = Ontology::from_store(&tmp_store, &staging_id, strict)?;
+    let hash = blake3::hash(bytes).to_hex().to_string();
+    ontology.set_content_hash(hash);
     ontology.with_last_updated(Utc::now());
     let id = ontology.id();
     let graphname: GraphName = id.graphname()?;

lib/src/io.rs

@@ -334,6 +334,8 @@ pub struct Ontology {
     version_properties: HashMap<NamedNode, String>,
     #[serde(default)]
     namespace_map: HashMap<String, String>,
+    #[serde(default)]
+    content_hash: Option<String>,
 }
 
 // impl display; name + location + last updated, then indented version properties
@@ -366,6 +368,7 @@ impl Default for Ontology {
             last_updated: None,
             version_properties: HashMap::new(),
             namespace_map: HashMap::new(),
+            content_hash: None,
         }
     }
 }
@@ -382,6 +385,14 @@ impl Ontology {
         self.location = Some(location);
     }
 
+    pub fn set_content_hash(&mut self, hash: String) {
+        self.content_hash = Some(hash);
+    }
+
+    pub fn content_hash(&self) -> Option<&str> {
+        self.content_hash.as_deref()
+    }
+
     pub fn id(&self) -> &GraphIdentifier {
         &self.id
     }
@@ -610,6 +621,7 @@ impl Ontology {
             version_properties,
             last_updated: None,
             namespace_map,
+            content_hash: None,
         })
     }