Merge pull request #2 from DeciSym/update-0.7.3

Update 0.7.3

Author: GregHanson

.github/workflows/lint_and_test.yml

@@ -25,7 +25,7 @@ jobs:
           # - os: windows-latest
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           submodules: 'true'
       - name: Install Rust

CITATION.cff

@@ -9,11 +9,11 @@ authors:
     email: konrad.hoeffner@uni-leipzig.de
     affiliation: Institute for Medical Informatics, Statistics and Epidemiology (IMISE), Leipzig, Germany
     orcid: https://orcid.org/0000-0001-7358-3217
-  - given-names: Baccaert
-    family-names: Tim
+  - given-names: Tim
+    family-names: Baccaert
     affiliation: Independent Researcher, Belgium
-  - given-names: Hanson
-    family-names: Greg
+  - given-names: Greg
+    family-names: Hanson
     affiliation: DeciSym
 repository-code: 'https://github.com/konradhoeffner/hdt'
 url: 'https://crates.io/crates/hdt'

Cargo.toml

@@ -1,25 +1,26 @@
 [package]
 name = "hdt"
-version = "0.5.0"
-repository = "https://github.com/konradhoeffner/hdt"
-authors = ["Tim Baccaert <tbaccaer@vub.be>", "Konrad Höffner"]
+version = "0.7.3"
+repository = "https://github.com/KonradHoeffner/hdt"
+authors = ["Tim Baccaert <tbaccaer@vub.be>", "Konrad Höffner", "Greg Hanson"]
 license = "MIT"
 description = "Library for the Header Dictionary Triples (HDT) RDF compression format."
 keywords = ["rdf", "hdt", "compression", "file-format"]
 categories = ["compression", "filesystem", "parsing", "web-programming"]
 readme = "README.md"
 edition = "2024"
 rust-version = "1.87"
+exclude = ["tests/resources/rdf-tests"]
 
 [package.metadata."docs.rs"]
 all-features = true
 
+
 [dependencies]
 bytesize = "2"
 crc = "3"
-oxttl = "0.2"
 oxrdf = "0.3"
-sophia = { version = "0.9", optional = true }
+sophia = { version = "0.10", optional = true }
 thiserror = "2"
 log = "0.4"
 env_logger = { version = "0.11", default-features = false, features = ["auto-color"] }
@@ -34,20 +35,21 @@ color-eyre = { version = "0.6", optional = true }
 clap = { version = "4.5", optional = true, features = ["derive"] }
 fs-err = { version = "3.1.0", optional = true }
 bitset-core = { version = "0.1.1", optional = true }
-lasso = {  version = "0.7.3", features = ["multi-threaded"], optional = true }
-qwt = "0.3.4"
-mem_dbg = "0.3.4"
+oxttl = { version = "0.2.3" }
+qwt = "0.4.0"
 memmap2 = "0.9"
 fs2 = { version = "0.4", optional = true }
 tempfile = { version = "3.27", optional = true }
+hashbrown = { version = "0.17", optional = true }
+ahash = { version = "0.8", optional = true }
 
 [features]
 default = ["sophia", "cache"]
 sophia = ["dep:sophia", "dep:mownstr"]
 cli = ["sophia", "nt", "dep:eyre", "dep:color-eyre", "dep:clap","dep:fs-err"]
 cache = ["dep:serde", "dep:bincode", "dep:fs2", "dep:tempfile"]
 sparql = ["dep:spareval", "dep:spargebra"]
-nt = ["dep:lasso","dep:bitset-core"]
+nt = ["dep:hashbrown","dep:ahash","dep:bitset-core"]
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 wasm-bindgen = "0.2"
@@ -80,8 +82,12 @@ opt-level = 1
 pretty_assertions = "1"
 env_logger = { version = "0.11", default-features = false, features = ["auto-color"] }
 criterion = { version = "0.8", default-features = false, features = ["cargo_bench_support", "html_reports"] }
-gungraun = "0.17"
+gungraun = "0.19"
 color-eyre = "0.6"
 fs-err = "3.1.0"
 sparesults = "0.3"
 walkdir = "2.5.0"
+
+[[example]]
+name = "write"
+required-features = ["sophia"]

README.md

@@ -82,11 +82,13 @@ Web Assembly allows purely client-side HDT in the browser.
 It is gated behind the wasm32-unknown-unknown target and not a feature but is still experimental.
 We don't release the compiled artifacts to NPM yet (please open an issue if you need this or something else) so for now you need to build them yourself.
 Open [test-wasm.html](tests/resources/test-wasm.html) to test it and see the source code for how to load and query HDT with Wasm.
+Due to CORS browser restrictions you may have to run a local server e.g. on port 8000 and then open <http://localhost:8000/tests/resources/test-wasm.html>.
 
 ```sh
 rustup target add wasm32-unknown-unknown
-cargo install wasm_pack
+cargo install wasm-pack
 RUSTFLAGS="-C target-feature=+simd128" wasm-pack build --target web --release
+python -m http.server 8000
 ```
 
 ## API Documentation
@@ -130,7 +132,7 @@ cargo bench --bench gungraun
 ```
 
 * requires [persondata\_en\_10k.hdt](https://github.com/KonradHoeffner/hdt/releases/download/benchmarkdata/persondata_en_10k.hdt.bz2) placed in `tests/resources`
-* requires [Valgrind](https://valgrind.org/) to be installed
+* requires [Valgrind](https://valgrind.org/) and [gungraun-runner](https://gungraun.github.io/gungraun/latest/html/installation/gungraun.html) to be installed
 * may require a conservative target CPU like `RUSTFLAGS="-C target-cpu=skylake" cargo bench --bench gungraun`
 
 ## Comparative benchmark suite

benches/criterion.rs

@@ -1,8 +1,8 @@
 use color_eyre::Result;
 use color_eyre::eyre::WrapErr;
-use criterion::{Criterion, criterion_group, criterion_main};
-use fs_err::File;
-use hdt::triples::*;
+use criterion::{Criterion, criterion_group};
+use fs_err::{File, exists};
+use hdt::triples::{ObjectIter, PredicateIter, PredicateObjectIter, SubjectIter};
 use hdt::{Hdt, IdKind};
 use sophia::api::graph::Graph;
 use sophia::api::term::matcher::Any;
@@ -12,10 +12,10 @@ const VINCENT: &str = "http://dbpedia.org/resource/Vincent_Descombes_Sevoie";
 const TYPE: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
 const PERSON: &str = "http://dbpedia.org/ontology/Person";
 const H: &str = "tests/resources/persondata_en.hdt";
+#[cfg(feature = "nt")]
 const N: &str = "tests/resources/persondata_en.nt";
 
 fn load() -> Result<Hdt> {
-    color_eyre::install().unwrap();
     let f = File::open(H).wrap_err(format!("Error opening {H}, did you download it? See README.md."))?;
     //let file = File::open("tests/resources/lscomplete2015.hdt").expect("error opening file");
     //let file = File::open("tests/resources/snikmeta.hdt").expect("error opening file");
@@ -44,47 +44,47 @@ fn query(c: &mut Criterion) {
     let mut group = c.benchmark_group("S??");
     //let mut group = c.benchmark_group("query");
     group.bench_function("1.1 (vincent, ?, ?) triple IDs", |b| {
-        b.iter(|| SubjectIter::with_pattern(triples, [vincent_id, 0, 0]).count())
+        b.iter(|| SubjectIter::with_pattern(triples, [vincent_id, 0, 0]).count());
     });
     group.bench_function("1.2 (vincent, ?, ?) str triples", |b| b.iter(|| twp(Some(VINCENT), None, None).count()));
     group.bench_function("1.3 (vincent, ?, ?) Sophia triples", |b| {
-        b.iter(|| hdt.triples_matching(Some(&vincent_term), Any, Any).count())
+        b.iter(|| hdt.triples_matching(Some(&vincent_term), Any, Any).count());
     });
     group.finish();
 
     let mut group = c.benchmark_group(format!("?P? {} triples", PredicateIter::new(triples, type_id).count()));
     group.sample_size(10);
     group.bench_function("2.1 (?, type, ?) triple IDs", |b| {
-        b.iter(|| PredicateIter::new(triples, type_id).count())
+        b.iter(|| PredicateIter::new(triples, type_id).count());
     });
     group.bench_function("2.2 (?, type, ?) str triples", |b| b.iter(|| twp(None, Some(TYPE), None).count()));
     group.bench_function("2.3 (?, type, ?) Sophia triples", |b| {
-        b.iter(|| hdt.triples_matching(Any, Some(&type_term), Any).count())
+        b.iter(|| hdt.triples_matching(Any, Some(&type_term), Any).count());
     });
     group.finish();
     let mut group = c.benchmark_group(format!("??O {} triples", ObjectIter::new(triples, person_id).count()));
     group.bench_function("3.1 (?, ?, person) triple IDs", |b| {
-        b.iter(|| ObjectIter::new(triples, person_id).count())
+        b.iter(|| ObjectIter::new(triples, person_id).count());
     });
     group.bench_function("3.2 (?, ?, person) str triples", |b| b.iter(|| twp(None, None, Some(PERSON)).count()));
     group.bench_function("3.3 (?, ?, person) Sophia triples", |b| {
-        b.iter(|| hdt.triples_matching(Any, Any, Some(&person_term)).count())
+        b.iter(|| hdt.triples_matching(Any, Any, Some(&person_term)).count());
     });
     group.finish();
     let mut group = c
         .benchmark_group(format!("?PO {} triples", PredicateObjectIter::new(triples, type_id, person_id).count()));
     group.sample_size(10);
     group.bench_function("4.1 (?, type, person) triple IDs", |b| {
-        b.iter(|| PredicateObjectIter::new(triples, type_id, person_id).count())
+        b.iter(|| PredicateObjectIter::new(triples, type_id, person_id).count());
     });
     group.bench_function("4.2 (?, type, person) str subjects", |b| {
-        b.iter(|| hdt.subjects_with_po(TYPE, PERSON).count())
+        b.iter(|| hdt.subjects_with_po(TYPE, PERSON).count());
     });
     group.bench_function("4.3 (?, type, person) str triples", |b| {
-        b.iter(|| twp(None, Some(TYPE), Some(PERSON)).count())
+        b.iter(|| twp(None, Some(TYPE), Some(PERSON)).count());
     });
     group.bench_function("4.4 (?, type, person) Sophia triples", |b| {
-        b.iter(|| hdt.triples_matching(Any, Some(&type_term), Some(&person_term)).count())
+        b.iter(|| hdt.triples_matching(Any, Some(&type_term), Some(&person_term)).count());
     });
     group.finish();
 }
@@ -99,8 +99,29 @@ fn read_nt_benchmarks(c: &mut Criterion) {
     group.finish();
 }
 
+fn read_hdt_benchmarks(c: &mut Criterion) {
+    use std::path::Path;
+    let path = Path::new(H);
+    assert!(exists(path).unwrap(), "{H} does not exist, please create it. See README.md.");
+    let mut group = c.benchmark_group("read_hdt");
+    group.sample_size(10);
+
+    group.bench_function("read_hdt_uncached", |b| b.iter(load));
+    #[cfg(feature = "cache")]
+    {
+        let c = format!("{H}.{}", hdt::triples::CACHE_EXT);
+        assert!(exists(Path::new(&c)).unwrap(), "cache file {c} does not exist");
+        group.bench_function("read_hdt_cached", |b| b.iter(|| Hdt::read_from_path(path)));
+    }
+    group.finish();
+}
+
 #[cfg(feature = "nt")]
-criterion_group!(criterion, query, read_nt_benchmarks);
+criterion_group!(criterion, query, read_nt_benchmarks, read_hdt_benchmarks);
 #[cfg(not(feature = "nt"))]
-criterion_group!(criterion, query);
-criterion_main!(criterion);
+criterion_group!(criterion, query, read_hdt_benchmarks);
+
+fn main() {
+    color_eyre::install().unwrap();
+    criterion();
+}

codemeta.json

@@ -10,19 +10,25 @@
         },
         {
             "affiliation": {
-                "name": "Institut für Medizinische Informatik, Statistik und Epidemiologie (IMISE), Leipzig University",
+                "name": "Institute for Medical Informatics, Statistics and Epidemiology (IMISE), Leipzig, Germany",
                 "@type": "Organization"
             },
             "familyName": "Höffner",
             "givenName": "Konrad",
             "@id": "https://github.com/KonradHoeffner",
             "@type": "Person"
+        },
+        {
+            "familyName": "Hanson",
+            "givenName": "Greg",
+            "@id": "https://github.com/GregHanson",
+            "@type": "Person"
         }
     ],
     "codeRepository": "https://github.com/KonradHoeffner/hdt",
     "dateCreated": "2022-09-26",
     "datePublished": "2022-11-17",
-    "description": "Library for the Header Dictionary Triples (HDT) compression file format for RDF data. All features other than \"sophia\" are experimental and are neither guaranteed to work in all combinations nor adher to semver: they may change or be removed in future versions including minor or patch releases.\n \n A Rust library for the [Header Dictionary Triples](https://www.rdfhdt.org/) compressed RDF format, including: \n* loading the HDT default format as created by [hdt-cpp](https://github.com/rdfhdt/hdt-cpp)\n* converting N-Triples to HDT\n* efficient querying by triple patterns\n* serializing into other formats like RDF Turtle and N-Triples using the [Sophia](https://crates.io/crates/sophia) adapter\n* running SPARQL queries (with the experimental \"sparql\" feature but HDT is not optimized for that) \n* load other HDT variants\n* swap data to disk\n* modify the RDF graph in memory \nIf you need any of the those features, consider using a SPARQL endpoint instead.\nFor acknowledgement of all the original authors, please look at the reference implementations in C++ and Java by the [https://github.com/rdfhdt](https://github.com/rdfhdt) organisation.\n \n The performance of a query depends on the size of the graph, the type of triple pattern and the size of the result set.\nWhen using large HDT files, make sure to enable the release profile, such as through `cargo build --release`, as this can be much faster than using the dev profile.\n \n If you want to optimize the code, you can use a profiler.\nThe provided test data is very small in order to keep the size of the crate down; locally modifying the tests to use a large HDT file returns more meaningful results.\n \n",
+    "description": "Rust library for the Header Dictionary Triples (HDT) compression file format for RDF data.",
     "identifier": "10.5281/zenodo.7871065",
     "keywords": [
         {
@@ -61,7 +67,7 @@
             ],
             "datePublished": "2023",
             "identifier": "10.21105/joss.05114",
-            "name": "https://orcid.org/0000-0000-0000-0000",
+            "name": "hdt-rs: A Rust library for the Header Dictionary Triples binary RDF compression format",
             "url": "https://doi.org/10.21105/joss.05114",
             "@type": "ScholarlyArticle"
         }

examples/write.rs

@@ -1,4 +1,3 @@
-#![cfg(feature = "sophia")]
 use hdt::Hdt;
 use sophia::api::prelude::{Stringifier, TripleSerializer};
 use sophia::turtle::serializer::turtle::{TurtleConfig, TurtleSerializer};