diff --git a/Cargo.lock b/Cargo.lock index 4d45a8e..a9008bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,12 +8,6 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - [[package]] name = "base-x" version = "0.2.11" @@ -22,9 +16,9 @@ checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" [[package]] name = "bindgen" -version = "0.64.0" +version = "0.66.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" dependencies = [ "bitflags", "cexpr", @@ -33,26 +27,21 @@ dependencies = [ "lazycell", "log", "peeking_take_while", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn", + "syn 2.0.28", "which", ] [[package]] name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bytes" -version = "1.4.0" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" [[package]] name = "cbor4ii" @@ -60,12 +49,6 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" -[[package]] -name = "cc" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" - [[package]] name = "cexpr" version = "0.6.0" @@ -152,7 +135,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" dependencies = [ "data-encoding", - "syn", + "syn 1.0.109", ] [[package]] @@ -161,42 +144,6 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" -[[package]] -name = "errno" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "form_urlencoded" version = "1.1.0" @@ -212,62 +159,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "indexmap" -version = "1.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" -dependencies = [ - "autocfg", - "hashbrown", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" -dependencies = [ - "libc", - "windows-sys 0.45.0", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -296,12 +193,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "linux-raw-sys" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" - [[package]] name = "log" version = "0.4.17" @@ -358,15 +249,9 @@ dependencies = [ "unsigned-varint", ] -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - [[package]] name = "nginx-car-range" -version = "0.3.0" +version = "0.6.0" dependencies = [ "anyhow", "bindgen", @@ -375,8 +260,6 @@ dependencies = [ "form_urlencoded", "hex", "num_enum", - "prost", - "prost-build", "serde", "serde_ipld_dagcbor", ] @@ -408,7 +291,7 @@ checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -429,107 +312,34 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" -[[package]] -name = "petgraph" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" -dependencies = [ - "fixedbitset", - "indexmap", -] - [[package]] name = "prettyplease" -version = "0.1.23" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" +checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.28", ] [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c828f93f5ca4826f97fedcbd3f9a536c16b12cff3dbbb4a007f932bbad95b12" -dependencies = [ - "bytes", - "heck", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea9b0f8cbe5e15a8a042d030bd96668db28ecb567ec37d691971ff5731d2b1b" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379119666929a1afd7a043aa6cf96fa67a6dce9af60c88095a4686dbce4c9c88" -dependencies = [ - "prost", -] - [[package]] name = "quote" -version = "1.0.23" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - [[package]] name = "regex" version = "1.7.1" @@ -551,20 +361,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustix" -version = "0.36.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" -dependencies = [ - "bitflags", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys", - "windows-sys 0.45.0", -] - [[package]] name = "scopeguard" version = "1.1.0" @@ -606,7 +402,7 @@ checksum = "d7e29c4601e36bcec74a223228dce795f4cd3616341a4af93520ca1a837c087d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -639,16 +435,14 @@ dependencies = [ ] [[package]] -name = "tempfile" -version = "3.4.0" +name = "syn" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ - "cfg-if", - "fastrand", - "redox_syscall", - "rustix", - "windows-sys 0.42.0", + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] @@ -695,84 +489,3 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" diff --git a/Cargo.toml b/Cargo.toml index 91b6768..b533c99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ serde_ipld_dagcbor = {version = "0.2.2", default-features = false } cid = { version = "0.10.1", default-features = false, features = ["serde-codec"] } serde = { version = "1.0", default-features = false, features = ["derive", "alloc"] } anyhow = { version = "1.0.69", default-features = false } -prost = { version = "0.11", default-features = false, features = ["prost-derive"] } num_enum = { version = "0.5.11", default-features = false } core2 = { version = "0.4", default-features = false } @@ -23,8 +22,7 @@ core2 = { version = "0.4", default-features = false } hex = "0.4.3" [build-dependencies] -prost-build = "0.11.1" -bindgen = "0.64" +bindgen = "0.66" # help debugging diff --git a/Dockerfile b/Dockerfile index 14ad6af..dfcd244 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,8 @@ RUN curl -LO https://github.com/ipld/go-car/releases/download/v2.8.0/go-car_2.8. # download CAR fixture RUN curl https://ipfs.io/ipfs/bafybeifpz6onienrgwvb3mw5rg7piq5jh63ystjn7s5wk6ttezy2gy5xwu/Mexico.JPG?format=car > /usr/local/nginx/html/midfixture.car \ - && curl https://saturn.ms/ipfs/QmafUYju2Ab4ETi5HJG1cqjmnjs2xw9PUuBKzU7Hi3zvXU/MC_TheSource.mp4?format=car > /usr/local/nginx/html/bigfixture.car + && curl https://saturn.ms/ipfs/QmafUYju2Ab4ETi5HJG1cqjmnjs2xw9PUuBKzU7Hi3zvXU/MC_TheSource.mp4?format=car > /usr/local/nginx/html/bigfixture.car \ + && curl https://saturn.ms/ipfs/bafybeieyd7lbyfjexjaqb6wvryaixjcmgnpljaca5umfwkbit72ozaotem?format=car > /usr/local/nginx/html/fixture3.car # build the plugin COPY . . diff --git a/build.rs b/build.rs index f38f269..9b9c314 100644 --- a/build.rs +++ b/build.rs @@ -3,11 +3,6 @@ use std::path::PathBuf; use std::process::Command; fn main() { - prost_build::Config::new() - .bytes([".unixfs_pb.Data", ".merkledag_pb.PBNode.Data"]) - .compile_protos(&["src/unixfs.proto", "src/merkledag.proto"], &["src"]) - .expect("unable to generate unixfs protobufs"); - // Path to the nginx repo in the local file system let nginx_dir = env::var("NGINX_DIR").unwrap_or(String::from("../nginx")); diff --git a/ci.sh b/ci.sh index 423c5f6..7df9b31 100644 --- a/ci.sh +++ b/ci.sh @@ -26,4 +26,12 @@ test_range_request "1048576:2097152" "midfixture" test_range_request "555555:999999" "bigfixture" +test_range_request "0:2097151" "fixture3" + +curl -o partial.car -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:8080/fixture3.car" +/usr/local/bin/car ls -v partial.car + +# try a cache hit +test_range_request "1048576:2097151" "fixture3" + cat /var/log/nginx/error.log diff --git a/fixture3.car b/fixture3.car new file mode 100644 index 0000000..c465cab --- /dev/null +++ b/fixture3.car @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:462dec2f7c18490f762dd3eaa22dd6fb8eae21f0e17ea48a6354c524396fa976 +size 6133758 diff --git a/src/car_reader.rs b/src/car_reader.rs index c879902..797b3a3 100644 --- a/src/car_reader.rs +++ b/src/car_reader.rs @@ -7,14 +7,6 @@ use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use std::ops::{Bound, Range, RangeBounds}; -mod unixfs_pb { - include!(concat!(env!("OUT_DIR"), "/unixfs_pb.rs")); -} - -mod dag_pb { - include!(concat!(env!("OUT_DIR"), "/merkledag_pb.rs")); -} - fn lt_bound(bound: Bound<&u64>, val: u64) -> bool { match bound { Bound::Included(&b) => b >= val, @@ -1624,4 +1616,49 @@ mod tests { } } } + + #[test] + fn test_range_start_stall() { + use std::fs::File; + use std::io::{BufReader, Read}; + + let f = File::open("./fixture3.car").unwrap(); + let mut reader = BufReader::new(f); + + let mut car_data = vec![]; + reader.read_to_end(&mut car_data).unwrap(); + + let mut reader = Framed::new(1048576..2097151); + + let mut buf = vec![]; + + let parts = reader + .next(&car_data[..]) + .expect("failed to read all bytes"); + for (start, end) in parts { + println!("=> start {} end {}", start, end); + buf.extend_from_slice(&car_data[start..end]); + } + + check_car( + &buf[..], + vec![ + "bafybeieyd7lbyfjexjaqb6wvryaixjcmgnpljaca5umfwkbit72ozaotem" + .parse() + .unwrap(), + "bafkreiafwydontni6cnaaikpyukjnxcs6npacnzw33yzla2ovst63mmmzu" + .parse() + .unwrap(), + "bafkreiejzhxinfxxobf5drbzcpvyc7dpcapoki3x47p2oy4g2ysw2wqtma" + .parse() + .unwrap(), + "bafkreidxdztf4f5pimr2evnz36eskzmf35sa2vb3wea6fqyl5tipcxip74" + .parse() + .unwrap(), + "bafkreicqajwgsr5z2b4qprhy6c7fexrwmqtvaldcaaiswhcgrkcbmwexmu" + .parse() + .unwrap(), + ], + ); + } }