Merge branch 'main' into antoine/table-filter

Author: abey79

.github/pull_request_template.md

@@ -19,4 +19,6 @@ We track various metrics at <https://build.rerun.io>.
 For maintainers:
 * To run all checks from `main`, comment on the PR with `@rerun-bot full-check`.
 * To deploy documentation changes immediately after merging this PR, add the `deploy docs` label.
+
+For more details check the PR section on <https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md>.
 -->

.github/workflows/labels.yml

@@ -39,7 +39,7 @@ jobs:
         with:
           mode: minimum
           count: 1
-          labels: "📊 analytics, 🟦 blueprint, 🪳 bug, 🌊 C++ API, CLI, codegen/idl, 🧑‍💻 dev experience, dependencies, 📖 documentation, 💬 discussion, examples, exclude from changelog, 🪵 Log & send APIs, 📉 performance, 🐍 Python API, ⛃ re_datastore, 🔍 re_query, 📺 re_viewer, 🔺 re_renderer, 🚜 refactor, ⛴ release, 🦀 Rust API, 🔨 testing, ui, 🕸️ web"
+          labels: "📊 analytics, 🟦 blueprint, 🪳 bug, CLI, codegen/idl, 🧑‍💻 dev experience, dependencies, 📖 documentation, 💬 discussion, examples, exclude from changelog, 🪵 Log & send APIs, 📉 performance, sdk-python, sdk-cpp, sdk-rust, ⛃ re_datastore, 🔍 re_query, 📺 re_viewer, 🔺 re_renderer, 🚜 refactor, ⛴ release, 🔨 testing, ui, 🕸️ web"
 
   wasm-bindgen-check:
     name: Check wasm-bindgen version

CONTRIBUTING.md

@@ -36,6 +36,41 @@ Members of the `rerun-io` organization and collaborators in the `rerun-io/rerun`
 
 ![PR comment with the text `@rerun-bot approve`](https://github.com/rerun-io/rerun/assets/1665677/b5f07f3f-ea95-44a4-8eb7-f07c905f96c3)
 
+
+### Labeling of PRs & changelog generation
+
+Members of the `rerun-io` organization _have_ to add label PRs since they're part of how we generate [changelogs](https://github.com/rerun-io/rerun/blob/main/CHANGELOG.md).
+
+If `include in changelog` is present, the **title** of the PR will be used as a line in the detailed section of the changelog.
+Therefore, make sure that it's informative & concise without any additional context other than the category.
+If a PR title should not be part of the changelog, you _have_ to label the PR with `exclude from changelog`.
+
+The category of the changelog entry is governed by additional labels of which you have to provide at least one.
+The exact list may change over time, see the [CI job](./.github/workflows/labels.yml) checking for it for a full list.
+
+Beyond changelog categorization, it's encouraged to add too many rather than too few labels as they help with search.
+
+#### What should go to the changelog?
+
+Whether a PR should be added to the changelog isn't always clear cut, but it's advised to err on the side of
+adding too many entries rather than too few!
+Generally, whenever you believe that this is a value-add for a user browsing the detailed changelog, add it.
+Also, credit where credit is due, be more generous adding contributions from outside the `rerun-io` org to the changelog!
+
+We typically don't include:
+pure refactors, testing, CI fixes, fixes for bugs that showed up since the last release, minor doc changes (like typos) etc.
+
+#### Other special labels
+
+* `deploy docs`:
+  PRs marked with this will automatically be cherry-picked to the `docs-latest` branch which then will kick off a rebuild of the public [doc page](https://www.rerun.io/docs)
+  Use this for landing doc fixes that are relevant to the latest release.
+* `do-not-merge`:
+  Will fail the CI unconditionally.
+  Useful e.g. for opening PRs that target branches other than `main` which you want to rebase prior to merging
+  or when you're still waiting on a test result.
+  Alternatively, you can also use checkboxes in the PR description - any unticked checkboxes will make the ci fail automatically ✨
+
 ## Contributing to CI
 
 Every CI job would in its ideal state consist of only a single `pixi` (or similar) script invocation that works locally as-is.

Cargo.lock

@@ -642,7 +642,7 @@ dependencies = [
  "memchr",
  "num",
  "regex",
- "regex-syntax 0.8.5",
+ "regex-syntax",
 ]
 
 [[package]]
@@ -1288,7 +1288,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
 dependencies = [
  "memchr",
- "regex-automata 0.4.9",
+ "regex-automata",
  "serde",
 ]
 
@@ -2722,7 +2722,7 @@ dependencies = [
  "log",
  "recursive",
  "regex",
- "regex-syntax 0.8.5",
+ "regex-syntax",
 ]
 
 [[package]]
@@ -5212,11 +5212,11 @@ dependencies = [
 
 [[package]]
 name = "matchers"
-version = "0.1.0"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
+checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
 dependencies = [
- "regex-automata 0.1.10",
+ "regex-automata",
 ]
 
 [[package]]
@@ -5617,12 +5617,11 @@ dependencies = [
 
 [[package]]
 name = "nu-ansi-term"
-version = "0.46.0"
+version = "0.50.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
+checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399"
 dependencies = [
- "overload",
- "winapi",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -6255,12 +6254,6 @@ dependencies = [
  "pin-project-lite",
 ]
 
-[[package]]
-name = "overload"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
-
 [[package]]
 name = "owned_ttf_parser"
 version = "0.25.0"
@@ -9234,17 +9227,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.4.9",
- "regex-syntax 0.8.5",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
-dependencies = [
- "regex-syntax 0.6.29",
+ "regex-automata",
+ "regex-syntax",
 ]
 
 [[package]]
@@ -9255,7 +9239,7 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.8.5",
+ "regex-syntax",
 ]
 
 [[package]]
@@ -9264,12 +9248,6 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a"
 
-[[package]]
-name = "regex-syntax"
-version = "0.6.29"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
-
 [[package]]
 name = "regex-syntax"
 version = "0.8.5"
@@ -11025,14 +11003,14 @@ dependencies = [
 
 [[package]]
 name = "tracing-subscriber"
-version = "0.3.19"
+version = "0.3.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
+checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5"
 dependencies = [
  "matchers",
  "nu-ansi-term",
  "once_cell",
- "regex",
+ "regex-automata",
  "serde",
  "serde_json",
  "sharded-slab",

crates/build/re_types_builder/src/codegen/rust/deserializer.rs

@@ -600,12 +600,10 @@ fn quote_arrow_field_deserializer(
         }
 
         DataType::Utf8 => {
-            let quoted_downcast = quote_array_downcast(
-                obj_field_fqname,
-                data_src,
-                quote!(StringArray),
-                quoted_datatype,
-            );
+            let quoted_downcast = {
+                let cast_as = quote!(StringArray);
+                quote_array_downcast(obj_field_fqname, data_src, cast_as, quoted_datatype)
+            };
 
             let quoted_iter_transparency = quote_iterator_transparency(
                 objects,
@@ -641,8 +639,7 @@ fn quote_arrow_field_deserializer(
                                 (start, end), #data_src_buf.len(),
                             ));
                         }
-
-                        #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)]
+                        #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] // TODO(apache/arrow-rs#6900): slice_with_length_unchecked unsafe when https://github.com/apache/arrow-rs/pull/6901 is merged and released
                         let data = #data_src_buf.slice_with_length(start, len);
 
                         Ok(data)

crates/store/re_protos/proto/rerun/v1alpha1/cloud.proto

@@ -74,7 +74,11 @@ service RerunCloudService {
   // Search a previously created index.
   rpc SearchDataset(SearchDatasetRequest) returns (stream SearchDatasetResponse) {}
 
-  // Perform Rerun-native queries on a dataset, returning the matching chunk IDs.
+  // Perform Rerun-native queries on a dataset, returning the matching chunk IDs, as well
+  // as information that can be sent back to Rerun Cloud to fetch the actual chunks as part
+  // of `FetchChunks` request. In this 2-step query process, 1st step is getting information
+  // from the server about the chunks that contain relevant information. 2nd step is fetching
+  // those chunks (the actual data).
   //
   // These Rerun-native queries include:
   // * Filtering by specific partition and chunk IDs.
@@ -96,6 +100,13 @@ service RerunCloudService {
   // To fetch only the actual chunk IDs rather than the chunks themselves, see `QueryDataset`.
   rpc GetChunks(GetChunksRequest) returns (stream GetChunksResponse) {}
 
+  // Fetch specific chunks from Rerun Cloud. In a 2-step query process, result of 1st phase,
+  // that is, the result of `QueryDataset` should include all the necessary information to send
+  // the actual chunk requests, which is the 2nd step of the query process.
+  //
+  // See `FetchChunksRequest` for details on the fields that describe each individual chunk.
+  rpc FetchChunks(FetchChunksRequest) returns (stream FetchChunksResponse) {}
+
   // --- Tables ---
   // TODO(jleibs): This will be replaced / extended by Arrow Flight
 
@@ -533,6 +544,21 @@ message GetChunksResponse {
   repeated rerun.log_msg.v1alpha1.ArrowMsg chunks = 1;
 }
 
+message FetchChunksRequest {
+  // Information about the chunks to fetch. These dataframes have to include the following columns:
+  // * `chunk_id` - Chunk unique identifier
+  // * `partition_id` - partition this chunk belongs to. Currently needed as we pass this metadata back and forth
+  // * `partition_layer` - specific partition layer. Currently needed as we pass this metadata back and forth
+  // * `chunk_key` - chunk location details
+  repeated rerun.common.v1alpha1.DataframePart chunk_infos = 1;
+}
+
+message FetchChunksResponse {
+  // Every gRPC response, even within the confines of a stream, involves HTTP2 overhead, which isn't
+  // cheap by any means, which is why we're returning a batch of `ArrowMsg` rather than a single one.
+  repeated rerun.log_msg.v1alpha1.ArrowMsg chunks = 1;
+}
+
 // --- Table Apis ---
 
 message GetTableSchemaRequest {

crates/store/re_protos/src/v1alpha1/rerun.cloud.v1alpha1.ext.rs

@@ -1255,7 +1255,7 @@ impl ScanPartitionTableResponse {
 // --- DataSource --
 
 // NOTE: Match the values of the Protobuf definition to keep life simple.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
 pub enum DataSourceKind {
     Rrd = 1,
 }