Merge branch 'main' into grtlr/hybrid-latest-at-tagged

Author: grtlr

.typos.toml

@@ -6,9 +6,10 @@
 extend-exclude = [
   ".typos.toml",
   "*.snap",
-  "crates/viewer/re_ui/data/design_tokens.json",
   "crates/viewer/re_ui/src/design_tokens.rs",
   "examples/assets",
+  "rerun_cpp/src/rerun/archetypes/image.hpp",     # TODO(emilk): remove once we remove from_greyscale8
+  "rerun_cpp/src/rerun/archetypes/image_ext.cpp", # TODO(emilk): remove once we remove from_greyscale8
   "rerun_cpp/src/rerun/third_party/cxxopts.hpp",
 ]
 
@@ -74,6 +75,8 @@ flavour = "flavor"
 fulfil = "fufill"
 gaol = "jail"
 grey = "gray"
+greys = "grays"
+greyscale = "grayscale"
 harbour = "habor"
 honour = "honor"
 humour = "humor"

CHANGELOG.md

@@ -1259,7 +1259,7 @@ _All four tetrahedron meshes on this screen share the same vertices and are inst
 - Add an explicit "mode" view property to the dataframe view [#6927](https://github.com/rerun-io/rerun/pull/6927)
 - Introduce a "Selectable Toggle" widget and use it for the 3D view's camera kind [#7064](https://github.com/rerun-io/rerun/pull/7064)
 - Improve entity stats when hovered [#7074](https://github.com/rerun-io/rerun/pull/7074)
-- Update the UI colors to use our (blueish) ramp instead of pure greys [#7075](https://github.com/rerun-io/rerun/pull/7075)
+- Update the UI colors to use our (blueish) ramp instead of pure grays [#7075](https://github.com/rerun-io/rerun/pull/7075)
 - Query editor for the dataframe view [#7071](https://github.com/rerun-io/rerun/pull/7071)
 - Better ui for `Blob`s, especially those representing images [#7128](https://github.com/rerun-io/rerun/pull/7128)
 - Add button for copying and saving images [#7156](https://github.com/rerun-io/rerun/pull/7156)

Cargo.lock

@@ -8123,15 +8123,13 @@ dependencies = [
  "once_cell",
  "parking_lot",
  "pollster 0.4.0",
- "re_build_info",
  "re_capabilities",
  "re_chunk",
  "re_chunk_store",
  "re_data_source",
  "re_entity_db",
  "re_format",
  "re_log",
- "re_log_encoding",
  "re_log_types",
  "re_math",
  "re_query",

Cargo.toml

@@ -378,6 +378,83 @@ opt-level = 1 # Make debug builds run faster
 # Use the `debugging` profile (see below) if you need debug symbols.
 debug = false
 
+
+# Use the following command to grab the list of workspace crates:
+#   cargo metadata --no-deps --format-version 1 | jq '.packages[].name' | rg '"re_'
+#
+# There is, as far we know, no way to do this without enumerating all crates manually.
+
+[profile.dev.package]
+"re_analytics".debug = true
+"re_arrow_util".debug = true
+"re_auth".debug = true
+"re_blueprint_tree".debug = true
+"re_build_info".debug = true
+"re_build_tools".debug = true
+"re_byte_size".debug = true
+"re_capabilities".debug = true
+"re_case".debug = true
+"re_chunk".debug = true
+"re_chunk_store".debug = true
+"re_chunk_store_ui".debug = true
+"re_component_ui".debug = true
+"re_context_menu".debug = true
+"re_crash_handler".debug = true
+"re_data_loader".debug = true
+"re_data_source".debug = true
+"re_data_ui".debug = true
+"re_dataframe".debug = true
+"re_dataframe_ui".debug = true
+"re_datafusion".debug = true
+"re_dev_tools".debug = true
+"re_entity_db".debug = true
+"re_error".debug = true
+"re_format".debug = true
+"re_format_arrow".debug = true
+"re_grpc_client".debug = true
+"re_grpc_server".debug = true
+"re_int_histogram".debug = true
+"re_log".debug = true
+"re_log_encoding".debug = true
+"re_log_types".debug = true
+"re_memory".debug = true
+"re_protos".debug = true
+"re_protos_builder".debug = true
+"re_query".debug = true
+"re_redap_browser".debug = true
+"re_renderer".debug = true
+"re_renderer_examples".debug = true
+"re_sdk".debug = true
+"re_selection_panel".debug = true
+"re_smart_channel".debug = true
+"re_sorbet".debug = true
+"re_string_interner".debug = true
+"re_time_panel".debug = true
+"re_tracing".debug = true
+"re_tuid".debug = true
+"re_types".debug = true
+"re_types_builder".debug = true
+"re_types_core".debug = true
+"re_ui".debug = true
+"re_uri".debug = true
+"re_video".debug = true
+"re_view".debug = true
+"re_view_bar_chart".debug = true
+"re_view_dataframe".debug = true
+"re_view_graph".debug = true
+"re_view_map".debug = true
+"re_view_spatial".debug = true
+"re_view_tensor".debug = true
+"re_view_text_document".debug = true
+"re_view_text_log".debug = true
+"re_view_time_series".debug = true
+"re_viewer".debug = true
+"re_viewer_context".debug = true
+"re_viewport".debug = true
+"re_viewport_blueprint".debug = true
+"re_web_viewer_server".debug = true
+
+
 # panic = "abort" leads to better optimizations and smaller binaries (and is the default in Wasm anyways),
 # but it also means backtraces don't work with the `backtrace` library (https://github.com/rust-lang/backtrace-rs/issues/397).
 # egui has a feature where if you hold down all modifiers keys on your keyboard and hover any UI widget,

crates/store/re_data_loader/src/loader_lerobot.rs

@@ -1,26 +1,32 @@
-use std::sync::Arc;
-use std::sync::mpsc::Sender;
-use std::thread;
+use std::{
+    sync::{Arc, mpsc::Sender},
+    thread,
+};
 
 use anyhow::{Context as _, anyhow};
-use arrow::array::{
-    ArrayRef, BinaryArray, FixedSizeListArray, Int64Array, RecordBatch, StringArray, StructArray,
+use arrow::{
+    array::{
+        ArrayRef, BinaryArray, FixedSizeListArray, Int64Array, RecordBatch, StringArray,
+        StructArray,
+    },
+    compute::cast,
+    datatypes::{DataType, Field},
 };
-use arrow::compute::cast;
-use arrow::datatypes::{DataType, Field};
 use itertools::Either;
+
 use re_arrow_util::ArrowArrayDowncastRef as _;
 use re_chunk::{
     ArrowArray, Chunk, ChunkId, EntityPath, RowId, TimeColumn, TimeInt, TimePoint, Timeline,
+    TimelineName, external::nohash_hasher::IntMap,
 };
-use re_chunk::{TimelineName, external::nohash_hasher::IntMap};
-
 use re_log_types::{ApplicationId, StoreId};
-use re_types::archetypes::{
-    AssetVideo, DepthImage, EncodedImage, TextDocument, VideoFrameReference,
+use re_types::{
+    Archetype, Component, ComponentBatch,
+    archetypes::{
+        AssetVideo, DepthImage, EncodedImage, Scalars, TextDocument, VideoFrameReference,
+    },
+    components::{Name, VideoTimestamp},
 };
-use re_types::components::{Name, Scalar, VideoTimestamp};
-use re_types::{Archetype, Component, ComponentBatch};
 
 use crate::lerobot::{
     DType, EpisodeIndex, Feature, LeRobotDataset, TaskIndex, is_lerobot_dataset,
@@ -503,7 +509,23 @@ fn load_scalar(
                 make_scalar_batch_entity_chunks(entity_path, feature, timelines, fixed_size_array)?;
             Ok(ScalarChunkIterator::Batch(Box::new(batch_chunks)))
         }
-        DataType::Float32 => {
+        DataType::List(_field) => {
+            let list_array = data
+                .column_by_name(feature_key)
+                .and_then(|col| col.downcast_array_ref::<arrow::array::ListArray>())
+                .ok_or_else(|| {
+                    DataLoaderError::Other(anyhow!("Failed to downcast feature to ListArray"))
+                })?;
+
+            let sliced = extract_list_array_elements_as_f64(list_array).with_context(|| {
+                format!("Failed to cast scalar feature {entity_path} to Float64")
+            })?;
+
+            Ok(ScalarChunkIterator::Single(std::iter::once(
+                make_scalar_entity_chunk(entity_path, timelines, &sliced)?,
+            )))
+        }
+        DataType::Float32 | DataType::Float64 => {
             let feature_data = data.column_by_name(feature_key).ok_or_else(|| {
                 DataLoaderError::Other(anyhow!(
                     "Failed to get LeRobot dataset column data for: {:?}",
@@ -540,7 +562,7 @@ fn make_scalar_batch_entity_chunks(
 
     let mut chunks = Vec::with_capacity(num_elements);
 
-    let sliced = extract_list_elements_as_f64(data)
+    let sliced = extract_fixed_size_list_array_elements_as_f64(data)
         .with_context(|| format!("Failed to cast scalar feature {entity_path} to Float64"))?;
 
     chunks.push(make_scalar_entity_chunk(
@@ -592,11 +614,7 @@ fn make_scalar_entity_chunk(
         ChunkId::new(),
         entity_path,
         timelines.clone(),
-        std::iter::once((
-            <Scalar as Component>::descriptor().clone(),
-            data_field_array,
-        ))
-        .collect(),
+        std::iter::once((Scalars::descriptor_scalars().clone(), data_field_array)).collect(),
     )?)
 }
 
@@ -610,7 +628,20 @@ fn extract_scalar_slices_as_f64(data: &ArrayRef) -> anyhow::Result<Vec<ArrayRef>
         .collect::<Vec<_>>())
 }
 
-fn extract_list_elements_as_f64(data: &FixedSizeListArray) -> anyhow::Result<Vec<ArrayRef>> {
+fn extract_fixed_size_list_array_elements_as_f64(
+    data: &FixedSizeListArray,
+) -> anyhow::Result<Vec<ArrayRef>> {
+    (0..data.len())
+        .map(|idx| {
+            cast(&data.value(idx), &DataType::Float64)
+                .with_context(|| format!("Failed to cast {:?} to Float64", data.data_type()))
+        })
+        .collect::<Result<Vec<_>, _>>()
+}
+
+fn extract_list_array_elements_as_f64(
+    data: &arrow::array::ListArray,
+) -> anyhow::Result<Vec<ArrayRef>> {
     (0..data.len())
         .map(|idx| {
             cast(&data.value(idx), &DataType::Float64)

crates/store/re_entity_db/src/lib.rs

@@ -7,6 +7,7 @@
 pub mod entity_db;
 pub mod entity_tree;
 mod instance_path;
+mod store_bundle;
 mod time_histogram_per_timeline;
 mod times_per_timeline;
 mod versioned_instance_path;
@@ -15,6 +16,7 @@ pub use self::{
     entity_db::{DEFAULT_GC_TIME_BUDGET, EntityDb},
     entity_tree::EntityTree,
     instance_path::{InstancePath, InstancePathHash},
+    store_bundle::{StoreBundle, StoreLoadError},
     time_histogram_per_timeline::{TimeHistogram, TimeHistogramPerTimeline},
     times_per_timeline::{TimeCounts, TimesPerTimeline},
     versioned_instance_path::{VersionedInstancePath, VersionedInstancePathHash},

crates/viewer/re_viewer_context/src/store_bundle.rs renamed to crates/store/re_entity_db/src/store_bundle.rs

@@ -1,15 +1,16 @@
 use itertools::Itertools as _;
 
-use re_entity_db::EntityDb;
 use re_log_types::{StoreId, StoreKind};
 
+use crate::EntityDb;
+
 #[derive(thiserror::Error, Debug)]
 pub enum StoreLoadError {
     #[error(transparent)]
     Decode(#[from] re_log_encoding::decoder::DecodeError),
 
     #[error(transparent)]
-    ChunkStore(#[from] re_entity_db::Error),
+    ChunkStore(#[from] crate::Error),
 }
 
 /// Stores many [`EntityDb`]s of recordings and blueprints.

crates/utils/re_log/src/lib.rs

@@ -71,11 +71,15 @@ const CRATES_AT_WARN_LEVEL: &[&str] = &[
 ];
 
 /// Never log anything less serious than a `INFO` from these crates.
+///
+/// These creates are quite spammy on debug, drowning out what we care about:
 const CRATES_AT_INFO_LEVEL: &[&str] = &[
-    // These are quite spammy on debug, drowning out what we care about:
+    "datafusion_optimizer",
+    "datafusion",
     "h2",
     "hyper",
     "prost_build",
+    "sqlparser",
     "tower",
     "ureq",
     // only let rustls log in debug mode: https://github.com/rerun-io/rerun/issues/3104

crates/utils/re_video/src/decode/mod.rs

@@ -14,7 +14,7 @@
 //!   * is either 8 or 16
 //!   * that's how the decoder stores for us but the per component we have either 8 or 10 or 12 bits -> see `picture.bits_per_component()`
 //! * `picture.pixel_layout()`
-//!   * `4:0:0` greyscale
+//!   * `4:0:0` grayscale
 //!   * `4:2:0` half horizontal and half vertical resolution for chroma
 //!   * `4:2:2` half horizontal resolution for chroma
 //!   * `4:4:4` full resolution for chroma

crates/viewer/re_blueprint_tree/src/blueprint_tree.rs

@@ -1140,10 +1140,7 @@ fn add_new_view_or_container_menu_button(
     ui: &mut egui::Ui,
 ) {
     if ui
-        .add(egui::Button::image_and_text(
-            &re_ui::icons::ADD,
-            "Add view or container…",
-        ))
+        .add(re_ui::icons::ADD.as_button_with_label(ui.design_tokens(), "Add view or container…"))
         .clicked()
     {
         ui.close();
@@ -1188,7 +1185,8 @@ fn set_blueprint_to_default_menu_buttons(ctx: &ViewerContext<'_>, ui: &mut egui:
     let mut response = ui
         .add_enabled(
             enabled,
-            egui::Button::image_and_text(&re_ui::icons::RESET, "Reset to default blueprint"),
+            re_ui::icons::RESET
+                .as_button_with_label(ui.design_tokens(), "Reset to default blueprint"),
         )
         .on_hover_text("Reset to the default blueprint for this app");
 
@@ -1214,7 +1212,8 @@ fn set_blueprint_to_auto_menu_button(ctx: &ViewerContext<'_>, ui: &mut egui::Ui)
     if ui
         .add_enabled(
             enabled,
-            egui::Button::image_and_text(&re_ui::icons::RESET, "Reset to heuristic blueprint"),
+            re_ui::icons::RESET
+                .as_button_with_label(ui.design_tokens(), "Reset to heuristic blueprint"),
         )
         .on_hover_text("Re-populate viewport with automatically chosen views")
         .clicked()