diff --git a/CHANGELOG.md b/CHANGELOG.md index a05a5ce..9622e12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +- add LAS/COPC-native column types in `copc-core`, materialized column reads in + `copc-reader`, and a `copc-writer` `ColumnBatchSource` adapter for writing + neutral `LasColumnBatch` values directly +- keep row iteration supported while documenting that column reads are owned, + materialized buffers decoded from compressed LAZ chunks, not zero-copy views + into COPC files +- keep Arrow/DataFusion conversion out of `copc-rust`; downstream engines can + adapt `LasColumnBatch` into their own models or a future optional feature can + add Arrow-specific conversion + ## 0.2.0 - 2026-06-10 - reject COPC files whose VLR/EVLR sections, hierarchy pages, or child hierarchy pages extend past EOF; cap VLR/EVLR counts at 4,096, one hierarchy page at 64 MiB, and recursively loaded hierarchy pages at 256 MiB; add truncation tests that assert errors instead of panics diff --git a/README.md b/README.md index 1c77252..3301d16 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ read-only memory mapping of writer spill files. | Crate | Description | |---|---| -| `copc-core` | Shared COPC metadata, hierarchy entries, voxel keys, bounds, streaming LAS records, and errors | -| `copc-reader` | COPC header/info parsing, recursive hierarchy access, and chunked-LAZ point iteration | -| `copc-writer` | COPC writer with source-trait point access, native LOD distribution, mmap spill support, and streaming LAS/LAZ intake | +| `copc-core` | Shared COPC metadata, hierarchy entries, voxel keys, bounds, LAS-native column batches, streaming LAS records, and errors | +| `copc-reader` | COPC header/info parsing, recursive hierarchy access, chunked-LAZ row iteration, and materialized column reads | +| `copc-writer` | COPC writer with source-trait point access, column-batch source support, native LOD distribution, mmap spill support, and streaming LAS/LAZ intake | ## Usage @@ -40,6 +40,26 @@ for point in reader.points(LodSelection::All, BoundsSelection::All)? { } ``` +```rust +use copc_core::{ColumnData, LasDimension}; +use copc_reader::{ColumnSelection, CopcReader, PointQuery}; + +let mut reader = CopcReader::from_path("cloud.copc.laz")?; +let batch = reader.read_columns( + PointQuery::all(), + ColumnSelection::from_dimensions([ + LasDimension::X, + LasDimension::Y, + LasDimension::Z, + LasDimension::Classification, + ]), +)?; + +if let Some(ColumnData::F64(xs)) = batch.column(LasDimension::X) { + println!("decoded {} x coordinates", xs.len()); +} +``` + ```rust use copc_writer::{convert_las_to_copc_streaming, CopcWriterParams}; @@ -52,20 +72,45 @@ convert_las_to_copc_streaming( )?; ``` +## Column Ownership Model + +`copc-core` owns the LAS/COPC-native column model: `LasDimension`, +`ColumnSpec`, `ColumnData`, `ColumnView`, `ColumnSelection`, and +`LasColumnBatch`. These types are dependency-light and do not depend on Arrow, +DataFusion, or engine-specific point-cloud crates. + +`copc-reader` exposes materialized column batches with +`CopcReader::read_columns` and `CopcReader::read_columns_with_cancel`. +Existing row iteration with `points`, `points_for_query`, and +`points_with_cancel` remains supported. + +The column API is materialized. COPC point data is still read from compressed +LAZ chunks, decoded, filtered, transformed, and appended into owned column +buffers. It is not a zero-copy view into compressed COPC files. + +Downstream engines should adapt `LasColumnBatch` into their own canonical +memory model. For example, `roteiro-engine` maps these native batches into its +`PointCloud` struct-of-arrays representation. Arrow conversion is intentionally +out of scope for `copc-rust` today; it belongs in downstream engine code or +behind a future optional feature. + ## Supported Now - Public COPC hierarchy types for availability, indexing, and tile serving - COPC info VLR and recursive hierarchy page parsing - Chunked-LAZ point iteration in `copc-reader` - All-points, LOD-selected, and bounds-selected reader point iteration +- Materialized LAS/COPC-native column batches in `copc-reader` - Source-trait writer API for caller-owned point storage +- COPC writing from neutral `LasColumnBatch` values via `ColumnBatchSource` - Streaming LAS/LAZ-to-COPC conversion through a disk-backed mmap spill - LAS 1.4 point formats 6 and 7 with LAZ variable-size chunks - Interior-node representative points for native LOD reads ## Not Yet Supported -- Materialized point-column convenience APIs +- Zero-copy column views directly over compressed COPC/LAZ point data +- Built-in Arrow or DataFusion conversion ## Testing diff --git a/copc-writer/src/lib.rs b/copc-writer/src/lib.rs index 25659fe..89cfed8 100644 --- a/copc-writer/src/lib.rs +++ b/copc-writer/src/lib.rs @@ -6,5 +6,6 @@ mod writer; pub use spill::{SpillReader, SpillWriter}; pub use writer::{ convert_las_to_copc_streaming, write_source, write_source_with_cancel, - write_streaming_with_cancel, CopcPointFields, CopcPointSource, CopcWriterParams, + write_streaming_with_cancel, ColumnBatchSource, CopcPointFields, CopcPointSource, + CopcWriterParams, }; diff --git a/copc-writer/src/writer.rs b/copc-writer/src/writer.rs index 3000798..7f4f485 100644 --- a/copc-writer/src/writer.rs +++ b/copc-writer/src/writer.rs @@ -4,8 +4,8 @@ use std::path::Path; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use copc_core::{ - Bounds, CancelCheck, CopcInfo, Entry, Error, LasPointRecord, NeverCancel, Result, - StreamingLayout, VoxelKey, HIERARCHY_ENTRY_BYTES, + Bounds, CancelCheck, ColumnData, CopcInfo, Entry, Error, LasColumnBatch, LasDimension, + LasPointRecord, NeverCancel, Result, StreamingLayout, VoxelKey, HIERARCHY_ENTRY_BYTES, }; use las::{point::Format as LasFormat, raw, Color, Read as _}; use laz::{LasZipCompressor, LazVlrBuilder}; @@ -58,6 +58,245 @@ pub trait CopcPointSource { } } +/// COPC writer source backed directly by a neutral LAS column batch. +pub struct ColumnBatchSource<'a> { + batch: &'a LasColumnBatch, + x: &'a [f64], + y: &'a [f64], + z: &'a [f64], + intensity: Option<&'a [u16]>, + return_number: Option<&'a [u8]>, + number_of_returns: Option<&'a [u8]>, + synthetic: Option<&'a [bool]>, + key_point: Option<&'a [bool]>, + withheld: Option<&'a [bool]>, + overlap: Option<&'a [bool]>, + scan_channel: Option<&'a [u8]>, + scan_direction_flag: Option<&'a [bool]>, + edge_of_flight_line: Option<&'a [bool]>, + classification: Option<&'a [u8]>, + user_data: Option<&'a [u8]>, + scan_angle_rank: Option<&'a [i16]>, + point_source_id: Option<&'a [u16]>, + gps_time: Option<&'a [f64]>, + red: Option<&'a [u16]>, + green: Option<&'a [u16]>, + blue: Option<&'a [u16]>, +} + +impl<'a> ColumnBatchSource<'a> { + pub fn new(batch: &'a LasColumnBatch) -> Result { + batch.validate()?; + validate_column_batch_writer_support(batch)?; + + let x = required_f64_column(batch, LasDimension::X)?; + let y = required_f64_column(batch, LasDimension::Y)?; + let z = required_f64_column(batch, LasDimension::Z)?; + let red = optional_u16_column(batch, LasDimension::Red)?; + let green = optional_u16_column(batch, LasDimension::Green)?; + let blue = optional_u16_column(batch, LasDimension::Blue)?; + validate_color_columns(red, green, blue)?; + + Ok(Self { + batch, + x, + y, + z, + intensity: optional_u16_column(batch, LasDimension::Intensity)?, + return_number: optional_u8_column(batch, LasDimension::ReturnNumber)?, + number_of_returns: optional_u8_column(batch, LasDimension::NumberOfReturns)?, + synthetic: optional_bool_column(batch, LasDimension::Synthetic)?, + key_point: optional_bool_column(batch, LasDimension::KeyPoint)?, + withheld: optional_bool_column(batch, LasDimension::Withheld)?, + overlap: optional_bool_column(batch, LasDimension::Overlap)?, + scan_channel: optional_u8_column(batch, LasDimension::ScanChannel)?, + scan_direction_flag: optional_bool_column(batch, LasDimension::ScanDirectionFlag)?, + edge_of_flight_line: optional_bool_column(batch, LasDimension::EdgeOfFlightLine)?, + classification: optional_u8_column(batch, LasDimension::Classification)?, + user_data: optional_u8_column(batch, LasDimension::UserData)?, + scan_angle_rank: optional_i16_column(batch, LasDimension::ScanAngleRank)?, + point_source_id: optional_u16_column(batch, LasDimension::PointSourceId)?, + gps_time: optional_f64_column(batch, LasDimension::GpsTime)?, + red, + green, + blue, + }) + } + + pub fn batch(&self) -> &LasColumnBatch { + self.batch + } + + pub fn has_color(&self) -> bool { + self.red.is_some() && self.green.is_some() && self.blue.is_some() + } + + pub fn bounds(&self) -> Result { + if self.is_empty() { + return Err(Error::InvalidInput( + "cannot compute bounds for empty column batch".into(), + )); + } + let mut bounds = Bounds::point(self.x[0], self.y[0], self.z[0]); + for index in 1..self.len() { + bounds.extend(self.x[index], self.y[index], self.z[index]); + } + Ok(bounds) + } +} + +impl CopcPointSource for ColumnBatchSource<'_> { + fn len(&self) -> usize { + self.batch.len() + } + + #[inline] + fn xyz(&self, index: usize) -> (f64, f64, f64) { + (self.x[index], self.y[index], self.z[index]) + } + + fn fields(&self, index: usize) -> Result { + Ok(CopcPointFields { + x: self.x[index], + y: self.y[index], + z: self.z[index], + intensity: at_u16(self.intensity, index), + return_number: at_u8(self.return_number, index), + number_of_returns: at_u8(self.number_of_returns, index), + synthetic: at_bool_u8(self.synthetic, index), + key_point: at_bool_u8(self.key_point, index), + withheld: at_bool_u8(self.withheld, index), + overlap: at_bool_u8(self.overlap, index), + scan_channel: at_u8(self.scan_channel, index), + scan_direction_flag: at_bool_u8(self.scan_direction_flag, index), + edge_of_flight_line: at_bool_u8(self.edge_of_flight_line, index), + classification: at_u8(self.classification, index), + user_data: at_u8(self.user_data, index), + scan_angle: self + .scan_angle_rank + .map(|column| column[index] as f32 * 90.0 / 180.0) + .unwrap_or(0.0), + point_source_id: at_u16(self.point_source_id, index), + gps_time: self.gps_time.map(|column| column[index]).unwrap_or(0.0), + red: at_u16(self.red, index), + green: at_u16(self.green, index), + blue: at_u16(self.blue, index), + }) + } +} + +fn at_bool_u8(column: Option<&[bool]>, index: usize) -> u8 { + column.map(|values| u8::from(values[index])).unwrap_or(0) +} + +fn at_u8(column: Option<&[u8]>, index: usize) -> u8 { + column.map(|values| values[index]).unwrap_or(0) +} + +fn at_u16(column: Option<&[u16]>, index: usize) -> u16 { + column.map(|values| values[index]).unwrap_or(0) +} + +fn validate_column_batch_writer_support(batch: &LasColumnBatch) -> Result<()> { + let unsupported: Vec<_> = batch + .columns + .iter() + .filter_map(|(spec, _)| match spec.dimension { + LasDimension::Nir => Some("NIR point data"), + LasDimension::WaveformPacketDescriptorIndex + | LasDimension::WaveformPacketByteOffset + | LasDimension::WaveformPacketSize + | LasDimension::WavePacketReturnPointWaveformLocation => Some("waveform point data"), + LasDimension::ExtraBytes => Some("extra point bytes"), + _ => None, + }) + .collect(); + if unsupported.is_empty() { + Ok(()) + } else { + Err(Error::Unsupported(format!( + "COPC writer cannot preserve {}", + unsupported.join(", ") + ))) + } +} + +fn validate_color_columns( + red: Option<&[u16]>, + green: Option<&[u16]>, + blue: Option<&[u16]>, +) -> Result<()> { + let present = + usize::from(red.is_some()) + usize::from(green.is_some()) + usize::from(blue.is_some()); + if present == 0 || present == 3 { + Ok(()) + } else { + Err(Error::InvalidInput( + "Red, Green, and Blue columns must be supplied together".into(), + )) + } +} + +fn required_f64_column(batch: &LasColumnBatch, dimension: LasDimension) -> Result<&[f64]> { + match batch.column(dimension) { + Some(ColumnData::F64(values)) => Ok(values), + Some(other) => Err(unexpected_column_type(dimension, "F64", other)), + None => Err(Error::InvalidInput(format!( + "ColumnBatchSource requires {dimension:?} column" + ))), + } +} + +fn optional_f64_column(batch: &LasColumnBatch, dimension: LasDimension) -> Result> { + match batch.column(dimension) { + Some(ColumnData::F64(values)) => Ok(Some(values)), + Some(other) => Err(unexpected_column_type(dimension, "F64", other)), + None => Ok(None), + } +} + +fn optional_i16_column(batch: &LasColumnBatch, dimension: LasDimension) -> Result> { + match batch.column(dimension) { + Some(ColumnData::I16(values)) => Ok(Some(values)), + Some(other) => Err(unexpected_column_type(dimension, "I16", other)), + None => Ok(None), + } +} + +fn optional_u16_column(batch: &LasColumnBatch, dimension: LasDimension) -> Result> { + match batch.column(dimension) { + Some(ColumnData::U16(values)) => Ok(Some(values)), + Some(other) => Err(unexpected_column_type(dimension, "U16", other)), + None => Ok(None), + } +} + +fn optional_u8_column(batch: &LasColumnBatch, dimension: LasDimension) -> Result> { + match batch.column(dimension) { + Some(ColumnData::U8(values)) => Ok(Some(values)), + Some(other) => Err(unexpected_column_type(dimension, "U8", other)), + None => Ok(None), + } +} + +fn optional_bool_column( + batch: &LasColumnBatch, + dimension: LasDimension, +) -> Result> { + match batch.column(dimension) { + Some(ColumnData::Bool(values)) => Ok(Some(values)), + Some(other) => Err(unexpected_column_type(dimension, "Bool", other)), + None => Ok(None), + } +} + +fn unexpected_column_type(dimension: LasDimension, expected: &str, actual: &ColumnData) -> Error { + Error::InvalidInput(format!( + "{dimension:?} column must be {expected}, found {:?}", + actual.scalar() + )) +} + struct SpillSource<'a> { reader: &'a SpillReader, } diff --git a/copc-writer/tests/column_read.rs b/copc-writer/tests/column_read.rs index 3d6f779..95bf0f9 100644 --- a/copc-writer/tests/column_read.rs +++ b/copc-writer/tests/column_read.rs @@ -1,6 +1,8 @@ -use copc_core::{Bounds, ColumnData, LasColumnBatch, LasDimension}; +use copc_core::{Bounds, ColumnData, ColumnSpec, LasColumnBatch, LasDimension}; use copc_reader::{BoundsSelection, ColumnSelection, CopcReader, LodSelection, PointQuery}; -use copc_writer::{write_source, CopcPointFields, CopcPointSource, CopcWriterParams}; +use copc_writer::{ + write_source, ColumnBatchSource, CopcPointFields, CopcPointSource, CopcWriterParams, +}; struct VecSource { points: Vec, @@ -177,6 +179,35 @@ fn read_columns_matches_synthetic_copc_rows() { assert_eq!(lod_columns.len(), lod_points.len()); } +#[test] +fn column_batch_source_writes_columns_readable_by_reader() { + let batch = column_batch(384); + let source = ColumnBatchSource::new(&batch).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("column-batch-source.copc.laz"); + + write_source( + &path, + &source, + source.has_color(), + source.bounds().unwrap(), + &CopcWriterParams { + max_points_per_node: 1_024, + max_depth: 4, + }, + ) + .unwrap(); + + let roundtripped = read_columns(&path, PointQuery::all(), ColumnSelection::all()); + assert_eq!(roundtripped.len(), batch.len()); + for (spec, expected) in &batch.columns { + let actual = roundtripped + .column(spec.dimension) + .unwrap_or_else(|| panic!("missing {:?} column", spec.dimension)); + assert_column_data_eq(spec.dimension, expected, actual); + } +} + fn read_rows(path: &std::path::Path, query: PointQuery) -> Vec { let mut reader = CopcReader::from_path(path).unwrap(); reader @@ -228,6 +259,101 @@ fn grid_points(count: usize) -> Vec { .collect() } +fn column_batch(count: usize) -> LasColumnBatch { + let mut x = Vec::with_capacity(count); + let mut y = Vec::with_capacity(count); + let mut z = Vec::with_capacity(count); + let mut intensity = Vec::with_capacity(count); + let mut return_number = Vec::with_capacity(count); + let mut number_of_returns = Vec::with_capacity(count); + let mut classification = Vec::with_capacity(count); + let mut scan_direction_flag = Vec::with_capacity(count); + let mut edge_of_flight_line = Vec::with_capacity(count); + let mut scan_angle_rank = Vec::with_capacity(count); + let mut user_data = Vec::with_capacity(count); + let mut point_source_id = Vec::with_capacity(count); + let mut synthetic = Vec::with_capacity(count); + let mut key_point = Vec::with_capacity(count); + let mut withheld = Vec::with_capacity(count); + let mut overlap = Vec::with_capacity(count); + let mut scan_channel = Vec::with_capacity(count); + let mut gps_time = Vec::with_capacity(count); + let mut red = Vec::with_capacity(count); + let mut green = Vec::with_capacity(count); + let mut blue = Vec::with_capacity(count); + + for i in 0..count { + x.push((i % 24) as f64 * 0.5 - 6.0); + y.push(((i / 24) % 16) as f64 * 0.25 - 2.0); + z.push((i / (24 * 16)) as f64 * 0.125 + 10.0); + intensity.push((20_000 + i) as u16); + return_number.push(((i % 4) + 1) as u8); + number_of_returns.push(4); + classification.push(if i % 2 == 0 { 2 } else { 6 }); + scan_direction_flag.push(i % 2 == 0); + edge_of_flight_line.push(i % 5 == 0); + scan_angle_rank.push((i as i16 % 181) - 90); + user_data.push((i % 251) as u8); + point_source_id.push((1_000 + i) as u16); + synthetic.push(i % 7 == 0); + key_point.push(i % 11 == 0); + withheld.push(i % 13 == 0); + overlap.push(i % 17 == 0); + scan_channel.push((i % 4) as u8); + gps_time.push(1.0e9 + i as f64 * 0.25); + red.push((i * 3) as u16); + green.push((i * 5) as u16); + blue.push((i * 7) as u16); + } + + LasColumnBatch::new(vec![ + column(LasDimension::X, ColumnData::F64(x)), + column(LasDimension::Y, ColumnData::F64(y)), + column(LasDimension::Z, ColumnData::F64(z)), + column(LasDimension::Intensity, ColumnData::U16(intensity)), + column(LasDimension::ReturnNumber, ColumnData::U8(return_number)), + column( + LasDimension::NumberOfReturns, + ColumnData::U8(number_of_returns), + ), + column(LasDimension::Classification, ColumnData::U8(classification)), + column( + LasDimension::ScanDirectionFlag, + ColumnData::Bool(scan_direction_flag), + ), + column( + LasDimension::EdgeOfFlightLine, + ColumnData::Bool(edge_of_flight_line), + ), + column( + LasDimension::ScanAngleRank, + ColumnData::I16(scan_angle_rank), + ), + column(LasDimension::UserData, ColumnData::U8(user_data)), + column( + LasDimension::PointSourceId, + ColumnData::U16(point_source_id), + ), + column(LasDimension::Synthetic, ColumnData::Bool(synthetic)), + column(LasDimension::KeyPoint, ColumnData::Bool(key_point)), + column(LasDimension::Withheld, ColumnData::Bool(withheld)), + column(LasDimension::Overlap, ColumnData::Bool(overlap)), + column(LasDimension::ScanChannel, ColumnData::U8(scan_channel)), + column(LasDimension::GpsTime, ColumnData::F64(gps_time)), + column(LasDimension::Red, ColumnData::U16(red)), + column(LasDimension::Green, ColumnData::U16(green)), + column(LasDimension::Blue, ColumnData::U16(blue)), + ]) + .unwrap() +} + +fn column(dimension: LasDimension, data: ColumnData) -> (ColumnSpec, ColumnData) { + ( + ColumnSpec::default_for(dimension).expect("fixed LAS column spec"), + data, + ) +} + fn source_bounds(points: &[CopcPointFields]) -> Bounds { points.iter().fold( Bounds::point(points[0].x, points[0].y, points[0].z), @@ -267,3 +393,52 @@ fn column_u8(batch: &LasColumnBatch, dimension: LasDimension) -> &[u8] { ), } } + +fn assert_column_data_eq(dimension: LasDimension, expected: &ColumnData, actual: &ColumnData) { + match (expected, actual) { + (ColumnData::F64(expected), ColumnData::F64(actual)) => { + assert_eq!(expected.len(), actual.len(), "{dimension:?} length"); + for (index, (&expected, &actual)) in expected.iter().zip(actual).enumerate() { + assert!( + (expected - actual).abs() <= 1e-9, + "{dimension:?} differs at row {index}: expected {expected}, got {actual}" + ); + } + } + (ColumnData::F32(expected), ColumnData::F32(actual)) => { + assert_eq!(expected, actual, "{dimension:?}"); + } + (ColumnData::I64(expected), ColumnData::I64(actual)) => { + assert_eq!(expected, actual, "{dimension:?}"); + } + (ColumnData::I32(expected), ColumnData::I32(actual)) => { + assert_eq!(expected, actual, "{dimension:?}"); + } + (ColumnData::I16(expected), ColumnData::I16(actual)) => { + assert_eq!(expected, actual, "{dimension:?}"); + } + (ColumnData::I8(expected), ColumnData::I8(actual)) => { + assert_eq!(expected, actual, "{dimension:?}"); + } + (ColumnData::U64(expected), ColumnData::U64(actual)) => { + assert_eq!(expected, actual, "{dimension:?}"); + } + (ColumnData::U32(expected), ColumnData::U32(actual)) => { + assert_eq!(expected, actual, "{dimension:?}"); + } + (ColumnData::U16(expected), ColumnData::U16(actual)) => { + assert_eq!(expected, actual, "{dimension:?}"); + } + (ColumnData::U8(expected), ColumnData::U8(actual)) => { + assert_eq!(expected, actual, "{dimension:?}"); + } + (ColumnData::Bool(expected), ColumnData::Bool(actual)) => { + assert_eq!(expected, actual, "{dimension:?}"); + } + _ => panic!( + "{dimension:?} scalar mismatch: expected {:?}, got {:?}", + expected.scalar(), + actual.scalar() + ), + } +}