Spec structure improvements (#1984)

Mostly moves around content to make the `format` section more focused on
the actual format definitions.

Also adds hooks in the flatbuffers files that we can use to inline
sections in the docs (following @ianhi 's approach in a previous PR)

cc @ianhi

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: TomNicholas

icechunk-format/flatbuffers/common.fbs

@@ -1,15 +1,20 @@
 namespace generated;
 
+// --8<-- [start:object_id_12]
 // used for SnapshotIds, ChunkIds, etc
 struct ObjectId12 {
   bytes:[uint8:12];
 }
+// --8<-- [end:object_id_12]
 
+// --8<-- [start:object_id_8]
 // used for NodeIds
 struct ObjectId8 {
   bytes:[uint8:8];
 }
+// --8<-- [end:object_id_8]
 
+// --8<-- [start:metadata_item]
 // a single key-value of snapshot metadata
 table MetadataItem {
   // the name of the attribute
@@ -19,3 +24,4 @@ table MetadataItem {
   // TODO: better serialization format
   value: [uint8] (required);
 }
+// --8<-- [end:metadata_item]

icechunk-format/flatbuffers/manifest.fbs

@@ -2,6 +2,7 @@ include "common.fbs";
 
 namespace generated;
 
+// --8<-- [start:chunk_ref]
 // We don't use unions and datastructures for the different types of refs
 // If we do that, the manifest grows in size a lot, because of the extra
 // offsets needed. This makes the code more complex because we need to
@@ -44,7 +45,9 @@ table ChunkRef {
   // Introduced in spec version 2
   extra: [uint8];
 }
+// --8<-- [end:chunk_ref]
 
+// --8<-- [start:array_manifest]
 table ArrayManifest {
   // the id of the node the chunk refs belong to
   node_id: ObjectId8 (required);
@@ -57,7 +60,9 @@ table ArrayManifest {
   // Introduced in spec version 2
   extra: [uint8];
 }
+// --8<-- [end:array_manifest]
 
+// --8<-- [start:manifest_table]
 table Manifest {
   // the manifest id
   id: ObjectId12 (required);
@@ -79,5 +84,6 @@ table Manifest {
   // Introduced in spec version 2
   extra: [uint8];
 }
+// --8<-- [end:manifest_table]
 
 root_type Manifest;

icechunk-format/flatbuffers/repo.fbs

@@ -2,11 +2,14 @@ include "common.fbs";
 
 namespace generated;
 
+// --8<-- [start:ref]
 table Ref {
     name: string (required);
     snapshot_index: uint32;
 }
+// --8<-- [end:ref]
 
+// --8<-- [start:snapshot_info]
 table SnapshotInfo {
     id: ObjectId12 (required);
     // -1 means no parent
@@ -16,15 +19,21 @@ table SnapshotInfo {
     message: string (required);
     metadata: [MetadataItem];
 }
+// --8<-- [end:snapshot_info]
 
+// --8<-- [start:repo_availability]
 enum RepoAvailability : ubyte { Online = 0, ReadOnly, Offline }
+// --8<-- [end:repo_availability]
 
+// --8<-- [start:repo_status]
 table RepoStatus {
     availability: RepoAvailability;
     set_at: uint64;
     limited_availability_reason: string;
 }
+// --8<-- [end:repo_status]
 
+// --8<-- [start:update_types]
 table RepoInitializedUpdate {
 }
 table RepoMigratedUpdate {
@@ -78,7 +87,9 @@ table FeatureFlagChangedUpdate {
     new_value: bool;
     is_set: bool;
 }
+// --8<-- [end:update_types]
 
+// --8<-- [start:update_type_union]
 union UpdateType {
   RepoInitializedUpdate,
   RepoMigratedUpdate,
@@ -97,7 +108,9 @@ union UpdateType {
   FeatureFlagChangedUpdate,
   RepoStatusChangedUpdate,
 }
+// --8<-- [end:update_type_union]
 
+// --8<-- [start:update]
 table Update {
     update_type: UpdateType (required);
 
@@ -107,7 +120,9 @@ table Update {
     // on updates the repo object is backed up
     backup_path: string;
   }
+// --8<-- [end:update]
 
+// --8<-- [start:repo_table]
 table Repo {
 
   spec_version: uint8;
@@ -145,5 +160,6 @@ table Repo {
   // Introduced in spec version 2
   extra: [uint8];
 }
+// --8<-- [end:repo_table]
 
 root_type Repo;

icechunk-format/flatbuffers/snapshot.fbs

@@ -2,6 +2,7 @@ include "common.fbs";
 
 namespace generated;
 
+// --8<-- [start:manifest_file_info]
 // a pointer to a manifest file
 struct ManifestFileInfo {
     // id of the object in the repo's object store
@@ -13,7 +14,9 @@ struct ManifestFileInfo {
     // number of chunk refs in the manifest
     num_chunk_refs: uint32;
 }
+// --8<-- [end:manifest_file_info]
 
+// --8<-- [start:manifest_file_info_v2]
 // table version of ManifestFileInfo, allowing optional extra data
 // introduced in spec V2
 table ManifestFileInfoV2 {
@@ -29,7 +32,9 @@ table ManifestFileInfoV2 {
     // Reserved for future use. Opaque byte vector for extensibility.
     extra: [uint8];
 }
+// --8<-- [end:manifest_file_info_v2]
 
+// --8<-- [start:chunk_index_range]
 // A range of chunk indexes
 struct ChunkIndexRange {
   // inclusive
@@ -38,7 +43,9 @@ struct ChunkIndexRange {
   // exclusive
   to: uint32;
 }
+// --8<-- [end:chunk_index_range]
 
+// --8<-- [start:manifest_ref]
 // a pointer to a manifest
 table ManifestRef {
     // id of the object in the repo's object store
@@ -47,7 +54,9 @@ table ManifestRef {
     // one element per dimension of the array, same order as in metadata
     extents: [ChunkIndexRange] (required);
 }
+// --8<-- [end:manifest_ref]
 
+// --8<-- [start:dimension_shape]
 // the shape of the array along a given dimension
 struct DimensionShape {
   array_length: uint64;
@@ -60,20 +69,28 @@ struct DimensionShape {
   // It was only ever used to estimate `num_chunks`.
   chunk_length: uint64;
 }
+// --8<-- [end:dimension_shape]
 
+// --8<-- [start:dimension_shape_v2]
 table DimensionShapeV2 {
   array_length: uint64;
   num_chunks: uint32;
 }
+// --8<-- [end:dimension_shape_v2]
 
+// --8<-- [start:dimension_name]
 table DimensionName {
   // optional
   name: string;
 }
+// --8<-- [end:dimension_name]
 
+// --8<-- [start:group_node_data]
 // a marker for a group node
 table GroupNodeData {}
+// --8<-- [end:group_node_data]
 
+// --8<-- [start:array_node_data]
 // data for an array node
 table ArrayNodeData {
   shape: [DimensionShape] (required);
@@ -90,12 +107,15 @@ table ArrayNodeData {
   // ultimately determines whether this is a V2 snapshot.
   shape_v2: [DimensionShapeV2];
 }
+// --8<-- [end:array_node_data]
 
+// --8<-- [start:node_data]
 // the node contents, that can be either a group or an array
 union NodeData {
   Array :ArrayNodeData,
   Group :GroupNodeData,
 }
+// --8<-- [end:node_data]
 
 // --8<-- [start:node_snapshot]
 // a node

icechunk-format/flatbuffers/transaction_log.fbs

@@ -2,10 +2,13 @@ include "common.fbs";
 
 namespace generated;
 
+// --8<-- [start:chunk_indices]
 table ChunkIndices {
   coords: [uint32] (required);
 }
+// --8<-- [end:chunk_indices]
 
+// --8<-- [start:array_updated_chunks]
 table ArrayUpdatedChunks {
   // the node id of the array to which the chunks belong to
   node_id: ObjectId8 (required);
@@ -14,9 +17,13 @@ table ArrayUpdatedChunks {
   // sorted in ascending lexicographical order
   chunks: [ChunkIndices] (required);
 }
+// --8<-- [end:array_updated_chunks]
 
+// --8<-- [start:node_type]
 enum NodeType: ubyte { Group = 0, Array }
+// --8<-- [end:node_type]
 
+// --8<-- [start:move_operation]
 // moves are fully collapsed: there is only one from/to pair,
 // and overlapping moves (from -> to, to -> new_to) get merged
 // (from -> new_to in this example).
@@ -28,7 +35,9 @@ table MoveOperation {
   node_id: ObjectId8;
   node_type: NodeType;
 }
+// --8<-- [end:move_operation]
 
+// --8<-- [start:transaction_log_table]
 table TransactionLog {
   // id of the transaction log file,
   // it will be the same as the corresponding snapshot
@@ -70,5 +79,6 @@ table TransactionLog {
   // Introduced in spec version 2
   extra: [uint8];
 }
+// --8<-- [end:transaction_log_table]
 
 root_type TransactionLog;