[feat](cloud) Support Cloud partition inverted format rollout

### What problem does this PR solve?

Issue Number: None

Related PR: None

Problem Summary: Cloud tables previously had a single table-level inverted-index storage format. That made V2-to-V3 upgrades all-or-nothing: updating the default could not preserve the format selected when existing partitions and their tablet metadata were created.

This change adds the Cloud-only partition.inverted_index_storage_format table property. It controls the format for subsequently created partitions, while each existing partition retains its resolved V2 or V3 format. The resolved value is durable in partition metadata and is carried through journal replay, recycle/recover, truncate, insert overwrite, and schema-change paths. Tablet and rowset metadata record the physical format separately so mixed-format partitions remain distinguishable during a rolling upgrade. Partition inspection commands read the value from FE partition metadata.

### Release note

Cloud OLAP tables can roll out inverted-index storage formats by partition. New partitions can use V3 while existing V2 partitions remain online and queryable.

### Check List (For Author)

- Test: Regression test / Unit Test
    - FE unit tests covering partition properties, persistence, recycle/recover, and Cloud schema-change handling
    - BE unit tests covering rowset and tablet metadata serialization
    - Cloud regression test: test_partition_cloud_inverted_index_storage_format_meta_write_switch
- Behavior changed: Yes. Cloud tables accept partition.inverted_index_storage_format for new partitions and expose the resolved format in partition metadata commands.
- Does this need documentation: No

Author: hoshinojyunn

be/src/cloud/cloud_rowset_writer.cpp

@@ -73,7 +73,14 @@ Status CloudRowsetWriter::init(const RowsetWriterContext& rowset_writer_context)
         DCHECK_NE(_context.newest_write_timestamp, -1);
         _rowset_meta->set_newest_write_timestamp(_context.newest_write_timestamp);
     }
-    _rowset_meta->set_tablet_schema(_context.tablet_schema);
+    auto schema = _context.tablet_schema->need_record_variant_extended_schema()
+                          ? _context.tablet_schema
+                          : _context.tablet_schema->copy_without_variant_extracted_columns();
+    _rowset_meta->set_tablet_schema(schema);
+    if (_context.persist_inverted_index_storage_format &&
+        _context.inverted_index_storage_format.has_value()) {
+        _rowset_meta->set_inverted_index_storage_format(*_context.inverted_index_storage_format);
+    }
     _rowset_meta->set_job_id(_context.job_id);
     _context.segment_collector = std::make_shared<SegmentCollectorT<BaseBetaRowsetWriter>>(this);
     _context.file_writer_creator = std::make_shared<FileWriterCreatorT<BaseBetaRowsetWriter>>(this);

be/src/cloud/cloud_tablet.cpp

@@ -839,6 +839,9 @@ Result<std::unique_ptr<RowsetWriter>> CloudTablet::create_rowset_writer(
     context.partition_id = partition_id();
     context.enable_unique_key_merge_on_write = enable_unique_key_merge_on_write();
     context.encrypt_algorithm = tablet_meta()->encryption_algorithm();
+    context.inverted_index_storage_format = tablet_meta()->inverted_index_storage_format();
+    context.persist_inverted_index_storage_format =
+            tablet_meta()->has_inverted_index_storage_format();
     return RowsetFactory::create_rowset_writer(_engine, context, vertical);
 }
 
@@ -881,6 +884,9 @@ Result<std::unique_ptr<RowsetWriter>> CloudTablet::create_transient_rowset_write
     context.enable_unique_key_merge_on_write = enable_unique_key_merge_on_write();
     context.txn_expiration = txn_expiration;
     context.encrypt_algorithm = tablet_meta()->encryption_algorithm();
+    context.inverted_index_storage_format = tablet_meta()->inverted_index_storage_format();
+    context.persist_inverted_index_storage_format =
+            tablet_meta()->has_inverted_index_storage_format();
     // TODO(liaoxin) enable packed file for transient rowset
     context.allow_packed_file = false;
 

be/src/cloud/pb_convert.cpp

@@ -108,6 +108,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in)
     if (in.has_reference_instance_id()) {
         out->set_reference_instance_id(in.reference_instance_id());
     }
+    if (in.has_inverted_index_storage_format()) {
+        out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+    }
     auto* slice_locations = out->mutable_packed_slice_locations();
     slice_locations->clear();
     slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end());
@@ -204,6 +207,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
     if (in.has_reference_instance_id()) {
         out->set_reference_instance_id(in.reference_instance_id());
     }
+    if (in.has_inverted_index_storage_format()) {
+        out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+    }
     auto* slice_locations = out->mutable_packed_slice_locations();
     slice_locations->clear();
     slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end());
@@ -304,6 +310,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in)
     if (in.has___split_schema()) {
         out->mutable___split_schema()->CopyFrom(in.__split_schema());
     }
+    if (in.has_inverted_index_storage_format()) {
+        out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+    }
     if (in.has_visible_ts_ms()) {
         out->set_visible_ts_ms(in.visible_ts_ms());
     }
@@ -399,6 +408,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
     if (in.has___split_schema()) {
         out->mutable___split_schema()->Swap(in.mutable___split_schema());
     }
+    if (in.has_inverted_index_storage_format()) {
+        out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+    }
     if (in.has_visible_ts_ms()) {
         out->set_visible_ts_ms(in.visible_ts_ms());
     }
@@ -796,6 +808,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB& in)
     if (in.has_encryption_algorithm()) {
         out->set_encryption_algorithm(in.encryption_algorithm());
     }
+    if (in.has_inverted_index_storage_format()) {
+        out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+    }
 }
 
 void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
@@ -890,6 +905,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
     if (in.has_encryption_algorithm()) {
         out->set_encryption_algorithm(in.encryption_algorithm());
     }
+    if (in.has_inverted_index_storage_format()) {
+        out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+    }
 }
 
 TabletMetaPB cloud_tablet_meta_to_doris(const TabletMetaCloudPB& in) {
@@ -988,6 +1006,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB& in)
     if (in.has_encryption_algorithm()) {
         out->set_encryption_algorithm(in.encryption_algorithm());
     }
+    if (in.has_inverted_index_storage_format()) {
+        out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+    }
 }
 
 void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
@@ -1082,6 +1103,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
     if (in.has_encryption_algorithm()) {
         out->set_encryption_algorithm(in.encryption_algorithm());
     }
+    if (in.has_inverted_index_storage_format()) {
+        out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+    }
 }
 
 } // namespace doris::cloud

be/src/service/http/action/show_nested_index_file_action.cpp

@@ -22,6 +22,7 @@
 #include <exception>
 #include <string>
 
+#include "cloud/cloud_tablet.h"
 #include "common/status.h"
 #include "service/http/http_channel.h"
 #include "service/http/http_headers.h"
@@ -54,6 +55,10 @@ Status ShowNestedIndexFileAction::_handle_show_nested_index_file(HttpRequest* re
     }
 
     auto tablet = DORIS_TRY(ExecEnv::get_tablet(tablet_id));
+    if (auto cloud_tablet = std::dynamic_pointer_cast<CloudTablet>(tablet)) {
+        // The debug endpoint must inspect all visible cloud rowsets, not only the local cache.
+        RETURN_IF_ERROR(cloud_tablet->sync_rowsets());
+    }
     RETURN_IF_ERROR(tablet->show_nested_index_file(json_meta));
     return Status::OK();
 }

be/src/storage/rowset/beta_rowset_writer.cpp

@@ -387,7 +387,14 @@ Status BaseBetaRowsetWriter::init(const RowsetWriterContext& rowset_writer_conte
         _rowset_meta->set_newest_write_timestamp(_context.newest_write_timestamp);
     }
     _rowset_meta->set_tablet_uid(_context.tablet_uid);
-    _rowset_meta->set_tablet_schema(_context.tablet_schema);
+    auto schema = _context.tablet_schema->need_record_variant_extended_schema()
+                          ? _context.tablet_schema
+                          : _context.tablet_schema->copy_without_variant_extracted_columns();
+    _rowset_meta->set_tablet_schema(schema);
+    if (_context.persist_inverted_index_storage_format &&
+        _context.inverted_index_storage_format.has_value()) {
+        _rowset_meta->set_inverted_index_storage_format(*_context.inverted_index_storage_format);
+    }
     _rowset_meta->set_compaction_level(_context.compaction_level);
     if (_context.write_binlog_opt().enable) {
         _rowset_meta->mark_row_binlog();

be/src/storage/rowset/rowset_meta.cpp

@@ -70,8 +70,19 @@ bool RowsetMeta::init(const RowsetMeta* rowset_meta) {
 }
 
 bool RowsetMeta::init_from_pb(const RowsetMetaPB& rowset_meta_pb) {
+    if (rowset_meta_pb.has_inverted_index_storage_format()) {
+        _rowset_meta_pb.set_inverted_index_storage_format(
+                rowset_meta_pb.inverted_index_storage_format());
+    } else {
+        _rowset_meta_pb.clear_inverted_index_storage_format();
+    }
     if (rowset_meta_pb.has_tablet_schema()) {
-        set_tablet_schema(rowset_meta_pb.tablet_schema());
+        TabletSchemaPB schema_pb = rowset_meta_pb.tablet_schema();
+        if (rowset_meta_pb.has_inverted_index_storage_format()) {
+            schema_pb.set_inverted_index_storage_format(
+                    rowset_meta_pb.inverted_index_storage_format());
+        }
+        set_tablet_schema(schema_pb);
     }
     // Release ownership of TabletSchemaPB from `rowset_meta_pb` and then set it back to `rowset_meta_pb`,
     // this won't break const semantics of `rowset_meta_pb`, because `rowset_meta_pb` is not changed
@@ -209,6 +220,10 @@ void RowsetMeta::to_rowset_pb(RowsetMetaPB* rs_meta_pb, bool skip_schema) const
         if (!skip_schema) {
             // For cloud, separate tablet schema from rowset meta to reduce persistent size.
             _schema->to_schema_pb(rs_meta_pb->mutable_tablet_schema());
+            if (rs_meta_pb->has_inverted_index_storage_format()) {
+                rs_meta_pb->mutable_tablet_schema()->set_inverted_index_storage_format(
+                        rs_meta_pb->inverted_index_storage_format());
+            }
         }
     }
     rs_meta_pb->set_has_variant_type_in_schema(has_variant_type_in_schema());
@@ -221,6 +236,12 @@ RowsetMetaPB RowsetMeta::get_rowset_pb(bool skip_schema) const {
 }
 
 void RowsetMeta::set_tablet_schema(const TabletSchemaSPtr& tablet_schema) {
+    if (_rowset_meta_pb.has_inverted_index_storage_format()) {
+        TabletSchemaPB schema_pb;
+        tablet_schema->to_schema_pb(&schema_pb);
+        set_tablet_schema(schema_pb);
+        return;
+    }
     if (_handle) {
         TabletSchemaCache::instance()->release(_handle);
     }
@@ -230,22 +251,42 @@ void RowsetMeta::set_tablet_schema(const TabletSchemaSPtr& tablet_schema) {
 }
 
 void RowsetMeta::set_tablet_schema(const TabletSchemaPB& tablet_schema) {
+    TabletSchemaPB resolved_schema = tablet_schema;
+    if (_rowset_meta_pb.has_inverted_index_storage_format()) {
+        resolved_schema.set_inverted_index_storage_format(
+                _rowset_meta_pb.inverted_index_storage_format());
+    }
     if (_handle) {
         TabletSchemaCache::instance()->release(_handle);
     }
     auto pair = TabletSchemaCache::instance()->insert(
-            TabletSchema::deterministic_string_serialize(tablet_schema));
+            TabletSchema::deterministic_string_serialize(resolved_schema));
     _handle = pair.first;
     _schema = pair.second;
 }
 
+void RowsetMeta::set_inverted_index_storage_format(InvertedIndexStorageFormatPB format) {
+    _rowset_meta_pb.set_inverted_index_storage_format(format);
+    if (_schema && _schema->get_inverted_index_storage_format() != format) {
+        TabletSchemaPB schema_pb;
+        _schema->to_schema_pb(&schema_pb);
+        schema_pb.set_inverted_index_storage_format(format);
+        set_tablet_schema(schema_pb);
+    }
+}
+
 bool RowsetMeta::_deserialize_from_pb(std::string_view value) {
     if (!_rowset_meta_pb.ParseFromArray(value.data(), cast_set<int32_t>(value.size()))) {
         _rowset_meta_pb.Clear();
         return false;
     }
     if (_rowset_meta_pb.has_tablet_schema()) {
-        set_tablet_schema(_rowset_meta_pb.tablet_schema());
+        TabletSchemaPB schema_pb = _rowset_meta_pb.tablet_schema();
+        if (_rowset_meta_pb.has_inverted_index_storage_format()) {
+            schema_pb.set_inverted_index_storage_format(
+                    _rowset_meta_pb.inverted_index_storage_format());
+        }
+        set_tablet_schema(schema_pb);
         _rowset_meta_pb.set_allocated_tablet_schema(nullptr);
     }
     return true;
@@ -258,6 +299,10 @@ bool RowsetMeta::_serialize_to_pb(std::string* value) {
     RowsetMetaPB rowset_meta_pb = _rowset_meta_pb;
     if (_schema) {
         _schema->to_schema_pb(rowset_meta_pb.mutable_tablet_schema());
+        if (rowset_meta_pb.has_inverted_index_storage_format()) {
+            rowset_meta_pb.mutable_tablet_schema()->set_inverted_index_storage_format(
+                    rowset_meta_pb.inverted_index_storage_format());
+        }
     }
     return rowset_meta_pb.SerializeToString(value);
 }

be/src/storage/rowset/rowset_meta.h

@@ -107,6 +107,16 @@ class RowsetMeta : public MetadataAdder<RowsetMeta> {
 
     void set_index_id(int64_t index_id) { _rowset_meta_pb.set_index_id(index_id); }
 
+    bool has_inverted_index_storage_format() const {
+        return _rowset_meta_pb.has_inverted_index_storage_format();
+    }
+
+    InvertedIndexStorageFormatPB inverted_index_storage_format() const {
+        return _rowset_meta_pb.inverted_index_storage_format();
+    }
+
+    void set_inverted_index_storage_format(InvertedIndexStorageFormatPB format);
+
     TabletUid tablet_uid() const { return _rowset_meta_pb.tablet_uid(); }
 
     void set_tablet_uid(TabletUid tablet_uid) {

be/src/storage/rowset/rowset_writer_context.h

@@ -70,7 +70,13 @@ struct RowsetWriterContext {
     RowsetTypePB rowset_type {BETA_ROWSET};
 
     TabletSchemaSPtr tablet_schema;
-
+    // Immutable inverted-index file format inherited from the owner tablet.
+    std::optional<InvertedIndexStorageFormatPB> inverted_index_storage_format;
+    // Whether the owner tablet persists the format in its top-level metadata.
+    // This is derived from TabletMeta and keeps rowset metadata consistent with it.
+    bool persist_inverted_index_storage_format = false;
+    // for variant schema update
+    TabletSchemaSPtr merged_tablet_schema;
     // PREPARED/COMMITTED for pending rowset
     // VISIBLE for non-pending rowset
     RowsetStatePB rowset_state {PREPARED};

be/src/storage/tablet/tablet.cpp

@@ -2302,6 +2302,7 @@ void Tablet::_init_context_common_fields(RowsetWriterContext& context) {
     context.tablet_id = tablet_id();
     context.partition_id = partition_id();
     context.tablet_schema_hash = schema_hash();
+    context.inverted_index_storage_format = tablet_meta()->inverted_index_storage_format();
     context.rowset_type = tablet_meta()->preferred_rowset_type();
     // Alpha Rowset will be removed in the future, so that if the tablet's default rowset type is
     // alpha rowset, then set the newly created rowset to storage engine's default rowset.

be/src/storage/tablet/tablet_meta.cpp

@@ -315,8 +315,8 @@ TabletMeta::TabletMeta(const TabletMeta& b)
           _time_series_compaction_empty_rowsets_threshold(
                   b._time_series_compaction_empty_rowsets_threshold),
           _time_series_compaction_level_threshold(b._time_series_compaction_level_threshold),
-          _vertical_compaction_num_columns_per_group(
-                  b._vertical_compaction_num_columns_per_group) {};
+          _vertical_compaction_num_columns_per_group(b._vertical_compaction_num_columns_per_group),
+          _inverted_index_storage_format(b._inverted_index_storage_format) {};
 
 void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn,
                                           ColumnPB* column) {
@@ -827,6 +827,10 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) {
     _cumulative_layer_point = tablet_meta_pb.cumulative_layer_point();
     _tablet_uid = TabletUid(tablet_meta_pb.tablet_uid());
     _ttl_seconds = tablet_meta_pb.ttl_seconds();
+    _inverted_index_storage_format.reset();
+    if (tablet_meta_pb.has_inverted_index_storage_format()) {
+        _inverted_index_storage_format = tablet_meta_pb.inverted_index_storage_format();
+    }
     if (tablet_meta_pb.has_tablet_type()) {
         _tablet_type = tablet_meta_pb.tablet_type();
     } else {
@@ -857,7 +861,11 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) {
 
     // init _schema
     TabletSchemaSPtr schema = std::make_shared<TabletSchema>();
-    schema->init_from_pb(tablet_meta_pb.schema());
+    TabletSchemaPB schema_pb = tablet_meta_pb.schema();
+    if (_inverted_index_storage_format.has_value()) {
+        schema_pb.set_inverted_index_storage_format(*_inverted_index_storage_format);
+    }
+    schema->init_from_pb(schema_pb);
     if (_handle) {
         TabletSchemaCache::instance()->release(_handle);
     }
@@ -1014,6 +1022,11 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb, bool cloud_get_rowset_
     }
 
     _schema->to_schema_pb(tablet_meta_pb->mutable_schema());
+    if (_inverted_index_storage_format.has_value()) {
+        tablet_meta_pb->set_inverted_index_storage_format(*_inverted_index_storage_format);
+        tablet_meta_pb->mutable_schema()->set_inverted_index_storage_format(
+                *_inverted_index_storage_format);
+    }
 
     if (_row_binlog_schema != nullptr) {
         _row_binlog_schema->to_schema_pb(tablet_meta_pb->mutable_row_binlog_schema());