feat: Add Paimon connector data structures and scaffolding (facebookincubator#16674)

Summary:
Pull Request resolved: facebookincubator#16674

CONTEXT: Paimon is a lake format that organizes data files in an LSM-tree
structure per partition/bucket. This diff bootstraps the Paimon integration
under velox/connectors/hive/paimon/, following the same pattern as Iceberg
(hive/iceberg/).

WHAT: Adds the foundational data structures and connector scaffolding for
Paimon table format support:

- PaimonConnectorSplit: Extends ConnectorSplit. Represents a Paimon DataSplit
  (one partition x bucket) containing multiple data files across LSM-tree
  levels. Includes rawConvertible validation (deleteRowCount must be 0).
- PaimonConnectorSplitBuilder: Fluent builder for test convenience.
- PaimonDataFile: Per-file metadata including path, size, rowCount, level,
  sequence numbers, deleteRowCount, Type (kData/kChangelog), Source
  (kAppend/kCompact), and optional PaimonDeletionFile.
- PaimonDeletionFile: Deletion bitmap metadata (path, offset, length,
  cardinality) with constructor validation.
- PaimonRowKind: Row-level change type enum (+I/-U/+U/-D) matching Paimon
  Java's RowKind values, with kRowKindColumn constant for the hidden
  _rowkind system column.
- PaimonChangelogMode: Table-level changelog producer mode enum
  (kNone/kInput/kLookup/kFullCompaction).
- PaimonConnector: Extends HiveConnector, creates PaimonDataSource instances.
- PaimonDataSource: Extends HiveDataSource with addSplit/next stubs
  (VELOX_NYI). Implementation in follow-up diff.
- PaimonConfig: Placeholder for Paimon-specific configuration.
- Full serialization/deserialization support for all types.

Reviewed By: srsuryadev, zzhao0, tanjialiang

Differential Revision: D95760777

fbshipit-source-id: ce2bef799dabaf936a485590110177dbc2247902

Author: xiaoxmeng

velox/connectors/hive/CMakeLists.txt

@@ -16,6 +16,7 @@ velox_add_library(velox_hive_config OBJECT HiveConfig.cpp)
 velox_link_libraries(velox_hive_config velox_core velox_exception)
 
 add_subdirectory(iceberg)
+add_subdirectory(paimon)
 
 velox_add_library(
   velox_hive_connector

velox/connectors/hive/paimon/CMakeLists.txt

@@ -0,0 +1,44 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+velox_add_library(
+  velox_hive_paimon_split
+  PaimonConnectorSplit.cpp
+  PaimonDataFileMeta.cpp
+  PaimonDeletionFile.cpp
+  PaimonRowKind.cpp
+)
+
+velox_link_libraries(
+  velox_hive_paimon_split
+  velox_connector
+  velox_hive_connector
+  fmt::fmt
+)
+
+velox_add_library(
+  velox_hive_paimon_connector
+  PaimonConnector.cpp
+  PaimonDataSource.cpp
+)
+
+velox_link_libraries(
+  velox_hive_paimon_connector
+  velox_hive_paimon_split
+  velox_hive_connector
+)
+
+if(${VELOX_BUILD_TESTING})
+  add_subdirectory(tests)
+endif()

velox/connectors/hive/paimon/PaimonConfig.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "velox/common/config/Config.h"
+
+namespace facebook::velox::connector::hive::paimon {
+
+/// Paimon-specific connector configuration.
+/// Wraps the shared ConfigBase and provides accessors for Paimon settings.
+class PaimonConfig {
+ public:
+  explicit PaimonConfig(std::shared_ptr<const config::ConfigBase> config)
+      : config_(std::move(config)) {
+    VELOX_CHECK_NOT_NULL(config_);
+  }
+
+  const std::shared_ptr<const config::ConfigBase>& config() const {
+    return config_;
+  }
+
+ private:
+  const std::shared_ptr<const config::ConfigBase> config_;
+};
+
+} // namespace facebook::velox::connector::hive::paimon

velox/connectors/hive/paimon/PaimonConnector.cpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "velox/connectors/hive/paimon/PaimonConnector.h"
+
+#include "velox/connectors/hive/paimon/PaimonDataSource.h"
+
+namespace facebook::velox::connector::hive::paimon {
+
+PaimonConnector::PaimonConnector(
+    const std::string& id,
+    std::shared_ptr<const config::ConfigBase> config,
+    folly::Executor* ioExecutor)
+    : HiveConnector(id, config, ioExecutor),
+      paimonConfig_(std::make_shared<PaimonConfig>(connectorConfig())) {}
+
+std::unique_ptr<DataSource> PaimonConnector::createDataSource(
+    const RowTypePtr& outputType,
+    const ConnectorTableHandlePtr& tableHandle,
+    const ColumnHandleMap& columnHandles,
+    ConnectorQueryCtx* connectorQueryCtx) {
+  return std::make_unique<PaimonDataSource>(
+      outputType,
+      tableHandle,
+      columnHandles,
+      &fileHandleFactory_,
+      ioExecutor_,
+      connectorQueryCtx,
+      hiveConfig_);
+}
+
+} // namespace facebook::velox::connector::hive::paimon

velox/connectors/hive/paimon/PaimonConnector.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "velox/connectors/hive/HiveConnector.h"
+#include "velox/connectors/hive/paimon/PaimonConfig.h"
+
+namespace facebook::velox::connector::hive::paimon {
+
+/// Provides Paimon table format support by extending HiveConnector.
+///
+/// Creates PaimonDataSource instances that handle Paimon's multi-file splits
+/// (one split = one bucket with multiple data files across LSM-tree levels).
+/// Reuses HiveConnector's ORC/Parquet readers directly — no Arrow bridge.
+class PaimonConnector final : public HiveConnector {
+ public:
+  PaimonConnector(
+      const std::string& id,
+      std::shared_ptr<const config::ConfigBase> config,
+      folly::Executor* ioExecutor);
+
+  /// Creates PaimonDataSource for reading from Paimon tables.
+  std::unique_ptr<DataSource> createDataSource(
+      const RowTypePtr& outputType,
+      const ConnectorTableHandlePtr& tableHandle,
+      const ColumnHandleMap& columnHandles,
+      ConnectorQueryCtx* connectorQueryCtx) override;
+
+ private:
+  const std::shared_ptr<PaimonConfig> paimonConfig_;
+};
+
+class PaimonConnectorFactory final : public ConnectorFactory {
+ public:
+  static constexpr const char* kPaimonConnectorName = "paimon";
+
+  PaimonConnectorFactory() : ConnectorFactory(kPaimonConnectorName) {}
+
+  std::shared_ptr<Connector> newConnector(
+      const std::string& id,
+      std::shared_ptr<const config::ConfigBase> config,
+      folly::Executor* ioExecutor = nullptr,
+      [[maybe_unused]] folly::Executor* cpuExecutor = nullptr) override {
+    return std::make_shared<PaimonConnector>(id, config, ioExecutor);
+  }
+};
+
+} // namespace facebook::velox::connector::hive::paimon