Skip to content

Proof of idea consolidation #5358

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Proof of idea consolidation #5358

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2de7466
d
ChengJi-db Oct 17, 2025
81857e8
fix
ChengJi-db Oct 17, 2025
6fb33bf
d
ChengJi-db Oct 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -923,7 +923,7 @@ lazy val iceberg = (project in file("iceberg"))
)
// scalastyle:on println

val icebergShadedVersion = "1.8.0"
val icebergShadedVersion = "1.9.0"
lazy val icebergShaded = (project in file("icebergShaded"))
.dependsOn(spark % "provided")
.disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
Expand Down
18 changes: 9 additions & 9 deletions iceberg/src/main/scala/org/apache/iceberg/transforms/IcebergPartitionUtil.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,18 @@
* limitations under the License.
*/

package org.apache.iceberg.transforms
package shadedForDelta.org.apache.iceberg.transforms

import scala.collection.JavaConverters._

import org.apache.spark.sql.delta.DeltaColumnMapping
import org.apache.spark.sql.delta.commands.convert.TypeToSparkTypeWithCustomCast
import org.apache.spark.sql.delta.sources.DeltaSourceUtils.GENERATION_EXPRESSION_METADATA_KEY
import org.apache.spark.sql.delta.util.{DateFormatter, TimestampFormatter}
import org.apache.iceberg.{PartitionField, PartitionSpec, Schema, StructLike}
import org.apache.iceberg.types.Type.TypeID
import org.apache.iceberg.types.Types
import org.apache.iceberg.types.TypeUtil
import shadedForDelta.org.apache.iceberg.{PartitionField, PartitionSpec, Schema, StructLike}
import shadedForDelta.org.apache.iceberg.types.Type.TypeID
import shadedForDelta.org.apache.iceberg.types.Types
import shadedForDelta.org.apache.iceberg.types.TypeUtil

import org.apache.spark.sql.types.{DateType, IntegerType, MetadataBuilder, StringType, StructField}

Expand Down Expand Up @@ -140,10 +140,10 @@ object IcebergPartitionUtil {
.putLong(DeltaColumnMapping.COLUMN_MAPPING_METADATA_ID_KEY, sourceField.fieldId())
("", TypeUtil.visit(sourceType, new TypeToSparkTypeWithCustomCast(castTimeType)))

case Timestamps.YEAR | Dates.YEAR =>
case Timestamps.MICROS_TO_YEAR | Dates.YEAR =>
(s"year($sourceColumnName)", IntegerType)

case Timestamps.DAY | Dates.DAY =>
case Timestamps.MICROS_TO_DAY | Dates.DAY =>
(s"cast($sourceColumnName as date)", DateType)

case t: Truncate[_] if sourceType.typeId() == TypeID.STRING =>
Expand All @@ -154,10 +154,10 @@ object IcebergPartitionUtil {
(icebergNumericTruncateExpression(sourceColumnName, t.width().toLong),
TypeUtil.visit(sourceType, new TypeToSparkTypeWithCustomCast(castTimeType)))

case Timestamps.MONTH | Dates.MONTH =>
case Timestamps.MICROS_TO_MONTH | Dates.MONTH =>
(s"date_format($sourceColumnName, 'yyyy-MM')", StringType)

case Timestamps.HOUR =>
case Timestamps.MICROS_TO_HOUR =>
(s"date_format($sourceColumnName, 'yyyy-MM-dd-HH')", StringType)

case other =>
Expand Down
6 changes: 3 additions & 3 deletions iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ import org.apache.spark.sql.delta.commands.convert.IcebergTable.ERR_MULTIPLE_PAR
import org.apache.spark.sql.delta.logging.DeltaLogKeys
import org.apache.spark.sql.delta.sources.DeltaSQLConf
import org.apache.hadoop.fs.Path
import org.apache.iceberg.{BaseTable, DataFile, DataFiles, DeleteFile, FileContent, FileFormat, ManifestContent, ManifestFile, ManifestFiles, PartitionData, PartitionSpec, RowLevelOperationMode, Schema, StructLike, Table, TableProperties}
import org.apache.iceberg.transforms.IcebergPartitionUtil
import org.apache.iceberg.types.Type.TypeID
import shadedForDelta.org.apache.iceberg.{BaseTable, DataFile, DataFiles, DeleteFile, FileContent, FileFormat, ManifestContent, ManifestFile, ManifestFiles, PartitionData, PartitionSpec, RowLevelOperationMode, Schema, StructLike, Table, TableProperties}
import shadedForDelta.org.apache.iceberg.transforms.IcebergPartitionUtil
import shadedForDelta.org.apache.iceberg.types.Type.TypeID

import org.apache.spark.SparkThrowable
import org.apache.spark.internal.{LoggingShims, MDC}
Expand Down
10 changes: 5 additions & 5 deletions iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergPartitionConverter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ import scala.collection.JavaConverters._

import org.apache.spark.sql.delta.DeltaColumnMapping
import org.apache.spark.sql.delta.util.{DateFormatter, TimestampFormatter}
import org.apache.iceberg.{PartitionData, PartitionField, PartitionSpec, Schema, StructLike, Table}
import org.apache.iceberg.transforms.IcebergPartitionUtil
import org.apache.iceberg.types.{Conversions, Type => IcebergType}
import org.apache.iceberg.types.Type.{PrimitiveType => IcebergPrimitiveType, TypeID}
import org.apache.iceberg.types.Types.{
import shadedForDelta.org.apache.iceberg.{PartitionData, PartitionField, PartitionSpec, Schema, StructLike, Table}
import shadedForDelta.org.apache.iceberg.transforms.IcebergPartitionUtil
import shadedForDelta.org.apache.iceberg.types.{Conversions, Type => IcebergType}
import shadedForDelta.org.apache.iceberg.types.Type.{PrimitiveType => IcebergPrimitiveType, TypeID}
import shadedForDelta.org.apache.iceberg.types.Types.{
ListType => IcebergListType,
MapType => IcebergMapType,
NestedField,
Expand Down
4 changes: 2 additions & 2 deletions iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergSchemaUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ package org.apache.spark.sql.delta.commands.convert

import org.apache.spark.sql.delta.DeltaColumnMapping
import org.apache.spark.sql.delta.schema.SchemaMergingUtils
import org.apache.iceberg.Schema
import org.apache.iceberg.types.TypeUtil
import shadedForDelta.org.apache.iceberg.Schema
import shadedForDelta.org.apache.iceberg.types.TypeUtil

import org.apache.spark.sql.types.{MetadataBuilder, StructType}

Expand Down
4 changes: 2 additions & 2 deletions iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergSparkWrappers.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ import java.util.stream.Collectors
import scala.collection.JavaConverters._

import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor
import org.apache.iceberg.{DataFile, DeleteFile, FileContent, FileFormat, ManifestContent, ManifestFile, PartitionData, StructLike}
import org.apache.iceberg.ManifestFile.PartitionFieldSummary
import shadedForDelta.org.apache.iceberg.{DataFile, DeleteFile, FileContent, FileFormat, ManifestContent, ManifestFile, PartitionData, StructLike}
import shadedForDelta.org.apache.iceberg.ManifestFile.PartitionFieldSummary

/**
* The classes in this file are wrappers of Iceberg classes
Expand Down
10 changes: 5 additions & 5 deletions iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergStatsUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ import org.apache.spark.sql.delta.metering.DeltaLogging
import org.apache.spark.sql.delta.sources.DeltaSQLConf
import org.apache.spark.sql.delta.stats.DeltaStatistics._
import org.apache.spark.sql.delta.util.JsonUtils
import org.apache.iceberg.{DataFile, PartitionData, PartitionField, Schema, StructLike, Table}
import org.apache.iceberg.types.{Conversions, Type => IcebergType}
import org.apache.iceberg.types.Type.{PrimitiveType => IcebergPrimitiveType, TypeID}
import org.apache.iceberg.types.Types.{
import shadedForDelta.org.apache.iceberg.{DataFile, PartitionData, PartitionField, Schema, StructLike, Table}
import shadedForDelta.org.apache.iceberg.types.{Conversions, Type => IcebergType}
import shadedForDelta.org.apache.iceberg.types.Type.{PrimitiveType => IcebergPrimitiveType, TypeID}
import shadedForDelta.org.apache.iceberg.types.Types.{
DateType => IcebergDateType,
ListType => IcebergListType,
MapType => IcebergMapType,
Expand All @@ -39,7 +39,7 @@ import org.apache.iceberg.types.Types.{
StructType => IcebergStructType,
TimestampType => IcebergTimestampType
}
import org.apache.iceberg.util.DateTimeUtil
import shadedForDelta.org.apache.iceberg.util.DateTimeUtil

import org.apache.spark.sql.SparkSession

Expand Down
10 changes: 5 additions & 5 deletions iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergTable.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaColumnMappingMode, D
import org.apache.spark.sql.delta.DeltaErrors.{cloneFromIcebergSourceWithoutSpecs, cloneFromIcebergSourceWithPartitionEvolution}
import org.apache.spark.sql.delta.schema.SchemaMergingUtils
import org.apache.spark.sql.delta.sources.DeltaSQLConf
import org.apache.iceberg.{PartitionSpec, Schema, Snapshot => IcebergSnapshot, Table, TableProperties}
import org.apache.iceberg.hadoop.HadoopTables
import org.apache.iceberg.io.FileIO
import org.apache.iceberg.transforms.{Bucket, IcebergPartitionUtil}
import org.apache.iceberg.util.PropertyUtil
import shadedForDelta.org.apache.iceberg.{PartitionSpec, Schema, Snapshot => IcebergSnapshot, Table, TableProperties}
import shadedForDelta.org.apache.iceberg.hadoop.HadoopTables
import shadedForDelta.org.apache.iceberg.io.FileIO
import shadedForDelta.org.apache.iceberg.transforms.{Bucket, IcebergPartitionUtil}
import shadedForDelta.org.apache.iceberg.util.PropertyUtil

import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.spark.sql.execution.datasources.PartitioningUtils
Expand Down
14 changes: 7 additions & 7 deletions iceberg/src/main/scala/org/apache/spark/sql/delta/TypeToSparkTypeWithCustomCast.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,13 @@ import java.util

import scala.collection.JavaConverters._

import org.apache.iceberg.MetadataColumns
import org.apache.iceberg.Schema
import org.apache.iceberg.relocated.com.google.common.collect.Lists
import org.apache.iceberg.types.Type
import org.apache.iceberg.types.Type.TypeID._
import org.apache.iceberg.types.Types
import org.apache.iceberg.types.TypeUtil
import shadedForDelta.org.apache.iceberg.MetadataColumns
import shadedForDelta.org.apache.iceberg.Schema
import shadedForDelta.org.apache.iceberg.relocated.com.google.common.collect.Lists
import shadedForDelta.org.apache.iceberg.types.Type
import shadedForDelta.org.apache.iceberg.types.Type.TypeID._
import shadedForDelta.org.apache.iceberg.types.Types
import shadedForDelta.org.apache.iceberg.types.TypeUtil

import org.apache.spark.sql.types.ArrayType
import org.apache.spark.sql.types.BinaryType
Expand Down
8 changes: 4 additions & 4 deletions ...st/scala/org/apache/spark/sql/delta/commands/convert/IcebergPartitionConverterSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ import java.util.{List => JList}

import scala.collection.JavaConverters._

import org.apache.iceberg.{PartitionData, PartitionSpec, Schema}
import org.apache.iceberg.transforms._
import org.apache.iceberg.types.Conversions
import org.apache.iceberg.types.Types._
import shadedForDelta.org.apache.iceberg.{PartitionData, PartitionSpec, Schema}
import shadedForDelta.org.apache.iceberg.transforms._
import shadedForDelta.org.apache.iceberg.types.Conversions
import shadedForDelta.org.apache.iceberg.types.Types._

import org.apache.spark.SparkFunSuite

Expand Down
12 changes: 6 additions & 6 deletions ...g/src/test/scala/org/apache/spark/sql/delta/commands/convert/IcebergStatsUtilsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@ import scala.collection.JavaConverters._

import org.apache.spark.sql.delta.sources.DeltaSQLConf
import org.apache.spark.sql.delta.util.JsonUtils
import org.apache.iceberg.{DataFile, FileContent, FileFormat, PartitionData, PartitionSpec, Schema, StructLike}
import org.apache.iceberg.transforms._
import org.apache.iceberg.types.Conversions
import org.apache.iceberg.types.Type
import org.apache.iceberg.types.Type.TypeID
import org.apache.iceberg.types.Types._
import shadedForDelta.org.apache.iceberg.{DataFile, FileContent, FileFormat, PartitionData, PartitionSpec, Schema, StructLike}
import shadedForDelta.org.apache.iceberg.transforms._
import shadedForDelta.org.apache.iceberg.types.Conversions
import shadedForDelta.org.apache.iceberg.types.Type
import shadedForDelta.org.apache.iceberg.types.Type.TypeID
import shadedForDelta.org.apache.iceberg.types.Types._

import org.apache.spark.SparkFunSuite
import org.apache.spark.internal.config.ConfigEntry
Expand Down
Loading
Loading