Skip to content

Commit 4d96013

Browse files
JingsongLiJingsongLi
committed
fix
1 parent 8ba4051 commit 4d96013

File tree

4 files changed

+40
-52
lines changed

4 files changed

+40
-52
lines changed

‎paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@
3434

3535
import java.io.Serializable;
3636

37-
import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE;
38-
3937
/** Catalog environment in table which contains log factory, metastore client factory. */
4038
public class CatalogEnvironment implements Serializable {
4139

@@ -80,17 +78,7 @@ public PartitionHandler partitionHandler() {
8078
return null;
8179
}
8280
Catalog catalog = catalogLoader.load();
83-
Table table;
84-
try {
85-
table = catalog.getTable(identifier);
86-
if (!Boolean.parseBoolean(
87-
table.options().getOrDefault(METASTORE_PARTITIONED_TABLE.key(), "FALSE"))) {
88-
return null;
89-
}
90-
} catch (Catalog.TableNotExistException e) {
91-
throw new RuntimeException(e);
92-
}
93-
return PartitionHandler.create(catalog, table, identifier);
81+
return PartitionHandler.create(catalog, identifier);
9482
}
9583

9684
@Nullable

‎paimon-core/src/main/java/org/apache/paimon/table/PartitionHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ void alterPartitions(List<PartitionStatistics> partitions)
3939
void markDonePartitions(List<Map<String, String>> partitions)
4040
throws Catalog.TableNotExistException;
4141

42-
static PartitionHandler create(Catalog catalog, Table table, Identifier identifier) {
42+
static PartitionHandler create(Catalog catalog, Identifier identifier) {
4343
return new PartitionHandler() {
4444

4545
@Override

‎paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ public void dropPartitions(Identifier identifier, List<Map<String, String>> part
430430
}
431431
}
432432
if (!tagToPart) {
433-
dropPartitions(identifier, partitions);
433+
super.dropPartitions(identifier, partitions);
434434
}
435435
}
436436

‎paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateFileProcedureTest.scala‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -26,29 +26,29 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
2626
Seq("parquet", "orc", "avro").foreach(
2727
format => {
2828
test(s"Paimon migrate file procedure: migrate $format non-partitioned table") {
29-
withTable("hive_tbl", "paimon_tbl") {
29+
withTable(s"hive_tbl$format", s"paimon_tbl$format") {
3030
// create hive table
3131
spark.sql(s"""
32-
|CREATE TABLE hive_tbl (id STRING, name STRING, pt STRING)
32+
|CREATE TABLE hive_tbl$format (id STRING, name STRING, pt STRING)
3333
|USING $format
3434
|""".stripMargin)
3535

36-
spark.sql(s"INSERT INTO hive_tbl VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
36+
spark.sql(s"INSERT INTO hive_tbl$format VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
3737

3838
// create paimon table
3939
spark.sql(s"""
40-
|CREATE TABLE paimon_tbl (id STRING, name STRING, pt STRING)
40+
|CREATE TABLE paimon_tbl$format (id STRING, name STRING, pt STRING)
4141
|USING PAIMON
4242
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
4343
|""".stripMargin)
4444

45-
spark.sql(s"INSERT INTO paimon_tbl VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
45+
spark.sql(s"INSERT INTO paimon_tbl$format VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
4646

4747
spark.sql(
48-
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl', target_table => '$hiveDbName.paimon_tbl')")
48+
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl$format', target_table => '$hiveDbName.paimon_tbl$format')")
4949

5050
checkAnswer(
51-
spark.sql("SELECT * FROM paimon_tbl ORDER BY id"),
51+
spark.sql(s"SELECT * FROM paimon_tbl$format ORDER BY id"),
5252
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1") :: Row(
5353
"4",
5454
"d",
@@ -61,29 +61,29 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
6161
format => {
6262
test(
6363
s"Paimon migrate file procedure: migrate $format non-partitioned table with parallelism") {
64-
withTable("hive_tbl_02", "paimon_tbl_02") {
64+
withTable(s"hive_tbl_02$format", s"paimon_tbl_02$format") {
6565
// create hive table
6666
spark.sql(s"""
67-
|CREATE TABLE hive_tbl_02 (id STRING, name STRING, pt STRING)
67+
|CREATE TABLE hive_tbl_02$format (id STRING, name STRING, pt STRING)
6868
|USING $format
6969
|""".stripMargin)
7070

71-
spark.sql(s"INSERT INTO hive_tbl_02 VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
71+
spark.sql(s"INSERT INTO hive_tbl_02$format VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
7272

7373
// create paimon table
7474
spark.sql(s"""
75-
|CREATE TABLE paimon_tbl_02 (id STRING, name STRING, pt STRING)
75+
|CREATE TABLE paimon_tbl_02$format (id STRING, name STRING, pt STRING)
7676
|USING PAIMON
7777
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
7878
|""".stripMargin)
7979

80-
spark.sql(s"INSERT INTO paimon_tbl_02 VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
80+
spark.sql(s"INSERT INTO paimon_tbl_02$format VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
8181

8282
spark.sql(
83-
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl_02', target_table => '$hiveDbName.paimon_tbl_02', parallelism => 6)")
83+
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl_02$format', target_table => '$hiveDbName.paimon_tbl_02$format', parallelism => 6)")
8484

8585
checkAnswer(
86-
spark.sql("SELECT * FROM paimon_tbl_02 ORDER BY id"),
86+
spark.sql(s"SELECT * FROM paimon_tbl_02$format ORDER BY id"),
8787
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1") :: Row(
8888
"4",
8989
"d",
@@ -96,31 +96,31 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
9696
format => {
9797
test(
9898
s"Paimon migrate file procedure: migrate $format non-partitioned table with delete source table") {
99-
withTable("hive_tbl", "paimon_tbl") {
99+
withTable(s"hive_tbl$format", s"paimon_tbl$format") {
100100
// create hive table
101101
spark.sql(s"""
102-
|CREATE TABLE hive_tbl (id STRING, name STRING, pt STRING)
102+
|CREATE TABLE hive_tbl$format (id STRING, name STRING, pt STRING)
103103
|USING $format
104104
|""".stripMargin)
105105

106-
spark.sql(s"INSERT INTO hive_tbl VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
106+
spark.sql(s"INSERT INTO hive_tbl$format VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
107107

108108
// create paimon table
109109
spark.sql(s"""
110-
|CREATE TABLE paimon_tbl (id STRING, name STRING, pt STRING)
110+
|CREATE TABLE paimon_tbl$format (id STRING, name STRING, pt STRING)
111111
|USING PAIMON
112112
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
113113
|""".stripMargin)
114114

115-
spark.sql(s"INSERT INTO paimon_tbl VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
115+
spark.sql(s"INSERT INTO paimon_tbl$format VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
116116

117117
spark.sql(
118-
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl', target_table => '$hiveDbName.paimon_tbl', delete_origin => false)")
118+
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl$format', target_table => '$hiveDbName.paimon_tbl$format', delete_origin => false)")
119119

120-
checkAnswer(spark.sql("SELECT * FROM hive_tbl ORDER BY id"), Nil)
120+
checkAnswer(spark.sql(s"SELECT * FROM hive_tbl$format ORDER BY id"), Nil)
121121

122122
checkAnswer(
123-
spark.sql("SELECT * FROM paimon_tbl ORDER BY id"),
123+
spark.sql(s"SELECT * FROM paimon_tbl$format ORDER BY id"),
124124
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1") :: Row(
125125
"4",
126126
"d",
@@ -132,31 +132,31 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
132132
Seq("parquet", "orc", "avro").foreach(
133133
format => {
134134
test(s"Paimon migrate file procedure: migrate $format partitioned table") {
135-
withTable("hive_tbl", "paimon_tbl") {
135+
withTable(s"hive_tbl$format", s"paimon_tbl$format") {
136136
// create hive table
137137
spark.sql(s"""
138-
|CREATE TABLE hive_tbl (id STRING, name STRING, pt STRING)
138+
|CREATE TABLE hive_tbl$format (id STRING, name STRING, pt STRING)
139139
|USING $format
140140
|PARTITIONED BY (pt)
141141
|""".stripMargin)
142142

143-
spark.sql(s"INSERT INTO hive_tbl VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
143+
spark.sql(s"INSERT INTO hive_tbl$format VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
144144

145145
// create paimon table
146146
spark.sql(s"""
147-
|CREATE TABLE paimon_tbl (id STRING, name STRING, pt STRING)
147+
|CREATE TABLE paimon_tbl$format (id STRING, name STRING, pt STRING)
148148
|USING PAIMON
149149
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
150150
|PARTITIONED BY (pt)
151151
|""".stripMargin)
152152

153-
spark.sql(s"INSERT INTO paimon_tbl VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
153+
spark.sql(s"INSERT INTO paimon_tbl$format VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
154154

155155
spark.sql(
156-
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl', target_table => '$hiveDbName.paimon_tbl')")
156+
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl$format', target_table => '$hiveDbName.paimon_tbl$format')")
157157

158158
checkAnswer(
159-
spark.sql("SELECT * FROM paimon_tbl ORDER BY id"),
159+
spark.sql(s"SELECT * FROM paimon_tbl$format ORDER BY id"),
160160
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1") :: Row(
161161
"4",
162162
"d",
@@ -169,37 +169,37 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
169169
format => {
170170
test(
171171
s"Paimon migrate file procedure: migrate $format partitioned table with delete source table") {
172-
withTable("hive_tbl", "paimon_tbl") {
172+
withTable(s"hive_tbl$format", s"paimon_tbl$format") {
173173
// create hive table
174174
spark.sql(s"""
175-
|CREATE TABLE hive_tbl (id STRING, name STRING, pt STRING)
175+
|CREATE TABLE hive_tbl$format (id STRING, name STRING, pt STRING)
176176
|USING $format
177177
|PARTITIONED BY (pt)
178178
|""".stripMargin)
179179

180-
spark.sql(s"INSERT INTO hive_tbl VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
180+
spark.sql(s"INSERT INTO hive_tbl$format VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
181181

182182
// create paimon table
183183
spark.sql(s"""
184-
|CREATE TABLE paimon_tbl (id STRING, name STRING, pt STRING)
184+
|CREATE TABLE paimon_tbl$format (id STRING, name STRING, pt STRING)
185185
|USING PAIMON
186186
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
187187
|PARTITIONED BY (pt)
188188
|""".stripMargin)
189189

190-
spark.sql(s"INSERT INTO paimon_tbl VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
190+
spark.sql(s"INSERT INTO paimon_tbl$format VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
191191

192192
spark.sql(
193-
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl', target_table => '$hiveDbName.paimon_tbl', delete_origin => false)")
193+
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl$format', target_table => '$hiveDbName.paimon_tbl$format', delete_origin => false)")
194194

195195
checkAnswer(
196-
spark.sql("SELECT * FROM paimon_tbl ORDER BY id"),
196+
spark.sql(s"SELECT * FROM paimon_tbl$format ORDER BY id"),
197197
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1") :: Row(
198198
"4",
199199
"d",
200200
"p2") :: Nil)
201201

202-
checkAnswer(spark.sql("SELECT * FROM hive_tbl ORDER BY id"), Nil)
202+
checkAnswer(spark.sql(s"SELECT * FROM hive_tbl$format ORDER BY id"), Nil)
203203
}
204204
}
205205
})

0 commit comments

Comments
 (0)