Skip to content

Commit daaca52

Browse files
JingsongLiJingsongLi
committed
[test] Add format end to Spark migrate file test
1 parent 78f73eb commit daaca52

File tree

1 file changed

+47
-37
lines changed

1 file changed

+47
-37
lines changed

‎paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateFileProcedureTest.scala‎

Lines changed: 47 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -31,29 +31,31 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
3131
Seq("parquet", "orc", "avro").foreach(
3232
format => {
3333
test(s"Paimon migrate file procedure: migrate $format non-partitioned table") {
34-
withTable(s"hive_tbl1$random", s"paimon_tbl1$random") {
34+
withTable(s"hive_tbl1$random$format", s"paimon_tbl1$random$format") {
3535
// create hive table
3636
spark.sql(s"""
37-
|CREATE TABLE hive_tbl1$random (id STRING, name STRING, pt STRING)
37+
|CREATE TABLE hive_tbl1$random$format (id STRING, name STRING, pt STRING)
3838
|USING $format
3939
|""".stripMargin)
4040

41-
spark.sql(s"INSERT INTO hive_tbl1$random VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
41+
spark.sql(
42+
s"INSERT INTO hive_tbl1$random$format VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
4243

4344
// create paimon table
4445
spark.sql(s"""
45-
|CREATE TABLE paimon_tbl1$random (id STRING, name STRING, pt STRING)
46+
|CREATE TABLE paimon_tbl1$random$format (id STRING, name STRING, pt STRING)
4647
|USING PAIMON
4748
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
4849
|""".stripMargin)
4950

50-
spark.sql(s"INSERT INTO paimon_tbl1$random VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
51+
spark.sql(
52+
s"INSERT INTO paimon_tbl1$random$format VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
5153

5254
spark.sql(
53-
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl1$random', target_table => '$hiveDbName.paimon_tbl1$random')")
55+
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl1$random$format', target_table => '$hiveDbName.paimon_tbl1$random$format')")
5456

5557
checkAnswer(
56-
spark.sql(s"SELECT * FROM paimon_tbl1$random ORDER BY id"),
58+
spark.sql(s"SELECT * FROM paimon_tbl1$random$format ORDER BY id"),
5759
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1") :: Row(
5860
"4",
5961
"d",
@@ -66,29 +68,31 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
6668
format => {
6769
test(
6870
s"Paimon migrate file procedure: migrate $format non-partitioned table with parallelism") {
69-
withTable(s"hive_tbl_02$random", s"paimon_tbl_02$random") {
71+
withTable(s"hive_tbl_02$random$format", s"paimon_tbl_02$random$format") {
7072
// create hive table
7173
spark.sql(s"""
72-
|CREATE TABLE hive_tbl_02$random (id STRING, name STRING, pt STRING)
74+
|CREATE TABLE hive_tbl_02$random$format (id STRING, name STRING, pt STRING)
7375
|USING $format
7476
|""".stripMargin)
7577

76-
spark.sql(s"INSERT INTO hive_tbl_02$random VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
78+
spark.sql(
79+
s"INSERT INTO hive_tbl_02$random$format VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
7780

7881
// create paimon table
7982
spark.sql(s"""
80-
|CREATE TABLE paimon_tbl_02$random (id STRING, name STRING, pt STRING)
83+
|CREATE TABLE paimon_tbl_02$random$format (id STRING, name STRING, pt STRING)
8184
|USING PAIMON
8285
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
8386
|""".stripMargin)
8487

85-
spark.sql(s"INSERT INTO paimon_tbl_02$random VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
88+
spark.sql(
89+
s"INSERT INTO paimon_tbl_02$random$format VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
8690

8791
spark.sql(
88-
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl_02$random', target_table => '$hiveDbName.paimon_tbl_02$random', parallelism => 6)")
92+
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl_02$random$format', target_table => '$hiveDbName.paimon_tbl_02$random$format', parallelism => 6)")
8993

9094
checkAnswer(
91-
spark.sql(s"SELECT * FROM paimon_tbl_02$random ORDER BY id"),
95+
spark.sql(s"SELECT * FROM paimon_tbl_02$random$format ORDER BY id"),
9296
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1") :: Row(
9397
"4",
9498
"d",
@@ -101,31 +105,33 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
101105
format => {
102106
test(
103107
s"Paimon migrate file procedure: migrate $format non-partitioned table with delete source table") {
104-
withTable(s"hive_tbl3$random", s"paimon_tbl3$random") {
108+
withTable(s"hive_tbl3$random$format", s"paimon_tbl3$random$format") {
105109
// create hive table
106110
spark.sql(s"""
107-
|CREATE TABLE hive_tbl3$random (id STRING, name STRING, pt STRING)
111+
|CREATE TABLE hive_tbl3$random$format (id STRING, name STRING, pt STRING)
108112
|USING $format
109113
|""".stripMargin)
110114

111-
spark.sql(s"INSERT INTO hive_tbl3$random VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
115+
spark.sql(
116+
s"INSERT INTO hive_tbl3$random$format VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
112117

113118
// create paimon table
114119
spark.sql(s"""
115-
|CREATE TABLE paimon_tbl3$random (id STRING, name STRING, pt STRING)
120+
|CREATE TABLE paimon_tbl3$random$format (id STRING, name STRING, pt STRING)
116121
|USING PAIMON
117122
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
118123
|""".stripMargin)
119124

120-
spark.sql(s"INSERT INTO paimon_tbl3$random VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
125+
spark.sql(
126+
s"INSERT INTO paimon_tbl3$random$format VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
121127

122128
spark.sql(
123-
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl3$random', target_table => '$hiveDbName.paimon_tbl3$random', delete_origin => false)")
129+
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl3$random$format', target_table => '$hiveDbName.paimon_tbl3$random$format', delete_origin => false)")
124130

125-
checkAnswer(spark.sql(s"SELECT * FROM hive_tbl3$random ORDER BY id"), Nil)
131+
checkAnswer(spark.sql(s"SELECT * FROM hive_tbl3$random$format ORDER BY id"), Nil)
126132

127133
checkAnswer(
128-
spark.sql(s"SELECT * FROM paimon_tbl3$random ORDER BY id"),
134+
spark.sql(s"SELECT * FROM paimon_tbl3$random$format ORDER BY id"),
129135
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1") :: Row(
130136
"4",
131137
"d",
@@ -137,31 +143,33 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
137143
Seq("parquet", "orc", "avro").foreach(
138144
format => {
139145
test(s"Paimon migrate file procedure: migrate $format partitioned table") {
140-
withTable(s"hive_tbl4$random", s"paimon_tbl4$random") {
146+
withTable(s"hive_tbl4$random$format", s"paimon_tbl4$random$format") {
141147
// create hive table
142148
spark.sql(s"""
143-
|CREATE TABLE hive_tbl4$random (id STRING, name STRING, pt STRING)
149+
|CREATE TABLE hive_tbl4$random$format (id STRING, name STRING, pt STRING)
144150
|USING $format
145151
|PARTITIONED BY (pt)
146152
|""".stripMargin)
147153

148-
spark.sql(s"INSERT INTO hive_tbl4$random VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
154+
spark.sql(
155+
s"INSERT INTO hive_tbl4$random$format VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
149156

150157
// create paimon table
151158
spark.sql(s"""
152-
|CREATE TABLE paimon_tbl4$random (id STRING, name STRING, pt STRING)
159+
|CREATE TABLE paimon_tbl4$random$format (id STRING, name STRING, pt STRING)
153160
|USING PAIMON
154161
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
155162
|PARTITIONED BY (pt)
156163
|""".stripMargin)
157164

158-
spark.sql(s"INSERT INTO paimon_tbl4$random VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
165+
spark.sql(
166+
s"INSERT INTO paimon_tbl4$random$format VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
159167

160168
spark.sql(
161-
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl4$random', target_table => '$hiveDbName.paimon_tbl4$random')")
169+
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl4$random$format', target_table => '$hiveDbName.paimon_tbl4$random$format')")
162170

163171
checkAnswer(
164-
spark.sql(s"SELECT * FROM paimon_tbl4$random ORDER BY id"),
172+
spark.sql(s"SELECT * FROM paimon_tbl4$random$format ORDER BY id"),
165173
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1") :: Row(
166174
"4",
167175
"d",
@@ -174,37 +182,39 @@ class MigrateFileProcedureTest extends PaimonHiveTestBase {
174182
format => {
175183
test(
176184
s"Paimon migrate file procedure: migrate $format partitioned table with delete source table") {
177-
withTable(s"hive_tbl5$random", s"paimon_tbl5$random") {
185+
withTable(s"hive_tbl5$random$format", s"paimon_tbl5$random$format") {
178186
// create hive table
179187
spark.sql(s"""
180-
|CREATE TABLE hive_tbl5$random (id STRING, name STRING, pt STRING)
188+
|CREATE TABLE hive_tbl5$random$format (id STRING, name STRING, pt STRING)
181189
|USING $format
182190
|PARTITIONED BY (pt)
183191
|""".stripMargin)
184192

185-
spark.sql(s"INSERT INTO hive_tbl5$random VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
193+
spark.sql(
194+
s"INSERT INTO hive_tbl5$random$format VALUES ('1', 'a', 'p1'), ('2', 'b', 'p2')")
186195

187196
// create paimon table
188197
spark.sql(s"""
189-
|CREATE TABLE paimon_tbl5$random (id STRING, name STRING, pt STRING)
198+
|CREATE TABLE paimon_tbl5$random$format (id STRING, name STRING, pt STRING)
190199
|USING PAIMON
191200
|TBLPROPERTIES ('file.format'='$format', 'bucket'='-1')
192201
|PARTITIONED BY (pt)
193202
|""".stripMargin)
194203

195-
spark.sql(s"INSERT INTO paimon_tbl5$random VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
204+
spark.sql(
205+
s"INSERT INTO paimon_tbl5$random$format VALUES ('3', 'c', 'p1'), ('4', 'd', 'p2')")
196206

197207
spark.sql(
198-
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl5$random', target_table => '$hiveDbName.paimon_tbl5$random', delete_origin => false)")
208+
s"CALL sys.migrate_table(source_type => 'hive', table => '$hiveDbName.hive_tbl5$random$format', target_table => '$hiveDbName.paimon_tbl5$random$format', delete_origin => false)")
199209

200210
checkAnswer(
201-
spark.sql(s"SELECT * FROM paimon_tbl5$random ORDER BY id"),
211+
spark.sql(s"SELECT * FROM paimon_tbl5$random$format ORDER BY id"),
202212
Row("1", "a", "p1") :: Row("2", "b", "p2") :: Row("3", "c", "p1") :: Row(
203213
"4",
204214
"d",
205215
"p2") :: Nil)
206216

207-
checkAnswer(spark.sql(s"SELECT * FROM hive_tbl5$random ORDER BY id"), Nil)
217+
checkAnswer(spark.sql(s"SELECT * FROM hive_tbl5$random$format ORDER BY id"), Nil)
208218
}
209219
}
210220
})

0 commit comments

Comments
 (0)