Skip to content

Commit a31ac4a

Browse files
author
welsh_wen
committed
[E2E Test][Improvement] Add Test Case for LocalFile CSV Source
1 parent dadb35d commit a31ac4a

File tree

4 files changed

+225
-0
lines changed

4 files changed

+225
-0
lines changed

Diff for: ‎seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java

+7
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,11 @@ public class LocalFileIT extends TestSuiteBase {
237237
Path txtLzo = convertToLzoFile(ContainerUtil.getResourcesFile("/text/e2e.txt"));
238238
ContainerUtil.copyFileIntoContainers(
239239
txtLzo, "/seatunnel/read/lzo_text/e2e.txt", container);
240+
ContainerUtil.copyFileIntoContainers(
241+
"/excel/e2e.csv",
242+
"/seatunnel/read/csv/name=tyrantlucifer/hobby=coding/e2e.csv",
243+
container);
244+
240245
ContainerUtil.copyFileIntoContainers(
241246
"/excel/e2e.xlsx",
242247
"/seatunnel/read/excel/name=tyrantlucifer/hobby=coding/e2e.xlsx",
@@ -303,6 +308,8 @@ public class LocalFileIT extends TestSuiteBase {
303308
public void testLocalFileReadAndWrite(TestContainer container)
304309
throws IOException, InterruptedException {
305310
TestHelper helper = new TestHelper(container);
311+
helper.execute("/excel/local_csv_to_assert.conf");
312+
helper.execute("/excel/local_csv_to_assert_with_multipletable.conf");
306313
helper.execute("/excel/fake_to_local_csv.conf");
307314
helper.execute("/excel/fake_to_local_excel.conf");
308315
helper.execute("/excel/local_excel_to_assert.conf");

Diff for: ‎seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/e2e.csv

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
c_map c_array c_string c_boolean c_tinyint c_smallint c_int c_bigint c_float c_double c_bytes c_date c_decimal c_timestamp c_row
2+
{"OQBqH":"wTKAH","rkvlO":"KXStv","pCMEX":"CyJKx","DAgdj":"SMbQe","dsJag":"jyFsb"} [111,222,333,444,555] rDAya true 25 22478 1333226130 3261 3.26072 3.2606 [77, 116, 89, 118, 119] 2023-06-13 3.26072348493343 2023-05-17 00:36:12 "gfBji":"emeXF","BCStD":"cUZwy","XyMkz":"CWBGW","nTHbQ":"Oaaos","IwjTs":"zyGjf"};[1939637044,1066960875,2141621606,1638720300,1269572120];YjilG;false;11;26418;1110174340;4728476106429842432;2.0147008E38;8.338846635117556E307;UxtrR;2023-12-18;76273205606379580629.661783233489743766;2023-03-04 07:08:58
3+
{"OQBqH":"wTKAH","rkvlO":"KXStv","pCMEX":"CyJKx","DAgdj":"SMbQe","dsJag":"jyFsb"} [111,222,333,444,555] rDAya true 22 22472 1333226132 3262 3.26072 3.2602 [77, 116, 89, 118, 112] 2023-06-12 3.26072348493342 2023-05-17 00:36:12 "gfBji":"emeXF","BCStD":"cUZwy","XyMkz":"CWBGW","nTHbQ":"Oaaos","IwjTs":"zyGjf"};[1939637044,1066960875,2141621606,1638720300,1269572120];YjilG;false;11;26418;1110174340;4728476106429842432;2.0147008E38;8.338846635117556E307;UxtrR;2023-12-18;76273205606379580629.661783233489743766;2023-03-04 07:08:58

Diff for: ‎seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/local_csv_to_assert.conf

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
env {
19+
parallelism = 1
20+
job.mode = "BATCH"
21+
# You can set spark configuration here
22+
spark.app.name = "SeaTunnel"
23+
spark.executor.instances = 2
24+
spark.executor.cores = 1
25+
spark.executor.memory = "1g"
26+
spark.master = local
27+
job.mode = "BATCH"
28+
}
29+
30+
source {
31+
LocalFile {
32+
path = "/seatunnel/read/csv"
33+
plugin_output = "fake"
34+
file_format_type = csv
35+
field_delimiter = "\t"
36+
row_delimiter = "\n"
37+
skip_header_row_number = 1
38+
schema = {
39+
table = "fake01"
40+
fields {
41+
c_map = "map<string, string>"
42+
c_array = "array<int>"
43+
c_string = string
44+
c_boolean = boolean
45+
c_tinyint = tinyint
46+
c_smallint = smallint
47+
c_int = int
48+
c_bigint = bigint
49+
c_float = float
50+
c_double = double
51+
c_bytes = bytes
52+
c_date = date
53+
c_decimal = "decimal(38, 18)"
54+
c_timestamp = timestamp,
55+
c_row = {
56+
c_map = "map<string, string>"
57+
c_array = "array<int>"
58+
c_string = string
59+
c_boolean = boolean
60+
c_tinyint = tinyint
61+
c_smallint = smallint
62+
c_int = int
63+
c_bigint = bigint
64+
c_float = float
65+
c_double = double
66+
c_bytes = bytes
67+
c_date = date
68+
c_decimal = "decimal(38, 18)"
69+
c_timestamp = timestamp
70+
}
71+
}
72+
}
73+
}
74+
}
75+
76+
sink {
77+
Assert {
78+
rules {
79+
row_rules = [
80+
{
81+
rule_type = MAX_ROW
82+
rule_value = 2
83+
}
84+
],
85+
table-names = ["fake01"]
86+
}
87+
}
88+
}

Diff for: ‎seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/excel/local_csv_to_assert_with_multipletable.conf

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
env {
19+
parallelism = 1
20+
job.mode = "BATCH"
21+
# You can set spark configuration here
22+
spark.app.name = "SeaTunnel"
23+
spark.executor.instances = 2
24+
spark.executor.cores = 1
25+
spark.executor.memory = "1g"
26+
spark.master = local
27+
}
28+
29+
source {
30+
LocalFile {
31+
tables_configs = [
32+
{
33+
path = "/seatunnel/read/csv"
34+
file_format_type = csv
35+
field_delimiter = "\t"
36+
row_delimiter = "\n"
37+
skip_header_row_number = 1
38+
schema = {
39+
table = "fake01"
40+
fields {
41+
c_map = "map<string, string>"
42+
c_array = "array<int>"
43+
c_string = string
44+
c_boolean = boolean
45+
c_tinyint = tinyint
46+
c_smallint = smallint
47+
c_int = int
48+
c_bigint = bigint
49+
c_float = float
50+
c_double = double
51+
c_bytes = bytes
52+
c_date = date
53+
c_decimal = "decimal(38, 18)"
54+
c_timestamp = timestamp
55+
c_row = {
56+
c_map = "map<string, string>"
57+
c_array = "array<int>"
58+
c_string = string
59+
c_boolean = boolean
60+
c_tinyint = tinyint
61+
c_smallint = smallint
62+
c_int = int
63+
c_bigint = bigint
64+
c_float = float
65+
c_double = double
66+
c_bytes = bytes
67+
c_date = date
68+
c_decimal = "decimal(38, 18)"
69+
c_timestamp = timestamp
70+
}
71+
}
72+
}
73+
},
74+
{
75+
path = "/seatunnel/read/csv"
76+
file_format_type = csv
77+
field_delimiter = "\t"
78+
row_delimiter = "\n"
79+
skip_header_row_number = 1
80+
schema = {
81+
table = "fake02"
82+
fields {
83+
c_map = "map<string, string>"
84+
c_array = "array<int>"
85+
c_string = string
86+
c_boolean = boolean
87+
c_tinyint = tinyint
88+
c_smallint = smallint
89+
c_int = int
90+
c_bigint = bigint
91+
c_float = float
92+
c_double = double
93+
c_bytes = bytes
94+
c_date = date
95+
c_decimal = "decimal(38, 18)"
96+
c_timestamp = timestamp
97+
c_row = {
98+
c_map = "map<string, string>"
99+
c_array = "array<int>"
100+
c_string = string
101+
c_boolean = boolean
102+
c_tinyint = tinyint
103+
c_smallint = smallint
104+
c_int = int
105+
c_bigint = bigint
106+
c_float = float
107+
c_double = double
108+
c_bytes = bytes
109+
c_date = date
110+
c_decimal = "decimal(38, 18)"
111+
c_timestamp = timestamp
112+
}
113+
}
114+
}
115+
}
116+
]
117+
plugin_output = "fake"
118+
}
119+
}
120+
121+
sink {
122+
Assert {
123+
rules {
124+
table-names = ["fake01", "fake02"]
125+
}
126+
}
127+
}

0 commit comments

Comments
 (0)