Skip to content

Commit a45ed2a

Browse files
author
welsh_wen
committed
[E2E Test][Improvement] Add Test Case for LocalFile CSV Sink
1 parent 49ec541 commit a45ed2a

File tree

2 files changed

+92
-1
lines changed

2 files changed

+92
-1
lines changed

seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java

+12-1
Original file line numberDiff line numberDiff line change
@@ -295,9 +295,15 @@ public class LocalFileIT extends TestSuiteBase {
295295
};
296296

297297
@TestTemplate
298+
@DisabledOnContainer(
299+
value = {TestContainerId.SPARK_2_4},
300+
type = {EngineType.SPARK, EngineType.FLINK},
301+
disabledReason =
302+
"Fink test is multi-node, LocalFile connector will use different containers for obtaining files")
298303
public void testLocalFileReadAndWrite(TestContainer container)
299304
throws IOException, InterruptedException {
300305
TestHelper helper = new TestHelper(container);
306+
helper.execute("/excel/fake_to_local_csv.conf");
301307
helper.execute("/excel/fake_to_local_excel.conf");
302308
helper.execute("/excel/local_excel_to_assert.conf");
303309
helper.execute("/excel/local_excel_projection_to_assert.conf");
@@ -391,7 +397,7 @@ public void testLocalFileReadAndWrite(TestContainer container)
391397
@TestTemplate
392398
@DisabledOnContainer(
393399
value = {TestContainerId.SPARK_2_4},
394-
type = {EngineType.FLINK},
400+
type = {EngineType.SPARK, EngineType.FLINK},
395401
disabledReason =
396402
"Fink test is multi-node, LocalFile connector will use different containers for obtaining files")
397403
public void testLocalFileReadAndWriteWithSaveMode(TestContainer container)
@@ -437,6 +443,11 @@ private List<String> getFileListFromContainer(String path) {
437443
}
438444

439445
@TestTemplate
446+
@DisabledOnContainer(
447+
value = {TestContainerId.SPARK_2_4},
448+
type = {EngineType.SPARK, EngineType.FLINK},
449+
disabledReason =
450+
"Fink test is multi-node, LocalFile connector will use different containers for obtaining files")
440451
public void testLocalFileCatalog(TestContainer container)
441452
throws IOException, InterruptedException {
442453
final LocalFileCatalog localFileCatalog =
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
env {
19+
parallelism = 1
20+
job.mode = "BATCH"
21+
22+
# You can set spark configuration here
23+
spark.app.name = "SeaTunnel"
24+
spark.executor.instances = 2
25+
spark.executor.cores = 1
26+
spark.executor.memory = "1g"
27+
spark.master = local
28+
}
29+
30+
source {
31+
FakeSource {
32+
plugin_output = "fake"
33+
schema = {
34+
fields {
35+
c_map = "map<string, string>"
36+
c_array = "array<int>"
37+
c_string = string
38+
c_boolean = boolean
39+
c_tinyint = tinyint
40+
c_smallint = smallint
41+
c_int = int
42+
c_bigint = bigint
43+
c_float = float
44+
c_double = double
45+
c_bytes = bytes
46+
c_date = date
47+
c_decimal = "decimal(38, 18)"
48+
c_timestamp = timestamp
49+
c_row = {
50+
c_map = "map<string, string>"
51+
c_array = "array<int>"
52+
c_string = string
53+
c_boolean = boolean
54+
c_tinyint = tinyint
55+
c_smallint = smallint
56+
c_int = int
57+
c_bigint = bigint
58+
c_float = float
59+
c_double = double
60+
c_bytes = bytes
61+
c_date = date
62+
c_decimal = "decimal(38, 18)"
63+
c_timestamp = timestamp
64+
}
65+
}
66+
}
67+
}
68+
}
69+
70+
sink {
71+
LocalFile {
72+
path = "/tmp/seatunnel/csv"
73+
partition_dir_expression = "${k0}=${v0}"
74+
is_partition_field_write_in_file = true
75+
file_name_expression = "${transactionId}_${now}"
76+
file_format_type = "csv"
77+
filename_time_format = "yyyy.MM.dd"
78+
is_enable_transaction = true
79+
}
80+
}

0 commit comments

Comments
 (0)