Skip to content

Commit 6468a1b

Browse files
liunaijieliunaijie
authored
[improve] add connector options class exist check (#8600)
1 parent de44573 commit 6468a1b

File tree

1 file changed

+252
-0
lines changed

1 file changed

+252
-0
lines changed

‎seatunnel-ci-tools/src/test/java/org/apache/seatunnel/api/ConnectorOptionCheckTest.java

+252
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.seatunnel.api;
19+
20+
import org.junit.jupiter.api.Assertions;
21+
import org.junit.jupiter.api.Test;
22+
23+
import com.github.javaparser.JavaParser;
24+
import com.github.javaparser.ParseResult;
25+
import com.github.javaparser.ast.CompilationUnit;
26+
import com.github.javaparser.ast.NodeList;
27+
import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration;
28+
import com.github.javaparser.ast.type.ClassOrInterfaceType;
29+
import lombok.extern.slf4j.Slf4j;
30+
31+
import java.io.File;
32+
import java.io.IOException;
33+
import java.nio.file.FileVisitOption;
34+
import java.nio.file.Files;
35+
import java.nio.file.Path;
36+
import java.nio.file.Paths;
37+
import java.util.HashSet;
38+
import java.util.List;
39+
import java.util.Set;
40+
import java.util.stream.Collectors;
41+
import java.util.stream.Stream;
42+
43+
@Slf4j
44+
public class ConnectorOptionCheckTest {
45+
46+
private static final String javaPathFragment =
47+
"src" + File.separator + "main" + File.separator + "java";
48+
private static final String JAVA_FILE_EXTENSION = ".java";
49+
private static final String CONNECTOR_DIR = "seatunnel-connectors-v2";
50+
private static final JavaParser JAVA_PARSER = new JavaParser();
51+
52+
@Test
53+
public void checkConnectorOptionExist() {
54+
Set<String> connectorOptionFileNames = new HashSet<>();
55+
Set<String> whiteListConnectorOptionFileNames = buildWhiteList();
56+
try (Stream<Path> paths = Files.walk(Paths.get(".."), FileVisitOption.FOLLOW_LINKS)) {
57+
List<Path> connectorClassPaths =
58+
paths.filter(
59+
path -> {
60+
String pathString = path.toString();
61+
return pathString.endsWith(JAVA_FILE_EXTENSION)
62+
&& pathString.contains(CONNECTOR_DIR)
63+
&& pathString.contains(javaPathFragment);
64+
})
65+
.collect(Collectors.toList());
66+
connectorClassPaths.forEach(
67+
path -> {
68+
try {
69+
ParseResult<CompilationUnit> parseResult =
70+
JAVA_PARSER.parse(Files.newInputStream(path));
71+
parseResult
72+
.getResult()
73+
.ifPresent(
74+
compilationUnit -> {
75+
List<ClassOrInterfaceDeclaration> classes =
76+
compilationUnit.findAll(
77+
ClassOrInterfaceDeclaration.class);
78+
for (ClassOrInterfaceDeclaration classDeclaration :
79+
classes) {
80+
if (classDeclaration.isAbstract()
81+
|| classDeclaration.isInterface()) {
82+
continue;
83+
}
84+
NodeList<ClassOrInterfaceType>
85+
implementedTypes =
86+
classDeclaration
87+
.getImplementedTypes();
88+
implementedTypes.forEach(
89+
implementedType -> {
90+
if (implementedType
91+
.getNameAsString()
92+
.equals(
93+
"SeaTunnelSource")
94+
|| implementedType
95+
.getNameAsString()
96+
.equals(
97+
"SeaTunnelSink")) {
98+
connectorOptionFileNames.add(
99+
path.getFileName()
100+
.toString()
101+
.replace(
102+
JAVA_FILE_EXTENSION,
103+
"")
104+
.concat(
105+
"Options"));
106+
}
107+
});
108+
NodeList<ClassOrInterfaceType> extendedTypes =
109+
classDeclaration.getExtendedTypes();
110+
extendedTypes.forEach(
111+
extendedType -> {
112+
if (extendedType
113+
.getNameAsString()
114+
.equals(
115+
"AbstractSimpleSink")
116+
|| extendedType
117+
.getNameAsString()
118+
.equals(
119+
"AbstractSingleSplitSource")
120+
|| extendedType
121+
.getNameAsString()
122+
.equals(
123+
"IncrementalSource")) {
124+
connectorOptionFileNames.add(
125+
path.getFileName()
126+
.toString()
127+
.replace(
128+
JAVA_FILE_EXTENSION,
129+
"")
130+
.concat(
131+
"Options"));
132+
}
133+
});
134+
}
135+
});
136+
} catch (IOException e) {
137+
throw new RuntimeException(e);
138+
}
139+
});
140+
connectorClassPaths.forEach(
141+
path -> {
142+
String className =
143+
path.getFileName().toString().replace(JAVA_FILE_EXTENSION, "");
144+
connectorOptionFileNames.remove(className);
145+
});
146+
147+
whiteListConnectorOptionFileNames.forEach(
148+
whiteListConnectorOptionFileName -> {
149+
Assertions.assertTrue(
150+
connectorOptionFileNames.remove(whiteListConnectorOptionFileName),
151+
"This [Options] class is in white list, but not found related connector classes, please check: ["
152+
+ whiteListConnectorOptionFileName
153+
+ "]\n");
154+
});
155+
156+
Assertions.assertEquals(
157+
0,
158+
connectorOptionFileNames.size(),
159+
() ->
160+
"Connector class does not have correspondingly [Options] class. "
161+
+ "The connector need put all parameter into <ConnectorClassName>Options classes, like [ActivemqSink] and [ActivemqSinkOptions].\n"
162+
+ "Those [Options] class are missing: \n"
163+
+ String.join("\n", connectorOptionFileNames)
164+
+ "\n");
165+
} catch (IOException e) {
166+
throw new RuntimeException(e);
167+
}
168+
}
169+
170+
private Set<String> buildWhiteList() {
171+
Set<String> whiteList = new HashSet<>();
172+
whiteList.add("ElasticsearchSourceOptions");
173+
whiteList.add("JdbcSinkOptions");
174+
whiteList.add("TypesenseSourceOptions");
175+
whiteList.add("RabbitmqSourceOptions");
176+
whiteList.add("TypesenseSinkOptions");
177+
whiteList.add("EmailSinkOptions");
178+
whiteList.add("HudiSinkOptions");
179+
whiteList.add("PulsarSinkOptions");
180+
whiteList.add("HttpSinkOptions");
181+
whiteList.add("SlsSinkOptions");
182+
whiteList.add("DingTalkSinkOptions");
183+
whiteList.add("Neo4jSinkOptions");
184+
whiteList.add("SlackSinkOptions");
185+
whiteList.add("MaxcomputeSinkOptions");
186+
whiteList.add("PaimonSinkOptions");
187+
whiteList.add("TDengineSourceOptions");
188+
whiteList.add("PulsarSourceOptions");
189+
whiteList.add("RedisSinkOptions");
190+
whiteList.add("DruidSinkOptions");
191+
whiteList.add("FakeSourceOptions");
192+
whiteList.add("HbaseSinkOptions");
193+
whiteList.add("MongodbSinkOptions");
194+
whiteList.add("IoTDBSinkOptions");
195+
whiteList.add("ActivemqSinkOptions");
196+
whiteList.add("EasysearchSourceOptions");
197+
whiteList.add("RabbitmqSinkOptions");
198+
whiteList.add("StarRocksSourceOptions");
199+
whiteList.add("AmazonSqsSinkOptions");
200+
whiteList.add("IcebergSourceOptions");
201+
whiteList.add("HbaseSourceOptions");
202+
whiteList.add("PaimonSourceOptions");
203+
whiteList.add("IoTDBSourceOptions");
204+
whiteList.add("SlsSourceOptions");
205+
whiteList.add("SentrySinkOptions");
206+
whiteList.add("EasysearchSinkOptions");
207+
whiteList.add("QdrantSinkOptions");
208+
whiteList.add("ElasticsearchSinkOptions");
209+
whiteList.add("MilvusSourceOptions");
210+
whiteList.add("RocketMqSinkOptions");
211+
whiteList.add("ClickhouseFileSinkOptions");
212+
whiteList.add("IcebergSinkOptions");
213+
whiteList.add("KafkaSourceOptions");
214+
whiteList.add("AssertSinkOptions");
215+
whiteList.add("MaxcomputeSourceOptions");
216+
whiteList.add("InfluxDBSourceOptions");
217+
whiteList.add("InfluxDBSinkOptions");
218+
whiteList.add("CassandraSinkOptions");
219+
whiteList.add("KuduSourceOptions");
220+
whiteList.add("SocketSinkOptions");
221+
whiteList.add("DataHubSinkOptions");
222+
whiteList.add("ClickhouseSinkOptions");
223+
whiteList.add("SelectDBSinkOptions");
224+
whiteList.add("ConsoleSinkOptions");
225+
whiteList.add("PrometheusSinkOptions");
226+
whiteList.add("FirestoreSinkOptions");
227+
whiteList.add("ClickhouseSourceOptions");
228+
whiteList.add("MilvusSinkOptions");
229+
whiteList.add("RocketMqSourceOptions");
230+
whiteList.add("TablestoreSinkOptions");
231+
whiteList.add("TableStoreDBSourceOptions");
232+
whiteList.add("AmazonDynamoDBSinkOptions");
233+
whiteList.add("KuduSinkOptions");
234+
whiteList.add("KafkaSinkOptions");
235+
whiteList.add("TDengineSinkOptions");
236+
whiteList.add("Neo4jSourceOptions");
237+
whiteList.add("HttpSourceOptions");
238+
whiteList.add("QdrantSourceOptions");
239+
whiteList.add("SheetsSourceOptions");
240+
whiteList.add("SocketSourceOptions");
241+
whiteList.add("OpenMldbSourceOptions");
242+
whiteList.add("Web3jSourceOptions");
243+
whiteList.add("CassandraSourceOptions");
244+
whiteList.add("RedisSourceOptions");
245+
whiteList.add("PostgresIncrementalSourceOptions");
246+
whiteList.add("SqlServerIncrementalSourceOptions");
247+
whiteList.add("OracleIncrementalSourceOptions");
248+
whiteList.add("MySqlIncrementalSourceOptions");
249+
whiteList.add("MongodbIncrementalSourceOptions");
250+
return whiteList;
251+
}
252+
}

0 commit comments

Comments
 (0)