Skip to content

Commit 6dafc16

Browse files
CosmosNihawk9821
CosmosNi
authored and
hawk9821
committed
[Feature][Checkpoint] Add check script for source/sink state class serialVersionUID missing (apache#9118)
1 parent 9b8f61f commit 6dafc16

File tree

88 files changed

+459
-23
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

88 files changed

+459
-23
lines changed

‎seatunnel-ci-tools/pom.xml

+6
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,12 @@
3636
<version>${javaparser.version}</version>
3737
<scope>test</scope>
3838
</dependency>
39+
<dependency>
40+
<groupId>com.github.javaparser</groupId>
41+
<artifactId>javaparser-symbol-solver-core</artifactId>
42+
<version>${javaparser.version}</version>
43+
<scope>test</scope>
44+
</dependency>
3945
</dependencies>
4046

4147
<build>

‎seatunnel-ci-tools/src/test/java/org/apache/seatunnel/api/SerialVersionUIDCheckerTest.java

+321
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,321 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.seatunnel.api;
19+
20+
import org.junit.jupiter.api.Test;
21+
import org.junit.jupiter.api.extension.ExtendWith;
22+
import org.junit.jupiter.api.extension.ExtensionContext;
23+
import org.junit.jupiter.api.extension.TestWatcher;
24+
import org.slf4j.Logger;
25+
import org.slf4j.LoggerFactory;
26+
27+
import com.github.javaparser.JavaParser;
28+
import com.github.javaparser.ParseResult;
29+
import com.github.javaparser.ast.CompilationUnit;
30+
import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration;
31+
import com.github.javaparser.ast.type.ClassOrInterfaceType;
32+
import com.github.javaparser.ast.type.Type;
33+
import com.github.javaparser.resolution.declarations.ResolvedReferenceTypeDeclaration;
34+
import com.github.javaparser.resolution.types.ResolvedReferenceType;
35+
import com.github.javaparser.symbolsolver.JavaSymbolSolver;
36+
import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
37+
import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver;
38+
import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
39+
40+
import java.io.File;
41+
import java.io.IOException;
42+
import java.nio.file.FileVisitOption;
43+
import java.nio.file.Files;
44+
import java.nio.file.Path;
45+
import java.nio.file.Paths;
46+
import java.util.ArrayList;
47+
import java.util.HashMap;
48+
import java.util.HashSet;
49+
import java.util.List;
50+
import java.util.Map;
51+
import java.util.Set;
52+
import java.util.stream.Collectors;
53+
import java.util.stream.Stream;
54+
55+
import static org.junit.jupiter.api.Assertions.fail;
56+
57+
@ExtendWith(SerialVersionUIDCheckerTest.TestResultLogger.class)
58+
public class SerialVersionUIDCheckerTest {
59+
private static final Logger LOG = LoggerFactory.getLogger(SerialVersionUIDCheckerTest.class);
60+
private static final String JAVA_FILE_EXTENSION = ".java";
61+
private static final String CONNECTOR_DIR = "seatunnel-connectors-v2";
62+
private static final String JAVA_PATH_FRAGMENT =
63+
"src" + File.separator + "main" + File.separator + "java";
64+
private static final JavaParser JAVA_PARSER;
65+
private static final Set<String> checkedClasses = new HashSet<>();
66+
private static final Map<String, ClassOrInterfaceDeclaration> classDeclarationMap =
67+
new HashMap<>();
68+
69+
static {
70+
CombinedTypeSolver typeSolver = new CombinedTypeSolver();
71+
typeSolver.add(new ReflectionTypeSolver());
72+
setupTypeSolver(typeSolver);
73+
JavaSymbolSolver symbolSolver = new JavaSymbolSolver(typeSolver);
74+
JAVA_PARSER = new JavaParser();
75+
JAVA_PARSER.getParserConfiguration().setSymbolResolver(symbolSolver);
76+
}
77+
78+
private static void setupTypeSolver(CombinedTypeSolver typeSolver) {
79+
try (Stream<Path> paths = Files.walk(Paths.get(".."), FileVisitOption.FOLLOW_LINKS)) {
80+
paths.filter(path -> path.toString().contains("src/main/java"))
81+
.forEach(
82+
path -> {
83+
try {
84+
typeSolver.add(new JavaParserTypeSolver(path.toFile()));
85+
} catch (Exception e) {
86+
// ignore
87+
}
88+
});
89+
} catch (IOException e) {
90+
LOG.error("Failed to setup type solver", e);
91+
}
92+
}
93+
94+
@Test
95+
public void checkSerialVersionUID() {
96+
List<String> missingSerialVersionUID = new ArrayList<>();
97+
List<Path> connectorClassPaths = findConnectorClassPaths();
98+
LOG.info("Found {} connector class files to check", connectorClassPaths.size());
99+
100+
// First, populate the classDeclarationMap with all classes
101+
for (Path path : connectorClassPaths) {
102+
populateClassDeclarationMap(path);
103+
}
104+
LOG.info("Populated class declaration map with {} classes", classDeclarationMap.size());
105+
106+
// Then check each class path for serialVersionUID
107+
for (Path path : connectorClassPaths) {
108+
checkClassPath(path, missingSerialVersionUID);
109+
}
110+
111+
LOG.info("Check completed. Checked {} connector classes.", connectorClassPaths.size());
112+
if (!missingSerialVersionUID.isEmpty()) {
113+
String errorMessage = generateErrorMessage(missingSerialVersionUID);
114+
LOG.error("Test failed: {}", errorMessage);
115+
fail(errorMessage);
116+
}
117+
LOG.info("All checked classes have correct serialVersionUID.");
118+
}
119+
120+
private List<Path> findConnectorClassPaths() {
121+
try (Stream<Path> paths = Files.walk(Paths.get(".."), FileVisitOption.FOLLOW_LINKS)) {
122+
return paths.filter(
123+
path -> {
124+
String pathString = path.toString();
125+
return pathString.endsWith(JAVA_FILE_EXTENSION)
126+
&& pathString.contains(CONNECTOR_DIR)
127+
&& pathString.contains(JAVA_PATH_FRAGMENT);
128+
})
129+
.collect(Collectors.toList());
130+
} catch (IOException e) {
131+
throw new RuntimeException("Failed to walk through connector directories", e);
132+
}
133+
}
134+
135+
/** Populate the classDeclarationMap with all class declarations from the given path. */
136+
private void populateClassDeclarationMap(Path path) {
137+
try {
138+
ParseResult<CompilationUnit> parseResult =
139+
JAVA_PARSER.parse(Files.newInputStream(path));
140+
parseResult
141+
.getResult()
142+
.ifPresent(
143+
compilationUnit -> {
144+
List<ClassOrInterfaceDeclaration> classes =
145+
compilationUnit.findAll(ClassOrInterfaceDeclaration.class);
146+
for (ClassOrInterfaceDeclaration classDeclaration : classes) {
147+
String className =
148+
classDeclaration.getFullyQualifiedName().orElse("");
149+
if (!className.isEmpty()) {
150+
classDeclarationMap.put(className, classDeclaration);
151+
}
152+
}
153+
});
154+
} catch (IOException e) {
155+
LOG.warn("Could not parse file: {}", path, e);
156+
}
157+
}
158+
159+
/**
160+
* Check the class path for classes that implement SeaTunnelSource or SeaTunnelSink and verify
161+
* they have serialVersionUID.
162+
*/
163+
private void checkClassPath(Path path, List<String> missingSerialVersionUID) {
164+
try {
165+
ParseResult<CompilationUnit> parseResult =
166+
JAVA_PARSER.parse(Files.newInputStream(path));
167+
parseResult
168+
.getResult()
169+
.ifPresent(
170+
compilationUnit -> {
171+
List<ClassOrInterfaceDeclaration> classes =
172+
compilationUnit.findAll(ClassOrInterfaceDeclaration.class);
173+
for (ClassOrInterfaceDeclaration classDeclaration : classes) {
174+
if (implementsSeaTunnelSourceOrSink(classDeclaration)) {
175+
checkImplementedTypes(
176+
classDeclaration, missingSerialVersionUID);
177+
}
178+
}
179+
});
180+
} catch (IOException e) {
181+
LOG.warn("Could not parse file: {}", path, e);
182+
}
183+
}
184+
185+
private boolean implementsSeaTunnelSourceOrSink(ClassOrInterfaceDeclaration classDeclaration) {
186+
return classDeclaration.getImplementedTypes().stream()
187+
.anyMatch(
188+
type -> {
189+
String typeName = type.getNameAsString();
190+
return typeName.equals("SeaTunnelSource")
191+
|| typeName.equals("SeaTunnelSink");
192+
});
193+
}
194+
195+
private void checkImplementedTypes(
196+
ClassOrInterfaceDeclaration classDeclaration, List<String> missingSerialVersionUID) {
197+
classDeclaration
198+
.getImplementedTypes()
199+
.forEach(
200+
implementedType -> {
201+
implementedType
202+
.getTypeArguments()
203+
.ifPresent(
204+
typeArgs -> {
205+
for (Type typeArg : typeArgs) {
206+
if (typeArg.isClassOrInterfaceType()) {
207+
checkClassType(
208+
typeArg.asClassOrInterfaceType(),
209+
missingSerialVersionUID);
210+
}
211+
}
212+
});
213+
});
214+
}
215+
216+
private void checkClassType(
217+
ClassOrInterfaceType classType, List<String> missingSerialVersionUID) {
218+
219+
try {
220+
ResolvedReferenceType resolvedType = classType.resolve().asReferenceType();
221+
if (resolvedType == null) {
222+
return;
223+
}
224+
if (isSerializable(resolvedType)) {
225+
ResolvedReferenceTypeDeclaration typeDeclaration =
226+
resolvedType.getTypeDeclaration().orElse(null);
227+
if (typeDeclaration == null) {
228+
return;
229+
}
230+
String paramTypeName = typeDeclaration.getQualifiedName();
231+
if (!checkedClasses.contains(paramTypeName)) {
232+
// Check if the class is abstract and return early if it is
233+
if (isAbstractClass(typeDeclaration)) {
234+
checkedClasses.add(paramTypeName);
235+
return;
236+
}
237+
238+
if (!hasSerialVersionUID(typeDeclaration)) {
239+
missingSerialVersionUID.add(paramTypeName);
240+
LOG.warn("Class {} is missing serialVersionUID field", paramTypeName);
241+
}
242+
checkedClasses.add(paramTypeName);
243+
}
244+
}
245+
} catch (Exception e) {
246+
LOG.warn("Could not resolve type: {} in file: {}", classType.getNameAsString(), e);
247+
}
248+
}
249+
250+
private boolean isSerializable(ResolvedReferenceType resolvedType) {
251+
return resolvedType.getQualifiedName().equals("java.io.Serializable")
252+
|| resolvedType.getAllAncestors().stream()
253+
.anyMatch(
254+
ancestor ->
255+
ancestor.getQualifiedName().equals("java.io.Serializable"));
256+
}
257+
258+
private boolean hasSerialVersionUID(ResolvedReferenceTypeDeclaration typeDeclaration) {
259+
return typeDeclaration.isInterface()
260+
|| typeDeclaration.getAllFields().stream()
261+
.anyMatch(field -> field.getName().equals("serialVersionUID"));
262+
}
263+
264+
private boolean isAbstractClass(ResolvedReferenceTypeDeclaration typeDeclaration) {
265+
// Only check classes, not interfaces
266+
if (!typeDeclaration.isClass()) {
267+
return false;
268+
}
269+
270+
String className = typeDeclaration.getQualifiedName();
271+
272+
// First check if we have the class declaration in our map
273+
ClassOrInterfaceDeclaration classDeclaration = classDeclarationMap.get(className);
274+
if (classDeclaration != null) {
275+
// Directly check if the class is abstract using the declaration
276+
return classDeclaration.isAbstract();
277+
}
278+
279+
return false;
280+
}
281+
282+
private String generateErrorMessage(List<String> missingSerialVersionUID) {
283+
StringBuilder errorMessage = new StringBuilder();
284+
errorMessage.append("=================================================================\n");
285+
errorMessage.append(
286+
"Test failed: The following classes are missing serialVersionUID fields\n");
287+
errorMessage.append("=================================================================\n");
288+
errorMessage
289+
.append("A total of ")
290+
.append(missingSerialVersionUID.size())
291+
.append(" Question:\n\n");
292+
293+
for (int i = 0; i < missingSerialVersionUID.size(); i++) {
294+
errorMessage
295+
.append(i + 1)
296+
.append(". ")
297+
.append(missingSerialVersionUID.get(i))
298+
.append("\n");
299+
}
300+
301+
errorMessage.append(
302+
"\n=================================================================\n");
303+
errorMessage.append(
304+
"Please add a serialVersionUID field to the above class and make sure its value is not -1L, for example:\n");
305+
errorMessage.append("private static final long serialVersionUID = 5967888460683065669L;\n");
306+
errorMessage.append("=================================================================\n");
307+
return errorMessage.toString();
308+
}
309+
310+
public static class TestResultLogger implements TestWatcher {
311+
@Override
312+
public void testSuccessful(ExtensionContext context) {
313+
LOG.info("Test successful: {}", context.getDisplayName());
314+
}
315+
316+
@Override
317+
public void testFailed(ExtensionContext context, Throwable cause) {
318+
LOG.error("Test failed: {}", context.getDisplayName(), cause);
319+
}
320+
}
321+
}

‎seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazonDynamoDBSourceSplit.java

+1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
@Setter
2929
public class AmazonDynamoDBSourceSplit implements SourceSplit {
3030

31+
private static final long serialVersionUID = -5148142613656330674L;
3132
private Integer splitId;
3233
private Integer totalSegments;
3334
private Integer itemCount;

‎seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazonDynamoDBSourceState.java

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
@Setter
3030
@AllArgsConstructor
3131
public class AmazonDynamoDBSourceState implements Serializable {
32+
private static final long serialVersionUID = -8614736648787520123L;
3233
private boolean shouldEnumerate;
3334
private Map<Integer, List<AmazonDynamoDBSourceSplit>> pendingSplits;
3435
}

‎seatunnel-connectors-v2/connector-cdc/connector-cdc-tidb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/tidb/source/enumerator/TiDBSourceCheckpointState.java

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
@AllArgsConstructor
3333
@ToString
3434
public class TiDBSourceCheckpointState implements Serializable {
35+
private static final long serialVersionUID = 6292978509042158791L;
3536
private boolean shouldEnumerate;
3637
private Map<Integer, TiDBSourceSplit> pendingSplit;
3738
}

‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/state/CKAggCommitInfo.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,6 @@
1919

2020
import java.io.Serializable;
2121

22-
public class CKAggCommitInfo implements Serializable {}
22+
public class CKAggCommitInfo implements Serializable {
23+
private static final long serialVersionUID = 7725191558817348241L;
24+
}

‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/state/CKCommitInfo.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,6 @@
1919

2020
import java.io.Serializable;
2121

22-
public class CKCommitInfo implements Serializable {}
22+
public class CKCommitInfo implements Serializable {
23+
private static final long serialVersionUID = -3467325029403882141L;
24+
}

‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/state/CKFileAggCommitInfo.java

+1
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,6 @@
3030
@AllArgsConstructor
3131
public class CKFileAggCommitInfo implements Serializable {
3232

33+
private static final long serialVersionUID = 1815170158201953697L;
3334
private Map<Shard, List<String>> detachedFiles;
3435
}

‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/state/CKFileCommitInfo.java

+1
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,6 @@
3030
@AllArgsConstructor
3131
public class CKFileCommitInfo implements Serializable {
3232

33+
private static final long serialVersionUID = 5967888460683065639L;
3334
private Map<Shard, List<String>> detachedFiles;
3435
}

‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/state/ClickhouseSinkState.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,6 @@
1919

2020
import java.io.Serializable;
2121

22-
public class ClickhouseSinkState implements Serializable {}
22+
public class ClickhouseSinkState implements Serializable {
23+
private static final long serialVersionUID = -2781233847929140233L;
24+
}

0 commit comments

Comments
 (0)