Skip to content

Commit 8285555

Browse files
nscuronscuro
authored
Add EUVD importer (#56)
1 parent 13eea9b commit 8285555

File tree

7 files changed

+208
-1
lines changed

7 files changed

+208
-1
lines changed

‎.github/workflows/update-database.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ jobs:
3333
strategy:
3434
matrix:
3535
source:
36+
- euvd
3637
- github
3738
- nvd
3839
- osv
@@ -82,7 +83,7 @@ jobs:
8283
echo '${{ secrets.GITHUB_TOKEN }}' | oras login ghcr.io -u github --password-stdin
8384
- name: Download source databases
8485
run: |
85-
for source_name in github nvd osv; do
86+
for source_name in euvd github nvd osv; do
8687
oras pull "ghcr.io/${GITHUB_REPOSITORY,,}/source/${source_name}:${{ inputs.database-version }}"
8788
done
8889
- name: Decompress source databases

‎pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
<lib.open-vulnerability-clients.version>8.0.0</lib.open-vulnerability-clients.version>
2424
<lib.packageurl-java.version>1.5.0</lib.packageurl-java.version>
2525
<lib.picocli.version>4.7.7</lib.picocli.version>
26+
<lib.resilience4j.version>2.3.0</lib.resilience4j.version>
2627
<lib.slf4j.version>2.0.17</lib.slf4j.version>
2728
<lib.sqlite-jdbc.version>3.49.1.0</lib.sqlite-jdbc.version>
2829
<lib.versatile.version>0.12.0</lib.versatile.version>
@@ -110,6 +111,12 @@
110111
<version>${lib.picocli.version}</version>
111112
</dependency>
112113

114+
<dependency>
115+
<groupId>io.github.resilience4j</groupId>
116+
<artifactId>resilience4j-retry</artifactId>
117+
<version>${lib.resilience4j.version}</version>
118+
</dependency>
119+
113120
<dependency>
114121
<groupId>org.slf4j</groupId>
115122
<artifactId>slf4j-api</artifactId>

‎src/main/java/org/dependencytrack/vulndb/source/euvd/EuvdImporter.java‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
package org.dependencytrack.vulndb.source.euvd;
2+
3+
import com.fasterxml.jackson.databind.DeserializationFeature;
4+
import com.fasterxml.jackson.databind.ObjectMapper;
5+
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
6+
import io.github.resilience4j.retry.Retry;
7+
import io.github.resilience4j.retry.RetryConfig;
8+
import io.github.resilience4j.retry.RetryRegistry;
9+
import org.dependencytrack.vulndb.api.Database;
10+
import org.dependencytrack.vulndb.api.Importer;
11+
import org.dependencytrack.vulndb.api.Rating;
12+
import org.dependencytrack.vulndb.api.Reference;
13+
import org.dependencytrack.vulndb.api.Source;
14+
import org.dependencytrack.vulndb.api.Vulnerability;
15+
import org.metaeffekt.core.security.cvss.CvssVector;
16+
import org.metaeffekt.core.security.cvss.v2.Cvss2;
17+
import org.metaeffekt.core.security.cvss.v3.Cvss3;
18+
import org.metaeffekt.core.security.cvss.v4P0.Cvss4P0;
19+
import org.slf4j.Logger;
20+
import org.slf4j.LoggerFactory;
21+
import org.slf4j.MDC;
22+
23+
import java.net.URI;
24+
import java.net.http.HttpClient;
25+
import java.net.http.HttpRequest;
26+
import java.net.http.HttpResponse;
27+
import java.time.Duration;
28+
import java.util.List;
29+
30+
import static io.github.resilience4j.core.IntervalFunction.ofExponentialRandomBackoff;
31+
32+
public final class EuvdImporter implements Importer {
33+
34+
private static final Logger LOGGER = LoggerFactory.getLogger(EuvdImporter.class);
35+
36+
private Database database;
37+
private HttpClient httpClient;
38+
private ObjectMapper objectMapper;
39+
private Retry retry;
40+
41+
@Override
42+
public Source source() {
43+
return new Source("euvd", "European Union Vulnerability Database", null, "https://euvd.enisa.europa.eu/");
44+
}
45+
46+
@Override
47+
public void init(final Database database) {
48+
this.database = database;
49+
this.httpClient = HttpClient.newHttpClient();
50+
this.objectMapper = new ObjectMapper()
51+
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
52+
.registerModule(new JavaTimeModule());
53+
this.retry = RetryRegistry.of(
54+
RetryConfig.<HttpResponse<?>>custom()
55+
.retryOnResult(response -> response.statusCode() == 403)
56+
.intervalFunction(ofExponentialRandomBackoff(
57+
Duration.ofSeconds(15),
58+
/* multiplier */ 2.0,
59+
/* randomizationFactor */ 0.5,
60+
Duration.ofMinutes(3)))
61+
.maxAttempts(12)
62+
.build())
63+
.retry("euvd-api");
64+
this.retry.getEventPublisher().onRetry(
65+
event -> LOGGER.warn(
66+
"Performing retry attempt {} in {}",
67+
event.getNumberOfRetryAttempts(),
68+
event.getWaitInterval()));
69+
}
70+
71+
@Override
72+
public void runImport() throws Exception {
73+
boolean hasMore;
74+
int pageNumber = 0;
75+
int vulnsImported = 0;
76+
do {
77+
final HttpResponse<byte[]> response = retry.executeCallable(
78+
() -> httpClient.send(
79+
HttpRequest.newBuilder(URI.create(
80+
"https://euvdservices.enisa.europa.eu/api/search?size=100&page=%d".formatted(pageNumber)))
81+
.GET()
82+
.header("User-Agent", "github.com/DependencyTrack/vuln-db")
83+
.build(),
84+
HttpResponse.BodyHandlers.ofByteArray()));
85+
if (response.statusCode() != 200) {
86+
throw new Exception("Unexpected response code: " + response.statusCode());
87+
}
88+
89+
final var vulnsPage = objectMapper.readValue(response.body(), EuvdVulnerabilitiesPage.class);
90+
final List<Vulnerability> vulns = vulnsPage.items().stream()
91+
.map(euvdVuln -> {
92+
try (var ignored = MDC.putCloseable("vulnId", euvdVuln.id())) {
93+
return convert(euvdVuln);
94+
}
95+
})
96+
.toList();
97+
98+
database.storeVulnerabilities(vulns);
99+
100+
vulnsImported += vulns.size();
101+
hasMore = vulnsImported < vulnsPage.total();
102+
LOGGER.info("Imported {}/{} vulnerabilities", vulnsImported, vulnsPage.total());
103+
} while (hasMore);
104+
}
105+
106+
private Vulnerability convert(final EuvdVulnerability euvdVuln) {
107+
return new Vulnerability(
108+
euvdVuln.id(),
109+
euvdVuln.aliases(),
110+
/* related */ null,
111+
euvdVuln.description(),
112+
/* cwes */ null,
113+
getRatings(euvdVuln),
114+
euvdVuln.references() != null
115+
? euvdVuln.references().stream().map(referenceUrl -> new Reference(referenceUrl, null)).toList()
116+
: null,
117+
/* matchingCriteria */ null,
118+
/* createdAt */ null,
119+
euvdVuln.datePublished() != null ? euvdVuln.datePublished().toInstant() : null,
120+
euvdVuln.dateUpdated() != null ? euvdVuln.dateUpdated().toInstant() : null,
121+
/* rejectedAt */ null);
122+
}
123+
124+
private List<Rating> getRatings(final EuvdVulnerability euvdVuln) {
125+
if (euvdVuln.baseScoreVector() == null) {
126+
return null;
127+
}
128+
129+
final CvssVector vector = CvssVector.parseVector(euvdVuln.baseScoreVector());
130+
if (vector != null) {
131+
final Rating.Method method = switch (vector) {
132+
case Cvss2 ignored -> Rating.Method.CVSSv2;
133+
case Cvss3 ignored -> Rating.Method.CVSSv3;
134+
case Cvss4P0 ignored -> Rating.Method.CVSSv4;
135+
default -> null;
136+
};
137+
if (method == null) {
138+
LOGGER.warn("Unexpected CVSS type {}", vector.getClass().getName());
139+
}
140+
141+
return List.of(
142+
new Rating(
143+
method,
144+
Rating.Severity.ofCvss(vector),
145+
vector.toString(),
146+
vector.getBaseScore()));
147+
} else {
148+
LOGGER.warn("Failed to parse CVSS vector {}", euvdVuln.baseScoreVector());
149+
}
150+
151+
return null;
152+
}
153+
154+
}

‎src/main/java/org/dependencytrack/vulndb/source/euvd/EuvdVulnerabilitiesPage.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package org.dependencytrack.vulndb.source.euvd;
2+
3+
import java.util.List;
4+
5+
public record EuvdVulnerabilitiesPage(List<EuvdVulnerability> items, int total) {
6+
}

‎src/main/java/org/dependencytrack/vulndb/source/euvd/EuvdVulnerability.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package org.dependencytrack.vulndb.source.euvd;
2+
3+
import com.fasterxml.jackson.annotation.JsonFormat;
4+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
5+
6+
import java.time.ZonedDateTime;
7+
import java.util.List;
8+
9+
public record EuvdVulnerability(
10+
String id,
11+
String description,
12+
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "MMM d, yyyy, h:m:s a", locale = "en_US", timezone = /* TODO */ "UTC") ZonedDateTime datePublished,
13+
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "MMM d, yyyy, h:m:s a", locale = "en_US", timezone = /* TODO */ "UTC") ZonedDateTime dateUpdated,
14+
Double baseScore,
15+
String baseScoreVersion,
16+
String baseScoreVector,
17+
@JsonDeserialize(using = NewlineDelimitedListDeserializer.class) List<String> aliases,
18+
@JsonDeserialize(using = NewlineDelimitedListDeserializer.class) List<String> references) {
19+
}

‎src/main/java/org/dependencytrack/vulndb/source/euvd/NewlineDelimitedListDeserializer.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package org.dependencytrack.vulndb.source.euvd;
2+
3+
import com.fasterxml.jackson.core.JsonParser;
4+
import com.fasterxml.jackson.databind.DeserializationContext;
5+
import com.fasterxml.jackson.databind.JsonDeserializer;
6+
7+
import java.io.IOException;
8+
import java.util.List;
9+
10+
public final class NewlineDelimitedListDeserializer extends JsonDeserializer<List<String>> {
11+
12+
@Override
13+
public List<String> deserialize(
14+
final JsonParser jsonParser,
15+
final DeserializationContext deserializationContext) throws IOException {
16+
return jsonParser.readValueAs(String.class).lines().toList();
17+
}
18+
19+
}

‎src/main/resources/META-INF/services/org.dependencytrack.vulndb.api.Importer‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
org.dependencytrack.vulndb.source.euvd.EuvdImporter
12
org.dependencytrack.vulndb.source.github.GitHubImporter
23
org.dependencytrack.vulndb.source.nvd.NvdImporter
34
org.dependencytrack.vulndb.source.osv.OsvImporter

0 commit comments

Comments
 (0)