Skip to content

Commit 5f04522

Browse files
joegallojoegallo
authored
Fix illegal_access_exception: class com.maxmind.db.Decoder from ip_location processor (#137479) (#137545)
1 parent 8b19e22 commit 5f04522

File tree

9 files changed

+109
-7
lines changed

9 files changed

+109
-7
lines changed

‎docs/changelog/137479.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 137479
2+
summary: "Fix illegal_access_exception: class com.maxmind.db.Decoder from `ip_location`\
3+
\ processor"
4+
area: Ingest Node
5+
type: bug
6+
issues: []

‎modules/ingest-geoip/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,4 +99,6 @@ artifacts {
9999
tasks.named("yamlRestCompatTestTransform").configure({ task ->
100100
task.skipTest("ingest_geoip/40_geoip_databases/Test adding, getting, and removing geoip databases",
101101
"get databases behavior began returning more results in 8.16")
102+
task.skipTest("ingest_geoip/60_ip_location_databases/Test adding, getting, and removing ip location databases",
103+
"get databases behavior began returning more results in 9.2.1")
102104
})

‎modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,20 @@
2121
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
2222
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
2323
import org.junit.Before;
24+
import org.junit.BeforeClass;
2425
import org.junit.ClassRule;
2526
import org.junit.rules.RuleChain;
27+
import org.junit.rules.TemporaryFolder;
2628
import org.junit.rules.TestRule;
2729

30+
import java.nio.file.Files;
31+
import java.nio.file.Path;
32+
import java.util.Objects;
33+
2834
import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.assertDatabasesLoaded;
35+
import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.getRootPath;
2936
import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.putGeoipPipeline;
37+
import static org.hamcrest.Matchers.is;
3038

3139
@FixForMultiProject(description = "Potentially remove this test after https://elasticco.atlassian.net/browse/ES-12094 is implemented")
3240
public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjectsClientYamlSuiteTestCase {
@@ -35,7 +43,10 @@ public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjec
3543

3644
private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture);
3745

46+
public static TemporaryFolder configDir = new TemporaryFolder();
47+
3848
private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
49+
.withConfigDir(() -> getRootPath(configDir))
3950
.module("reindex")
4051
.module("ingest-geoip")
4152
.systemProperty("ingest.geoip.downloader.enabled.default", "true")
@@ -51,7 +62,7 @@ public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjec
5162
.build();
5263

5364
@ClassRule
54-
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
65+
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(configDir).around(cluster);
5566

5667
@Override
5768
protected String getTestRestCluster() {
@@ -67,6 +78,19 @@ public static Iterable<Object[]> parameters() throws Exception {
6778
return ESClientYamlSuiteTestCase.createParameters();
6879
}
6980

81+
@BeforeClass
82+
public static void copyExtraDatabase() throws Exception {
83+
Path configPath = getRootPath(configDir);
84+
assertThat(Files.exists(configPath), is(true));
85+
Path ingestGeoipDatabaseDir = configPath.resolve("ingest-geoip");
86+
Files.createDirectory(ingestGeoipDatabaseDir);
87+
final var clazz = IngestGeoIpClientYamlTestSuiteIT.class; // long line prevention
88+
Files.copy(
89+
Objects.requireNonNull(clazz.getResourceAsStream("/ipinfo/asn_sample.mmdb")),
90+
ingestGeoipDatabaseDir.resolve("asn.mmdb")
91+
);
92+
}
93+
7094
@Before
7195
public void waitForDatabases() throws Exception {
7296
putGeoipPipeline("pipeline-with-geoip");

‎modules/ingest-geoip/src/main/java/module-info.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,5 @@
1919
exports org.elasticsearch.ingest.geoip.direct to org.elasticsearch.server;
2020
exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server;
2121

22-
exports org.elasticsearch.ingest.geoip to org.elasticsearch.logstashbridge;
22+
exports org.elasticsearch.ingest.geoip to org.elasticsearch.logstashbridge, com.maxmind.db;
2323
}

‎modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,29 @@
1919
import org.elasticsearch.client.Request;
2020
import org.elasticsearch.common.bytes.BytesReference;
2121
import org.elasticsearch.core.Booleans;
22+
import org.elasticsearch.core.SuppressForbidden;
2223
import org.elasticsearch.test.cluster.ElasticsearchCluster;
2324
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
2425
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
2526
import org.elasticsearch.xcontent.XContentBuilder;
2627
import org.elasticsearch.xcontent.json.JsonXContent;
2728
import org.junit.Before;
29+
import org.junit.BeforeClass;
2830
import org.junit.ClassRule;
2931
import org.junit.rules.RuleChain;
32+
import org.junit.rules.TemporaryFolder;
3033
import org.junit.rules.TestRule;
3134

3235
import java.io.IOException;
36+
import java.nio.file.Files;
37+
import java.nio.file.Path;
3338
import java.util.List;
3439
import java.util.Map;
40+
import java.util.Objects;
3541

3642
import static org.hamcrest.Matchers.containsInAnyOrder;
3743
import static org.hamcrest.Matchers.equalTo;
44+
import static org.hamcrest.Matchers.is;
3845
import static org.hamcrest.Matchers.notNullValue;
3946

4047
public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -43,7 +50,10 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
4350

4451
private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture);
4552

53+
public static TemporaryFolder configDir = new TemporaryFolder();
54+
4655
private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
56+
.withConfigDir(() -> getRootPath(configDir))
4757
.module("reindex")
4858
.module("ingest-geoip")
4959
.systemProperty("ingest.geoip.downloader.enabled.default", "true")
@@ -56,7 +66,7 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
5666
.build();
5767

5868
@ClassRule
59-
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
69+
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(configDir).around(cluster);
6070

6171
@Override
6272
protected String getTestRestCluster() {
@@ -72,6 +82,19 @@ public static Iterable<Object[]> parameters() throws Exception {
7282
return ESClientYamlSuiteTestCase.createParameters();
7383
}
7484

85+
@BeforeClass
86+
public static void copyExtraDatabase() throws Exception {
87+
Path configPath = getRootPath(configDir);
88+
assertThat(Files.exists(configPath), is(true));
89+
Path ingestGeoipDatabaseDir = configPath.resolve("ingest-geoip");
90+
Files.createDirectory(ingestGeoipDatabaseDir);
91+
final var clazz = IngestGeoIpClientYamlTestSuiteIT.class; // long line prevention
92+
Files.copy(
93+
Objects.requireNonNull(clazz.getResourceAsStream("/ipinfo/asn_sample.mmdb")),
94+
ingestGeoipDatabaseDir.resolve("asn.mmdb")
95+
);
96+
}
97+
7598
@Before
7699
public void waitForDatabases() throws Exception {
77100
putGeoipPipeline("pipeline-with-geoip");
@@ -123,13 +146,23 @@ static void assertDatabasesLoaded() throws Exception {
123146
Map<?, ?> nodes = (Map<?, ?>) response.get("nodes");
124147
assertThat(nodes.size(), equalTo(1));
125148
Map<?, ?> node = (Map<?, ?>) nodes.values().iterator().next();
149+
150+
// confirm the downloaded databases are all correct
126151
List<?> databases = ((List<?>) node.get("databases"));
127152
assertThat(databases, notNullValue());
128153
List<String> databaseNames = databases.stream().map(o -> (String) ((Map<?, ?>) o).get("name")).toList();
129154
assertThat(
130155
databaseNames,
131156
containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb", "MyCustomGeoLite2-City.mmdb")
132157
);
158+
159+
// ensure that the extra config database has been set up, too:
160+
assertThat(node.get("config_databases"), equalTo(List.of("asn.mmdb")));
133161
});
134162
}
163+
164+
@SuppressForbidden(reason = "fixtures use java.io.File based APIs")
165+
public static Path getRootPath(TemporaryFolder folder) {
166+
return folder.getRoot().toPath();
167+
}
135168
}

‎modules/ingest-geoip/src/yamlRestTest/resources/ipinfo/asn_sample.mmdb‎

25 KB
Binary file not shown.

‎modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ teardown:
6969

7070
- do:
7171
ingest.get_geoip_database: {}
72-
- length: { databases: 6 }
72+
- length: { databases: 7 }
7373

7474
- do:
7575
ingest.get_geoip_database:
@@ -95,7 +95,7 @@ teardown:
9595

9696
- do:
9797
ingest.get_geoip_database: {}
98-
- length: { databases: 5 }
98+
- length: { databases: 6 }
9999

100100
- do:
101101
ingest.get_geoip_database:

‎modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,40 @@
3737
- match: { _source.ip_location.country_name: "Sweden" }
3838
- match: { _source.ip_location.region_name: "Östergötland County" }
3939
- match: { _source.ip_location.continent_name: "Europe" }
40+
41+
---
42+
"Test ip_location processor with a config directory database":
43+
- do:
44+
ingest.put_pipeline:
45+
id: "my_pipeline_2"
46+
body: >
47+
{
48+
"description": "_description",
49+
"processors": [
50+
{
51+
"ip_location" : {
52+
"field" : "field1",
53+
"database_file" : "asn.mmdb"
54+
}
55+
}
56+
]
57+
}
58+
- match: { acknowledged: true }
59+
60+
- do:
61+
index:
62+
index: test
63+
id: "1"
64+
pipeline: "my_pipeline_2"
65+
body: {field1: "5.104.168.0"}
66+
67+
- do:
68+
get:
69+
index: test
70+
id: "1"
71+
- match: { _source.field1: "5.104.168.0" }
72+
- length: { _source.ip_location: 4 }
73+
- match: { _source.ip_location.organization_name: "Telehouse EAD" }
74+
- match: { _source.ip_location.asn: 57344 }
75+
- match: { _source.ip_location.ip: "5.104.168.0" }
76+
- match: { _source.ip_location.network: "5.104.168.0/23" }

‎modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ teardown:
8484

8585
- do:
8686
ingest.get_ip_location_database: {}
87-
- length: { databases: 7 }
87+
- length: { databases: 8 }
8888

8989
- do:
9090
ingest.get_ip_location_database:
@@ -110,7 +110,7 @@ teardown:
110110

111111
- do:
112112
ingest.get_ip_location_database: {}
113-
- length: { databases: 6 }
113+
- length: { databases: 7 }
114114

115115
- do:
116116
ingest.get_ip_location_database:

0 commit comments

Comments
 (0)