Skip to content

Commit 6494265

Browse files
courtneyehcourtneyeh
committed
Retry loading of external validators if failed
1 parent 80687ba commit 6494265

File tree

7 files changed

+266
-24
lines changed

7 files changed

+266
-24
lines changed

‎infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java

+4
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,10 @@ public void reconstructedHistoricalBlocks(
260260
totalToRecord);
261261
}
262262

263+
public void failedToLoadPublicKeysFromUrl(final String url) {
264+
log.error("Failed to load public keys from URL: {}", url);
265+
}
266+
263267
public void failedToStartValidatorClient(final String message) {
264268
log.fatal(
265269
"An error was encountered during validator client service start up. Error: {}", message);

‎teku/src/main/java/tech/pegasys/teku/cli/subcommand/VoluntaryExitCommand.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,9 @@ private void initialise() {
423423
new RejectingSlashingProtector(),
424424
slashingProtectionLogger,
425425
new PublicKeyLoader(
426-
externalSignerHttpClientFactory, validatorConfig.getValidatorExternalSignerUrl()),
426+
externalSignerHttpClientFactory,
427+
validatorConfig.getValidatorExternalSignerUrl(),
428+
asyncRunner),
427429
asyncRunner,
428430
metricsSystem,
429431
dataDirLayout,

‎validator/client/src/main/java/tech/pegasys/teku/validator/client/ValidatorClientService.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,8 @@ private static ValidatorLoader createValidatorLoader(
430430
slashingProtectionLogger,
431431
new PublicKeyLoader(
432432
externalSignerHttpClientFactory,
433-
config.getValidatorConfig().getValidatorExternalSignerUrl()),
433+
config.getValidatorConfig().getValidatorExternalSignerUrl(),
434+
asyncRunner),
434435
asyncRunner,
435436
services.getMetricsSystem(),
436437
config.getValidatorRestApiConfig().isRestApiEnabled()

‎validator/client/src/main/java/tech/pegasys/teku/validator/client/loader/ExternalUrlKeyReader.java

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Copyright Consensys Software Inc., 2024
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
5+
* the License. You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
10+
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
* specific language governing permissions and limitations under the License.
12+
*/
13+
14+
package tech.pegasys.teku.validator.client.loader;
15+
16+
import static tech.pegasys.teku.infrastructure.logging.StatusLogger.STATUS_LOG;
17+
18+
import com.fasterxml.jackson.databind.ObjectMapper;
19+
import java.io.IOException;
20+
import java.net.MalformedURLException;
21+
import java.net.URL;
22+
import java.time.Duration;
23+
import java.util.Arrays;
24+
import java.util.stream.Stream;
25+
import org.apache.tuweni.bytes.Bytes;
26+
import tech.pegasys.teku.bls.BLSPublicKey;
27+
import tech.pegasys.teku.infrastructure.async.AsyncRunner;
28+
import tech.pegasys.teku.infrastructure.async.SafeFuture;
29+
import tech.pegasys.teku.infrastructure.exceptions.InvalidConfigurationException;
30+
31+
public class ExternalUrlKeyReader {
32+
private static final Duration FIXED_DELAY = Duration.ofSeconds(5);
33+
34+
private final String url;
35+
private final ObjectMapper mapper;
36+
private final AsyncRunner asyncRunner;
37+
38+
public ExternalUrlKeyReader(
39+
final String url, final ObjectMapper mapper, final AsyncRunner asyncRunner) {
40+
this.url = url;
41+
this.mapper = mapper;
42+
this.asyncRunner = asyncRunner;
43+
}
44+
45+
public Stream<BLSPublicKey> readKeys() {
46+
final String[] keys =
47+
readUrl()
48+
.exceptionallyCompose(
49+
ex -> {
50+
if (ex instanceof MalformedURLException) {
51+
return SafeFuture.failedFuture(
52+
new InvalidConfigurationException(
53+
"Failed to load public keys from invalid URL: " + url, ex));
54+
}
55+
return retry();
56+
})
57+
.join();
58+
return Arrays.stream(keys).map(key -> BLSPublicKey.fromSSZBytes(Bytes.fromHexString(key)));
59+
}
60+
61+
public SafeFuture<String[]> readUrl() {
62+
try {
63+
return SafeFuture.completedFuture(mapper.readValue(new URL(url), String[].class));
64+
} catch (IOException e) {
65+
return SafeFuture.failedFuture(e);
66+
}
67+
}
68+
69+
public SafeFuture<String[]> retry() {
70+
return asyncRunner
71+
.runWithRetry(
72+
() -> {
73+
STATUS_LOG.failedToLoadPublicKeysFromUrl(url);
74+
return readUrl();
75+
},
76+
FIXED_DELAY,
77+
59)
78+
.exceptionallyCompose(
79+
ex ->
80+
SafeFuture.failedFuture(
81+
new InvalidConfigurationException(
82+
"Failed to load public keys from URL: " + url, ex)));
83+
}
84+
}

‎validator/client/src/main/java/tech/pegasys/teku/validator/client/loader/PublicKeyLoader.java

+18-13
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,13 @@
2727
import java.util.Collections;
2828
import java.util.List;
2929
import java.util.Set;
30+
import java.util.function.Function;
3031
import java.util.function.Supplier;
3132
import java.util.stream.Collectors;
3233
import java.util.stream.Stream;
3334
import org.apache.tuweni.bytes.Bytes;
3435
import tech.pegasys.teku.bls.BLSPublicKey;
36+
import tech.pegasys.teku.infrastructure.async.AsyncRunner;
3537
import tech.pegasys.teku.infrastructure.exceptions.InvalidConfigurationException;
3638
import tech.pegasys.teku.infrastructure.http.HttpStatusCodes;
3739

@@ -42,6 +44,7 @@ public class PublicKeyLoader {
4244
final ObjectMapper objectMapper;
4345
final Supplier<HttpClient> externalSignerHttpClientFactory;
4446
final URL externalSignerUrl;
47+
final AsyncRunner asyncRunner;
4548

4649
@VisibleForTesting
4750
PublicKeyLoader() {
@@ -50,24 +53,35 @@ public class PublicKeyLoader {
5053
() -> {
5154
throw new UnsupportedOperationException();
5255
},
56+
null,
5357
null);
5458
}
5559

5660
public PublicKeyLoader(
57-
final Supplier<HttpClient> externalSignerHttpClientFactory, final URL externalSignerUrl) {
58-
this(new ObjectMapper(), externalSignerHttpClientFactory, externalSignerUrl);
61+
final Supplier<HttpClient> externalSignerHttpClientFactory,
62+
final URL externalSignerUrl,
63+
final AsyncRunner asyncRunner) {
64+
this(new ObjectMapper(), externalSignerHttpClientFactory, externalSignerUrl, asyncRunner);
5965
}
6066

6167
public PublicKeyLoader(
6268
final ObjectMapper objectMapper,
6369
final Supplier<HttpClient> externalSignerHttpClientFactory,
64-
final URL externalSignerUrl) {
70+
final URL externalSignerUrl,
71+
final AsyncRunner asyncRunner) {
6572
this.objectMapper = objectMapper;
6673
this.externalSignerHttpClientFactory = externalSignerHttpClientFactory;
6774
this.externalSignerUrl = externalSignerUrl;
75+
this.asyncRunner = asyncRunner;
6876
}
6977

7078
public List<BLSPublicKey> getPublicKeys(final List<String> sources) {
79+
return getPublicKeys(
80+
sources, (url) -> new ExternalUrlKeyReader(url, objectMapper, asyncRunner));
81+
}
82+
83+
public List<BLSPublicKey> getPublicKeys(
84+
final List<String> sources, final Function<String, ExternalUrlKeyReader> urlReader) {
7185
if (sources == null || sources.isEmpty()) {
7286
return Collections.emptyList();
7387
}
@@ -81,7 +95,7 @@ public List<BLSPublicKey> getPublicKeys(final List<String> sources) {
8195
return readKeysFromExternalSigner();
8296
}
8397
if (key.contains(":")) {
84-
return readKeysFromUrl(key);
98+
return urlReader.apply(key).readKeys();
8599
}
86100

87101
return Stream.of(BLSPublicKey.fromSSZBytes(Bytes.fromHexString(key)));
@@ -95,15 +109,6 @@ public List<BLSPublicKey> getPublicKeys(final List<String> sources) {
95109
}
96110
}
97111

98-
private Stream<BLSPublicKey> readKeysFromUrl(final String url) {
99-
try {
100-
final String[] keys = objectMapper.readValue(new URL(url), String[].class);
101-
return Arrays.stream(keys).map(key -> BLSPublicKey.fromSSZBytes(Bytes.fromHexString(key)));
102-
} catch (IOException ex) {
103-
throw new InvalidConfigurationException("Failed to load public keys from URL " + url, ex);
104-
}
105-
}
106-
107112
private Stream<BLSPublicKey> readKeysFromExternalSigner() {
108113
try {
109114
final URI uri = externalSignerUrl.toURI().resolve(EXTERNAL_SIGNER_PUBKEYS_ENDPOINT);

‎validator/client/src/test/java/tech/pegasys/teku/validator/client/loader/ExternalUrlKeyReaderTest.java

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/*
2+
* Copyright Consensys Software Inc., 2024
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
5+
* the License. You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
10+
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
* specific language governing permissions and limitations under the License.
12+
*/
13+
14+
package tech.pegasys.teku.validator.client.loader;
15+
16+
import static org.assertj.core.api.Assertions.assertThat;
17+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
18+
import static org.mockito.ArgumentMatchers.any;
19+
import static org.mockito.ArgumentMatchers.eq;
20+
import static org.mockito.Mockito.mock;
21+
import static org.mockito.Mockito.times;
22+
import static org.mockito.Mockito.verify;
23+
import static org.mockito.Mockito.verifyNoInteractions;
24+
import static org.mockito.Mockito.when;
25+
26+
import com.fasterxml.jackson.databind.ObjectMapper;
27+
import java.io.IOException;
28+
import java.net.ConnectException;
29+
import java.net.MalformedURLException;
30+
import java.net.URL;
31+
import java.net.UnknownHostException;
32+
import java.time.Duration;
33+
import java.util.concurrent.CompletionException;
34+
import java.util.stream.Collectors;
35+
import org.junit.jupiter.api.Test;
36+
import tech.pegasys.teku.bls.BLSPublicKey;
37+
import tech.pegasys.teku.infrastructure.async.SafeFuture;
38+
import tech.pegasys.teku.infrastructure.async.StubAsyncRunner;
39+
import tech.pegasys.teku.infrastructure.exceptions.InvalidConfigurationException;
40+
import tech.pegasys.teku.infrastructure.time.StubTimeProvider;
41+
import tech.pegasys.teku.spec.TestSpecFactory;
42+
import tech.pegasys.teku.spec.util.DataStructureUtil;
43+
44+
class ExternalUrlKeyReaderTest {
45+
private static final Duration DELAY = Duration.ofSeconds(5);
46+
private static final String VALID_URL = "http://test:0000/api/v1/eth2/publicKeys";
47+
private final ObjectMapper mapper = mock(ObjectMapper.class);
48+
49+
private final StubTimeProvider timeProvider = StubTimeProvider.withTimeInMillis(0);
50+
private final StubAsyncRunner asyncRunner = new StubAsyncRunner(timeProvider);
51+
52+
private final DataStructureUtil dataStructureUtil =
53+
new DataStructureUtil(TestSpecFactory.createDefault());
54+
55+
final BLSPublicKey publicKey1 = dataStructureUtil.randomPublicKey();
56+
final BLSPublicKey publicKey2 = dataStructureUtil.randomPublicKey();
57+
final String[] expectedKeys = new String[] {publicKey1.toHexString(), publicKey2.toHexString()};
58+
59+
@Test
60+
void readKeys_validUrlReturnsValidKeys() throws IOException {
61+
when(mapper.readValue(any(URL.class), eq(String[].class))).thenReturn(expectedKeys);
62+
final ExternalUrlKeyReader reader = new ExternalUrlKeyReader(VALID_URL, mapper, asyncRunner);
63+
64+
assertThat(reader.readKeys()).contains(publicKey1, publicKey2);
65+
verify(mapper).readValue(any(URL.class), eq(String[].class));
66+
}
67+
68+
@Test
69+
void readKeys_validUrlReturnsEmptyKeys() throws IOException {
70+
when(mapper.readValue(any(URL.class), eq(String[].class))).thenReturn(new String[] {});
71+
final ExternalUrlKeyReader reader = new ExternalUrlKeyReader(VALID_URL, mapper, asyncRunner);
72+
73+
assertThat(reader.readKeys()).isEmpty();
74+
verify(mapper).readValue(any(URL.class), eq(String[].class));
75+
}
76+
77+
@Test
78+
void readKeys_validUrlReturnsInvalidKeys() throws IOException {
79+
when(mapper.readValue(any(URL.class), eq(String[].class)))
80+
.thenReturn(new String[] {"invalid", "keys"});
81+
final ExternalUrlKeyReader reader = new ExternalUrlKeyReader(VALID_URL, mapper, asyncRunner);
82+
83+
assertThatThrownBy(() -> reader.readKeys().collect(Collectors.toSet()))
84+
.isInstanceOf(IllegalArgumentException.class)
85+
.hasMessage("Invalid odd-length hex binary representation");
86+
verify(mapper).readValue(any(URL.class), eq(String[].class));
87+
}
88+
89+
@Test
90+
void readKeys_malformedUrlString() {
91+
final String invalidUrl = "invalid:url";
92+
final ExternalUrlKeyReader reader = new ExternalUrlKeyReader(invalidUrl, mapper, asyncRunner);
93+
94+
assertThatThrownBy(reader::readKeys)
95+
.isInstanceOf(CompletionException.class)
96+
.hasCauseInstanceOf(InvalidConfigurationException.class)
97+
.hasMessageContaining("Failed to load public keys from invalid URL: " + invalidUrl)
98+
.hasRootCauseInstanceOf(MalformedURLException.class);
99+
verifyNoInteractions(mapper);
100+
}
101+
102+
@Test
103+
void readKeysWithRetry_unreachableUrlRetryUntilReachable() throws IOException {
104+
final UnknownHostException exception = new UnknownHostException("Unknown host");
105+
when(mapper.readValue(any(URL.class), eq(String[].class)))
106+
.thenThrow(exception, exception, exception)
107+
.thenReturn(expectedKeys);
108+
final ExternalUrlKeyReader reader = new ExternalUrlKeyReader(VALID_URL, mapper, asyncRunner);
109+
110+
final SafeFuture<String[]> keys = reader.retry();
111+
for (int i = 0; i < 3; i++) {
112+
assertThat(keys).isNotCompleted();
113+
timeProvider.advanceTimeBy(DELAY);
114+
asyncRunner.executeQueuedActions();
115+
}
116+
assertThat(keys).isCompletedWithValue(expectedKeys);
117+
verify(mapper, times(4)).readValue(any(URL.class), eq(String[].class));
118+
}
119+
120+
@Test
121+
void readKeysWithRetry_unreachableUrlRetryUntilMaxRetries() throws IOException {
122+
final IOException exception = new ConnectException("Connection refused");
123+
when(mapper.readValue(any(URL.class), eq(String[].class))).thenThrow(exception);
124+
final ExternalUrlKeyReader reader = new ExternalUrlKeyReader(VALID_URL, mapper, asyncRunner);
125+
126+
final SafeFuture<String[]> keys = reader.retry();
127+
for (int i = 0; i < 59; i++) {
128+
assertThat(keys).isNotCompleted();
129+
timeProvider.advanceTimeBy(DELAY);
130+
asyncRunner.executeQueuedActions();
131+
}
132+
assertThat(keys).isCompletedExceptionally();
133+
assertThatThrownBy(keys::join)
134+
.isInstanceOf(CompletionException.class)
135+
.hasCauseInstanceOf(InvalidConfigurationException.class)
136+
.hasRootCause(exception);
137+
verify(mapper, times(60)).readValue(any(URL.class), eq(String[].class));
138+
}
139+
}

0 commit comments

Comments
 (0)