Skip to content

Commit 90ba908

Browse files
jasperpottsNana-EC
jasperpotts
and
Nana-EC
authored
feat: Implement Cloud Archive Plugin (#1043)
Adds new base functionality of a S3 compatible client lib. Minimal with no dependencies, just want we need. Then a first implementation of cloud archive Signed-off-by: jasperpotts <[email protected]> Signed-off-by: Nana Essilfie-Conduah <[email protected]> Co-authored-by: jasperpotts <[email protected]> Co-authored-by: Nana Essilfie-Conduah <[email protected]>
1 parent f5e77f8 commit 90ba908

File tree

34 files changed

+2252
-101
lines changed

34 files changed

+2252
-101
lines changed

‎block-node/app/build.gradle.kts

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,12 @@ mainModuleInfo {
5353
// List of all "plugin modules" we need at runtime.
5454
// In the future, we may get Gradle to automatically infer this block
5555
// https://github.com/gradlex-org/java-module-dependencies/issues/174
56+
runtimeOnly("org.hiero.block.node.archive")
5657
runtimeOnly("org.hiero.block.node.messaging")
5758
runtimeOnly("org.hiero.block.node.health")
5859
runtimeOnly("org.hiero.block.node.publisher")
5960
runtimeOnly("org.hiero.block.node.subscriber")
6061
runtimeOnly("org.hiero.block.node.verification")
61-
runtimeOnly("org.hiero.block.node.blocks.cloud.archive")
6262
runtimeOnly("org.hiero.block.node.blocks.cloud.historic")
6363
runtimeOnly("org.hiero.block.node.blocks.files.historic")
6464
runtimeOnly("org.hiero.block.node.blocks.files.recent")

‎block-node/app/src/main/java/org/hiero/block/node/app/BlockNodeApp.java

+14
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,20 @@ public class BlockNodeApp implements HealthFacility {
8080
CleanColorfulFormatter.makeLoggingColorful();
8181
// tell helidon to use the same logging configuration
8282
System.setProperty("io.helidon.logging.config.disabled", "true");
83+
// ==== LOG HIERO MODULES ======================================================================================
84+
// this can be useful when debugging issues with modules/plugins not being loaded
85+
LOGGER.log(INFO, "=".repeat(120));
86+
LOGGER.log(INFO, "Loaded Hiero Java modules:");
87+
// log all the modules loaded by the class loader
88+
final String moduleClassPath = System.getProperty("jdk.module.path");
89+
if (moduleClassPath != null) {
90+
final String[] moduleClassPathArray = moduleClassPath.split(":");
91+
for (String module : moduleClassPathArray) {
92+
if (module.contains("hiero")) {
93+
LOGGER.log(INFO, GREY + " " + module);
94+
}
95+
}
96+
}
8397
// ==== FACILITY & PLUGIN LOADING ==============================================================================
8498
// Load Block Messaging Service plugin - for now allow nulls
8599
final BlockMessagingFacility blockMessagingService = serviceLoader

‎block-node/app/src/main/java/org/hiero/block/node/app/config/ConfigLogger.java

-2
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,6 @@ public static void log(@NonNull final Configuration configuration) {
3535
if (LOGGER.isLoggable(INFO)) {
3636
// Header
3737
LOGGER.log(INFO, BANNER_LINE);
38-
LOGGER.log(INFO, "Block Node Configuration");
39-
LOGGER.log(INFO, BANNER_LINE);
4038
// Log loaded configuration data types
4139
LOGGER.log(INFO, "Configuration data types:");
4240
for (final var type : configuration.getConfigDataTypes()) {

‎block-node/app/src/testFixtures/java/org/hiero/block/node/app/fixtures/blocks/SimpleTestBlockItemBuilder.java

+143-5
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,44 @@
11
// SPDX-License-Identifier: Apache-2.0
22
package org.hiero.block.node.app.fixtures.blocks;
33

4+
import com.hedera.hapi.block.stream.Block;
45
import com.hedera.hapi.block.stream.BlockItem;
56
import com.hedera.hapi.block.stream.BlockItem.ItemOneOfType;
67
import com.hedera.hapi.block.stream.BlockProof;
8+
import com.hedera.hapi.block.stream.input.EventHeader;
79
import com.hedera.hapi.block.stream.input.RoundHeader;
810
import com.hedera.hapi.block.stream.output.BlockHeader;
911
import com.hedera.hapi.node.base.BlockHashAlgorithm;
1012
import com.hedera.hapi.node.base.SemanticVersion;
1113
import com.hedera.hapi.node.base.Timestamp;
14+
import com.hedera.hapi.platform.event.EventCore;
1215
import com.hedera.pbj.runtime.OneOf;
1316
import com.hedera.pbj.runtime.io.buffer.Bytes;
17+
import java.time.Duration;
18+
import java.time.Instant;
1419
import java.util.ArrayList;
20+
import java.util.Arrays;
1521
import java.util.Collections;
1622
import java.util.List;
23+
import java.util.Random;
24+
import java.util.stream.LongStream;
1725
import org.hiero.block.internal.BlockItemUnparsed;
1826
import org.hiero.block.node.spi.blockmessaging.BlockItems;
27+
import org.hiero.block.node.spi.historicalblocks.BlockAccessor;
1928

2029
/**
2130
* A utility class to create sample BlockItem objects for testing purposes.
2231
*/
2332
public final class SimpleTestBlockItemBuilder {
33+
private static final Bytes RANDOM_HALF_MB;
34+
35+
static {
36+
final Random random = new Random(1435134141854542L);
37+
final byte[] randomBytes = new byte[512 * 1024];
38+
random.nextBytes(randomBytes);
39+
RANDOM_HALF_MB = Bytes.wrap(randomBytes);
40+
}
41+
2442
public static BlockHeader createBlockHeader(final long blockNumber) {
2543
return new BlockHeader(
2644
new SemanticVersion(1, 2, 3, "a", "b"),
@@ -65,10 +83,43 @@ public static BlockItemUnparsed sampleBlockHeaderUnparsed(final long blockNumber
6583
.build();
6684
}
6785

86+
/**
87+
* Creates a sample BlockItem representing a block header with the given block number and consensus time.
88+
*/
89+
public static BlockItem sampleBlockHeader(final long blockNumber, Instant consensusTime) {
90+
return new BlockItem(new OneOf<>(
91+
ItemOneOfType.BLOCK_HEADER,
92+
new BlockHeader(
93+
new SemanticVersion(1, 2, 3, "a", "b"),
94+
new SemanticVersion(4, 5, 6, "c", "d"),
95+
blockNumber,
96+
new Timestamp(consensusTime.getEpochSecond(), consensusTime.getNano()),
97+
BlockHashAlgorithm.SHA2_384)));
98+
}
99+
100+
/**
101+
* Creates a sample BlockItemUnparsed representing a block header with the given block number and consensus time.
102+
*/
103+
public static BlockItemUnparsed sampleBlockHeaderUnparsed(final long blockNumber, Instant consensusTime) {
104+
//noinspection DataFlowIssue
105+
return BlockItemUnparsed.newBuilder()
106+
.blockHeader(BlockHeader.PROTOBUF.toBytes(
107+
sampleBlockHeader(blockNumber, consensusTime).blockHeader()))
108+
.build();
109+
}
110+
68111
public static BlockItem sampleRoundHeader(final long roundNumber) {
69112
return new BlockItem(new OneOf<>(ItemOneOfType.ROUND_HEADER, createRoundHeader(roundNumber)));
70113
}
71114

115+
/**
116+
* Create an EventHeader with a large 0.5MB signature data.
117+
*/
118+
public static BlockItem sampleLargeEventHeader() {
119+
return new BlockItem(
120+
new OneOf<>(ItemOneOfType.EVENT_HEADER, new EventHeader(EventCore.DEFAULT, RANDOM_HALF_MB)));
121+
}
122+
72123
public static BlockItemUnparsed sampleRoundHeaderUnparsed(final long roundNumber) {
73124
return BlockItemUnparsed.newBuilder()
74125
.roundHeader(createRoundHeaderUnparsed(roundNumber))
@@ -97,26 +148,113 @@ public static BlockItem[] createNumberOfVerySimpleBlocks(final int numberOfBlock
97148

98149
/**
99150
* Creates an array of BlockItem objects representing a very simple block stream of blocks from startBlockNumber to
100-
* but not including endBlockNumber.
151+
* endBlockNumber inclusive.
101152
*
102153
* @param startBlockNumber the starting block number
103154
* @param endBlockNumber the ending block number, inclusive
104155
* @return an array of BlockItem objects
105156
*/
106-
public static BlockItem[] createNumberOfVerySimpleBlocks(final int startBlockNumber, final int endBlockNumber) {
157+
public static BlockItem[] createNumberOfVerySimpleBlocks(final long startBlockNumber, final long endBlockNumber) {
107158
assert startBlockNumber <= endBlockNumber;
108159
assert startBlockNumber >= 0;
109-
final int numberOfBlocks = endBlockNumber - startBlockNumber + 1;
160+
final int numberOfBlocks = (int) (endBlockNumber - startBlockNumber + 1);
110161
final BlockItem[] blockItems = new BlockItem[numberOfBlocks * 3];
111-
for (int blockNumber = startBlockNumber; blockNumber <= endBlockNumber; blockNumber++) {
112-
final int i = (blockNumber - startBlockNumber) * 3;
162+
for (int blockNumber = (int) startBlockNumber; blockNumber <= endBlockNumber; blockNumber++) {
163+
final int i = (blockNumber - (int) startBlockNumber) * 3;
113164
blockItems[i] = sampleBlockHeader(blockNumber);
114165
blockItems[i + 1] = sampleRoundHeader(blockNumber * 10L);
115166
blockItems[i + 2] = sampleBlockProof(blockNumber);
116167
}
117168
return blockItems;
118169
}
119170

171+
/**
172+
* Creates an array of BlockItem objects representing a simple block stream of blocks with large 2.5MB data each
173+
* from startBlockNumber to endBlockNumber inclusive.
174+
*
175+
* @param startBlockNumber the starting block number
176+
* @param endBlockNumber the ending block number, inclusive
177+
* @return an array of BlockItem objects
178+
*/
179+
public static BlockItem[] createNumberOfLargeBlocks(final long startBlockNumber, final long endBlockNumber) {
180+
assert startBlockNumber <= endBlockNumber;
181+
assert startBlockNumber >= 0;
182+
final int numberOfBlocks = (int) (endBlockNumber - startBlockNumber + 1);
183+
final BlockItem[] blockItems = new BlockItem[numberOfBlocks * 8];
184+
for (int blockNumber = (int) startBlockNumber; blockNumber <= endBlockNumber; blockNumber++) {
185+
final int i = (blockNumber - (int) startBlockNumber) * 8;
186+
blockItems[i] = sampleBlockHeader(blockNumber);
187+
blockItems[i + 1] = sampleRoundHeader(blockNumber * 10L);
188+
blockItems[i + 2] = sampleLargeEventHeader();
189+
blockItems[i + 3] = sampleLargeEventHeader();
190+
blockItems[i + 4] = sampleLargeEventHeader();
191+
blockItems[i + 5] = sampleLargeEventHeader();
192+
blockItems[i + 6] = sampleLargeEventHeader();
193+
blockItems[i + 7] = sampleBlockProof(blockNumber);
194+
}
195+
return blockItems;
196+
}
197+
198+
/**
199+
* Creates an array of BlockItem objects representing a very simple block stream of blocks from startBlockNumber to
200+
* but not including endBlockNumber.
201+
*
202+
* @param startBlockNumber the starting block number
203+
* @param endBlockNumber the ending block number, inclusive
204+
* @param firstBlockConsensusTime the consensus time of the first block
205+
* @param consensusTimeBetweenBlocks the time between blocks starts with the first block at 2025-01-01T00:00:00Z
206+
* @return an array of BlockItem objects
207+
*/
208+
public static BlockItemUnparsed[] createNumberOfVerySimpleBlocksUnparsed(
209+
final long startBlockNumber,
210+
final long endBlockNumber,
211+
Instant firstBlockConsensusTime,
212+
Duration consensusTimeBetweenBlocks) {
213+
assert startBlockNumber <= endBlockNumber;
214+
assert startBlockNumber >= 0;
215+
final int numberOfBlocks = (int) (endBlockNumber - startBlockNumber + 1);
216+
final BlockItemUnparsed[] blockItems = new BlockItemUnparsed[numberOfBlocks * 3];
217+
Instant blockTime = firstBlockConsensusTime;
218+
for (int blockNumber = (int) startBlockNumber; blockNumber <= endBlockNumber; blockNumber++) {
219+
final int i = (blockNumber - (int) startBlockNumber) * 3;
220+
blockItems[i] = sampleBlockHeaderUnparsed(blockNumber, blockTime);
221+
blockItems[i + 1] = sampleRoundHeaderUnparsed(blockNumber * 10L);
222+
blockItems[i + 2] = sampleBlockProofUnparsed(blockNumber);
223+
// Increment the block time by the consensus time between blocks
224+
blockTime = blockTime.plus(consensusTimeBetweenBlocks);
225+
}
226+
return blockItems;
227+
}
228+
229+
/**
230+
* Creates an array of BlockAccessor objects representing a very simple block stream of blocks from startBlockNumber
231+
* to but endBlockNumber inclusive.
232+
*
233+
* @param startBlockNumber the starting block number
234+
* @param endBlockNumber the ending block number, inclusive
235+
* @return an array of BlockAccessor objects
236+
*/
237+
public static BlockAccessor[] createNumberOfVerySimpleBlockAccessors(
238+
final int startBlockNumber, final int endBlockNumber) {
239+
return LongStream.range(startBlockNumber, endBlockNumber + 1)
240+
.mapToObj(bn -> {
241+
BlockItem[] blockItems = createNumberOfVerySimpleBlocks(bn, bn);
242+
Block block = new Block(Arrays.asList(blockItems));
243+
return new BlockAccessor() {
244+
@Override
245+
public long blockNumber() {
246+
return bn;
247+
}
248+
249+
@Override
250+
public Block block() {
251+
return block;
252+
}
253+
};
254+
})
255+
.toArray(BlockAccessor[]::new);
256+
}
257+
120258
/**
121259
* Creates an array of BlockItem objects representing a very simple block stream of N blocks.
122260
*

‎block-node/app/src/testFixtures/java/org/hiero/block/node/app/fixtures/plugintest/GrpcPluginTestBase.java

+18-9
Original file line numberDiff line numberDiff line change
@@ -25,22 +25,31 @@
2525
* plugin. It's parent PluginTestBase mocks out all the base functionality of the block node, including the
2626
* configuration, metrics, health, block messaging, and historical block facilities. See {@link PluginTestBase} for
2727
* more details.
28+
* <p>
29+
* Implementations of this class should call one of the start() methods. This will start the plugin and initialize the
30+
* test fixture.
2831
*/
2932
public abstract class GrpcPluginTestBase<P extends BlockNodePlugin> extends PluginTestBase<P>
3033
implements ServiceBuilder {
3134
private record ReqOptions(Optional<String> authority, boolean isProtobuf, boolean isJson, String contentType)
3235
implements ServiceInterface.RequestOptions {}
33-
34-
/** The logger for this class. */
35-
private final System.Logger LOGGER = System.getLogger(getClass().getName());
36-
37-
protected final List<Bytes> fromPluginBytes = new ArrayList<>();
38-
protected final Pipeline<? super Bytes> toPluginPipe;
39-
protected final Pipeline<Bytes> fromPluginPipe;
36+
/** The GRPC bytes received from the plugin. */
37+
protected List<Bytes> fromPluginBytes = new ArrayList<>();
38+
/** The pipeline for GRPC bytes to the plugin. */
39+
protected Pipeline<? super Bytes> toPluginPipe;
40+
/** The pipeline for GRPC bytes from the plugin. */
41+
protected Pipeline<Bytes> fromPluginPipe;
42+
/** The GRPC service interface for the plugin. */
4043
protected ServiceInterface serviceInterface;
4144

42-
public GrpcPluginTestBase(P plugin, Method method, HistoricalBlockFacility historicalBlockFacility) {
43-
super(plugin, historicalBlockFacility);
45+
/**
46+
* Start the test fixture with the given plugin and historical block facility.
47+
*
48+
* @param plugin the plugin to be tested
49+
* @param historicalBlockFacility the historical block facility to be used
50+
*/
51+
public void start(P plugin, Method method, HistoricalBlockFacility historicalBlockFacility) {
52+
super.start(plugin, historicalBlockFacility);
4453
// setup to receive bytes from the plugin
4554
fromPluginPipe = new Pipeline<>() {
4655
@Override

‎block-node/app/src/testFixtures/java/org/hiero/block/node/app/fixtures/plugintest/PluginTestBase.java

+49-12
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import com.swirlds.metrics.api.Metrics;
99
import edu.umd.cs.findbugs.annotations.NonNull;
1010
import io.helidon.webserver.http.HttpService;
11+
import java.util.Collections;
12+
import java.util.Map;
1113
import org.hiero.block.node.spi.BlockNodeContext;
1214
import org.hiero.block.node.spi.BlockNodePlugin;
1315
import org.hiero.block.node.spi.ServiceBuilder;
@@ -24,29 +26,59 @@
2426
* It mocks out all the base functionality of the block node, including the configuration, metrics, health,
2527
* block messaging, and historical block facilities.
2628
* <p>
27-
* The messaging services is mocked out with two concurrent queues, one for block items
29+
* The messaging services are mocked out with two concurrent queues, one for block items
2830
* {@link PluginTestBase :sentBlockBlockItems} and one for block notifications
2931
* {@link PluginTestBase :sentBlockNotifications}. You can look at these queues to see what was sent to the messaging
3032
* service by the plugin.
33+
* <p>
34+
* Implementations of this class should call one of the start() methods. This will start the plugin and initialize the
35+
* test fixture. This fixture uses start() methods vs. a constructor to allow for subclasses to do work before calling
36+
* start(). Such as computing the configuration or starting other things needed like minio.
3137
*
3238
* @param <P> the type of plugin being tested
3339
*/
3440
public abstract class PluginTestBase<P extends BlockNodePlugin> {
35-
private final DefaultMetricsProvider metricsProvider;
36-
protected final BlockNodeContext blockNodeContext;
37-
protected final TestBlockMessagingFacility blockMessaging = new TestBlockMessagingFacility();
38-
protected final P plugin;
41+
/** The logger for this class. */
42+
protected final System.Logger LOGGER = System.getLogger(getClass().getName());
43+
/** The metrics provider for the test. */
44+
private DefaultMetricsProvider metricsProvider;
45+
/** The block node context, for access to core facilities. */
46+
protected BlockNodeContext blockNodeContext;
47+
/** The test block messaging facility, for mocking out the messaging service. */
48+
protected TestBlockMessagingFacility blockMessaging = new TestBlockMessagingFacility();
49+
/** The plugin to be tested */
50+
protected P plugin;
51+
52+
/**
53+
* Start the test fixture with the given plugin and historical block facility.
54+
*
55+
* @param plugin the plugin to be tested
56+
* @param historicalBlockFacility the historical block facility to be used
57+
*/
58+
public void start(P plugin, HistoricalBlockFacility historicalBlockFacility) {
59+
start(plugin, historicalBlockFacility, Collections.emptyMap());
60+
}
3961

40-
public PluginTestBase(P plugin, HistoricalBlockFacility historicalBlockFacility) {
62+
/**
63+
* Start the test fixture with the given plugin, historical block facility, and configuration overrides.
64+
*
65+
* @param plugin the plugin to be tested
66+
* @param historicalBlockFacility the historical block facility to be used
67+
* @param configOverrides a map of configuration overrides to be applied to loaded configuration
68+
*/
69+
public void start(P plugin, HistoricalBlockFacility historicalBlockFacility, Map<String, String> configOverrides) {
4170
this.plugin = plugin;
4271
org.hiero.block.node.app.fixtures.logging.CleanColorfulFormatter.makeLoggingColorful();
4372
// Build the configuration
4473
//noinspection unchecked
45-
final Configuration configuration = ConfigurationBuilder.create()
74+
ConfigurationBuilder configurationBuilder = ConfigurationBuilder.create()
4675
.withConfigDataType(com.swirlds.common.metrics.config.MetricsConfig.class)
47-
.withConfigDataType(com.swirlds.common.metrics.platform.prometheus.PrometheusConfig.class)
4876
.withConfigDataTypes(plugin.configDataTypes().toArray(new Class[0]))
49-
.build();
77+
.withConfigDataType(com.swirlds.common.metrics.platform.prometheus.PrometheusConfig.class);
78+
for (var override : configOverrides.entrySet()) {
79+
configurationBuilder = configurationBuilder.withValue(override.getKey(), override.getValue());
80+
}
81+
final Configuration configuration = configurationBuilder.build();
5082
// create metrics provider
5183
metricsProvider = new DefaultMetricsProvider(configuration);
5284
final Metrics metrics = metricsProvider.createGlobalMetrics();
@@ -71,7 +103,8 @@ public void registerHttpService(String path, HttpService... service) {}
71103
@Override
72104
public void registerGrpcService(@NonNull ServiceInterface service) {}
73105
};
74-
106+
// initialize the block messaging facility
107+
historicalBlockFacility.init(blockNodeContext, mockServiceBuilder);
75108
// if HistoricalBlockFacility is a BlockItemHandler, register it with the messaging facility
76109
if (historicalBlockFacility instanceof BlockItemHandler blockItemHandler) {
77110
blockMessaging.registerBlockItemHandler(
@@ -84,12 +117,16 @@ public void registerGrpcService(@NonNull ServiceInterface service) {}
84117
false,
85118
historicalBlockFacility.getClass().getSimpleName());
86119
}
87-
88-
// start plugin
120+
// init plugin
89121
plugin.init(blockNodeContext, mockServiceBuilder);
122+
// start everything
123+
historicalBlockFacility.start();
90124
plugin.start();
91125
}
92126

127+
/**
128+
* Tears down the test fixture by stopping the metrics provider.
129+
*/
93130
@AfterEach
94131
public void tearDown() {
95132
metricsProvider.stop();

0 commit comments

Comments
 (0)