Skip to content

StorageValue #8482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 58 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
58 commits
Select commit Hold shift + click to select a range
e8be057
StorageValue introduced
Scooletz Apr 8, 2025
9e9e68a
more by ref
Scooletz Apr 8, 2025
ef5cbb8
spacing
Scooletz Apr 8, 2025
4e67b16
compile
Scooletz Apr 8, 2025
e994649
base test fixed
Scooletz Apr 8, 2025
2e42d7f
comparison fixed
Scooletz Apr 8, 2025
95ccb5c
healing test fixed
Scooletz Apr 8, 2025
10f6ccd
tostring
Scooletz Apr 8, 2025
17acda8
resolution
Scooletz Apr 8, 2025
2bd79b4
force update on after equal to zero
Scooletz Apr 8, 2025
dfeef27
fixed test
Scooletz Apr 8, 2025
c392731
Merge branch 'master' into storage-value
benaadams Apr 9, 2025
2fd44e2
hash store amendment
Scooletz Apr 10, 2025
c19e0ff
uint256 property
Scooletz Apr 10, 2025
b9a0cab
Merge branch 'storage-value' of https://github.com/NethermindEth/neth…
Scooletz Apr 10, 2025
349c065
direct stack push
Scooletz Apr 10, 2025
6415c2d
sstore amended
Scooletz Apr 10, 2025
0e4313c
Trailing zero bytes made 2x faster
Scooletz Apr 10, 2025
c56e1dd
trailing zeros fixed
Scooletz Apr 10, 2025
478d678
benchmarks redone
Scooletz Apr 11, 2025
276bad6
tracers
Scooletz Apr 11, 2025
17a830b
benchmark moved
Scooletz Apr 11, 2025
5f42613
benchmarks for storage retrieval
Scooletz Apr 11, 2025
730b2ea
by pointer
Scooletz Apr 15, 2025
e4a4f8e
updates
Scooletz Apr 15, 2025
36a8eb0
local allocation
Scooletz Apr 16, 2025
bd6296c
memory reuse
Scooletz Apr 17, 2025
a9bb991
merged
Scooletz Apr 17, 2025
bdfae5b
minors
Scooletz Apr 17, 2025
0ca0013
no materliazation for sstore
Scooletz Apr 17, 2025
f461292
epoch based storage value map
Scooletz Apr 23, 2025
262988d
StorageValueMap is concurrent and is used in a concurrent way
Scooletz Apr 23, 2025
7357f67
benchmarks
Scooletz Apr 23, 2025
82e04bc
benchmark
Scooletz Apr 24, 2025
36facc5
updated bench
Scooletz Apr 24, 2025
5d01e5a
mapping made smaller
Scooletz Apr 24, 2025
0708319
Merge branch 'master' into storage-value
Scooletz Apr 24, 2025
a77cfcb
coarse grained benchmark
Scooletz Apr 25, 2025
a613c9e
Merge branch 'master' into storage-value
Scooletz Apr 25, 2025
7211928
benchmark fix
Scooletz Apr 25, 2025
ec123b2
review remarks addressed
Scooletz Apr 28, 2025
6577982
Merge branch 'master' into storage-value
Scooletz Apr 28, 2025
0fa3463
Merge branch 'master' into storage-value
benaadams Apr 30, 2025
1d85516
amendments
Scooletz May 1, 2025
cb1b80a
tentative shared ownership over storage map
Scooletz May 1, 2025
a0431ff
fixups
Scooletz May 1, 2025
a341d0b
build
Scooletz May 1, 2025
ba38b27
Merge branch 'master' into storage-value
Scooletz May 5, 2025
c39d51b
ordering parameters
Scooletz May 5, 2025
317166c
benchmark fix
Scooletz May 5, 2025
d0dd465
map undo;
Scooletz May 6, 2025
746224e
storage access benchmark
Scooletz May 6, 2025
27b48c2
ChangeTrace is pooled now
Scooletz May 6, 2025
be7e38a
Merge branch 'master' into storage-value
Scooletz May 6, 2025
33c98b1
fixups after merge and others
Scooletz May 6, 2025
3a837dc
Merge branch 'master' into storage-value
Scooletz May 6, 2025
2acc2cd
fixup
Scooletz May 6, 2025
5c66b5b
disposable
Scooletz May 6, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -362,24 +362,24 @@ private List<string> RunAssertions(BlockchainTest test, Block headBlock, IWorldS
differencesBefore = differences.Count;

KeyValuePair<UInt256, byte[]>[] clearedStorages = new KeyValuePair<UInt256, byte[]>[0];
if (test.Pre.ContainsKey(acountAddress))
if (test.Pre.TryGetValue(acountAddress, out var v))
{
clearedStorages = test.Pre[acountAddress].Storage.Where(s => !accountState.Storage.ContainsKey(s.Key)).ToArray();
clearedStorages = v.Storage.Where(s => !accountState.Storage.ContainsKey(s.Key)).ToArray();
}

foreach (KeyValuePair<UInt256, byte[]> clearedStorage in clearedStorages)
{
ReadOnlySpan<byte> value = !stateProvider.AccountExists(acountAddress) ? Bytes.Empty : stateProvider.Get(new StorageCell(acountAddress, clearedStorage.Key));
if (!value.IsZero())
StorageValue value = !stateProvider.AccountExists(acountAddress) ? StorageValue.Zero : stateProvider.Get(new StorageCell(acountAddress, clearedStorage.Key));
if (!value.IsZero)
{
differences.Add($"{acountAddress} storage[{clearedStorage.Key}] exp: 0x00, actual: {value.ToHexString(true)}");
}
}

foreach (KeyValuePair<UInt256, byte[]> storageItem in accountState.Storage)
{
ReadOnlySpan<byte> value = !stateProvider.AccountExists(acountAddress) ? Bytes.Empty : stateProvider.Get(new StorageCell(acountAddress, storageItem.Key));
if (!Bytes.AreEqual(storageItem.Value, value))
StorageValue value = !stateProvider.AccountExists(acountAddress) ? StorageValue.Zero : stateProvider.Get(new StorageCell(acountAddress, storageItem.Key));
if (!new StorageValue(storageItem.Value).Equals(value))
{
differences.Add($"{acountAddress} storage[{storageItem.Key}] exp: {storageItem.Value.ToHexString(true)}, actual: {value.ToHexString(true)}");
}
Expand Down
106 changes: 106 additions & 0 deletions src/Nethermind/Nethermind.Benchmark/State/StorageAccessBenchmark.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
// SPDX-License-Identifier: LGPL-3.0-only

using BenchmarkDotNet.Attributes;
using Nethermind.Core;
using Nethermind.Core.Crypto;
using Nethermind.Db;
using Nethermind.Int256;
using Nethermind.Logging;
using Nethermind.Specs.Forks;
using Nethermind.State;
using Nethermind.Trie.Pruning;

namespace Nethermind.Benchmarks.State;

[DisassemblyDiagnoser]
[MemoryDiagnoser]
public class StorageAccessBenchmark
{
private WorldState _preCached;
private WorldState _notCached;

private const uint MaxPrecalculatedIndex = 1024;

private static readonly Address Account = new(Keccak.Compute("test"));
private static readonly StorageCell A = new(Account, MaxPrecalculatedIndex / 2);
private static readonly StorageCell B = new(Account, MaxPrecalculatedIndex / 2 - 1);
private static readonly StorageCell C = new(Account, MaxPrecalculatedIndex / 2 - 2);
private static readonly StorageCell D = new(Account, MaxPrecalculatedIndex / 2 - 3);

private static readonly StorageValue ValueA = new(new UInt256(1));
private static readonly StorageValue ValueB = new(new UInt256(2));
private static readonly StorageValue ValueC = new(new UInt256(3));
private static readonly StorageValue ValueD = new(new UInt256(4));

[GlobalSetup]
public void Setup()
{
var preCache = new PreBlockCaches();
var code = new MemDb();

_preCached = new WorldState(new TrieStore(new MemDb("storage"), NullLogManager.Instance), code,
LimboLogs.Instance, preCache, false);

_notCached = new WorldState(new TrieStore(new MemDb("storage"), NullLogManager.Instance), code,
LimboLogs.Instance, null, false);

_notCached.CreateAccount(Account, 100, 100);

for (uint i = 0; i < MaxPrecalculatedIndex; i++)
{
var cell = new StorageCell(Account, i);
var value = new StorageValue(i);

preCache.StorageCache[cell] = value;
_preCached.Set(cell, value);
_notCached.Set(cell, value);
}

_preCached.Commit(Prague.Instance);
_preCached.CommitTree(123);
_notCached.Commit(Prague.Instance);
_notCached.CommitTree(123);
}

[Benchmark]
public void Just_reset()
{
_notCached.Reset(true);
}

[Benchmark(OperationsPerInvoke = 4)]
public bool PreCached_small_indexes()
{
_preCached.Reset(true);

return
_preCached.Get(A).IsZero &&
_preCached.Get(B).IsZero &&
_preCached.Get(C).IsZero &&
_preCached.Get(D).IsZero;
}

[Benchmark(OperationsPerInvoke = 4)]
public bool NotCached_small_indexes()
{
_notCached.Reset(true);

return
_notCached.Get(A).IsZero &&
_notCached.Get(B).IsZero &&
_notCached.Get(C).IsZero &&
_notCached.Get(D).IsZero;
}

[Benchmark(OperationsPerInvoke = 4)]
public void Set()
{
_notCached.Reset(true);

_notCached.Set(A, ValueA);
_notCached.Set(B, ValueB);
_notCached.Set(C, ValueC);
_notCached.Set(D, ValueD);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ public class StorageCellBenchmark

private const int OperationsPerInvoke = 1000;


[GlobalSetup]
public void Setup()
{
Expand Down Expand Up @@ -46,11 +45,11 @@ private class StorageTracer : IStorageTracer
{
public bool IsTracingStorage => throw new NotImplementedException();

public void ReportStorageChange(in ReadOnlySpan<byte> key, in ReadOnlySpan<byte> value) =>
public void ReportStorageChange(in ReadOnlySpan<byte> key, in StorageValue value) =>
throw new NotImplementedException();

[MethodImpl(MethodImplOptions.NoInlining)]
public void ReportStorageChange(in StorageCell storageCell, byte[] before, byte[] after) { }
public void ReportStorageChange(in StorageCell storageCell, in StorageValue before, in StorageValue after) { }

[MethodImpl(MethodImplOptions.NoInlining)]
public void ReportStorageRead(in StorageCell storageCell) { }
Expand Down
40 changes: 40 additions & 0 deletions src/Nethermind/Nethermind.Benchmark/State/StorageValueBenchmark.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
// SPDX-License-Identifier: LGPL-3.0-only

using System;
using BenchmarkDotNet.Attributes;
using Nethermind.Core;

namespace Nethermind.Benchmarks.State;

public class StorageValueBenchmark
{
[Benchmark(OperationsPerInvoke = 8)]
[Arguments(0)]
[Arguments(1)]
[Arguments(8)]
[Arguments(9)]
[Arguments(17)]
[Arguments(18)]
[Arguments(23)]
[Arguments(24)]
[Arguments(31)]
public int LeadingZeros(int nonZero)
{
Span<byte> span = stackalloc byte[32];
span[nonZero] = 1;

var v = new StorageValue(span);

return
v.BytesWithNoLeadingZeroes.Length +
v.BytesWithNoLeadingZeroes.Length +
v.BytesWithNoLeadingZeroes.Length +
v.BytesWithNoLeadingZeroes.Length +

v.BytesWithNoLeadingZeroes.Length +
v.BytesWithNoLeadingZeroes.Length +
v.BytesWithNoLeadingZeroes.Length +
v.BytesWithNoLeadingZeroes.Length;
}
}
8 changes: 3 additions & 5 deletions src/Nethermind/Nethermind.Blockchain/Blocks/BlockhashStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ namespace Nethermind.Blockchain.Blocks;
public class BlockhashStore(ISpecProvider specProvider, IWorldState worldState)
: IBlockhashStore
{
private static readonly byte[] EmptyBytes = [0];

public void ApplyBlockhashStateChanges(BlockHeader blockHeader)
{
IReleaseSpec spec = specProvider.GetSpec(blockHeader);
Expand All @@ -30,7 +28,7 @@ public void ApplyBlockhashStateChanges(BlockHeader blockHeader)
Hash256 parentBlockHash = blockHeader.ParentHash;
var parentBlockIndex = new UInt256((ulong)((blockHeader.Number - 1) % Eip2935Constants.RingBufferSize));
StorageCell blockHashStoreCell = new(eip2935Account, parentBlockIndex);
worldState.Set(blockHashStoreCell, parentBlockHash!.Bytes.WithoutLeadingZeros().ToArray());
worldState.Set(blockHashStoreCell, new StorageValue(parentBlockHash!.Bytes));
}

public Hash256? GetBlockHashFromState(BlockHeader currentHeader, long requiredBlockNumber)
Expand All @@ -44,7 +42,7 @@ public void ApplyBlockhashStateChanges(BlockHeader blockHeader)
var blockIndex = new UInt256((ulong)(requiredBlockNumber % Eip2935Constants.RingBufferSize));
Address? eip2935Account = spec.Eip2935ContractAddress ?? Eip2935Constants.BlockHashHistoryAddress;
StorageCell blockHashStoreCell = new(eip2935Account, blockIndex);
ReadOnlySpan<byte> data = worldState.Get(blockHashStoreCell);
return data.SequenceEqual(EmptyBytes) ? null : Hash256.FromBytesWithPadding(data);
ref readonly StorageValue data = ref worldState.Get(blockHashStoreCell);
return data.IsZero ? null : Hash256.FromBytesWithPadding(data.Bytes);
}
}
2 changes: 1 addition & 1 deletion src/Nethermind/Nethermind.Blockchain/GenesisLoader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ private void Preallocate(Block genesis)
foreach (KeyValuePair<UInt256, byte[]> storage in allocation.Storage)
{
_stateProvider.Set(new StorageCell(address, storage.Key),
storage.Value.WithoutLeadingZeros().ToArray());
new StorageValue(storage.Value));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ protected virtual async Task<TestBlockchain> Build(Action<ContainerBuilder>? con
byte[] code = Bytes.FromHexString("0xabcd");
state.InsertCode(TestItem.AddressA, code, SpecProvider.GenesisSpec);

state.Set(new StorageCell(TestItem.AddressA, UInt256.One), Bytes.FromHexString("0xabcdef"));
state.Set(new StorageCell(TestItem.AddressA, UInt256.One), new StorageValue(Bytes.FromHexString("0xabcdef")));

state.Commit(SpecProvider.GenesisSpec);
state.CommitTree(0);
Expand Down
52 changes: 26 additions & 26 deletions src/Nethermind/Nethermind.Core.Test/RlpTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ public void Serializing_sequence_with_one_int_regression()
}

[Test]
[Explicit("That was a regression test but now it is failing again and cannot find the reason we needed this behaviour in the first place. Sync works all fine. Leaving it here as it may resurface - make sure to add more explanation to it in such case.")]
[Explicit(
"That was a regression test but now it is failing again and cannot find the reason we needed this behaviour in the first place. Sync works all fine. Leaving it here as it may resurface - make sure to add more explanation to it in such case.")]
public void Serializing_object_int_regression()
{
Rlp output = Rlp.Encode(new[] { Rlp.Encode(1) });
Expand Down Expand Up @@ -99,11 +100,9 @@ public void Long_negative()
public void Empty_byte_array()
{
byte[] bytes = [];
Rlp rlp = Rlp.Encode(bytes);
Rlp rlpSpan = Rlp.Encode(bytes.AsSpan());
Rlp expectedResult = new(new byte[] { 128 });
Assert.That(rlp, Is.EqualTo(expectedResult), "byte array");
Assert.That(rlpSpan, Is.EqualTo(expectedResult), "span");

AssertAllEncodings(expectedResult, bytes);
}

[TestCase(0)]
Expand All @@ -112,23 +111,19 @@ public void Empty_byte_array()
public void Byte_array_of_length_1_and_first_byte_value_less_than_128(byte value)
{
byte[] bytes = { value };
Rlp rlp = Rlp.Encode(bytes);
Rlp rlpSpan = Rlp.Encode(bytes.AsSpan());
Rlp expectedResult = new(new[] { value });
Assert.That(rlp, Is.EqualTo(expectedResult), "byte array");
Assert.That(rlpSpan, Is.EqualTo(expectedResult), "span");

AssertAllEncodings(expectedResult, bytes);
}

[TestCase(128)]
[TestCase(255)]
public void Byte_array_of_length_1_and_first_byte_value_equal_or_more_than_128(byte value)
{
byte[] bytes = { value };
Rlp rlp = Rlp.Encode(bytes);
Rlp rlpSpan = Rlp.Encode(bytes.AsSpan());
Rlp expectedResult = new(new[] { (byte)129, value });
Assert.That(rlp, Is.EqualTo(expectedResult), "byte array");
Assert.That(rlpSpan, Is.EqualTo(expectedResult), "span");

AssertAllEncodings(expectedResult, bytes);
}

[Test]
Expand All @@ -147,8 +142,7 @@ public void Byte_array_of_length_55()

Rlp expectedResult = new(expectedResultBytes);

Assert.That(Rlp.Encode(input), Is.EqualTo(expectedResult), "byte array");
Assert.That(Rlp.Encode(input.AsSpan()), Is.EqualTo(expectedResult), "span");
AssertAllEncodings(expectedResult, input);
}

[Test]
Expand All @@ -168,8 +162,7 @@ public void Byte_array_of_length_56()

Rlp expectedResult = new(expectedResultBytes);

Assert.That(Rlp.Encode(input), Is.EqualTo(expectedResult), "byte array");
Assert.That(Rlp.Encode(input.AsSpan()), Is.EqualTo(expectedResult), "span");
AssertAllEncodings(expectedResult, input);
}

[Test]
Expand All @@ -190,22 +183,22 @@ public void Long_byte_array()

Rlp expectedResult = new(expectedResultBytes);

Assert.That(Rlp.Encode(input), Is.EqualTo(expectedResult), "byte array");
Assert.That(Rlp.Encode(input.AsSpan()), Is.EqualTo(expectedResult), "span");
AssertAllEncodings(expectedResult, input);
}

[TestCase(new byte[] { 127, 1, 2, 2 }, false)]
[TestCase(new byte[] { 130, 1, 0 }, true)]
[TestCase(new byte[] { 130, 0, 2, 2 }, false)]
[TestCase(new byte[] { 130, 0, 2, 2 }, false)]
[TestCase(new byte[]
{184, 56,
1,0,0,0,0,0,0,0,
1,0,0,0,0,0,0,0,
1,0,0,0,0,0,0,0,
1,0,0,0,0,0,0,0,
1,0,0,0,0,0,0,0,
1,0,0,0,0,0,0,0
{
184, 56,
1, 0, 0, 0, 0, 0, 0, 0,
1, 0, 0, 0, 0, 0, 0, 0,
1, 0, 0, 0, 0, 0, 0, 0,
1, 0, 0, 0, 0, 0, 0, 0,
1, 0, 0, 0, 0, 0, 0, 0,
1, 0, 0, 0, 0, 0, 0, 0
}, true)]
public void Strange_bool(byte[] rlp, bool expectedBool)
{
Expand Down Expand Up @@ -272,5 +265,12 @@ public void RlpContextWithSliceMemory_shouldNotCopyUnderlyingData(bool sliceValu
isACopy.Should().NotBe(sliceValue);
}
}

private static void AssertAllEncodings(Rlp expected, byte[] bytes)
{
Assert.That(Rlp.Encode(bytes), Is.EqualTo(expected), "byte array");
Assert.That(Rlp.Encode(bytes.AsSpan()), Is.EqualTo(expected), "span");
Rlp.EncodeToArray(bytes).Should().BeEquivalentTo(expected.Bytes, "to array");
}
}
}
Loading