From c62f192459626a602b90a4959eab047b326c1b4a Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 11 Jun 2024 15:28:30 +0200 Subject: [PATCH] Improve genesis state performance at startup (#6977) * Refactor genesis options file management Signed-off-by: Fabio Di Fabio * Make loading allocation from genesis lazy Signed-off-by: Fabio Di Fabio * Update tests Signed-off-by: Fabio Di Fabio * Memory optimization with streaming Signed-off-by: Fabio Di Fabio * Improve loading and storgin genesis state at startup Signed-off-by: Fabio Di Fabio * Remove comments Signed-off-by: Fabio Di Fabio * Avoid parsing genesis file allocations twice Signed-off-by: Fabio Di Fabio * Update javadoc Signed-off-by: Fabio Di Fabio * Fix Signed-off-by: Fabio Di Fabio * Fix Signed-off-by: Fabio Di Fabio * Ignore unknown objects in allocations Signed-off-by: Fabio Di Fabio * avoid keeping genesis allocation data in memory Signed-off-by: Fabio Di Fabio * Update CHANGELOG Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio Signed-off-by: ahamlat Co-authored-by: ahamlat --- CHANGELOG.md | 1 + .../controller/BesuControllerBuilder.java | 50 ++-- .../besu/ForkIdsNetworkConfigTest.java | 3 +- .../besu/config/GenesisAccount.java | 42 +++ .../besu/config/GenesisAllocation.java | 109 -------- .../besu/config/GenesisConfigFile.java | 51 ++-- .../besu/config/GenesisReader.java | 242 ++++++++++++++++++ .../org/hyperledger/besu/config/JsonUtil.java | 148 ++++++++++- .../besu/config/GenesisConfigFileTest.java | 59 +++-- .../besu/config/GenesisReaderTest.java | 98 +++++++ .../hyperledger/besu/config/JsonUtilTest.java | 23 +- .../MergeGenesisConfigHelper.java | 7 +- .../besu/ethereum/chain/GenesisState.java | 156 +++-------- .../besu/ethereum/chain/GenesisStateTest.java | 64 ++--- .../bonsai/AbstractIsolationTests.java | 15 +- .../bonsai/BonsaiSnapshotIsolationTests.java | 2 +- 16 files changed, 713 insertions(+), 357 deletions(-) create mode 100644 config/src/main/java/org/hyperledger/besu/config/GenesisAccount.java delete mode 100644 config/src/main/java/org/hyperledger/besu/config/GenesisAllocation.java create mode 100644 config/src/main/java/org/hyperledger/besu/config/GenesisReader.java create mode 100644 config/src/test/java/org/hyperledger/besu/config/GenesisReaderTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b64946ae0..435ecea9e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ ### Additions and Improvements - Add two counters to DefaultBlockchain in order to be able to calculate TPS and Mgas/s [#7105](https://github.com/hyperledger/besu/pull/7105) +- Improve genesis state performance at startup [#6977](https://github.com/hyperledger/besu/pull/6977) - Enable --Xbonsai-limit-trie-logs-enabled by default, unless sync-mode=FULL [#7181](https://github.com/hyperledger/besu/pull/7181) - Promote experimental --Xbonsai-limit-trie-logs-enabled to production-ready, --bonsai-limit-trie-logs-enabled [#7192](https://github.com/hyperledger/besu/pull/7192) - Promote experimental --Xbonsai-trie-logs-pruning-window-size to production-ready, --bonsai-trie-logs-pruning-window-size [#7192](https://github.com/hyperledger/besu/pull/7192) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 4e3929e8ae..aab0000592 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -41,6 +41,7 @@ import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.chain.VariablesStorage; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; @@ -552,30 +553,9 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides prepForBuild(); final ProtocolSchedule protocolSchedule = createProtocolSchedule(); - final GenesisState genesisState; final VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); - Optional genesisStateHash = Optional.empty(); - if (variablesStorage != null && this.genesisStateHashCacheEnabled) { - genesisStateHash = variablesStorage.getGenesisStateHash(); - } - - if (genesisStateHash.isPresent()) { - genesisState = - GenesisState.fromConfig(genesisStateHash.get(), genesisConfigFile, protocolSchedule); - } else { - genesisState = - GenesisState.fromConfig(dataStorageConfiguration, genesisConfigFile, protocolSchedule); - if (variablesStorage != null) { - VariablesStorage.Updater updater = variablesStorage.updater(); - if (updater != null) { - updater.setGenesisStateHash(genesisState.getBlock().getHeader().getStateRoot()); - updater.commit(); - } - } - } - final WorldStateStorageCoordinator worldStateStorageCoordinator = storageProvider.createWorldStateStorageCoordinator(dataStorageConfiguration); @@ -583,6 +563,13 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides storageProvider.createBlockchainStorage( protocolSchedule, variablesStorage, dataStorageConfiguration); + final var maybeStoredGenesisBlockHash = blockchainStorage.getBlockHash(0L); + + final var genesisState = + getGenesisState( + maybeStoredGenesisBlockHash.flatMap(blockchainStorage::getBlockHeader), + protocolSchedule); + final MutableBlockchain blockchain = DefaultBlockchain.createMutable( genesisState.getBlock(), @@ -591,7 +578,6 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides reorgLoggingThreshold, dataDirectory.toString(), numberOfBlocksToCache); - final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = besuComponent .map(BesuComponent::getCachedMerkleTrieLoader) @@ -601,7 +587,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides createWorldStateArchive( worldStateStorageCoordinator, blockchain, bonsaiCachedMerkleTrieLoader); - if (blockchain.getChainHeadBlockNumber() < 1) { + if (maybeStoredGenesisBlockHash.isEmpty()) { genesisState.writeStateTo(worldStateArchive.getMutable()); } @@ -772,6 +758,24 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides dataStorageConfiguration); } + private GenesisState getGenesisState( + final Optional maybeGenesisBlockHeader, + final ProtocolSchedule protocolSchedule) { + final Optional maybeGenesisStateRoot = + genesisStateHashCacheEnabled + ? maybeGenesisBlockHeader.map(BlockHeader::getStateRoot) + : Optional.empty(); + + return maybeGenesisStateRoot + .map( + genesisStateRoot -> + GenesisState.fromStorage(genesisStateRoot, genesisConfigFile, protocolSchedule)) + .orElseGet( + () -> + GenesisState.fromConfig( + dataStorageConfiguration, genesisConfigFile, protocolSchedule)); + } + private TrieLogPruner createTrieLogPruner( final WorldStateKeyValueStorage worldStateStorage, final Blockchain blockchain, diff --git a/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java b/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java index 7d7c6391ff..ae338f27ca 100644 --- a/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java +++ b/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java @@ -18,7 +18,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.hyperledger.besu.cli.config.EthNetworkConfig; import org.hyperledger.besu.cli.config.NetworkName; import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.config.GenesisConfigOptions; @@ -138,7 +137,7 @@ public class ForkIdsNetworkConfigTest { @MethodSource("parameters") public void testForkId(final NetworkName chainName, final List expectedForkIds) { final GenesisConfigFile genesisConfigFile = - GenesisConfigFile.fromConfig(EthNetworkConfig.jsonConfig(chainName)); + GenesisConfigFile.fromResource(chainName.getGenesisFile()); final MilestoneStreamingTransitionProtocolSchedule schedule = createSchedule(genesisConfigFile); final GenesisState genesisState = GenesisState.fromConfig(genesisConfigFile, schedule); final Blockchain mockBlockchain = mock(Blockchain.class); diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisAccount.java b/config/src/main/java/org/hyperledger/besu/config/GenesisAccount.java new file mode 100644 index 0000000000..70bb80e042 --- /dev/null +++ b/config/src/main/java/org/hyperledger/besu/config/GenesisAccount.java @@ -0,0 +1,42 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.config; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; + +import java.util.Map; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * Genesis account + * + * @param address of the account + * @param nonce nonce of the account at genesis + * @param balance balance of the account at genesis + * @param code code of the account at genesis, can be null + * @param storage storage of the account at genesis + * @param privateKey of the account, only use for testing + */ +public record GenesisAccount( + Address address, + long nonce, + Wei balance, + Bytes code, + Map storage, + Bytes32 privateKey) {} diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisAllocation.java b/config/src/main/java/org/hyperledger/besu/config/GenesisAllocation.java deleted file mode 100644 index c53ded9e86..0000000000 --- a/config/src/main/java/org/hyperledger/besu/config/GenesisAllocation.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.config; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import com.fasterxml.jackson.databind.node.ObjectNode; - -/** The Genesis allocation configuration. */ -public class GenesisAllocation { - private final String address; - private final ObjectNode data; - - /** - * Instantiates a new Genesis allocation. - * - * @param address the address - * @param data the data - */ - GenesisAllocation(final String address, final ObjectNode data) { - this.address = address; - this.data = data; - } - - /** - * Gets address. - * - * @return the address - */ - public String getAddress() { - return address; - } - - /** - * Gets private key. - * - * @return the private key - */ - public Optional getPrivateKey() { - return Optional.ofNullable(JsonUtil.getString(data, "privatekey", null)); - } - - /** - * Gets balance. - * - * @return the balance - */ - public String getBalance() { - return JsonUtil.getValueAsString(data, "balance", "0"); - } - - /** - * Gets code. - * - * @return the code - */ - public String getCode() { - return JsonUtil.getString(data, "code", null); - } - - /** - * Gets nonce. - * - * @return the nonce - */ - public String getNonce() { - return JsonUtil.getValueAsString(data, "nonce", "0"); - } - - /** - * Gets version. - * - * @return the version - */ - public String getVersion() { - return JsonUtil.getValueAsString(data, "version", null); - } - - /** - * Gets storage map. - * - * @return fields under storage as a map - */ - public Map getStorage() { - final Map map = new HashMap<>(); - JsonUtil.getObjectNode(data, "storage") - .orElse(JsonUtil.createEmptyObjectNode()) - .fields() - .forEachRemaining( - (entry) -> { - map.put(entry.getKey(), entry.getValue().asText()); - }); - return map; - } -} diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java b/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java index cf5a13d855..bd1e117773 100644 --- a/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java +++ b/config/src/main/java/org/hyperledger/besu/config/GenesisConfigFile.java @@ -14,8 +14,6 @@ */ package org.hyperledger.besu.config; -import static org.hyperledger.besu.config.JsonUtil.normalizeKeys; - import org.hyperledger.besu.datatypes.Wei; import java.net.URL; @@ -30,22 +28,23 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Streams; /** The Genesis config file. */ public class GenesisConfigFile { /** The constant DEFAULT. */ public static final GenesisConfigFile DEFAULT = - new GenesisConfigFile(JsonUtil.createEmptyObjectNode()); + new GenesisConfigFile(new GenesisReader.FromObjectNode(JsonUtil.createEmptyObjectNode())); /** The constant BASEFEE_AT_GENESIS_DEFAULT_VALUE. */ public static final Wei BASEFEE_AT_GENESIS_DEFAULT_VALUE = Wei.of(1_000_000_000L); + private final GenesisReader loader; private final ObjectNode genesisRoot; - private GenesisConfigFile(final ObjectNode config) { - this.genesisRoot = config; + private GenesisConfigFile(final GenesisReader loader) { + this.loader = loader; + this.genesisRoot = loader.getRoot(); } /** @@ -70,21 +69,31 @@ public class GenesisConfigFile { /** * Genesis file from resource. * - * @param jsonResource the resource name + * @param resourceName the resource name + * @return the genesis config file + */ + public static GenesisConfigFile fromResource(final String resourceName) { + return fromConfig(GenesisConfigFile.class.getResource(resourceName)); + } + + /** + * From config genesis config file. + * + * @param jsonSource the json string * @return the genesis config file */ - public static GenesisConfigFile fromResource(final String jsonResource) { - return fromSource(GenesisConfigFile.class.getResource(jsonResource)); + public static GenesisConfigFile fromConfig(final URL jsonSource) { + return new GenesisConfigFile(new GenesisReader.FromURL(jsonSource)); } /** * From config genesis config file. * - * @param jsonString the json string + * @param json the json string * @return the genesis config file */ - public static GenesisConfigFile fromConfig(final String jsonString) { - return fromConfig(JsonUtil.objectNodeFromString(jsonString, false)); + public static GenesisConfigFile fromConfig(final String json) { + return fromConfig(JsonUtil.objectNodeFromString(json, false)); } /** @@ -94,7 +103,7 @@ public class GenesisConfigFile { * @return the genesis config file */ public static GenesisConfigFile fromConfig(final ObjectNode config) { - return new GenesisConfigFile(normalizeKeys(config)); + return new GenesisConfigFile(new GenesisReader.FromObjectNode(config)); } /** @@ -113,8 +122,7 @@ public class GenesisConfigFile { * @return the config options */ public GenesisConfigOptions getConfigOptions(final Map overrides) { - final ObjectNode config = - JsonUtil.getObjectNode(genesisRoot, "config").orElse(JsonUtil.createEmptyObjectNode()); + final ObjectNode config = loader.getConfig(); Map overridesRef = overrides; @@ -134,15 +142,8 @@ public class GenesisConfigFile { * * @return the stream */ - public Stream streamAllocations() { - return JsonUtil.getObjectNode(genesisRoot, "alloc").stream() - .flatMap( - allocations -> - Streams.stream(allocations.fieldNames()) - .map( - key -> - new GenesisAllocation( - key, JsonUtil.getObjectNode(allocations, key).get()))); + public Stream streamAllocations() { + return loader.streamAllocations(); } /** @@ -344,7 +345,7 @@ public class GenesisConfigFile { + "genesisRoot=" + genesisRoot + ", allocations=" - + streamAllocations().map(GenesisAllocation::toString).collect(Collectors.joining(",")) + + loader.streamAllocations().map(GenesisAccount::toString).collect(Collectors.joining(",")) + '}'; } } diff --git a/config/src/main/java/org/hyperledger/besu/config/GenesisReader.java b/config/src/main/java/org/hyperledger/besu/config/GenesisReader.java new file mode 100644 index 0000000000..316aa73d04 --- /dev/null +++ b/config/src/main/java/org/hyperledger/besu/config/GenesisReader.java @@ -0,0 +1,242 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.config; + +import static org.hyperledger.besu.config.JsonUtil.normalizeKey; +import static org.hyperledger.besu.config.JsonUtil.normalizeKeys; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; + +import java.io.IOException; +import java.math.BigInteger; +import java.net.URL; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Streams; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +interface GenesisReader { + String CONFIG_FIELD = "config"; + String ALLOCATION_FIELD = "alloc"; + + ObjectNode getRoot(); + + ObjectNode getConfig(); + + Stream streamAllocations(); + + class FromObjectNode implements GenesisReader { + private final ObjectNode allocations; + private final ObjectNode rootWithoutAllocations; + + public FromObjectNode(final ObjectNode root) { + final var removedAllocations = root.remove(ALLOCATION_FIELD); + this.allocations = + removedAllocations != null + ? (ObjectNode) removedAllocations + : JsonUtil.createEmptyObjectNode(); + this.rootWithoutAllocations = normalizeKeys(root); + } + + @Override + public ObjectNode getRoot() { + return rootWithoutAllocations; + } + + @Override + public ObjectNode getConfig() { + return JsonUtil.getObjectNode(rootWithoutAllocations, CONFIG_FIELD) + .orElse(JsonUtil.createEmptyObjectNode()); + } + + @Override + public Stream streamAllocations() { + return Streams.stream(allocations.fields()) + .map( + entry -> { + final var on = normalizeKeys((ObjectNode) entry.getValue()); + return new GenesisAccount( + Address.fromHexString(entry.getKey()), + JsonUtil.getString(on, "nonce").map(ParserUtils::parseUnsignedLong).orElse(0L), + JsonUtil.getString(on, "balance") + .map(ParserUtils::parseBalance) + .orElse(Wei.ZERO), + JsonUtil.getBytes(on, "code", null), + ParserUtils.getStorageMap(on, "storage"), + JsonUtil.getBytes(on, "privatekey").map(Bytes32::wrap).orElse(null)); + }); + } + } + + class FromURL implements GenesisReader { + private final URL url; + private final ObjectNode rootWithoutAllocations; + + public FromURL(final URL url) { + this.url = url; + this.rootWithoutAllocations = + normalizeKeys(JsonUtil.objectNodeFromURL(url, false, ALLOCATION_FIELD)); + } + + @Override + public ObjectNode getRoot() { + return rootWithoutAllocations; + } + + @Override + public ObjectNode getConfig() { + return JsonUtil.getObjectNode(rootWithoutAllocations, CONFIG_FIELD) + .orElse(JsonUtil.createEmptyObjectNode()); + } + + @Override + public Stream streamAllocations() { + final var parser = JsonUtil.jsonParserFromURL(url, false); + + try { + parser.nextToken(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + if (ALLOCATION_FIELD.equals(parser.getCurrentName())) { + parser.nextToken(); + parser.nextToken(); + break; + } else { + parser.skipChildren(); + } + } + } catch (final IOException e) { + throw new RuntimeException(e); + } + + return Streams.stream(new AllocationIterator(parser)); + } + + private static class AllocationIterator implements Iterator { + final JsonParser parser; + + public AllocationIterator(final JsonParser parser) { + this.parser = parser; + } + + @Override + public boolean hasNext() { + final var end = parser.currentToken() == JsonToken.END_OBJECT; + if (end) { + try { + parser.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return !end; + } + + @Override + public GenesisAccount next() { + try { + final Address address = Address.fromHexString(parser.currentName()); + long nonce = 0; + Wei balance = Wei.ZERO; + Bytes code = null; + Map storage = Map.of(); + Bytes32 privateKey = null; + parser.nextToken(); // consume start object + while (parser.nextToken() != JsonToken.END_OBJECT) { + switch (normalizeKey(parser.currentName())) { + case "nonce": + parser.nextToken(); + nonce = ParserUtils.parseUnsignedLong(parser.getText()); + break; + case "balance": + parser.nextToken(); + balance = ParserUtils.parseBalance(parser.getText()); + break; + case "code": + parser.nextToken(); + code = Bytes.fromHexStringLenient(parser.getText()); + break; + case "privatekey": + parser.nextToken(); + privateKey = Bytes32.fromHexStringLenient(parser.getText()); + break; + case "storage": + parser.nextToken(); + storage = new HashMap<>(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + final var key = UInt256.fromHexString(parser.currentName()); + parser.nextToken(); + final var value = UInt256.fromHexString(parser.getText()); + storage.put(key, value); + } + break; + } + if (parser.currentToken() == JsonToken.START_OBJECT) { + // ignore any unknown nested object + parser.skipChildren(); + } + } + parser.nextToken(); + return new GenesisAccount(address, nonce, balance, code, storage, privateKey); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } + + class ParserUtils { + static long parseUnsignedLong(final String value) { + String v = value.toLowerCase(Locale.US); + if (v.startsWith("0x")) { + v = v.substring(2); + } + return Long.parseUnsignedLong(v, 16); + } + + static Wei parseBalance(final String balance) { + final BigInteger val; + if (balance.startsWith("0x")) { + val = new BigInteger(1, Bytes.fromHexStringLenient(balance).toArrayUnsafe()); + } else { + val = new BigInteger(balance); + } + + return Wei.of(val); + } + + static Map getStorageMap(final ObjectNode json, final String key) { + return JsonUtil.getObjectNode(json, key) + .map( + storageMap -> + Streams.stream(storageMap.fields()) + .collect( + Collectors.toMap( + e -> UInt256.fromHexString(e.getKey()), + e -> UInt256.fromHexString(e.getValue().asText())))) + .orElse(Map.of()); + } + } +} diff --git a/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java b/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java index fba62dd906..bcb89c64ed 100644 --- a/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java +++ b/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java @@ -23,17 +23,30 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; +import java.util.Set; +import java.util.function.Function; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonParser.Feature; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.filter.FilteringParserDelegate; +import com.fasterxml.jackson.core.filter.TokenFilter; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.tuweni.bytes.Bytes; /** The Json util class. */ public class JsonUtil { + private static final JsonFactory JSON_FACTORY = + JsonFactory.builder() + .disable(JsonFactory.Feature.INTERN_FIELD_NAMES) + .disable(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES) + .build(); + /** Default constructor. */ private JsonUtil() {} @@ -53,7 +66,7 @@ public class JsonUtil { entry -> { final String key = entry.getKey(); final JsonNode value = entry.getValue(); - final String normalizedKey = key.toLowerCase(Locale.US); + final String normalizedKey = normalizeKey(key); if (value instanceof ObjectNode) { normalized.set(normalizedKey, normalizeKeys((ObjectNode) value)); } else if (value instanceof ArrayNode) { @@ -65,6 +78,17 @@ public class JsonUtil { return normalized; } + /** + * Converts the key to lowercase for easier lookup. This is useful in cases such as the + * 'genesis.json' file where all keys are assumed to be case insensitive. + * + * @param key the key to be normalized + * @return key in lower case. + */ + public static String normalizeKey(final String key) { + return key.toLowerCase(Locale.US); + } + private static ArrayNode normalizeKeysInArray(final ArrayNode arrayNode) { final ArrayNode normalizedArray = JsonUtil.createEmptyArrayNode(); arrayNode.forEach( @@ -263,6 +287,35 @@ public class JsonUtil { return getBoolean(node, key).orElse(defaultValue); } + /** + * Gets Bytes. + * + * @param json the json + * @param key the key + * @return the Bytes + */ + public static Optional getBytes(final ObjectNode json, final String key) { + return getParsedValue(json, key, Bytes::fromHexString); + } + + /** + * Gets Wei. + * + * @param json the json + * @param key the key + * @param defaultValue the default value + * @return the Wei + */ + public static Bytes getBytes(final ObjectNode json, final String key, final Bytes defaultValue) { + return getBytes(json, key).orElse(defaultValue); + } + + private static Optional getParsedValue( + final ObjectNode json, final String name, final Function parser) { + + return getValue(json, name).map(JsonNode::asText).map(parser); + } + /** * Create empty object node object node. * @@ -308,18 +361,75 @@ public class JsonUtil { * * @param jsonData the json data * @param allowComments true to allow comments + * @param excludeFields names of the fields to not read * @return the object node */ public static ObjectNode objectNodeFromString( - final String jsonData, final boolean allowComments) { - final ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.configure(Feature.ALLOW_COMMENTS, allowComments); + final String jsonData, final boolean allowComments, final String... excludeFields) { try { - final JsonNode jsonNode = objectMapper.readTree(jsonData); + return objectNodeFromParser( + JSON_FACTORY.createParser(jsonData), allowComments, excludeFields); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Object node from string object node. + * + * @param jsonSource the json data + * @param allowComments true to allow comments + * @param excludeFields names of the fields to not read + * @return the object node + */ + public static ObjectNode objectNodeFromURL( + final URL jsonSource, final boolean allowComments, final String... excludeFields) { + try { + return objectNodeFromParser( + JSON_FACTORY.createParser(jsonSource).enable(Feature.AUTO_CLOSE_SOURCE), + allowComments, + excludeFields); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Get a JsonParser to parse JSON from URL. + * + * @param jsonSource the json source + * @param allowComments true to allow comments + * @return the json parser + */ + public static JsonParser jsonParserFromURL(final URL jsonSource, final boolean allowComments) { + try { + return JSON_FACTORY + .createParser(jsonSource) + .enable(Feature.AUTO_CLOSE_SOURCE) + .configure(Feature.ALLOW_COMMENTS, allowComments); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static ObjectNode objectNodeFromParser( + final JsonParser baseParser, final boolean allowComments, final String... excludeFields) { + try { + final var parser = + excludeFields.length > 0 + ? new FilteringParserDelegate( + baseParser, + new NameExcludeFilter(excludeFields), + TokenFilter.Inclusion.INCLUDE_ALL_AND_PATH, + true) + : baseParser; + parser.configure(Feature.ALLOW_COMMENTS, allowComments); + + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonNode = objectMapper.readTree(parser); validateType(jsonNode, JsonNodeType.OBJECT); return (ObjectNode) jsonNode; } catch (final IOException e) { - // Reading directly from a string should not raise an IOException, just catch and rethrow throw new RuntimeException(e); } } @@ -490,4 +600,30 @@ public class JsonUtil { } return true; } + + private static class NameExcludeFilter extends TokenFilter { + private final Set names; + + public NameExcludeFilter(final String... names) { + this.names = Set.of(names); + } + + @Override + public TokenFilter includeProperty(final String name) { + if (names.contains(name)) { + return null; + } + return this; + } + + @Override + public boolean includeEmptyObject(final boolean contentsFiltered) { + return !contentsFiltered; + } + + @Override + public boolean includeEmptyArray(final boolean contentsFiltered) { + return !contentsFiltered; + } + } } diff --git a/config/src/test/java/org/hyperledger/besu/config/GenesisConfigFileTest.java b/config/src/test/java/org/hyperledger/besu/config/GenesisConfigFileTest.java index 2605f0d5e8..9f014b7d4d 100644 --- a/config/src/test/java/org/hyperledger/besu/config/GenesisConfigFileTest.java +++ b/config/src/test/java/org/hyperledger/besu/config/GenesisConfigFileTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hyperledger.besu.config.GenesisConfigFile.fromConfig; +import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import java.io.IOException; @@ -50,7 +51,11 @@ class GenesisConfigFileTest { // Sanity check some basic properties to confirm this is the mainnet file. assertThat(config.getConfigOptions().isEthHash()).isTrue(); assertThat(config.getConfigOptions().getChainId()).hasValue(MAINNET_CHAIN_ID); - assertThat(config.streamAllocations().map(GenesisAllocation::getAddress)) + assertThat( + config + .streamAllocations() + .map(GenesisAccount::address) + .map(Address::toUnprefixedHexString)) .contains( "000d836201318ec6899a67540690382780743280", "001762430ea9c3a26e5749afdb70da5f78ddbb8c", @@ -63,7 +68,11 @@ class GenesisConfigFileTest { // Sanity check some basic properties to confirm this is the dev file. assertThat(config.getConfigOptions().isEthHash()).isTrue(); assertThat(config.getConfigOptions().getChainId()).hasValue(DEVELOPMENT_CHAIN_ID); - assertThat(config.streamAllocations().map(GenesisAllocation::getAddress)) + assertThat( + config + .streamAllocations() + .map(GenesisAccount::address) + .map(Address::toUnprefixedHexString)) .contains( "fe3b557e8fb62b89f4916b721be55ceb828dbd73", "627306090abab3a6e1400e9345bc60c78a8bef57", @@ -271,31 +280,41 @@ class GenesisConfigFileTest { + " }" + "}"); - final Map allocations = + final Map allocations = config .streamAllocations() - .collect(Collectors.toMap(GenesisAllocation::getAddress, Function.identity())); - assertThat(allocations) - .containsOnlyKeys( + .collect(Collectors.toMap(GenesisAccount::address, Function.identity())); + assertThat(allocations.keySet()) + .map(Address::toUnprefixedHexString) + .containsOnly( "fe3b557e8fb62b89f4916b721be55ceb828dbd73", "627306090abab3a6e1400e9345bc60c78a8bef57", "f17f52151ebef6c7334fad080c5704d77216b732"); - final GenesisAllocation alloc1 = allocations.get("fe3b557e8fb62b89f4916b721be55ceb828dbd73"); - final GenesisAllocation alloc2 = allocations.get("627306090abab3a6e1400e9345bc60c78a8bef57"); - final GenesisAllocation alloc3 = allocations.get("f17f52151ebef6c7334fad080c5704d77216b732"); - - assertThat(alloc1.getBalance()).isEqualTo("0xad78ebc5ac6200000"); - assertThat(alloc2.getBalance()).isEqualTo("1000"); - assertThat(alloc3.getBalance()).isEqualTo("90000000000000000000000"); - assertThat(alloc3.getStorage()).hasSize(2); - assertThat(alloc3.getStorage()) + final GenesisAccount alloc1 = + allocations.get(Address.fromHexString("fe3b557e8fb62b89f4916b721be55ceb828dbd73")); + final GenesisAccount alloc2 = + allocations.get(Address.fromHexString("627306090abab3a6e1400e9345bc60c78a8bef57")); + final GenesisAccount alloc3 = + allocations.get(Address.fromHexString("f17f52151ebef6c7334fad080c5704d77216b732")); + + assertThat(alloc1.balance()) + .isEqualTo(GenesisReader.ParserUtils.parseBalance("0xad78ebc5ac6200000")); + assertThat(alloc2.balance()).isEqualTo(GenesisReader.ParserUtils.parseBalance("1000")); + assertThat(alloc3.balance()) + .isEqualTo(GenesisReader.ParserUtils.parseBalance("90000000000000000000000")); + assertThat(alloc3.storage()).hasSize(2); + assertThat(alloc3.storage()) .containsEntry( - "0xc4c3a3f99b26e5e534b71d6f33ca6ea5c174decfb16dd7237c60eff9774ef4a4", - "0x937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0"); - assertThat(alloc3.getStorage()) + UInt256.fromHexString( + "0xc4c3a3f99b26e5e534b71d6f33ca6ea5c174decfb16dd7237c60eff9774ef4a4"), + UInt256.fromHexString( + "0x937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0")); + assertThat(alloc3.storage()) .containsEntry( - "0xc4c3a3f99b26e5e534b71d6f33ca6ea5c174decfb16dd7237c60eff9774ef4a3", - "0x6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012"); + UInt256.fromHexString( + "0xc4c3a3f99b26e5e534b71d6f33ca6ea5c174decfb16dd7237c60eff9774ef4a3"), + UInt256.fromHexString( + "0x6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012")); } @Test diff --git a/config/src/test/java/org/hyperledger/besu/config/GenesisReaderTest.java b/config/src/test/java/org/hyperledger/besu/config/GenesisReaderTest.java new file mode 100644 index 0000000000..5d73a2829f --- /dev/null +++ b/config/src/test/java/org/hyperledger/besu/config/GenesisReaderTest.java @@ -0,0 +1,98 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.config.GenesisReader.ALLOCATION_FIELD; +import static org.hyperledger.besu.config.GenesisReader.CONFIG_FIELD; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class GenesisReaderTest { + private final ObjectMapper mapper = new ObjectMapper(); + + @Test + public void readGenesisFromObjectNode() { + final var configNode = mapper.createObjectNode(); + configNode.put("londonBlock", 1); + final var allocNode = mapper.createObjectNode(); + allocNode.put(Address.BLS12_G2MUL.toUnprefixedHexString(), generateAllocation(Wei.ONE)); + final var rootNode = mapper.createObjectNode(); + rootNode.put("chainId", 12); + rootNode.put(CONFIG_FIELD, configNode); + rootNode.put(ALLOCATION_FIELD, allocNode); + final var genesisReader = new GenesisReader.FromObjectNode(rootNode); + + assertThat(genesisReader.getRoot().get("chainid").asInt()).isEqualTo(12); + assertThat(genesisReader.getRoot().has(ALLOCATION_FIELD)).isFalse(); + assertThat(genesisReader.getConfig().get("londonblock").asInt()).isEqualTo(1); + assertThat(genesisReader.streamAllocations()) + .containsExactly(new GenesisAccount(Address.BLS12_G2MUL, 0, Wei.ONE, null, Map.of(), null)); + } + + @Test + public void readGenesisFromURL(@TempDir final Path folder) throws IOException { + final String jsonStr = + """ + { + "chainId":11, + "config": { + "londonBlock":1 + }, + "alloc": { + "000d836201318ec6899a67540690382780743280": { + "balance": "0xad78ebc5ac6200000" + } + }, + "gasLimit": "0x1" + } + """; + + final var genesisFile = Files.writeString(folder.resolve("genesis.json"), jsonStr); + + final var genesisReader = new GenesisReader.FromURL(genesisFile.toUri().toURL()); + + assertThat(genesisReader.getRoot().get("chainid").asInt()).isEqualTo(11); + assertThat(genesisReader.getRoot().get("gaslimit").asText()).isEqualTo("0x1"); + assertThat(genesisReader.getRoot().has(ALLOCATION_FIELD)).isFalse(); + assertThat(genesisReader.getConfig().get("londonblock").asInt()).isEqualTo(1); + assertThat(genesisReader.streamAllocations()) + .containsExactly( + new GenesisAccount( + Address.fromHexString("000d836201318ec6899a67540690382780743280"), + 0, + Wei.fromHexString("0xad78ebc5ac6200000"), + null, + Map.of(), + null)); + } + + private ObjectNode generateAllocation(final Wei balance) { + final ObjectNode entry = mapper.createObjectNode(); + entry.put("balance", balance.toShortHexString()); + return entry; + } +} diff --git a/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java b/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java index 1b251d5cd1..1e1df382e7 100644 --- a/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java +++ b/config/src/test/java/org/hyperledger/besu/config/JsonUtilTest.java @@ -659,7 +659,24 @@ public class JsonUtilTest { } @Test - public void objectNodeFromURL(@TempDir final Path folder) throws IOException { + public void objectNodeFromString_excludingField() { + final String jsonStr = + """ + { + "a":1, + "b":2, + "c":3 + } + """; + + final ObjectNode result = JsonUtil.objectNodeFromString(jsonStr, false, "b"); + assertThat(result.get("a").asInt()).isEqualTo(1); + assertThat(result.has("b")).isFalse(); + assertThat(result.get("c").asInt()).isEqualTo(3); + } + + @Test + public void objectNodeFromURL_excludingField(@TempDir final Path folder) throws IOException { final String jsonStr = """ { @@ -670,9 +687,9 @@ public class JsonUtilTest { """; final var genesisFile = Files.writeString(folder.resolve("genesis.json"), jsonStr); - final ObjectNode result = JsonUtil.objectNodeFromURL(genesisFile.toUri().toURL(), false); + final ObjectNode result = JsonUtil.objectNodeFromURL(genesisFile.toUri().toURL(), false, "b"); assertThat(result.get("a").asInt()).isEqualTo(1); - assertThat(result.get("b").asInt()).isEqualTo(2); + assertThat(result.has("b")).isFalse(); assertThat(result.get("c").asInt()).isEqualTo(3); } diff --git a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeGenesisConfigHelper.java b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeGenesisConfigHelper.java index 4f510ec7e3..54099f868c 100644 --- a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeGenesisConfigHelper.java +++ b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeGenesisConfigHelper.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.consensus.merge.blockcreation; -import org.hyperledger.besu.config.GenesisAllocation; +import org.hyperledger.besu.config.GenesisAccount; import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.consensus.merge.MergeProtocolSchedule; import org.hyperledger.besu.datatypes.Address; @@ -48,10 +48,7 @@ public interface MergeGenesisConfigHelper { } default Stream
genesisAllocations(final GenesisConfigFile configFile) { - return configFile - .streamAllocations() - .map(GenesisAllocation::getAddress) - .map(Address::fromHexString); + return configFile.streamAllocations().map(GenesisAccount::address); } default ProtocolSchedule getMergeProtocolSchedule() { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java index ae9acc89a6..d42f2d39a3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java @@ -17,12 +17,11 @@ package org.hyperledger.besu.ethereum.chain; import static java.util.Collections.emptyList; import static org.hyperledger.besu.ethereum.trie.common.GenesisWorldStateProvider.createGenesisWorldState; -import org.hyperledger.besu.config.GenesisAllocation; +import org.hyperledger.besu.config.GenesisAccount; import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.BlobGas; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; @@ -38,29 +37,27 @@ import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.log.LogsBloomFilter; import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import java.math.BigInteger; -import java.util.HashMap; +import java.net.URL; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Optional; import java.util.OptionalLong; import java.util.function.Function; import java.util.stream.Stream; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; public final class GenesisState { private final Block block; - private final List genesisAccounts; + private final GenesisConfigFile genesisConfigFile; - private GenesisState(final Block block, final List genesisAccounts) { + private GenesisState(final Block block, final GenesisConfigFile genesisConfigFile) { this.block = block; - this.genesisAccounts = genesisAccounts; + this.genesisConfigFile = genesisConfigFile; } /** @@ -75,24 +72,25 @@ public final class GenesisState { } /** - * Construct a {@link GenesisState} from a JSON string. + * Construct a {@link GenesisState} from a URL * * @param dataStorageConfiguration A {@link DataStorageConfiguration} describing the storage * configuration - * @param json A JSON string describing the genesis block + * @param jsonSource A URL pointing to JSON genesis file * @param protocolSchedule A protocol Schedule associated with * @return A new {@link GenesisState}. */ - public static GenesisState fromJson( + @VisibleForTesting + static GenesisState fromJsonSource( final DataStorageConfiguration dataStorageConfiguration, - final String json, + final URL jsonSource, final ProtocolSchedule protocolSchedule) { return fromConfig( - dataStorageConfiguration, GenesisConfigFile.fromConfig(json), protocolSchedule); + dataStorageConfiguration, GenesisConfigFile.fromConfig(jsonSource), protocolSchedule); } /** - * Construct a {@link GenesisState} from a JSON object. + * Construct a {@link GenesisState} from a genesis file object. * * @param config A {@link GenesisConfigFile} describing the genesis block. * @param protocolSchedule A protocol Schedule associated with @@ -108,41 +106,40 @@ public final class GenesisState { * * @param dataStorageConfiguration A {@link DataStorageConfiguration} describing the storage * configuration - * @param config A {@link GenesisConfigFile} describing the genesis block. + * @param genesisConfigFile A {@link GenesisConfigFile} describing the genesis block. * @param protocolSchedule A protocol Schedule associated with * @return A new {@link GenesisState}. */ public static GenesisState fromConfig( final DataStorageConfiguration dataStorageConfiguration, - final GenesisConfigFile config, + final GenesisConfigFile genesisConfigFile, final ProtocolSchedule protocolSchedule) { - final List genesisAccounts = parseAllocations(config).toList(); + final var genesisStateRoot = + calculateGenesisStateRoot(dataStorageConfiguration, genesisConfigFile); final Block block = new Block( - buildHeader( - config, - calculateGenesisStateHash(dataStorageConfiguration, genesisAccounts), - protocolSchedule), - buildBody(config)); - return new GenesisState(block, genesisAccounts); + buildHeader(genesisConfigFile, genesisStateRoot, protocolSchedule), + buildBody(genesisConfigFile)); + return new GenesisState(block, genesisConfigFile); } /** * Construct a {@link GenesisState} from a JSON object. * - * @param genesisStateHash The hash of the genesis state. - * @param config A {@link GenesisConfigFile} describing the genesis block. + * @param genesisStateRoot The root of the genesis state. + * @param genesisConfigFile A {@link GenesisConfigFile} describing the genesis block. * @param protocolSchedule A protocol Schedule associated with * @return A new {@link GenesisState}. */ - public static GenesisState fromConfig( - final Hash genesisStateHash, - final GenesisConfigFile config, + public static GenesisState fromStorage( + final Hash genesisStateRoot, + final GenesisConfigFile genesisConfigFile, final ProtocolSchedule protocolSchedule) { - final List genesisAccounts = parseAllocations(config).toList(); final Block block = - new Block(buildHeader(config, genesisStateHash, protocolSchedule), buildBody(config)); - return new GenesisState(block, genesisAccounts); + new Block( + buildHeader(genesisConfigFile, genesisStateRoot, protocolSchedule), + buildBody(genesisConfigFile)); + return new GenesisState(block, genesisConfigFile); } private static BlockBody buildBody(final GenesisConfigFile config) { @@ -164,31 +161,31 @@ public final class GenesisState { * @param target WorldView to write genesis state to */ public void writeStateTo(final MutableWorldState target) { - writeAccountsTo(target, genesisAccounts, block.getHeader()); + writeAccountsTo(target, genesisConfigFile.streamAllocations(), block.getHeader()); } private static void writeAccountsTo( final MutableWorldState target, - final List genesisAccounts, + final Stream genesisAccounts, final BlockHeader rootHeader) { final WorldUpdater updater = target.updater(); genesisAccounts.forEach( genesisAccount -> { - final MutableAccount account = updater.getOrCreate(genesisAccount.address); - account.setNonce(genesisAccount.nonce); - account.setBalance(genesisAccount.balance); - account.setCode(genesisAccount.code); - genesisAccount.storage.forEach(account::setStorageValue); + final MutableAccount account = updater.createAccount(genesisAccount.address()); + account.setNonce(genesisAccount.nonce()); + account.setBalance(genesisAccount.balance()); + account.setCode(genesisAccount.code()); + genesisAccount.storage().forEach(account::setStorageValue); }); updater.commit(); target.persist(rootHeader); } - private static Hash calculateGenesisStateHash( + private static Hash calculateGenesisStateRoot( final DataStorageConfiguration dataStorageConfiguration, - final List genesisAccounts) { + final GenesisConfigFile genesisConfigFile) { try (var worldState = createGenesisWorldState(dataStorageConfiguration)) { - writeAccountsTo(worldState, genesisAccounts, null); + writeAccountsTo(worldState, genesisConfigFile.streamAllocations(), null); return worldState.rootHash(); } catch (Exception e) { throw new RuntimeException(e); @@ -265,10 +262,6 @@ public final class GenesisState { return withNiceErrorMessage("mixHash", genesis.getMixHash(), Hash::fromHexStringLenient); } - private static Stream parseAllocations(final GenesisConfigFile genesis) { - return genesis.streamAllocations().map(GenesisAccount::fromAllocation); - } - private static long parseNonce(final GenesisConfigFile genesis) { return withNiceErrorMessage("nonce", genesis.getNonce(), GenesisState::parseUnsignedLong); } @@ -340,75 +333,6 @@ public final class GenesisState { @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("block", block) - .add("genesisAccounts", genesisAccounts) - .toString(); - } - - private static final class GenesisAccount { - - final long nonce; - final Address address; - final Wei balance; - final Map storage; - final Bytes code; - - static GenesisAccount fromAllocation(final GenesisAllocation allocation) { - return new GenesisAccount( - allocation.getNonce(), - allocation.getAddress(), - allocation.getBalance(), - allocation.getStorage(), - allocation.getCode()); - } - - private GenesisAccount( - final String hexNonce, - final String hexAddress, - final String balance, - final Map storage, - final String hexCode) { - this.nonce = withNiceErrorMessage("nonce", hexNonce, GenesisState::parseUnsignedLong); - this.address = withNiceErrorMessage("address", hexAddress, Address::fromHexString); - this.balance = withNiceErrorMessage("balance", balance, this::parseBalance); - this.code = hexCode != null ? Bytes.fromHexString(hexCode) : null; - this.storage = parseStorage(storage); - } - - private Wei parseBalance(final String balance) { - final BigInteger val; - if (balance.startsWith("0x")) { - val = new BigInteger(1, Bytes.fromHexStringLenient(balance).toArrayUnsafe()); - } else { - val = new BigInteger(balance); - } - - return Wei.of(val); - } - - private Map parseStorage(final Map storage) { - final Map parsedStorage = new HashMap<>(); - storage.forEach( - (key1, value1) -> { - final UInt256 key = withNiceErrorMessage("storage key", key1, UInt256::fromHexString); - final UInt256 value = - withNiceErrorMessage("storage value", value1, UInt256::fromHexString); - parsedStorage.put(key, value); - }); - - return parsedStorage; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("address", address) - .add("nonce", nonce) - .add("balance", balance) - .add("storage", storage) - .add("code", code) - .toString(); - } + return MoreObjects.toStringHelper(this).add("block", block).toString(); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java index 6e6217e08b..ea1eb04414 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java @@ -29,8 +29,6 @@ import org.hyperledger.besu.evm.account.Account; import java.util.stream.Stream; -import com.google.common.base.Charsets; -import com.google.common.io.Resources; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; import org.bouncycastle.util.encoders.Hex; @@ -64,12 +62,11 @@ final class GenesisStateTest { @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - public void createFromJsonWithAllocs(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + public void createFromJsonWithAllocs(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString(GenesisStateTest.class.getResource("genesis1.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis1.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getStateRoot()) @@ -95,12 +92,11 @@ final class GenesisStateTest { @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void createFromJsonNoAllocs(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + void createFromJsonNoAllocs(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString(GenesisStateTest.class.getResource("genesis2.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis2.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getStateRoot()).isEqualTo(Hash.EMPTY_TRIE_HASH); @@ -114,12 +110,11 @@ final class GenesisStateTest { private void assertContractInvariants( final DataStorageConfiguration dataStorageConfiguration, final String sourceFile, - final String blockHash) - throws Exception { + final String blockHash) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString(GenesisStateTest.class.getResource(sourceFile), Charsets.UTF_8), + GenesisStateTest.class.getResource(sourceFile), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getHash()).isEqualTo(Hash.fromHexString(blockHash)); @@ -141,8 +136,7 @@ final class GenesisStateTest { @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void createFromJsonWithContract(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + void createFromJsonWithContract(final DataStorageConfiguration dataStorageConfiguration) { assertContractInvariants( dataStorageConfiguration, "genesis3.json", @@ -151,13 +145,11 @@ final class GenesisStateTest { @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void createFromJsonWithNonce(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + void createFromJsonWithNonce(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString( - GenesisStateTest.class.getResource("genesisNonce.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesisNonce.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getHash()) @@ -168,13 +160,11 @@ final class GenesisStateTest { @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void encodeOlympicBlock(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + void encodeOlympicBlock(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString( - GenesisStateTest.class.getResource("genesis-olympic.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis-olympic.json"), ProtocolScheduleFixture.MAINNET); final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); genesisState.getBlock().writeTo(tmp); @@ -190,13 +180,11 @@ final class GenesisStateTest { @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void genesisFromShanghai(final DataStorageConfiguration dataStorageConfiguration) - throws Exception { + void genesisFromShanghai(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString( - GenesisStateTest.class.getResource("genesis_shanghai.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis_shanghai.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getHash()) @@ -241,12 +229,11 @@ final class GenesisStateTest { @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void genesisFromCancun(final DataStorageConfiguration dataStorageConfiguration) throws Exception { + void genesisFromCancun(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString( - GenesisStateTest.class.getResource("genesis_cancun.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis_cancun.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getHash()) @@ -292,12 +279,11 @@ final class GenesisStateTest { @ParameterizedTest @ArgumentsSource(GenesisStateTestArguments.class) - void genesisFromPrague(final DataStorageConfiguration dataStorageConfiguration) throws Exception { + void genesisFromPrague(final DataStorageConfiguration dataStorageConfiguration) { final GenesisState genesisState = - GenesisState.fromJson( + GenesisState.fromJsonSource( dataStorageConfiguration, - Resources.toString( - GenesisStateTest.class.getResource("genesis_prague.json"), Charsets.UTF_8), + GenesisStateTest.class.getResource("genesis_prague.json"), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); assertThat(header.getHash()) diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java index d9b9d5d195..62adf26ecc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java @@ -20,7 +20,7 @@ import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.hyperledger.besu.config.GenesisAllocation; +import org.hyperledger.besu.config.GenesisAccount; import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.crypto.KeyPair; import org.hyperledger.besu.crypto.SECPPrivateKey; @@ -85,7 +85,6 @@ import java.util.List; import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -98,10 +97,10 @@ public abstract class AbstractIsolationTests { protected ProtocolContext protocolContext; protected EthContext ethContext; protected EthScheduler ethScheduler = new DeterministicEthScheduler(); - final Function asKeyPair = + final Function asKeyPair = key -> SignatureAlgorithmFactory.getInstance() - .createKeyPair(SECPPrivateKey.create(Bytes32.fromHexString(key), "ECDSA")); + .createKeyPair(SECPPrivateKey.create(key, "ECDSA")); protected final ProtocolSchedule protocolSchedule = MainnetProtocolSchedule.fromConfig( GenesisConfigFile.fromResource("/dev.json").getConfigOptions(), @@ -139,13 +138,13 @@ public abstract class AbstractIsolationTests { new BlobCache(), MiningParameters.newDefault())); - protected final List accounts = + protected final List accounts = GenesisConfigFile.fromResource("/dev.json") .streamAllocations() - .filter(ga -> ga.getPrivateKey().isPresent()) - .collect(Collectors.toList()); + .filter(ga -> ga.privateKey() != null) + .toList(); - KeyPair sender1 = asKeyPair.apply(accounts.get(0).getPrivateKey().get()); + KeyPair sender1 = Optional.ofNullable(accounts.get(0).privateKey()).map(asKeyPair).orElseThrow(); TransactionPool transactionPool; @TempDir private Path tempData; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java index ffdc3fa897..322aacb7eb 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java @@ -39,7 +39,7 @@ public class BonsaiSnapshotIsolationTests extends AbstractIsolationTests { var postTruncatedWorldState = archive.getMutable(genesisState.getBlock().getHeader(), false); assertThat(postTruncatedWorldState).isEmpty(); // assert that trying to access pre-worldstate does not segfault after truncating - preTruncatedWorldState.get().get(Address.fromHexString(accounts.get(0).getAddress())); + preTruncatedWorldState.get().get(accounts.get(0).address()); assertThat(true).isTrue(); }