mirror of https://github.com/hyperledger/besu
Improve genesis state performance at startup (#6977)
* Refactor genesis options file management Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Make loading allocation from genesis lazy Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Update tests Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Memory optimization with streaming Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Improve loading and storgin genesis state at startup Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Remove comments Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Avoid parsing genesis file allocations twice Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Update javadoc Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Fix Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Fix Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Ignore unknown objects in allocations Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * avoid keeping genesis allocation data in memory Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> * Update CHANGELOG Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> --------- Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> Signed-off-by: ahamlat <ameziane.hamlat@consensys.net> Co-authored-by: ahamlat <ameziane.hamlat@consensys.net>pull/6972/head
Fabio Di Fabio
5 months ago
committed by
GitHub
parent
b1ac5acd60
commit
c62f192459
@ -0,0 +1,42 @@ |
||||
/* |
||||
* Copyright contributors to Hyperledger Besu. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.config; |
||||
|
||||
import org.hyperledger.besu.datatypes.Address; |
||||
import org.hyperledger.besu.datatypes.Wei; |
||||
|
||||
import java.util.Map; |
||||
|
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.apache.tuweni.units.bigints.UInt256; |
||||
|
||||
/** |
||||
* Genesis account |
||||
* |
||||
* @param address of the account |
||||
* @param nonce nonce of the account at genesis |
||||
* @param balance balance of the account at genesis |
||||
* @param code code of the account at genesis, can be null |
||||
* @param storage storage of the account at genesis |
||||
* @param privateKey of the account, only use for testing |
||||
*/ |
||||
public record GenesisAccount( |
||||
Address address, |
||||
long nonce, |
||||
Wei balance, |
||||
Bytes code, |
||||
Map<UInt256, UInt256> storage, |
||||
Bytes32 privateKey) {} |
||||
@ -1,109 +0,0 @@ |
||||
/* |
||||
* Copyright ConsenSys AG. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.config; |
||||
|
||||
import java.util.HashMap; |
||||
import java.util.Map; |
||||
import java.util.Optional; |
||||
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode; |
||||
|
||||
/** The Genesis allocation configuration. */ |
||||
public class GenesisAllocation { |
||||
private final String address; |
||||
private final ObjectNode data; |
||||
|
||||
/** |
||||
* Instantiates a new Genesis allocation. |
||||
* |
||||
* @param address the address |
||||
* @param data the data |
||||
*/ |
||||
GenesisAllocation(final String address, final ObjectNode data) { |
||||
this.address = address; |
||||
this.data = data; |
||||
} |
||||
|
||||
/** |
||||
* Gets address. |
||||
* |
||||
* @return the address |
||||
*/ |
||||
public String getAddress() { |
||||
return address; |
||||
} |
||||
|
||||
/** |
||||
* Gets private key. |
||||
* |
||||
* @return the private key |
||||
*/ |
||||
public Optional<String> getPrivateKey() { |
||||
return Optional.ofNullable(JsonUtil.getString(data, "privatekey", null)); |
||||
} |
||||
|
||||
/** |
||||
* Gets balance. |
||||
* |
||||
* @return the balance |
||||
*/ |
||||
public String getBalance() { |
||||
return JsonUtil.getValueAsString(data, "balance", "0"); |
||||
} |
||||
|
||||
/** |
||||
* Gets code. |
||||
* |
||||
* @return the code |
||||
*/ |
||||
public String getCode() { |
||||
return JsonUtil.getString(data, "code", null); |
||||
} |
||||
|
||||
/** |
||||
* Gets nonce. |
||||
* |
||||
* @return the nonce |
||||
*/ |
||||
public String getNonce() { |
||||
return JsonUtil.getValueAsString(data, "nonce", "0"); |
||||
} |
||||
|
||||
/** |
||||
* Gets version. |
||||
* |
||||
* @return the version |
||||
*/ |
||||
public String getVersion() { |
||||
return JsonUtil.getValueAsString(data, "version", null); |
||||
} |
||||
|
||||
/** |
||||
* Gets storage map. |
||||
* |
||||
* @return fields under storage as a map |
||||
*/ |
||||
public Map<String, String> getStorage() { |
||||
final Map<String, String> map = new HashMap<>(); |
||||
JsonUtil.getObjectNode(data, "storage") |
||||
.orElse(JsonUtil.createEmptyObjectNode()) |
||||
.fields() |
||||
.forEachRemaining( |
||||
(entry) -> { |
||||
map.put(entry.getKey(), entry.getValue().asText()); |
||||
}); |
||||
return map; |
||||
} |
||||
} |
||||
@ -0,0 +1,242 @@ |
||||
/* |
||||
* Copyright contributors to Hyperledger Besu. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.config; |
||||
|
||||
import static org.hyperledger.besu.config.JsonUtil.normalizeKey; |
||||
import static org.hyperledger.besu.config.JsonUtil.normalizeKeys; |
||||
|
||||
import org.hyperledger.besu.datatypes.Address; |
||||
import org.hyperledger.besu.datatypes.Wei; |
||||
|
||||
import java.io.IOException; |
||||
import java.math.BigInteger; |
||||
import java.net.URL; |
||||
import java.util.HashMap; |
||||
import java.util.Iterator; |
||||
import java.util.Locale; |
||||
import java.util.Map; |
||||
import java.util.stream.Collectors; |
||||
import java.util.stream.Stream; |
||||
|
||||
import com.fasterxml.jackson.core.JsonParser; |
||||
import com.fasterxml.jackson.core.JsonToken; |
||||
import com.fasterxml.jackson.databind.node.ObjectNode; |
||||
import com.google.common.collect.Streams; |
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.apache.tuweni.units.bigints.UInt256; |
||||
|
||||
interface GenesisReader { |
||||
String CONFIG_FIELD = "config"; |
||||
String ALLOCATION_FIELD = "alloc"; |
||||
|
||||
ObjectNode getRoot(); |
||||
|
||||
ObjectNode getConfig(); |
||||
|
||||
Stream<GenesisAccount> streamAllocations(); |
||||
|
||||
class FromObjectNode implements GenesisReader { |
||||
private final ObjectNode allocations; |
||||
private final ObjectNode rootWithoutAllocations; |
||||
|
||||
public FromObjectNode(final ObjectNode root) { |
||||
final var removedAllocations = root.remove(ALLOCATION_FIELD); |
||||
this.allocations = |
||||
removedAllocations != null |
||||
? (ObjectNode) removedAllocations |
||||
: JsonUtil.createEmptyObjectNode(); |
||||
this.rootWithoutAllocations = normalizeKeys(root); |
||||
} |
||||
|
||||
@Override |
||||
public ObjectNode getRoot() { |
||||
return rootWithoutAllocations; |
||||
} |
||||
|
||||
@Override |
||||
public ObjectNode getConfig() { |
||||
return JsonUtil.getObjectNode(rootWithoutAllocations, CONFIG_FIELD) |
||||
.orElse(JsonUtil.createEmptyObjectNode()); |
||||
} |
||||
|
||||
@Override |
||||
public Stream<GenesisAccount> streamAllocations() { |
||||
return Streams.stream(allocations.fields()) |
||||
.map( |
||||
entry -> { |
||||
final var on = normalizeKeys((ObjectNode) entry.getValue()); |
||||
return new GenesisAccount( |
||||
Address.fromHexString(entry.getKey()), |
||||
JsonUtil.getString(on, "nonce").map(ParserUtils::parseUnsignedLong).orElse(0L), |
||||
JsonUtil.getString(on, "balance") |
||||
.map(ParserUtils::parseBalance) |
||||
.orElse(Wei.ZERO), |
||||
JsonUtil.getBytes(on, "code", null), |
||||
ParserUtils.getStorageMap(on, "storage"), |
||||
JsonUtil.getBytes(on, "privatekey").map(Bytes32::wrap).orElse(null)); |
||||
}); |
||||
} |
||||
} |
||||
|
||||
class FromURL implements GenesisReader { |
||||
private final URL url; |
||||
private final ObjectNode rootWithoutAllocations; |
||||
|
||||
public FromURL(final URL url) { |
||||
this.url = url; |
||||
this.rootWithoutAllocations = |
||||
normalizeKeys(JsonUtil.objectNodeFromURL(url, false, ALLOCATION_FIELD)); |
||||
} |
||||
|
||||
@Override |
||||
public ObjectNode getRoot() { |
||||
return rootWithoutAllocations; |
||||
} |
||||
|
||||
@Override |
||||
public ObjectNode getConfig() { |
||||
return JsonUtil.getObjectNode(rootWithoutAllocations, CONFIG_FIELD) |
||||
.orElse(JsonUtil.createEmptyObjectNode()); |
||||
} |
||||
|
||||
@Override |
||||
public Stream<GenesisAccount> streamAllocations() { |
||||
final var parser = JsonUtil.jsonParserFromURL(url, false); |
||||
|
||||
try { |
||||
parser.nextToken(); |
||||
while (parser.nextToken() != JsonToken.END_OBJECT) { |
||||
if (ALLOCATION_FIELD.equals(parser.getCurrentName())) { |
||||
parser.nextToken(); |
||||
parser.nextToken(); |
||||
break; |
||||
} else { |
||||
parser.skipChildren(); |
||||
} |
||||
} |
||||
} catch (final IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
|
||||
return Streams.stream(new AllocationIterator(parser)); |
||||
} |
||||
|
||||
private static class AllocationIterator implements Iterator<GenesisAccount> { |
||||
final JsonParser parser; |
||||
|
||||
public AllocationIterator(final JsonParser parser) { |
||||
this.parser = parser; |
||||
} |
||||
|
||||
@Override |
||||
public boolean hasNext() { |
||||
final var end = parser.currentToken() == JsonToken.END_OBJECT; |
||||
if (end) { |
||||
try { |
||||
parser.close(); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
return !end; |
||||
} |
||||
|
||||
@Override |
||||
public GenesisAccount next() { |
||||
try { |
||||
final Address address = Address.fromHexString(parser.currentName()); |
||||
long nonce = 0; |
||||
Wei balance = Wei.ZERO; |
||||
Bytes code = null; |
||||
Map<UInt256, UInt256> storage = Map.of(); |
||||
Bytes32 privateKey = null; |
||||
parser.nextToken(); // consume start object
|
||||
while (parser.nextToken() != JsonToken.END_OBJECT) { |
||||
switch (normalizeKey(parser.currentName())) { |
||||
case "nonce": |
||||
parser.nextToken(); |
||||
nonce = ParserUtils.parseUnsignedLong(parser.getText()); |
||||
break; |
||||
case "balance": |
||||
parser.nextToken(); |
||||
balance = ParserUtils.parseBalance(parser.getText()); |
||||
break; |
||||
case "code": |
||||
parser.nextToken(); |
||||
code = Bytes.fromHexStringLenient(parser.getText()); |
||||
break; |
||||
case "privatekey": |
||||
parser.nextToken(); |
||||
privateKey = Bytes32.fromHexStringLenient(parser.getText()); |
||||
break; |
||||
case "storage": |
||||
parser.nextToken(); |
||||
storage = new HashMap<>(); |
||||
while (parser.nextToken() != JsonToken.END_OBJECT) { |
||||
final var key = UInt256.fromHexString(parser.currentName()); |
||||
parser.nextToken(); |
||||
final var value = UInt256.fromHexString(parser.getText()); |
||||
storage.put(key, value); |
||||
} |
||||
break; |
||||
} |
||||
if (parser.currentToken() == JsonToken.START_OBJECT) { |
||||
// ignore any unknown nested object
|
||||
parser.skipChildren(); |
||||
} |
||||
} |
||||
parser.nextToken(); |
||||
return new GenesisAccount(address, nonce, balance, code, storage, privateKey); |
||||
} catch (IOException e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
class ParserUtils { |
||||
static long parseUnsignedLong(final String value) { |
||||
String v = value.toLowerCase(Locale.US); |
||||
if (v.startsWith("0x")) { |
||||
v = v.substring(2); |
||||
} |
||||
return Long.parseUnsignedLong(v, 16); |
||||
} |
||||
|
||||
static Wei parseBalance(final String balance) { |
||||
final BigInteger val; |
||||
if (balance.startsWith("0x")) { |
||||
val = new BigInteger(1, Bytes.fromHexStringLenient(balance).toArrayUnsafe()); |
||||
} else { |
||||
val = new BigInteger(balance); |
||||
} |
||||
|
||||
return Wei.of(val); |
||||
} |
||||
|
||||
static Map<UInt256, UInt256> getStorageMap(final ObjectNode json, final String key) { |
||||
return JsonUtil.getObjectNode(json, key) |
||||
.map( |
||||
storageMap -> |
||||
Streams.stream(storageMap.fields()) |
||||
.collect( |
||||
Collectors.toMap( |
||||
e -> UInt256.fromHexString(e.getKey()), |
||||
e -> UInt256.fromHexString(e.getValue().asText())))) |
||||
.orElse(Map.of()); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,98 @@ |
||||
/* |
||||
* Copyright contributors to Hyperledger Besu. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.config; |
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat; |
||||
import static org.hyperledger.besu.config.GenesisReader.ALLOCATION_FIELD; |
||||
import static org.hyperledger.besu.config.GenesisReader.CONFIG_FIELD; |
||||
|
||||
import org.hyperledger.besu.datatypes.Address; |
||||
import org.hyperledger.besu.datatypes.Wei; |
||||
|
||||
import java.io.IOException; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Path; |
||||
import java.util.Map; |
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper; |
||||
import com.fasterxml.jackson.databind.node.ObjectNode; |
||||
import org.junit.jupiter.api.Test; |
||||
import org.junit.jupiter.api.io.TempDir; |
||||
|
||||
public class GenesisReaderTest { |
||||
private final ObjectMapper mapper = new ObjectMapper(); |
||||
|
||||
@Test |
||||
public void readGenesisFromObjectNode() { |
||||
final var configNode = mapper.createObjectNode(); |
||||
configNode.put("londonBlock", 1); |
||||
final var allocNode = mapper.createObjectNode(); |
||||
allocNode.put(Address.BLS12_G2MUL.toUnprefixedHexString(), generateAllocation(Wei.ONE)); |
||||
final var rootNode = mapper.createObjectNode(); |
||||
rootNode.put("chainId", 12); |
||||
rootNode.put(CONFIG_FIELD, configNode); |
||||
rootNode.put(ALLOCATION_FIELD, allocNode); |
||||
final var genesisReader = new GenesisReader.FromObjectNode(rootNode); |
||||
|
||||
assertThat(genesisReader.getRoot().get("chainid").asInt()).isEqualTo(12); |
||||
assertThat(genesisReader.getRoot().has(ALLOCATION_FIELD)).isFalse(); |
||||
assertThat(genesisReader.getConfig().get("londonblock").asInt()).isEqualTo(1); |
||||
assertThat(genesisReader.streamAllocations()) |
||||
.containsExactly(new GenesisAccount(Address.BLS12_G2MUL, 0, Wei.ONE, null, Map.of(), null)); |
||||
} |
||||
|
||||
@Test |
||||
public void readGenesisFromURL(@TempDir final Path folder) throws IOException { |
||||
final String jsonStr = |
||||
""" |
||||
{ |
||||
"chainId":11, |
||||
"config": { |
||||
"londonBlock":1 |
||||
}, |
||||
"alloc": { |
||||
"000d836201318ec6899a67540690382780743280": { |
||||
"balance": "0xad78ebc5ac6200000" |
||||
} |
||||
}, |
||||
"gasLimit": "0x1" |
||||
} |
||||
"""; |
||||
|
||||
final var genesisFile = Files.writeString(folder.resolve("genesis.json"), jsonStr); |
||||
|
||||
final var genesisReader = new GenesisReader.FromURL(genesisFile.toUri().toURL()); |
||||
|
||||
assertThat(genesisReader.getRoot().get("chainid").asInt()).isEqualTo(11); |
||||
assertThat(genesisReader.getRoot().get("gaslimit").asText()).isEqualTo("0x1"); |
||||
assertThat(genesisReader.getRoot().has(ALLOCATION_FIELD)).isFalse(); |
||||
assertThat(genesisReader.getConfig().get("londonblock").asInt()).isEqualTo(1); |
||||
assertThat(genesisReader.streamAllocations()) |
||||
.containsExactly( |
||||
new GenesisAccount( |
||||
Address.fromHexString("000d836201318ec6899a67540690382780743280"), |
||||
0, |
||||
Wei.fromHexString("0xad78ebc5ac6200000"), |
||||
null, |
||||
Map.of(), |
||||
null)); |
||||
} |
||||
|
||||
private ObjectNode generateAllocation(final Wei balance) { |
||||
final ObjectNode entry = mapper.createObjectNode(); |
||||
entry.put("balance", balance.toShortHexString()); |
||||
return entry; |
||||
} |
||||
} |
||||
Loading…
Reference in new issue