mirror of https://github.com/hyperledger/besu
Bonsai based reference test worldstate (#5686)
* create a bonsai based reference test worldstate -> getOrCreate in BonsaiWorldStateUpdateAccumulator - do not throw if we discover an empty account in a non-null BonsaiValue<Account> -> add curentStateRoot to t8n -> storageEntriesFrom and streamAccounts implemented in BonsaiWorldStateKeyValueStorage -> add endKey version of streamFromKey * bonsai fix for self-destruct and create2 at the same address and same block Signed-off-by: garyschulte <garyschulte@gmail.com> Signed-off-by: Karim TAAM <karim.t2am@gmail.com> Co-authored-by: Karim TAAM <karim.t2am@gmail.com>pull/5882/head
garyschulte
1 year ago
committed by
GitHub
parent
35385611ae
commit
4b2ef689c1
@ -0,0 +1,68 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
* |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.bonsai.storage; |
||||
|
||||
import org.hyperledger.besu.datatypes.Address; |
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; |
||||
|
||||
import java.util.Optional; |
||||
|
||||
import com.google.common.collect.BiMap; |
||||
import com.google.common.collect.HashBiMap; |
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.apache.tuweni.units.bigints.UInt256; |
||||
|
||||
/** Acts as both a Hasher and PreImageStorage for Bonsai storage format. */ |
||||
public interface BonsaiPreImageProxy extends WorldStatePreimageStorage { |
||||
/** |
||||
* If this value is not already present, save in preImage store and return the hash value. |
||||
* |
||||
* @param value value to hash |
||||
* @return Hash of value |
||||
*/ |
||||
Hash hashAndSavePreImage(Bytes value); |
||||
|
||||
/** |
||||
* A caching PreImageProxy suitable for ReferenceTestWorldState which saves hashes in an unbounded |
||||
* BiMap. |
||||
*/ |
||||
class BonsaiReferenceTestPreImageProxy implements BonsaiPreImageProxy { |
||||
BiMap<Hash, Bytes> preImageCache = HashBiMap.create(); |
||||
|
||||
@Override |
||||
public synchronized Hash hashAndSavePreImage(final Bytes value) { |
||||
return preImageCache.inverse().computeIfAbsent(value, Hash::hash); |
||||
} |
||||
|
||||
@Override |
||||
public Optional<UInt256> getStorageTrieKeyPreimage(final Bytes32 trieKey) { |
||||
return Optional.ofNullable(preImageCache.get(trieKey)).map(UInt256::fromBytes); |
||||
} |
||||
|
||||
@Override |
||||
public Optional<Address> getAccountTrieKeyPreimage(final Bytes32 trieKey) { |
||||
return Optional.ofNullable(preImageCache.get(trieKey)).map(Address::wrap); |
||||
} |
||||
|
||||
@Override |
||||
public Updater updater() { |
||||
throw new UnsupportedOperationException( |
||||
"BonsaiReferenceTestPreImageProxy does not implement an updater"); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,44 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.referencetests; |
||||
|
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.datatypes.StorageSlotKey; |
||||
import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; |
||||
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; |
||||
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; |
||||
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; |
||||
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldView; |
||||
|
||||
import org.apache.tuweni.bytes.Bytes; |
||||
|
||||
public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdateAccumulator { |
||||
private final BonsaiPreImageProxy preImageProxy; |
||||
|
||||
public BonsaiReferenceTestUpdateAccumulator( |
||||
final BonsaiWorldView world, |
||||
final Consumer<BonsaiValue<BonsaiAccount>> accountPreloader, |
||||
final Consumer<StorageSlotKey> storagePreloader, |
||||
final BonsaiPreImageProxy preImageProxy) { |
||||
super(world, accountPreloader, storagePreloader); |
||||
this.preImageProxy = preImageProxy; |
||||
} |
||||
|
||||
@Override |
||||
protected Hash hashAndSavePreImage(final Bytes bytes) { |
||||
// by default do not save hash preImages
|
||||
return preImageProxy.hashAndSavePreImage(bytes); |
||||
} |
||||
} |
||||
@ -0,0 +1,182 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.referencetests; |
||||
|
||||
import org.hyperledger.besu.datatypes.Address; |
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; |
||||
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; |
||||
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent; |
||||
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; |
||||
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; |
||||
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; |
||||
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; |
||||
import org.hyperledger.besu.ethereum.core.BlockHeader; |
||||
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; |
||||
import org.hyperledger.besu.evm.worldstate.WorldUpdater; |
||||
import org.hyperledger.besu.metrics.ObservableMetricsSystem; |
||||
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; |
||||
import org.hyperledger.besu.plugin.services.trielogs.TrieLog; |
||||
import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; |
||||
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; |
||||
import org.hyperledger.besu.util.Subscribers; |
||||
|
||||
import java.util.Map; |
||||
import java.util.Optional; |
||||
import java.util.function.Function; |
||||
import java.util.stream.Stream; |
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator; |
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
|
||||
public class BonsaiReferenceTestWorldState extends BonsaiWorldState |
||||
implements ReferenceTestWorldState { |
||||
|
||||
private final BonsaiReferenceTestWorldStateStorage refTestStorage; |
||||
private final BonsaiPreImageProxy preImageProxy; |
||||
|
||||
protected BonsaiReferenceTestWorldState( |
||||
final BonsaiReferenceTestWorldStateStorage worldStateStorage, |
||||
final CachedMerkleTrieLoader cachedMerkleTrieLoader, |
||||
final TrieLogManager trieLogManager, |
||||
final BonsaiPreImageProxy preImageProxy) { |
||||
super(worldStateStorage, cachedMerkleTrieLoader, trieLogManager); |
||||
this.refTestStorage = worldStateStorage; |
||||
this.preImageProxy = preImageProxy; |
||||
setAccumulator( |
||||
new BonsaiReferenceTestUpdateAccumulator( |
||||
this, |
||||
(addr, value) -> |
||||
cachedMerkleTrieLoader.preLoadAccount( |
||||
getWorldStateStorage(), worldStateRootHash, addr), |
||||
(addr, value) -> |
||||
cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), |
||||
preImageProxy)); |
||||
} |
||||
|
||||
@Override |
||||
public ReferenceTestWorldState copy() { |
||||
var layerCopy = new BonsaiReferenceTestWorldStateStorage(worldStateStorage, preImageProxy); |
||||
return new BonsaiReferenceTestWorldState( |
||||
layerCopy, cachedMerkleTrieLoader, trieLogManager, preImageProxy); |
||||
} |
||||
|
||||
@JsonCreator |
||||
public static BonsaiReferenceTestWorldState create( |
||||
final Map<String, ReferenceTestWorldState.AccountMock> accounts) { |
||||
final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); |
||||
final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem); |
||||
final TrieLogManager trieLogManager = new NoOpTrieLogManager(); |
||||
final BonsaiPreImageProxy preImageProxy = |
||||
new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); |
||||
|
||||
final BonsaiReferenceTestWorldStateStorage worldStateStorage = |
||||
new BonsaiReferenceTestWorldStateStorage( |
||||
new BonsaiWorldStateKeyValueStorage( |
||||
new InMemoryKeyValueStorageProvider(), metricsSystem), |
||||
preImageProxy); |
||||
|
||||
final BonsaiReferenceTestWorldState worldState = |
||||
new BonsaiReferenceTestWorldState( |
||||
worldStateStorage, cachedMerkleTrieLoader, trieLogManager, preImageProxy); |
||||
|
||||
final WorldUpdater updater = worldState.updater(); |
||||
for (final Map.Entry<String, ReferenceTestWorldState.AccountMock> entry : accounts.entrySet()) { |
||||
ReferenceTestWorldState.insertAccount( |
||||
updater, Address.fromHexString(entry.getKey()), entry.getValue()); |
||||
} |
||||
updater.commit(); |
||||
return worldState; |
||||
} |
||||
|
||||
@Override |
||||
public Stream<StreamableAccount> streamAccounts(final Bytes32 startKeyHash, final int limit) { |
||||
return this.refTestStorage.streamAccounts(this, startKeyHash, limit); |
||||
} |
||||
|
||||
static class NoOpTrieLogManager implements TrieLogManager { |
||||
private final Subscribers<TrieLogEvent.TrieLogObserver> trieLogObservers = Subscribers.create(); |
||||
private final TrieLogFactory trieLogFactory = new TrieLogFactoryImpl(); |
||||
|
||||
@Override |
||||
public void saveTrieLog( |
||||
final BonsaiWorldStateUpdateAccumulator localUpdater, |
||||
final Hash forWorldStateRootHash, |
||||
final BlockHeader forBlockHeader, |
||||
final BonsaiWorldState forWorldState) { |
||||
// notify trie log added observers, synchronously
|
||||
TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); |
||||
trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); |
||||
} |
||||
|
||||
@Override |
||||
public void addCachedLayer( |
||||
final BlockHeader blockHeader, |
||||
final Hash worldStateRootHash, |
||||
final BonsaiWorldState forWorldState) {} |
||||
|
||||
@Override |
||||
public boolean containWorldStateStorage(final Hash blockHash) { |
||||
return false; |
||||
} |
||||
|
||||
@Override |
||||
public Optional<BonsaiWorldState> getWorldState(final Hash blockHash) { |
||||
return Optional.empty(); |
||||
} |
||||
|
||||
@Override |
||||
public Optional<BonsaiWorldState> getNearestWorldState(final BlockHeader blockHeader) { |
||||
return Optional.empty(); |
||||
} |
||||
|
||||
@Override |
||||
public Optional<BonsaiWorldState> getHeadWorldState( |
||||
final Function<Hash, Optional<BlockHeader>> hashBlockHeaderFunction) { |
||||
return Optional.empty(); |
||||
} |
||||
|
||||
@Override |
||||
public long getMaxLayersToLoad() { |
||||
return 0; |
||||
} |
||||
|
||||
@Override |
||||
public void reset() {} |
||||
|
||||
@Override |
||||
public Optional<? extends TrieLog> getTrieLogLayer(final Hash blockHash) { |
||||
return Optional.empty(); |
||||
} |
||||
|
||||
@Override |
||||
public synchronized long subscribe(final TrieLogEvent.TrieLogObserver sub) { |
||||
return trieLogObservers.subscribe(sub); |
||||
} |
||||
|
||||
@Override |
||||
public synchronized void unsubscribe(final long id) { |
||||
trieLogObservers.unsubscribe(id); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
protected Hash hashAndSavePreImage(final Bytes value) { |
||||
// by default do not save has preImages
|
||||
return preImageProxy.hashAndSavePreImage(value); |
||||
} |
||||
} |
||||
@ -0,0 +1,85 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.referencetests; |
||||
|
||||
import org.hyperledger.besu.datatypes.Address; |
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; |
||||
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; |
||||
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage; |
||||
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldView; |
||||
import org.hyperledger.besu.evm.account.AccountStorageEntry; |
||||
import org.hyperledger.besu.evm.worldstate.WorldState; |
||||
|
||||
import java.util.Comparator; |
||||
import java.util.NavigableMap; |
||||
import java.util.Optional; |
||||
import java.util.TreeMap; |
||||
import java.util.stream.Collectors; |
||||
import java.util.stream.Stream; |
||||
|
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.apache.tuweni.rlp.RLP; |
||||
import org.apache.tuweni.units.bigints.UInt256; |
||||
|
||||
public class BonsaiReferenceTestWorldStateStorage extends BonsaiWorldStateLayerStorage { |
||||
private final BonsaiPreImageProxy preImageProxy; |
||||
|
||||
public BonsaiReferenceTestWorldStateStorage( |
||||
final BonsaiWorldStateKeyValueStorage parent, final BonsaiPreImageProxy preImageProxy) { |
||||
super(parent); |
||||
this.preImageProxy = preImageProxy; |
||||
} |
||||
|
||||
@Override |
||||
public NavigableMap<Bytes32, AccountStorageEntry> storageEntriesFrom( |
||||
final Hash addressHash, final Bytes32 startKeyHash, final int limit) { |
||||
return streamFlatStorages(addressHash, startKeyHash, UInt256.MAX_VALUE, limit) |
||||
.entrySet() |
||||
// map back to slot keys using preImage provider:
|
||||
.stream() |
||||
.collect( |
||||
Collectors.toMap( |
||||
e -> e.getKey(), |
||||
e -> |
||||
AccountStorageEntry.create( |
||||
UInt256.fromBytes(RLP.decodeValue(e.getValue())), |
||||
Hash.wrap(e.getKey()), |
||||
preImageProxy.getStorageTrieKeyPreimage(e.getKey())), |
||||
(a, b) -> a, |
||||
TreeMap::new)); |
||||
} |
||||
|
||||
public Stream<WorldState.StreamableAccount> streamAccounts( |
||||
final BonsaiWorldView context, final Bytes32 startKeyHash, final int limit) { |
||||
return streamFlatAccounts(startKeyHash, UInt256.MAX_VALUE, limit) |
||||
.entrySet() |
||||
// map back to addresses using preImage provider:
|
||||
.stream() |
||||
.map( |
||||
entry -> |
||||
preImageProxy |
||||
.getAccountTrieKeyPreimage(entry.getKey()) |
||||
.map( |
||||
address -> |
||||
new WorldState.StreamableAccount( |
||||
Optional.of(address), |
||||
BonsaiAccount.fromRLP(context, address, entry.getValue(), false)))) |
||||
.filter(Optional::isPresent) |
||||
.map(Optional::get) |
||||
.sorted(Comparator.comparing(account -> account.getAddress().orElse(Address.ZERO))); |
||||
} |
||||
} |
||||
@ -0,0 +1,60 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.referencetests; |
||||
|
||||
import org.hyperledger.besu.datatypes.Address; |
||||
import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; |
||||
import org.hyperledger.besu.evm.worldstate.WorldState; |
||||
import org.hyperledger.besu.evm.worldstate.WorldUpdater; |
||||
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; |
||||
|
||||
import java.util.Map; |
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator; |
||||
|
||||
public class DefaultReferenceTestWorldState extends DefaultMutableWorldState |
||||
implements ReferenceTestWorldState { |
||||
|
||||
DefaultReferenceTestWorldState() { |
||||
super( |
||||
new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()), |
||||
new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage())); |
||||
} |
||||
|
||||
public DefaultReferenceTestWorldState(final WorldState worldState) { |
||||
super(worldState); |
||||
} |
||||
|
||||
@Override |
||||
public ReferenceTestWorldState copy() { |
||||
return new DefaultReferenceTestWorldState(this); |
||||
} |
||||
|
||||
@JsonCreator |
||||
public static ReferenceTestWorldState create(final Map<String, AccountMock> accounts) { |
||||
final ReferenceTestWorldState worldState = new DefaultReferenceTestWorldState(); |
||||
final WorldUpdater updater = worldState.updater(); |
||||
|
||||
for (final Map.Entry<String, AccountMock> entry : accounts.entrySet()) { |
||||
ReferenceTestWorldState.insertAccount( |
||||
updater, Address.fromHexString(entry.getKey()), entry.getValue()); |
||||
} |
||||
|
||||
updater.commit(); |
||||
return worldState; |
||||
} |
||||
} |
||||
Loading…
Reference in new issue