TrieLog shipping prep (#5317)

* trielog save event, observer, test
* add zero read slots to Accumulator storageToUpdate, omit zero read slots from trielog generation
* add account zero reads, mark self-destructed accounts, code and storage as cleared

Signed-off-by: garyschulte <garyschulte@gmail.com>
pull/5374/head
garyschulte 2 years ago committed by GitHub
parent de4aa7b8f5
commit 9eec16eed4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 28
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java
  2. 18
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java
  3. 35
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogAddedEvent.java
  4. 40
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogLayer.java
  5. 4
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java
  6. 90
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java
  7. 122
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogLayerTests.java
  8. 77
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java

@ -21,6 +21,9 @@ import org.hyperledger.besu.ethereum.rlp.RLPOutput;
import java.util.Objects;
import java.util.function.BiConsumer;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
public class BonsaiValue<T> {
private T prior;
private T updated;
@ -80,6 +83,10 @@ public class BonsaiValue<T> {
return Objects.equals(updated, prior);
}
public void setCleared() {
this.cleared = true;
}
public boolean isCleared() {
return cleared;
}
@ -95,4 +102,25 @@ public class BonsaiValue<T> {
+ cleared
+ '}';
}
@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
BonsaiValue<?> that = (BonsaiValue<?>) o;
return new EqualsBuilder()
.append(cleared, that.cleared)
.append(prior, that.prior)
.append(updated, that.updated)
.isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder(17, 37).append(prior).append(updated).append(cleared).toHashCode();
}
}

@ -19,11 +19,13 @@ import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedBonsaiWorldView;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiUpdater;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent.TrieLogAddedObserver;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.util.Subscribers;
import java.util.Map;
import java.util.Optional;
@ -36,12 +38,12 @@ import org.slf4j.LoggerFactory;
public abstract class AbstractTrieLogManager implements TrieLogManager {
private static final Logger LOG = LoggerFactory.getLogger(AbstractTrieLogManager.class);
public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks
protected final Blockchain blockchain;
protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage;
protected final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash;
protected final long maxLayersToLoad;
private final Subscribers<TrieLogAddedObserver> trieLogAddedObservers = Subscribers.create();
protected AbstractTrieLogManager(
final Blockchain blockchain,
@ -69,6 +71,10 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
try {
final TrieLogLayer trieLog = prepareTrieLog(forBlockHeader, localUpdater);
persistTrieLog(forBlockHeader, forWorldStateRootHash, trieLog, stateUpdater);
// notify trie log added observers, synchronously
trieLogAddedObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog)));
success = true;
} finally {
if (success) {
@ -137,4 +143,14 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
public Optional<TrieLogLayer> getTrieLogLayer(final Hash blockHash) {
return rootWorldStateStorage.getTrieLog(blockHash).map(TrieLogLayer::fromBytes);
}
@Override
public synchronized long subscribe(final TrieLogAddedObserver sub) {
return trieLogAddedObservers.subscribe(sub);
}
@Override
public synchronized void unsubscribe(final long id) {
trieLogAddedObservers.unsubscribe(id);
}
}

@ -0,0 +1,35 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.ethereum.bonsai.trielog;
public class TrieLogAddedEvent {
private final TrieLogLayer layer;
public TrieLogAddedEvent(final TrieLogLayer layer) {
this.layer = layer;
}
public TrieLogLayer getLayer() {
return layer;
}
@FunctionalInterface
interface TrieLogAddedObserver {
void onTrieLogAdded(TrieLogAddedEvent event);
}
}

@ -36,6 +36,8 @@ import java.util.TreeSet;
import java.util.function.Function;
import java.util.stream.Stream;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256;
@ -70,26 +72,29 @@ public class TrieLogLayer {
return blockHash;
}
public void setBlockHash(final Hash blockHash) {
public TrieLogLayer setBlockHash(final Hash blockHash) {
checkState(!frozen, "Layer is Frozen");
this.blockHash = blockHash;
return this;
}
public void addAccountChange(
public TrieLogLayer addAccountChange(
final Address address,
final StateTrieAccountValue oldValue,
final StateTrieAccountValue newValue) {
checkState(!frozen, "Layer is Frozen");
accounts.put(address, new BonsaiValue<>(oldValue, newValue));
return this;
}
public void addCodeChange(
public TrieLogLayer addCodeChange(
final Address address, final Bytes oldValue, final Bytes newValue, final Hash blockHash) {
checkState(!frozen, "Layer is Frozen");
code.put(
address,
new BonsaiValue<>(
oldValue == null ? Bytes.EMPTY : oldValue, newValue == null ? Bytes.EMPTY : newValue));
return this;
}
public void addStorageChange(
@ -317,4 +322,33 @@ public class TrieLogLayer {
}
return sb.toString();
}
@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
TrieLogLayer that = (TrieLogLayer) o;
return new EqualsBuilder()
.append(frozen, that.frozen)
.append(blockHash, that.blockHash)
.append(accounts, that.accounts)
.append(code, that.code)
.append(storage, that.storage)
.isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder(17, 37)
.append(blockHash)
.append(frozen)
.append(accounts)
.append(code)
.append(storage)
.toHashCode();
}
}

@ -48,4 +48,8 @@ public interface TrieLogManager {
void reset();
Optional<TrieLogLayer> getTrieLogLayer(final Hash blockHash);
long subscribe(final TrieLogAddedEvent.TrieLogAddedObserver sub);
void unsubscribe(final long id);
}

@ -62,7 +62,6 @@ public class BonsaiWorldStateUpdateAccumulator
private final AccountConsumingMap<BonsaiValue<BonsaiAccount>> accountsToUpdate;
private final Map<Address, BonsaiValue<Bytes>> codeToUpdate = new ConcurrentHashMap<>();
private final Set<Address> storageToClear = Collections.synchronizedSet(new HashSet<>());
private final Set<Bytes> emptySlot = Collections.synchronizedSet(new HashSet<>());
// storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to
// enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the
@ -98,7 +97,6 @@ public class BonsaiWorldStateUpdateAccumulator
storageToUpdate.putAll(source.storageToUpdate);
updatedAccounts.putAll(source.updatedAccounts);
deletedAccounts.addAll(source.deletedAccounts);
emptySlot.addAll(source.emptySlot);
this.isAccumulatorStateChanged = true;
}
@ -178,12 +176,14 @@ public class BonsaiWorldStateUpdateAccumulator
} else {
account = wrappedWorldView().get(address);
}
BonsaiAccount mutableAccount = null;
if (account instanceof BonsaiAccount) {
final BonsaiAccount mutableAccount =
new BonsaiAccount((BonsaiAccount) account, this, true);
mutableAccount = new BonsaiAccount((BonsaiAccount) account, this, true);
accountsToUpdate.put(address, new BonsaiValue<>((BonsaiAccount) account, mutableAccount));
return mutableAccount;
} else {
// add the empty read in accountsToUpdate
accountsToUpdate.put(address, new BonsaiValue<>(null, null));
return null;
}
} else {
@ -218,11 +218,11 @@ public class BonsaiWorldStateUpdateAccumulator
final BonsaiValue<BonsaiAccount> accountValue =
accountsToUpdate.computeIfAbsent(
deletedAddress,
__ -> loadAccountFromParent(deletedAddress, new BonsaiValue<>(null, null)));
__ -> loadAccountFromParent(deletedAddress, new BonsaiValue<>(null, null, true)));
storageToClear.add(deletedAddress);
final BonsaiValue<Bytes> codeValue = codeToUpdate.get(deletedAddress);
if (codeValue != null) {
codeValue.setUpdated(null);
codeValue.setUpdated(null).setCleared();
} else {
wrappedWorldView()
.getCode(
@ -233,7 +233,7 @@ public class BonsaiWorldStateUpdateAccumulator
.orElse(Hash.EMPTY))
.ifPresent(
deletedCode ->
codeToUpdate.put(deletedAddress, new BonsaiValue<>(deletedCode, null)));
codeToUpdate.put(deletedAddress, new BonsaiValue<>(deletedCode, null, true)));
}
// mark all updated storage as to be cleared
@ -251,7 +251,7 @@ public class BonsaiWorldStateUpdateAccumulator
if (updatedSlot.getPrior() == null || updatedSlot.getPrior().isZero()) {
iter.remove();
} else {
updatedSlot.setUpdated(null);
updatedSlot.setUpdated(null).setCleared();
}
}
@ -399,39 +399,29 @@ public class BonsaiWorldStateUpdateAccumulator
return Optional.ofNullable(value.getUpdated());
}
}
final Bytes slot = Bytes.concatenate(Hash.hash(address), slotHash);
if (emptySlot.contains(slot)) {
return Optional.empty();
} else {
try {
final Optional<UInt256> valueUInt =
(wrappedWorldView() instanceof BonsaiWorldState)
? ((BonsaiWorldState) wrappedWorldView())
.getStorageValueBySlotHash(
() ->
Optional.ofNullable(loadAccount(address, BonsaiValue::getPrior))
.map(BonsaiAccount::getStorageRoot),
address,
slotHash)
: wrappedWorldView().getStorageValueBySlotHash(address, slotHash);
valueUInt.ifPresentOrElse(
v ->
storageToUpdate
.computeIfAbsent(
address,
key ->
new StorageConsumingMap<>(
address, new ConcurrentHashMap<>(), storagePreloader))
.put(slotHash, new BonsaiValue<>(v, v)),
() -> {
emptySlot.add(Bytes.concatenate(Hash.hash(address), slotHash));
});
return valueUInt;
} catch (MerkleTrieException e) {
// need to throw to trigger the heal
throw new MerkleTrieException(
e.getMessage(), Optional.of(address), e.getHash(), e.getLocation());
}
try {
final Optional<UInt256> valueUInt =
(wrappedWorldView() instanceof BonsaiWorldState)
? ((BonsaiWorldState) wrappedWorldView())
.getStorageValueBySlotHash(
() ->
Optional.ofNullable(loadAccount(address, BonsaiValue::getPrior))
.map(BonsaiAccount::getStorageRoot),
address,
slotHash)
: wrappedWorldView().getStorageValueBySlotHash(address, slotHash);
storageToUpdate
.computeIfAbsent(
address,
key ->
new StorageConsumingMap<>(address, new ConcurrentHashMap<>(), storagePreloader))
.put(slotHash, new BonsaiValue<>(valueUInt.orElse(null), valueUInt.orElse(null)));
return valueUInt;
} catch (MerkleTrieException e) {
// need to throw to trigger the heal
throw new MerkleTrieException(
e.getMessage(), Optional.of(address), e.getHash(), e.getLocation());
}
}
@ -512,6 +502,10 @@ public class BonsaiWorldStateUpdateAccumulator
newValue.getBalance(),
newValue.getStorageRoot(),
newValue.getCodeHash());
if (oldValue == null && newValue == null) {
// by default do not persist empty reads of accounts to the trie log
continue;
}
layer.addAccountChange(updatedAccount.getKey(), oldAccount, newAccount);
}
@ -528,11 +522,14 @@ public class BonsaiWorldStateUpdateAccumulator
final Address address = updatesStorage.getKey();
for (final Map.Entry<Hash, BonsaiValue<UInt256>> slotUpdate :
updatesStorage.getValue().entrySet()) {
layer.addStorageChange(
address,
slotUpdate.getKey(),
slotUpdate.getValue().getPrior(),
slotUpdate.getValue().getUpdated());
var val = slotUpdate.getValue();
if (val.getPrior() == null && val.getUpdated() == null) {
// by default do not persist empty reads to the trie log
continue;
}
layer.addStorageChange(address, slotUpdate.getKey(), val.getPrior(), val.getUpdated());
}
}
}
@ -817,7 +814,6 @@ public class BonsaiWorldStateUpdateAccumulator
storageToUpdate.clear();
codeToUpdate.clear();
accountsToUpdate.clear();
emptySlot.clear();
resetAccumulatorStateChanged();
super.reset();
}

@ -0,0 +1,122 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.ethereum.bonsai.trielog;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import java.util.Optional;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256;
import org.assertj.core.api.Assertions;
import org.junit.Before;
import org.junit.Test;
public class TrieLogLayerTests {
private TrieLogLayer trieLogLayer;
private TrieLogLayer otherTrieLogLayer;
@Before
public void setUp() {
trieLogLayer = new TrieLogLayer();
otherTrieLogLayer = new TrieLogLayer();
}
@Test
public void testAddAccountChange() {
Address address = Address.fromHexString("0x00");
StateTrieAccountValue oldValue = new StateTrieAccountValue(0, Wei.ZERO, Hash.EMPTY, Hash.EMPTY);
StateTrieAccountValue newValue =
new StateTrieAccountValue(1, Wei.fromEth(1), Hash.EMPTY, Hash.EMPTY);
Address otherAddress = Address.fromHexString("0x000000");
StateTrieAccountValue otherOldValue =
new StateTrieAccountValue(0, Wei.ZERO, Hash.EMPTY, Hash.EMPTY);
StateTrieAccountValue otherNewValue =
new StateTrieAccountValue(1, Wei.fromEth(1), Hash.EMPTY, Hash.EMPTY);
trieLogLayer.addAccountChange(address, oldValue, newValue);
otherTrieLogLayer.addAccountChange(otherAddress, otherOldValue, otherNewValue);
Assertions.assertThat(trieLogLayer).isEqualTo(otherTrieLogLayer);
Optional<StateTrieAccountValue> priorAccount = trieLogLayer.getPriorAccount(address);
Assertions.assertThat(priorAccount).isPresent();
Assertions.assertThat(priorAccount.get()).isEqualTo(oldValue);
Optional<StateTrieAccountValue> updatedAccount = trieLogLayer.getAccount(address);
Assertions.assertThat(updatedAccount).isPresent();
Assertions.assertThat(updatedAccount.get()).isEqualTo(newValue);
}
@Test
public void testAddCodeChange() {
Address address = Address.fromHexString("0xdeadbeef");
Bytes oldValue = Bytes.fromHexString("0x00");
Bytes newValue = Bytes.fromHexString("0x01030307");
Hash blockHash = Hash.fromHexStringLenient("0xfeedbeef02dead");
Address otherAddress = Address.fromHexString("0x0000deadbeef");
Bytes otherOldValue = Bytes.fromHexString("0x00");
Bytes otherNewValue = Bytes.fromHexString("0x01030307");
Hash otherBlockHash = Hash.fromHexStringLenient("0x00feedbeef02dead");
trieLogLayer.addCodeChange(address, oldValue, newValue, blockHash);
otherTrieLogLayer.addCodeChange(otherAddress, otherOldValue, otherNewValue, otherBlockHash);
Assertions.assertThat(trieLogLayer).isEqualTo(otherTrieLogLayer);
Optional<Bytes> priorCode = trieLogLayer.getPriorCode(address);
Assertions.assertThat(priorCode).isPresent();
Assertions.assertThat(priorCode.get()).isEqualTo(oldValue);
Optional<Bytes> updatedCode = trieLogLayer.getCode(address);
Assertions.assertThat(updatedCode).isPresent();
Assertions.assertThat(updatedCode.get()).isEqualTo(newValue);
}
@Test
public void testAddStorageChange() {
Address address = Address.fromHexString("0x0");
UInt256 oldValue = UInt256.ZERO;
UInt256 newValue = UInt256.ONE;
UInt256 slot = UInt256.ONE;
Address otherAddress = Address.fromHexString("0x0");
UInt256 otherOldValue = UInt256.ZERO;
UInt256 otherNewValue = UInt256.ONE;
UInt256 otherSlot = UInt256.ONE;
trieLogLayer.addStorageChange(address, Hash.hash(slot), oldValue, newValue);
otherTrieLogLayer.addStorageChange(
otherAddress, Hash.hash(otherSlot), otherOldValue, otherNewValue);
Assertions.assertThat(trieLogLayer).isEqualTo(otherTrieLogLayer);
Optional<UInt256> priorStorageValue =
trieLogLayer.getPriorStorageBySlotHash(address, Hash.hash(slot));
Assertions.assertThat(priorStorageValue).isPresent();
Assertions.assertThat(priorStorageValue.get()).isEqualTo(oldValue);
Optional<UInt256> updatedStorageValue =
trieLogLayer.getStorageBySlotHash(address, Hash.hash(slot));
Assertions.assertThat(updatedStorageValue).isPresent();
Assertions.assertThat(updatedStorageValue.get()).isEqualTo(newValue);
}
}

@ -0,0 +1,77 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.ethereum.bonsai.trielog;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.spy;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class TrieLogManagerTests {
BlockHeader blockHeader = new BlockHeaderTestFixture().buildHeader();
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
BonsaiWorldState bonsaiWorldState;
@Mock BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage;
@Mock BonsaiWorldState worldState;
@Mock BonsaiWorldStateProvider archive;
@Mock Blockchain blockchain;
BonsaiWorldStateUpdateAccumulator bonsaiUpdater =
spy(new BonsaiWorldStateUpdateAccumulator(worldState, (__, ___) -> {}, (__, ___) -> {}));
TrieLogManager trieLogManager;
@Before
public void setup() {
trieLogManager =
new CachedWorldStorageManager(
archive, blockchain, bonsaiWorldStateKeyValueStorage, new NoOpMetricsSystem(), 512);
}
@Test
public void testSaveTrieLogEvent() {
AtomicBoolean eventFired = new AtomicBoolean(false);
trieLogManager.subscribe(
layer -> {
assertThat(layer).isNotNull();
eventFired.set(true);
});
trieLogManager.saveTrieLog(bonsaiUpdater, Hash.ZERO, blockHeader, bonsaiWorldState);
assertThat(eventFired.get()).isTrue();
}
}
Loading…
Cancel
Save