From 37ea0282de17f393531a6ee7be2dbfe92d0910ac Mon Sep 17 00:00:00 2001 From: Dmitry <98899785+mdqst@users.noreply.github.com> Date: Tue, 19 Nov 2024 09:33:06 +0300 Subject: [PATCH 1/7] Typo Update Private-Txns-Migration.md (#7870) The phrase "private transaction created using v1.3.4" should be in the plural form: "private transactions created using v1.3.4". Corrected. Signed-off-by: Dmitry <98899785+mdqst@users.noreply.github.com> Co-authored-by: Sally MacFarlane --- docs/Private-Txns-Migration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Private-Txns-Migration.md b/docs/Private-Txns-Migration.md index 0ae06dae3c..042e83c8dd 100644 --- a/docs/Private-Txns-Migration.md +++ b/docs/Private-Txns-Migration.md @@ -12,7 +12,7 @@ state storage when upgrading to v1.4. It is not possible to upgrade to v1.4 with ## Private transactions created using v1.3.4 or earlier A critical issue for privacy users with private transactions created using Hyperledger Besu v1.3.4 -or earlier has been identified. If you have a network with private transaction created using v1.3.4 +or earlier has been identified. If you have a network with private transactions created using v1.3.4 or earlier, please read the following and take the appropriate steps: https://wiki.hyperledger.org/display/BESU/Critical+Issue+for+Privacy+Users @@ -41,4 +41,4 @@ your Besu database from backup and restart the migration process. ## Migration support If you have a long running network with a large volume of private transactions and/or would like to discuss -the migration process with us before upgrading, contact us at support@pegasys.tech \ No newline at end of file +the migration process with us before upgrading, contact us at support@pegasys.tech From dc81641c5832175b560579b144873267c71ef4cd Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 19 Nov 2024 08:15:09 -0500 Subject: [PATCH 2/7] narrows dependencies of BlockchainService (#7882) Signed-off-by: jflo --- .../dsl/node/ThreadBesuNodeRunner.java | 4 +++- .../org/hyperledger/besu/cli/BesuCommand.java | 2 +- .../besu/services/BlockchainServiceImpl.java | 18 +++++++----------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java index fc2a7d372e..4544872d5c 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java @@ -327,7 +327,9 @@ public class ThreadBesuNodeRunner implements BesuNodeRunner { @Singleton BlockchainServiceImpl provideBlockchainService(final BesuController besuController) { BlockchainServiceImpl retval = new BlockchainServiceImpl(); - retval.init(besuController.getProtocolContext(), besuController.getProtocolSchedule()); + retval.init( + besuController.getProtocolContext().getBlockchain(), + besuController.getProtocolSchedule()); return retval; } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 8f4a1472fb..9126bc8a2a 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1233,7 +1233,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { private void startPlugins(final Runner runner) { blockchainServiceImpl.init( - besuController.getProtocolContext(), besuController.getProtocolSchedule()); + besuController.getProtocolContext().getBlockchain(), besuController.getProtocolSchedule()); transactionSimulationServiceImpl.init( besuController.getProtocolContext().getBlockchain(), new TransactionSimulator( diff --git a/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java b/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java index e981a3e78d..76b4d7e150 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.services; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; @@ -40,7 +39,6 @@ import java.util.stream.Collectors; @Unstable public class BlockchainServiceImpl implements BlockchainService { - private ProtocolContext protocolContext; private ProtocolSchedule protocolSchedule; private MutableBlockchain blockchain; @@ -50,13 +48,12 @@ public class BlockchainServiceImpl implements BlockchainService { /** * Initialize the Blockchain service. * - * @param protocolContext the protocol context + * @param blockchain the blockchain * @param protocolSchedule the protocol schedule */ - public void init(final ProtocolContext protocolContext, final ProtocolSchedule protocolSchedule) { - this.protocolContext = protocolContext; + public void init(final MutableBlockchain blockchain, final ProtocolSchedule protocolSchedule) { this.protocolSchedule = protocolSchedule; - this.blockchain = protocolContext.getBlockchain(); + this.blockchain = blockchain; } /** @@ -67,25 +64,24 @@ public class BlockchainServiceImpl implements BlockchainService { */ @Override public Optional getBlockByNumber(final long number) { - return protocolContext - .getBlockchain() + return blockchain .getBlockByNumber(number) .map(block -> blockContext(block::getHeader, block::getBody)); } @Override public Hash getChainHeadHash() { - return protocolContext.getBlockchain().getChainHeadHash(); + return blockchain.getChainHeadHash(); } @Override public BlockHeader getChainHeadHeader() { - return protocolContext.getBlockchain().getChainHeadHeader(); + return blockchain.getChainHeadHeader(); } @Override public Optional getNextBlockBaseFee() { - final var chainHeadHeader = protocolContext.getBlockchain().getChainHeadHeader(); + final var chainHeadHeader = blockchain.getChainHeadHeader(); final var protocolSpec = protocolSchedule.getForNextBlockHeader(chainHeadHeader, System.currentTimeMillis()); return Optional.of(protocolSpec.getFeeMarket()) From 833a5ce5ddcb56c1ca67544a3d9f968128fb45bc Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 19 Nov 2024 15:51:04 +0100 Subject: [PATCH 3/7] Simplify ProtocolContext creation (#7792) Signed-off-by: Fabio Di Fabio --- .../controller/BesuControllerBuilder.java | 24 ++++++------ ...onsensusScheduleBesuControllerBuilder.java | 17 +++++---- .../controller/QbftBesuControllerBuilder.java | 5 +-- .../TransitionBesuControllerBuilder.java | 7 +--- ...nsusScheduleBesuControllerBuilderTest.java | 17 ++++----- .../blockcreation/CliqueBlockCreatorTest.java | 17 +++++---- .../CliqueMinerExecutorTest.java | 2 + ...xt.java => MigratingConsensusContext.java} | 4 +- .../common/MigratingProtocolContext.java | 36 ++---------------- .../common/MigratingProtocolContextTest.java | 6 ++- .../jsonrpc/JsonRpcTestMethodsFactory.java | 5 ++- ...ckByNumberLatestDesyncIntegrationTest.java | 4 +- .../AbstractEthGraphQLHttpServiceTest.java | 6 ++- .../blockcreation/BlockMinerTest.java | 7 ++-- .../ethereum/ConsensusContextFactory.java | 38 ------------------- .../besu/ethereum/ProtocolContext.java | 29 +------------- .../ethereum/core/BlockchainSetupUtil.java | 11 +----- .../core/ConsensusContextFixture.java | 24 ++++++++++++ .../core/ExecutionContextTestFixture.java | 3 +- .../bonsai/AbstractIsolationTests.java | 5 ++- .../fullsync/FullSyncTargetManagerTest.java | 4 +- ...neCommonAncestorTaskParameterizedTest.java | 7 +++- .../DetermineCommonAncestorTaskTest.java | 7 +++- .../ethereum/eth/transactions/TestNode.java | 4 +- .../BlockchainReferenceTestCaseSpec.java | 7 +++- 25 files changed, 127 insertions(+), 169 deletions(-) rename consensus/common/src/main/java/org/hyperledger/besu/consensus/common/{MigratingContext.java => MigratingConsensusContext.java} (88%) delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ConsensusContextFactory.java create mode 100644 ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ConsensusContextFixture.java diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 9059d2209b..b3cbab5cc9 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -26,7 +26,6 @@ import org.hyperledger.besu.consensus.qbft.BFTPivotSelectorFromPeers; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.ConsensusContext; -import org.hyperledger.besu.ethereum.ConsensusContextFactory; import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.jsonrpc.methods.JsonRpcMethods; @@ -605,9 +604,11 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides genesisState.writeStateTo(worldStateArchive.getMutable()); } + final var consensusContext = + createConsensusContext(blockchain, worldStateArchive, protocolSchedule); + final ProtocolContext protocolContext = - createProtocolContext( - blockchain, worldStateArchive, protocolSchedule, this::createConsensusContext); + createProtocolContext(blockchain, worldStateArchive, consensusContext); validateContext(protocolContext); protocolSchedule.setPublicWorldStateArchiveForPrivacyBlockProcessor( @@ -976,7 +977,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides } /** - * Create mining coordinator mining coordinator. + * Create mining coordinator. * * @param protocolSchedule the protocol schedule * @param protocolContext the protocol context @@ -1017,9 +1018,9 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides * @return the consensus context */ protected abstract ConsensusContext createConsensusContext( - Blockchain blockchain, - WorldStateArchive worldStateArchive, - ProtocolSchedule protocolSchedule); + final Blockchain blockchain, + final WorldStateArchive worldStateArchive, + final ProtocolSchedule protocolSchedule); /** * Gets supported protocol. @@ -1079,17 +1080,14 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides * * @param blockchain the blockchain * @param worldStateArchive the world state archive - * @param protocolSchedule the protocol schedule - * @param consensusContextFactory the consensus context factory + * @param consensusContext the consensus context * @return the protocol context */ protected ProtocolContext createProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, - final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory) { - return ProtocolContext.init( - blockchain, worldStateArchive, protocolSchedule, consensusContextFactory, badBlockManager); + final ConsensusContext consensusContext) { + return new ProtocolContext(blockchain, worldStateArchive, consensusContext, badBlockManager); } private Optional createSnapProtocolManager( diff --git a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java index 3ecd5a6c41..c70367aed6 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java @@ -20,13 +20,12 @@ import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.consensus.common.CombinedProtocolScheduleFactory; import org.hyperledger.besu.consensus.common.ForkSpec; import org.hyperledger.besu.consensus.common.ForksSchedule; -import org.hyperledger.besu.consensus.common.MigratingContext; +import org.hyperledger.besu.consensus.common.MigratingConsensusContext; import org.hyperledger.besu.consensus.common.MigratingMiningCoordinator; import org.hyperledger.besu.consensus.common.MigratingProtocolContext; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.ConsensusContext; -import org.hyperledger.besu.ethereum.ConsensusContextFactory; import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.jsonrpc.methods.JsonRpcMethods; @@ -168,10 +167,12 @@ public class ConsensusScheduleBesuControllerBuilder extends BesuControllerBuilde protected ProtocolContext createProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, - final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory) { - return MigratingProtocolContext.init( - blockchain, worldStateArchive, protocolSchedule, consensusContextFactory, badBlockManager); + final ConsensusContext consensusContext) { + return new MigratingProtocolContext( + blockchain, + worldStateArchive, + consensusContext.as(MigratingConsensusContext.class), + badBlockManager); } @Override @@ -188,10 +189,10 @@ public class ConsensusScheduleBesuControllerBuilder extends BesuControllerBuilde e.getValue() .createConsensusContext( blockchain, worldStateArchive, protocolSchedule))) - .collect(Collectors.toList()); + .toList(); final ForksSchedule consensusContextsSchedule = new ForksSchedule<>(consensusContextSpecs); - return new MigratingContext(consensusContextsSchedule); + return new MigratingConsensusContext(consensusContextsSchedule); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java index 75aff30871..b6f579ea68 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java @@ -112,10 +112,7 @@ public class QbftBesuControllerBuilder extends BftBesuControllerBuilder { @Override protected Supplier bftExtraDataCodec() { - return Suppliers.memoize( - () -> { - return new QbftExtraDataCodec(); - }); + return Suppliers.memoize(QbftExtraDataCodec::new); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index 3530621fc6..75abca6a57 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -24,7 +24,6 @@ import org.hyperledger.besu.consensus.merge.blockcreation.TransitionCoordinator; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.ConsensusContext; -import org.hyperledger.besu.ethereum.ConsensusContextFactory; import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator; @@ -193,11 +192,9 @@ public class TransitionBesuControllerBuilder extends BesuControllerBuilder { protected ProtocolContext createProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, - final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory) { + final ConsensusContext consensusContext) { final ProtocolContext protocolContext = - super.createProtocolContext( - blockchain, worldStateArchive, protocolSchedule, consensusContextFactory); + super.createProtocolContext(blockchain, worldStateArchive, consensusContext); transitionProtocolSchedule.setProtocolContext(protocolContext); return protocolContext; } diff --git a/besu/src/test/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilderTest.java index aefcda4c23..4e38e03c06 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilderTest.java @@ -24,7 +24,7 @@ import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.consensus.common.ForkSpec; import org.hyperledger.besu.consensus.common.ForksSchedule; -import org.hyperledger.besu.consensus.common.MigratingContext; +import org.hyperledger.besu.consensus.common.MigratingConsensusContext; import org.hyperledger.besu.consensus.common.MigratingMiningCoordinator; import org.hyperledger.besu.consensus.common.bft.blockcreation.BftMiningCoordinator; import org.hyperledger.besu.ethereum.ConsensusContext; @@ -166,8 +166,8 @@ public class ConsensusScheduleBesuControllerBuilderTest { @Test public void createsMigratingContext() { - final ConsensusContext context1 = Mockito.mock(ConsensusContext.class); - final ConsensusContext context2 = Mockito.mock(ConsensusContext.class); + final ConsensusContext context1 = mock(ConsensusContext.class); + final ConsensusContext context2 = mock(ConsensusContext.class); final Map besuControllerBuilderSchedule = new TreeMap<>(); besuControllerBuilderSchedule.put(0L, besuControllerBuilder1); @@ -180,15 +180,14 @@ public class ConsensusScheduleBesuControllerBuilderTest { new ConsensusScheduleBesuControllerBuilder(besuControllerBuilderSchedule); final ConsensusContext consensusContext = controllerBuilder.createConsensusContext( - Mockito.mock(Blockchain.class), - Mockito.mock(WorldStateArchive.class), - Mockito.mock(ProtocolSchedule.class)); + mock(Blockchain.class), mock(WorldStateArchive.class), mock(ProtocolSchedule.class)); - assertThat(consensusContext).isInstanceOf(MigratingContext.class); - final MigratingContext migratingContext = (MigratingContext) consensusContext; + assertThat(consensusContext).isInstanceOf(MigratingConsensusContext.class); + final MigratingConsensusContext migratingConsensusContext = + (MigratingConsensusContext) consensusContext; final ForksSchedule contextSchedule = - migratingContext.getConsensusContextSchedule(); + migratingConsensusContext.getConsensusContextSchedule(); final NavigableSet> expectedConsensusContextSpecs = new TreeSet<>(ForkSpec.COMPARATOR); diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java index fdc068ddef..12467cda4e 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueBlockCreatorTest.java @@ -52,6 +52,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.ImmutableMiningConfiguration; import org.hyperledger.besu.ethereum.core.ImmutableMiningConfiguration.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningConfiguration; +import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Util; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; @@ -99,11 +100,20 @@ public class CliqueBlockCreatorTest { @BeforeEach void setup() { + final Address otherAddress = Util.publicKeyToAddress(otherKeyPair.getPublicKey()); + validatorList.add(otherAddress); + + validatorProvider = mock(ValidatorProvider.class); + voteProvider = mock(VoteProvider.class); + when(validatorProvider.getVoteProviderAtHead()).thenReturn(Optional.of(voteProvider)); + when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); + protocolSchedule = CliqueProtocolSchedule.create( GenesisConfigFile.DEFAULT.getConfigOptions(), new ForksSchedule<>(List.of()), proposerNodeKey, + PrivacyParameters.DEFAULT, false, EvmConfiguration.DEFAULT, MiningConfiguration.MINING_DISABLED, @@ -111,13 +121,6 @@ public class CliqueBlockCreatorTest { false, new NoOpMetricsSystem()); - final Address otherAddress = Util.publicKeyToAddress(otherKeyPair.getPublicKey()); - validatorList.add(otherAddress); - - validatorProvider = mock(ValidatorProvider.class); - voteProvider = mock(VoteProvider.class); - when(validatorProvider.getVoteProviderAtHead()).thenReturn(Optional.of(voteProvider)); - when(validatorProvider.getValidatorsAfterBlock(any())).thenReturn(validatorList); final CliqueContext cliqueContext = new CliqueContext(validatorProvider, null, blockInterface); final Block genesis = diff --git a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java index 3d6dbd77db..7000d145b6 100644 --- a/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java +++ b/consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java @@ -42,6 +42,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.ImmutableMiningConfiguration; import org.hyperledger.besu.ethereum.core.ImmutableMiningConfiguration.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningConfiguration; +import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Util; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; @@ -103,6 +104,7 @@ public class CliqueMinerExecutorTest { GENESIS_CONFIG_OPTIONS, new ForksSchedule<>(List.of()), proposerNodeKey, + PrivacyParameters.DEFAULT, false, EvmConfiguration.DEFAULT, MiningConfiguration.MINING_DISABLED, diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingContext.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingConsensusContext.java similarity index 88% rename from consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingContext.java rename to consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingConsensusContext.java index f71e3ac7f3..91ace1a04b 100644 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingContext.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingConsensusContext.java @@ -17,7 +17,7 @@ package org.hyperledger.besu.consensus.common; import org.hyperledger.besu.ethereum.ConsensusContext; /** The Migrating context. */ -public class MigratingContext implements ConsensusContext { +public class MigratingConsensusContext implements ConsensusContext { private final ForksSchedule consensusContextSchedule; @@ -26,7 +26,7 @@ public class MigratingContext implements ConsensusContext { * * @param consensusContextSchedule the consensus context schedule */ - public MigratingContext(final ForksSchedule consensusContextSchedule) { + public MigratingConsensusContext(final ForksSchedule consensusContextSchedule) { this.consensusContextSchedule = consensusContextSchedule; } diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java index c0557485f3..27cce3157b 100644 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/MigratingProtocolContext.java @@ -15,11 +15,9 @@ package org.hyperledger.besu.consensus.common; import org.hyperledger.besu.ethereum.ConsensusContext; -import org.hyperledger.besu.ethereum.ConsensusContextFactory; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; /** The Migrating protocol context. */ @@ -32,42 +30,16 @@ public class MigratingProtocolContext extends ProtocolContext { * * @param blockchain the blockchain * @param worldStateArchive the world state archive - * @param consensusContextSchedule the consensus context schedule + * @param migratingConsensusContext the consensus context * @param badBlockManager the cache to use to keep invalid blocks */ public MigratingProtocolContext( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive, - final ForksSchedule consensusContextSchedule, + final MigratingConsensusContext migratingConsensusContext, final BadBlockManager badBlockManager) { - super(blockchain, worldStateArchive, null, badBlockManager); - this.consensusContextSchedule = consensusContextSchedule; - } - - /** - * Init protocol context. - * - * @param blockchain the blockchain - * @param worldStateArchive the world state archive - * @param protocolSchedule the protocol schedule - * @param consensusContextFactory the consensus context factory - * @param badBlockManager the cache to use to keep invalid blocks - * @return the protocol context - */ - public static ProtocolContext init( - final MutableBlockchain blockchain, - final WorldStateArchive worldStateArchive, - final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory, - final BadBlockManager badBlockManager) { - final ConsensusContext consensusContext = - consensusContextFactory.create(blockchain, worldStateArchive, protocolSchedule); - final MigratingContext migratingContext = consensusContext.as(MigratingContext.class); - return new MigratingProtocolContext( - blockchain, - worldStateArchive, - migratingContext.getConsensusContextSchedule(), - badBlockManager); + super(blockchain, worldStateArchive, migratingConsensusContext, badBlockManager); + this.consensusContextSchedule = migratingConsensusContext.getConsensusContextSchedule(); } @Override diff --git a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java index d62de57eac..39ab9f8cb2 100644 --- a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java +++ b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/MigratingProtocolContextTest.java @@ -43,9 +43,13 @@ public class MigratingProtocolContextTest { final ForksSchedule contextSchedule = new ForksSchedule<>(List.of(new ForkSpec<>(0L, context1), new ForkSpec<>(10L, context2))); + final MigratingProtocolContext migratingProtocolContext = new MigratingProtocolContext( - blockchain, worldStateArchive, contextSchedule, new BadBlockManager()); + blockchain, + worldStateArchive, + new MigratingConsensusContext(contextSchedule), + new BadBlockManager()); assertThat(migratingProtocolContext.getConsensusContext(ConsensusContext.class)) .isSameAs(context1); diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java index 902d2b0487..2a65b1e7cb 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java @@ -19,6 +19,7 @@ import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider import static org.mockito.Mockito.mock; import org.hyperledger.besu.config.StubGenesisConfigOptions; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.ImmutableApiConfiguration; import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration; @@ -82,7 +83,9 @@ public class JsonRpcTestMethodsFactory { this.blockchain = createInMemoryBlockchain(importer.getGenesisBlock()); this.stateArchive = createInMemoryWorldStateArchive(); this.importer.getGenesisState().writeStateTo(stateArchive.getMutable()); - this.context = new ProtocolContext(blockchain, stateArchive, null, new BadBlockManager()); + this.context = + new ProtocolContext( + blockchain, stateArchive, mock(ConsensusContext.class), new BadBlockManager()); this.protocolSchedule = importer.getProtocolSchedule(); this.synchronizer = mock(Synchronizer.class); diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java index 088eff8500..7e26a4d812 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.jsonrpc.BlockchainImporter; import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcTestMethodsFactory; @@ -67,7 +68,8 @@ public class EthGetBlockByNumberLatestDesyncIntegrationTest { InMemoryKeyValueStorageProvider.createInMemoryBlockchain(importer.getGenesisBlock()); WorldStateArchive state = InMemoryKeyValueStorageProvider.createInMemoryWorldStateArchive(); importer.getGenesisState().writeStateTo(state.getMutable()); - ProtocolContext context = new ProtocolContext(chain, state, null, new BadBlockManager()); + ProtocolContext context = + new ProtocolContext(chain, state, mock(ConsensusContext.class), new BadBlockManager()); for (final Block block : importer.getBlocks()) { final ProtocolSchedule protocolSchedule = importer.getProtocolSchedule(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java index 2b55b42ba2..08cfc3d7bd 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java @@ -19,6 +19,7 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.TransactionType; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.ImmutableApiConfiguration; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; @@ -110,7 +111,10 @@ public abstract class AbstractEthGraphQLHttpServiceTest { final MutableBlockchain blockchain = blockchainSetupUtil.getBlockchain(); ProtocolContext context = new ProtocolContext( - blockchain, blockchainSetupUtil.getWorldArchive(), null, new BadBlockManager()); + blockchain, + blockchainSetupUtil.getWorldArchive(), + mock(ConsensusContext.class), + new BadBlockManager()); final BlockchainQueries blockchainQueries = new BlockchainQueries( blockchainSetupUtil.getProtocolSchedule(), diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java index e5b46925aa..5b893d11f2 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockMinerTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.blockcreation.BlockCreator.BlockCreationResult; import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionSelectionResults; @@ -58,7 +59,7 @@ public class BlockMinerTest { headerBuilder.buildHeader(), new BlockBody(Lists.newArrayList(), Lists.newArrayList())); final ProtocolContext protocolContext = - new ProtocolContext(null, null, null, new BadBlockManager()); + new ProtocolContext(null, null, mock(ConsensusContext.class), new BadBlockManager()); final PoWBlockCreator blockCreator = mock(PoWBlockCreator.class); final Function blockCreatorSupplier = @@ -102,7 +103,7 @@ public class BlockMinerTest { headerBuilder.buildHeader(), new BlockBody(Lists.newArrayList(), Lists.newArrayList())); final ProtocolContext protocolContext = - new ProtocolContext(null, null, null, new BadBlockManager()); + new ProtocolContext(null, null, mock(ConsensusContext.class), new BadBlockManager()); final PoWBlockCreator blockCreator = mock(PoWBlockCreator.class); final Function blockCreatorSupplier = @@ -150,7 +151,7 @@ public class BlockMinerTest { headerBuilder.buildHeader(), new BlockBody(Lists.newArrayList(), Lists.newArrayList())); final ProtocolContext protocolContext = - new ProtocolContext(null, null, null, new BadBlockManager()); + new ProtocolContext(null, null, mock(ConsensusContext.class), new BadBlockManager()); final PoWBlockCreator blockCreator = mock(PoWBlockCreator.class); final Function blockCreatorSupplier = diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ConsensusContextFactory.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ConsensusContextFactory.java deleted file mode 100644 index a9381fa199..0000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ConsensusContextFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum; - -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; - -/** The ConsensusContextFactory interface defines a method for creating a consensus context. */ -@FunctionalInterface -public interface ConsensusContextFactory { - - /** - * Creates a consensus context with the given blockchain, world state archive, and protocol - * schedule. - * - * @param blockchain the blockchain - * @param worldStateArchive the world state archive - * @param protocolSchedule the protocol schedule - * @return the created consensus context - */ - ConsensusContext create( - Blockchain blockchain, - WorldStateArchive worldStateArchive, - ProtocolSchedule protocolSchedule); -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java index 53197ea9c5..33897c3e1e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.ethereum; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.Optional; @@ -29,8 +28,8 @@ import java.util.Optional; public class ProtocolContext { private final MutableBlockchain blockchain; private final WorldStateArchive worldStateArchive; - private final BadBlockManager badBlockManager; private final ConsensusContext consensusContext; + private final BadBlockManager badBlockManager; /** * Constructs a new ProtocolContext with the given blockchain, world state archive, consensus @@ -38,7 +37,7 @@ public class ProtocolContext { * * @param blockchain the blockchain of the protocol context * @param worldStateArchive the world state archive of the protocol context - * @param consensusContext the consensus context of the protocol context + * @param consensusContext the consensus context * @param badBlockManager the bad block manager of the protocol context */ public ProtocolContext( @@ -52,30 +51,6 @@ public class ProtocolContext { this.badBlockManager = badBlockManager; } - /** - * Initializes a new ProtocolContext with the given blockchain, world state archive, protocol - * schedule, consensus context factory, and bad block manager. - * - * @param blockchain the blockchain of the protocol context - * @param worldStateArchive the world state archive of the protocol context - * @param protocolSchedule the protocol schedule of the protocol context - * @param consensusContextFactory the consensus context factory of the protocol context - * @param badBlockManager the bad block manager of the protocol context - * @return the initialized ProtocolContext - */ - public static ProtocolContext init( - final MutableBlockchain blockchain, - final WorldStateArchive worldStateArchive, - final ProtocolSchedule protocolSchedule, - final ConsensusContextFactory consensusContextFactory, - final BadBlockManager badBlockManager) { - return new ProtocolContext( - blockchain, - worldStateArchive, - consensusContextFactory.create(blockchain, worldStateArchive, protocolSchedule), - badBlockManager); - } - /** * Gets the blockchain of the protocol context. * diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java index b7608bb5de..3391a0e162 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java @@ -21,7 +21,6 @@ import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider import static org.mockito.Mockito.mock; import org.hyperledger.besu.config.GenesisConfigFile; -import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -161,15 +160,7 @@ public class BlockchainSetupUtil { private static ProtocolContext mainnetProtocolContextProvider( final MutableBlockchain blockchain, final WorldStateArchive worldStateArchive) { return new ProtocolContext( - blockchain, - worldStateArchive, - new ConsensusContext() { - @Override - public C as(final Class klass) { - return null; - } - }, - new BadBlockManager()); + blockchain, worldStateArchive, new ConsensusContextFixture(), new BadBlockManager()); } private static BlockchainSetupUtil create( diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ConsensusContextFixture.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ConsensusContextFixture.java new file mode 100644 index 0000000000..198b3ab813 --- /dev/null +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ConsensusContextFixture.java @@ -0,0 +1,24 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.core; + +import org.hyperledger.besu.ethereum.ConsensusContext; + +public class ConsensusContextFixture implements ConsensusContext { + @Override + public C as(final Class klass) { + return klass.cast(this); + } +} diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java index b2b979e283..b4eaec04d3 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java @@ -76,7 +76,8 @@ public class ExecutionContextTestFixture { else this.stateArchive = createInMemoryWorldStateArchive(); this.protocolSchedule = protocolSchedule; this.protocolContext = - new ProtocolContext(blockchain, stateArchive, null, new BadBlockManager()); + new ProtocolContext( + blockchain, stateArchive, new ConsensusContextFixture(), new BadBlockManager()); genesisState.writeStateTo(stateArchive.getMutable()); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java index cfb3094726..275adc9ff4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java @@ -30,6 +30,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.BlockProcessingResult; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.blockcreation.AbstractBlockCreator; import org.hyperledger.besu.ethereum.chain.BadBlockManager; @@ -172,7 +173,9 @@ public abstract class AbstractIsolationTests { throwingWorldStateHealerSupplier()); var ws = archive.getMutable(); genesisState.writeStateTo(ws); - protocolContext = new ProtocolContext(blockchain, archive, null, new BadBlockManager()); + protocolContext = + new ProtocolContext( + blockchain, archive, mock(ConsensusContext.class), new BadBlockManager()); ethContext = mock(EthContext.class, RETURNS_DEEP_STUBS); when(ethContext.getEthPeers().subscribeConnect(any())).thenReturn(1L); transactionPool = diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java index 027bd270d2..9132842ccb 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManagerTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -77,7 +78,8 @@ public class FullSyncTargetManagerTest { final ProtocolSchedule protocolSchedule = ProtocolScheduleFixture.MAINNET; final ProtocolContext protocolContext = - new ProtocolContext(localBlockchain, localWorldState, null, new BadBlockManager()); + new ProtocolContext( + localBlockchain, localWorldState, mock(ConsensusContext.class), new BadBlockManager()); ethProtocolManager = EthProtocolManagerTestUtil.create( protocolSchedule, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java index 73d5e5138b..5686b4ae90 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskParameterizedTest.java @@ -19,6 +19,7 @@ import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryWorldStateArchive; import static org.mockito.Mockito.mock; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -151,7 +152,11 @@ public class DetermineCommonAncestorTaskParameterizedTest { final EthContext ethContext = ethProtocolManager.ethContext(); final ProtocolContext protocolContext = - new ProtocolContext(localBlockchain, worldStateArchive, null, new BadBlockManager()); + new ProtocolContext( + localBlockchain, + worldStateArchive, + mock(ConsensusContext.class), + new BadBlockManager()); final EthTask task = DetermineCommonAncestorTask.create( diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java index 1b19a076d2..f85e4dd31c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DetermineCommonAncestorTaskTest.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -88,7 +89,11 @@ public class DetermineCommonAncestorTaskTest { EthProtocolConfiguration.defaultConfig()); ethContext = ethProtocolManager.ethContext(); protocolContext = - new ProtocolContext(localBlockchain, worldStateArchive, null, new BadBlockManager()); + new ProtocolContext( + localBlockchain, + worldStateArchive, + mock(ConsensusContext.class), + new BadBlockManager()); } @Test diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java index 0081dc7d94..d042996ce7 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TestNode.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.crypto.KeyPair; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.cryptoservices.NodeKeyUtils; +import org.hyperledger.besu.ethereum.ConsensusContext; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.GenesisState; @@ -135,7 +136,8 @@ public class TestNode implements Closeable { final WorldStateArchive worldStateArchive = createInMemoryWorldStateArchive(); genesisState.writeStateTo(worldStateArchive.getMutable()); final ProtocolContext protocolContext = - new ProtocolContext(blockchain, worldStateArchive, null, new BadBlockManager()); + new ProtocolContext( + blockchain, worldStateArchive, mock(ConsensusContext.class), new BadBlockManager()); final SyncState syncState = mock(SyncState.class); final SynchronizerConfiguration syncConfig = mock(SynchronizerConfiguration.class); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java index 7ab0091090..771ae4672a 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; +import org.hyperledger.besu.ethereum.core.ConsensusContextFixture; import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.MutableWorldState; @@ -108,7 +109,11 @@ public class BlockchainReferenceTestCaseSpec { this.blockchain = buildBlockchain(genesisBlockHeader); this.sealEngine = sealEngine; this.protocolContext = - new ProtocolContext(this.blockchain, this.worldStateArchive, null, new BadBlockManager()); + new ProtocolContext( + this.blockchain, + this.worldStateArchive, + new ConsensusContextFixture(), + new BadBlockManager()); } public String getNetwork() { From 02722bcd8a71527fd28b600433a4606368d6aa16 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 19 Nov 2024 17:10:03 +0100 Subject: [PATCH 4/7] Consolidate all metric collectors into the metrics system (#7877) * Consolidate all metric collectors into the metrics system Signed-off-by: Fabio Di Fabio * Fixes Signed-off-by: Fabio Di Fabio * Do not recreate the Prometheus metric system Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio --- .../jsonrpc/internal/methods/TraceBlock.java | 37 ++--- .../jsonrpc/internal/methods/TraceFilter.java | 28 ++-- .../methods/TraceReplayBlockTransactions.java | 25 ++-- .../methods/JsonRpcMethodsFactory.java | 6 +- .../jsonrpc/methods/TraceJsonRpcMethods.java | 15 +- .../internal/methods/TraceFilterTest.java | 4 +- ethereum/core/build.gradle | 2 - .../ethereum/chain/DefaultBlockchain.java | 32 ++--- .../cache/BonsaiCachedMerkleTrieLoader.java | 13 +- ethereum/p2p/build.gradle | 1 - metrics/core/build.gradle | 1 + .../besu/metrics/ObservableMetricsSystem.java | 31 +---- .../besu/metrics/noop/NoOpMetricsSystem.java | 17 +++ .../opentelemetry/OpenTelemetrySystem.java | 15 ++ .../prometheus/PrometheusMetricsSystem.java | 130 +++++++++++++----- .../besu/metrics/StubMetricsSystem.java | 21 +++ metrics/rocksdb/build.gradle | 2 - .../besu/metrics/rocksdb/RocksDBStats.java | 87 +++++------- plugin-api/build.gradle | 2 +- .../besu/plugin/services/MetricsSystem.java | 46 +++++++ .../services/metrics/ExternalSummary.java | 36 +++++ plugins/rocksdb/build.gradle | 1 - .../rocksdb/RocksDBMetricsFactory.java | 6 +- 23 files changed, 349 insertions(+), 209 deletions(-) create mode 100644 plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/ExternalSummary.java diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java index adacf01e76..16798160d8 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java @@ -25,7 +25,6 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcPara import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.FlatTraceGenerator; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.RewardTraceGenerator; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.api.util.ArrayNodeWrapper; @@ -41,12 +40,11 @@ import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.services.pipeline.Pipeline; -import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -58,10 +56,21 @@ public class TraceBlock extends AbstractBlockParameterMethod { private static final Logger LOG = LoggerFactory.getLogger(TraceBlock.class); private static final ObjectMapper MAPPER = new ObjectMapper(); protected final ProtocolSchedule protocolSchedule; + private final LabelledMetric outputCounter; - public TraceBlock(final ProtocolSchedule protocolSchedule, final BlockchainQueries queries) { + public TraceBlock( + final ProtocolSchedule protocolSchedule, + final BlockchainQueries queries, + final MetricsSystem metricsSystem) { super(queries); this.protocolSchedule = protocolSchedule; + this.outputCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.BLOCKCHAIN, + "transactions_traceblock_pipeline_processed_total", + "Number of transactions processed for each block", + "step", + "action"); } @Override @@ -115,14 +124,6 @@ public class TraceBlock extends AbstractBlockParameterMethod { final ChainUpdater chainUpdater = new ChainUpdater(traceableState); TransactionSource transactionSource = new TransactionSource(block); - final LabelledMetric outputCounter = - new PrometheusMetricsSystem(BesuMetricCategory.DEFAULT_METRIC_CATEGORIES, false) - .createLabelledCounter( - BesuMetricCategory.BLOCKCHAIN, - "transactions_traceblock_pipeline_processed_total", - "Number of transactions processed for each block", - "step", - "action"); DebugOperationTracer debugOperationTracer = new DebugOperationTracer(new TraceOptions(false, false, true), false); ExecuteTransactionStep executeTransactionStep = @@ -173,18 +174,6 @@ public class TraceBlock extends AbstractBlockParameterMethod { .orElse(emptyResult()); } - protected void generateTracesFromTransactionTraceAndBlock( - final Optional filterParameter, - final List transactionTraces, - final Block block, - final ArrayNodeWrapper resultArrayNode) { - transactionTraces.forEach( - transactionTrace -> - FlatTraceGenerator.generateFromTransactionTraceAndBlock( - protocolSchedule, transactionTrace, block) - .forEachOrdered(resultArrayNode::addPOJO)); - } - protected void generateRewardsFromBlock( final Optional maybeFilterParameter, final Block block, diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java index 74bfec87c2..6be65149cb 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonR import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.FilterParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; @@ -45,7 +44,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.services.pipeline.Pipeline; @@ -58,7 +57,6 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -68,17 +66,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TraceFilter extends TraceBlock { - private static final Logger LOG = LoggerFactory.getLogger(TraceFilter.class); private final Long maxRange; + private final LabelledMetric outputCounter; public TraceFilter( - final Supplier blockTracerSupplier, final ProtocolSchedule protocolSchedule, final BlockchainQueries blockchainQueries, - final Long maxRange) { - super(protocolSchedule, blockchainQueries); + final Long maxRange, + final MetricsSystem metricsSystem) { + super(protocolSchedule, blockchainQueries, metricsSystem); this.maxRange = maxRange; + this.outputCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.BLOCKCHAIN, + "transactions_tracefilter_pipeline_processed_total", + "Number of transactions processed for trace_filter", + "step", + "action"); } @Override @@ -157,15 +162,6 @@ public class TraceFilter extends TraceBlock { final MainnetTransactionProcessor transactionProcessor = protocolSpec.getTransactionProcessor(); final ChainUpdater chainUpdater = new ChainUpdater(traceableState); - final LabelledMetric outputCounter = - new PrometheusMetricsSystem( - BesuMetricCategory.DEFAULT_METRIC_CATEGORIES, false) - .createLabelledCounter( - BesuMetricCategory.BLOCKCHAIN, - "transactions_tracefilter_pipeline_processed_total", - "Number of transactions processed for trace_filter", - "step", - "action"); DebugOperationTracer debugOperationTracer = new DebugOperationTracer(new TraceOptions(false, false, true), false); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java index 202d2ddbbb..51eeebb959 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java @@ -39,7 +39,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.services.pipeline.Pipeline; @@ -57,13 +57,23 @@ import org.slf4j.LoggerFactory; public class TraceReplayBlockTransactions extends AbstractBlockParameterMethod { private static final Logger LOG = LoggerFactory.getLogger(TraceReplayBlockTransactions.class); - private final ProtocolSchedule protocolSchedule; private static final ObjectMapper MAPPER = new ObjectMapper(); + private final ProtocolSchedule protocolSchedule; + private final LabelledMetric outputCounter; public TraceReplayBlockTransactions( - final ProtocolSchedule protocolSchedule, final BlockchainQueries queries) { + final ProtocolSchedule protocolSchedule, + final BlockchainQueries queries, + final MetricsSystem metricsSystem) { super(queries); this.protocolSchedule = protocolSchedule; + this.outputCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.BLOCKCHAIN, + "transactions_tracereplayblock_pipeline_processed_total", + "Number of transactions processed for each block", + "step", + "action"); } @Override @@ -131,14 +141,7 @@ public class TraceReplayBlockTransactions extends AbstractBlockParameterMethod { final ChainUpdater chainUpdater = new ChainUpdater(traceableState); final TransactionSource transactionSource = new TransactionSource(block); - final LabelledMetric outputCounter = - new PrometheusMetricsSystem(BesuMetricCategory.DEFAULT_METRIC_CATEGORIES, false) - .createLabelledCounter( - BesuMetricCategory.BLOCKCHAIN, - "transactions_tracereplayblock_pipeline_processed_total", - "Number of transactions processed for each block", - "step", - "action"); + final DebugOperationTracer debugOperationTracer = new DebugOperationTracer(new TraceOptions(false, false, true), false); final ExecuteTransactionStep executeTransactionStep = diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/JsonRpcMethodsFactory.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/JsonRpcMethodsFactory.java index 7be34f0229..2e68576fca 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/JsonRpcMethodsFactory.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/JsonRpcMethodsFactory.java @@ -151,7 +151,11 @@ public class JsonRpcMethodsFactory { blockchainQueries, protocolSchedule, transactionPool, privacyParameters), new Web3JsonRpcMethods(clientNodeName), new TraceJsonRpcMethods( - blockchainQueries, protocolSchedule, protocolContext, apiConfiguration), + blockchainQueries, + protocolSchedule, + protocolContext, + apiConfiguration, + metricsSystem), new TxPoolJsonRpcMethods(transactionPool), new PluginsJsonRpcMethods(namedPlugins)); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java index 6a36bebfcd..4638e90a62 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; +import org.hyperledger.besu.plugin.services.MetricsSystem; import java.util.Map; @@ -38,19 +39,21 @@ public class TraceJsonRpcMethods extends ApiGroupJsonRpcMethods { private final BlockchainQueries blockchainQueries; private final ProtocolSchedule protocolSchedule; - private final ApiConfiguration apiConfiguration; private final ProtocolContext protocolContext; + private final MetricsSystem metricsSystem; TraceJsonRpcMethods( final BlockchainQueries blockchainQueries, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, - final ApiConfiguration apiConfiguration) { + final ApiConfiguration apiConfiguration, + final MetricsSystem metricsSystem) { this.blockchainQueries = blockchainQueries; this.protocolSchedule = protocolSchedule; this.protocolContext = protocolContext; this.apiConfiguration = apiConfiguration; + this.metricsSystem = metricsSystem; } @Override @@ -63,16 +66,16 @@ public class TraceJsonRpcMethods extends ApiGroupJsonRpcMethods { final BlockReplay blockReplay = new BlockReplay(protocolSchedule, protocolContext, blockchainQueries.getBlockchain()); return mapOf( - new TraceReplayBlockTransactions(protocolSchedule, blockchainQueries), + new TraceReplayBlockTransactions(protocolSchedule, blockchainQueries, metricsSystem), new TraceFilter( - () -> new BlockTracer(blockReplay), protocolSchedule, blockchainQueries, - apiConfiguration.getMaxTraceFilterRange()), + apiConfiguration.getMaxTraceFilterRange(), + metricsSystem), new TraceGet(() -> new BlockTracer(blockReplay), blockchainQueries, protocolSchedule), new TraceTransaction( () -> new BlockTracer(blockReplay), protocolSchedule, blockchainQueries), - new TraceBlock(protocolSchedule, blockchainQueries), + new TraceBlock(protocolSchedule, blockchainQueries, metricsSystem), new TraceCall( blockchainQueries, protocolSchedule, diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java index c5c03f6b11..8f64d2cb0b 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcRespon import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.function.Supplier; @@ -69,7 +70,8 @@ public class TraceFilterTest { new JsonRpcRequest("2.0", "trace_filter", new Object[] {filterParameter})); method = - new TraceFilter(blockTracerSupplier, protocolSchedule, blockchainQueries, maxFilterRange); + new TraceFilter( + protocolSchedule, blockchainQueries, maxFilterRange, new NoOpMetricsSystem()); final JsonRpcResponse response = method.response(request); assertThat(response).isInstanceOf(JsonRpcErrorResponse.class); diff --git a/ethereum/core/build.gradle b/ethereum/core/build.gradle index c5ab446266..c0f04a7644 100644 --- a/ethereum/core/build.gradle +++ b/ethereum/core/build.gradle @@ -65,8 +65,6 @@ dependencies { implementation 'org.immutables:value-annotations' implementation 'tech.pegasys:jc-kzg-4844' - implementation 'io.prometheus:simpleclient_guava' - implementation 'org.xerial.snappy:snappy-java' annotationProcessor 'org.immutables:value' diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java index bce94a1915..f6e06f82d2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; +import static org.hyperledger.besu.metrics.BesuMetricCategory.BLOCKCHAIN; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.BlockchainStorage.Updater; @@ -32,7 +33,6 @@ import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.util.InvalidConfigurationException; @@ -56,7 +56,6 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; import com.google.common.collect.Streams; -import io.prometheus.client.guava.cache.CacheMetricsCollector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -134,13 +133,12 @@ public class DefaultBlockchain implements MutableBlockchain { totalDifficultyCache = Optional.of( CacheBuilder.newBuilder().recordStats().maximumSize(numberOfBlocksToCache).build()); - CacheMetricsCollector cacheMetrics = new CacheMetricsCollector(); - cacheMetrics.addCache("blockHeaders", blockHeadersCache.get()); - cacheMetrics.addCache("blockBodies", blockBodiesCache.get()); - cacheMetrics.addCache("transactionReceipts", transactionReceiptsCache.get()); - cacheMetrics.addCache("totalDifficulty", totalDifficultyCache.get()); - if (metricsSystem instanceof PrometheusMetricsSystem prometheusMetricsSystem) - prometheusMetricsSystem.addCollector(BesuMetricCategory.BLOCKCHAIN, () -> cacheMetrics); + metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockHeaders", blockHeadersCache.get()); + metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "blockBodies", blockBodiesCache.get()); + metricsSystem.createGuavaCacheCollector( + BLOCKCHAIN, "transactionReceipts", transactionReceiptsCache.get()); + metricsSystem.createGuavaCacheCollector( + BLOCKCHAIN, "totalDifficulty", totalDifficultyCache.get()); } else { blockHeadersCache = Optional.empty(); blockBodiesCache = Optional.empty(); @@ -155,11 +153,11 @@ public class DefaultBlockchain implements MutableBlockchain { private void createCounters(final MetricsSystem metricsSystem) { gasUsedCounter = metricsSystem.createCounter( - BesuMetricCategory.BLOCKCHAIN, "chain_head_gas_used_counter", "Counter for Gas used"); + BLOCKCHAIN, "chain_head_gas_used_counter", "Counter for Gas used"); numberOfTransactionsCounter = metricsSystem.createCounter( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_transaction_count_counter", "Counter for the number of transactions"); } @@ -184,37 +182,37 @@ public class DefaultBlockchain implements MutableBlockchain { this::getSafeBlockNumber); metricsSystem.createGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "difficulty_total", "Total difficulty of the chainhead", () -> this.getChainHead().getTotalDifficulty().toBigInteger().doubleValue()); metricsSystem.createLongGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_timestamp", "Timestamp from the current chain head", () -> getChainHeadHeader().getTimestamp()); metricsSystem.createLongGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_gas_used", "Gas used by the current chain head block", () -> getChainHeadHeader().getGasUsed()); metricsSystem.createLongGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_gas_limit", "Block gas limit of the current chain head block", () -> getChainHeadHeader().getGasLimit()); metricsSystem.createIntegerGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_transaction_count", "Number of transactions in the current chain head block", () -> chainHeadTransactionCount); metricsSystem.createIntegerGauge( - BesuMetricCategory.BLOCKCHAIN, + BLOCKCHAIN, "chain_head_ommer_count", "Number of ommers in the current chain head block", () -> chainHeadOmmerCount); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java index 9b9960b5c6..402402bc06 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; +import static org.hyperledger.besu.metrics.BesuMetricCategory.BLOCKCHAIN; + import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; @@ -22,9 +24,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -33,7 +33,6 @@ import java.util.function.Function; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import io.prometheus.client.guava.cache.CacheMetricsCollector; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -47,12 +46,8 @@ public class BonsaiCachedMerkleTrieLoader implements StorageSubscriber { CacheBuilder.newBuilder().recordStats().maximumSize(STORAGE_CACHE_SIZE).build(); public BonsaiCachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { - - CacheMetricsCollector cacheMetrics = new CacheMetricsCollector(); - cacheMetrics.addCache("accountsNodes", accountNodes); - cacheMetrics.addCache("storageNodes", storageNodes); - if (metricsSystem instanceof PrometheusMetricsSystem prometheusMetricsSystem) - prometheusMetricsSystem.addCollector(BesuMetricCategory.BLOCKCHAIN, () -> cacheMetrics); + metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "accountsNodes", accountNodes); + metricsSystem.createGuavaCacheCollector(BLOCKCHAIN, "storageNodes", storageNodes); } public void preLoadAccount( diff --git a/ethereum/p2p/build.gradle b/ethereum/p2p/build.gradle index 971504a25a..7cfcf2f8d5 100644 --- a/ethereum/p2p/build.gradle +++ b/ethereum/p2p/build.gradle @@ -43,7 +43,6 @@ dependencies { implementation 'com.google.guava:guava' implementation 'dnsjava:dnsjava' implementation 'io.netty:netty-transport-native-unix-common' - implementation 'io.prometheus:simpleclient' implementation 'io.vertx:vertx-core' implementation 'io.tmio:tuweni-bytes' diff --git a/metrics/core/build.gradle b/metrics/core/build.gradle index 1438a198ae..160093c749 100644 --- a/metrics/core/build.gradle +++ b/metrics/core/build.gradle @@ -57,6 +57,7 @@ dependencies { implementation 'io.prometheus:simpleclient' implementation 'io.prometheus:simpleclient_common' + implementation 'io.prometheus:simpleclient_guava' implementation 'io.prometheus:simpleclient_hotspot' implementation 'io.prometheus:simpleclient_pushgateway' implementation 'io.vertx:vertx-core' diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/ObservableMetricsSystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/ObservableMetricsSystem.java index 8090d28fc1..98e7a8b634 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/ObservableMetricsSystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/ObservableMetricsSystem.java @@ -17,42 +17,25 @@ package org.hyperledger.besu.metrics; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; -import java.util.Set; import java.util.stream.Stream; -/** The interface Observable metrics system. */ +/** The observable metrics system is used to inspect metrics for debug reasons */ public interface ObservableMetricsSystem extends MetricsSystem { - /** - * Stream observations. + * Stream observations by category * * @param category the category - * @return the stream + * @return the observations stream */ Stream streamObservations(MetricCategory category); /** - * Stream observations. + * Stream observations * - * @return the stream + * @return the observations stream */ Stream streamObservations(); - /** - * Provides an immutable view into the metric categories enabled for metric collection. - * - * @return the set of enabled metric categories. - */ - Set getEnabledCategories(); - - /** - * Checks if a particular category of metrics is enabled. - * - * @param category the category to check - * @return true if the category is enabled, false otherwise - */ - default boolean isCategoryEnabled(final MetricCategory category) { - return getEnabledCategories().stream() - .anyMatch(metricCategory -> metricCategory.getName().equals(category.getName())); - } + /** Unregister all the collectors and perform other cleanup tasks */ + void shutdown(); } diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java index 5f876fa4d8..d3b6c4aadc 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.metrics.noop; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.Observation; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; @@ -27,9 +28,11 @@ import java.util.Collections; import java.util.List; import java.util.Set; import java.util.function.DoubleSupplier; +import java.util.function.Supplier; import java.util.stream.Stream; import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; /** The NoOp metrics system. */ public class NoOpMetricsSystem implements ObservableMetricsSystem { @@ -113,6 +116,13 @@ public class NoOpMetricsSystem implements ObservableMetricsSystem { return getOperationTimerLabelledMetric(labelNames.length); } + @Override + public void trackExternalSummary( + final MetricCategory category, + final String name, + final String help, + final Supplier summarySupplier) {} + @Override public LabelledMetric createLabelledTimer( final MetricCategory category, @@ -144,6 +154,10 @@ public class NoOpMetricsSystem implements ObservableMetricsSystem { final String help, final DoubleSupplier valueSupplier) {} + @Override + public void createGuavaCacheCollector( + final MetricCategory category, final String name, final Cache cache) {} + @Override public LabelledGauge createLabelledGauge( final MetricCategory category, @@ -187,6 +201,9 @@ public class NoOpMetricsSystem implements ObservableMetricsSystem { return Collections.emptySet(); } + @Override + public void shutdown() {} + /** * The Label counting NoOp metric. * diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java index a399b28373..a17c773813 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.metrics.Observation; import org.hyperledger.besu.metrics.StandardMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; @@ -40,9 +41,11 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.function.DoubleSupplier; +import java.util.function.Supplier; import java.util.stream.Stream; import javax.inject.Singleton; +import com.google.common.cache.Cache; import com.google.common.collect.ImmutableSet; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; @@ -242,6 +245,13 @@ public class OpenTelemetrySystem implements ObservableMetricsSystem { return createLabelledTimer(category, name, help, labelNames); } + @Override + public void trackExternalSummary( + final MetricCategory category, + final String name, + final String help, + final Supplier summarySupplier) {} + @Override public LabelledMetric createLabelledTimer( final MetricCategory category, @@ -277,6 +287,10 @@ public class OpenTelemetrySystem implements ObservableMetricsSystem { } } + @Override + public void createGuavaCacheCollector( + final MetricCategory category, final String name, final Cache cache) {} + @Override public LabelledGauge createLabelledGauge( final MetricCategory category, @@ -376,6 +390,7 @@ public class OpenTelemetrySystem implements ObservableMetricsSystem { } /** Shuts down the OpenTelemetry exporters, blocking until they have completed orderly. */ + @Override public void shutdown() { final CompletableResultCode result = CompletableResultCode.ofAll( diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java index 653f448311..b001eb0b3b 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.Observation; import org.hyperledger.besu.metrics.StandardMetricCategory; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; @@ -34,6 +35,7 @@ import java.util.function.DoubleSupplier; import java.util.function.Supplier; import java.util.stream.Stream; +import com.google.common.cache.Cache; import com.google.common.collect.ImmutableSet; import io.prometheus.client.Collector; import io.prometheus.client.Collector.MetricFamilySamples; @@ -42,6 +44,7 @@ import io.prometheus.client.CollectorRegistry; import io.prometheus.client.Counter; import io.prometheus.client.Histogram; import io.prometheus.client.Summary; +import io.prometheus.client.guava.cache.CacheMetricsCollector; import io.prometheus.client.hotspot.BufferPoolsExports; import io.prometheus.client.hotspot.ClassLoadingExports; import io.prometheus.client.hotspot.GarbageCollectorExports; @@ -52,6 +55,7 @@ import io.vertx.core.impl.ConcurrentHashSet; /** The Prometheus metrics system. */ public class PrometheusMetricsSystem implements ObservableMetricsSystem { + private static final List EXTERNAL_SUMMARY_LABELS = List.of("quantile"); private final Map> collectors = new ConcurrentHashMap<>(); private final CollectorRegistry registry = new CollectorRegistry(true); @@ -60,6 +64,9 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { private final Map> cachedTimers = new ConcurrentHashMap<>(); private final Set totalSuffixedCounters = new ConcurrentHashSet<>(); + private final Map guavaCacheCollectors = + new ConcurrentHashMap<>(); + private final Set guavaCacheNames = new ConcurrentHashSet<>(); private final Set enabledCategories; private final boolean timersEnabled; @@ -78,12 +85,16 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { /** Init. */ public void init() { - addCollector(StandardMetricCategory.PROCESS, StandardExports::new); - addCollector(StandardMetricCategory.JVM, MemoryPoolsExports::new); - addCollector(StandardMetricCategory.JVM, BufferPoolsExports::new); - addCollector(StandardMetricCategory.JVM, GarbageCollectorExports::new); - addCollector(StandardMetricCategory.JVM, ThreadExports::new); - addCollector(StandardMetricCategory.JVM, ClassLoadingExports::new); + if (isCategoryEnabled(StandardMetricCategory.PROCESS)) { + registerCollector(StandardMetricCategory.PROCESS, new StandardExports()); + } + if (isCategoryEnabled(StandardMetricCategory.JVM)) { + registerCollector(StandardMetricCategory.JVM, new MemoryPoolsExports()); + registerCollector(StandardMetricCategory.JVM, new BufferPoolsExports()); + registerCollector(StandardMetricCategory.JVM, new GarbageCollectorExports()); + registerCollector(StandardMetricCategory.JVM, new ThreadExports()); + registerCollector(StandardMetricCategory.JVM, new ClassLoadingExports()); + } } @Override @@ -103,7 +114,7 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { (k) -> { if (isCategoryEnabled(category)) { final Counter counter = Counter.build(metricName, help).labelNames(labelNames).create(); - addCollectorUnchecked(category, counter); + registerCollector(category, counter); return new PrometheusCounter(counter); } else { return NoOpMetricsSystem.getCounterLabelledMetric(labelNames.length); @@ -132,7 +143,7 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { .quantile(1.0, 0) .labelNames(labelNames) .create(); - addCollectorUnchecked(category, summary); + registerCollector(category, summary); return new PrometheusTimer(summary); } else { return NoOpMetricsSystem.getOperationTimerLabelledMetric(labelNames.length); @@ -153,7 +164,7 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { if (timersEnabled && isCategoryEnabled(category)) { final Histogram histogram = Histogram.build(metricName, help).labelNames(labelNames).buckets(1D).create(); - addCollectorUnchecked(category, histogram); + registerCollector(category, histogram); return new PrometheusSimpleTimer(histogram); } else { return NoOpMetricsSystem.getOperationTimerLabelledMetric(labelNames.length); @@ -170,7 +181,61 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { final String metricName = convertToPrometheusName(category, name); if (isCategoryEnabled(category)) { final Collector collector = new CurrentValueCollector(metricName, help, valueSupplier); - addCollectorUnchecked(category, collector); + registerCollector(category, collector); + } + } + + @Override + public void trackExternalSummary( + final MetricCategory category, + final String name, + final String help, + final Supplier summarySupplier) { + if (isCategoryEnabled(category)) { + final var externalSummaryCollector = + new Collector() { + @Override + public List collect() { + final var externalSummary = summarySupplier.get(); + + final var quantileValues = + externalSummary.quantiles().stream() + .map( + quantile -> + new Sample( + name, + EXTERNAL_SUMMARY_LABELS, + List.of(Double.toString(quantile.quantile())), + quantile.value())) + .toList(); + + return List.of( + new MetricFamilySamples( + name, Type.SUMMARY, "RocksDB histogram for " + name, quantileValues)); + } + }; + + registerCollector(category, externalSummaryCollector); + } + } + + @Override + public void createGuavaCacheCollector( + final MetricCategory category, final String name, final Cache cache) { + if (isCategoryEnabled(category)) { + if (guavaCacheNames.contains(name)) { + throw new IllegalStateException("Cache already registered: " + name); + } + guavaCacheNames.add(name); + final var guavaCacheCollector = + guavaCacheCollectors.computeIfAbsent( + category, + unused -> { + final var cmc = new CacheMetricsCollector(); + registerCollector(category, cmc); + return cmc; + }); + guavaCacheCollector.addCache(name, cache); } } @@ -183,46 +248,33 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { final String metricName = convertToPrometheusName(category, name); if (isCategoryEnabled(category)) { final PrometheusGauge gauge = new PrometheusGauge(metricName, help, List.of(labelNames)); - addCollectorUnchecked(category, gauge); + registerCollector(category, gauge); return gauge; } return NoOpMetricsSystem.getLabelledGauge(labelNames.length); } - /** - * Add collector. - * - * @param category the category - * @param metricSupplier the metric supplier - */ - public void addCollector( - final MetricCategory category, final Supplier metricSupplier) { - if (isCategoryEnabled(category)) { - addCollectorUnchecked(category, metricSupplier.get()); - } - } - - private void addCollectorUnchecked(final MetricCategory category, final Collector metric) { - final Collection metrics = + private void registerCollector(final MetricCategory category, final Collector collector) { + final Collection categoryCollectors = this.collectors.computeIfAbsent( category, key -> Collections.newSetFromMap(new ConcurrentHashMap<>())); final List newSamples = - metric.collect().stream().map(metricFamilySamples -> metricFamilySamples.name).toList(); + collector.collect().stream().map(metricFamilySamples -> metricFamilySamples.name).toList(); - metrics.stream() + categoryCollectors.stream() .filter( - collector -> - collector.collect().stream() + c -> + c.collect().stream() .anyMatch(metricFamilySamples -> newSamples.contains(metricFamilySamples.name))) .findFirst() .ifPresent( - collector -> { - metrics.remove(collector); - registry.unregister(collector); + c -> { + categoryCollectors.remove(c); + registry.unregister(c); }); - metrics.add(metric.register(registry)); + categoryCollectors.add(collector.register(registry)); } @Override @@ -237,6 +289,16 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { return collectors.keySet().stream().flatMap(this::streamObservations); } + @Override + public void shutdown() { + registry.clear(); + collectors.clear(); + cachedCounters.clear(); + cachedTimers.clear(); + guavaCacheCollectors.clear(); + guavaCacheNames.clear(); + } + private Stream convertSamplesToObservations( final MetricCategory category, final MetricFamilySamples familySamples) { return familySamples.samples.stream() diff --git a/metrics/core/src/test-support/java/org/hyperledger/besu/metrics/StubMetricsSystem.java b/metrics/core/src/test-support/java/org/hyperledger/besu/metrics/StubMetricsSystem.java index 2e0ea006db..13f16ef0a0 100644 --- a/metrics/core/src/test-support/java/org/hyperledger/besu/metrics/StubMetricsSystem.java +++ b/metrics/core/src/test-support/java/org/hyperledger/besu/metrics/StubMetricsSystem.java @@ -18,6 +18,7 @@ import static java.util.Arrays.asList; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; @@ -29,8 +30,11 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.DoubleSupplier; +import java.util.function.Supplier; import java.util.stream.Stream; +import com.google.common.cache.Cache; + public class StubMetricsSystem implements ObservableMetricsSystem { private final Map counters = new HashMap<>(); @@ -84,6 +88,13 @@ public class StubMetricsSystem implements ObservableMetricsSystem { return labelValues -> NoOpMetricsSystem.NO_OP_OPERATION_TIMER; } + @Override + public void trackExternalSummary( + final MetricCategory category, + final String name, + final String help, + final Supplier summarySupplier) {} + @Override public void createGauge( final MetricCategory category, @@ -93,6 +104,10 @@ public class StubMetricsSystem implements ObservableMetricsSystem { gauges.put(name, valueSupplier); } + @Override + public void createGuavaCacheCollector( + final MetricCategory category, final String name, final Cache cache) {} + public double getGaugeValue(final String name) { final DoubleSupplier gauge = gauges.get(name); if (gauge == null) { @@ -116,6 +131,12 @@ public class StubMetricsSystem implements ObservableMetricsSystem { return Collections.emptySet(); } + @Override + public void shutdown() { + counters.clear(); + gauges.clear(); + } + public static class StubLabelledCounter implements LabelledMetric { private final Map, StubCounter> metrics = new HashMap<>(); diff --git a/metrics/rocksdb/build.gradle b/metrics/rocksdb/build.gradle index d46488daf8..de312ab708 100644 --- a/metrics/rocksdb/build.gradle +++ b/metrics/rocksdb/build.gradle @@ -40,7 +40,5 @@ dependencies { implementation project(':metrics:core') implementation project(':plugin-api') - implementation 'com.google.guava:guava' - implementation 'io.prometheus:simpleclient' implementation 'org.rocksdb:rocksdbjni' } diff --git a/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java b/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java index edb837985d..c3c5fef2c1 100644 --- a/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java +++ b/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java @@ -16,15 +16,14 @@ package org.hyperledger.besu.metrics.rocksdb; import static org.hyperledger.besu.metrics.BesuMetricCategory.KVSTORE_ROCKSDB_STATS; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary.Quantile; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Locale; -import io.prometheus.client.Collector; import org.rocksdb.HistogramData; import org.rocksdb.HistogramType; import org.rocksdb.Statistics; @@ -32,22 +31,9 @@ import org.rocksdb.TickerType; /** The Rocks db stats. */ public class RocksDBStats { - - /** The Labels. */ - static final List LABELS = Collections.singletonList("quantile"); - - /** The Label 50. */ - static final List LABEL_50 = Collections.singletonList("0.5"); - - /** The Label 95. */ - static final List LABEL_95 = Collections.singletonList("0.95"); - - /** The Label 99. */ - static final List LABEL_99 = Collections.singletonList("0.99"); - - /** The constant TICKERS. */ + /** The constant TICKER_TYPES. */ // Tickers - RocksDB equivalent of counters - static final TickerType[] TICKERS = { + static final TickerType[] TICKER_TYPES = { TickerType.BLOCK_CACHE_ADD, TickerType.BLOCK_CACHE_HIT, TickerType.BLOCK_CACHE_ADD_FAILURES, @@ -133,9 +119,9 @@ public class RocksDBStats { TickerType.NUMBER_MULTIGET_KEYS_FOUND, }; - /** The constant HISTOGRAMS. */ + /** The constant HISTOGRAM_TYPES. */ // Histograms - treated as prometheus summaries - static final HistogramType[] HISTOGRAMS = { + static final HistogramType[] HISTOGRAM_TYPES = { HistogramType.DB_GET, HistogramType.DB_WRITE, HistogramType.COMPACTION_TIME, @@ -175,47 +161,40 @@ public class RocksDBStats { * @param category the category */ public static void registerRocksDBMetrics( - final Statistics stats, - final PrometheusMetricsSystem metricsSystem, - final MetricCategory category) { - if (!metricsSystem.isCategoryEnabled(category)) { - return; - } - for (final TickerType ticker : TICKERS) { - final String promCounterName = ticker.name().toLowerCase(Locale.ROOT); + final Statistics stats, final MetricsSystem metricsSystem, final MetricCategory category) { + + for (final var tickerType : TICKER_TYPES) { + final String promCounterName = tickerType.name().toLowerCase(Locale.ROOT); metricsSystem.createLongGauge( category, promCounterName, - "RocksDB reported statistics for " + ticker.name(), - () -> stats.getTickerCount(ticker)); + "RocksDB reported statistics for " + tickerType.name(), + () -> stats.getTickerCount(tickerType)); } - for (final HistogramType histogram : HISTOGRAMS) { - metricsSystem.addCollector(category, () -> histogramToCollector(stats, histogram)); + for (final var histogramType : HISTOGRAM_TYPES) { + + metricsSystem.trackExternalSummary( + KVSTORE_ROCKSDB_STATS, + KVSTORE_ROCKSDB_STATS.getName() + "_" + histogramType.name().toLowerCase(Locale.ROOT), + "RocksDB histogram for " + histogramType.name(), + () -> provideExternalSummary(stats, histogramType)); } } - private static Collector histogramToCollector( - final Statistics stats, final HistogramType histogram) { - return new Collector() { - final String metricName = - KVSTORE_ROCKSDB_STATS.getName() + "_" + histogram.name().toLowerCase(Locale.ROOT); + private static ExternalSummary provideExternalSummary( + final Statistics stats, final HistogramType histogramType) { + + final HistogramData data = stats.getHistogramData(histogramType); - @Override - public List collect() { - final HistogramData data = stats.getHistogramData(histogram); - return Collections.singletonList( - new MetricFamilySamples( - metricName, - Type.SUMMARY, - "RocksDB histogram for " + metricName, - Arrays.asList( - new MetricFamilySamples.Sample(metricName, LABELS, LABEL_50, data.getMedian()), - new MetricFamilySamples.Sample( - metricName, LABELS, LABEL_95, data.getPercentile95()), - new MetricFamilySamples.Sample( - metricName, LABELS, LABEL_99, data.getPercentile99())))); - } - }; + return new ExternalSummary( + data.getCount(), + data.getSum(), + List.of( + new Quantile(0.0, data.getMin()), + new Quantile(0.5, data.getMedian()), + new Quantile(0.95, data.getPercentile95()), + new Quantile(0.99, data.getPercentile99()), + new Quantile(1.0, data.getMax()))); } } diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index c7ec672867..2d3d5191df 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -71,7 +71,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = '8rPIE3fYl48RPRQXxYhMk559e/r+wHSKU9bGSJmruKQ=' + knownHash = 'aYWbsgPoKTGDgq9d4QUBvQEaZYbKNJGMiBufzyKnusA=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/MetricsSystem.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/MetricsSystem.java index 80e02a6dba..744bca4eb3 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/MetricsSystem.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/MetricsSystem.java @@ -15,14 +15,19 @@ package org.hyperledger.besu.plugin.services; import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.ExternalSummary; import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.hyperledger.besu.plugin.services.metrics.OperationTimer; +import java.util.Set; import java.util.function.DoubleSupplier; import java.util.function.IntSupplier; import java.util.function.LongSupplier; +import java.util.function.Supplier; + +import com.google.common.cache.Cache; /** An interface for creating various Metrics components. */ public interface MetricsSystem extends BesuService { @@ -159,4 +164,45 @@ public interface MetricsSystem extends BesuService { final LongSupplier valueSupplier) { createGauge(category, name, help, () -> (double) valueSupplier.getAsLong()); } + + /** + * Track a summary that is computed externally to this metric system. Useful when existing + * libraries calculate the summary data on their own, and we want to export that summary via the + * configured metric system. A notable example are RocksDB statistics. + * + * @param category The {@link MetricCategory} this external summary is assigned to. + * @param name A name for the metric. + * @param help A human readable description of the metric. + * @param summarySupplier A supplier to retrieve the summary data when needed. + */ + void trackExternalSummary( + MetricCategory category, String name, String help, Supplier summarySupplier); + + /** + * Collect metrics from Guava cache. + * + * @param category The {@link MetricCategory} this Guava cache is assigned to. + * @param name the name to identify this Guava cache, must be unique. + * @param cache the Guava cache + */ + void createGuavaCacheCollector(MetricCategory category, String name, Cache cache); + + /** + * Provides an immutable view into the metric categories enabled for metric collection. + * + * @return the set of enabled metric categories. + */ + Set getEnabledCategories(); + + /** + * Checks if a particular category of metrics is enabled. + * + * @param category the category to check + * @return true if the category is enabled, false otherwise + */ + default boolean isCategoryEnabled(final MetricCategory category) { + return getEnabledCategories().stream() + .map(MetricCategory::getName) + .anyMatch(category.getName()::equals); + } } diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/ExternalSummary.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/ExternalSummary.java new file mode 100644 index 0000000000..5e7e38b76a --- /dev/null +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/metrics/ExternalSummary.java @@ -0,0 +1,36 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.services.metrics; + +import java.util.List; + +/** + * Record of summary data the is kept outside the metric system. Useful when existing libraries + * calculate the summary data on their own, and we want to export that summary via the configured + * metric system. A notable example are RocksDB statistics. + * + * @param count the number of observations + * @param sum the sum of all the observations + * @param quantiles a list of quantiles with values + */ +public record ExternalSummary(long count, double sum, List quantiles) { + /** + * Represent a single quantile and its value + * + * @param quantile the quantile + * @param value the value + */ + public record Quantile(double quantile, double value) {} +} diff --git a/plugins/rocksdb/build.gradle b/plugins/rocksdb/build.gradle index d487cf9824..a2315398e1 100644 --- a/plugins/rocksdb/build.gradle +++ b/plugins/rocksdb/build.gradle @@ -46,7 +46,6 @@ dependencies { implementation 'com.google.guava:guava' implementation 'info.picocli:picocli' implementation 'io.opentelemetry:opentelemetry-api' - implementation 'io.prometheus:simpleclient' implementation 'io.tmio:tuweni-bytes' implementation 'org.rocksdb:rocksdbjni' implementation project(path: ':ethereum:core') diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java index bf781d300a..f5c666184b 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBMetricsFactory.java @@ -15,7 +15,6 @@ package org.hyperledger.besu.plugin.services.storage.rocksdb; import org.hyperledger.besu.metrics.BesuMetricCategory; -import org.hyperledger.besu.metrics.prometheus.PrometheusMetricsSystem; import org.hyperledger.besu.metrics.rocksdb.RocksDBStats; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; @@ -107,10 +106,7 @@ public class RocksDBMetricsFactory { "database") .labels(rocksDbConfiguration.getLabel()); - if (metricsSystem instanceof PrometheusMetricsSystem) { - RocksDBStats.registerRocksDBMetrics( - stats, (PrometheusMetricsSystem) metricsSystem, statsDbMetricCategory); - } + RocksDBStats.registerRocksDBMetrics(stats, metricsSystem, statsDbMetricCategory); metricsSystem.createLongGauge( rocksDbMetricCategory, From 58acfcea78164714d19dfbd373f240e6de10f68f Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 19 Nov 2024 18:19:53 +0100 Subject: [PATCH 5/7] Add support for TransactionPoolService in acceptance tests (#7861) Signed-off-by: Fabio Di Fabio --- .../besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java index 4544872d5c..6f9ed42828 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java @@ -75,6 +75,7 @@ import org.hyperledger.besu.plugin.services.TransactionSelectionService; import org.hyperledger.besu.plugin.services.TransactionSimulationService; import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin; +import org.hyperledger.besu.plugin.services.transactionpool.TransactionPoolService; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.services.BesuEventsImpl; import org.hyperledger.besu.services.BesuPluginContextImpl; @@ -85,6 +86,7 @@ import org.hyperledger.besu.services.PrivacyPluginServiceImpl; import org.hyperledger.besu.services.RpcEndpointServiceImpl; import org.hyperledger.besu.services.SecurityModuleServiceImpl; import org.hyperledger.besu.services.StorageServiceImpl; +import org.hyperledger.besu.services.TransactionPoolServiceImpl; import org.hyperledger.besu.services.TransactionPoolValidatorServiceImpl; import org.hyperledger.besu.services.TransactionSelectionServiceImpl; import org.hyperledger.besu.services.TransactionSimulationServiceImpl; @@ -215,6 +217,9 @@ public class ThreadBesuNodeRunner implements BesuNodeRunner { besuController.getTransactionPool(), besuController.getSyncState(), besuController.getProtocolContext().getBadBlockManager())); + besuPluginContext.addService( + TransactionPoolService.class, + new TransactionPoolServiceImpl(besuController.getTransactionPool())); component.rpcEndpointService().init(runner.getInProcessRpcMethods()); From 2384c09a82b0b6ef69a54cbe6a1c935d666a7f1b Mon Sep 17 00:00:00 2001 From: Vaidik Date: Wed, 20 Nov 2024 02:14:55 +0530 Subject: [PATCH 6/7] Fix #7810 (#7867) * fix-#7810 Signed-off-by: vaidikcode * fix-#7810 Signed-off-by: vaidikcode --------- Co-authored-by: Sally MacFarlane --- .../besu/controller/BesuControllerBuilder.java | 11 +---------- .../ConsensusScheduleBesuControllerBuilder.java | 5 ----- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index b3cbab5cc9..831429a02f 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -625,7 +625,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides ethereumWireProtocolConfiguration.isLegacyEth64ForkIdEnabled()); final EthPeers ethPeers = new EthPeers( - getSupportedProtocol(), + EthProtocol.NAME, currentProtocolSpecSupplier, clock, metricsSystem, @@ -1022,15 +1022,6 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides final WorldStateArchive worldStateArchive, final ProtocolSchedule protocolSchedule); - /** - * Gets supported protocol. - * - * @return the supported protocol - */ - protected String getSupportedProtocol() { - return EthProtocol.NAME; - } - /** * Create eth protocol manager eth protocol manager. * diff --git a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java index c70367aed6..31f84b6ae0 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java @@ -228,11 +228,6 @@ public class ConsensusScheduleBesuControllerBuilder extends BesuControllerBuilde besuControllerBuilderSchedule.get(GENESIS_BLOCK_NUMBER).validateContext(context); } - @Override - protected String getSupportedProtocol() { - return besuControllerBuilderSchedule.get(0L).getSupportedProtocol(); - } - @Override protected EthProtocolManager createEthProtocolManager( final ProtocolContext protocolContext, From 8b7ca2017697f08bbf49edef2c80e10513bdd3a7 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Wed, 20 Nov 2024 08:16:33 +1000 Subject: [PATCH 7/7] eth_call overrides (#7801) * add state and account overrides Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane Co-authored-by: Fabio Di Fabio --- CHANGELOG.md | 1 + .../api/jsonrpc/internal/methods/EthCall.java | 20 ++ .../internal/methods/TraceCallMany.java | 7 +- .../jsonrpc/internal/methods/EthCallTest.java | 81 ++++++-- .../jsonrpc/eth/eth_call_overrides_empty.json | 24 +++ .../jsonrpc/eth/eth_call_stateOverride.json | 32 +++ ...all_stateOverride_insufficientBalance.json | 34 ++++ .../transaction/TransactionSimulator.java | 66 +++++- .../besu/ethereum/util/AccountOverride.java | 147 ++++++++++++++ .../ethereum/util/AccountOverrideMap.java | 27 +++ .../transaction/TransactionSimulatorTest.java | 41 ++++ .../util/AccountOverrideParameterTest.java | 188 ++++++++++++++++++ 12 files changed, 647 insertions(+), 21 deletions(-) create mode 100644 ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_overrides_empty.json create mode 100644 ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride.json create mode 100644 ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride_insufficientBalance.json create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverride.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverrideMap.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/util/AccountOverrideParameterTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index a0a53280fe..2efa312d26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Update Java dependencies [#7786](https://github.com/hyperledger/besu/pull/7786) - Add a method to get all the transaction in the pool, to the `TransactionPoolService`, to easily access the transaction pool content from plugins [#7813](https://github.com/hyperledger/besu/pull/7813) - Add a method to check if a metric category is enabled to the plugin API [#7832](https://github.com/hyperledger/besu/pull/7832) +- Add account and state overrides to `eth_call` and `eth_estimateGas` [#7801](https://github.com/hyperledger/besu/pull/7801) ### Bug fixes - Fix registering new metric categories from plugins [#7825](https://github.com/hyperledger/besu/pull/7825) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java index 0e0318f9c2..5f736e8d5a 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcErrorConverter; import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcRequestException; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameterOrBlockHash; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException; @@ -40,8 +41,13 @@ import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; import org.hyperledger.besu.ethereum.transaction.TransactionSimulatorResult; +import org.hyperledger.besu.ethereum.util.AccountOverrideMap; import org.hyperledger.besu.evm.tracing.OperationTracer; +import java.util.Optional; + +import com.google.common.annotations.VisibleForTesting; + public class EthCall extends AbstractBlockParameterOrBlockHashMethod { private final TransactionSimulator transactionSimulator; @@ -81,10 +87,13 @@ public class EthCall extends AbstractBlockParameterOrBlockHashMethod { protected Object resultByBlockHeader( final JsonRpcRequestContext request, final BlockHeader header) { JsonCallParameter callParams = JsonCallParameterUtil.validateAndGetCallParams(request); + Optional maybeStateOverrides = getAddressAccountOverrideMap(request); + // TODO implement for block overrides return transactionSimulator .process( callParams, + maybeStateOverrides, buildTransactionValidationParams(header, callParams), OperationTracer.NO_TRACING, (mutableWorldState, transactionSimulatorResult) -> @@ -108,6 +117,17 @@ public class EthCall extends AbstractBlockParameterOrBlockHashMethod { .orElse(errorResponse(request, INTERNAL_ERROR)); } + @VisibleForTesting + protected Optional getAddressAccountOverrideMap( + final JsonRpcRequestContext request) { + try { + return request.getOptionalParameter(2, AccountOverrideMap.class); + } catch (JsonRpcParameterException e) { + throw new InvalidJsonRpcRequestException( + "Invalid account overrides parameter (index 2)", RpcErrorType.INVALID_CALL_PARAMS, e); + } + } + @Override public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { return (JsonRpcResponse) handleParamTypes(requestContext); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java index 10d4018bce..1601241db6 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java @@ -160,7 +160,12 @@ public class TraceCallMany extends TraceCall implements JsonRpcMethod { new DebugOperationTracer(buildTraceOptions(traceTypes), false); final Optional maybeSimulatorResult = transactionSimulator.processWithWorldUpdater( - callParameter, buildTransactionValidationParams(), tracer, header, worldUpdater); + callParameter, + Optional.empty(), + buildTransactionValidationParams(), + tracer, + header, + worldUpdater); LOG.trace("Executing {} call for transaction {}", traceTypeParameter, callParameter); if (maybeSimulatorResult.isEmpty()) { diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java index 8b8ff67cd8..1ba9839b66 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java @@ -51,6 +51,8 @@ import org.hyperledger.besu.ethereum.transaction.CallParameter; import org.hyperledger.besu.ethereum.transaction.PreCloseStateHandler; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; import org.hyperledger.besu.ethereum.transaction.TransactionSimulatorResult; +import org.hyperledger.besu.ethereum.util.AccountOverride; +import org.hyperledger.besu.ethereum.util.AccountOverrideMap; import java.util.Optional; @@ -92,6 +94,33 @@ public class EthCallTest { assertThat(method.getName()).isEqualTo("eth_call"); } + @Test + public void noAccountOverrides() { + final JsonRpcRequestContext request = ethCallRequest(callParameter(), "latest"); + Optional overrideMap = method.getAddressAccountOverrideMap(request); + assertThat(overrideMap.isPresent()).isFalse(); + } + + @Test + public void someAccountOverrides() { + AccountOverrideMap expectedOverrides = new AccountOverrideMap(); + AccountOverride override = new AccountOverride.Builder().withNonce(88L).build(); + final Address address = Address.fromHexString("0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3"); + expectedOverrides.put(address, override); + + final JsonRpcRequestContext request = + ethCallRequestWithStateOverrides(callParameter(), "latest", expectedOverrides); + + Optional maybeOverrideMap = method.getAddressAccountOverrideMap(request); + assertThat(maybeOverrideMap.isPresent()).isTrue(); + AccountOverrideMap overrideMap = maybeOverrideMap.get(); + assertThat(overrideMap.keySet()).hasSize(1); + assertThat(overrideMap.values()).hasSize(1); + + assertThat(overrideMap).containsKey(address); + assertThat(overrideMap).containsValue(override); + } + @Test public void shouldReturnInternalErrorWhenProcessorReturnsEmpty() { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "latest"); @@ -99,7 +128,7 @@ public class EthCallTest { when(blockchainQueries.getBlockchain()).thenReturn(blockchain); when(blockchain.getChainHead()).thenReturn(chainHead); - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), any(), any(), any(), any(), any())) .thenReturn(Optional.empty()); final BlockHeader blockHeader = mock(BlockHeader.class); @@ -109,7 +138,7 @@ public class EthCallTest { final JsonRpcResponse response = method.response(request); assertThat(response).usingRecursiveComparison().isEqualTo(expectedResponse); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), any(), any(), any(), any(), any()); } @Test @@ -130,12 +159,13 @@ public class EthCallTest { when(result.isSuccessful()).thenReturn(true); when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.getOutput()).thenReturn(Bytes.of()); - verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); + verify(transactionSimulator) + .process( + eq(callParameter), eq(Optional.empty()), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); assertThat(response).usingRecursiveComparison().isEqualTo(expectedResponse); - verify(transactionSimulator).process(eq(callParameter), any(), any(), any(), any()); } @Test @@ -158,7 +188,8 @@ public class EthCallTest { when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.getOutput()).thenReturn(Bytes.of(1)); verify(transactionSimulator) - .process(eq(callParameter()), any(), any(), mapperCaptor.capture(), any()); + .process( + eq(callParameter()), eq(Optional.empty()), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); @@ -196,7 +227,8 @@ public class EthCallTest { when(result.isSuccessful()).thenReturn(false); when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.result()).thenReturn(processingResult); - verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); + verify(transactionSimulator) + .process(any(), eq(Optional.empty()), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); @@ -235,7 +267,8 @@ public class EthCallTest { when(result.isSuccessful()).thenReturn(false); when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.result()).thenReturn(processingResult); - verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); + verify(transactionSimulator) + .process(any(), eq(Optional.empty()), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); @@ -277,7 +310,8 @@ public class EthCallTest { when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.result()).thenReturn(processingResult); - verify(transactionSimulator).process(any(), any(), any(), mapperCaptor.capture(), any()); + verify(transactionSimulator) + .process(any(), eq(Optional.empty()), any(), any(), mapperCaptor.capture(), any()); assertThat(mapperCaptor.getValue().apply(mock(MutableWorldState.class), Optional.of(result))) .isEqualTo(Optional.of(expectedResponse)); @@ -291,7 +325,7 @@ public class EthCallTest { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "latest"); when(blockchainQueries.getBlockchain()).thenReturn(blockchain); when(blockchain.getChainHead()).thenReturn(chainHead); - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), eq(Optional.empty()), any(), any(), any(), any())) .thenReturn(Optional.empty()); final BlockHeader blockHeader = mock(BlockHeader.class); @@ -301,7 +335,7 @@ public class EthCallTest { method.response(request); verify(blockchainQueries, atLeastOnce()).getBlockchain(); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), eq(Optional.empty()), any(), any(), any(), any()); } @Test @@ -315,7 +349,7 @@ public class EthCallTest { method.response(request); verify(blockchainQueries).getBlockHeaderByHash(eq(Hash.ZERO)); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), eq(Optional.empty()), any(), any(), any(), any()); } @Test @@ -323,13 +357,13 @@ public class EthCallTest { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "safe"); when(blockchainQueries.getBlockHeaderByHash(Hash.ZERO)).thenReturn(Optional.of(blockHeader)); when(blockchainQueries.safeBlockHeader()).thenReturn(Optional.of(blockHeader)); - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), eq(Optional.empty()), any(), any(), any(), any())) .thenReturn(Optional.empty()); method.response(request); verify(blockchainQueries).getBlockHeaderByHash(Hash.ZERO); verify(blockchainQueries).safeBlockHeader(); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), eq(Optional.empty()), any(), any(), any(), any()); } @Test @@ -337,13 +371,13 @@ public class EthCallTest { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "finalized"); when(blockchainQueries.getBlockHeaderByHash(Hash.ZERO)).thenReturn(Optional.of(blockHeader)); when(blockchainQueries.finalizedBlockHeader()).thenReturn(Optional.of(blockHeader)); - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), eq(Optional.empty()), any(), any(), any(), any())) .thenReturn(Optional.empty()); method.response(request); verify(blockchainQueries).getBlockHeaderByHash(Hash.ZERO); verify(blockchainQueries).finalizedBlockHeader(); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), eq(Optional.empty()), any(), any(), any(), any()); } @Test @@ -353,13 +387,13 @@ public class EthCallTest { when(blockchainQueries.getBlockHashByNumber(anyLong())).thenReturn(Optional.of(Hash.ZERO)); when(blockchainQueries.getBlockHeaderByHash(Hash.ZERO)) .thenReturn(Optional.of(mock(BlockHeader.class))); - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), eq(Optional.empty()), any(), any(), any(), any())) .thenReturn(Optional.empty()); method.response(request); verify(blockchainQueries).getBlockHeaderByHash(eq(Hash.ZERO)); - verify(transactionSimulator).process(any(), any(), any(), any(), any()); + verify(transactionSimulator).process(any(), eq(Optional.empty()), any(), any(), any(), any()); } @Test @@ -431,7 +465,7 @@ public class EthCallTest { .build(); verify(transactionSimulator) - .process(any(), eq(transactionValidationParams), any(), any(), any()); + .process(any(), eq(Optional.empty()), eq(transactionValidationParams), any(), any(), any()); } private JsonCallParameter callParameter() { @@ -458,8 +492,17 @@ public class EthCallTest { new JsonRpcRequest("2.0", "eth_call", new Object[] {callParameter, blockNumberInHex})); } + private JsonRpcRequestContext ethCallRequestWithStateOverrides( + final CallParameter callParameter, + final String blockNumberInHex, + final AccountOverrideMap overrides) { + return new JsonRpcRequestContext( + new JsonRpcRequest( + "2.0", "eth_call", new Object[] {callParameter, blockNumberInHex, overrides})); + } + private void mockTransactionProcessorSuccessResult(final JsonRpcResponse jsonRpcResponse) { - when(transactionSimulator.process(any(), any(), any(), any(), any())) + when(transactionSimulator.process(any(), eq(Optional.empty()), any(), any(), any(), any())) .thenReturn(Optional.of(jsonRpcResponse)); } } diff --git a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_overrides_empty.json b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_overrides_empty.json new file mode 100644 index 0000000000..6280773f3d --- /dev/null +++ b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_overrides_empty.json @@ -0,0 +1,24 @@ +{ + "request": { + "id": 3, + "jsonrpc": "2.0", + "method": "eth_call", + "params": [ + { + "to": "0x6295ee1b4f6dd65047762f924ecd367c17eabf8f", + "from": "a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "data": "0x12a7b914" + }, + "latest", + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": {} + } + ] + }, + "response": { + "jsonrpc": "2.0", + "id": 3, + "result": "0x0000000000000000000000000000000000000000000000000000000000000001" + }, + "statusCode": 200 +} \ No newline at end of file diff --git a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride.json b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride.json new file mode 100644 index 0000000000..762202f96b --- /dev/null +++ b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride.json @@ -0,0 +1,32 @@ +{ + "request": { + "id": 3, + "jsonrpc": "2.0", + "method": "eth_call", + "params": [ + { + "to": "0x6295ee1b4f6dd65047762f924ecd367c17eabf8f", + "from": "a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "data": "0x12a7b914" + }, + "latest", + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0xde0b6b3a7640000", + "nonce": 88 + }, + "0xb9741079a300Cb3B8f324CdDB847c0d1d273a05E": { + "stateDiff": { + "0x1cf7945003fc5b59d2f6736f0704557aa805c4f2844084ccd1173b8d56946962": "0x000000000000000000000000000000000000000000000000000000110ed03bf7" + } + } + } + ] + }, + "response": { + "jsonrpc": "2.0", + "id": 3, + "result": "0x0000000000000000000000000000000000000000000000000000000000000001" + }, + "statusCode": 200 +} \ No newline at end of file diff --git a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride_insufficientBalance.json b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride_insufficientBalance.json new file mode 100644 index 0000000000..c3832c5689 --- /dev/null +++ b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_stateOverride_insufficientBalance.json @@ -0,0 +1,34 @@ +{ + "request": { + "id": 3, + "jsonrpc": "2.0", + "method": "eth_call", + "params": [ + { + "to": "0x6295ee1b4f6dd65047762f924ecd367c17eabf8f", + "from": "a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "value": "0x000002" + }, + "latest", + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x000001" + }, + "0xb9741079a300Cb3B8f324CdDB847c0d1d273a05E": { + "stateDiff": { + "0x1cf7945003fc5b59d2f6736f0704557aa805c4f2844084ccd1173b8d56946962": "0x000000000000000000000000000000000000000000000000000000110ed03bf7" + } + } + } + ] + }, + "response": { + "jsonrpc": "2.0", + "id": 3, + "error" : { + "code" : -32004, + "message" : "Upfront cost exceeds account balance" + } + }, + "statusCode": 200 +} \ No newline at end of file diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java index 321dc965ec..b0a7fa4325 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java @@ -34,10 +34,13 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; +import org.hyperledger.besu.ethereum.util.AccountOverride; +import org.hyperledger.besu.ethereum.util.AccountOverrideMap; import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.tracing.OperationTracer; import org.hyperledger.besu.evm.worldstate.WorldUpdater; @@ -46,8 +49,10 @@ import java.util.Optional; import java.util.function.Supplier; import javax.annotation.Nonnull; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Suppliers; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -152,6 +157,35 @@ public class TransactionSimulator { final OperationTracer operationTracer, final PreCloseStateHandler preWorldStateCloseGuard, final BlockHeader header) { + return process( + callParams, + Optional.empty(), + transactionValidationParams, + operationTracer, + preWorldStateCloseGuard, + header); + } + + /** + * Processes a transaction simulation with the provided parameters and executes pre-worldstate + * close actions. + * + * @param callParams The call parameters for the transaction. + * @param maybeStateOverrides The map of state overrides to apply to the state for this + * transaction. + * @param transactionValidationParams The validation parameters for the transaction. + * @param operationTracer The tracer for capturing operations during processing. + * @param preWorldStateCloseGuard The pre-worldstate close guard for executing pre-close actions. + * @param header The block header. + * @return An Optional containing the result of the processing. + */ + public Optional process( + final CallParameter callParams, + final Optional maybeStateOverrides, + final TransactionValidationParams transactionValidationParams, + final OperationTracer operationTracer, + final PreCloseStateHandler preWorldStateCloseGuard, + final BlockHeader header) { if (header == null) { return Optional.empty(); } @@ -169,7 +203,12 @@ public class TransactionSimulator { return preWorldStateCloseGuard.apply( ws, processWithWorldUpdater( - callParams, transactionValidationParams, operationTracer, header, updater)); + callParams, + maybeStateOverrides, + transactionValidationParams, + operationTracer, + header, + updater)); } catch (final Exception e) { return Optional.empty(); @@ -208,6 +247,7 @@ public class TransactionSimulator { @Nonnull public Optional processWithWorldUpdater( final CallParameter callParams, + final Optional maybeStateOverrides, final TransactionValidationParams transactionValidationParams, final OperationTracer operationTracer, final BlockHeader header, @@ -226,6 +266,12 @@ public class TransactionSimulator { .blockHeaderFunctions(protocolSpec.getBlockHeaderFunctions()) .buildBlockHeader(); } + if (maybeStateOverrides.isPresent()) { + for (Address accountToOverride : maybeStateOverrides.get().keySet()) { + final AccountOverride overrides = maybeStateOverrides.get().get(accountToOverride); + applyOverrides(updater.getOrCreate(accountToOverride), overrides); + } + } final Account sender = updater.get(senderAddress); final long nonce = sender != null ? sender.getNonce() : 0L; @@ -284,6 +330,24 @@ public class TransactionSimulator { return Optional.of(new TransactionSimulatorResult(transaction, result)); } + @VisibleForTesting + protected void applyOverrides(final MutableAccount account, final AccountOverride override) { + LOG.debug("applying overrides to state for account {}", account.getAddress()); + override.getNonce().ifPresent(account::setNonce); + if (override.getBalance().isPresent()) { + account.setBalance(override.getBalance().get()); + } + override.getCode().ifPresent(n -> account.setCode(Bytes.fromHexString(n))); + override + .getStateDiff() + .ifPresent( + d -> + d.forEach( + (key, value) -> + account.setStorageValue( + UInt256.fromHexString(key), UInt256.fromHexString(value)))); + } + private long calculateSimulationGasCap( final long userProvidedGasLimit, final long blockGasLimit) { final long simulationGasCap; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverride.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverride.java new file mode 100644 index 0000000000..3bae4af1d8 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverride.java @@ -0,0 +1,147 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.util; + +import org.hyperledger.besu.datatypes.Wei; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// similar to AccountDiff +// BUT +// there are more fields that need to be added +// stateDiff +// movePrecompileToAddress +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonDeserialize(builder = AccountOverride.Builder.class) +public class AccountOverride { + private static final Logger LOG = LoggerFactory.getLogger(AccountOverride.class); + + private final Optional balance; + private final Optional nonce; + private final Optional code; + private final Optional> stateDiff; + + private AccountOverride( + final Optional balance, + final Optional nonce, + final Optional code, + final Optional> stateDiff) { + this.balance = balance; + this.nonce = nonce; + this.code = code; + this.stateDiff = stateDiff; + } + + public Optional getBalance() { + return balance; + } + + public Optional getNonce() { + return nonce; + } + + public Optional getCode() { + return code; + } + + public Optional> getStateDiff() { + return stateDiff; + } + + public static class Builder { + private Optional balance = Optional.empty(); + private Optional nonce = Optional.empty(); + private Optional code = Optional.empty(); + private Optional> stateDiff = Optional.empty(); + + /** Default constructor. */ + public Builder() {} + + public Builder withBalance(final Wei balance) { + this.balance = Optional.ofNullable(balance); + return this; + } + + public Builder withNonce(final Long nonce) { + this.nonce = Optional.ofNullable(nonce); + return this; + } + + public Builder withCode(final String code) { + this.code = Optional.ofNullable(code); + return this; + } + + public Builder withStateDiff(final Map stateDiff) { + this.stateDiff = Optional.ofNullable(stateDiff); + return this; + } + + public AccountOverride build() { + return new AccountOverride(balance, nonce, code, stateDiff); + } + } + + @JsonAnySetter + public void withUnknownProperties(final String key, final Object value) { + LOG.debug( + "unknown property - {} with value - {} and type - {} caught during serialization", + key, + value, + value != null ? value.getClass() : "NULL"); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AccountOverride accountOverride = (AccountOverride) o; + return balance.equals(accountOverride.balance) + && nonce.equals(accountOverride.nonce) + && code.equals(accountOverride.code) + && stateDiff.equals(accountOverride.stateDiff); + } + + @Override + public int hashCode() { + return Objects.hash(balance, nonce, code, stateDiff); + } + + @Override + public String toString() { + return "AccountOverride{" + + "balance=" + + balance + + ", nonce=" + + nonce + + ", code=" + + code + + ", stateDiff=" + + stateDiff + + '}'; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverrideMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverrideMap.java new file mode 100644 index 0000000000..30fc808c9b --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/util/AccountOverrideMap.java @@ -0,0 +1,27 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.util; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.HashMap; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class AccountOverrideMap extends HashMap { + + public AccountOverrideMap() {} +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java index c6c676c415..0dfa7e924f 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java @@ -20,6 +20,7 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import org.hyperledger.besu.crypto.SECPSignature; @@ -47,17 +48,21 @@ import org.hyperledger.besu.ethereum.mainnet.blockhash.BlockHashProcessor; import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult.Status; +import org.hyperledger.besu.ethereum.util.AccountOverride; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.tracing.OperationTracer; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import java.math.BigInteger; +import java.util.Map; import java.util.Optional; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -100,6 +105,42 @@ public class TransactionSimulatorTest { new TransactionSimulator(blockchain, worldStateArchive, protocolSchedule, GAS_CAP); } + @Test + public void testOverrides_whenNoOverrides_noUpdates() { + MutableAccount mutableAccount = mock(MutableAccount.class); + when(mutableAccount.getAddress()).thenReturn(DEFAULT_FROM); // called from logging + AccountOverride.Builder builder = new AccountOverride.Builder(); + AccountOverride override = builder.build(); + transactionSimulator.applyOverrides(mutableAccount, override); + verify(mutableAccount).getAddress(); + verifyNoMoreInteractions(mutableAccount); + } + + @Test + public void testOverrides_whenBalanceOverrides_balanceIsUpdated() { + MutableAccount mutableAccount = mock(MutableAccount.class); + when(mutableAccount.getAddress()).thenReturn(DEFAULT_FROM); + AccountOverride.Builder builder = new AccountOverride.Builder().withBalance(Wei.of(99)); + AccountOverride override = builder.build(); + transactionSimulator.applyOverrides(mutableAccount, override); + verify(mutableAccount).setBalance(eq(Wei.of(99))); + } + + @Test + public void testOverrides_whenStateDiffOverrides_stateIsUpdated() { + MutableAccount mutableAccount = mock(MutableAccount.class); + when(mutableAccount.getAddress()).thenReturn(DEFAULT_FROM); + final String storageKey = "0x01a2"; + final String storageValue = "0x00ff"; + AccountOverride.Builder builder = + new AccountOverride.Builder().withStateDiff(Map.of(storageKey, storageValue)); + AccountOverride override = builder.build(); + transactionSimulator.applyOverrides(mutableAccount, override); + verify(mutableAccount) + .setStorageValue( + eq(UInt256.fromHexString(storageKey)), eq(UInt256.fromHexString(storageValue))); + } + @Test public void shouldReturnEmptyWhenBlockDoesNotExist() { when(blockchain.getBlockHeader(eq(1L))).thenReturn(Optional.empty()); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/util/AccountOverrideParameterTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/util/AccountOverrideParameterTest.java new file mode 100644 index 0000000000..8e7b5c3f0e --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/util/AccountOverrideParameterTest.java @@ -0,0 +1,188 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.util; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; + +import java.util.Optional; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +public class AccountOverrideParameterTest { + + private static final String ADDRESS_HEX1 = "0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3"; + private static final String ADDRESS_HEX2 = "0xd5E23607D5d73ff2293152f464C3caB005f87696"; + private static final String STORAGE_KEY = + "0x1cf7945003fc5b59d2f6736f0704557aa805c4f2844084ccd1173b8d56946962"; + private static final String STORAGE_VALUE = + "0x000000000000000000000000000000000000000000000000000000110ed03bf7"; + private static final String CODE_STRING = + "0xdbf4257000000000000000000000000000000000000000000000000000000000"; + + @Test + public void jsonDeserializesCorrectly() throws Exception { + final String json = + "{\"jsonrpc\":\"2.0\",\"method\":\"eth_call\",\"params\":[{" + + "\"from\":\"0x0\", \"to\": \"0x0\"}, " + + "\"latest\"," + + "{\"" + + ADDRESS_HEX1 + + "\":" + + "{" + + "\"balance\": \"0x01\"," + + "\"nonce\": 88" + + "}}],\"id\":1}"; + + final JsonRpcRequestContext request = new JsonRpcRequestContext(readJsonAsJsonRpcRequest(json)); + final AccountOverrideMap accountOverrideParam = + request.getRequiredParameter(2, AccountOverrideMap.class); + + final AccountOverride accountOverride = + accountOverrideParam.get(Address.fromHexString(ADDRESS_HEX1)); + + assertThat(accountOverride.getNonce()).isEqualTo(Optional.of(88L)); + assertThat(accountOverride.getBalance()).isEqualTo(Optional.of(Wei.of(1))); + assertFalse(accountOverride.getStateDiff().isPresent()); + } + + @Test + public void jsonWithCodeDeserializesCorrectly() throws Exception { + final String json = + "{\"jsonrpc\":\"2.0\",\"method\":\"eth_call\",\"params\":[{" + + "\"from\":\"0x0\", \"to\": \"0x0\"}, " + + "\"latest\"," + + "{\"" + + ADDRESS_HEX1 + + "\":" + + "{" + + "\"balance\": \"0x01\"," + + "\"code\": \"" + + CODE_STRING + + "\"" + + "}}],\"id\":1}"; + + final JsonRpcRequestContext request = new JsonRpcRequestContext(readJsonAsJsonRpcRequest(json)); + final AccountOverrideMap accountOverrideParam = + request.getRequiredParameter(2, AccountOverrideMap.class); + + final AccountOverride accountOverride = + accountOverrideParam.get(Address.fromHexString(ADDRESS_HEX1)); + + assertFalse(accountOverride.getNonce().isPresent()); + assertThat(accountOverride.getBalance()).isEqualTo(Optional.of(Wei.of(1))); + assertThat(accountOverride.getCode()).isEqualTo(Optional.of(CODE_STRING)); + assertFalse(accountOverride.getStateDiff().isPresent()); + } + + @Test + public void jsonWithStorageOverridesDeserializesCorrectly() throws Exception { + final String json = + "{\"jsonrpc\":\"2.0\",\"method\":\"eth_call\",\"params\":[{" + + "\"from\":\"0x0\", \"to\": \"0x0\"}, " + + "\"latest\"," + + "{\"" + + ADDRESS_HEX1 + + "\":" + + "{" + + "\"balance\": \"0x01\"," + + "\"nonce\": 88," + + "\"stateDiff\": {" + + "\"" + + STORAGE_KEY + + "\": \"" + + STORAGE_VALUE + + "\"" + + "}}}],\"id\":1}"; + + final JsonRpcRequestContext request = new JsonRpcRequestContext(readJsonAsJsonRpcRequest(json)); + + final AccountOverrideMap accountOverrideParam = + request.getRequiredParameter(2, AccountOverrideMap.class); + assertThat(accountOverrideParam.size()).isEqualTo(1); + + final AccountOverride accountOverride = + accountOverrideParam.get(Address.fromHexString(ADDRESS_HEX1)); + assertThat(accountOverride.getNonce()).isEqualTo(Optional.of(88L)); + + assertTrue(accountOverride.getStateDiff().isPresent()); + assertThat(accountOverride.getStateDiff().get().get(STORAGE_KEY)).isEqualTo(STORAGE_VALUE); + } + + @Test + public void jsonWithMultipleAccountOverridesDeserializesCorrectly() throws Exception { + final String json = + "{\"jsonrpc\":\"2.0\",\"method\":\"eth_call\",\"params\":[{" + + "\"from\":\"0x0\", \"to\": \"0x0\"}, " + + "\"latest\"," + + "{\"" + + ADDRESS_HEX1 + + "\":" + + "{" + + "\"balance\": \"0x01\"," + + "\"nonce\": 88," + + "\"stateDiff\": {" + + "\"" + + STORAGE_KEY + + "\": \"" + + STORAGE_VALUE + + "\"" + + "}}," + + "\"" + + ADDRESS_HEX2 + + "\":" + + "{" + + "\"balance\": \"0xFF\"," + + "\"nonce\": 99," + + "\"stateDiff\": {" + + "\"" + + STORAGE_KEY + + "\": \"" + + STORAGE_VALUE + + "\"" + + "}}}],\"id\":1}"; + + final JsonRpcRequestContext request = new JsonRpcRequestContext(readJsonAsJsonRpcRequest(json)); + + final AccountOverrideMap accountOverrideParam = + request.getRequiredParameter(2, AccountOverrideMap.class); + assertThat(accountOverrideParam.size()).isEqualTo(2); + + final AccountOverride accountOverride1 = + accountOverrideParam.get(Address.fromHexString(ADDRESS_HEX1)); + assertThat(accountOverride1.getNonce()).isEqualTo(Optional.of(88L)); + assertThat(accountOverride1.getBalance()).isEqualTo(Optional.of(Wei.fromHexString("0x01"))); + assertTrue(accountOverride1.getStateDiff().isPresent()); + assertThat(accountOverride1.getStateDiff().get().get(STORAGE_KEY)).isEqualTo(STORAGE_VALUE); + + final AccountOverride accountOverride2 = + accountOverrideParam.get(Address.fromHexString(ADDRESS_HEX2)); + assertThat(accountOverride2.getNonce()).isEqualTo(Optional.of(99L)); + assertThat(accountOverride2.getBalance()).isEqualTo(Optional.of(Wei.fromHexString("0xFF"))); + assertTrue(accountOverride2.getStateDiff().isPresent()); + assertThat(accountOverride2.getStateDiff().get().get(STORAGE_KEY)).isEqualTo(STORAGE_VALUE); + } + + private JsonRpcRequest readJsonAsJsonRpcRequest(final String json) throws java.io.IOException { + return new ObjectMapper().readValue(json, JsonRpcRequest.class); + } +}