From 423fe1d48129c13b1a04d37febba294cd6d3e682 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 9 May 2023 14:38:56 +0200 Subject: [PATCH] Layered Transaction Pool (#5290) * Introduce experimental layered transaction pool Signed-off-by: Fabio Di Fabio * new Xlayered-tx-pool flag to enabled the new tx pool Signed-off-by: Fabio Di Fabio * Move pending transaction sorter tests in the sorter folder Signed-off-by: Fabio Di Fabio * Unit tests for new and old transaction pool Signed-off-by: Fabio Di Fabio * Fix: do not decrease size when promoting ready txs Signed-off-by: Fabio Di Fabio * Fix: remove tx from orderByFee when replaced Signed-off-by: Fabio Di Fabio * Fix: decrease size when removing confirmed txs Signed-off-by: Fabio Di Fabio * Fix: always recreate orderByFee for London fee market Signed-off-by: Fabio Di Fabio * Fix: transaction removal counter when txs added to block Signed-off-by: Fabio Di Fabio * Fix: update expected nonce when demoting a prioritized transaction Signed-off-by: Fabio Di Fabio * Fix: correctly remove expected nonce entry when removing the last prioritized transaction for the sender Signed-off-by: Fabio Di Fabio * Fix NullPointerException when the replaced tx is not prioritized Signed-off-by: Fabio Di Fabio * Replace postponed with spare transactions Signed-off-by: Fabio Di Fabio * WIP Signed-off-by: Fabio Di Fabio * WIP Signed-off-by: Fabio Di Fabio * Fix merge from main Signed-off-by: Fabio Di Fabio * Fixed most tests Signed-off-by: Fabio Di Fabio * Fix more tests Signed-off-by: Fabio Di Fabio * Rename and reorg some classes Signed-off-by: Fabio Di Fabio * More renaming and code clean up Signed-off-by: Fabio Di Fabio * Refactor transaction pool metrics Signed-off-by: Fabio Di Fabio * Stats log refined Signed-off-by: Fabio Di Fabio * Cleanup unit tests Signed-off-by: Fabio Di Fabio * Improve stats log Signed-off-by: Fabio Di Fabio * Remove unnecessary test parameters Signed-off-by: Fabio Di Fabio * Fix unit test Signed-off-by: Fabio Di Fabio # Conflicts: # ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java # ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java * Cancel older block creation tasks upon receiving a new one Signed-off-by: Simon Dudley * Fixes to expected next nonce for sender Signed-off-by: Fabio Di Fabio * Fix promotion filter and use synchronized methods instead of blocks Signed-off-by: Fabio Di Fabio * Fix metrics concurrent access issue Signed-off-by: Fabio Di Fabio * Fixes Signed-off-by: Fabio Di Fabio * Configuration options for the layered txpool Signed-off-by: Fabio Di Fabio * Use long instead of Instant for PendingTransaction add time Signed-off-by: Fabio Di Fabio * Fixes Signed-off-by: Fabio Di Fabio * Move layered txpool clasess in a dedicated package Signed-off-by: Fabio Di Fabio * Fixes Signed-off-by: Fabio Di Fabio * WIP Signed-off-by: Fabio Di Fabio * Remove confirmed transaction from sparse set too Signed-off-by: Fabio Di Fabio * WIP Signed-off-by: Fabio Di Fabio * WIP Signed-off-by: Fabio Di Fabio * Fill gap on added tx Signed-off-by: Fabio Di Fabio * Fix eviction on sparse layer Signed-off-by: Fabio Di Fabio * Fix remove from ready layer Signed-off-by: Fabio Di Fabio * Fix remove from sparse layer Signed-off-by: Fabio Di Fabio * Fix for block added and confirmed txs Signed-off-by: Fabio Di Fabio * Fixes to sparse layer Signed-off-by: Fabio Di Fabio * Fix the filling of the gap when adding transactions Signed-off-by: Fabio Di Fabio * Layered pending transactions test and fixes Signed-off-by: Fabio Di Fabio * Distinguish between layer and comulative space used Signed-off-by: Fabio Di Fabio * unit tests for expected next nonce for sender Signed-off-by: Fabio Di Fabio * Adding test for transaction selection Signed-off-by: Fabio Di Fabio * Re-enable prioritized transaction tests and more fixes Signed-off-by: Fabio Di Fabio * log stats Signed-off-by: Fabio Di Fabio * syncronized some methods, metrics update and dump transactions for replay Signed-off-by: Fabio Di Fabio * Test that replay tx and fix for tx replacement across layers Signed-off-by: Fabio Di Fabio * Add missing copyright and fix replay test Signed-off-by: Fabio Di Fabio * Add consistency check asserts Signed-off-by: Fabio Di Fabio * Fix ready internalRemove Signed-off-by: Fabio Di Fabio * Metrics tests improvements Signed-off-by: Fabio Di Fabio * WIP: Transaction memory size estimation Signed-off-by: Fabio Di Fabio * Complete pending transaction memory used computation Signed-off-by: Fabio Di Fabio * Improve metrics Signed-off-by: Fabio Di Fabio * Rename to specify that the limit is per layer Signed-off-by: Fabio Di Fabio * Update metric names in tests Signed-off-by: Fabio Di Fabio * Adjust tx layer max capacity according to new tx memory size calculation Signed-off-by: Fabio Di Fabio * Fix legacy transaction expiration tests Signed-off-by: Fabio Di Fabio * Fix IndexOutOfBoundsException in sparse layer Signed-off-by: Fabio Di Fabio * Unique senders metric Signed-off-by: Fabio Di Fabio * Ignore ReplayTest by default, fix logging of stats Signed-off-by: Fabio Di Fabio * Log for replay renamings Signed-off-by: Fabio Di Fabio * Document howto generate txpool replay Signed-off-by: Fabio Di Fabio * Reduce max layer capacity Signed-off-by: Fabio Di Fabio * exclude transaction replay resource Signed-off-by: Fabio Di Fabio * Improve compareByFee when effectivePriorityFee is 0 Signed-off-by: Fabio Di Fabio * More debug logs during transaction selection for a block Signed-off-by: Fabio Di Fabio * Use only one thread for building blocks so there is no risk of overlapping Signed-off-by: Fabio Di Fabio * Improve transaction trace log making wei human readable Signed-off-by: Fabio Di Fabio * Use List instead of Set when getting all pending transactions Signed-off-by: Fabio Di Fabio * More detailed log on adding remote txs Signed-off-by: Fabio Di Fabio * Execute transaction broadcast aysnc after adding them to the txpool Signed-off-by: Fabio Di Fabio * Log time taken to add remote txs before their broadcast Signed-off-by: Fabio Di Fabio * Fix test Signed-off-by: Fabio Di Fabio * Add missing header Signed-off-by: Fabio Di Fabio * Fix unit tests Signed-off-by: Fabio Di Fabio * Add CHANGELOG entry Signed-off-by: Fabio Di Fabio * Delete unneeded file Signed-off-by: Fabio Di Fabio * Rename some layered txpool metrics to avoid conflict with existing metrics Signed-off-by: Fabio Di Fabio * Fix pushing to next layers txs following an invalid one Signed-off-by: Fabio Di Fabio * In case of an unexpected error, log more data and do a consistency check Signed-off-by: Fabio Di Fabio * Fix null check on wrong var Signed-off-by: Fabio Di Fabio * Fix some codeql alerts Signed-off-by: Fabio Di Fabio * Fix sparse gap calculation when invalidating Signed-off-by: Fabio Di Fabio * Apply suggestions from doce review Signed-off-by: Fabio Di Fabio * Only trigger consistency check if trace log is enable in case of unexpected error Signed-off-by: Fabio Di Fabio * Fix replay of blocks with no transactions Signed-off-by: Fabio Di Fabio * Fix for negative gap when there is a reorg Signed-off-by: Fabio Di Fabio * Implement code review suggestions Signed-off-by: Fabio Di Fabio * Fix for a case when deleting tx with zero gap in sparse layer Signed-off-by: Fabio Di Fabio * Delete redoundant tests Signed-off-by: Fabio Di Fabio * Update CHANGELOG.md Co-authored-by: Sally MacFarlane Signed-off-by: Fabio Di Fabio * Update ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java Co-authored-by: Sally MacFarlane Signed-off-by: Fabio Di Fabio * Update ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolMetrics.java Co-authored-by: Sally MacFarlane Signed-off-by: Fabio Di Fabio * Update ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java Co-authored-by: Sally MacFarlane Signed-off-by: Fabio Di Fabio * Update ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactionsTestBase.java Co-authored-by: Sally MacFarlane Signed-off-by: Fabio Di Fabio * Update ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java Co-authored-by: Sally MacFarlane Signed-off-by: Fabio Di Fabio * Update besu/src/main/java/org/hyperledger/besu/cli/options/unstable/TransactionPoolOptions.java Co-authored-by: Sally MacFarlane Signed-off-by: Fabio Di Fabio * Update besu/src/main/java/org/hyperledger/besu/cli/options/unstable/TransactionPoolOptions.java Co-authored-by: Sally MacFarlane Signed-off-by: Fabio Di Fabio * Address code review suggestions Signed-off-by: Fabio Di Fabio * Improve logSender when there are no sparse txs Signed-off-by: Fabio Di Fabio * Fix off by one error Signed-off-by: Fabio Di Fabio * Update ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java Co-authored-by: Simon Dudley Signed-off-by: Fabio Di Fabio * Address code review suggestions Signed-off-by: Fabio Di Fabio * Rename fix Signed-off-by: Fabio Di Fabio * Simplify the way reorgs are handled, by detecting a negative gap and deleting and readding all the txs for that sender Signed-off-by: Fabio Di Fabio * Do not run consistency check on internal error since it is too expensive, instead force a reorg of the sender txs Signed-off-by: Fabio Di Fabio * Remove invalid txs after the selection is complete to avoid ConcurrentModificationException Signed-off-by: Fabio Di Fabio * Update txpool defaults Signed-off-by: Fabio Di Fabio * Tune default Signed-off-by: Fabio Di Fabio * Fix merge Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio Signed-off-by: Simon Dudley Co-authored-by: Simon Dudley Co-authored-by: Sally MacFarlane Co-authored-by: garyschulte --- .gitignore | 4 +- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 3 +- .../unstable/TransactionPoolOptions.java | 79 +- .../options/TransactionPoolOptionsTest.java | 19 +- .../EthGetFilterChangesIntegrationTest.java | 6 +- .../EthGetFilterChangesIntegrationTest.java | 6 +- .../TxPoolBesuPendingTransactions.java | 4 +- .../methods/TxPoolBesuStatistics.java | 5 +- .../results/PendingTransactionResult.java | 2 +- .../results/PendingTransactionsResult.java | 14 +- .../pool/PendingTransactionFilter.java | 9 +- ...PendingTransactionSubscriptionService.java | 4 +- .../AbstractEthGraphQLHttpServiceTest.java | 12 +- .../AbstractJsonRpcHttpServiceTest.java | 5 +- .../api/jsonrpc/LatestNonceProviderTest.java | 39 +- .../methods/EthGetTransactionByHashTest.java | 9 +- .../methods/EthGetTransactionCountTest.java | 56 +- .../TxPoolBesuPendingTransactionsTest.java | 9 +- .../methods/TxPoolBesuStatisticsTest.java | 4 +- .../methods/TxPoolBesuTransactionsTest.java | 10 +- .../pool/PendingTransactionFilterTest.java | 6 +- .../BlockTransactionSelector.java | 7 +- .../besu/ethereum/core/Transaction.java | 30 +- .../bonsai/AbstractIsolationTests.java | 38 +- ethereum/eth/build.gradle | 1 + ...dGetPooledTransactionsFromPeerFetcher.java | 34 +- ...oledTransactionHashesMessageProcessor.java | 32 +- .../transactions/PeerTransactionTracker.java | 2 +- .../eth/transactions/PendingTransaction.java | 156 +- ...a => PendingTransactionAddedListener.java} | 2 +- .../eth/transactions/PendingTransactions.java | 60 +- .../transactions/TransactionAddedResult.java | 129 ++ .../transactions/TransactionAddedStatus.java | 42 - .../transactions/TransactionBroadcaster.java | 9 +- .../eth/transactions/TransactionPool.java | 202 ++- .../TransactionPoolConfiguration.java | 32 +- .../transactions/TransactionPoolFactory.java | 109 +- .../transactions/TransactionPoolMetrics.java | 173 +++ .../TransactionsMessageProcessor.java | 42 +- .../AbstractPrioritizedTransactions.java | 139 ++ .../AbstractSequentialTransactionsLayer.java | 160 ++ .../layered/AbstractTransactionsLayer.java | 596 ++++++++ .../BaseFeePrioritizedTransactions.java | 152 ++ .../eth/transactions/layered/EndLayer.java | 171 +++ .../GasPricePrioritizedTransactions.java | 80 + .../layered/LayeredPendingTransactions.java | 479 ++++++ .../layered/ReadyTransactions.java | 221 +++ .../layered/SparseTransactions.java | 374 +++++ .../layered/TransactionsLayer.java | 103 ++ .../transactions/layered/package-info.java | 76 + .../AbstractPendingTransactionsSorter.java | 129 +- .../BaseFeePendingTransactionsSorter.java | 9 +- .../GasPricePendingTransactionsSorter.java | 5 +- ...PooledTransactionsFromPeerFetcherTest.java | 14 +- ...GetPooledTransactionsFromPeerTaskTest.java | 4 +- .../AbstractTransactionPoolTest.java | 8 +- ...actionsLayeredPendingTransactionsTest.java | 697 +++++++++ ...TransactionHashesMessageProcessorTest.java | 10 +- ...ledTransactionHashesMessageSenderTest.java | 18 +- .../PendingMultiTypesTransactionsTest.java | 382 ----- ...ingTransactionEstimatedMemorySizeTest.java | 414 +++++ .../TransactionBroadcasterTest.java | 5 +- .../TransactionPoolFactoryTest.java | 4 +- .../TransactionsMessageProcessorTest.java | 15 +- ...stractPrioritizedTransactionsTestBase.java | 185 +++ .../BaseFeePrioritizedTransactionsTest.java | 173 +++ .../layered/BaseTransactionPoolTest.java | 178 +++ .../layered/EvictCollectorLayer.java | 47 + .../GasPricePrioritizedTransactionsTest.java | 91 ++ .../LayeredPendingTransactionsLegacyTest.java | 241 +++ .../LayeredPendingTransactionsLondonTest.java | 294 ++++ .../LayeredPendingTransactionsTest.java | 713 +++++++++ .../eth/transactions/layered/LayersTest.java | 1331 +++++++++++++++++ .../eth/transactions/layered/ReplayTest.java | 297 ++++ .../AbstractPendingTransactionsTestBase.java | 18 +- .../BaseFeePendingTransactionsTest.java | 3 +- .../GasPricePendingTransactionsTest.java | 3 +- .../besu/ethereum/rlp/RLPInput.java | 2 +- gradle/verification-metadata.xml | 13 + gradle/versions.gradle | 1 + 81 files changed, 8339 insertions(+), 922 deletions(-) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/{PendingTransactionListener.java => PendingTransactionAddedListener.java} (94%) create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionAddedResult.java delete mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionAddedStatus.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolMetrics.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactions.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/EndLayer.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactions.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactions.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/package-info.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/AbstractTransactionsLayeredPendingTransactionsTest.java delete mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingMultiTypesTransactionsTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactionsTestBase.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactionsTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/EvictCollectorLayer.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactionsTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsLegacyTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsLondonTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java diff --git a/.gitignore b/.gitignore index cf501ba9f2..0465ea5e30 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ *~.nib *.iml *.launch -*.swp *.log .classpath .DS_Store @@ -29,4 +28,5 @@ site/ /kubernetes/reports/ /kubernetes/besu-*.tar.gz **/src/*/generated -jitpack.yml \ No newline at end of file +jitpack.yml +/ethereum/eth/src/test/resources/tx.csv.gz diff --git a/CHANGELOG.md b/CHANGELOG.md index db8afb68a8..8f058545bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - EIP-4844: Zero blob transactions are invalid [#5425](https://github.com/hyperledger/besu/pull/5425) - Transaction pool flag to disable specific behaviors for locally submitted transactions [#5418](https://github.com/hyperledger/besu/pull/5418) - New optional feature to save the txpool content to file on shutdown and reloading it on startup [#5434](https://github.com/hyperledger/besu/pull/5434) +- Early access - layered transaction pool implementation [#5290](https://github.com/hyperledger/besu/pull/5290) ### Bug Fixes - Fix eth_feeHistory response for the case in which blockCount is higher than highestBlock requested. [#5397](https://github.com/hyperledger/besu/pull/5397) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 5e5c2f74e8..84ce3f918c 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1202,7 +1202,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable { description = "Maximum number of pending transactions that will be kept in the transaction pool (default: ${DEFAULT-VALUE})", arity = "1") - private final Integer txPoolMaxSize = TransactionPoolConfiguration.MAX_PENDING_TRANSACTIONS; + private final Integer txPoolMaxSize = + TransactionPoolConfiguration.DEFAULT_MAX_PENDING_TRANSACTIONS; @Option( names = {"--tx-pool-retention-hours"}, diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/TransactionPoolOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/TransactionPoolOptions.java index a733a042ad..a6fdd55f84 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/TransactionPoolOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/TransactionPoolOptions.java @@ -25,11 +25,15 @@ import java.time.Duration; import java.util.Arrays; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import picocli.CommandLine; /** The Transaction pool Cli options. */ public class TransactionPoolOptions implements CLIOptions { + private static final Logger LOG = LoggerFactory.getLogger(TransactionPoolOptions.class); + private static final String TX_MESSAGE_KEEP_ALIVE_SEC_FLAG = "--Xincoming-tx-messages-keep-alive-seconds"; @@ -47,6 +51,14 @@ public class TransactionPoolOptions private static final String SAVE_RESTORE_FLAG = "--tx-pool-enable-save-restore"; private static final String SAVE_FILE = "--tx-pool-save-file"; + private static final String LAYERED_TX_POOL_ENABLED_FLAG = "--Xlayered-tx-pool"; + private static final String LAYERED_TX_POOL_LAYER_MAX_CAPACITY = + "--Xlayered-tx-pool-layer-max-capacity"; + private static final String LAYERED_TX_POOL_MAX_PRIORITIZED = + "--Xlayered-tx-pool-max-prioritized"; + private static final String LAYERED_TX_POOL_MAX_FUTURE_BY_SENDER = + "--Xlayered-tx-pool-max-future-by-sender"; + @CommandLine.Option( names = {STRICT_TX_REPLAY_PROTECTION_ENABLED_FLAG}, paramLabel = "", @@ -84,7 +96,46 @@ public class TransactionPoolOptions "Maximum portion of the transaction pool which a single account may occupy with future transactions (default: ${DEFAULT-VALUE})", arity = "1") private Float txPoolLimitByAccountPercentage = - TransactionPoolConfiguration.LIMIT_TXPOOL_BY_ACCOUNT_PERCENTAGE; + TransactionPoolConfiguration.DEFAULT_LIMIT_TX_POOL_BY_ACCOUNT_PERCENTAGE; + + @CommandLine.Option( + names = {LAYERED_TX_POOL_ENABLED_FLAG}, + paramLabel = "", + hidden = true, + description = "Enable the Layered Transaction Pool (default: ${DEFAULT-VALUE})", + arity = "0..1") + private Boolean layeredTxPoolEnabled = + TransactionPoolConfiguration.DEFAULT_LAYERED_TX_POOL_ENABLED; + + @CommandLine.Option( + names = {LAYERED_TX_POOL_LAYER_MAX_CAPACITY}, + paramLabel = "", + hidden = true, + description = + "Max amount of memory space, in bytes, that any layer within the transaction pool could occupy (default: ${DEFAULT-VALUE})", + arity = "1") + private long layeredTxPoolLayerMaxCapacity = + TransactionPoolConfiguration.DEFAULT_PENDING_TRANSACTIONS_LAYER_MAX_CAPACITY_BYTES; + + @CommandLine.Option( + names = {LAYERED_TX_POOL_MAX_PRIORITIZED}, + paramLabel = "", + hidden = true, + description = + "Max number of pending transactions that are prioritized and thus kept sorted (default: ${DEFAULT-VALUE})", + arity = "1") + private int layeredTxPoolMaxPrioritized = + TransactionPoolConfiguration.DEFAULT_MAX_PRIORITIZED_TRANSACTIONS; + + @CommandLine.Option( + names = {LAYERED_TX_POOL_MAX_FUTURE_BY_SENDER}, + paramLabel = "", + hidden = true, + description = + "Max number of future pending transactions allowed for a single sender (default: ${DEFAULT-VALUE})", + arity = "1") + private int layeredTxPoolMaxFutureBySender = + TransactionPoolConfiguration.DEFAULT_MAX_FUTURE_BY_SENDER; @CommandLine.Option( names = {DISABLE_LOCAL_TXS_FLAG}, @@ -139,11 +190,21 @@ public class TransactionPoolOptions options.disableLocalTxs = config.getDisableLocalTransactions(); options.saveRestoreEnabled = config.getEnableSaveRestore(); options.saveFile = config.getSaveFile(); + options.layeredTxPoolEnabled = config.getLayeredTxPoolEnabled(); + options.layeredTxPoolLayerMaxCapacity = config.getPendingTransactionsLayerMaxCapacityBytes(); + options.layeredTxPoolMaxPrioritized = config.getMaxPrioritizedTransactions(); + options.layeredTxPoolMaxFutureBySender = config.getMaxFutureBySender(); return options; } @Override public ImmutableTransactionPoolConfiguration.Builder toDomainObject() { + if (layeredTxPoolEnabled) { + LOG.warn( + "Layered transaction pool enabled, ignoring settings for " + + "--tx-pool-max-size and --tx-pool-limit-by-account-percentage"); + } + return ImmutableTransactionPoolConfiguration.builder() .strictTransactionReplayProtectionEnabled(strictTxReplayProtectionEnabled) .txMessageKeepAliveSeconds(txMessageKeepAliveSeconds) @@ -151,7 +212,12 @@ public class TransactionPoolOptions .txPoolLimitByAccountPercentage(txPoolLimitByAccountPercentage) .disableLocalTransactions(disableLocalTxs) .enableSaveRestore(saveRestoreEnabled) - .saveFile(saveFile); + .saveFile(saveFile) + .txPoolLimitByAccountPercentage(txPoolLimitByAccountPercentage) + .layeredTxPoolEnabled(layeredTxPoolEnabled) + .pendingTransactionsLayerMaxCapacityBytes(layeredTxPoolLayerMaxCapacity) + .maxPrioritizedTransactions(layeredTxPoolMaxPrioritized) + .maxFutureBySender(layeredTxPoolMaxFutureBySender); } @Override @@ -166,7 +232,14 @@ public class TransactionPoolOptions TX_MESSAGE_KEEP_ALIVE_SEC_FLAG, OptionParser.format(txMessageKeepAliveSeconds), ETH65_TX_ANNOUNCED_BUFFERING_PERIOD_FLAG, - OptionParser.format(eth65TrxAnnouncedBufferingPeriod)); + OptionParser.format(eth65TrxAnnouncedBufferingPeriod), + LAYERED_TX_POOL_ENABLED_FLAG + "=" + layeredTxPoolEnabled, + LAYERED_TX_POOL_LAYER_MAX_CAPACITY, + OptionParser.format(layeredTxPoolLayerMaxCapacity), + LAYERED_TX_POOL_MAX_PRIORITIZED, + OptionParser.format(layeredTxPoolMaxPrioritized), + LAYERED_TX_POOL_MAX_FUTURE_BY_SENDER, + OptionParser.format(layeredTxPoolMaxFutureBySender)); } /** diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/TransactionPoolOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/TransactionPoolOptionsTest.java index 12d6a8ca8b..2f44c2d145 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/TransactionPoolOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/TransactionPoolOptionsTest.java @@ -16,7 +16,7 @@ package org.hyperledger.besu.cli.options; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration.LIMIT_TXPOOL_BY_ACCOUNT_PERCENTAGE; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration.DEFAULT_LIMIT_TX_POOL_BY_ACCOUNT_PERCENTAGE; import org.hyperledger.besu.cli.options.unstable.TransactionPoolOptions; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; @@ -101,7 +101,7 @@ public class TransactionPoolOptionsTest final TransactionPoolOptions options = getOptionsFromBesuCommand(cmd); final TransactionPoolConfiguration config = options.toDomainObject().build(); assertThat(config.getTxPoolLimitByAccountPercentage()) - .isEqualTo(LIMIT_TXPOOL_BY_ACCOUNT_PERCENTAGE); + .isEqualTo(DEFAULT_LIMIT_TX_POOL_BY_ACCOUNT_PERCENTAGE); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)) @@ -239,7 +239,13 @@ public class TransactionPoolOptionsTest .txPoolLimitByAccountPercentage(defaultValue.getTxPoolLimitByAccountPercentage()) .disableLocalTransactions(defaultValue.getDisableLocalTransactions()) .enableSaveRestore(defaultValue.getEnableSaveRestore()) - .saveFile(defaultValue.getSaveFile()); + .saveFile(defaultValue.getSaveFile()) + .txPoolLimitByAccountPercentage(defaultValue.getTxPoolLimitByAccountPercentage()) + .layeredTxPoolEnabled(defaultValue.getLayeredTxPoolEnabled()) + .pendingTransactionsLayerMaxCapacityBytes( + defaultValue.getPendingTransactionsLayerMaxCapacityBytes()) + .maxPrioritizedTransactions(defaultValue.getMaxPrioritizedTransactions()) + .maxFutureBySender(defaultValue.getMaxFutureBySender()); } @Override @@ -253,7 +259,12 @@ public class TransactionPoolOptionsTest .txPoolLimitByAccountPercentage(0.5f) .disableLocalTransactions(true) .enableSaveRestore(true) - .saveFile(new File("abc.xyz")); + .saveFile(new File("abc.xyz")) + .txPoolLimitByAccountPercentage(0.5f) + .layeredTxPoolEnabled(true) + .pendingTransactionsLayerMaxCapacityBytes(1_000_000L) + .maxPrioritizedTransactions(1000) + .maxFutureBySender(10); } @Override diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/fork/frontier/EthGetFilterChangesIntegrationTest.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/fork/frontier/EthGetFilterChangesIntegrationTest.java index e46efb6b8d..19bc2c9d68 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/fork/frontier/EthGetFilterChangesIntegrationTest.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/fork/frontier/EthGetFilterChangesIntegrationTest.java @@ -49,9 +49,11 @@ import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeers; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import org.hyperledger.besu.ethereum.eth.transactions.TransactionBroadcaster; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.data.TransactionType; @@ -84,7 +86,7 @@ public class EthGetFilterChangesIntegrationTest { private TransactionPool transactionPool; private final MetricsSystem metricsSystem = new NoOpMetricsSystem(); - private GasPricePendingTransactionsSorter transactions; + private PendingTransactions transactions; private static final int MAX_TRANSACTIONS = 5; private static final KeyPair keyPair = SignatureAlgorithmFactory.getInstance().generateKeyPair(); @@ -116,7 +118,7 @@ public class EthGetFilterChangesIntegrationTest { batchAddedListener, ethContext, new MiningParameters.Builder().minTransactionGasPrice(Wei.ZERO).build(), - metricsSystem, + new TransactionPoolMetrics(metricsSystem), TransactionPoolConfiguration.DEFAULT); final BlockchainQueries blockchainQueries = new BlockchainQueries(blockchain, protocolContext.getWorldStateArchive()); diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/fork/london/EthGetFilterChangesIntegrationTest.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/fork/london/EthGetFilterChangesIntegrationTest.java index 9e95650f04..04a8891fbc 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/fork/london/EthGetFilterChangesIntegrationTest.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/fork/london/EthGetFilterChangesIntegrationTest.java @@ -49,9 +49,11 @@ import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeers; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import org.hyperledger.besu.ethereum.eth.transactions.TransactionBroadcaster; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; import org.hyperledger.besu.ethereum.eth.transactions.sorter.BaseFeePendingTransactionsSorter; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.data.TransactionType; @@ -84,7 +86,7 @@ public class EthGetFilterChangesIntegrationTest { private TransactionPool transactionPool; private final MetricsSystem metricsSystem = new NoOpMetricsSystem(); - private BaseFeePendingTransactionsSorter transactions; + private PendingTransactions transactions; private static final int MAX_TRANSACTIONS = 5; private static final KeyPair keyPair = SignatureAlgorithmFactory.getInstance().generateKeyPair(); @@ -116,7 +118,7 @@ public class EthGetFilterChangesIntegrationTest { batchAddedListener, ethContext, new MiningParameters.Builder().minTransactionGasPrice(Wei.ZERO).build(), - metricsSystem, + new TransactionPoolMetrics(metricsSystem), TransactionPoolConfiguration.DEFAULT); final BlockchainQueries blockchainQueries = new BlockchainQueries(blockchain, protocolContext.getWorldStateArchive()); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuPendingTransactions.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuPendingTransactions.java index 226fd80e17..d1cf49fd0d 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuPendingTransactions.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuPendingTransactions.java @@ -25,9 +25,9 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.transaction.po import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; +import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; public class TxPoolBesuPendingTransactions implements JsonRpcMethod { @@ -56,7 +56,7 @@ public class TxPoolBesuPendingTransactions implements JsonRpcMethod { .map(PendingTransactionsParams::filters) .orElse(Collections.emptyList()); - final Set pendingTransactionsFiltered = + final Collection pendingTransactionsFiltered = pendingTransactionFilter.reduce( pendingTransactions.getPendingTransactions(), filters, limit); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuStatistics.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuStatistics.java index 58b7475272..1950d771ce 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuStatistics.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuStatistics.java @@ -22,7 +22,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.PendingTransac import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; -import java.util.Set; +import java.util.Collection; public class TxPoolBesuStatistics implements JsonRpcMethod { @@ -43,7 +43,8 @@ public class TxPoolBesuStatistics implements JsonRpcMethod { } private PendingTransactionsStatisticsResult statistics() { - final Set pendingTransaction = pendingTransactions.getPendingTransactions(); + final Collection pendingTransaction = + pendingTransactions.getPendingTransactions(); final long localCount = pendingTransaction.stream().filter(PendingTransaction::isReceivedFromLocalSource).count(); final long remoteCount = pendingTransaction.size() - localCount; diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/PendingTransactionResult.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/PendingTransactionResult.java index d5355365fd..c87b34da0f 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/PendingTransactionResult.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/PendingTransactionResult.java @@ -31,7 +31,7 @@ public class PendingTransactionResult implements TransactionResult { public PendingTransactionResult(final PendingTransaction pendingTransaction) { hash = pendingTransaction.getHash().toString(); isReceivedFromLocalSource = pendingTransaction.isReceivedFromLocalSource(); - addedToPoolAt = pendingTransaction.getAddedToPoolAt(); + addedToPoolAt = Instant.ofEpochMilli(pendingTransaction.getAddedAt()); } @JsonGetter(value = "hash") diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/PendingTransactionsResult.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/PendingTransactionsResult.java index 87a4871d12..39308fd38f 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/PendingTransactionsResult.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/PendingTransactionsResult.java @@ -16,24 +16,22 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.results; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; -import java.util.Set; -import java.util.stream.Collectors; +import java.util.Collection; +import java.util.List; import com.fasterxml.jackson.annotation.JsonValue; public class PendingTransactionsResult implements TransactionResult { - private final Set pendingTransactionResults; + private final List pendingTransactionResults; - public PendingTransactionsResult(final Set pendingTransactionSet) { + public PendingTransactionsResult(final Collection pendingTransactionSet) { pendingTransactionResults = - pendingTransactionSet.stream() - .map(PendingTransactionResult::new) - .collect(Collectors.toSet()); + pendingTransactionSet.stream().map(PendingTransactionResult::new).toList(); } @JsonValue - public Set getResults() { + public List getResults() { return pendingTransactionResults; } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/transaction/pool/PendingTransactionFilter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/transaction/pool/PendingTransactionFilter.java index f62881aedd..958b225420 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/transaction/pool/PendingTransactionFilter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/transaction/pool/PendingTransactionFilter.java @@ -23,10 +23,9 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonR import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; /** * This class allows to filter a list of pending transactions @@ -43,8 +42,8 @@ public class PendingTransactionFilter { public static final String VALUE_FIELD = "value"; public static final String NONCE_FIELD = "nonce"; - public Set reduce( - final Set pendingTransactions, + public Collection reduce( + final Collection pendingTransactions, final List filters, final int limit) throws InvalidJsonRpcParameters { @@ -52,7 +51,7 @@ public class PendingTransactionFilter { .filter(pendingTx -> applyFilters(pendingTx, filters)) .limit(limit) .map(PendingTransaction::getTransaction) - .collect(Collectors.toSet()); + .toList(); } private boolean applyFilters( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionSubscriptionService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionSubscriptionService.java index 26ce0ad10a..c015a7bddd 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionSubscriptionService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionSubscriptionService.java @@ -18,11 +18,11 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.subscription.Subscrip import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.subscription.SubscriptionManager; import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.subscription.request.SubscriptionType; import org.hyperledger.besu.ethereum.core.Transaction; -import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionListener; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionAddedListener; import java.util.List; -public class PendingTransactionSubscriptionService implements PendingTransactionListener { +public class PendingTransactionSubscriptionService implements PendingTransactionAddedListener { private final SubscriptionManager subscriptionManager; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java index 0594633cfd..cb8c253ad3 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java @@ -34,8 +34,8 @@ import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; import org.hyperledger.besu.ethereum.mainnet.HeaderValidationMode; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; @@ -51,7 +51,6 @@ import org.hyperledger.besu.testutil.BlockTestUtil; import java.net.URL; import java.nio.file.Paths; -import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -139,21 +138,18 @@ public abstract class AbstractEthGraphQLHttpServiceTest { transactionPoolMock.addTransactionViaApi( ArgumentMatchers.argThat(tx -> tx.getNonce() == 16))) .thenReturn(ValidationResult.invalid(TransactionInvalidReason.NONCE_TOO_LOW)); - final GasPricePendingTransactionsSorter pendingTransactionsMock = - Mockito.mock(GasPricePendingTransactionsSorter.class); + final PendingTransactions pendingTransactionsMock = Mockito.mock(PendingTransactions.class); Mockito.when(transactionPoolMock.getPendingTransactions()).thenReturn(pendingTransactionsMock); Mockito.when(pendingTransactionsMock.getPendingTransactions()) .thenReturn( Collections.singleton( - new PendingTransaction( + new PendingTransaction.Local( Transaction.builder() .type(TransactionType.FRONTIER) .nonce(42) .gasLimit(654321) .gasPrice(Wei.ONE) - .build(), - true, - Instant.ofEpochSecond(Integer.MAX_VALUE)))); + .build()))); final WorldStateArchive stateArchive = createInMemoryWorldStateArchive(); GENESIS_CONFIG.writeStateTo(stateArchive.getMutable()); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/AbstractJsonRpcHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/AbstractJsonRpcHttpServiceTest.java index 9e9c6c7c1f..d2979a1bf6 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/AbstractJsonRpcHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/AbstractJsonRpcHttpServiceTest.java @@ -38,8 +38,8 @@ import org.hyperledger.besu.ethereum.core.Synchronizer; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.EthProtocol; import org.hyperledger.besu.ethereum.eth.manager.EthPeers; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; import org.hyperledger.besu.ethereum.mainnet.ValidationResult; import org.hyperledger.besu.ethereum.p2p.network.P2PNetwork; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.Capability; @@ -140,8 +140,7 @@ public abstract class AbstractJsonRpcHttpServiceTest { // nonce too low tests uses a tx with nonce=16 when(transactionPoolMock.addTransactionViaApi(argThat(tx -> tx.getNonce() == 16))) .thenReturn(ValidationResult.invalid(TransactionInvalidReason.NONCE_TOO_LOW)); - final GasPricePendingTransactionsSorter pendingTransactionsMock = - mock(GasPricePendingTransactionsSorter.class); + final PendingTransactions pendingTransactionsMock = mock(PendingTransactions.class); when(transactionPoolMock.getPendingTransactions()).thenReturn(pendingTransactionsMock); final PrivacyParameters privacyParameters = mock(PrivacyParameters.class); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/LatestNonceProviderTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/LatestNonceProviderTest.java index 32fc37ad6b..bc3312fe05 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/LatestNonceProviderTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/LatestNonceProviderTest.java @@ -15,42 +15,29 @@ package org.hyperledger.besu.ethereum.api.jsonrpc; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.BaseFeePendingTransactionsSorter; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; -import java.util.Arrays; -import java.util.Collection; import java.util.OptionalLong; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; -@RunWith(Parameterized.class) +@RunWith(MockitoJUnitRunner.class) public class LatestNonceProviderTest { - private final Address senderAdress = Address.fromHexString("1"); + private final Address senderAddress = Address.fromHexString("1"); - private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); + @Mock private BlockchainQueries blockchainQueries; private LatestNonceProvider nonceProvider; - @Parameterized.Parameter public PendingTransactions pendingTransactions; - - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList( - new Object[][] { - {mock(GasPricePendingTransactionsSorter.class)}, - {mock(BaseFeePendingTransactionsSorter.class)} - }); - } + @Mock private PendingTransactions pendingTransactions; @Before public void setUp() { @@ -60,19 +47,19 @@ public class LatestNonceProviderTest { @Test public void nextNonceUsesTxPool() { final long highestNonceInPendingTransactions = 123; - when(pendingTransactions.getNextNonceForSender(senderAdress)) + when(pendingTransactions.getNextNonceForSender(senderAddress)) .thenReturn(OptionalLong.of(highestNonceInPendingTransactions)); - assertThat(nonceProvider.getNonce(senderAdress)).isEqualTo(highestNonceInPendingTransactions); + assertThat(nonceProvider.getNonce(senderAddress)).isEqualTo(highestNonceInPendingTransactions); } @Test public void nextNonceIsTakenFromBlockchainIfNoPendingTransactionResponse() { final long headBlockNumber = 8; - final long nonceInBLockchain = 56; - when(pendingTransactions.getNextNonceForSender(senderAdress)).thenReturn(OptionalLong.empty()); + final long nonceInBlockchain = 56; + when(pendingTransactions.getNextNonceForSender(senderAddress)).thenReturn(OptionalLong.empty()); when(blockchainQueries.headBlockNumber()).thenReturn(headBlockNumber); - when(blockchainQueries.getTransactionCount(senderAdress, headBlockNumber)) - .thenReturn(nonceInBLockchain); - assertThat(nonceProvider.getNonce(senderAdress)).isEqualTo(nonceInBLockchain); + when(blockchainQueries.getTransactionCount(senderAddress, headBlockNumber)) + .thenReturn(nonceInBlockchain); + assertThat(nonceProvider.getNonce(senderAddress)).isEqualTo(nonceInBlockchain); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionByHashTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionByHashTest.java index 3ba8444421..ecdb8fb67d 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionByHashTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionByHashTest.java @@ -33,10 +33,9 @@ import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.api.query.TransactionWithMetadata; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import org.hyperledger.besu.plugin.data.Transaction; -import java.time.Instant; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -59,7 +58,7 @@ public class EthGetTransactionByHashTest { private final String JSON_RPC_VERSION = "2.0"; private final String ETH_METHOD = "eth_getTransactionByHash"; - @Mock private GasPricePendingTransactionsSorter pendingTransactions; + @Mock private PendingTransactions pendingTransactions; @Before public void setUp() { @@ -195,9 +194,7 @@ public class EthGetTransactionByHashTest { Transaction pendingTransaction = gen.transaction(); System.out.println(pendingTransaction.getHash()); return gen.transactionsWithAllTypes(4).stream() - .map( - transaction -> - new PendingTransaction(transaction, true, Instant.ofEpochSecond(Integer.MAX_VALUE))) + .map(transaction -> new PendingTransaction.Local(transaction)) .collect(Collectors.toUnmodifiableSet()); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionCountTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionCountTest.java index 8ec2e6bd15..65fc57fa35 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionCountTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionCountTest.java @@ -28,17 +28,13 @@ import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.ChainHead; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.BaseFeePendingTransactionsSorter; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; -import java.util.Arrays; -import java.util.Collection; import java.util.OptionalLong; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -class EthGetTransactionCountTest { +public class EthGetTransactionCountTest { private final Blockchain blockchain = mock(Blockchain.class); private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); private final ChainHead chainHead = mock(ChainHead.class); @@ -46,19 +42,16 @@ class EthGetTransactionCountTest { private EthGetTransactionCount ethGetTransactionCount; private final String pendingTransactionString = "0x00000000000000000000000000000000000000AA"; private final Object[] pendingParams = new Object[] {pendingTransactionString, "pending"}; + private PendingTransactions pendingTransactions; - public static Collection data() { - return Arrays.asList( - new Object[][] { - {mock(GasPricePendingTransactionsSorter.class)}, - {mock(BaseFeePendingTransactionsSorter.class)} - }); + @BeforeEach + public void setup() { + pendingTransactions = mock(PendingTransactions.class); + ethGetTransactionCount = new EthGetTransactionCount(blockchainQueries, pendingTransactions); } - @ParameterizedTest - @MethodSource("data") - void shouldUsePendingTransactionsWhenToldTo(final PendingTransactions pendingTransactions) { - setup(pendingTransactions); + @Test + public void shouldUsePendingTransactionsWhenToldTo() { final Address address = Address.fromHexString(pendingTransactionString); when(pendingTransactions.getNextNonceForSender(address)).thenReturn(OptionalLong.of(12)); @@ -71,11 +64,8 @@ class EthGetTransactionCountTest { assertThat(response.getResult()).isEqualTo("0xc"); } - @ParameterizedTest - @MethodSource("data") - void shouldUseLatestTransactionsWhenNoPendingTransactions( - final PendingTransactions pendingTransactions) { - setup(pendingTransactions); + @Test + public void shouldUseLatestTransactionsWhenNoPendingTransactions() { final Address address = Address.fromHexString(pendingTransactionString); when(pendingTransactions.getNextNonceForSender(address)).thenReturn(OptionalLong.empty()); @@ -88,10 +78,8 @@ class EthGetTransactionCountTest { assertThat(response.getResult()).isEqualTo("0x7"); } - @ParameterizedTest - @MethodSource("data") - void shouldUseLatestWhenItIsBiggerThanPending(final PendingTransactions pendingTransactions) { - setup(pendingTransactions); + @Test + public void shouldUseLatestWhenItIsBiggerThanPending() { final Address address = Address.fromHexString(pendingTransactionString); mockGetTransactionCount(address, 8); @@ -105,10 +93,8 @@ class EthGetTransactionCountTest { assertThat(response.getResult()).isEqualTo("0x8"); } - @ParameterizedTest - @MethodSource("data") - void shouldReturnPendingWithHighNonce(final PendingTransactions pendingTransactions) { - setup(pendingTransactions); + @Test + public void shouldReturnPendingWithHighNonce() { final Address address = Address.fromHexString(pendingTransactionString); when(pendingTransactions.getNextNonceForSender(address)) @@ -122,10 +108,8 @@ class EthGetTransactionCountTest { assertThat(response.getResult()).isEqualTo("0xfffffffffffffffe"); } - @ParameterizedTest - @MethodSource("data") - void shouldReturnLatestWithHighNonce(final PendingTransactions pendingTransactions) { - setup(pendingTransactions); + @Test + public void shouldReturnLatestWithHighNonce() { final Address address = Address.fromHexString(pendingTransactionString); when(pendingTransactions.getNextNonceForSender(address)) @@ -139,10 +123,6 @@ class EthGetTransactionCountTest { assertThat(response.getResult()).isEqualTo("0xfffffffffffffffe"); } - private void setup(final PendingTransactions pendingTransactions) { - ethGetTransactionCount = new EthGetTransactionCount(blockchainQueries, pendingTransactions); - } - private void mockGetTransactionCount(final Address address, final long transactionCount) { when(blockchainQueries.getBlockchain()).thenReturn(blockchain); when(blockchain.getChainHead()).thenReturn(chainHead); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuPendingTransactionsTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuPendingTransactionsTest.java index d43ee4ef45..09b4752fae 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuPendingTransactionsTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuPendingTransactionsTest.java @@ -26,9 +26,8 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSucces import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.TransactionPendingResult; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; -import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -45,7 +44,7 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) public class TxPoolBesuPendingTransactionsTest { - @Mock private GasPricePendingTransactionsSorter pendingTransactions; + @Mock private PendingTransactions pendingTransactions; private TxPoolBesuPendingTransactions method; private final String JSON_RPC_VERSION = "2.0"; private final String TXPOOL_PENDING_TRANSACTIONS_METHOD = "txpool_besuPendingTransactions"; @@ -258,9 +257,7 @@ public class TxPoolBesuPendingTransactionsTest { final BlockDataGenerator gen = new BlockDataGenerator(); return gen.transactionsWithAllTypes(4).stream() - .map( - transaction -> - new PendingTransaction(transaction, true, Instant.ofEpochSecond(Integer.MAX_VALUE))) + .map(transaction -> new PendingTransaction.Local(transaction)) .collect(Collectors.toUnmodifiableSet()); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuStatisticsTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuStatisticsTest.java index a1e48813b9..b9646e18ff 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuStatisticsTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuStatisticsTest.java @@ -23,7 +23,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.PendingTransactionsStatisticsResult; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import com.google.common.collect.Sets; import org.junit.Before; @@ -35,7 +35,7 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) public class TxPoolBesuStatisticsTest { - @Mock private GasPricePendingTransactionsSorter pendingTransactions; + @Mock private PendingTransactions pendingTransactions; private TxPoolBesuStatistics method; private final String JSON_RPC_VERSION = "2.0"; private final String TXPOOL_PENDING_TRANSACTIONS_METHOD = "txpool_besuStatistics"; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuTransactionsTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuTransactionsTest.java index 22071e5e08..8af9011596 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuTransactionsTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TxPoolBesuTransactionsTest.java @@ -25,7 +25,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSucces import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.PendingTransactionResult; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.PendingTransactionsResult; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import java.time.Instant; @@ -39,7 +39,7 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) public class TxPoolBesuTransactionsTest { - @Mock private GasPricePendingTransactionsSorter pendingTransactions; + @Mock private PendingTransactions pendingTransactions; private TxPoolBesuTransactions method; private final String JSON_RPC_VERSION = "2.0"; private final String TXPOOL_PENDING_TRANSACTIONS_METHOD = "txpool_besuTransactions"; @@ -58,7 +58,7 @@ public class TxPoolBesuTransactionsTest { @Test public void shouldReturnPendingTransactions() { - Instant addedAt = Instant.ofEpochMilli(10_000_000); + long addedAt = 10_000_000; final JsonRpcRequestContext request = new JsonRpcRequestContext( new JsonRpcRequest( @@ -67,7 +67,7 @@ public class TxPoolBesuTransactionsTest { PendingTransaction pendingTransaction = mock(PendingTransaction.class); when(pendingTransaction.getHash()).thenReturn(Hash.fromHexString(TRANSACTION_HASH)); when(pendingTransaction.isReceivedFromLocalSource()).thenReturn(true); - when(pendingTransaction.getAddedToPoolAt()).thenReturn(addedAt); + when(pendingTransaction.getAddedAt()).thenReturn(addedAt); when(pendingTransactions.getPendingTransactions()) .thenReturn(Sets.newHashSet(pendingTransaction)); @@ -78,6 +78,6 @@ public class TxPoolBesuTransactionsTest { assertThat(actualResult.getHash()).isEqualTo(TRANSACTION_HASH); assertThat(actualResult.isReceivedFromLocalSource()).isTrue(); - assertThat(actualResult.getAddedToPoolAt()).isEqualTo(addedAt.toString()); + assertThat(actualResult.getAddedToPoolAt()).isEqualTo(Instant.ofEpochMilli(addedAt).toString()); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/transaction/pool/PendingTransactionFilterTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/transaction/pool/PendingTransactionFilterTest.java index 79bf5c2d3f..13c68f7c90 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/transaction/pool/PendingTransactionFilterTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/transaction/pool/PendingTransactionFilterTest.java @@ -32,7 +32,6 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.transaction.po import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; -import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; @@ -114,7 +113,7 @@ public class PendingTransactionFilterTest { @Test public void localAndRemoteAddressShouldNotStartWithForwardSlash() { - final Set filteredList = + final Collection filteredList = pendingTransactionFilter.reduce(getPendingTransactions(), filters, limit); assertThat(filteredList.size()).isEqualTo(expectedListOfTransactionHash.size()); @@ -139,8 +138,7 @@ public class PendingTransactionFilterTest { if (i == numberTrx - 1) { when(transaction.isContractCreation()).thenReturn(true); } - pendingTransactionList.add( - new PendingTransaction(transaction, true, Instant.ofEpochSecond(Integer.MAX_VALUE))); + pendingTransactionList.add(new PendingTransaction.Local(transaction)); } return new LinkedHashSet<>(pendingTransactionList); } diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelector.java index 21eec1644f..9cc17b601d 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelector.java @@ -260,10 +260,9 @@ public class BlockTransactionSelector { in this throwing an CancellationException). */ public TransactionSelectionResults buildTransactionListForBlock() { - LOG.debug("Transaction pool size {}", pendingTransactions.size()); - LOG.atTrace() - .setMessage("Transaction pool content {}") - .addArgument(() -> pendingTransactions.toTraceLog(false, false)) + LOG.atDebug() + .setMessage("Transaction pool stats {}") + .addArgument(pendingTransactions.logStats()) .log(); pendingTransactions.selectTransactions( pendingTransaction -> evaluateTransaction(pendingTransaction, false)); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java index 093e1e0486..3a444e5528 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java @@ -1033,19 +1033,27 @@ public class Transaction public String toTraceLog() { final StringBuilder sb = new StringBuilder(); + sb.append(getHash()).append("={"); sb.append(isContractCreation() ? "ContractCreation" : "MessageCall").append(", "); + sb.append(getNonce()).append(", "); sb.append(getSender()).append(", "); sb.append(getType()).append(", "); - sb.append(getNonce()).append(", "); - getGasPrice().ifPresent(gasPrice -> sb.append(gasPrice.toBigInteger()).append(", ")); + getGasPrice() + .ifPresent( + gasPrice -> sb.append("gp: ").append(gasPrice.toHumanReadableString()).append(", ")); if (getMaxPriorityFeePerGas().isPresent() && getMaxFeePerGas().isPresent()) { - sb.append(getMaxPriorityFeePerGas().map(Wei::toBigInteger).get()).append(", "); - sb.append(getMaxFeePerGas().map(Wei::toBigInteger).get()).append(", "); - getMaxFeePerDataGas().ifPresent(wei -> sb.append(wei.toShortHexString()).append(", ")); + sb.append("mf: ") + .append(getMaxFeePerGas().map(Wei::toHumanReadableString).get()) + .append(", "); + sb.append("pf: ") + .append(getMaxPriorityFeePerGas().map(Wei::toHumanReadableString).get()) + .append(", "); + getMaxFeePerDataGas() + .ifPresent(wei -> sb.append("df: ").append(wei.toHumanReadableString()).append(", ")); } - sb.append(getGasLimit()).append(", "); - sb.append(getValue().toBigInteger()).append(", "); - if (getTo().isPresent()) sb.append(getTo().get()).append(", "); + sb.append("gl: ").append(getGasLimit()).append(", "); + sb.append("v: ").append(getValue().toHumanReadableString()).append(", "); + getTo().ifPresent(to -> sb.append(to)); return sb.append("}").toString(); } @@ -1057,6 +1065,7 @@ public class Transaction } public static class Builder { + private static final Optional> EMPTY_ACCESS_LIST = Optional.of(List.of()); protected TransactionType transactionType; @@ -1149,7 +1158,10 @@ public class Transaction } public Builder accessList(final List accessList) { - this.accessList = Optional.ofNullable(accessList); + this.accessList = + accessList == null + ? Optional.empty() + : accessList.isEmpty() ? EMPTY_ACCESS_LIST : Optional.of(accessList); return this; } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java index a3f8b931e8..af25a1afcd 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java @@ -42,8 +42,14 @@ import org.hyperledger.besu.ethereum.core.SealableBlockHeader; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolReplacementHandler; +import org.hyperledger.besu.ethereum.eth.transactions.layered.EndLayer; +import org.hyperledger.besu.ethereum.eth.transactions.layered.GasPricePrioritizedTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.layered.LayeredPendingTransactions; import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.storage.StorageProvider; @@ -59,11 +65,11 @@ import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksD import java.io.File; import java.io.IOException; import java.nio.file.Path; -import java.time.Clock; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -87,12 +93,30 @@ public abstract class AbstractIsolationTests { protected final GenesisState genesisState = GenesisState.fromConfig(GenesisConfigFile.development(), protocolSchedule); protected final MutableBlockchain blockchain = createInMemoryBlockchain(genesisState.getBlock()); + + protected final TransactionPoolConfiguration poolConfiguration = + ImmutableTransactionPoolConfiguration.builder().txPoolMaxSize(100).build(); + + protected final TransactionPoolReplacementHandler transactionReplacementHandler = + new TransactionPoolReplacementHandler(poolConfiguration.getPriceBump()); + + protected final BiFunction + transactionReplacementTester = + (t1, t2) -> + transactionReplacementHandler.shouldReplace( + t1, t2, protocolContext.getBlockchain().getChainHeadHeader()); + + protected TransactionPoolMetrics txPoolMetrics = + new TransactionPoolMetrics(new NoOpMetricsSystem()); + protected final PendingTransactions sorter = - new GasPricePendingTransactionsSorter( - ImmutableTransactionPoolConfiguration.builder().txPoolMaxSize(100).build(), - Clock.systemUTC(), - new NoOpMetricsSystem(), - blockchain::getChainHeadHeader); + new LayeredPendingTransactions( + poolConfiguration, + new GasPricePrioritizedTransactions( + poolConfiguration, + new EndLayer(txPoolMetrics), + txPoolMetrics, + transactionReplacementTester)); protected final List accounts = GenesisConfigFile.development() diff --git a/ethereum/eth/build.gradle b/ethereum/eth/build.gradle index 4cd367dbb5..0f68fc521d 100644 --- a/ethereum/eth/build.gradle +++ b/ethereum/eth/build.gradle @@ -80,6 +80,7 @@ dependencies { testImplementation 'org.junit.jupiter:junit-jupiter' testImplementation 'org.mockito:mockito-core' testImplementation 'org.mockito:mockito-junit-jupiter' + testImplementation 'org.openjdk.jol:jol-core' testRuntimeOnly 'org.junit.vintage:junit-vintage-engine' diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/task/BufferedGetPooledTransactionsFromPeerFetcher.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/task/BufferedGetPooledTransactionsFromPeerFetcher.java index 5a373a2a32..215202f834 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/task/BufferedGetPooledTransactionsFromPeerFetcher.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/task/BufferedGetPooledTransactionsFromPeerFetcher.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.ethereum.eth.manager.task; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration.MAX_PENDING_TRANSACTIONS; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration.DEFAULT_MAX_PENDING_TRANSACTIONS; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.Transaction; @@ -22,9 +22,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.transactions.PeerTransactionTracker; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; -import org.hyperledger.besu.metrics.BesuMetricCategory; -import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; import java.util.ArrayList; import java.util.Collection; @@ -42,16 +40,15 @@ public class BufferedGetPooledTransactionsFromPeerFetcher { private static final Logger LOG = LoggerFactory.getLogger(BufferedGetPooledTransactionsFromPeerFetcher.class); private static final int MAX_HASHES = 256; - private static final String HASHES = "hashes"; private final TransactionPool transactionPool; private final PeerTransactionTracker transactionTracker; private final EthContext ethContext; - private final MetricsSystem metricsSystem; + private final TransactionPoolMetrics metrics; + private final String metricLabel; private final ScheduledFuture scheduledFuture; private final EthPeer peer; private final Queue txAnnounces; - private final Counter alreadySeenTransactionsCounter; public BufferedGetPooledTransactionsFromPeerFetcher( final EthContext ethContext, @@ -59,23 +56,17 @@ public class BufferedGetPooledTransactionsFromPeerFetcher { final EthPeer peer, final TransactionPool transactionPool, final PeerTransactionTracker transactionTracker, - final MetricsSystem metricsSystem) { + final TransactionPoolMetrics metrics, + final String metricLabel) { this.ethContext = ethContext; this.scheduledFuture = scheduledFuture; this.peer = peer; this.transactionPool = transactionPool; this.transactionTracker = transactionTracker; - this.metricsSystem = metricsSystem; - this.txAnnounces = Queues.synchronizedQueue(EvictingQueue.create(MAX_PENDING_TRANSACTIONS)); - - this.alreadySeenTransactionsCounter = - metricsSystem - .createLabelledCounter( - BesuMetricCategory.TRANSACTION_POOL, - "remote_already_seen_total", - "Total number of received transactions already seen", - "source") - .labels(HASHES); + this.metrics = metrics; + this.metricLabel = metricLabel; + this.txAnnounces = + Queues.synchronizedQueue(EvictingQueue.create(DEFAULT_MAX_PENDING_TRANSACTIONS)); } public ScheduledFuture getScheduledFuture() { @@ -86,7 +77,8 @@ public class BufferedGetPooledTransactionsFromPeerFetcher { List txHashesAnnounced; while (!(txHashesAnnounced = getTxHashesAnnounced()).isEmpty()) { final GetPooledTransactionsFromPeerTask task = - GetPooledTransactionsFromPeerTask.forHashes(ethContext, txHashesAnnounced, metricsSystem); + GetPooledTransactionsFromPeerTask.forHashes( + ethContext, txHashesAnnounced, metrics.getMetricsSystem()); task.assignPeer(peer); ethContext .getScheduler() @@ -125,7 +117,7 @@ public class BufferedGetPooledTransactionsFromPeerFetcher { } final int alreadySeenCount = discarded; - alreadySeenTransactionsCounter.inc(alreadySeenCount); + metrics.incrementAlreadySeenTransactions(metricLabel, alreadySeenCount); LOG.atTrace() .setMessage( "Transaction hashes to request from peer {}, fresh count {}, already seen count {}") diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageProcessor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageProcessor.java index 790cec9382..4717e2d9e1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageProcessor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageProcessor.java @@ -23,10 +23,6 @@ import org.hyperledger.besu.ethereum.eth.manager.task.BufferedGetPooledTransacti import org.hyperledger.besu.ethereum.eth.messages.NewPooledTransactionHashesMessage; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason; import org.hyperledger.besu.ethereum.rlp.RLPException; -import org.hyperledger.besu.metrics.BesuMetricCategory; -import org.hyperledger.besu.metrics.RunnableCounter; -import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.Counter; import java.time.Duration; import java.time.Instant; @@ -40,43 +36,32 @@ import org.slf4j.LoggerFactory; public class NewPooledTransactionHashesMessageProcessor { - private static final int SKIPPED_MESSAGES_LOGGING_THRESHOLD = 1000; - private static final Logger LOG = LoggerFactory.getLogger(NewPooledTransactionHashesMessageProcessor.class); + static final String METRIC_LABEL = "new_pooled_transaction_hashes"; + private final ConcurrentHashMap scheduledTasks; private final PeerTransactionTracker transactionTracker; - private final Counter totalSkippedNewPooledTransactionHashesMessageCounter; private final TransactionPool transactionPool; private final TransactionPoolConfiguration transactionPoolConfiguration; private final EthContext ethContext; - private final MetricsSystem metricsSystem; + private final TransactionPoolMetrics metrics; public NewPooledTransactionHashesMessageProcessor( final PeerTransactionTracker transactionTracker, final TransactionPool transactionPool, final TransactionPoolConfiguration transactionPoolConfiguration, final EthContext ethContext, - final MetricsSystem metricsSystem) { + final TransactionPoolMetrics metrics) { this.transactionTracker = transactionTracker; this.transactionPool = transactionPool; this.transactionPoolConfiguration = transactionPoolConfiguration; this.ethContext = ethContext; - this.metricsSystem = metricsSystem; - this.totalSkippedNewPooledTransactionHashesMessageCounter = - new RunnableCounter( - metricsSystem.createCounter( - BesuMetricCategory.TRANSACTION_POOL, - "new_pooled_transaction_hashes_messages_skipped_total", - "Total number of new pooled transaction hashes messages skipped by the processor."), - () -> - LOG.warn( - "{} expired new pooled transaction hashes messages have been skipped.", - SKIPPED_MESSAGES_LOGGING_THRESHOLD), - SKIPPED_MESSAGES_LOGGING_THRESHOLD); + this.metrics = metrics; + metrics.initExpiredMessagesCounter(METRIC_LABEL); this.scheduledTasks = new ConcurrentHashMap<>(); } @@ -89,7 +74,7 @@ public class NewPooledTransactionHashesMessageProcessor { if (startedAt.plus(keepAlive).isAfter(now())) { this.processNewPooledTransactionHashesMessage(peer, transactionsMessage); } else { - totalSkippedNewPooledTransactionHashesMessageCounter.inc(); + metrics.incrementExpiredMessages(METRIC_LABEL); } } @@ -125,7 +110,8 @@ public class NewPooledTransactionHashesMessageProcessor { peer, transactionPool, transactionTracker, - metricsSystem); + metrics, + METRIC_LABEL); }); bufferedTask.addHashes( diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PeerTransactionTracker.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PeerTransactionTracker.java index 2815a426b6..fc6b0f1d81 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PeerTransactionTracker.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PeerTransactionTracker.java @@ -88,7 +88,7 @@ public class PeerTransactionTracker implements EthPeer.DisconnectCallback { private Set createTransactionsSet() { return Collections.newSetFromMap( - new LinkedHashMap(1 << 4, 0.75f, true) { + new LinkedHashMap<>(1 << 4, 0.75f, true) { @Override protected boolean removeEldestEntry(final Map.Entry eldest) { return size() > MAX_TRACKED_SEEN_TRANSACTIONS; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java index 903349d7d3..656dc2c802 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java @@ -18,32 +18,37 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.evm.AccessListEntry; -import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; /** * Tracks the additional metadata associated with transactions to enable prioritization for mining * and deciding which transactions to drop when the transaction pool reaches its size limit. */ -public class PendingTransaction { - +public abstract class PendingTransaction { + static final int NOT_INITIALIZED = -1; + static final int FRONTIER_BASE_MEMORY_SIZE = 944; + static final int ACCESS_LIST_BASE_MEMORY_SIZE = 944; + static final int EIP1559_BASE_MEMORY_SIZE = 1056; + static final int OPTIONAL_TO_MEMORY_SIZE = 92; + static final int PAYLOAD_BASE_MEMORY_SIZE = 32; + static final int ACCESS_LIST_STORAGE_KEY_MEMORY_SIZE = 32; + static final int ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE = 128; + static final int OPTIONAL_ACCESS_LIST_MEMORY_SIZE = 24; + static final int PENDING_TRANSACTION_MEMORY_SIZE = 40; private static final AtomicLong TRANSACTIONS_ADDED = new AtomicLong(); private final Transaction transaction; - private final boolean receivedFromLocalSource; - private final Instant addedToPoolAt; + private final long addedAt; private final long sequence; // Allows prioritization based on order transactions are added - public PendingTransaction( - final Transaction transaction, - final boolean receivedFromLocalSource, - final Instant addedToPoolAt) { + private int memorySize = NOT_INITIALIZED; + + protected PendingTransaction(final Transaction transaction, final long addedAt) { this.transaction = transaction; - this.receivedFromLocalSource = receivedFromLocalSource; - this.addedToPoolAt = addedToPoolAt; + this.addedAt = addedAt; this.sequence = TRANSACTIONS_ADDED.getAndIncrement(); } @@ -67,23 +72,85 @@ public class PendingTransaction { return transaction.getSender(); } - public boolean isReceivedFromLocalSource() { - return receivedFromLocalSource; - } + public abstract boolean isReceivedFromLocalSource(); public Hash getHash() { return transaction.getHash(); } - public Instant getAddedToPoolAt() { - return addedToPoolAt; + public long getAddedAt() { + return addedAt; + } + + public int memorySize() { + if (memorySize == NOT_INITIALIZED) { + memorySize = computeMemorySize(); + } + return memorySize; + } + + private int computeMemorySize() { + return switch (transaction.getType()) { + case FRONTIER -> computeFrontierMemorySize(); + case ACCESS_LIST -> computeAccessListMemorySize(); + case EIP1559 -> computeEIP1559MemorySize(); + case BLOB -> computeBlobMemorySize(); + } + + PENDING_TRANSACTION_MEMORY_SIZE; + } + + private int computeFrontierMemorySize() { + return FRONTIER_BASE_MEMORY_SIZE + computePayloadMemorySize() + computeToMemorySize(); + } + + private int computeAccessListMemorySize() { + return ACCESS_LIST_BASE_MEMORY_SIZE + + computePayloadMemorySize() + + computeToMemorySize() + + computeAccessListEntriesMemorySize(); + } + + private int computeEIP1559MemorySize() { + return EIP1559_BASE_MEMORY_SIZE + + computePayloadMemorySize() + + computeToMemorySize() + + computeAccessListEntriesMemorySize(); + } + + private int computeBlobMemorySize() { + // ToDo 4844: adapt for blobs + return computeEIP1559MemorySize(); + } + + private int computePayloadMemorySize() { + return PAYLOAD_BASE_MEMORY_SIZE + transaction.getPayload().size(); + } + + private int computeToMemorySize() { + if (transaction.getTo().isPresent()) { + return OPTIONAL_TO_MEMORY_SIZE; + } + return 0; + } + + private int computeAccessListEntriesMemorySize() { + return transaction + .getAccessList() + .map( + al -> { + int totalSize = OPTIONAL_ACCESS_LIST_MEMORY_SIZE; + totalSize += al.size() * ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE; + totalSize += + al.stream().map(AccessListEntry::getStorageKeys).mapToInt(List::size).sum() + * ACCESS_LIST_STORAGE_KEY_MEMORY_SIZE; + return totalSize; + }) + .orElse(0); } public static List toTransactionList( final Collection transactionsInfo) { - return transactionsInfo.stream() - .map(PendingTransaction::getTransaction) - .collect(Collectors.toUnmodifiableList()); + return transactionsInfo.stream().map(PendingTransaction::getTransaction).toList(); } @Override @@ -105,13 +172,60 @@ public class PendingTransaction { return 31 * (int) (sequence ^ (sequence >>> 32)); } + @Override + public String toString() { + return "Hash=" + + transaction.getHash().toShortHexString() + + ", nonce=" + + transaction.getNonce() + + ", sender=" + + transaction.getSender().toShortHexString() + + ", addedAt=" + + addedAt + + ", sequence=" + + sequence + + '}'; + } + public String toTraceLog() { return "{sequence: " + sequence + ", addedAt: " - + addedToPoolAt + + addedAt + ", " + transaction.toTraceLog() + "}"; } + + public static class Local extends PendingTransaction { + + public Local(final Transaction transaction, final long addedAt) { + super(transaction, addedAt); + } + + public Local(final Transaction transaction) { + this(transaction, System.currentTimeMillis()); + } + + @Override + public boolean isReceivedFromLocalSource() { + return true; + } + } + + public static class Remote extends PendingTransaction { + + public Remote(final Transaction transaction, final long addedAt) { + super(transaction, addedAt); + } + + public Remote(final Transaction transaction) { + this(transaction, System.currentTimeMillis()); + } + + @Override + public boolean isReceivedFromLocalSource() { + return false; + } + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionListener.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionAddedListener.java similarity index 94% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionListener.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionAddedListener.java index d925b6311a..6e8adee710 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionListener.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionAddedListener.java @@ -17,7 +17,7 @@ package org.hyperledger.besu.ethereum.eth.transactions; import org.hyperledger.besu.ethereum.core.Transaction; @FunctionalInterface -public interface PendingTransactionListener { +public interface PendingTransactionAddedListener { void onTransactionAdded(Transaction transaction); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactions.java index ff22e558d6..1b00e82f64 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactions.java @@ -16,14 +16,15 @@ package org.hyperledger.besu.ethereum.eth.transactions; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; import org.hyperledger.besu.evm.account.Account; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.OptionalLong; -import java.util.Set; public interface PendingTransactions { @@ -33,45 +34,56 @@ public interface PendingTransactions { List getLocalTransactions(); - TransactionAddedStatus addRemoteTransaction( - final Transaction transaction, final Optional maybeSenderAccount); + TransactionAddedResult addRemoteTransaction( + Transaction transaction, Optional maybeSenderAccount); - TransactionAddedStatus addLocalTransaction( - final Transaction transaction, final Optional maybeSenderAccount); + TransactionAddedResult addLocalTransaction( + Transaction transaction, Optional maybeSenderAccount); - void removeTransaction(final Transaction transaction); - - void transactionAddedToBlock(final Transaction transaction); - - void selectTransactions(final TransactionSelector selector); + void selectTransactions(TransactionSelector selector); long maxSize(); int size(); - boolean containsTransaction(final Hash transactionHash); + boolean containsTransaction(Transaction transaction); + + Optional getTransactionByHash(Hash transactionHash); - Optional getTransactionByHash(final Hash transactionHash); + Collection getPendingTransactions(); - Set getPendingTransactions(); + long subscribePendingTransactions(PendingTransactionAddedListener listener); - long subscribePendingTransactions(final PendingTransactionListener listener); + void unsubscribePendingTransactions(long id); - void unsubscribePendingTransactions(final long id); + long subscribeDroppedTransactions(PendingTransactionDroppedListener listener); - long subscribeDroppedTransactions(final PendingTransactionDroppedListener listener); + void unsubscribeDroppedTransactions(long id); - void unsubscribeDroppedTransactions(final long id); + OptionalLong getNextNonceForSender(Address sender); - OptionalLong getNextNonceForSender(final Address sender); + void manageBlockAdded( + BlockHeader blockHeader, + List confirmedTransactions, + final List reorgTransactions, + FeeMarket feeMarket); - void manageBlockAdded(final Block block); + String toTraceLog(); - String toTraceLog(final boolean withTransactionsBySender, final boolean withLowestInvalidNonce); + String logStats(); - List signalInvalidAndGetDependentTransactions(final Transaction transaction); + default List signalInvalidAndGetDependentTransactions( + final Transaction transaction) { + // ToDo: remove when the legacy tx pool is removed + return List.of(); + } + + default void signalInvalidAndRemoveDependentTransactions(final Transaction transaction) { + // ToDo: remove when the legacy tx pool is removed + // no-op + } - boolean isLocalSender(final Address sender); + boolean isLocalSender(Address sender); enum TransactionSelectionResult { DELETE_TRANSACTION_AND_CONTINUE, @@ -81,6 +93,6 @@ public interface PendingTransactions { @FunctionalInterface interface TransactionSelector { - TransactionSelectionResult evaluateTransaction(final Transaction transaction); + TransactionSelectionResult evaluateTransaction(Transaction transaction); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionAddedResult.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionAddedResult.java new file mode 100644 index 0000000000..c1a139ef89 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionAddedResult.java @@ -0,0 +1,129 @@ +/* + * Copyright Besu contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions; + +import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; + +import java.util.Objects; +import java.util.Optional; + +public final class TransactionAddedResult { + private enum Status { + INVALID, + REPLACED, + DROPPED, + TRY_NEXT_LAYER, + ADDED, + REORG_SENDER, + INTERNAL_ERROR + } + + public static final TransactionAddedResult ALREADY_KNOWN = + new TransactionAddedResult(TransactionInvalidReason.TRANSACTION_ALREADY_KNOWN); + public static final TransactionAddedResult REJECTED_UNDERPRICED_REPLACEMENT = + new TransactionAddedResult(TransactionInvalidReason.TRANSACTION_REPLACEMENT_UNDERPRICED); + public static final TransactionAddedResult NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER = + new TransactionAddedResult(TransactionInvalidReason.NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER); + public static final TransactionAddedResult LOWER_NONCE_INVALID_TRANSACTION_KNOWN = + new TransactionAddedResult(TransactionInvalidReason.LOWER_NONCE_INVALID_TRANSACTION_EXISTS); + + public static final TransactionAddedResult ADDED = new TransactionAddedResult(Status.ADDED); + public static final TransactionAddedResult TRY_NEXT_LAYER = + new TransactionAddedResult(Status.TRY_NEXT_LAYER); + + public static final TransactionAddedResult REORG_SENDER = + new TransactionAddedResult(Status.REORG_SENDER); + + public static final TransactionAddedResult DROPPED = new TransactionAddedResult(Status.DROPPED); + + public static final TransactionAddedResult INTERNAL_ERROR = + new TransactionAddedResult(Status.INTERNAL_ERROR); + + private final Optional rejectReason; + + private final Optional replacedTransaction; + + private final Status status; + + private TransactionAddedResult(final PendingTransaction replacedTransaction) { + this.replacedTransaction = Optional.of(replacedTransaction); + this.rejectReason = Optional.empty(); + this.status = Status.REPLACED; + } + + private TransactionAddedResult(final TransactionInvalidReason rejectReason) { + this.replacedTransaction = Optional.empty(); + this.rejectReason = Optional.of(rejectReason); + this.status = Status.INVALID; + } + + private TransactionAddedResult(final Status status) { + this.replacedTransaction = Optional.empty(); + this.rejectReason = Optional.empty(); + this.status = status; + } + + public boolean isSuccess() { + return !isRejected() && status != Status.INTERNAL_ERROR; + } + + public boolean isRejected() { + return status == Status.INVALID; + } + + public boolean isReplacement() { + return replacedTransaction.isPresent(); + } + + public Optional maybeInvalidReason() { + return rejectReason; + } + + public Optional maybeReplacedTransaction() { + return replacedTransaction; + } + + public static TransactionAddedResult createForReplacement( + final PendingTransaction replacedTransaction) { + return new TransactionAddedResult(replacedTransaction); + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TransactionAddedResult that = (TransactionAddedResult) o; + return Objects.equals(rejectReason, that.rejectReason) + && Objects.equals(replacedTransaction, that.replacedTransaction) + && status == that.status; + } + + @Override + public int hashCode() { + return Objects.hash(rejectReason, replacedTransaction, status); + } + + @Override + public String toString() { + return "TransactionAddedResult{" + + "rejectReason=" + + rejectReason + + ", replacedTransaction=" + + replacedTransaction + + ", status=" + + status + + '}'; + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionAddedStatus.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionAddedStatus.java deleted file mode 100644 index 8f974b40f6..0000000000 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionAddedStatus.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Besu contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.eth.transactions; - -import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; - -import java.util.Optional; - -public enum TransactionAddedStatus { - ALREADY_KNOWN(TransactionInvalidReason.TRANSACTION_ALREADY_KNOWN), - REJECTED_UNDERPRICED_REPLACEMENT(TransactionInvalidReason.TRANSACTION_REPLACEMENT_UNDERPRICED), - NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER(TransactionInvalidReason.NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER), - LOWER_NONCE_INVALID_TRANSACTION_KNOWN( - TransactionInvalidReason.LOWER_NONCE_INVALID_TRANSACTION_EXISTS), - ADDED(); - - private final Optional invalidReason; - - TransactionAddedStatus() { - this.invalidReason = Optional.empty(); - } - - TransactionAddedStatus(final TransactionInvalidReason invalidReason) { - this.invalidReason = Optional.of(invalidReason); - } - - public Optional getInvalidReason() { - return invalidReason; - } -} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java index 5a9609d79d..088849c192 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java @@ -30,7 +30,6 @@ import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -64,13 +63,13 @@ public class TransactionBroadcaster implements TransactionBatchAddedListener { } public void relayTransactionPoolTo(final EthPeer peer) { - Set pendingPendingTransaction = + final Collection allPendingTransactions = pendingTransactions.getPendingTransactions(); - if (!pendingPendingTransaction.isEmpty()) { + if (!allPendingTransactions.isEmpty()) { if (peer.hasSupportForMessage(EthPV65.NEW_POOLED_TRANSACTION_HASHES)) { - sendTransactionHashes(toTransactionList(pendingPendingTransaction), List.of(peer)); + sendTransactionHashes(toTransactionList(allPendingTransactions), List.of(peer)); } else { - sendFullTransactions(toTransactionList(pendingPendingTransaction), List.of(peer)); + sendFullTransactions(toTransactionList(allPendingTransactions), List.of(peer)); } } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java index e777db78d1..4860d5a682 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java @@ -14,9 +14,6 @@ */ package org.hyperledger.besu.ethereum.eth.transactions; -import static java.util.Collections.singletonList; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus.ADDED; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus.ALREADY_KNOWN; import static org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason.CHAIN_HEAD_NOT_AVAILABLE; import static org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason.CHAIN_HEAD_WORLD_STATE_NOT_AVAILABLE; import static org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason.INTERNAL_ERROR; @@ -43,11 +40,7 @@ import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.fluent.SimpleAccount; -import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.data.TransactionType; -import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.Counter; -import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -55,15 +48,18 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.IntSummaryStatistics; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.slf4j.Logger; @@ -77,17 +73,14 @@ import org.slf4j.LoggerFactory; *

This class is safe for use across multiple threads. */ public class TransactionPool implements BlockAddedObserver { - private static final Logger LOG = LoggerFactory.getLogger(TransactionPool.class); - - private static final String REMOTE = "remote"; - private static final String LOCAL = "local"; + private static final Logger LOG_FOR_REPLAY = LoggerFactory.getLogger("LOG_FOR_REPLAY"); private final PendingTransactions pendingTransactions; private final ProtocolSchedule protocolSchedule; private final ProtocolContext protocolContext; private final TransactionBroadcaster transactionBroadcaster; private final MiningParameters miningParameters; - private final LabelledMetric duplicateTransactionCounter; + private final TransactionPoolMetrics metrics; private final TransactionPoolConfiguration configuration; private final AtomicBoolean isPoolEnabled = new AtomicBoolean(true); @@ -98,27 +91,36 @@ public class TransactionPool implements BlockAddedObserver { final TransactionBroadcaster transactionBroadcaster, final EthContext ethContext, final MiningParameters miningParameters, - final MetricsSystem metricsSystem, + final TransactionPoolMetrics metrics, final TransactionPoolConfiguration configuration) { this.pendingTransactions = pendingTransactions; this.protocolSchedule = protocolSchedule; this.protocolContext = protocolContext; this.transactionBroadcaster = transactionBroadcaster; this.miningParameters = miningParameters; + this.metrics = metrics; this.configuration = configuration; - - duplicateTransactionCounter = - metricsSystem.createLabelledCounter( - BesuMetricCategory.TRANSACTION_POOL, - "transactions_duplicates_total", - "Total number of duplicate transactions received", - "source"); - ethContext.getEthPeers().subscribeConnect(this::handleConnect); - + initLogForReplay(); CompletableFuture.runAsync(this::loadFromDisk); } + private void initLogForReplay() { + LOG_FOR_REPLAY + .atTrace() + .setMessage("{},{},{},{}") + .addArgument(() -> getChainHeadBlockHeader().map(BlockHeader::getNumber).orElse(0L)) + .addArgument( + () -> + getChainHeadBlockHeader() + .flatMap(BlockHeader::getBaseFee) + .map(Wei::getAsBigInteger) + .orElse(BigInteger.ZERO)) + .addArgument(() -> getChainHeadBlockHeader().map(BlockHeader::getGasUsed).orElse(0L)) + .addArgument(() -> getChainHeadBlockHeader().map(BlockHeader::getGasLimit).orElse(0L)) + .log(); + } + public void saveToDisk() { if (configuration.getEnableSaveRestore()) { final File saveFile = configuration.getSaveFile(); @@ -217,25 +219,24 @@ public class TransactionPool implements BlockAddedObserver { if (validationResult.result.isValid()) { - final TransactionAddedStatus transactionAddedStatus = + final TransactionAddedResult transactionAddedResult = pendingTransactions.addLocalTransaction(transaction, validationResult.maybeAccount); - if (!transactionAddedStatus.equals(ADDED)) { - if (transactionAddedStatus.equals(ALREADY_KNOWN)) { - duplicateTransactionCounter.labels(LOCAL).inc(); - } - return ValidationResult.invalid( - transactionAddedStatus - .getInvalidReason() + if (transactionAddedResult.isRejected()) { + final var rejectReason = + transactionAddedResult + .maybeInvalidReason() .orElseGet( () -> { - LOG.warn("Missing invalid reason for status {}", transactionAddedStatus); + LOG.warn("Missing invalid reason for status {}", transactionAddedResult); return INTERNAL_ERROR; - })); + }); + return ValidationResult.invalid(rejectReason); } - final Collection txs = singletonList(transaction); - transactionBroadcaster.onTransactionsAdded(txs); + transactionBroadcaster.onTransactionsAdded(List.of(transaction)); + } else { + metrics.incrementRejected(true, validationResult.result.getInvalidReason(), "txpool"); } return validationResult.result; @@ -251,80 +252,99 @@ public class TransactionPool implements BlockAddedObserver { .orElse(true); } - public void addRemoteTransactions(final Collection transactions) { - final List addedTransactions = new ArrayList<>(transactions.size()); - LOG.trace("Adding {} remote transactions", transactions.size()); - - for (final Transaction transaction : transactions) { + private Stream sortedBySenderAndNonce(final Collection transactions) { + return transactions.stream() + .sorted(Comparator.comparing(Transaction::getSender).thenComparing(Transaction::getNonce)); + } - final var result = addRemoteTransaction(transaction); - if (result.isValid()) { - addedTransactions.add(transaction); - } - } + public void addRemoteTransactions(final Collection transactions) { + final long started = System.currentTimeMillis(); + final int initialCount = transactions.size(); + final List addedTransactions = new ArrayList<>(initialCount); + LOG.debug("Adding {} remote transactions", initialCount); + + sortedBySenderAndNonce(transactions) + .forEach( + transaction -> { + final var result = addRemoteTransaction(transaction); + if (result.isValid()) { + addedTransactions.add(transaction); + } + }); + + LOG_FOR_REPLAY + .atTrace() + .setMessage("S,{}") + .addArgument(() -> pendingTransactions.logStats()) + .log(); + + LOG.atDebug() + .setMessage( + "Added {} transactions to the pool in {}ms, {} not added, current pool stats {}") + .addArgument(addedTransactions::size) + .addArgument(() -> System.currentTimeMillis() - started) + .addArgument(() -> initialCount - addedTransactions.size()) + .addArgument(pendingTransactions::logStats) + .log(); if (!addedTransactions.isEmpty()) { transactionBroadcaster.onTransactionsAdded(addedTransactions); - LOG.atTrace() - .setMessage("Added {} transactions to the pool, current pool size {}, content {}") - .addArgument(addedTransactions::size) - .addArgument(pendingTransactions::size) - .addArgument(() -> pendingTransactions.toTraceLog(true, true)) - .log(); } } private ValidationResult addRemoteTransaction( final Transaction transaction) { - if (pendingTransactions.containsTransaction(transaction.getHash())) { + if (pendingTransactions.containsTransaction(transaction)) { LOG.atTrace() .setMessage("Discard already present transaction {}") .addArgument(transaction::toTraceLog) .log(); // We already have this transaction, don't even validate it. - duplicateTransactionCounter.labels(REMOTE).inc(); + metrics.incrementRejected(false, TRANSACTION_ALREADY_KNOWN, "txpool"); return ValidationResult.invalid(TRANSACTION_ALREADY_KNOWN); } final ValidationResultAndAccount validationResult = validateRemoteTransaction(transaction); if (validationResult.result.isValid()) { - final var status = + final TransactionAddedResult status = pendingTransactions.addRemoteTransaction(transaction, validationResult.maybeAccount); - switch (status) { - case ADDED: - LOG.atTrace() - .setMessage("Added remote transaction {}") - .addArgument(transaction::toTraceLog) - .log(); - break; - case ALREADY_KNOWN: - LOG.atTrace() - .setMessage("Duplicate remote transaction {}") - .addArgument(transaction::toTraceLog) - .log(); - duplicateTransactionCounter.labels(REMOTE).inc(); - return ValidationResult.invalid(TRANSACTION_ALREADY_KNOWN); - default: - LOG.atTrace().setMessage("Transaction added status {}").addArgument(status::name).log(); - return ValidationResult.invalid(status.getInvalidReason().get()); + if (status.isSuccess()) { + LOG.atTrace() + .setMessage("Added remote transaction {}") + .addArgument(transaction::toTraceLog) + .log(); + } else { + final var rejectReason = + status + .maybeInvalidReason() + .orElseGet( + () -> { + LOG.warn("Missing invalid reason for status {}", status); + return INTERNAL_ERROR; + }); + LOG.atTrace() + .setMessage("Transaction {} rejected reason {}") + .addArgument(transaction::toTraceLog) + .addArgument(rejectReason) + .log(); + metrics.incrementRejected(false, rejectReason, "txpool"); + return ValidationResult.invalid(rejectReason); } - } else { LOG.atTrace() .setMessage("Discard invalid transaction {}, reason {}") .addArgument(transaction::toTraceLog) .addArgument(validationResult.result::getInvalidReason) .log(); - pendingTransactions - .signalInvalidAndGetDependentTransactions(transaction) - .forEach(pendingTransactions::removeTransaction); + metrics.incrementRejected(false, validationResult.result.getInvalidReason(), "txpool"); + pendingTransactions.signalInvalidAndRemoveDependentTransactions(transaction); } return validationResult.result; } - public long subscribePendingTransactions(final PendingTransactionListener listener) { + public long subscribePendingTransactions(final PendingTransactionAddedListener listener) { return pendingTransactions.subscribePendingTransactions(listener); } @@ -343,10 +363,16 @@ public class TransactionPool implements BlockAddedObserver { @Override public void onBlockAdded(final BlockAddedEvent event) { LOG.trace("Block added event {}", event); - if (isPoolEnabled.get()) { - event.getAddedTransactions().forEach(pendingTransactions::transactionAddedToBlock); - pendingTransactions.manageBlockAdded(event.getBlock()); - reAddTransactions(event.getRemovedTransactions()); + if (event.getEventType().equals(BlockAddedEvent.EventType.HEAD_ADVANCED) + || event.getEventType().equals(BlockAddedEvent.EventType.CHAIN_REORG)) { + if (isPoolEnabled.get()) { + pendingTransactions.manageBlockAdded( + event.getBlock().getHeader(), + event.getAddedTransactions(), + event.getRemovedTransactions(), + protocolSchedule.getByBlockHeader(event.getBlock().getHeader()).getFeeMarket()); + reAddTransactions(event.getRemovedTransactions()); + } } } @@ -363,16 +389,28 @@ public class TransactionPool implements BlockAddedObserver { var reAddLocalTxs = txsByOrigin.get(true); var reAddRemoteTxs = txsByOrigin.get(false); if (!reAddLocalTxs.isEmpty()) { - LOG.trace("Re-adding {} local transactions from a block event", reAddLocalTxs.size()); - reAddLocalTxs.forEach(this::addLocalTransaction); + logReAddedTransactions(reAddLocalTxs, "local"); + sortedBySenderAndNonce(reAddLocalTxs).forEach(this::addLocalTransaction); } if (!reAddRemoteTxs.isEmpty()) { - LOG.trace("Re-adding {} remote transactions from a block event", reAddRemoteTxs.size()); + logReAddedTransactions(reAddRemoteTxs, "remote"); addRemoteTransactions(reAddRemoteTxs); } } } + private static void logReAddedTransactions( + final List reAddedTxs, final String source) { + LOG.atTrace() + .setMessage("Re-adding {} {} transactions from a block event: {}") + .addArgument(reAddedTxs::size) + .addArgument(source) + .addArgument( + () -> + reAddedTxs.stream().map(Transaction::toTraceLog).collect(Collectors.joining("; "))) + .log(); + } + private MainnetTransactionValidator getTransactionValidator() { return protocolSchedule .getByBlockHeader(protocolContext.getBlockchain().getChainHeadHeader()) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolConfiguration.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolConfiguration.java index 29e53eca3c..ddcb06f8a9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolConfiguration.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolConfiguration.java @@ -27,8 +27,8 @@ import org.immutables.value.Value; public interface TransactionPoolConfiguration { String DEFAULT_SAVE_FILE_NAME = "txpool.dump"; int DEFAULT_TX_MSG_KEEP_ALIVE = 60; - int MAX_PENDING_TRANSACTIONS = 4096; - float LIMIT_TXPOOL_BY_ACCOUNT_PERCENTAGE = 0.001f; // 0.1% + int DEFAULT_MAX_PENDING_TRANSACTIONS = 4096; + float DEFAULT_LIMIT_TX_POOL_BY_ACCOUNT_PERCENTAGE = 0.001f; // 0.1% int DEFAULT_TX_RETENTION_HOURS = 13; boolean DEFAULT_STRICT_TX_REPLAY_PROTECTION_ENABLED = false; Percentage DEFAULT_PRICE_BUMP = Percentage.fromInt(10); @@ -38,17 +38,21 @@ public interface TransactionPoolConfiguration { boolean DEFAULT_ENABLE_SAVE_RESTORE = false; File DEFAULT_SAVE_FILE = new File(DEFAULT_SAVE_FILE_NAME); + long DEFAULT_PENDING_TRANSACTIONS_LAYER_MAX_CAPACITY_BYTES = 50_000_000L; + int DEFAULT_MAX_PRIORITIZED_TRANSACTIONS = 2000; + int DEFAULT_MAX_FUTURE_BY_SENDER = 200; + boolean DEFAULT_LAYERED_TX_POOL_ENABLED = false; TransactionPoolConfiguration DEFAULT = ImmutableTransactionPoolConfiguration.builder().build(); @Value.Default default int getTxPoolMaxSize() { - return MAX_PENDING_TRANSACTIONS; + return DEFAULT_MAX_PENDING_TRANSACTIONS; } @Value.Default default float getTxPoolLimitByAccountPercentage() { - return LIMIT_TXPOOL_BY_ACCOUNT_PERCENTAGE; + return DEFAULT_LIMIT_TX_POOL_BY_ACCOUNT_PERCENTAGE; } @Value.Derived @@ -100,4 +104,24 @@ public interface TransactionPoolConfiguration { default File getSaveFile() { return DEFAULT_SAVE_FILE; } + + @Value.Default + default Boolean getLayeredTxPoolEnabled() { + return DEFAULT_LAYERED_TX_POOL_ENABLED; + } + + @Value.Default + default long getPendingTransactionsLayerMaxCapacityBytes() { + return DEFAULT_PENDING_TRANSACTIONS_LAYER_MAX_CAPACITY_BYTES; + } + + @Value.Default + default int getMaxPrioritizedTransactions() { + return DEFAULT_MAX_PRIORITIZED_TRANSACTIONS; + } + + @Value.Default + default int getMaxFutureBySender() { + return DEFAULT_MAX_FUTURE_BY_SENDER; + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java index 6a805b4182..9d903a128b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java @@ -20,13 +20,23 @@ import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.messages.EthPV62; import org.hyperledger.besu.ethereum.eth.messages.EthPV65; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; +import org.hyperledger.besu.ethereum.eth.transactions.layered.AbstractPrioritizedTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.layered.BaseFeePrioritizedTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.layered.EndLayer; +import org.hyperledger.besu.ethereum.eth.transactions.layered.GasPricePrioritizedTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.layered.LayeredPendingTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.layered.ReadyTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.layered.SparseTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.sorter.AbstractPendingTransactionsSorter; import org.hyperledger.besu.ethereum.eth.transactions.sorter.BaseFeePendingTransactionsSorter; import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket; import org.hyperledger.besu.plugin.services.BesuEvents; import org.hyperledger.besu.plugin.services.MetricsSystem; import java.time.Clock; +import java.util.function.BiFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,9 +54,11 @@ public class TransactionPoolFactory { final MiningParameters miningParameters, final TransactionPoolConfiguration transactionPoolConfiguration) { + final TransactionPoolMetrics metrics = new TransactionPoolMetrics(metricsSystem); + final PendingTransactions pendingTransactions = createPendingTransactions( - protocolSchedule, protocolContext, clock, metricsSystem, transactionPoolConfiguration); + protocolSchedule, protocolContext, clock, metrics, transactionPoolConfiguration); final PeerTransactionTracker transactionTracker = new PeerTransactionTracker(); final TransactionsMessageSender transactionsMessageSender = @@ -59,7 +71,7 @@ public class TransactionPoolFactory { protocolSchedule, protocolContext, ethContext, - metricsSystem, + metrics, syncState, miningParameters, transactionPoolConfiguration, @@ -73,7 +85,7 @@ public class TransactionPoolFactory { final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, - final MetricsSystem metricsSystem, + final TransactionPoolMetrics metrics, final SyncState syncState, final MiningParameters miningParameters, final TransactionPoolConfiguration transactionPoolConfiguration, @@ -81,6 +93,7 @@ public class TransactionPoolFactory { final PeerTransactionTracker transactionTracker, final TransactionsMessageSender transactionsMessageSender, final NewPooledTransactionHashesMessageSender newPooledTransactionHashesMessageSender) { + final TransactionPool transactionPool = new TransactionPool( pendingTransactions, @@ -94,13 +107,13 @@ public class TransactionPoolFactory { newPooledTransactionHashesMessageSender), ethContext, miningParameters, - metricsSystem, + metrics, transactionPoolConfiguration); final TransactionsMessageHandler transactionsMessageHandler = new TransactionsMessageHandler( ethContext.getScheduler(), - new TransactionsMessageProcessor(transactionTracker, transactionPool, metricsSystem), + new TransactionsMessageProcessor(transactionTracker, transactionPool, metrics), transactionPoolConfiguration.getTxMessageKeepAliveSeconds()); final NewPooledTransactionHashesMessageHandler pooledTransactionsMessageHandler = @@ -111,7 +124,7 @@ public class TransactionPoolFactory { transactionPool, transactionPoolConfiguration, ethContext, - metricsSystem), + metrics), transactionPoolConfiguration.getTxMessageKeepAliveSeconds()); subscribeTransactionHandlers( @@ -173,11 +186,37 @@ public class TransactionPoolFactory { final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final Clock clock, - final MetricsSystem metricsSystem, + final TransactionPoolMetrics metrics, final TransactionPoolConfiguration transactionPoolConfiguration) { + boolean isFeeMarketImplementBaseFee = protocolSchedule.anyMatch( scheduledSpec -> scheduledSpec.spec().getFeeMarket().implementsBaseFee()); + + if (transactionPoolConfiguration.getLayeredTxPoolEnabled()) { + LOG.info("Using layered transaction pool"); + return createLayeredPendingTransactions( + protocolSchedule, + protocolContext, + metrics, + transactionPoolConfiguration, + isFeeMarketImplementBaseFee); + } else { + return createPendingTransactionSorter( + protocolContext, + clock, + metrics.getMetricsSystem(), + transactionPoolConfiguration, + isFeeMarketImplementBaseFee); + } + } + + private static AbstractPendingTransactionsSorter createPendingTransactionSorter( + final ProtocolContext protocolContext, + final Clock clock, + final MetricsSystem metricsSystem, + final TransactionPoolConfiguration transactionPoolConfiguration, + final boolean isFeeMarketImplementBaseFee) { if (isFeeMarketImplementBaseFee) { return new BaseFeePendingTransactionsSorter( transactionPoolConfiguration, @@ -192,4 +231,60 @@ public class TransactionPoolFactory { protocolContext.getBlockchain()::getChainHeadHeader); } } + + private static PendingTransactions createLayeredPendingTransactions( + final ProtocolSchedule protocolSchedule, + final ProtocolContext protocolContext, + final TransactionPoolMetrics metrics, + final TransactionPoolConfiguration transactionPoolConfiguration, + final boolean isFeeMarketImplementBaseFee) { + + final TransactionPoolReplacementHandler transactionReplacementHandler = + new TransactionPoolReplacementHandler(transactionPoolConfiguration.getPriceBump()); + + final BiFunction transactionReplacementTester = + (t1, t2) -> + transactionReplacementHandler.shouldReplace( + t1, t2, protocolContext.getBlockchain().getChainHeadHeader()); + + final EndLayer endLayer = new EndLayer(metrics); + + final SparseTransactions sparseTransactions = + new SparseTransactions( + transactionPoolConfiguration, endLayer, metrics, transactionReplacementTester); + + final ReadyTransactions readyTransactions = + new ReadyTransactions( + transactionPoolConfiguration, + sparseTransactions, + metrics, + transactionReplacementTester); + + final AbstractPrioritizedTransactions pendingTransactionsSorter; + if (isFeeMarketImplementBaseFee) { + final BaseFeeMarket baseFeeMarket = + (BaseFeeMarket) + protocolSchedule + .getByBlockHeader(protocolContext.getBlockchain().getChainHeadHeader()) + .getFeeMarket(); + + pendingTransactionsSorter = + new BaseFeePrioritizedTransactions( + transactionPoolConfiguration, + protocolContext.getBlockchain()::getChainHeadHeader, + readyTransactions, + metrics, + transactionReplacementTester, + baseFeeMarket); + } else { + pendingTransactionsSorter = + new GasPricePrioritizedTransactions( + transactionPoolConfiguration, + readyTransactions, + metrics, + transactionReplacementTester); + } + + return new LayeredPendingTransactions(transactionPoolConfiguration, pendingTransactionsSorter); + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolMetrics.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolMetrics.java new file mode 100644 index 0000000000..30ddb9db3d --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolMetrics.java @@ -0,0 +1,173 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions; + +import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; +import org.hyperledger.besu.metrics.BesuMetricCategory; +import org.hyperledger.besu.metrics.RunnableCounter; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.LabelledGauge; +import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.DoubleSupplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TransactionPoolMetrics { + private static final Logger LOG = LoggerFactory.getLogger(TransactionPoolMetrics.class); + public static final String ADDED_COUNTER_NAME = "added_total"; + public static final String REMOVED_COUNTER_NAME = "removed_total"; + public static final String REJECTED_COUNTER_NAME = "rejected_total"; + public static final String EXPIRED_MESSAGES_COUNTER_NAME = "messages_expired_total"; + private static final int SKIPPED_MESSAGES_LOGGING_THRESHOLD = 1000; + private final MetricsSystem metricsSystem; + private final LabelledMetric addedCounter; + private final LabelledMetric removedCounter; + private final LabelledMetric rejectedCounter; + private final LabelledGauge spaceUsed; + private final LabelledGauge transactionCount; + private final LabelledGauge uniqueSenderCount; + private final LabelledMetric expiredMessagesCounter; + private final Map expiredMessagesRunnableCounters = new HashMap<>(); + private final LabelledMetric alreadySeenTransactionsCounter; + + public TransactionPoolMetrics(final MetricsSystem metricsSystem) { + this.metricsSystem = metricsSystem; + + addedCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.TRANSACTION_POOL, + ADDED_COUNTER_NAME, + "Count of transactions added to the transaction pool", + "source", + "layer"); + + removedCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.TRANSACTION_POOL, + REMOVED_COUNTER_NAME, + "Count of transactions removed from the transaction pool", + "source", + "operation", + "layer"); + + rejectedCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.TRANSACTION_POOL, + REJECTED_COUNTER_NAME, + "Count of transactions not accepted to the transaction pool", + "source", + "reason", + "layer"); + + spaceUsed = + metricsSystem.createLabelledGauge( + BesuMetricCategory.TRANSACTION_POOL, + "space_used", + "The amount of space used by the transactions in the layer", + "layer"); + + transactionCount = + metricsSystem.createLabelledGauge( + BesuMetricCategory.TRANSACTION_POOL, + "number_of_transactions", + "The number of transactions currently present in the layer", + "layer"); + + uniqueSenderCount = + metricsSystem.createLabelledGauge( + BesuMetricCategory.TRANSACTION_POOL, + "unique_senders", + "The number of senders with at least one transaction currently present in the layer", + "layer"); + + expiredMessagesCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.TRANSACTION_POOL, + EXPIRED_MESSAGES_COUNTER_NAME, + "Total number of received transaction pool messages expired and not processed.", + "message"); + + alreadySeenTransactionsCounter = + metricsSystem.createLabelledCounter( + BesuMetricCategory.TRANSACTION_POOL, + "remote_transactions_already_seen_total", + "Total number of received transactions already seen", + "message"); + } + + public MetricsSystem getMetricsSystem() { + return metricsSystem; + } + + public void initSpaceUsed(final DoubleSupplier spaceUsedSupplier, final String layer) { + spaceUsed.labels(spaceUsedSupplier, layer); + } + + public void initTransactionCount( + final DoubleSupplier transactionCountSupplier, final String layer) { + transactionCount.labels(transactionCountSupplier, layer); + } + + public void initUniqueSenderCount( + final DoubleSupplier uniqueSenderCountSupplier, final String layer) { + uniqueSenderCount.labels(uniqueSenderCountSupplier, layer); + } + + public void initExpiredMessagesCounter(final String message) { + expiredMessagesRunnableCounters.put( + message, + new RunnableCounter( + expiredMessagesCounter.labels(message), + () -> + LOG.warn( + "{} expired {} messages have been skipped.", + SKIPPED_MESSAGES_LOGGING_THRESHOLD, + message), + SKIPPED_MESSAGES_LOGGING_THRESHOLD)); + } + + public void incrementAdded(final boolean receivedFromLocalSource, final String layer) { + addedCounter.labels(location(receivedFromLocalSource), layer).inc(); + } + + public void incrementRemoved( + final boolean receivedFromLocalSource, final String operation, final String layer) { + removedCounter.labels(location(receivedFromLocalSource), operation, layer).inc(); + } + + public void incrementRejected( + final boolean receivedFromLocalSource, + final TransactionInvalidReason rejectReason, + final String layer) { + rejectedCounter.labels(location(receivedFromLocalSource), rejectReason.name(), layer).inc(); + } + + public void incrementExpiredMessages(final String message) { + expiredMessagesCounter.labels(message).inc(); + } + + public void incrementAlreadySeenTransactions(final String message, final long count) { + alreadySeenTransactionsCounter.labels(message).inc(count); + } + + private String location(final boolean receivedFromLocalSource) { + return receivedFromLocalSource ? "local" : "remote"; + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessor.java index e3718b6119..aee229990a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessor.java @@ -22,10 +22,6 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.messages.TransactionsMessage; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason; import org.hyperledger.besu.ethereum.rlp.RLPException; -import org.hyperledger.besu.metrics.BesuMetricCategory; -import org.hyperledger.besu.metrics.RunnableCounter; -import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.Counter; import java.time.Duration; import java.time.Instant; @@ -38,40 +34,20 @@ import org.slf4j.LoggerFactory; class TransactionsMessageProcessor { private static final Logger LOG = LoggerFactory.getLogger(TransactionsMessageProcessor.class); - private static final int SKIPPED_MESSAGES_LOGGING_THRESHOLD = 1000; - private static final String TRANSACTIONS = "transactions"; - + static final String METRIC_LABEL = "transactions"; private final PeerTransactionTracker transactionTracker; private final TransactionPool transactionPool; - private final Counter totalSkippedTransactionsMessageCounter; - private final Counter alreadySeenTransactionsCounter; + + private final TransactionPoolMetrics metrics; public TransactionsMessageProcessor( final PeerTransactionTracker transactionTracker, final TransactionPool transactionPool, - final MetricsSystem metricsSystem) { + final TransactionPoolMetrics metrics) { this.transactionTracker = transactionTracker; this.transactionPool = transactionPool; - this.totalSkippedTransactionsMessageCounter = - new RunnableCounter( - metricsSystem.createCounter( - BesuMetricCategory.TRANSACTION_POOL, - "transactions_messages_skipped_total", - "Total number of transactions messages skipped by the processor."), - () -> - LOG.warn( - "{} expired transaction messages have been skipped.", - SKIPPED_MESSAGES_LOGGING_THRESHOLD), - SKIPPED_MESSAGES_LOGGING_THRESHOLD); - - alreadySeenTransactionsCounter = - metricsSystem - .createLabelledCounter( - BesuMetricCategory.TRANSACTION_POOL, - "remote_already_seen_total", - "Total number of received transactions already seen", - "source") - .labels(TRANSACTIONS); + this.metrics = metrics; + metrics.initExpiredMessagesCounter(METRIC_LABEL); } void processTransactionsMessage( @@ -83,7 +59,7 @@ class TransactionsMessageProcessor { if (startedAt.plus(keepAlive).isAfter(now())) { this.processTransactionsMessage(peer, transactionsMessage); } else { - totalSkippedTransactionsMessageCounter.inc(); + metrics.incrementExpiredMessages(METRIC_LABEL); } } @@ -95,8 +71,8 @@ class TransactionsMessageProcessor { transactionTracker.markTransactionsAsSeen(peer, incomingTransactions); - alreadySeenTransactionsCounter.inc( - (long) incomingTransactions.size() - freshTransactions.size()); + metrics.incrementAlreadySeenTransactions( + METRIC_LABEL, incomingTransactions.size() - freshTransactions.size()); LOG.atTrace() .setMessage( "Received transactions message from {}, incoming transactions {}, incoming list {}" diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java new file mode 100644 index 0000000000..3d47ad6efd --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java @@ -0,0 +1,139 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; + +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.BiFunction; +import java.util.function.Predicate; +import java.util.stream.Stream; + +public abstract class AbstractPrioritizedTransactions extends AbstractSequentialTransactionsLayer { + protected final TreeSet orderByFee; + + public AbstractPrioritizedTransactions( + final TransactionPoolConfiguration poolConfig, + final TransactionsLayer prioritizedTransactions, + final TransactionPoolMetrics metrics, + final BiFunction + transactionReplacementTester) { + super(poolConfig, prioritizedTransactions, transactionReplacementTester, metrics); + this.orderByFee = new TreeSet<>(this::compareByFee); + } + + @Override + public void reset() { + super.reset(); + orderByFee.clear(); + } + + @Override + public String name() { + return "prioritized"; + } + + @Override + protected TransactionAddedResult canAdd( + final PendingTransaction pendingTransaction, final int gap) { + final var senderTxs = txsBySender.get(pendingTransaction.getSender()); + + if (hasExpectedNonce(senderTxs, pendingTransaction, gap) && hasPriority(pendingTransaction)) { + + return TransactionAddedResult.ADDED; + } + + return TransactionAddedResult.TRY_NEXT_LAYER; + } + + @Override + protected void internalAdd( + final NavigableMap senderTxs, final PendingTransaction addedTx) { + orderByFee.add(addedTx); + } + + @Override + protected void internalReplaced(final PendingTransaction replacedTx) { + orderByFee.remove(replacedTx); + } + + private boolean hasPriority(final PendingTransaction pendingTransaction) { + if (orderByFee.size() < poolConfig.getMaxPrioritizedTransactions()) { + return true; + } + return compareByFee(pendingTransaction, orderByFee.first()) > 0; + } + + @Override + protected int maxTransactionsNumber() { + return poolConfig.getMaxPrioritizedTransactions(); + } + + @Override + protected PendingTransaction getEvictable() { + return orderByFee.first(); + } + + protected abstract int compareByFee(final PendingTransaction pt1, final PendingTransaction pt2); + + @Override + protected void internalRemove( + final NavigableMap senderTxs, + final PendingTransaction removedTx, + final RemovalReason removalReason) { + orderByFee.remove(removedTx); + } + + @Override + public PendingTransaction promote(final Predicate promotionFilter) { + return null; + } + + @Override + public Stream stream() { + return orderByFee.descendingSet().stream(); + } + + @Override + protected long cacheFreeSpace() { + return Integer.MAX_VALUE; + } + + @Override + protected void internalConsistencyCheck( + final Map> prevLayerTxsBySender) { + super.internalConsistencyCheck(prevLayerTxsBySender); + + final var controlOrderByFee = new TreeSet<>(this::compareByFee); + controlOrderByFee.addAll(pendingTransactions.values()); + + final var itControl = controlOrderByFee.iterator(); + final var itCurrent = orderByFee.iterator(); + + while (itControl.hasNext()) { + assert itControl.next().equals(itCurrent.next()) + : "orderByFee does not match pendingTransactions"; + } + + assert itCurrent.hasNext() == false : "orderByFee has more elements that pendingTransactions"; + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java new file mode 100644 index 0000000000..f9ca67347a --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java @@ -0,0 +1,160 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.EVICTED; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.FOLLOW_INVALIDATED; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; + +import java.util.Map; +import java.util.NavigableMap; +import java.util.OptionalLong; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.BiFunction; + +public abstract class AbstractSequentialTransactionsLayer extends AbstractTransactionsLayer { + + public AbstractSequentialTransactionsLayer( + final TransactionPoolConfiguration poolConfig, + final TransactionsLayer nextLayer, + final BiFunction + transactionReplacementTester, + final TransactionPoolMetrics metrics) { + super(poolConfig, nextLayer, transactionReplacementTester, metrics); + } + + @Override + public void remove(final PendingTransaction invalidatedTx, final RemovalReason reason) { + nextLayer.remove(invalidatedTx, reason); + + final var senderTxs = txsBySender.get(invalidatedTx.getSender()); + final long invalidNonce = invalidatedTx.getNonce(); + if (senderTxs != null && invalidNonce <= senderTxs.lastKey()) { + // on sequential layers we need to push to next layer all the txs following the invalid one, + // even if it belongs to a previous layer + + if (senderTxs.remove(invalidNonce) != null) { + // invalid tx removed in this layer + processRemove(senderTxs, invalidatedTx.getTransaction(), reason); + } + + // push following to next layer + pushDown(senderTxs, invalidNonce, 1); + + if (senderTxs.isEmpty()) { + txsBySender.remove(invalidatedTx.getSender()); + } + } + } + + private void pushDown( + final NavigableMap senderTxs, + final long afterNonce, + final int gap) { + senderTxs.tailMap(afterNonce, false).values().stream().toList().stream() + .peek( + txToRemove -> { + senderTxs.remove(txToRemove.getNonce()); + processRemove(senderTxs, txToRemove.getTransaction(), FOLLOW_INVALIDATED); + }) + .forEach(followingTx -> nextLayer.add(followingTx, gap)); + } + + @Override + protected boolean gapsAllowed() { + return false; + } + + @Override + protected void internalConfirmed( + final NavigableMap senderTxs, + final Address sender, + final long maxConfirmedNonce, + final PendingTransaction highestNonceRemovedTx) { + // no -op + } + + @Override + protected void internalEvict( + final NavigableMap senderTxs, final PendingTransaction evictedTx) { + internalRemove(senderTxs, evictedTx, EVICTED); + } + + @Override + public OptionalLong getNextNonceFor(final Address sender) { + final OptionalLong nextLayerRes = nextLayer.getNextNonceFor(sender); + if (nextLayerRes.isEmpty()) { + final var senderTxs = txsBySender.get(sender); + if (senderTxs != null) { + return OptionalLong.of(senderTxs.lastKey() + 1); + } + } + return nextLayerRes; + } + + @Override + protected void internalNotifyAdded( + final NavigableMap senderTxs, + final PendingTransaction pendingTransaction) { + // no-op + } + + protected boolean hasExpectedNonce( + final NavigableMap senderTxs, + final PendingTransaction pendingTransaction, + final long gap) { + if (senderTxs == null) { + return gap == 0; + } + + // true if prepend or append + return (senderTxs.lastKey() + 1) == pendingTransaction.getNonce() + || (senderTxs.firstKey() - 1) == pendingTransaction.getNonce(); + } + + @Override + protected void internalConsistencyCheck( + final Map> prevLayerTxsBySender) { + txsBySender.values().stream() + .filter(senderTxs -> senderTxs.size() > 1) + .map(NavigableMap::entrySet) + .map(Set::iterator) + .forEach( + itNonce -> { + PendingTransaction firstTx = itNonce.next().getValue(); + + prevLayerTxsBySender.computeIfPresent( + firstTx.getSender(), + (sender, txsByNonce) -> { + assert txsByNonce.lastKey() + 1 == firstTx.getNonce() + : "first nonce is not sequential with previous layer last nonce"; + return txsByNonce; + }); + + long prevNonce = firstTx.getNonce(); + + while (itNonce.hasNext()) { + final long currNonce = itNonce.next().getKey(); + assert prevNonce + 1 == currNonce : "non sequential nonce"; + prevNonce = currNonce; + } + }); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java new file mode 100644 index 0000000000..d2d20272cf --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java @@ -0,0 +1,596 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ADDED; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ALREADY_KNOWN; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.REJECTED_UNDERPRICED_REPLACEMENT; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.REORG_SENDER; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.TRY_NEXT_LAYER; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.CONFIRMED; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.CROSS_LAYER_REPLACED; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.EVICTED; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.PROMOTED; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.REPLACED; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionAddedListener; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionDroppedListener; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.util.Subscribers; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.TreeMap; +import java.util.function.BiFunction; +import java.util.function.BinaryOperator; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class AbstractTransactionsLayer implements TransactionsLayer { + private static final Logger LOG = LoggerFactory.getLogger(AbstractTransactionsLayer.class); + private static final NavigableMap EMPTY_SENDER_TXS = new TreeMap<>(); + protected final TransactionPoolConfiguration poolConfig; + protected final TransactionsLayer nextLayer; + protected final BiFunction + transactionReplacementTester; + protected final TransactionPoolMetrics metrics; + protected final Map pendingTransactions = new HashMap<>(); + protected final Map> txsBySender = + new HashMap<>(); + private final Subscribers onAddedListeners = + Subscribers.create(); + private final Subscribers onDroppedListeners = + Subscribers.create(); + private OptionalLong nextLayerOnAddedListenerId = OptionalLong.empty(); + private OptionalLong nextLayerOnDroppedListenerId = OptionalLong.empty(); + protected long spaceUsed = 0; + + public AbstractTransactionsLayer( + final TransactionPoolConfiguration poolConfig, + final TransactionsLayer nextLayer, + final BiFunction + transactionReplacementTester, + final TransactionPoolMetrics metrics) { + this.poolConfig = poolConfig; + this.nextLayer = nextLayer; + this.transactionReplacementTester = transactionReplacementTester; + this.metrics = metrics; + metrics.initSpaceUsed(this::getLayerSpaceUsed, name()); + metrics.initTransactionCount(pendingTransactions::size, name()); + metrics.initUniqueSenderCount(txsBySender::size, name()); + } + + protected abstract boolean gapsAllowed(); + + @Override + public void reset() { + pendingTransactions.clear(); + txsBySender.clear(); + spaceUsed = 0; + nextLayer.reset(); + } + + @Override + public Optional getByHash(final Hash transactionHash) { + final var currLayerTx = pendingTransactions.get(transactionHash); + if (currLayerTx == null) { + return nextLayer.getByHash(transactionHash); + } + return Optional.of(currLayerTx.getTransaction()); + } + + @Override + public boolean contains(final Transaction transaction) { + return pendingTransactions.containsKey(transaction.getHash()) + || nextLayer.contains(transaction); + } + + @Override + public List getAll() { + final List allNextLayers = nextLayer.getAll(); + final List allTxs = + new ArrayList<>(pendingTransactions.size() + allNextLayers.size()); + allTxs.addAll(pendingTransactions.values()); + allTxs.addAll(allNextLayers); + return allTxs; + } + + @Override + public long getCumulativeUsedSpace() { + return getLayerSpaceUsed() + nextLayer.getCumulativeUsedSpace(); + } + + protected long getLayerSpaceUsed() { + return spaceUsed; + } + + protected abstract TransactionAddedResult canAdd( + final PendingTransaction pendingTransaction, final int gap); + + @Override + public TransactionAddedResult add(final PendingTransaction pendingTransaction, final int gap) { + + // is replacing an existing one? + TransactionAddedResult addStatus = maybeReplaceTransaction(pendingTransaction); + if (addStatus == null) { + addStatus = canAdd(pendingTransaction, gap); + } + + if (addStatus.equals(TRY_NEXT_LAYER)) { + return addToNextLayer(pendingTransaction, gap); + } + + if (addStatus.isSuccess()) { + processAdded(pendingTransaction); + addStatus.maybeReplacedTransaction().ifPresent(this::replaced); + + nextLayer.notifyAdded(pendingTransaction); + + if (!maybeFull()) { + // if there is space try to see if the added tx filled some gaps + tryFillGap(addStatus, pendingTransaction); + } + + notifyTransactionAdded(pendingTransaction); + } else { + final var rejectReason = addStatus.maybeInvalidReason().orElseThrow(); + metrics.incrementRejected(false, rejectReason, name()); + LOG.atTrace() + .setMessage("Transaction {} rejected reason {}") + .addArgument(pendingTransaction::toTraceLog) + .addArgument(rejectReason) + .log(); + } + + return addStatus; + } + + private boolean maybeFull() { + final long cacheFreeSpace = cacheFreeSpace(); + final int overflowTxsCount = pendingTransactions.size() - maxTransactionsNumber(); + if (cacheFreeSpace < 0 || overflowTxsCount > 0) { + LOG.atDebug() + .setMessage("Layer full: {}") + .addArgument( + () -> + cacheFreeSpace < 0 + ? "need to free " + (-cacheFreeSpace) + " space" + : "need to evict " + overflowTxsCount + " transaction(s)") + .log(); + + evict(-cacheFreeSpace, overflowTxsCount); + return true; + } + return false; + } + + private void tryFillGap( + final TransactionAddedResult addStatus, final PendingTransaction pendingTransaction) { + // it makes sense to fill gaps only if the add is not a replacement and this layer does not + // allow gaps + if (!addStatus.isReplacement() && !gapsAllowed()) { + final PendingTransaction promotedTx = + nextLayer.promoteFor(pendingTransaction.getSender(), pendingTransaction.getNonce()); + if (promotedTx != null) { + processAdded(promotedTx); + if (!maybeFull()) { + tryFillGap(ADDED, promotedTx); + } + } + } + } + + @Override + public void notifyAdded(final PendingTransaction pendingTransaction) { + final Address sender = pendingTransaction.getSender(); + final var senderTxs = txsBySender.get(sender); + if (senderTxs != null) { + if (senderTxs.firstKey() < pendingTransaction.getNonce()) { + // in the case the world state has been updated but the confirmed txs have not yet been + // processed + confirmed(sender, pendingTransaction.getNonce()); + } else if (senderTxs.firstKey() == pendingTransaction.getNonce()) { + // it is a cross layer replacement, namely added to a previous layer + final PendingTransaction replacedTx = senderTxs.pollFirstEntry().getValue(); + processRemove(senderTxs, replacedTx.getTransaction(), CROSS_LAYER_REPLACED); + + if (senderTxs.isEmpty()) { + txsBySender.remove(sender); + } + } else { + internalNotifyAdded(senderTxs, pendingTransaction); + } + } + nextLayer.notifyAdded(pendingTransaction); + } + + protected abstract void internalNotifyAdded( + final NavigableMap senderTxs, + final PendingTransaction pendingTransaction); + + @Override + public PendingTransaction promoteFor(final Address sender, final long nonce) { + final var senderTxs = txsBySender.get(sender); + if (senderTxs != null) { + long expectedNonce = nonce + 1; + if (senderTxs.firstKey() == expectedNonce) { + final PendingTransaction promotedTx = senderTxs.pollFirstEntry().getValue(); + processRemove(senderTxs, promotedTx.getTransaction(), PROMOTED); + metrics.incrementRemoved(promotedTx.isReceivedFromLocalSource(), "promoted", name()); + + if (senderTxs.isEmpty()) { + txsBySender.remove(sender); + } + return promotedTx; + } + } + return nextLayer.promoteFor(sender, nonce); + } + + private TransactionAddedResult addToNextLayer( + final PendingTransaction pendingTransaction, final int distance) { + return addToNextLayer( + txsBySender.getOrDefault(pendingTransaction.getSender(), EMPTY_SENDER_TXS), + pendingTransaction, + distance); + } + + private TransactionAddedResult addToNextLayer( + final NavigableMap senderTxs, + final PendingTransaction pendingTransaction, + final int distance) { + final int nextLayerDistance; + if (senderTxs.isEmpty()) { + nextLayerDistance = distance; + } else { + nextLayerDistance = (int) (pendingTransaction.getNonce() - (senderTxs.lastKey() + 1)); + if (nextLayerDistance < 0) { + return REORG_SENDER; + } + } + return nextLayer.add(pendingTransaction, nextLayerDistance); + } + + private void processAdded(final PendingTransaction addedTx) { + pendingTransactions.put(addedTx.getHash(), addedTx); + final var senderTxs = txsBySender.computeIfAbsent(addedTx.getSender(), s -> new TreeMap<>()); + senderTxs.put(addedTx.getNonce(), addedTx); + increaseSpaceUsed(addedTx); + metrics.incrementAdded(addedTx.isReceivedFromLocalSource(), name()); + internalAdd(senderTxs, addedTx); + } + + protected abstract void internalAdd( + final NavigableMap senderTxs, final PendingTransaction addedTx); + + protected abstract int maxTransactionsNumber(); + + private void evict(final long spaceToFree, final int txsToEvict) { + final var evictableTx = getEvictable(); + if (evictableTx != null) { + final var lessReadySender = evictableTx.getSender(); + final var lessReadySenderTxs = txsBySender.get(lessReadySender); + + long evictedSize = 0; + int evictedCount = 0; + PendingTransaction lastTx; + // lastTx must never be null, because the sender have at least the lessReadyTx + while ((evictedSize < spaceToFree || txsToEvict > evictedCount) + && !lessReadySenderTxs.isEmpty()) { + lastTx = lessReadySenderTxs.pollLastEntry().getValue(); + processEvict(lessReadySenderTxs, lastTx); + ++evictedCount; + evictedSize += lastTx.memorySize(); + // evicted can always be added to the next layer + addToNextLayer(lessReadySenderTxs, lastTx, 0); + } + + if (lessReadySenderTxs.isEmpty()) { + txsBySender.remove(lessReadySender); + } + + final long newSpaceToFree = spaceToFree - evictedSize; + final int newTxsToEvict = txsToEvict - evictedCount; + + if ((newSpaceToFree > 0 || newTxsToEvict > 0) && !txsBySender.isEmpty()) { + // try next less valuable sender + evict(newSpaceToFree, newTxsToEvict); + } + } + } + + protected void replaced(final PendingTransaction replacedTx) { + pendingTransactions.remove(replacedTx.getHash()); + decreaseSpaceUsed(replacedTx); + metrics.incrementRemoved(replacedTx.isReceivedFromLocalSource(), REPLACED.label(), name()); + internalReplaced(replacedTx); + } + + protected abstract void internalReplaced(final PendingTransaction replacedTx); + + private TransactionAddedResult maybeReplaceTransaction(final PendingTransaction incomingTx) { + + final var existingTxs = txsBySender.get(incomingTx.getSender()); + + if (existingTxs != null) { + final var existingReadyTx = existingTxs.get(incomingTx.getNonce()); + if (existingReadyTx != null) { + + if (existingReadyTx.getHash().equals(incomingTx.getHash())) { + return ALREADY_KNOWN; + } + + if (!transactionReplacementTester.apply(existingReadyTx, incomingTx)) { + return REJECTED_UNDERPRICED_REPLACEMENT; + } + return TransactionAddedResult.createForReplacement(existingReadyTx); + } + } + return null; + } + + protected PendingTransaction processRemove( + final NavigableMap senderTxs, + final Transaction transaction, + final RemovalReason removalReason) { + final PendingTransaction removedTx = pendingTransactions.remove(transaction.getHash()); + if (removedTx != null) { + decreaseSpaceUsed(removedTx); + metrics.incrementRemoved( + removedTx.isReceivedFromLocalSource(), removalReason.label(), name()); + internalRemove(senderTxs, removedTx, removalReason); + } + return removedTx; + } + + protected PendingTransaction processEvict( + final NavigableMap senderTxs, final PendingTransaction evictedTx) { + final PendingTransaction removedTx = pendingTransactions.remove(evictedTx.getHash()); + if (removedTx != null) { + decreaseSpaceUsed(evictedTx); + metrics.incrementRemoved(evictedTx.isReceivedFromLocalSource(), EVICTED.label(), name()); + internalEvict(senderTxs, removedTx); + } + return removedTx; + } + + protected abstract void internalEvict( + final NavigableMap lessReadySenderTxs, + final PendingTransaction evictedTx); + + @Override + public final void blockAdded( + final FeeMarket feeMarket, + final BlockHeader blockHeader, + final Map maxConfirmedNonceBySender) { + LOG.atDebug() + .setMessage("Managing new added block {}") + .addArgument(blockHeader::toLogString) + .log(); + + nextLayer.blockAdded(feeMarket, blockHeader, maxConfirmedNonceBySender); + maxConfirmedNonceBySender.forEach(this::confirmed); + internalBlockAdded(blockHeader, feeMarket); + } + + protected abstract void internalBlockAdded( + final BlockHeader blockHeader, final FeeMarket feeMarket); + + final void promoteTransactions() { + int freeSlots = maxTransactionsNumber() - pendingTransactions.size(); + + while (cacheFreeSpace() > 0 && freeSlots > 0) { + final var promotedTx = nextLayer.promote(this::promotionFilter); + if (promotedTx != null) { + processAdded(promotedTx); + --freeSlots; + } else { + break; + } + } + } + + private void confirmed(final Address sender, final long maxConfirmedNonce) { + final var senderTxs = txsBySender.get(sender); + + if (senderTxs != null) { + final var confirmedTxs = senderTxs.headMap(maxConfirmedNonce, true); + final var highestNonceRemovedTx = + confirmedTxs.isEmpty() ? null : confirmedTxs.lastEntry().getValue(); + + final var itConfirmedTxs = confirmedTxs.values().iterator(); + while (itConfirmedTxs.hasNext()) { + final var confirmedTx = itConfirmedTxs.next(); + itConfirmedTxs.remove(); + processRemove(senderTxs, confirmedTx.getTransaction(), CONFIRMED); + + metrics.incrementRemoved(confirmedTx.isReceivedFromLocalSource(), "confirmed", name()); + LOG.atTrace() + .setMessage("Removed confirmed pending transactions {}") + .addArgument(confirmedTx::toTraceLog) + .log(); + } + + if (senderTxs.isEmpty()) { + txsBySender.remove(sender); + } else { + internalConfirmed(senderTxs, sender, maxConfirmedNonce, highestNonceRemovedTx); + } + } + + promoteTransactions(); + } + + protected abstract void internalConfirmed( + final NavigableMap senderTxs, + final Address sender, + final long maxConfirmedNonce, + final PendingTransaction highestNonceRemovedTx); + + protected abstract void internalRemove( + final NavigableMap senderTxs, + final PendingTransaction pendingTransaction, + final RemovalReason removalReason); + + protected abstract PendingTransaction getEvictable(); + + protected void increaseSpaceUsed(final PendingTransaction pendingTransaction) { + spaceUsed += pendingTransaction.memorySize(); + } + + protected void decreaseSpaceUsed(final PendingTransaction pendingTransaction) { + spaceUsed -= pendingTransaction.memorySize(); + } + + protected abstract long cacheFreeSpace(); + + protected abstract boolean promotionFilter(PendingTransaction pendingTransaction); + + @Override + public List getAllLocal() { + final var localTxs = + pendingTransactions.values().stream() + .filter(PendingTransaction::isReceivedFromLocalSource) + .map(PendingTransaction::getTransaction) + .collect(Collectors.toCollection(ArrayList::new)); + localTxs.addAll(nextLayer.getAllLocal()); + return localTxs; + } + + Stream stream(final Address sender) { + return txsBySender.getOrDefault(sender, EMPTY_SENDER_TXS).values().stream(); + } + + @Override + public List getAllFor(final Address sender) { + return Stream.concat(stream(sender), nextLayer.getAllFor(sender).stream()).toList(); + } + + abstract Stream stream(); + + @Override + public int count() { + return pendingTransactions.size() + nextLayer.count(); + } + + protected void notifyTransactionAdded(final PendingTransaction pendingTransaction) { + onAddedListeners.forEach( + listener -> listener.onTransactionAdded(pendingTransaction.getTransaction())); + } + + protected void notifyTransactionDropped(final PendingTransaction pendingTransaction) { + onDroppedListeners.forEach( + listener -> listener.onTransactionDropped(pendingTransaction.getTransaction())); + } + + @Override + public long subscribeToAdded(final PendingTransactionAddedListener listener) { + nextLayerOnAddedListenerId = OptionalLong.of(nextLayer.subscribeToAdded(listener)); + return onAddedListeners.subscribe(listener); + } + + @Override + public void unsubscribeFromAdded(final long id) { + nextLayerOnAddedListenerId.ifPresent(nextLayer::unsubscribeFromAdded); + onAddedListeners.unsubscribe(id); + } + + @Override + public long subscribeToDropped(final PendingTransactionDroppedListener listener) { + nextLayerOnDroppedListenerId = OptionalLong.of(nextLayer.subscribeToDropped(listener)); + return onDroppedListeners.subscribe(listener); + } + + @Override + public void unsubscribeFromDropped(final long id) { + nextLayerOnDroppedListenerId.ifPresent(nextLayer::unsubscribeFromDropped); + onDroppedListeners.unsubscribe(id); + } + + @Override + public String logStats() { + return internalLogStats() + " | " + nextLayer.logStats(); + } + + @Override + public String logSender(final Address sender) { + final var senderTxs = txsBySender.get(sender); + return name() + + "[" + + (Objects.isNull(senderTxs) ? "Empty" : senderTxs.keySet()) + + "] " + + nextLayer.logSender(sender); + } + + protected abstract String internalLogStats(); + + boolean consistencyCheck( + final Map> prevLayerTxsBySender) { + final BinaryOperator noMergeExpected = + (a, b) -> { + throw new IllegalArgumentException(); + }; + final var controlTxsBySender = + pendingTransactions.values().stream() + .collect( + Collectors.groupingBy( + PendingTransaction::getSender, + Collectors.toMap( + PendingTransaction::getNonce, + Function.identity(), + noMergeExpected, + TreeMap::new))); + + assert txsBySender.equals(controlTxsBySender) + : "pendingTransactions and txsBySender do not contain the same txs"; + + assert pendingTransactions.values().stream().mapToInt(PendingTransaction::memorySize).sum() + == spaceUsed + : "space used does not match"; + + internalConsistencyCheck(prevLayerTxsBySender); + + if (nextLayer instanceof AbstractTransactionsLayer) { + txsBySender.forEach( + (sender, txsByNonce) -> + prevLayerTxsBySender + .computeIfAbsent(sender, s -> new TreeMap<>()) + .putAll(txsByNonce)); + return ((AbstractTransactionsLayer) nextLayer).consistencyCheck(prevLayerTxsBySender); + } + return true; + } + + protected abstract void internalConsistencyCheck( + final Map> prevLayerTxsBySender); +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactions.java new file mode 100644 index 0000000000..e95d431bc6 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactions.java @@ -0,0 +1,152 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; + +import java.util.Comparator; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Holds the current set of pending transactions with the ability to iterate them based on priority + * for mining or look-up by hash. + * + *

This class is safe for use across multiple threads. + */ +public class BaseFeePrioritizedTransactions extends AbstractPrioritizedTransactions { + + private static final Logger LOG = LoggerFactory.getLogger(BaseFeePrioritizedTransactions.class); + private Optional nextBlockBaseFee; + + public BaseFeePrioritizedTransactions( + final TransactionPoolConfiguration poolConfig, + final Supplier chainHeadHeaderSupplier, + final TransactionsLayer nextLayer, + final TransactionPoolMetrics metrics, + final BiFunction + transactionReplacementTester, + final BaseFeeMarket baseFeeMarket) { + super(poolConfig, nextLayer, metrics, transactionReplacementTester); + this.nextBlockBaseFee = + Optional.of(calculateNextBlockBaseFee(baseFeeMarket, chainHeadHeaderSupplier.get())); + } + + @Override + protected int compareByFee(final PendingTransaction pt1, final PendingTransaction pt2) { + return Comparator.comparing( + (PendingTransaction pendingTransaction) -> + pendingTransaction.getTransaction().getEffectivePriorityFeePerGas(nextBlockBaseFee)) + .thenComparing( + (PendingTransaction pendingTransaction) -> + pendingTransaction.getTransaction().getMaxGasPrice()) + .thenComparing(Comparator.comparing(PendingTransaction::getNonce).reversed()) + .thenComparing(PendingTransaction::getSequence) + .compare(pt1, pt2); + } + + @Override + protected void internalBlockAdded(final BlockHeader blockHeader, final FeeMarket feeMarket) { + final BaseFeeMarket baseFeeMarket = (BaseFeeMarket) feeMarket; + final Wei newNextBlockBaseFee = calculateNextBlockBaseFee(baseFeeMarket, blockHeader); + + LOG.atTrace() + .setMessage("Updating base fee from {} to {}") + .addArgument(nextBlockBaseFee.get()::toHumanReadableString) + .addArgument(newNextBlockBaseFee::toHumanReadableString) + .log(); + + nextBlockBaseFee = Optional.of(newNextBlockBaseFee); + orderByFee.clear(); + orderByFee.addAll(pendingTransactions.values()); + } + + private Wei calculateNextBlockBaseFee( + final BaseFeeMarket baseFeeMarket, final BlockHeader blockHeader) { + return baseFeeMarket.computeBaseFee( + blockHeader.getNumber() + 1, + blockHeader.getBaseFee().orElse(Wei.ZERO), + blockHeader.getGasUsed(), + baseFeeMarket.targetGasUsed(blockHeader)); + } + + @Override + protected boolean promotionFilter(final PendingTransaction pendingTransaction) { + return nextBlockBaseFee + .map( + baseFee -> + pendingTransaction + .getTransaction() + .getEffectiveGasPrice(nextBlockBaseFee) + .greaterOrEqualThan(baseFee)) + .orElse(false); + } + + @Override + protected String internalLogStats() { + + if (orderByFee.isEmpty()) { + return "Basefee Prioritized: Empty"; + } + + final var baseFeePartition = + stream() + .map(PendingTransaction::getTransaction) + .collect( + Collectors.partitioningBy( + tx -> tx.getMaxGasPrice().greaterOrEqualThan(nextBlockBaseFee.get()), + Collectors.counting())); + final Transaction highest = orderByFee.last().getTransaction(); + final Transaction lowest = orderByFee.first().getTransaction(); + + return "Basefee Prioritized: " + + "count: " + + pendingTransactions.size() + + ", space used: " + + spaceUsed + + ", unique senders: " + + txsBySender.size() + + ", highest priority tx: [max fee: " + + highest.getMaxGasPrice().toHumanReadableString() + + ", curr prio fee: " + + highest.getEffectivePriorityFeePerGas(nextBlockBaseFee).toHumanReadableString() + + ", hash: " + + highest.getHash() + + "], lowest priority tx: [max fee: " + + lowest.getMaxGasPrice().toHumanReadableString() + + ", curr prio fee: " + + lowest.getEffectivePriorityFeePerGas(nextBlockBaseFee).toHumanReadableString() + + ", hash: " + + lowest.getHash() + + "], next block base fee: " + + nextBlockBaseFee.get().toHumanReadableString() + + ", above next base fee: " + + baseFeePartition.get(true) + + ", below next base fee: " + + baseFeePartition.get(false); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/EndLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/EndLayer.java new file mode 100644 index 0000000000..dfc9326748 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/EndLayer.java @@ -0,0 +1,171 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.DROPPED; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionAddedListener; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionDroppedListener; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.util.Subscribers; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.function.Predicate; + +public class EndLayer implements TransactionsLayer { + + private final TransactionPoolMetrics metrics; + private final Subscribers onAddedListeners = + Subscribers.create(); + + private final Subscribers onDroppedListeners = + Subscribers.create(); + + private long droppedCount = 0; + + public EndLayer(final TransactionPoolMetrics metrics) { + this.metrics = metrics; + } + + @Override + public String name() { + return "end"; + } + + @Override + public void reset() { + droppedCount = 0; + } + + @Override + public Optional getByHash(final Hash transactionHash) { + return Optional.empty(); + } + + @Override + public boolean contains(final Transaction transaction) { + return false; + } + + @Override + public List getAll() { + return List.of(); + } + + @Override + public TransactionAddedResult add(final PendingTransaction pendingTransaction, final int gap) { + notifyTransactionDropped(pendingTransaction); + metrics.incrementRemoved( + pendingTransaction.isReceivedFromLocalSource(), DROPPED.label(), name()); + ++droppedCount; + return TransactionAddedResult.DROPPED; + } + + @Override + public void remove(final PendingTransaction pendingTransaction, final RemovalReason reason) {} + + @Override + public void blockAdded( + final FeeMarket feeMarket, + final BlockHeader blockHeader, + final Map maxConfirmedNonceBySender) { + // no-op + } + + @Override + public List getAllLocal() { + return List.of(); + } + + @Override + public int count() { + return 0; + } + + @Override + public OptionalLong getNextNonceFor(final Address sender) { + return OptionalLong.empty(); + } + + @Override + public PendingTransaction promote(final Predicate promotionFilter) { + return null; + } + + @Override + public long subscribeToAdded(final PendingTransactionAddedListener listener) { + return onAddedListeners.subscribe(listener); + } + + @Override + public void unsubscribeFromAdded(final long id) { + onAddedListeners.unsubscribe(id); + } + + @Override + public long subscribeToDropped(final PendingTransactionDroppedListener listener) { + return onDroppedListeners.subscribe(listener); + } + + @Override + public void unsubscribeFromDropped(final long id) { + onDroppedListeners.unsubscribe(id); + } + + protected void notifyTransactionDropped(final PendingTransaction pendingTransaction) { + onDroppedListeners.forEach( + listener -> listener.onTransactionDropped(pendingTransaction.getTransaction())); + } + + @Override + public PendingTransaction promoteFor(final Address sender, final long nonce) { + return null; + } + + @Override + public void notifyAdded(final PendingTransaction pendingTransaction) { + // no-op + } + + @Override + public long getCumulativeUsedSpace() { + return 0; + } + + @Override + public String logStats() { + return "Dropped: " + droppedCount; + } + + @Override + public String logSender(final Address sender) { + return ""; + } + + @Override + public List getAllFor(final Address sender) { + return List.of(); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactions.java new file mode 100644 index 0000000000..725b5138d3 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactions.java @@ -0,0 +1,80 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static java.util.Comparator.comparing; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; + +import java.util.function.BiFunction; + +/** + * Holds the current set of pending transactions with the ability to iterate them based on priority + * for mining or look-up by hash. + * + *

This class is safe for use across multiple threads. + */ +public class GasPricePrioritizedTransactions extends AbstractPrioritizedTransactions { + + public GasPricePrioritizedTransactions( + final TransactionPoolConfiguration poolConfig, + final TransactionsLayer nextLayer, + final TransactionPoolMetrics metrics, + final BiFunction + transactionReplacementTester) { + super(poolConfig, nextLayer, metrics, transactionReplacementTester); + } + + @Override + protected int compareByFee(final PendingTransaction pt1, final PendingTransaction pt2) { + return comparing(PendingTransaction::isReceivedFromLocalSource) + .thenComparing(PendingTransaction::getGasPrice) + .thenComparing(PendingTransaction::getSequence) + .compare(pt1, pt2); + } + + @Override + protected void internalBlockAdded(final BlockHeader blockHeader, final FeeMarket feeMarket) { + // no-op + } + + @Override + protected boolean promotionFilter(final PendingTransaction pendingTransaction) { + return true; + } + + @Override + public String internalLogStats() { + if (orderByFee.isEmpty()) { + return "GasPrice Prioritized: Empty"; + } + + return "GasPrice Prioritized: " + + "count: " + + pendingTransactions.size() + + " space used: " + + spaceUsed + + " unique senders: " + + txsBySender.size() + + ", highest fee tx: " + + orderByFee.last().getTransaction().getGasPrice().get().toHumanReadableString() + + ", lowest fee tx: " + + orderByFee.first().getTransaction().getGasPrice().get().toHumanReadableString(); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactions.java new file mode 100644 index 0000000000..4623273799 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactions.java @@ -0,0 +1,479 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.reducing; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ALREADY_KNOWN; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.INTERNAL_ERROR; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.REORG_SENDER; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.INVALIDATED; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.REORG; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionAddedListener; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionDroppedListener; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.account.AccountState; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +import kotlin.ranges.LongRange; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LayeredPendingTransactions implements PendingTransactions { + private static final Logger LOG = LoggerFactory.getLogger(LayeredPendingTransactions.class); + private static final Logger LOG_FOR_REPLAY = LoggerFactory.getLogger("LOG_FOR_REPLAY"); + private final TransactionPoolConfiguration poolConfig; + private final Set

localSenders = new HashSet<>(); + private final AbstractPrioritizedTransactions prioritizedTransactions; + + public LayeredPendingTransactions( + final TransactionPoolConfiguration poolConfig, + final AbstractPrioritizedTransactions prioritizedTransactions) { + this.poolConfig = poolConfig; + this.prioritizedTransactions = prioritizedTransactions; + } + + @Override + public synchronized void reset() { + prioritizedTransactions.reset(); + } + + @Override + public synchronized TransactionAddedResult addRemoteTransaction( + final Transaction transaction, final Optional maybeSenderAccount) { + + return addTransaction(new PendingTransaction.Remote(transaction), maybeSenderAccount); + } + + @Override + public synchronized TransactionAddedResult addLocalTransaction( + final Transaction transaction, final Optional maybeSenderAccount) { + + final TransactionAddedResult addedResult = + addTransaction(new PendingTransaction.Local(transaction), maybeSenderAccount); + if (addedResult.isSuccess()) { + localSenders.add(transaction.getSender()); + } + return addedResult; + } + + TransactionAddedResult addTransaction( + final PendingTransaction pendingTransaction, final Optional maybeSenderAccount) { + + final long senderNonce = maybeSenderAccount.map(AccountState::getNonce).orElse(0L); + + logTransactionForReplayAdd(pendingTransaction, senderNonce); + + final long nonceDistance = pendingTransaction.getNonce() - senderNonce; + + final TransactionAddedResult nonceChecksResult = + nonceChecks(pendingTransaction, senderNonce, nonceDistance); + if (nonceChecksResult != null) { + return nonceChecksResult; + } + + try { + TransactionAddedResult result = + prioritizedTransactions.add(pendingTransaction, (int) nonceDistance); + + if (result.equals(REORG_SENDER)) { + result = reorgSenderOf(pendingTransaction, (int) nonceDistance); + } + + return result; + } catch (final Throwable throwable) { + // in case something unexpected happened, log this sender txs and force a reorg of his txs + LOG.warn( + "Unexpected error {} when adding transaction {}, current sender status {}", + throwable, + pendingTransaction.toTraceLog(), + prioritizedTransactions.logSender(pendingTransaction.getSender())); + LOG.warn("Stack trace", throwable); + reorgSenderOf(pendingTransaction, (int) nonceDistance); + return INTERNAL_ERROR; + } + } + + private TransactionAddedResult reorgSenderOf( + final PendingTransaction pendingTransaction, final int nonceDistance) { + final var existingSenderTxs = prioritizedTransactions.getAllFor(pendingTransaction.getSender()); + + // it is more performant to invalidate backward + for (int i = existingSenderTxs.size() - 1; i >= 0; --i) { + prioritizedTransactions.remove(existingSenderTxs.get(i), REORG); + } + + // add the new one and re-add all the previous + final var result = prioritizedTransactions.add(pendingTransaction, nonceDistance); + existingSenderTxs.forEach(ptx -> prioritizedTransactions.add(ptx, nonceDistance)); + LOG.atTrace() + .setMessage( + "Pending transaction {} with nonce distance {} triggered a reorg for sender {} with {} existing transactions: {}") + .addArgument(pendingTransaction::toTraceLog) + .addArgument(nonceDistance) + .addArgument(pendingTransaction::getSender) + .addArgument(existingSenderTxs::size) + .addArgument( + () -> + existingSenderTxs.stream() + .map(PendingTransaction::toTraceLog) + .collect(Collectors.joining("; "))) + .log(); + return result; + } + + private void logTransactionForReplayAdd( + final PendingTransaction pendingTransaction, final long senderNonce) { + // csv fields: sequence, addedAt, sender, sender_nonce, nonce, type, hash, rlp + LOG_FOR_REPLAY + .atTrace() + .setMessage("T,{},{},{},{},{},{},{},{}") + .addArgument(pendingTransaction.getSequence()) + .addArgument(pendingTransaction.getAddedAt()) + .addArgument(pendingTransaction.getSender()) + .addArgument(senderNonce) + .addArgument(pendingTransaction.getNonce()) + .addArgument(pendingTransaction.getTransaction().getType()) + .addArgument(pendingTransaction::getHash) + .addArgument( + () -> { + final BytesValueRLPOutput rlp = new BytesValueRLPOutput(); + pendingTransaction.getTransaction().writeTo(rlp); + return rlp.encoded().toHexString(); + }) + .log(); + } + + private void logTransactionForReplayDelete(final PendingTransaction pendingTransaction) { + // csv fields: sequence, addedAt, sender, nonce, type, hash, rlp + LOG_FOR_REPLAY + .atTrace() + .setMessage("D,{},{},{},{},{},{},{}") + .addArgument(pendingTransaction.getSequence()) + .addArgument(pendingTransaction.getAddedAt()) + .addArgument(pendingTransaction.getSender()) + .addArgument(pendingTransaction.getNonce()) + .addArgument(pendingTransaction.getTransaction().getType()) + .addArgument(pendingTransaction::getHash) + .addArgument( + () -> { + final BytesValueRLPOutput rlp = new BytesValueRLPOutput(); + pendingTransaction.getTransaction().writeTo(rlp); + return rlp.encoded().toHexString(); + }) + .log(); + } + + private TransactionAddedResult nonceChecks( + final PendingTransaction pendingTransaction, + final long senderNonce, + final long nonceDistance) { + if (nonceDistance < 0) { + LOG.atTrace() + .setMessage("Drop already confirmed transaction {}, since current sender nonce is {}") + .addArgument(pendingTransaction::toTraceLog) + .addArgument(senderNonce) + .log(); + return ALREADY_KNOWN; + } else if (nonceDistance >= poolConfig.getMaxFutureBySender()) { + LOG.atTrace() + .setMessage( + "Drop too much in the future transaction {}, since current sender nonce is {}") + .addArgument(pendingTransaction::toTraceLog) + .addArgument(senderNonce) + .log(); + return NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER; + } + return null; + } + + @Override + public void evictOldTransactions() {} + + @Override + public synchronized List getLocalTransactions() { + return prioritizedTransactions.getAllLocal(); + } + + @Override + public synchronized boolean isLocalSender(final Address sender) { + return localSenders.contains(sender); + } + + @Override + // There's a small edge case here we could encounter. + // When we pass an upgrade block that has a new transaction type, we start allowing transactions + // of that new type into our pool. + // If we then reorg to a block lower than the upgrade block height _and_ we create a block, that + // block could end up with transactions of the new type. + // This seems like it would be very rare but worth it to document that we don't handle that case + // right now. + public synchronized void selectTransactions( + final PendingTransactions.TransactionSelector selector) { + final List invalidTransactions = new ArrayList<>(); + final Set alreadyChecked = new HashSet<>(); + final AtomicBoolean completed = new AtomicBoolean(false); + + prioritizedTransactions.stream() + .takeWhile(unused -> !completed.get()) + .peek( + highPrioPendingTx -> + LOG.atDebug() + .setMessage("highPrioPendingTx {}, senderTxs {}") + .addArgument(highPrioPendingTx::toTraceLog) + .addArgument( + () -> + prioritizedTransactions.stream(highPrioPendingTx.getSender()) + .map(PendingTransaction::toTraceLog) + .collect(Collectors.joining(", "))) + .log()) + .forEach( + highPrioPendingTx -> + prioritizedTransactions.stream(highPrioPendingTx.getSender()) + .takeWhile(unused -> !completed.get()) + .filter( + candidatePendingTx -> + !alreadyChecked.contains(candidatePendingTx.getHash())) + .filter( + candidatePendingTx -> + candidatePendingTx.getNonce() <= highPrioPendingTx.getNonce()) + .forEach( + candidatePendingTx -> { + alreadyChecked.add(candidatePendingTx.getHash()); + switch (selector.evaluateTransaction( + candidatePendingTx.getTransaction())) { + case CONTINUE: + LOG.atTrace() + .setMessage("CONTINUE: Transaction {}") + .addArgument(candidatePendingTx::toTraceLog) + .log(); + break; + case DELETE_TRANSACTION_AND_CONTINUE: + invalidTransactions.add(candidatePendingTx); + LOG.atTrace() + .setMessage("DELETE_TRANSACTION_AND_CONTINUE: Transaction {}") + .addArgument(candidatePendingTx::toTraceLog) + .log(); + logTransactionForReplayDelete(candidatePendingTx); + break; + case COMPLETE_OPERATION: + completed.set(true); + LOG.atTrace() + .setMessage("COMPLETE_OPERATION: Transaction {}") + .addArgument(candidatePendingTx::toTraceLog) + .log(); + break; + } + })); + + invalidTransactions.forEach( + invalidTx -> prioritizedTransactions.remove(invalidTx, INVALIDATED)); + } + + @Override + public long maxSize() { + return -1; + } + + @Override + public synchronized int size() { + return prioritizedTransactions.count(); + } + + @Override + public synchronized boolean containsTransaction(final Transaction transaction) { + return prioritizedTransactions.contains(transaction); + } + + @Override + public synchronized Optional getTransactionByHash(final Hash transactionHash) { + return prioritizedTransactions.getByHash(transactionHash); + } + + @Override + public synchronized List getPendingTransactions() { + return prioritizedTransactions.getAll(); + } + + @Override + public long subscribePendingTransactions(final PendingTransactionAddedListener listener) { + return prioritizedTransactions.subscribeToAdded(listener); + } + + @Override + public void unsubscribePendingTransactions(final long id) { + prioritizedTransactions.unsubscribeFromAdded(id); + } + + @Override + public long subscribeDroppedTransactions(final PendingTransactionDroppedListener listener) { + return prioritizedTransactions.subscribeToDropped(listener); + } + + @Override + public void unsubscribeDroppedTransactions(final long id) { + prioritizedTransactions.unsubscribeFromDropped(id); + } + + @Override + public OptionalLong getNextNonceForSender(final Address sender) { + return prioritizedTransactions.getNextNonceFor(sender); + } + + @Override + public synchronized void manageBlockAdded( + final BlockHeader blockHeader, + final List confirmedTransactions, + final List reorgTransactions, + final FeeMarket feeMarket) { + LOG.atDebug() + .setMessage("Managing new added block {}") + .addArgument(blockHeader::toLogString) + .log(); + + final var maxConfirmedNonceBySender = maxNonceBySender(confirmedTransactions); + + final var reorgNonceRangeBySender = nonceRangeBySender(reorgTransactions); + + try { + prioritizedTransactions.blockAdded(feeMarket, blockHeader, maxConfirmedNonceBySender); + } catch (final Throwable throwable) { + LOG.warn( + "Unexpected error {} when managing added block {}, maxNonceBySender {}, reorgNonceRangeBySender {}", + throwable, + blockHeader.toLogString(), + maxConfirmedNonceBySender, + reorgTransactions); + LOG.warn("Stack trace", throwable); + } + + logBlockHeaderForReplay(blockHeader, maxConfirmedNonceBySender, reorgNonceRangeBySender); + } + + private void logBlockHeaderForReplay( + final BlockHeader blockHeader, + final Map maxConfirmedNonceBySender, + final Map reorgNonceRangeBySender) { + // block number, block hash, sender, max nonce ..., rlp + LOG_FOR_REPLAY + .atTrace() + .setMessage("B,{},{},{},R,{},{}") + .addArgument(blockHeader.getNumber()) + .addArgument(blockHeader.getBlockHash()) + .addArgument( + () -> + maxConfirmedNonceBySender.entrySet().stream() + .map(e -> e.getKey().toHexString() + "," + e.getValue()) + .collect(Collectors.joining(","))) + .addArgument( + () -> + reorgNonceRangeBySender.entrySet().stream() + .map( + e -> + e.getKey().toHexString() + + "," + + e.getValue().getStart() + + "," + + e.getValue().getEndInclusive()) + .collect(Collectors.joining(","))) + .addArgument( + () -> { + final BytesValueRLPOutput rlp = new BytesValueRLPOutput(); + blockHeader.writeTo(rlp); + return rlp.encoded().toHexString(); + }) + .log(); + } + + private Map maxNonceBySender(final List confirmedTransactions) { + return confirmedTransactions.stream() + .collect( + groupingBy( + Transaction::getSender, mapping(Transaction::getNonce, reducing(0L, Math::max)))); + } + + private Map nonceRangeBySender( + final List confirmedTransactions) { + + class MutableLongRange { + long start = Long.MAX_VALUE; + long end = 0; + + void update(final long nonce) { + if (nonce < start) { + start = nonce; + } + if (nonce > end) { + end = nonce; + } + } + + MutableLongRange combine(final MutableLongRange other) { + update(other.start); + update(other.end); + return this; + } + + LongRange toImmutable() { + return new LongRange(start, end); + } + } + + return confirmedTransactions.stream() + .collect( + groupingBy( + Transaction::getSender, + mapping( + Transaction::getNonce, + Collector.of( + MutableLongRange::new, + MutableLongRange::update, + MutableLongRange::combine, + MutableLongRange::toImmutable)))); + } + + @Override + public synchronized String toTraceLog() { + return ""; + } + + @Override + public synchronized String logStats() { + return prioritizedTransactions.logStats(); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java new file mode 100644 index 0000000000..6527031c91 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java @@ -0,0 +1,221 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.PROMOTED; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; + +import java.util.Comparator; +import java.util.Map; +import java.util.NavigableMap; +import java.util.NavigableSet; +import java.util.Optional; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.BiFunction; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ReadyTransactions extends AbstractSequentialTransactionsLayer { + + private final NavigableSet orderByMaxFee = + new TreeSet<>( + Comparator.comparing((PendingTransaction pt) -> pt.getTransaction().getMaxGasPrice()) + .thenComparing(PendingTransaction::getSequence)); + + public ReadyTransactions( + final TransactionPoolConfiguration poolConfig, + final TransactionsLayer nextLayer, + final TransactionPoolMetrics metrics, + final BiFunction + transactionReplacementTester) { + super(poolConfig, nextLayer, transactionReplacementTester, metrics); + } + + @Override + public String name() { + return "ready"; + } + + @Override + public void reset() { + super.reset(); + orderByMaxFee.clear(); + } + + @Override + protected long cacheFreeSpace() { + return poolConfig.getPendingTransactionsLayerMaxCapacityBytes() - getLayerSpaceUsed(); + } + + @Override + protected TransactionAddedResult canAdd( + final PendingTransaction pendingTransaction, final int gap) { + final var senderTxs = txsBySender.get(pendingTransaction.getSender()); + + if (hasExpectedNonce(senderTxs, pendingTransaction, gap)) { + return TransactionAddedResult.ADDED; + } + + return TransactionAddedResult.TRY_NEXT_LAYER; + } + + @Override + protected void internalAdd( + final NavigableMap senderTxs, + final PendingTransaction pendingTransaction) { + if (senderTxs.firstKey() == pendingTransaction.getNonce()) { + // replace previous if exists + if (senderTxs.size() > 1) { + final PendingTransaction secondTx = senderTxs.get(pendingTransaction.getNonce() + 1); + orderByMaxFee.remove(secondTx); + } + orderByMaxFee.add(pendingTransaction); + } + } + + @Override + protected int maxTransactionsNumber() { + return Integer.MAX_VALUE; + } + + @Override + protected void internalRemove( + final NavigableMap senderTxs, + final PendingTransaction removedTx, + final RemovalReason removalReason) { + orderByMaxFee.remove(removedTx); + if (!senderTxs.isEmpty()) { + orderByMaxFee.add(senderTxs.firstEntry().getValue()); + } + } + + @Override + protected void internalReplaced(final PendingTransaction replacedTx) { + orderByMaxFee.remove(replacedTx); + } + + @Override + protected void internalBlockAdded(final BlockHeader blockHeader, final FeeMarket feeMarket) { + // no-op + } + + @Override + protected PendingTransaction getEvictable() { + return orderByMaxFee.first(); + } + + @Override + protected boolean promotionFilter(final PendingTransaction pendingTransaction) { + return true; + } + + @Override + public Stream stream() { + return orderByMaxFee.descendingSet().stream() + .map(PendingTransaction::getSender) + .flatMap(sender -> txsBySender.get(sender).values().stream()); + } + + @Override + public PendingTransaction promote(final Predicate promotionFilter) { + + final var maybePromotedTx = + orderByMaxFee.descendingSet().stream() + .filter(candidateTx -> promotionFilter.test(candidateTx)) + .findFirst(); + + return maybePromotedTx + .map( + promotedTx -> { + final var senderTxs = txsBySender.get(promotedTx.getSender()); + // we always promote the first tx of a sender, so remove the first entry + senderTxs.pollFirstEntry(); + processRemove(senderTxs, promotedTx.getTransaction(), PROMOTED); + + // now that we have space, promote from the next layer + promoteTransactions(); + + if (senderTxs.isEmpty()) { + txsBySender.remove(promotedTx.getSender()); + } + return promotedTx; + }) + .orElse(null); + } + + @Override + public String internalLogStats() { + if (orderByMaxFee.isEmpty()) { + return "Ready: Empty"; + } + + final Transaction top = orderByMaxFee.last().getTransaction(); + final Transaction last = orderByMaxFee.first().getTransaction(); + + return "Ready: " + + "count=" + + pendingTransactions.size() + + ", space used: " + + spaceUsed + + ", unique senders: " + + txsBySender.size() + + ", top by max fee[max fee:" + + top.getMaxGasPrice().toHumanReadableString() + + ", hash: " + + top.getHash() + + "], last by max fee [max fee: " + + last.getMaxGasPrice().toHumanReadableString() + + ", hash: " + + last.getHash() + + "]"; + } + + @Override + protected void internalConsistencyCheck( + final Map> prevLayerTxsBySender) { + super.internalConsistencyCheck(prevLayerTxsBySender); + + final var minNonceBySender = + pendingTransactions.values().stream() + .collect( + Collectors.groupingBy( + PendingTransaction::getSender, + Collectors.minBy(Comparator.comparingLong(PendingTransaction::getNonce)))); + + final var controlOrderByMaxFee = new TreeSet<>(orderByMaxFee.comparator()); + controlOrderByMaxFee.addAll(minNonceBySender.values().stream().map(Optional::get).toList()); + + final var itControl = controlOrderByMaxFee.iterator(); + final var itCurrent = orderByMaxFee.iterator(); + + while (itControl.hasNext()) { + assert itControl.next().equals(itCurrent.next()) + : "orderByMaxFee does not match pendingTransactions"; + } + + assert itCurrent.hasNext() == false + : "orderByMaxFee has more elements than pendingTransactions"; + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java new file mode 100644 index 0000000000..103fe74581 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java @@ -0,0 +1,374 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.INVALIDATED; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.PROMOTED; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.NavigableSet; +import java.util.Objects; +import java.util.OptionalLong; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.BiFunction; +import java.util.function.Predicate; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +public class SparseTransactions extends AbstractTransactionsLayer { + private final NavigableSet sparseEvictionOrder = + new TreeSet<>(Comparator.comparing(PendingTransaction::getSequence)); + private final Map gapBySender = new HashMap<>(); + private final List> orderByGap; + + public SparseTransactions( + final TransactionPoolConfiguration poolConfig, + final TransactionsLayer nextLayer, + final TransactionPoolMetrics metrics, + final BiFunction + transactionReplacementTester) { + super(poolConfig, nextLayer, transactionReplacementTester, metrics); + orderByGap = new ArrayList<>(poolConfig.getMaxFutureBySender()); + IntStream.range(0, poolConfig.getMaxFutureBySender()) + .forEach(i -> orderByGap.add(new HashSet<>())); + } + + @Override + public String name() { + return "sparse"; + } + + @Override + protected long cacheFreeSpace() { + return poolConfig.getPendingTransactionsLayerMaxCapacityBytes() - getLayerSpaceUsed(); + } + + @Override + protected boolean gapsAllowed() { + return true; + } + + @Override + public void reset() { + super.reset(); + sparseEvictionOrder.clear(); + gapBySender.clear(); + orderByGap.forEach(Set::clear); + } + + @Override + protected TransactionAddedResult canAdd( + final PendingTransaction pendingTransaction, final int gap) { + gapBySender.compute( + pendingTransaction.getSender(), + (sender, currGap) -> { + if (currGap == null) { + orderByGap.get(gap).add(sender); + return gap; + } + if (pendingTransaction.getNonce() < txsBySender.get(sender).firstKey()) { + orderByGap.get(currGap).remove(sender); + orderByGap.get(gap).add(sender); + return gap; + } + return currGap; + }); + + return TransactionAddedResult.ADDED; + } + + @Override + protected void internalAdd( + final NavigableMap senderTxs, final PendingTransaction addedTx) { + sparseEvictionOrder.add(addedTx); + } + + @Override + protected int maxTransactionsNumber() { + return Integer.MAX_VALUE; + } + + @Override + protected void internalReplaced(final PendingTransaction replacedTx) { + sparseEvictionOrder.remove(replacedTx); + } + + @Override + protected void internalBlockAdded(final BlockHeader blockHeader, final FeeMarket feeMarket) {} + + @Override + public PendingTransaction promote(final Predicate promotionFilter) { + final PendingTransaction promotedTx = + orderByGap.get(0).stream() + .map(txsBySender::get) + .map(NavigableMap::values) + .flatMap(Collection::stream) + .filter(promotionFilter) + .findFirst() + .orElse(null); + + if (promotedTx != null) { + final Address sender = promotedTx.getSender(); + final var senderTxs = txsBySender.get(sender); + senderTxs.pollFirstEntry(); + processRemove(senderTxs, promotedTx.getTransaction(), PROMOTED); + if (senderTxs.isEmpty()) { + txsBySender.remove(sender); + orderByGap.get(0).remove(sender); + gapBySender.remove(sender); + } else { + final long firstNonce = senderTxs.firstKey(); + final int newGap = (int) (firstNonce - (promotedTx.getNonce() + 1)); + if (newGap != 0) { + updateGap(sender, 0, newGap); + } + } + } + + return promotedTx; + } + + @Override + public void remove(final PendingTransaction invalidatedTx, final RemovalReason reason) { + + final var senderTxs = txsBySender.get(invalidatedTx.getSender()); + if (senderTxs != null && senderTxs.containsKey(invalidatedTx.getNonce())) { + // gaps are allowed here then just remove + senderTxs.remove(invalidatedTx.getNonce()); + processRemove(senderTxs, invalidatedTx.getTransaction(), reason); + if (senderTxs.isEmpty()) { + txsBySender.remove(invalidatedTx.getSender()); + } + } else { + nextLayer.remove(invalidatedTx, reason); + } + } + + @Override + protected void internalConfirmed( + final NavigableMap senderTxs, + final Address sender, + final long maxConfirmedNonce, + final PendingTransaction highestNonceRemovedTx) { + + if (highestNonceRemovedTx != null) { + final int currGap = gapBySender.get(sender); + final int newGap = (int) (senderTxs.firstKey() - (highestNonceRemovedTx.getNonce() + 1)); + if (currGap != newGap) { + updateGap(sender, currGap, newGap); + } + } else { + final int currGap = gapBySender.get(sender); + final int newGap = (int) (senderTxs.firstKey() - (maxConfirmedNonce + 1)); + if (newGap < currGap) { + updateGap(sender, currGap, newGap); + } + } + } + + @Override + protected void internalEvict( + final NavigableMap lessReadySenderTxs, + final PendingTransaction evictedTx) { + sparseEvictionOrder.remove(evictedTx); + + if (lessReadySenderTxs.isEmpty()) { + deleteGap(evictedTx.getSender()); + } + } + + @Override + protected void internalRemove( + final NavigableMap senderTxs, + final PendingTransaction removedTx, + final RemovalReason removalReason) { + + sparseEvictionOrder.remove(removedTx); + + final Address sender = removedTx.getSender(); + + if (senderTxs != null && !senderTxs.isEmpty()) { + final int deltaGap = (int) (senderTxs.firstKey() - removedTx.getNonce()); + if (deltaGap > 0) { + final int currGap = gapBySender.get(sender); + final int newGap; + if (removalReason.equals(INVALIDATED)) { + newGap = currGap + deltaGap; + } else { + newGap = deltaGap - 1; + } + if (currGap != newGap) { + updateGap(sender, currGap, newGap); + } + } + + } else { + deleteGap(sender); + } + } + + private void deleteGap(final Address sender) { + orderByGap.get(gapBySender.remove(sender)).remove(sender); + } + + @Override + protected PendingTransaction getEvictable() { + return sparseEvictionOrder.first(); + } + + @Override + protected boolean promotionFilter(final PendingTransaction pendingTransaction) { + return false; + } + + @Override + public Stream stream() { + return sparseEvictionOrder.descendingSet().stream(); + } + + @Override + public OptionalLong getNextNonceFor(final Address sender) { + final Integer gap = gapBySender.get(sender); + if (gap != null && gap == 0) { + final var senderTxs = txsBySender.get(sender); + var currNonce = senderTxs.firstKey(); + for (final var nextNonce : senderTxs.keySet()) { + if (nextNonce > currNonce + 1) { + break; + } + currNonce = nextNonce; + } + return OptionalLong.of(currNonce + 1); + } + return OptionalLong.empty(); + } + + @Override + protected void internalNotifyAdded( + final NavigableMap senderTxs, + final PendingTransaction pendingTransaction) { + final Address sender = pendingTransaction.getSender(); + final Integer currGap = gapBySender.get(sender); + if (currGap != null) { + final int newGap = (int) (senderTxs.firstKey() - (pendingTransaction.getNonce() + 1)); + if (newGap < currGap) { + updateGap(sender, currGap, newGap); + } + } + } + + @Override + public String logSender(final Address sender) { + final var senderTxs = txsBySender.get(sender); + return name() + + "[" + + (Objects.isNull(senderTxs) + ? "Empty" + : "gap(" + gapBySender.get(sender) + ") " + senderTxs.keySet()) + + "] " + + nextLayer.logSender(sender); + } + + @Override + public String internalLogStats() { + if (sparseEvictionOrder.isEmpty()) { + return "Sparse: Empty"; + } + + final Transaction newest = sparseEvictionOrder.last().getTransaction(); + final Transaction oldest = sparseEvictionOrder.first().getTransaction(); + + return "Sparse: " + + "count=" + + pendingTransactions.size() + + ", space used: " + + spaceUsed + + ", unique senders: " + + txsBySender.size() + + ", oldest [gap: " + + gapBySender.get(oldest.getSender()) + + ", max fee:" + + oldest.getMaxGasPrice().toHumanReadableString() + + ", hash: " + + oldest.getHash() + + "], newest [gap: " + + gapBySender.get(newest.getSender()) + + ", max fee: " + + newest.getMaxGasPrice().toHumanReadableString() + + ", hash: " + + newest.getHash() + + "]"; + } + + private void updateGap(final Address sender, final int currGap, final int newGap) { + orderByGap.get(currGap).remove(sender); + orderByGap.get(newGap).add(sender); + gapBySender.put(sender, newGap); + } + + @Override + protected void internalConsistencyCheck( + final Map> prevLayerTxsBySender) { + txsBySender.values().stream() + .filter(senderTxs -> senderTxs.size() > 1) + .map(NavigableMap::entrySet) + .map(Set::iterator) + .forEach( + itNonce -> { + PendingTransaction firstTx = itNonce.next().getValue(); + + prevLayerTxsBySender.computeIfPresent( + firstTx.getSender(), + (sender, txsByNonce) -> { + final long prevLayerMaxNonce = txsByNonce.lastKey(); + assert prevLayerMaxNonce < firstTx.getNonce() + : "first nonce is not greater than previous layer last nonce"; + + final int gap = (int) (firstTx.getNonce() - (prevLayerMaxNonce + 1)); + assert gapBySender.get(firstTx.getSender()).equals(gap) : "gap mismatch"; + assert orderByGap.get(gap).contains(firstTx.getSender()) + : "orderByGap sender not found"; + + return txsByNonce; + }); + + long prevNonce = firstTx.getNonce(); + + while (itNonce.hasNext()) { + final long currNonce = itNonce.next().getKey(); + assert prevNonce < currNonce : "non incremental nonce"; + prevNonce = currNonce; + } + }); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java new file mode 100644 index 0000000000..ab4ec3b5a7 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java @@ -0,0 +1,103 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionAddedListener; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionDroppedListener; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.function.Predicate; + +public interface TransactionsLayer { + + String name(); + + void reset(); + + Optional getByHash(Hash transactionHash); + + boolean contains(Transaction transaction); + + List getAll(); + + TransactionAddedResult add(PendingTransaction pendingTransaction, int gap); + + void remove(PendingTransaction pendingTransaction, RemovalReason reason); + + void blockAdded( + FeeMarket feeMarket, + BlockHeader blockHeader, + final Map maxConfirmedNonceBySender); + + List getAllLocal(); + + int count(); + + OptionalLong getNextNonceFor(Address sender); + + PendingTransaction promote(Predicate promotionFilter); + + long subscribeToAdded(PendingTransactionAddedListener listener); + + void unsubscribeFromAdded(long id); + + long subscribeToDropped(PendingTransactionDroppedListener listener); + + void unsubscribeFromDropped(long id); + + PendingTransaction promoteFor(Address sender, long nonce); + + void notifyAdded(PendingTransaction pendingTransaction); + + long getCumulativeUsedSpace(); + + String logStats(); + + String logSender(Address sender); + + List getAllFor(Address sender); + + enum RemovalReason { + CONFIRMED, + CROSS_LAYER_REPLACED, + EVICTED, + DROPPED, + FOLLOW_INVALIDATED, + INVALIDATED, + PROMOTED, + REPLACED, + REORG; + + private final String label; + + RemovalReason() { + this.label = name().toLowerCase(); + } + + public String label() { + return label; + } + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/package-info.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/package-info.java new file mode 100644 index 0000000000..fbc9da9fe5 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/package-info.java @@ -0,0 +1,76 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * This implements a new transaction pool (txpool for brevity), with the main goal to better manage + * nonce gaps, i.e. the possibility that the list of transactions that we see for a sender could not + * be in order neither contiguous, that could happen just of the way there are broadcast on the p2p + * network or intentionally to try to spam the txpool with non-executable transactions (transactions + * that could not be included in a future block), so the goal is to try to keep in the pool + * transactions that could be selected for a future block proposal, and at the same time, without + * penalizing legitimate unordered transactions, that are only temporary non-executable. + * + *

It is disabled by default, to enable use the option {@code Xlayered-tx-pool=true} + * + *

The main idea is to organize the txpool in an arbitrary number of layers, where each layer has + * specific rules and constraints that determine if a transaction belong or not to that layer and + * also the way transactions move across layers. + * + *

Some design choices that apply to all layers are that a transaction can only be in one layer + * at any time, and that layers are chained by priority, so the first layer has the transactions + * that are candidate for a block proposal, and the last layer basically is where transactions are + * dropped. Layers are meant to be added and removed in case of specific future needs. When adding a + * new transaction, it is first tried on the first layer, if it is not accepted then the next one is + * tried and so on. Layers could be limited by transaction number of by space, and when a layer if + * full, it overflows to the next one and so on, instead when some space is freed, usually when + * transactions are removed since confirmed in a block, transactions from the next layer are + * promoted until there is space. + * + *

The current implementation is based on 3 layers, plus the last one that just drop every + * transaction when the previous layers are full. The 3 layers are, in order: + * + *

    + *
  • Prioritized + *
  • Ready + *
  • Sparse + *
+ * + *

Prioritized: This is where candidate transactions are selected for creating a new block. + * Transactions ordered by the effective priority fee, and it is limited by size, 2000 by default, + * to reduce the overhead of the sorting and because that number is enough to fill any block, at the + * current gas limit. Does not allow nonce gaps, and the first transaction for each sender must be + * the next one for that sender. Eviction is done removing the transaction with the higher nonce for + * the sender of the less valuable transaction, to avoid creating nonce gaps, evicted transactions + * go into the next layer Ready. + * + *

Ready: Similar to the Prioritized, it does not allow nonce gaps, and the first transaction for + * each sender must be the next one for that sender, but it is limited by space instead of count, + * thus allowing many more transactions, think about this layer like a buffer for the Prioritized. + * Since it is meant to keep ten to hundreds of thousand of transactions, it does not have a full + * ordering, like the previous, but only the first transaction for each sender is ordered using a + * stable value that is the max fee per gas. Eviction is the same as the Prioritized, and evicted + * transaction go into the next layer Sparse. + * + *

Sparse: This is the first layer where nonce gaps are allowed and where the first transaction + * for a sender could not be the next expected one for that sender. The main purpose of this layer + * is to act as a purgatory for temporary unordered and/or non-contiguous transactions, so that they + * could become ready asap the missing transactions arrive, or they are eventually evicted. It also + * keeps the less valuable ready transactions, that are evicted from the previous layer. It is + * limited by space, and eviction select the oldest transaction first, that is sent to the End Layer + * that just drop it. When promoting to the prev layer Ready, only transactions that will not create + * nonce gaps are selected, for that we need to keep track of the nonce distance for each sender. So + * we can say that is ordered by nonce distance for promotion. + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsSorter.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsSorter.java index bde41913c7..530175934e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsSorter.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsSorter.java @@ -14,24 +14,25 @@ */ package org.hyperledger.besu.ethereum.eth.transactions.sorter; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus.ADDED; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus.LOWER_NONCE_INVALID_TRANSACTION_KNOWN; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus.NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus.REJECTED_UNDERPRICED_REPLACEMENT; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ADDED; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ALREADY_KNOWN; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.LOWER_NONCE_INVALID_TRANSACTION_KNOWN; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.REJECTED_UNDERPRICED_REPLACEMENT; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.AccountTransactionOrder; -import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionAddedListener; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionDroppedListener; -import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionListener; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; -import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolReplacementHandler; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.AccountState; import org.hyperledger.besu.metrics.BesuMetricCategory; @@ -41,8 +42,8 @@ import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; import org.hyperledger.besu.util.Subscribers; import java.time.Clock; -import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -83,7 +84,7 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa protected final LowestInvalidNonceCache lowestInvalidKnownNonceCache = new LowestInvalidNonceCache(DEFAULT_LOWEST_INVALID_KNOWN_NONCE_CACHE); - protected final Subscribers pendingTransactionSubscribers = + protected final Subscribers pendingTransactionSubscribers = Subscribers.create(); protected final Subscribers transactionDroppedListeners = @@ -144,11 +145,14 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa @Override public void evictOldTransactions() { - final Instant removeTransactionsBefore = - clock.instant().minus(poolConfig.getPendingTxRetentionPeriod(), ChronoUnit.HOURS); + final long removeTransactionsBefore = + clock + .instant() + .minus(poolConfig.getPendingTxRetentionPeriod(), ChronoUnit.HOURS) + .toEpochMilli(); pendingTransactions.values().stream() - .filter(transaction -> transaction.getAddedToPoolAt().isBefore(removeTransactionsBefore)) + .filter(transaction -> transaction.getAddedAt() < removeTransactionsBefore) .forEach( transactionInfo -> { LOG.atTrace() @@ -168,7 +172,7 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa } @Override - public TransactionAddedStatus addRemoteTransaction( + public TransactionAddedResult addRemoteTransaction( final Transaction transaction, final Optional maybeSenderAccount) { if (lowestInvalidKnownNonceCache.hasInvalidLowerNonce(transaction)) { @@ -181,8 +185,8 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa } final PendingTransaction pendingTransaction = - new PendingTransaction(transaction, false, clock.instant()); - final TransactionAddedStatus transactionAddedStatus = + new PendingTransaction.Remote(transaction, clock.millis()); + final TransactionAddedResult transactionAddedStatus = addTransaction(pendingTransaction, maybeSenderAccount); if (transactionAddedStatus.equals(ADDED)) { lowestInvalidKnownNonceCache.registerValidTransaction(transaction); @@ -192,11 +196,11 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa } @Override - public TransactionAddedStatus addLocalTransaction( + public TransactionAddedResult addLocalTransaction( final Transaction transaction, final Optional maybeSenderAccount) { - final TransactionAddedStatus transactionAdded = + final TransactionAddedResult transactionAdded = addTransaction( - new PendingTransaction(transaction, true, clock.instant()), maybeSenderAccount); + new PendingTransaction.Local(transaction, clock.millis()), maybeSenderAccount); if (transactionAdded.equals(ADDED)) { localSenders.add(transaction.getSender()); localTransactionAddedCounter.inc(); @@ -204,13 +208,25 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa return transactionAdded; } - @Override - public void removeTransaction(final Transaction transaction) { + void removeTransaction(final Transaction transaction) { removeTransaction(transaction, false); notifyTransactionDropped(transaction); } @Override + public void manageBlockAdded( + final BlockHeader blockHeader, + final List confirmedTransactions, + final List reorgTransactions, + final FeeMarket feeMarket) { + synchronized (lock) { + confirmedTransactions.forEach(this::transactionAddedToBlock); + manageBlockAdded(blockHeader); + } + } + + protected abstract void manageBlockAdded(final BlockHeader blockHeader); + public void transactionAddedToBlock(final Transaction transaction) { removeTransaction(transaction, true); lowestInvalidKnownNonceCache.registerValidTransaction(transaction); @@ -250,8 +266,8 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa switch (result) { case DELETE_TRANSACTION_AND_CONTINUE: transactionsToRemove.add(transactionToProcess); - signalInvalidAndGetDependentTransactions(transactionToProcess) - .forEach(transactionsToRemove::add); + transactionsToRemove.addAll( + signalInvalidAndGetDependentTransactions(transactionToProcess)); break; case CONTINUE: break; @@ -275,7 +291,7 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa .map(PendingTransaction::getTransaction)); } - private TransactionAddedStatus addTransactionForSenderAndNonce( + private TransactionAddedResult addTransactionForSenderAndNonce( final PendingTransaction pendingTransaction, final Optional maybeSenderAccount) { PendingTransactionsForSender pendingTxsForSender = @@ -357,8 +373,8 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa } @Override - public boolean containsTransaction(final Hash transactionHash) { - return pendingTransactions.containsKey(transactionHash); + public boolean containsTransaction(final Transaction transaction) { + return pendingTransactions.containsKey(transaction.getHash()); } @Override @@ -368,12 +384,12 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa } @Override - public Set getPendingTransactions() { - return new HashSet<>(pendingTransactions.values()); + public List getPendingTransactions() { + return new ArrayList<>(pendingTransactions.values()); } @Override - public long subscribePendingTransactions(final PendingTransactionListener listener) { + public long subscribePendingTransactions(final PendingTransactionAddedListener listener) { return pendingTransactionSubscribers.subscribe(listener); } @@ -401,9 +417,6 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa : pendingTransactionsForSender.maybeNextNonce(); } - @Override - public abstract void manageBlockAdded(final Block block); - private void removeTransaction(final Transaction transaction, final boolean addedToBlock) { synchronized (lock) { final PendingTransaction removedPendingTx = pendingTransactions.remove(transaction.getHash()); @@ -422,7 +435,7 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa protected abstract void prioritizeTransaction(final PendingTransaction pendingTransaction); - private TransactionAddedStatus addTransaction( + private TransactionAddedResult addTransaction( final PendingTransaction pendingTransaction, final Optional maybeSenderAccount) { final Transaction transaction = pendingTransaction.getTransaction(); synchronized (lock) { @@ -431,7 +444,7 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa .setMessage("Already known transaction {}") .addArgument(pendingTransaction::toTraceLog) .log(); - return TransactionAddedStatus.ALREADY_KNOWN; + return ALREADY_KNOWN; } if (transaction.getNonce() - maybeSenderAccount.map(AccountState::getNonce).orElse(0L) @@ -450,10 +463,10 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa return NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER; } - final TransactionAddedStatus transactionAddedStatus = + final TransactionAddedResult transactionAddedStatus = addTransactionForSenderAndNonce(pendingTransaction, maybeSenderAccount); - if (!transactionAddedStatus.equals(TransactionAddedStatus.ADDED)) { + if (!transactionAddedStatus.equals(ADDED)) { return transactionAddedStatus; } @@ -465,7 +478,7 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa } } notifyTransactionAdded(pendingTransaction.getTransaction()); - return TransactionAddedStatus.ADDED; + return ADDED; } protected abstract PendingTransaction getLeastPriorityTransaction(); @@ -483,46 +496,24 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa } @Override - public String toTraceLog( - final boolean withTransactionsBySender, final boolean withLowestInvalidNonce) { + public String logStats() { + return "Pending " + pendingTransactions.size(); + } + + @Override + public String toTraceLog() { synchronized (lock) { StringBuilder sb = new StringBuilder( - "Transactions in order { " + "Prioritized transactions { " + StreamSupport.stream( Spliterators.spliteratorUnknownSize( prioritizedTransactions(), Spliterator.ORDERED), false) - .map( - pendingTx -> { - PendingTransactionsForSender pendingTxsForSender = - transactionsBySender.get(pendingTx.getSender()); - long nonceDistance = - pendingTx.getNonce() - pendingTxsForSender.getSenderAccountNonce(); - return "nonceDistance: " - + nonceDistance - + ", senderAccount: " - + pendingTxsForSender.getSenderAccount() - + ", " - + pendingTx.toTraceLog(); - }) + .map(PendingTransaction::toTraceLog) .collect(Collectors.joining("; ")) + " }"); - if (withTransactionsBySender) { - sb.append( - ", Transactions by sender { " - + transactionsBySender.entrySet().stream() - .map(e -> "(" + e.getKey() + ") " + e.getValue().toTraceLog()) - .collect(Collectors.joining("; ")) - + " }"); - } - if (withLowestInvalidNonce) { - sb.append( - ", Lowest invalid nonce by sender cache {" - + lowestInvalidKnownNonceCache.toTraceLog() - + "}"); - } return sb.toString(); } } @@ -547,10 +538,14 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransa .map(PendingTransaction::getTransaction) .collect(Collectors.toList()); } - return List.of(); } + @Override + public void signalInvalidAndRemoveDependentTransactions(final Transaction transaction) { + signalInvalidAndGetDependentTransactions(transaction).forEach(this::removeTransaction); + } + @Override public boolean isLocalSender(final Address sender) { return poolConfig.getDisableLocalTransactions() ? false : localSenders.contains(sender); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/BaseFeePendingTransactionsSorter.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/BaseFeePendingTransactionsSorter.java index 6bb9e7caf0..aeeb2b3e46 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/BaseFeePendingTransactionsSorter.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/BaseFeePendingTransactionsSorter.java @@ -18,7 +18,6 @@ import static java.util.Comparator.comparing; import static java.util.stream.Collectors.toUnmodifiableList; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; @@ -74,7 +73,7 @@ public class BaseFeePendingTransactionsSorter extends AbstractPendingTransaction .orElse(Wei.ZERO) .getAsBigInteger() .longValue()) - .thenComparing(PendingTransaction::getAddedToPoolAt) + .thenComparing(PendingTransaction::getAddedAt) .thenComparing(PendingTransaction::getSequence) .reversed()); @@ -88,7 +87,7 @@ public class BaseFeePendingTransactionsSorter extends AbstractPendingTransaction .getMaxFeePerGas() .map(maxFeePerGas -> maxFeePerGas.getAsBigInteger().longValue()) .orElse(pendingTx.getGasPrice().toLong())) - .thenComparing(PendingTransaction::getAddedToPoolAt) + .thenComparing(PendingTransaction::getAddedAt) .thenComparing(PendingTransaction::getSequence) .reversed()); @@ -100,8 +99,8 @@ public class BaseFeePendingTransactionsSorter extends AbstractPendingTransaction } @Override - public void manageBlockAdded(final Block block) { - block.getHeader().getBaseFee().ifPresent(this::updateBaseFee); + public void manageBlockAdded(final BlockHeader blockHeader) { + blockHeader.getBaseFee().ifPresent(this::updateBaseFee); } @Override diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/GasPricePendingTransactionsSorter.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/GasPricePendingTransactionsSorter.java index 1d9ad3184f..29b3406b05 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/GasPricePendingTransactionsSorter.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/GasPricePendingTransactionsSorter.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.ethereum.eth.transactions.sorter; import static java.util.Comparator.comparing; -import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; @@ -40,7 +39,7 @@ public class GasPricePendingTransactionsSorter extends AbstractPendingTransactio new TreeSet<>( comparing(PendingTransaction::isReceivedFromLocalSource) .thenComparing(PendingTransaction::getGasPrice) - .thenComparing(PendingTransaction::getAddedToPoolAt) + .thenComparing(PendingTransaction::getAddedAt) .thenComparing(PendingTransaction::getSequence) .reversed()); @@ -59,7 +58,7 @@ public class GasPricePendingTransactionsSorter extends AbstractPendingTransactio } @Override - public void manageBlockAdded(final Block block) { + public void manageBlockAdded(final BlockHeader blockHeader) { // nothing to do } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/BufferedGetPooledTransactionsFromPeerFetcherTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/BufferedGetPooledTransactionsFromPeerFetcherTest.java index c9df3cc782..2f4049aa7e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/BufferedGetPooledTransactionsFromPeerFetcherTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/BufferedGetPooledTransactionsFromPeerFetcherTest.java @@ -32,6 +32,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.transactions.PeerTransactionTracker; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; import org.hyperledger.besu.metrics.StubMetricsSystem; import java.util.List; @@ -68,7 +69,13 @@ public class BufferedGetPooledTransactionsFromPeerFetcherTest { ScheduledFuture mock = mock(ScheduledFuture.class); fetcher = new BufferedGetPooledTransactionsFromPeerFetcher( - ethContext, mock, ethPeer, transactionPool, transactionTracker, metricsSystem); + ethContext, + mock, + ethPeer, + transactionPool, + transactionTracker, + new TransactionPoolMetrics(metricsSystem), + "new_pooled_transaction_hashes"); } @Test @@ -123,6 +130,9 @@ public class BufferedGetPooledTransactionsFromPeerFetcherTest { verifyNoInteractions(ethScheduler); verify(transactionPool, never()).addRemoteTransactions(List.of(transaction)); - assertThat(metricsSystem.getCounterValue("remote_already_seen_total", "hashes")).isEqualTo(1); + assertThat( + metricsSystem.getCounterValue( + "remote_transactions_already_seen_total", "new_pooled_transaction_hashes")) + .isEqualTo(1); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/GetPooledTransactionsFromPeerTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/GetPooledTransactionsFromPeerTaskTest.java index 0cc3fceabd..760e481bc6 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/GetPooledTransactionsFromPeerTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/GetPooledTransactionsFromPeerTaskTest.java @@ -23,7 +23,7 @@ import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.ethtaskutils.PeerMessageTaskTest; -import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -51,7 +51,7 @@ public class GetPooledTransactionsFromPeerTaskTest extends PeerMessageTaskTest syncTaskCapture; + + protected abstract PendingTransactions createPendingTransactionsSorter( + final TransactionPoolConfiguration poolConfig, + BiFunction transactionReplacementTester); + + protected abstract ExecutionContextTestFixture createExecutionContextTestFixture(); + + protected abstract FeeMarket getFeeMarket(); + + @Before + public void setUp() { + executionContext = createExecutionContextTestFixture(); + protocolContext = executionContext.getProtocolContext(); + blockchain = executionContext.getBlockchain(); + + when(protocolSpec.getTransactionValidator()).thenReturn(transactionValidator); + when(protocolSpec.getFeeMarket()).thenReturn(getFeeMarket()); + protocolSchedule = spy(executionContext.getProtocolSchedule()); + doReturn(protocolSpec).when(protocolSchedule).getByBlockHeader(any()); + blockGasLimit = blockchain.getChainHeadBlock().getHeader().getGasLimit(); + ethProtocolManager = EthProtocolManagerTestUtil.create(); + ethContext = spy(ethProtocolManager.ethContext()); + + final EthScheduler ethScheduler = mock(EthScheduler.class); + syncTaskCapture = ArgumentCaptor.forClass(Runnable.class); + doNothing().when(ethScheduler).scheduleSyncWorkerTask(syncTaskCapture.capture()); + doReturn(ethScheduler).when(ethContext).getScheduler(); + + peerTransactionTracker = new PeerTransactionTracker(); + + transactionPool = createTransactionPool(); + + blockchain.observeBlockAdded(transactionPool); + when(miningParameters.getMinTransactionGasPrice()).thenReturn(Wei.of(2)); + } + + protected TransactionPool createTransactionPool() { + return createTransactionPool(b -> {}); + } + + protected TransactionPool createTransactionPool( + final Consumer configConsumer) { + final ImmutableTransactionPoolConfiguration.Builder configBuilder = + ImmutableTransactionPoolConfiguration.builder(); + configConsumer.accept(configBuilder); + poolConfig = configBuilder.build(); + + final TransactionPoolReplacementHandler transactionReplacementHandler = + new TransactionPoolReplacementHandler(poolConfig.getPriceBump()); + + final BiFunction transactionReplacementTester = + (t1, t2) -> + transactionReplacementHandler.shouldReplace( + t1, t2, protocolContext.getBlockchain().getChainHeadHeader()); + + transactions = spy(createPendingTransactionsSorter(poolConfig, transactionReplacementTester)); + + transactionBroadcaster = + spy( + new TransactionBroadcaster( + ethContext, + transactions, + peerTransactionTracker, + transactionsMessageSender, + newPooledTransactionHashesMessageSender)); + + return new TransactionPool( + transactions, + protocolSchedule, + protocolContext, + transactionBroadcaster, + ethContext, + miningParameters, + new TransactionPoolMetrics(metricsSystem), + poolConfig); + } + + @Test + public void localTransactionHappyPath() { + final Transaction transaction = createTransaction(0); + + givenTransactionIsValid(transaction); + + addAndAssertLocalTransactionValid(transaction); + } + + @Test + public void shouldReturnExclusivelyLocalTransactionsWhenAppropriate() { + final Transaction localTransaction0 = createTransaction(0, KEY_PAIR2); + + givenTransactionIsValid(localTransaction0); + givenTransactionIsValid(transaction0); + givenTransactionIsValid(transaction1); + + addAndAssertLocalTransactionValid(localTransaction0); + addAndAssertRemoteTransactionValid(transaction0); + addAndAssertRemoteTransactionValid(transaction1); + + assertThat(transactions.size()).isEqualTo(3); + List localTransactions = transactions.getLocalTransactions(); + assertThat(localTransactions.size()).isEqualTo(1); + } + + @Test + public void shouldRemoveTransactionsFromPendingListWhenIncludedInBlockOnchain() { + transactions.addRemoteTransaction(transaction0, Optional.empty()); + assertTransactionPending(transaction0); + appendBlock(transaction0); + + assertTransactionNotPending(transaction0); + } + + @Test + public void shouldRemoveMultipleTransactionsAddedInOneBlock() { + transactions.addRemoteTransaction(transaction0, Optional.empty()); + transactions.addRemoteTransaction(transaction1, Optional.empty()); + appendBlock(transaction0, transaction1); + + assertTransactionNotPending(transaction0); + assertTransactionNotPending(transaction1); + assertThat(transactions.size()).isZero(); + } + + @Test + public void shouldIgnoreUnknownTransactionsThatAreAddedInABlock() { + transactions.addRemoteTransaction(transaction0, Optional.empty()); + appendBlock(transaction0, transaction1); + + assertTransactionNotPending(transaction0); + assertTransactionNotPending(transaction1); + assertThat(transactions.size()).isZero(); + } + + @Test + public void shouldNotRemovePendingTransactionsWhenABlockAddedToAFork() { + transactions.addRemoteTransaction(transaction0, Optional.empty()); + final BlockHeader commonParent = getHeaderForCurrentChainHead(); + final Block canonicalHead = appendBlock(Difficulty.of(1000), commonParent); + appendBlock(Difficulty.ONE, commonParent, transaction0); + + verifyChainHeadIs(canonicalHead); + + assertTransactionPending(transaction0); + } + + @Test + public void shouldRemovePendingTransactionsFromAllBlocksOnAForkWhenItBecomesTheCanonicalChain() { + transactions.addRemoteTransaction(transaction0, Optional.empty()); + transactions.addRemoteTransaction(transaction1, Optional.empty()); + final BlockHeader commonParent = getHeaderForCurrentChainHead(); + final Block originalChainHead = appendBlock(Difficulty.of(1000), commonParent); + + final Block forkBlock1 = appendBlock(Difficulty.ONE, commonParent, transaction0); + verifyChainHeadIs(originalChainHead); + + final Block forkBlock2 = appendBlock(Difficulty.of(2000), forkBlock1.getHeader(), transaction1); + verifyChainHeadIs(forkBlock2); + + assertTransactionNotPending(transaction0); + assertTransactionNotPending(transaction1); + } + + @Test + public void shouldReAddTransactionsFromThePreviousCanonicalHeadWhenAReorgOccurs() { + givenTransactionIsValid(transaction0); + givenTransactionIsValid(transactionOtherSender); + transactions.addLocalTransaction(transaction0, Optional.empty()); + transactions.addRemoteTransaction(transactionOtherSender, Optional.empty()); + final BlockHeader commonParent = getHeaderForCurrentChainHead(); + final Block originalFork1 = appendBlock(Difficulty.of(1000), commonParent, transaction0); + final Block originalFork2 = + appendBlock(Difficulty.ONE, originalFork1.getHeader(), transactionOtherSender); + assertTransactionNotPending(transaction0); + assertTransactionNotPending(transactionOtherSender); + assertThat(transactions.getLocalTransactions()).isEmpty(); + + final Block reorgFork1 = appendBlock(Difficulty.ONE, commonParent); + verifyChainHeadIs(originalFork2); + + transactions.subscribePendingTransactions(listener); + final Block reorgFork2 = appendBlock(Difficulty.of(2000), reorgFork1.getHeader()); + verifyChainHeadIs(reorgFork2); + + assertTransactionPending(transaction0); + assertTransactionPending(transactionOtherSender); + assertThat(transactions.getLocalTransactions()).contains(transaction0); + assertThat(transactions.getLocalTransactions()).doesNotContain(transactionOtherSender); + verify(listener).onTransactionAdded(transaction0); + verify(listener).onTransactionAdded(transactionOtherSender); + verifyNoMoreInteractions(listener); + } + + @Test + public void shouldNotReAddTransactionsThatAreInBothForksWhenReorgHappens() { + givenTransactionIsValid(transaction0); + givenTransactionIsValid(transaction1); + transactions.addRemoteTransaction(transaction0, Optional.empty()); + transactions.addRemoteTransaction(transaction1, Optional.empty()); + final BlockHeader commonParent = getHeaderForCurrentChainHead(); + final Block originalFork1 = appendBlock(Difficulty.of(1000), commonParent, transaction0); + final Block originalFork2 = + appendBlock(Difficulty.ONE, originalFork1.getHeader(), transaction1); + assertTransactionNotPending(transaction0); + assertTransactionNotPending(transaction1); + + final Block reorgFork1 = appendBlock(Difficulty.ONE, commonParent, transaction1); + verifyChainHeadIs(originalFork2); + + final Block reorgFork2 = appendBlock(Difficulty.of(2000), reorgFork1.getHeader()); + verifyChainHeadIs(reorgFork2); + + assertTransactionPending(transaction0); + assertTransactionNotPending(transaction1); + } + + @Test + public void addLocalTransaction_strictReplayProtectionOn_txWithChainId_chainIdIsConfigured() { + protocolSupportsTxReplayProtection(1337, true); + transactionPool = createTransactionPool(b -> b.strictTransactionReplayProtectionEnabled(true)); + final Transaction tx = createTransaction(0); + givenTransactionIsValid(tx); + + addAndAssertLocalTransactionValid(tx); + } + + @Test + public void addRemoteTransactions_strictReplayProtectionOn_txWithChainId_chainIdIsConfigured() { + protocolSupportsTxReplayProtection(1337, true); + transactionPool = createTransactionPool(b -> b.strictTransactionReplayProtectionEnabled(true)); + final Transaction tx = createTransaction(0); + givenTransactionIsValid(tx); + + addAndAssertRemoteTransactionValid(tx); + } + + @Test + public void shouldNotAddRemoteTransactionsWhenGasPriceBelowMinimum() { + final Transaction transaction = createTransaction(1, Wei.ONE); + transactionPool.addRemoteTransactions(singletonList(transaction)); + + assertTransactionNotPending(transaction); + verifyNoMoreInteractions(transactionValidator); + } + + @Test + public void shouldNotAddRemoteTransactionsThatAreInvalidAccordingToStateDependentChecks() { + givenTransactionIsValid(transaction0); + givenTransactionIsValid(transaction1); + when(transactionValidator.validateForSender( + eq(transaction1), eq(null), any(TransactionValidationParams.class))) + .thenReturn(ValidationResult.invalid(NONCE_TOO_LOW)); + transactionPool.addRemoteTransactions(asList(transaction0, transaction1)); + + assertTransactionPending(transaction0); + assertTransactionNotPending(transaction1); + verify(transactionBroadcaster).onTransactionsAdded(singletonList(transaction0)); + verify(transactionValidator).validate(eq(transaction0), any(Optional.class), any()); + verify(transactionValidator) + .validateForSender(eq(transaction0), eq(null), any(TransactionValidationParams.class)); + verify(transactionValidator).validate(eq(transaction1), any(Optional.class), any()); + verify(transactionValidator).validateForSender(eq(transaction1), any(), any()); + verifyNoMoreInteractions(transactionValidator); + } + + @Test + public void shouldAllowSequenceOfTransactionsWithIncreasingNonceFromSameSender() { + final Transaction transaction1 = createTransaction(0); + final Transaction transaction2 = createTransaction(1); + final Transaction transaction3 = createTransaction(2); + + givenTransactionIsValid(transaction1); + givenTransactionIsValid(transaction2); + givenTransactionIsValid(transaction3); + + addAndAssertLocalTransactionValid(transaction1); + addAndAssertLocalTransactionValid(transaction2); + addAndAssertLocalTransactionValid(transaction3); + } + + @Test + public void + shouldAllowSequenceOfTransactionsWithIncreasingNonceFromSameSenderWhenSentInBatchOutOfOrder() { + final Transaction transaction1 = createTransaction(0); + final Transaction transaction2 = createTransaction(1); + final Transaction transaction3 = createTransaction(2); + + givenTransactionIsValid(transaction1); + givenTransactionIsValid(transaction2); + givenTransactionIsValid(transaction3); + + transactionPool.addRemoteTransactions(List.of(transaction3, transaction1, transaction2)); + assertRemoteTransactionValid(transaction3); + assertRemoteTransactionValid(transaction1); + assertRemoteTransactionValid(transaction2); + } + + @Test + public void shouldDiscardRemoteTransactionThatAlreadyExistsBeforeValidation() { + doReturn(true).when(transactions).containsTransaction(transaction0); + transactionPool.addRemoteTransactions(singletonList(transaction0)); + + verify(transactions).containsTransaction(transaction0); + verifyNoInteractions(transactionValidator); + verifyNoMoreInteractions(transactions); + } + + @Test + public void shouldNotNotifyBatchListenerWhenRemoteTransactionDoesNotReplaceExisting() { + final Transaction transaction1 = createTransaction(0, Wei.of(100)); + final Transaction transaction2 = createTransaction(0, Wei.of(50)); + + givenTransactionIsValid(transaction1); + givenTransactionIsValid(transaction2); + + addAndAssertRemoteTransactionValid(transaction1); + addAndAssertRemoteTransactionInvalid(transaction2); + } + + @Test + public void shouldNotNotifyBatchListenerWhenLocalTransactionDoesNotReplaceExisting() { + final Transaction transaction1 = createTransaction(0, Wei.of(10)); + final Transaction transaction2 = createTransaction(0, Wei.of(9)); + + givenTransactionIsValid(transaction1); + givenTransactionIsValid(transaction2); + + addAndAssertLocalTransactionValid(transaction1); + addAndAssertLocalTransactionInvalid(transaction2, TRANSACTION_REPLACEMENT_UNDERPRICED); + } + + @Test + public void shouldRejectLocalTransactionsWhereGasLimitExceedBlockGasLimit() { + final Transaction transaction1 = + createBaseTransaction(0).gasLimit(blockGasLimit + 1).createTransaction(KEY_PAIR1); + + givenTransactionIsValid(transaction1); + + addAndAssertLocalTransactionInvalid(transaction1, EXCEEDS_BLOCK_GAS_LIMIT); + } + + @Test + public void shouldRejectRemoteTransactionsWhereGasLimitExceedBlockGasLimit() { + final Transaction transaction1 = + createBaseTransaction(0).gasLimit(blockGasLimit + 1).createTransaction(KEY_PAIR1); + + givenTransactionIsValid(transaction1); + + addAndAssertRemoteTransactionInvalid(transaction1); + } + + @Test + public void + shouldAcceptAsPostponedLocalTransactionsEvenIfAnInvalidTransactionWithLowerNonceExists() { + final Transaction invalidTx = + createBaseTransaction(0).gasLimit(blockGasLimit + 1).createTransaction(KEY_PAIR1); + + final Transaction nextTx = createBaseTransaction(1).gasLimit(1).createTransaction(KEY_PAIR1); + + givenTransactionIsValid(invalidTx); + givenTransactionIsValid(nextTx); + + addAndAssertLocalTransactionInvalid(invalidTx, EXCEEDS_BLOCK_GAS_LIMIT); + final ValidationResult result = + transactionPool.addLocalTransaction(nextTx); + + assertThat(result.isValid()).isTrue(); + } + + @Test + public void shouldRejectLocalTransactionsWhenNonceTooFarInFuture() { + final Transaction transaction1 = createTransaction(Integer.MAX_VALUE); + + givenTransactionIsValid(transaction1); + + addAndAssertLocalTransactionInvalid(transaction1, NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER); + } + + @Test + public void shouldNotNotifyBatchListenerIfNoTransactionsAreAdded() { + transactionPool.addRemoteTransactions(emptyList()); + verifyNoInteractions(transactionBroadcaster); + } + + @Test + public void shouldSendPooledTransactionHashesIfPeerSupportsEth65() { + EthPeer peer = mock(EthPeer.class); + when(peer.hasSupportForMessage(EthPV65.NEW_POOLED_TRANSACTION_HASHES)).thenReturn(true); + + givenTransactionIsValid(transaction0); + transactionPool.addLocalTransaction(transaction0); + transactionPool.handleConnect(peer); + syncTaskCapture.getValue().run(); + verify(newPooledTransactionHashesMessageSender).sendTransactionHashesToPeer(peer); + } + + @Test + public void shouldSendFullTransactionsIfPeerDoesNotSupportEth65() { + EthPeer peer = mock(EthPeer.class); + when(peer.hasSupportForMessage(EthPV65.NEW_POOLED_TRANSACTION_HASHES)).thenReturn(false); + + givenTransactionIsValid(transaction0); + transactionPool.addLocalTransaction(transaction0); + transactionPool.handleConnect(peer); + syncTaskCapture.getValue().run(); + verify(transactionsMessageSender).sendTransactionsToPeer(peer); + } + + @Test + public void shouldSendFullTransactionPoolToNewlyConnectedPeer() { + final Transaction transactionLocal = createTransaction(0); + final Transaction transactionRemote = createTransaction(1); + + givenTransactionIsValid(transactionLocal); + givenTransactionIsValid(transactionRemote); + + transactionPool.addLocalTransaction(transactionLocal); + transactionPool.addRemoteTransactions(Collections.singletonList(transactionRemote)); + + RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager); + + Set transactionsToSendToPeer = + peerTransactionTracker.claimTransactionsToSendToPeer(peer.getEthPeer()); + + assertThat(transactionsToSendToPeer).contains(transactionLocal, transactionRemote); + } + + @Test + public void shouldCallValidatorWithExpectedValidationParameters() { + final ArgumentCaptor txValidationParamCaptor = + ArgumentCaptor.forClass(TransactionValidationParams.class); + + when(transactionValidator.validate(eq(transaction0), any(Optional.class), any())) + .thenReturn(valid()); + when(transactionValidator.validateForSender(any(), any(), txValidationParamCaptor.capture())) + .thenReturn(valid()); + + final TransactionValidationParams expectedValidationParams = + TransactionValidationParams.transactionPool(); + + transactionPool.addLocalTransaction(transaction0); + + assertThat(txValidationParamCaptor.getValue()) + .usingRecursiveComparison() + .isEqualTo(expectedValidationParams); + } + + @Test + public void shouldIgnoreFeeCapIfSetZero() { + final Wei twoEthers = Wei.fromEth(2); + transactionPool = createTransactionPool(b -> b.txFeeCap(Wei.ZERO)); + final Transaction transaction = createTransaction(0, twoEthers.add(Wei.of(1))); + + givenTransactionIsValid(transaction); + + addAndAssertLocalTransactionValid(transaction); + } + + @Test + public void shouldRejectLocalTransactionIfFeeCapExceeded() { + final Wei twoEthers = Wei.fromEth(2); + transactionPool = createTransactionPool(b -> b.txFeeCap(twoEthers)); + + final Transaction transactionLocal = createTransaction(0, twoEthers.add(1)); + + givenTransactionIsValid(transactionLocal); + + addAndAssertLocalTransactionInvalid(transactionLocal, TX_FEECAP_EXCEEDED); + } + + @Test + public void shouldRejectZeroGasPriceTransactionWhenNotMining() { + when(miningParameters.isMiningEnabled()).thenReturn(false); + + final Transaction transaction = createTransaction(0, Wei.ZERO); + + givenTransactionIsValid(transaction); + + addAndAssertLocalTransactionInvalid(transaction, GAS_PRICE_TOO_LOW); + } + + private void assertTransactionPending(final Transaction t) { + assertThat(transactions.getTransactionByHash(t.getHash())).contains(t); + } + + private void assertTransactionNotPending(final Transaction transaction) { + assertThat(transactions.getTransactionByHash(transaction.getHash())).isEmpty(); + } + + private void verifyChainHeadIs(final Block forkBlock2) { + assertThat(blockchain.getChainHeadHash()).isEqualTo(forkBlock2.getHash()); + } + + private void appendBlock(final Transaction... transactionsToAdd) { + appendBlock(Difficulty.ONE, getHeaderForCurrentChainHead(), transactionsToAdd); + } + + private BlockHeader getHeaderForCurrentChainHead() { + return blockchain.getBlockHeader(blockchain.getChainHeadHash()).get(); + } + + protected abstract Block appendBlock( + final Difficulty difficulty, + final BlockHeader parentBlock, + final Transaction... transactionsToAdd); + + protected abstract Transaction createTransaction( + final int nonce, final Optional maybeChainId); + + protected abstract Transaction createTransaction(final int nonce, final Wei maxPrice); + + protected abstract TransactionTestFixture createBaseTransaction(final int nonce); + + private Transaction createTransaction(final int nonce) { + return createTransaction(nonce, Optional.of(BigInteger.ONE)); + } + + private Transaction createTransaction(final int nonce, final KeyPair keyPair) { + return createBaseTransaction(nonce).createTransaction(keyPair); + } + + protected void protocolSupportsTxReplayProtection( + final long chainId, final boolean isSupportedAtCurrentBlock) { + when(protocolSpec.isReplayProtectionSupported()).thenReturn(isSupportedAtCurrentBlock); + when(protocolSchedule.getChainId()).thenReturn(Optional.of(BigInteger.valueOf(chainId))); + } + + protected void givenTransactionIsValid(final Transaction transaction) { + when(transactionValidator.validate(eq(transaction), any(Optional.class), any())) + .thenReturn(valid()); + when(transactionValidator.validateForSender( + eq(transaction), nullable(Account.class), any(TransactionValidationParams.class))) + .thenReturn(valid()); + } + + protected void addAndAssertLocalTransactionInvalid( + final Transaction tx, final TransactionInvalidReason invalidReason) { + final ValidationResult result = + transactionPool.addLocalTransaction(tx); + + assertThat(result.isValid()).isFalse(); + assertThat(result.getInvalidReason()).isEqualTo(invalidReason); + assertTransactionNotPending(tx); + verify(transactionBroadcaster, never()).onTransactionsAdded(singletonList(tx)); + } + + protected void addAndAssertLocalTransactionValid(final Transaction tx) { + final ValidationResult result = + transactionPool.addLocalTransaction(tx); + + assertThat(result.isValid()).isTrue(); + assertTransactionPending(tx); + verify(transactionBroadcaster).onTransactionsAdded(singletonList(tx)); + assertThat(transactions.getLocalTransactions()).contains(tx); + } + + protected void addAndAssertRemoteTransactionValid(final Transaction tx) { + transactionPool.addRemoteTransactions(List.of(tx)); + + assertRemoteTransactionValid(tx); + } + + protected void assertRemoteTransactionValid(final Transaction tx) { + verify(transactionBroadcaster) + .onTransactionsAdded(argThat((List list) -> list.contains(tx))); + assertTransactionPending(tx); + assertThat(transactions.getLocalTransactions()).doesNotContain(tx); + } + + protected void addAndAssertRemoteTransactionInvalid(final Transaction tx) { + transactionPool.addRemoteTransactions(List.of(tx)); + + verify(transactionBroadcaster, never()).onTransactionsAdded(singletonList(tx)); + assertTransactionNotPending(tx); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageProcessorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageProcessorTest.java index 82d86a19ab..8cbf26deac 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageProcessorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageProcessorTest.java @@ -93,7 +93,7 @@ public class NewPooledTransactionHashesMessageProcessorTest { transactionPool, transactionPoolConfiguration, ethContext, - metricsSystem); + new TransactionPoolMetrics(metricsSystem)); when(ethContext.getScheduler()).thenReturn(ethScheduler); } @@ -150,7 +150,9 @@ public class NewPooledTransactionHashesMessageProcessorTest { ofMillis(1)); verifyNoInteractions(transactionTracker); assertThat( - metricsSystem.getCounterValue("new_pooled_transaction_hashes_messages_skipped_total")) + metricsSystem.getCounterValue( + TransactionPoolMetrics.EXPIRED_MESSAGES_COUNTER_NAME, + NewPooledTransactionHashesMessageProcessor.METRIC_LABEL)) .isEqualTo(1); } @@ -163,7 +165,9 @@ public class NewPooledTransactionHashesMessageProcessorTest { ofMillis(1)); verifyNoInteractions(transactionPool); assertThat( - metricsSystem.getCounterValue("new_pooled_transaction_hashes_messages_skipped_total")) + metricsSystem.getCounterValue( + TransactionPoolMetrics.EXPIRED_MESSAGES_COUNTER_NAME, + NewPooledTransactionHashesMessageProcessor.METRIC_LABEL)) .isEqualTo(1); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSenderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSenderTest.java index 2f29fdb553..07e15b5eac 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSenderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSenderTest.java @@ -33,12 +33,9 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.MockPeerConnection; import org.hyperledger.besu.ethereum.eth.messages.EthPV65; import org.hyperledger.besu.ethereum.eth.messages.NewPooledTransactionHashesMessage; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.BaseFeePendingTransactionsSorter; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.Set; @@ -47,11 +44,8 @@ import java.util.stream.Collectors; import com.google.common.collect.Sets; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; import org.mockito.ArgumentCaptor; -@RunWith(Parameterized.class) public class NewPooledTransactionHashesMessageSenderTest { private final EthPeer peer1 = mock(EthPeer.class); @@ -62,25 +56,17 @@ public class NewPooledTransactionHashesMessageSenderTest { private final Transaction transaction2 = generator.transaction(); private final Transaction transaction3 = generator.transaction(); - @Parameterized.Parameter public PendingTransactions pendingTransactions; + public PendingTransactions pendingTransactions; private PeerTransactionTracker transactionTracker; private NewPooledTransactionHashesMessageSender messageSender; - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList( - new Object[][] { - {mock(GasPricePendingTransactionsSorter.class)}, - {mock(BaseFeePendingTransactionsSorter.class)} - }); - } - @Before public void setUp() { transactionTracker = new PeerTransactionTracker(); messageSender = new NewPooledTransactionHashesMessageSender(transactionTracker); final Transaction tx = mock(Transaction.class); + pendingTransactions = mock(PendingTransactions.class); when(pendingTransactions.getTransactionByHash(any())).thenReturn(Optional.of(tx)); when(peer1.getConnection()) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingMultiTypesTransactionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingMultiTypesTransactionsTest.java deleted file mode 100644 index ab42e62725..0000000000 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingMultiTypesTransactionsTest.java +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.eth.transactions; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus.ALREADY_KNOWN; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import org.hyperledger.besu.crypto.KeyPair; -import org.hyperledger.besu.crypto.SignatureAlgorithm; -import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; -import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.Transaction; -import org.hyperledger.besu.ethereum.core.TransactionTestFixture; -import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions.TransactionSelectionResult; -import org.hyperledger.besu.ethereum.eth.transactions.sorter.BaseFeePendingTransactionsSorter; -import org.hyperledger.besu.metrics.StubMetricsSystem; -import org.hyperledger.besu.plugin.data.TransactionType; -import org.hyperledger.besu.testutil.TestClock; - -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.function.Supplier; - -import com.google.common.base.Suppliers; -import org.junit.Test; - -public class PendingMultiTypesTransactionsTest { - - private static final int MAX_TRANSACTIONS = 5; - private static final float MAX_TRANSACTIONS_BY_SENDER_PERCENTAGE = 0.8f; // evaluates to 4 - private static final Supplier SIGNATURE_ALGORITHM = - Suppliers.memoize(SignatureAlgorithmFactory::getInstance)::get; - private static final KeyPair KEYS1 = SIGNATURE_ALGORITHM.get().generateKeyPair(); - private static final KeyPair KEYS2 = SIGNATURE_ALGORITHM.get().generateKeyPair(); - private static final KeyPair KEYS3 = SIGNATURE_ALGORITHM.get().generateKeyPair(); - private static final KeyPair KEYS4 = SIGNATURE_ALGORITHM.get().generateKeyPair(); - private static final KeyPair KEYS5 = SIGNATURE_ALGORITHM.get().generateKeyPair(); - private static final KeyPair KEYS6 = SIGNATURE_ALGORITHM.get().generateKeyPair(); - private static final String ADDED_COUNTER = "transactions_added_total"; - private static final String REMOTE = "remote"; - private static final String LOCAL = "local"; - - private final BlockHeader blockHeader = mock(BlockHeader.class); - - private final StubMetricsSystem metricsSystem = new StubMetricsSystem(); - private final BaseFeePendingTransactionsSorter transactions = - new BaseFeePendingTransactionsSorter( - ImmutableTransactionPoolConfiguration.builder() - .txPoolMaxSize(MAX_TRANSACTIONS) - .txPoolLimitByAccountPercentage(MAX_TRANSACTIONS_BY_SENDER_PERCENTAGE) - .build(), - TestClock.system(ZoneId.systemDefault()), - metricsSystem, - () -> mockBlockHeader(Wei.of(7L))); - - @Test - public void shouldReturnExclusivelyLocal1559TransactionsWhenAppropriate() { - final Transaction localTransaction0 = create1559Transaction(0, 19, 20, KEYS1); - transactions.addLocalTransaction(localTransaction0, Optional.empty()); - assertThat(transactions.size()).isEqualTo(1); - - List localTransactions = transactions.getLocalTransactions(); - assertThat(localTransactions.size()).isEqualTo(1); - - final Transaction remoteTransaction1 = create1559Transaction(1, 19, 20, KEYS1); - transactions.addRemoteTransaction(remoteTransaction1, Optional.empty()); - assertThat(transactions.size()).isEqualTo(2); - - localTransactions = transactions.getLocalTransactions(); - assertThat(localTransactions.size()).isEqualTo(1); - } - - @Test - public void shouldReplaceTransactionWithLowestMaxFeePerGas() { - final Transaction localTransaction0 = create1559Transaction(0, 200, 20, KEYS1); - final Transaction localTransaction1 = create1559Transaction(0, 190, 20, KEYS2); - final Transaction localTransaction2 = create1559Transaction(0, 220, 20, KEYS3); - final Transaction localTransaction3 = create1559Transaction(0, 240, 20, KEYS4); - final Transaction localTransaction4 = create1559Transaction(0, 260, 20, KEYS5); - final Transaction localTransaction5 = create1559Transaction(0, 900, 20, KEYS6); - transactions.addLocalTransaction(localTransaction0, Optional.empty()); - transactions.addLocalTransaction(localTransaction1, Optional.empty()); - transactions.addLocalTransaction(localTransaction2, Optional.empty()); - transactions.addLocalTransaction(localTransaction3, Optional.empty()); - transactions.addLocalTransaction(localTransaction4, Optional.empty()); - - transactions.updateBaseFee(Wei.of(300L)); - - transactions.addLocalTransaction(localTransaction5, Optional.empty()); - assertThat(transactions.size()).isEqualTo(5); - - transactions.selectTransactions( - transaction -> { - assertThat(transaction.getNonce()).isNotEqualTo(1); - return TransactionSelectionResult.CONTINUE; - }); - } - - @Test - public void shouldEvictTransactionWithLowestMaxFeePerGasAndLowestTip() { - final Transaction localTransaction0 = create1559Transaction(0, 200, 20, KEYS1); - final Transaction localTransaction1 = create1559Transaction(0, 200, 19, KEYS2); - final Transaction localTransaction2 = create1559Transaction(0, 200, 18, KEYS3); - final Transaction localTransaction3 = create1559Transaction(0, 240, 20, KEYS4); - final Transaction localTransaction4 = create1559Transaction(0, 260, 20, KEYS5); - final Transaction localTransaction5 = create1559Transaction(0, 900, 20, KEYS6); - transactions.addLocalTransaction(localTransaction0, Optional.empty()); - transactions.addLocalTransaction(localTransaction1, Optional.empty()); - transactions.addLocalTransaction(localTransaction2, Optional.empty()); - transactions.addLocalTransaction(localTransaction3, Optional.empty()); - transactions.addLocalTransaction(localTransaction4, Optional.empty()); - transactions.addLocalTransaction(localTransaction5, Optional.empty()); // causes eviction - - assertThat(transactions.size()).isEqualTo(5); - - transactions.selectTransactions( - transaction -> { - assertThat(transaction.getNonce()).isNotEqualTo(2); - return TransactionSelectionResult.CONTINUE; - }); - } - - @Test - public void shouldEvictLegacyTransactionWithLowestEffectiveMaxPriorityFeePerGas() { - final Transaction localTransaction0 = create1559Transaction(0, 200, 20, KEYS1); - final Transaction localTransaction1 = createLegacyTransaction(0, 25, KEYS2); - final Transaction localTransaction2 = create1559Transaction(0, 200, 18, KEYS3); - final Transaction localTransaction3 = create1559Transaction(0, 240, 20, KEYS4); - final Transaction localTransaction4 = create1559Transaction(0, 260, 20, KEYS5); - final Transaction localTransaction5 = create1559Transaction(0, 900, 20, KEYS6); - transactions.addLocalTransaction(localTransaction0, Optional.empty()); - transactions.addLocalTransaction(localTransaction1, Optional.empty()); - transactions.addLocalTransaction(localTransaction2, Optional.empty()); - transactions.addLocalTransaction(localTransaction3, Optional.empty()); - transactions.addLocalTransaction(localTransaction4, Optional.empty()); - transactions.addLocalTransaction(localTransaction5, Optional.empty()); // causes eviction - assertThat(transactions.size()).isEqualTo(5); - - transactions.selectTransactions( - transaction -> { - assertThat(transaction.getNonce()).isNotEqualTo(1); - return TransactionSelectionResult.CONTINUE; - }); - } - - @Test - public void shouldEvictEIP1559TransactionWithLowestEffectiveMaxPriorityFeePerGas() { - final Transaction localTransaction0 = create1559Transaction(0, 200, 20, KEYS1); - final Transaction localTransaction1 = createLegacyTransaction(0, 26, KEYS2); - final Transaction localTransaction2 = create1559Transaction(0, 200, 18, KEYS3); - final Transaction localTransaction3 = create1559Transaction(0, 240, 20, KEYS4); - final Transaction localTransaction4 = create1559Transaction(0, 260, 20, KEYS5); - final Transaction localTransaction5 = create1559Transaction(0, 900, 20, KEYS6); - transactions.addLocalTransaction(localTransaction0, Optional.empty()); - transactions.addLocalTransaction(localTransaction1, Optional.empty()); - transactions.addLocalTransaction(localTransaction2, Optional.empty()); - transactions.addLocalTransaction(localTransaction3, Optional.empty()); - transactions.addLocalTransaction(localTransaction4, Optional.empty()); - transactions.addLocalTransaction(localTransaction5, Optional.empty()); // causes eviction - assertThat(transactions.size()).isEqualTo(5); - - transactions.selectTransactions( - transaction -> { - assertThat(transaction.getNonce()).isNotEqualTo(2); - return TransactionSelectionResult.CONTINUE; - }); - } - - @Test - public void shouldChangePriorityWhenBaseFeeIncrease() { - final Transaction localTransaction0 = create1559Transaction(1, 200, 18, KEYS1); - final Transaction localTransaction1 = create1559Transaction(1, 100, 20, KEYS2); - final Transaction localTransaction2 = create1559Transaction(2, 100, 19, KEYS2); - - transactions.addLocalTransaction(localTransaction0, Optional.empty()); - transactions.addLocalTransaction(localTransaction1, Optional.empty()); - transactions.addLocalTransaction(localTransaction2, Optional.empty()); - - final List iterationOrder = new ArrayList<>(); - transactions.selectTransactions( - transaction -> { - iterationOrder.add(transaction); - return TransactionSelectionResult.CONTINUE; - }); - - assertThat(iterationOrder) - .containsExactly(localTransaction1, localTransaction2, localTransaction0); - - transactions.updateBaseFee(Wei.of(110L)); - - final List iterationOrderAfterBaseIncreased = new ArrayList<>(); - transactions.selectTransactions( - transaction -> { - iterationOrderAfterBaseIncreased.add(transaction); - return TransactionSelectionResult.CONTINUE; - }); - - assertThat(iterationOrderAfterBaseIncreased) - .containsExactly(localTransaction0, localTransaction1, localTransaction2); - } - - @Test - public void shouldChangePriorityWhenBaseFeeDecrease() { - final Transaction localTransaction0 = create1559Transaction(1, 200, 18, KEYS1); - final Transaction localTransaction1 = create1559Transaction(1, 100, 20, KEYS2); - final Transaction localTransaction2 = create1559Transaction(2, 100, 19, KEYS2); - - transactions.updateBaseFee(Wei.of(110L)); - - transactions.addLocalTransaction(localTransaction0, Optional.empty()); - transactions.addLocalTransaction(localTransaction1, Optional.empty()); - transactions.addLocalTransaction(localTransaction2, Optional.empty()); - - final List iterationOrder = new ArrayList<>(); - transactions.selectTransactions( - transaction -> { - iterationOrder.add(transaction); - return TransactionSelectionResult.CONTINUE; - }); - - assertThat(iterationOrder) - .containsExactly(localTransaction0, localTransaction1, localTransaction2); - - transactions.updateBaseFee(Wei.of(50L)); - - final List iterationOrderAfterBaseIncreased = new ArrayList<>(); - transactions.selectTransactions( - transaction -> { - iterationOrderAfterBaseIncreased.add(transaction); - return TransactionSelectionResult.CONTINUE; - }); - - assertThat(iterationOrderAfterBaseIncreased) - .containsExactly(localTransaction1, localTransaction2, localTransaction0); - } - - @Test - public void shouldCorrectlyPrioritizeMultipleTransactionTypesBasedOnNonce() { - final Transaction localTransaction0 = create1559Transaction(1, 200, 18, KEYS1); - final Transaction localTransaction1 = create1559Transaction(1, 100, 20, KEYS2); - final Transaction localTransaction2 = create1559Transaction(2, 100, 19, KEYS2); - final Transaction localTransaction3 = createLegacyTransaction(0, 20, KEYS1); - - transactions.addLocalTransaction(localTransaction0, Optional.empty()); - transactions.addLocalTransaction(localTransaction1, Optional.empty()); - transactions.addLocalTransaction(localTransaction2, Optional.empty()); - transactions.addLocalTransaction(localTransaction3, Optional.empty()); - - final List iterationOrder = new ArrayList<>(); - transactions.selectTransactions( - transaction -> { - iterationOrder.add(transaction); - return TransactionSelectionResult.CONTINUE; - }); - - assertThat(iterationOrder) - .containsExactly( - localTransaction1, localTransaction2, localTransaction3, localTransaction0); - } - - @Test - public void shouldCorrectlyPrioritizeMultipleTransactionTypesBasedOnGasPayed() { - final Transaction localTransaction0 = create1559Transaction(0, 100, 19, KEYS2); - final Transaction localTransaction1 = createLegacyTransaction(0, 2000, KEYS1); - final Transaction localTransaction2 = createLegacyTransaction(0, 20, KEYS3); - final Transaction localTransaction3 = createLegacyTransaction(1, 2000, KEYS3); - - transactions.addLocalTransaction(localTransaction0, Optional.empty()); - transactions.addLocalTransaction(localTransaction1, Optional.empty()); - transactions.addLocalTransaction(localTransaction2, Optional.empty()); - transactions.addLocalTransaction(localTransaction3, Optional.empty()); - - final List iterationOrder = new ArrayList<>(); - transactions.selectTransactions( - transaction -> { - iterationOrder.add(transaction); - return TransactionSelectionResult.CONTINUE; - }); - - assertThat(iterationOrder) - .containsExactly( - localTransaction1, localTransaction0, localTransaction2, localTransaction3); - } - - @Test - public void shouldSelectNoTransactionsIfPoolEmpty() { - final List iterationOrder = new ArrayList<>(); - transactions.selectTransactions( - transaction -> { - iterationOrder.add(transaction); - return TransactionSelectionResult.CONTINUE; - }); - - assertThat(iterationOrder).isEmpty(); - } - - @Test - public void shouldAdd1559Transaction() { - final Transaction remoteTransaction0 = create1559Transaction(0, 19, 20, KEYS1); - transactions.addRemoteTransaction(remoteTransaction0, Optional.empty()); - assertThat(transactions.size()).isEqualTo(1); - assertThat(metricsSystem.getCounterValue(ADDED_COUNTER, REMOTE)).isEqualTo(1); - - final Transaction remoteTransaction1 = create1559Transaction(1, 19, 20, KEYS1); - transactions.addRemoteTransaction(remoteTransaction1, Optional.empty()); - assertThat(transactions.size()).isEqualTo(2); - assertThat(metricsSystem.getCounterValue(ADDED_COUNTER, REMOTE)).isEqualTo(2); - } - - @Test - public void shouldNotIncrementAddedCounterWhenRemote1559TransactionAlreadyPresent() { - final Transaction localTransaction0 = create1559Transaction(0, 19, 20, KEYS1); - transactions.addLocalTransaction(localTransaction0, Optional.empty()); - assertThat(transactions.size()).isEqualTo(1); - assertThat(metricsSystem.getCounterValue(ADDED_COUNTER, LOCAL)).isEqualTo(1); - assertThat(metricsSystem.getCounterValue(ADDED_COUNTER, REMOTE)).isEqualTo(0); - - assertThat(transactions.addRemoteTransaction(localTransaction0, Optional.empty())) - .isEqualTo(ALREADY_KNOWN); - assertThat(transactions.size()).isEqualTo(1); - assertThat(metricsSystem.getCounterValue(ADDED_COUNTER, LOCAL)).isEqualTo(1); - assertThat(metricsSystem.getCounterValue(ADDED_COUNTER, REMOTE)).isEqualTo(0); - } - - @Test - public void shouldAddMixedTransactions() { - final Transaction remoteTransaction0 = create1559Transaction(0, 19, 20, KEYS1); - transactions.addRemoteTransaction(remoteTransaction0, Optional.empty()); - assertThat(transactions.size()).isEqualTo(1); - assertThat(metricsSystem.getCounterValue(ADDED_COUNTER, REMOTE)).isEqualTo(1); - - final Transaction remoteTransaction1 = createLegacyTransaction(1, 5000, KEYS1); - transactions.addRemoteTransaction(remoteTransaction1, Optional.empty()); - assertThat(transactions.size()).isEqualTo(2); - assertThat(metricsSystem.getCounterValue(ADDED_COUNTER, REMOTE)).isEqualTo(2); - } - - private Transaction create1559Transaction( - final long transactionNumber, - final long maxFeePerGas, - final long maxPriorityFeePerGas, - final KeyPair keyPair) { - return new TransactionTestFixture() - .type(TransactionType.EIP1559) - .value(Wei.of(transactionNumber)) - .nonce(transactionNumber) - .maxFeePerGas(Optional.of(Wei.of(maxFeePerGas))) - .maxPriorityFeePerGas(Optional.of(Wei.of(maxPriorityFeePerGas))) - .createTransaction(keyPair); - } - - private Transaction createLegacyTransaction( - final long transactionNumber, final long gasPrice, final KeyPair keyPair) { - return new TransactionTestFixture() - .value(Wei.of(transactionNumber)) - .gasPrice(Wei.of(gasPrice)) - .nonce(transactionNumber) - .createTransaction(keyPair); - } - - private BlockHeader mockBlockHeader(final Wei baseFee) { - when(blockHeader.getBaseFee()).thenReturn(Optional.of(baseFee)); - return blockHeader; - } -} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java new file mode 100644 index 0000000000..48317545be --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java @@ -0,0 +1,414 @@ +/* + * Copyright Besu contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.crypto.SignatureAlgorithm; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.TransactionTestFixture; +import org.hyperledger.besu.ethereum.eth.transactions.layered.BaseTransactionPoolTest; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.evm.AccessListEntry; +import org.hyperledger.besu.plugin.data.TransactionType; + +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.LongAdder; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.openjdk.jol.info.ClassLayout; +import org.openjdk.jol.info.GraphPathRecord; +import org.openjdk.jol.info.GraphVisitor; +import org.openjdk.jol.info.GraphWalker; + +@Disabled("Need to handle different results on different OS") +public class PendingTransactionEstimatedMemorySizeTest extends BaseTransactionPoolTest { + private static final Set> SHARED_CLASSES = + Set.of(SignatureAlgorithm.class, TransactionType.class); + private static final Set EIP1559_CONSTANT_FIELD_PATHS = Set.of(".gasPrice"); + private static final Set EIP1559_VARIABLE_SIZE_PATHS = + Set.of(".to", ".payload", ".maybeAccessList"); + + private static final Set FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS = + Set.of(".maxFeePerGas", ".maxPriorityFeePerGas"); + private static final Set FRONTIER_ACCESS_LIST_VARIABLE_SIZE_PATHS = + Set.of(".to", ".payload", ".maybeAccessList"); + + @Test + public void toSize() { + TransactionTestFixture preparedTx = + prepareTransaction(TransactionType.ACCESS_LIST, 10, Wei.of(500), 10); + Transaction txTo = + preparedTx.to(Optional.of(Address.extract(Bytes32.random()))).createTransaction(KEYS1); + BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + txTo.writeTo(rlpOut); + + txTo = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + System.out.println(txTo.getSender()); + System.out.println(txTo.getHash()); + System.out.println(txTo.getSize()); + + Optional

to = txTo.getTo(); + final ClassLayout cl = ClassLayout.parseInstance(to); + System.out.println(cl.toPrintable()); + LongAdder size = new LongAdder(); + size.add(cl.instanceSize()); + System.out.println(size); + + GraphVisitor gv = + gpr -> { + // byte[] is shared so only count the specific part for each field + if (gpr.path().endsWith(".bytes")) { + if (gpr.path().contains("delegate")) { + size.add(20); + System.out.println( + "(" + + size + + ")[20 = fixed address size; overrides: " + + gpr.size() + + ", " + + gpr.path() + + ", " + + gpr.klass().toString() + + "]"); + } + } else { + size.add(gpr.size()); + System.out.println( + "(" + + size + + ")[" + + gpr.size() + + ", " + + gpr.path() + + ", " + + gpr.klass().toString() + + "]"); + } + }; + + GraphWalker gw = new GraphWalker(gv); + + gw.walk(to); + + System.out.println("Optional To size: " + size); + + assertThat(size.sum()).isEqualTo(PendingTransaction.OPTIONAL_TO_MEMORY_SIZE); + } + + @Test + public void payloadSize() { + + TransactionTestFixture preparedTx = + prepareTransaction(TransactionType.ACCESS_LIST, 10, Wei.of(500), 10); + Transaction txPayload = preparedTx.createTransaction(KEYS1); + BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + txPayload.writeTo(rlpOut); + + txPayload = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + System.out.println(txPayload.getSender()); + System.out.println(txPayload.getHash()); + System.out.println(txPayload.getSize()); + + final Bytes payload = txPayload.getPayload(); + final ClassLayout cl = ClassLayout.parseInstance(payload); + System.out.println(cl.toPrintable()); + LongAdder size = new LongAdder(); + size.add(cl.instanceSize()); + System.out.println("Base payload size: " + size); + + assertThat(size.sum()).isEqualTo(PendingTransaction.PAYLOAD_BASE_MEMORY_SIZE); + } + + @Test + public void pendingTransactionSize() { + + TransactionTestFixture preparedTx = + prepareTransaction(TransactionType.ACCESS_LIST, 10, Wei.of(500), 10); + Transaction txPayload = preparedTx.createTransaction(KEYS1); + BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + txPayload.writeTo(rlpOut); + + txPayload = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + System.out.println(txPayload.getSender()); + System.out.println(txPayload.getHash()); + System.out.println(txPayload.getSize()); + + final PendingTransaction pendingTx = new PendingTransaction.Remote(txPayload); + + final ClassLayout cl = ClassLayout.parseInstance(pendingTx); + System.out.println(cl.toPrintable()); + LongAdder size = new LongAdder(); + size.add(cl.instanceSize()); + System.out.println("PendingTransaction size: " + size); + + assertThat(size.sum()).isEqualTo(PendingTransaction.PENDING_TRANSACTION_MEMORY_SIZE); + } + + @Test + public void accessListSize() { + System.setProperty("jol.magicFieldOffset", "true"); + + final AccessListEntry ale1 = + new AccessListEntry(Address.extract(Bytes32.random()), List.of(Bytes32.random())); + + final List ales = List.of(ale1); + + TransactionTestFixture preparedTx = + prepareTransaction(TransactionType.ACCESS_LIST, 0, Wei.of(500), 0); + Transaction txAccessList = preparedTx.accessList(ales).createTransaction(KEYS1); + BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + txAccessList.writeTo(rlpOut); + + txAccessList = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + System.out.println(txAccessList.getSender()); + System.out.println(txAccessList.getHash()); + System.out.println(txAccessList.getSize()); + + final var optAL = txAccessList.getAccessList(); + + final ClassLayout cl1 = ClassLayout.parseInstance(optAL); + System.out.println(cl1.toPrintable()); + System.out.println("Optional size: " + cl1.instanceSize()); + + final ClassLayout cl2 = ClassLayout.parseInstance(optAL.get()); + System.out.println(cl2.toPrintable()); + System.out.println("Optional + list size: " + cl2.instanceSize()); + + assertThat(cl2.instanceSize()).isEqualTo(PendingTransaction.OPTIONAL_ACCESS_LIST_MEMORY_SIZE); + + final AccessListEntry ale = optAL.get().get(0); + + final ClassLayout cl3 = ClassLayout.parseInstance(ale); + System.out.println(cl3.toPrintable()); + System.out.println("AccessListEntry size: " + cl3.instanceSize()); + + LongAdder size = new LongAdder(); + size.add(cl3.instanceSize()); + + GraphVisitor gv = + gpr -> { + // byte[] is shared so only count the specific part for each field + if (gpr.path().endsWith(".bytes")) { + if (gpr.path().contains("address")) { + size.add(20); + System.out.println( + "(" + + size + + ")[20 = fixed address size; overrides: " + + gpr.size() + + ", " + + gpr.path() + + ", " + + gpr.klass().toString() + + "]"); + } + } else if (!gpr.path() + .contains( + "storageKeys.elementData[")) { // exclude elements since we want the container + // size + size.add(gpr.size()); + System.out.println( + "(" + + size + + ")[" + + gpr.size() + + ", " + + gpr.path() + + ", " + + gpr.klass().toString() + + "]"); + } + }; + + GraphWalker gw = new GraphWalker(gv); + + gw.walk(ale); + + System.out.println("AccessListEntry container size: " + size); + + assertThat(size.sum()).isEqualTo(PendingTransaction.ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE); + + final Bytes32 storageKey = ale.getStorageKeys().get(0); + final ClassLayout cl4 = ClassLayout.parseInstance(storageKey); + System.out.println(cl4.toPrintable()); + System.out.println("Single storage key size: " + cl4.instanceSize()); + + assertThat(cl4.instanceSize()) + .isEqualTo(PendingTransaction.ACCESS_LIST_STORAGE_KEY_MEMORY_SIZE); + } + + @Test + public void baseEIP1559TransactionMemorySize() { + System.setProperty("jol.magicFieldOffset", "true"); + Transaction txEip1559 = createEIP1559Transaction(1, KEYS1, 10); + BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + txEip1559.writeTo(rlpOut); + + txEip1559 = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + System.out.println(txEip1559.getSender()); + System.out.println(txEip1559.getHash()); + System.out.println(txEip1559.getSize()); + + final ClassLayout cl = ClassLayout.parseInstance(txEip1559); + System.out.println(cl.toPrintable()); + LongAdder eip1559size = new LongAdder(); + eip1559size.add(cl.instanceSize()); + System.out.println(eip1559size); + + final Set skipPrefixes = new HashSet<>(); + + GraphVisitor gv = + gpr -> { + if (!skipPrefixes.stream().anyMatch(sp -> gpr.path().startsWith(sp))) { + if (SHARED_CLASSES.stream().anyMatch(scz -> scz.isAssignableFrom(gpr.klass()))) { + skipPrefixes.add(gpr.path()); + } else if (!startWithAnyOf(EIP1559_CONSTANT_FIELD_PATHS, gpr) + && !startWithAnyOf(EIP1559_VARIABLE_SIZE_PATHS, gpr)) { + eip1559size.add(gpr.size()); + System.out.println( + "(" + + eip1559size + + ")[" + + gpr.size() + + ", " + + gpr.path() + + ", " + + gpr.klass().toString() + + "]"); + } + } + }; + + GraphWalker gw = new GraphWalker(gv); + + gw.walk(txEip1559); + + System.out.println("Base EIP1559 size: " + eip1559size); + assertThat(eip1559size.sum()).isEqualTo(PendingTransaction.EIP1559_BASE_MEMORY_SIZE); + } + + @Test + public void baseAccessListTransactionMemorySize() { + System.setProperty("jol.magicFieldOffset", "true"); + Transaction txAccessList = + createTransaction(TransactionType.ACCESS_LIST, 1, Wei.of(500), 0, KEYS1); + BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + txAccessList.writeTo(rlpOut); + + txAccessList = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + System.out.println(txAccessList.getSender()); + System.out.println(txAccessList.getHash()); + System.out.println(txAccessList.getSize()); + + final ClassLayout cl = ClassLayout.parseInstance(txAccessList); + System.out.println(cl.toPrintable()); + LongAdder accessListSize = new LongAdder(); + accessListSize.add(cl.instanceSize()); + System.out.println(accessListSize); + + final Set skipPrefixes = new HashSet<>(); + + GraphVisitor gv = + gpr -> { + if (!skipPrefixes.stream().anyMatch(sp -> gpr.path().startsWith(sp))) { + if (SHARED_CLASSES.stream().anyMatch(scz -> scz.isAssignableFrom(gpr.klass()))) { + skipPrefixes.add(gpr.path()); + } else if (!startWithAnyOf(FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS, gpr) + && !startWithAnyOf(FRONTIER_ACCESS_LIST_VARIABLE_SIZE_PATHS, gpr)) { + accessListSize.add(gpr.size()); + System.out.println( + "(" + + accessListSize + + ")[" + + gpr.size() + + ", " + + gpr.path() + + ", " + + gpr.klass().toString() + + "]"); + } + } + }; + + GraphWalker gw = new GraphWalker(gv); + + gw.walk(txAccessList); + System.out.println("Base Access List size: " + accessListSize); + assertThat(accessListSize.sum()).isEqualTo(PendingTransaction.ACCESS_LIST_BASE_MEMORY_SIZE); + } + + @Test + public void baseFrontierTransactionMemorySize() { + System.setProperty("jol.magicFieldOffset", "true"); + Transaction txFrontier = createTransaction(TransactionType.FRONTIER, 1, Wei.of(500), 0, KEYS1); + BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + txFrontier.writeTo(rlpOut); + + txFrontier = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)); + System.out.println(txFrontier.getSender()); + System.out.println(txFrontier.getHash()); + System.out.println(txFrontier.getSize()); + + final ClassLayout cl = ClassLayout.parseInstance(txFrontier); + System.out.println(cl.toPrintable()); + LongAdder frontierSize = new LongAdder(); + frontierSize.add(cl.instanceSize()); + System.out.println(frontierSize); + + final Set skipPrefixes = new HashSet<>(); + + GraphVisitor gv = + gpr -> { + if (!skipPrefixes.stream().anyMatch(sp -> gpr.path().startsWith(sp))) { + if (SHARED_CLASSES.stream().anyMatch(scz -> scz.isAssignableFrom(gpr.klass()))) { + skipPrefixes.add(gpr.path()); + } else if (!startWithAnyOf(FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS, gpr) + && !startWithAnyOf(FRONTIER_ACCESS_LIST_VARIABLE_SIZE_PATHS, gpr)) { + frontierSize.add(gpr.size()); + System.out.println( + "(" + + frontierSize + + ")[" + + gpr.size() + + ", " + + gpr.path() + + ", " + + gpr.klass().toString() + + "]"); + } + } + }; + + GraphWalker gw = new GraphWalker(gv); + + gw.walk(txFrontier); + System.out.println("Base Frontier size: " + frontierSize); + assertThat(frontierSize.sum()).isEqualTo(PendingTransaction.FRONTIER_BASE_MEMORY_SIZE); + } + + private boolean startWithAnyOf(final Set prefixes, final GraphPathRecord path) { + return prefixes.stream().anyMatch(prefix -> path.path().startsWith(prefix)); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcasterTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcasterTest.java index bf1c0c8cf8..fedbf68c35 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcasterTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcasterTest.java @@ -35,7 +35,6 @@ import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.messages.EthPV65; import org.hyperledger.besu.plugin.data.TransactionType; -import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -328,7 +327,7 @@ public class TransactionBroadcasterTest { private Set createPendingTransactionList(final int num, final boolean local) { return IntStream.range(0, num) .mapToObj(unused -> generator.transaction()) - .map(tx -> new PendingTransaction(tx, local, Instant.now())) + .map(tx -> local ? new PendingTransaction.Local(tx) : new PendingTransaction.Remote(tx)) .collect(Collectors.toSet()); } @@ -336,7 +335,7 @@ public class TransactionBroadcasterTest { final TransactionType type, final int num, final boolean local) { return IntStream.range(0, num) .mapToObj(unused -> generator.transaction(type)) - .map(tx -> new PendingTransaction(tx, local, Instant.now())) + .map(tx -> local ? new PendingTransaction.Local(tx) : new PendingTransaction.Remote(tx)) .collect(Collectors.toSet()); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java index b0943c8190..80482a01f9 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java @@ -79,7 +79,7 @@ public class TransactionPoolFactoryTest { @Mock EthMessages ethMessages; @Mock EthScheduler ethScheduler; - @Mock GasPricePendingTransactionsSorter pendingTransactions; + @Mock PendingTransactions pendingTransactions; @Mock PeerTransactionTracker peerTransactionTracker; @Mock TransactionsMessageSender transactionsMessageSender; @@ -243,7 +243,7 @@ public class TransactionPoolFactoryTest { schedule, context, ethContext, - new NoOpMetricsSystem(), + new TransactionPoolMetrics(new NoOpMetricsSystem()), syncState, new MiningParameters.Builder().minTransactionGasPrice(Wei.ONE).build(), ImmutableTransactionPoolConfiguration.builder() diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessorTest.java index 1f0719f90e..b8a3593b13 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessorTest.java @@ -54,7 +54,8 @@ public class TransactionsMessageProcessorTest { metricsSystem = new StubMetricsSystem(); messageHandler = - new TransactionsMessageProcessor(transactionTracker, transactionPool, metricsSystem); + new TransactionsMessageProcessor( + transactionTracker, transactionPool, new TransactionPoolMetrics(metricsSystem)); } @Test @@ -87,7 +88,11 @@ public class TransactionsMessageProcessorTest { now().minus(ofMinutes(1)), ofMillis(1)); verifyNoInteractions(transactionTracker); - assertThat(metricsSystem.getCounterValue("transactions_messages_skipped_total")).isEqualTo(1); + assertThat( + metricsSystem.getCounterValue( + TransactionPoolMetrics.EXPIRED_MESSAGES_COUNTER_NAME, + TransactionsMessageProcessor.METRIC_LABEL)) + .isEqualTo(1); } @Test @@ -98,6 +103,10 @@ public class TransactionsMessageProcessorTest { now().minus(ofMinutes(1)), ofMillis(1)); verifyNoInteractions(transactionPool); - assertThat(metricsSystem.getCounterValue("transactions_messages_skipped_total")).isEqualTo(1); + assertThat( + metricsSystem.getCounterValue( + TransactionPoolMetrics.EXPIRED_MESSAGES_COUNTER_NAME, + TransactionsMessageProcessor.METRIC_LABEL)) + .isEqualTo(1); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactionsTestBase.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactionsTestBase.java new file mode 100644 index 0000000000..86741de914 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactionsTestBase.java @@ -0,0 +1,185 @@ +/* + * Copyright Besu contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ADDED; + +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolReplacementHandler; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.function.BiFunction; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Test; + +public abstract class AbstractPrioritizedTransactionsTestBase extends BaseTransactionPoolTest { + protected static final int MAX_TRANSACTIONS = 5; + protected final TransactionPoolMetrics txPoolMetrics = new TransactionPoolMetrics(metricsSystem); + protected final EvictCollectorLayer evictCollector = new EvictCollectorLayer(txPoolMetrics); + protected AbstractPrioritizedTransactions transactions = + getSorter( + ImmutableTransactionPoolConfiguration.builder() + .maxPrioritizedTransactions(MAX_TRANSACTIONS) + .maxFutureBySender(MAX_TRANSACTIONS) + .build()); + + private AbstractPrioritizedTransactions getSorter(final TransactionPoolConfiguration poolConfig) { + return getSorter( + poolConfig, + evictCollector, + txPoolMetrics, + (pt1, pt2) -> transactionReplacementTester(poolConfig, pt1, pt2)); + } + + abstract AbstractPrioritizedTransactions getSorter( + final TransactionPoolConfiguration poolConfig, + final TransactionsLayer nextLayer, + final TransactionPoolMetrics txPoolMetrics, + final BiFunction + transactionReplacementTester); + + abstract BlockHeader mockBlockHeader(); + + private boolean transactionReplacementTester( + final TransactionPoolConfiguration poolConfig, + final PendingTransaction pt1, + final PendingTransaction pt2) { + final TransactionPoolReplacementHandler transactionReplacementHandler = + new TransactionPoolReplacementHandler(poolConfig.getPriceBump()); + return transactionReplacementHandler.shouldReplace(pt1, pt2, mockBlockHeader()); + } + + @Test + public void prioritizeLocalTransactionThenValue() { + final PendingTransaction localTransaction = + createLocalPendingTransaction(createTransaction(0, KEYS1)); + assertThat(prioritizeTransaction(localTransaction)).isEqualTo(ADDED); + + final List remoteTxs = new ArrayList<>(); + TransactionAddedResult prioritizeResult = null; + for (int i = 0; i < MAX_TRANSACTIONS; i++) { + final PendingTransaction highValueRemoteTx = + createRemotePendingTransaction( + createTransaction( + 0, + Wei.of(BigInteger.valueOf(100).pow(i)), + SIGNATURE_ALGORITHM.get().generateKeyPair())); + remoteTxs.add(highValueRemoteTx); + prioritizeResult = prioritizeTransaction(highValueRemoteTx); + assertThat(prioritizeResult).isEqualTo(ADDED); + } + + assertEvicted(remoteTxs.get(0)); + assertTransactionPrioritized(localTransaction); + remoteTxs.stream().skip(1).forEach(remoteTx -> assertTransactionPrioritized(remoteTx)); + } + + @Test + public void shouldStartDroppingLocalTransactionsWhenPoolIsFullOfLocalTransactions() { + final List localTransactions = new ArrayList<>(); + + for (int i = 0; i < MAX_TRANSACTIONS; i++) { + final var localTransaction = createLocalPendingTransaction(createTransaction(i)); + assertThat(prioritizeTransaction(localTransaction)).isEqualTo(ADDED); + localTransactions.add(localTransaction); + } + + assertThat(transactions.count()).isEqualTo(MAX_TRANSACTIONS); + + // this will be rejected since the prioritized set is full of txs from the same sender with + // lower nonce + final var lastLocalTransaction = + createLocalPendingTransaction(createTransaction(MAX_TRANSACTIONS)); + prioritizeTransaction(lastLocalTransaction); + assertEvicted(lastLocalTransaction); + + assertThat(transactions.count()).isEqualTo(MAX_TRANSACTIONS); + + localTransactions.forEach(this::assertTransactionPrioritized); + assertTransactionNotPrioritized(lastLocalTransaction); + } + + protected void shouldPrioritizeValueThenTimeAddedToPool( + final Iterator lowValueTxSupplier, + final PendingTransaction highValueTx, + final PendingTransaction expectedDroppedTx) { + + // Fill the pool with transactions from random senders + final List lowGasPriceTransactions = + IntStream.range(0, MAX_TRANSACTIONS) + .mapToObj( + i -> { + final var lowPriceTx = lowValueTxSupplier.next(); + final var prioritizeResult = transactions.add(lowPriceTx, 0); + + assertThat(prioritizeResult).isEqualTo(ADDED); + assertThat(evictCollector.getEvictedTransactions()).isEmpty(); + return lowPriceTx; + }) + .toList(); + + assertThat(transactions.count()).isEqualTo(MAX_TRANSACTIONS); + + // This should kick the oldest tx with the low gas price out, namely the first one we added + final var highValuePrioRes = transactions.add(highValueTx, 0); + assertThat(highValuePrioRes).isEqualTo(ADDED); + assertEvicted(expectedDroppedTx); + + assertTransactionPrioritized(highValueTx); + lowGasPriceTransactions.stream() + .filter(tx -> !tx.equals(expectedDroppedTx)) + .forEach(tx -> assertThat(transactions.getByHash(tx.getHash())).isPresent()); + } + + protected TransactionAddedResult prioritizeTransaction(final Transaction tx) { + return prioritizeTransaction(createRemotePendingTransaction(tx)); + } + + protected TransactionAddedResult prioritizeTransaction(final PendingTransaction tx) { + return transactions.add(tx, 0); + } + + protected void assertTransactionPrioritized(final PendingTransaction tx) { + assertThat(transactions.getByHash(tx.getHash())).isPresent(); + } + + protected void assertTransactionNotPrioritized(final PendingTransaction tx) { + assertThat(transactions.getByHash(tx.getHash())).isEmpty(); + } + + protected void assertTransactionPrioritized(final Transaction tx) { + assertThat(transactions.getByHash(tx.getHash())).isPresent(); + } + + protected void assertTransactionNotPrioritized(final Transaction tx) { + assertThat(transactions.getByHash(tx.getHash())).isEmpty(); + } + + protected void assertEvicted(final PendingTransaction tx) { + assertThat(evictCollector.getEvictedTransactions()).contains(tx); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactionsTest.java new file mode 100644 index 0000000000..f21c62e9d9 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseFeePrioritizedTransactionsTest.java @@ -0,0 +1,173 @@ +/* + * Copyright Besu contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.hyperledger.besu.plugin.data.TransactionType.EIP1559; +import static org.hyperledger.besu.plugin.data.TransactionType.FRONTIER; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.TransactionTestFixture; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.plugin.data.TransactionType; + +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.function.BiFunction; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Test; + +public class BaseFeePrioritizedTransactionsTest extends AbstractPrioritizedTransactionsTestBase { + + private static final Random randomizeTxType = new Random(); + + @Override + AbstractPrioritizedTransactions getSorter( + final TransactionPoolConfiguration poolConfig, + final TransactionsLayer nextLayer, + final TransactionPoolMetrics txPoolMetrics, + final BiFunction + transactionReplacementTester) { + + return new BaseFeePrioritizedTransactions( + poolConfig, + this::mockBlockHeader, + nextLayer, + txPoolMetrics, + transactionReplacementTester, + FeeMarket.london(0L)); + } + + @Override + protected BlockHeader mockBlockHeader() { + final BlockHeader blockHeader = mock(BlockHeader.class); + when(blockHeader.getBaseFee()).thenReturn(Optional.of(Wei.ONE)); + return blockHeader; + } + + @Override + protected Transaction createTransaction( + final long nonce, final Wei maxGasPrice, final KeyPair keys) { + + return createTransaction( + randomizeTxType.nextBoolean() ? EIP1559 : FRONTIER, nonce, maxGasPrice, keys); + } + + protected Transaction createTransaction( + final TransactionType type, final long nonce, final Wei maxGasPrice, final KeyPair keys) { + + var tx = new TransactionTestFixture().value(Wei.of(nonce)).nonce(nonce).type(type); + if (type.supports1559FeeMarket()) { + tx.maxFeePerGas(Optional.of(maxGasPrice)) + .maxPriorityFeePerGas(Optional.of(maxGasPrice.divide(10))); + } else { + tx.gasPrice(maxGasPrice); + } + return tx.createTransaction(keys); + } + + @Override + protected Transaction createTransactionReplacement( + final Transaction originalTransaction, final KeyPair keys) { + return createTransaction( + originalTransaction.getType(), + originalTransaction.getNonce(), + originalTransaction.getMaxGasPrice().multiply(2), + keys); + } + + @Test + public void shouldPrioritizePriorityFeeThenTimeAddedToPoolOnlyEIP1559Txs() { + shouldPrioritizePriorityFeeThenTimeAddedToPoolSameTypeTxs(EIP1559); + } + + @Test + public void shouldPrioritizeGasPriceThenTimeAddedToPoolOnlyFrontierTxs() { + shouldPrioritizePriorityFeeThenTimeAddedToPoolSameTypeTxs(FRONTIER); + } + + @Test + public void shouldPrioritizeEffectivePriorityFeeThenTimeAddedToPoolOnMixedTypes() { + final var nextBlockBaseFee = Optional.of(Wei.ONE); + + final PendingTransaction highGasPriceTransaction = + createRemotePendingTransaction(createTransaction(0, Wei.of(100), KEYS1)); + + final List lowValueTxs = + IntStream.range(0, MAX_TRANSACTIONS) + .mapToObj( + i -> + new PendingTransaction.Remote( + createTransaction( + 0, Wei.of(10), SIGNATURE_ALGORITHM.get().generateKeyPair()))) + .collect(Collectors.toUnmodifiableList()); + + final var lowestPriorityFee = + lowValueTxs.stream() + .sorted( + Comparator.comparing( + pt -> pt.getTransaction().getEffectivePriorityFeePerGas(nextBlockBaseFee))) + .findFirst() + .get() + .getTransaction() + .getEffectivePriorityFeePerGas(nextBlockBaseFee); + + final var firstLowValueTx = + lowValueTxs.stream() + .filter( + pt -> + pt.getTransaction() + .getEffectivePriorityFeePerGas(nextBlockBaseFee) + .equals(lowestPriorityFee)) + .findFirst() + .get(); + + shouldPrioritizeValueThenTimeAddedToPool( + lowValueTxs.iterator(), highGasPriceTransaction, firstLowValueTx); + } + + private void shouldPrioritizePriorityFeeThenTimeAddedToPoolSameTypeTxs( + final TransactionType transactionType) { + final PendingTransaction highGasPriceTransaction = + createRemotePendingTransaction(createTransaction(0, Wei.of(100), KEYS1)); + + final var lowValueTxs = + IntStream.range(0, MAX_TRANSACTIONS) + .mapToObj( + i -> + createRemotePendingTransaction( + createTransaction( + transactionType, + 0, + Wei.of(10), + 0, + SIGNATURE_ALGORITHM.get().generateKeyPair()))) + .collect(Collectors.toUnmodifiableList()); + + shouldPrioritizeValueThenTimeAddedToPool( + lowValueTxs.iterator(), highGasPriceTransaction, lowValueTxs.get(0)); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java new file mode 100644 index 0000000000..defafb1b36 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java @@ -0,0 +1,178 @@ +/* + * Copyright Besu contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.crypto.SignatureAlgorithm; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.TransactionTestFixture; +import org.hyperledger.besu.ethereum.core.Util; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.metrics.StubMetricsSystem; +import org.hyperledger.besu.plugin.data.TransactionType; + +import java.util.Optional; +import java.util.Random; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import org.apache.tuweni.bytes.Bytes; + +public class BaseTransactionPoolTest { + + protected static final Supplier SIGNATURE_ALGORITHM = + Suppliers.memoize(SignatureAlgorithmFactory::getInstance); + protected static final KeyPair KEYS1 = SIGNATURE_ALGORITHM.get().generateKeyPair(); + protected static final KeyPair KEYS2 = SIGNATURE_ALGORITHM.get().generateKeyPair(); + protected static final Address SENDER1 = Util.publicKeyToAddress(KEYS1.getPublicKey()); + protected static final Address SENDER2 = Util.publicKeyToAddress(KEYS2.getPublicKey()); + + private static final Random randomizeTxType = new Random(); + + protected final Transaction transaction0 = createTransaction(0); + protected final Transaction transaction1 = createTransaction(1); + protected final Transaction transaction2 = createTransaction(2); + + protected final StubMetricsSystem metricsSystem = new StubMetricsSystem(); + + protected Transaction createTransaction(final long nonce) { + return createTransaction(nonce, Wei.of(5000L), KEYS1); + } + + protected Transaction createTransaction(final long nonce, final KeyPair keys) { + return createTransaction(nonce, Wei.of(5000L), keys); + } + + protected Transaction createTransaction(final long nonce, final Wei maxGasPrice) { + return createTransaction(nonce, maxGasPrice, KEYS1); + } + + protected Transaction createTransaction(final long nonce, final int payloadSize) { + return createTransaction(nonce, Wei.of(5000L), payloadSize, KEYS1); + } + + protected Transaction createTransaction( + final long nonce, final Wei maxGasPrice, final KeyPair keys) { + return createTransaction(nonce, maxGasPrice, 0, keys); + } + + protected Transaction createEIP1559Transaction( + final long nonce, final KeyPair keys, final int gasFeeMultiplier) { + return createTransaction( + TransactionType.EIP1559, nonce, Wei.of(5000L).multiply(gasFeeMultiplier), 0, keys); + } + + protected Transaction createTransaction( + final long nonce, final Wei maxGasPrice, final int payloadSize, final KeyPair keys) { + + // ToDo 4844: include BLOB tx here + final TransactionType txType = TransactionType.values()[randomizeTxType.nextInt(3)]; + + return createTransaction(txType, nonce, maxGasPrice, payloadSize, keys); + } + + protected Transaction createTransaction( + final TransactionType type, + final long nonce, + final Wei maxGasPrice, + final int payloadSize, + final KeyPair keys) { + return prepareTransaction(type, nonce, maxGasPrice, payloadSize).createTransaction(keys); + } + + protected TransactionTestFixture prepareTransaction( + final TransactionType type, final long nonce, final Wei maxGasPrice, final int payloadSize) { + + var tx = + new TransactionTestFixture() + .to(Optional.of(Address.fromHexString("0x634316eA0EE79c701c6F67C53A4C54cBAfd2316d"))) + .value(Wei.of(nonce)) + .nonce(nonce) + .type(type); + if (payloadSize > 0) { + var payloadBytes = Bytes.repeat((byte) 1, payloadSize); + tx.payload(payloadBytes); + } + if (type.supports1559FeeMarket()) { + tx.maxFeePerGas(Optional.of(maxGasPrice)) + .maxPriorityFeePerGas(Optional.of(maxGasPrice.divide(10))); + } else { + tx.gasPrice(maxGasPrice); + } + return tx; + } + + protected Transaction createTransactionReplacement( + final Transaction originalTransaction, final KeyPair keys) { + return createTransaction( + originalTransaction.getType(), + originalTransaction.getNonce(), + originalTransaction.getMaxGasPrice().multiply(2), + 0, + keys); + } + + protected PendingTransaction createRemotePendingTransaction(final Transaction transaction) { + return new PendingTransaction.Remote(transaction); + } + + protected PendingTransaction createLocalPendingTransaction(final Transaction transaction) { + return new PendingTransaction.Local(transaction); + } + + protected void assertTransactionPending( + final PendingTransactions transactions, final Transaction t) { + assertThat(transactions.getTransactionByHash(t.getHash())).contains(t); + } + + protected void assertTransactionNotPending( + final PendingTransactions transactions, final Transaction t) { + assertThat(transactions.getTransactionByHash(t.getHash())).isEmpty(); + } + + protected void assertNoNextNonceForSender( + final PendingTransactions pendingTransactions, final Address sender) { + assertThat(pendingTransactions.getNextNonceForSender(sender)).isEmpty(); + } + + protected void assertNextNonceForSender( + final PendingTransactions pendingTransactions, final Address sender1, final int i) { + assertThat(pendingTransactions.getNextNonceForSender(sender1)).isPresent().hasValue(i); + } + + protected void addLocalTransactions( + final PendingTransactions sorter, final Account sender, final long... nonces) { + for (final long nonce : nonces) { + sorter.addLocalTransaction(createTransaction(nonce), Optional.of(sender)); + } + } + + protected long getAddedCount(final String source, final String layer) { + return metricsSystem.getCounterValue(TransactionPoolMetrics.ADDED_COUNTER_NAME, source, layer); + } + + protected long getRemovedCount(final String source, final String operation, final String layer) { + return metricsSystem.getCounterValue( + TransactionPoolMetrics.REMOVED_COUNTER_NAME, source, operation, layer); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/EvictCollectorLayer.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/EvictCollectorLayer.java new file mode 100644 index 0000000000..27f478d172 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/EvictCollectorLayer.java @@ -0,0 +1,47 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; + +import java.util.ArrayList; +import java.util.List; + +public class EvictCollectorLayer extends EndLayer { + static final String LAYER_NAME = "evict-collector"; + final List evictedTxs = new ArrayList<>(); + + public EvictCollectorLayer(final TransactionPoolMetrics metrics) { + super(metrics); + } + + @Override + public String name() { + return LAYER_NAME; + } + + @Override + public TransactionAddedResult add(final PendingTransaction pendingTransaction, final int gap) { + final var res = super.add(pendingTransaction, gap); + evictedTxs.add(pendingTransaction); + return res; + } + + public List getEvictedTransactions() { + return evictedTxs; + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactionsTest.java new file mode 100644 index 0000000000..e41130f4f6 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/GasPricePrioritizedTransactionsTest.java @@ -0,0 +1,91 @@ +/* + * Copyright Besu contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.TransactionTestFixture; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; + +import java.util.List; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Test; + +public class GasPricePrioritizedTransactionsTest extends AbstractPrioritizedTransactionsTestBase { + + @Override + AbstractPrioritizedTransactions getSorter( + final TransactionPoolConfiguration poolConfig, + final TransactionsLayer nextLayer, + final TransactionPoolMetrics txPoolMetrics, + final BiFunction + transactionReplacementTester) { + + return new GasPricePrioritizedTransactions( + poolConfig, nextLayer, txPoolMetrics, transactionReplacementTester); + } + + @Override + protected BlockHeader mockBlockHeader() { + final BlockHeader blockHeader = mock(BlockHeader.class); + when(blockHeader.getBaseFee()).thenReturn(Optional.empty()); + return blockHeader; + } + + @Override + protected Transaction createTransaction( + final long transactionNumber, final Wei maxGasPrice, final KeyPair keys) { + return new TransactionTestFixture() + .value(Wei.of(transactionNumber)) + .nonce(transactionNumber) + .gasPrice(maxGasPrice) + .createTransaction(keys); + } + + @Override + protected Transaction createTransactionReplacement( + final Transaction originalTransaction, final KeyPair keys) { + return createTransaction( + originalTransaction.getNonce(), originalTransaction.getMaxGasPrice().multiply(2), keys); + } + + @Test + public void shouldPrioritizeGasPriceThenTimeAddedToPool() { + final List lowValueTxs = + IntStream.range(0, MAX_TRANSACTIONS) + .mapToObj( + i -> + createRemotePendingTransaction( + createTransaction( + 0, Wei.of(10), SIGNATURE_ALGORITHM.get().generateKeyPair()))) + .toList(); + + final PendingTransaction highGasPriceTransaction = + createRemotePendingTransaction(createTransaction(0, Wei.of(100), KEYS1)); + + shouldPrioritizeValueThenTimeAddedToPool( + lowValueTxs.iterator(), highGasPriceTransaction, lowValueTxs.get(0)); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsLegacyTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsLegacyTest.java new file mode 100644 index 0000000000..c165944633 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsLegacyTest.java @@ -0,0 +1,241 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; +import static org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason.INVALID_TRANSACTION_FORMAT; +import static org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason.REPLAY_PROTECTED_SIGNATURE_REQUIRED; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockBody; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.core.ExecutionContextTestFixture; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.core.TransactionTestFixture; +import org.hyperledger.besu.ethereum.eth.transactions.AbstractTransactionsLayeredPendingTransactionsTest; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.plugin.data.TransactionType; + +import java.math.BigInteger; +import java.util.List; +import java.util.Optional; +import java.util.function.BiFunction; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@SuppressWarnings("unchecked") +@RunWith(MockitoJUnitRunner.class) +public class LayeredPendingTransactionsLegacyTest + extends AbstractTransactionsLayeredPendingTransactionsTest { + + @Override + protected PendingTransactions createPendingTransactionsSorter( + final TransactionPoolConfiguration poolConfig, + final BiFunction + transactionReplacementTester) { + + final var txPoolMetrics = new TransactionPoolMetrics(metricsSystem); + return new LayeredPendingTransactions( + poolConfig, + new GasPricePrioritizedTransactions( + poolConfig, new EndLayer(txPoolMetrics), txPoolMetrics, transactionReplacementTester)); + } + + @Override + protected Transaction createTransaction( + final int nonce, final Optional maybeChainId) { + return createBaseTransaction(nonce).chainId(maybeChainId).createTransaction(KEY_PAIR1); + } + + @Override + protected Transaction createTransaction(final int nonce, final Wei maxPrice) { + return createBaseTransaction(nonce).gasPrice(maxPrice).createTransaction(KEY_PAIR1); + } + + @Override + protected TransactionTestFixture createBaseTransaction(final int nonce) { + return new TransactionTestFixture() + .nonce(nonce) + .gasLimit(blockGasLimit) + .type(TransactionType.FRONTIER); + } + + @Override + protected ExecutionContextTestFixture createExecutionContextTestFixture() { + return ExecutionContextTestFixture.create(); + } + + @Override + protected FeeMarket getFeeMarket() { + return FeeMarket.legacy(); + } + + @Override + protected Block appendBlock( + final Difficulty difficulty, + final BlockHeader parentBlock, + final Transaction... transactionsToAdd) { + final List transactionList = asList(transactionsToAdd); + final Block block = + new Block( + new BlockHeaderTestFixture() + .difficulty(difficulty) + .gasLimit(parentBlock.getGasLimit()) + .parentHash(parentBlock.getHash()) + .number(parentBlock.getNumber() + 1) + .buildHeader(), + new BlockBody(transactionList, emptyList())); + final List transactionReceipts = + transactionList.stream() + .map(transaction -> new TransactionReceipt(1, 1, emptyList(), Optional.empty())) + .collect(toList()); + blockchain.appendBlock(block, transactionReceipts); + return block; + } + + @Test + public void + addLocalTransaction_strictReplayProtectionOn_txWithoutChainId_chainIdIsConfigured_protectionNotSupportedAtCurrentBlock() { + protocolSupportsTxReplayProtection(1337, false); + transactionPool = createTransactionPool(b -> b.strictTransactionReplayProtectionEnabled(true)); + final Transaction tx = createTransactionWithoutChainId(0); + givenTransactionIsValid(tx); + + addAndAssertLocalTransactionValid(tx); + } + + @Test + public void + addRemoteTransactions_strictReplayProtectionOff_txWithoutChainId_chainIdIsConfigured() { + protocolSupportsTxReplayProtection(1337, true); + transactionPool = createTransactionPool(b -> b.strictTransactionReplayProtectionEnabled(false)); + final Transaction tx = createTransactionWithoutChainId(0); + givenTransactionIsValid(tx); + + addAndAssertRemoteTransactionValid(tx); + } + + @Test + public void addLocalTransaction_strictReplayProtectionOff_txWithoutChainId_chainIdIsConfigured() { + protocolSupportsTxReplayProtection(1337, true); + transactionPool = createTransactionPool(b -> b.strictTransactionReplayProtectionEnabled(false)); + final Transaction tx = createTransactionWithoutChainId(0); + givenTransactionIsValid(tx); + + addAndAssertLocalTransactionValid(tx); + } + + @Test + public void addLocalTransaction_strictReplayProtectionOn_txWithoutChainId_chainIdIsConfigured() { + protocolSupportsTxReplayProtection(1337, true); + transactionPool = createTransactionPool(b -> b.strictTransactionReplayProtectionEnabled(true)); + final Transaction tx = createTransactionWithoutChainId(0); + givenTransactionIsValid(tx); + + addAndAssertLocalTransactionInvalid(tx, REPLAY_PROTECTED_SIGNATURE_REQUIRED); + } + + @Test + public void + addRemoteTransactions_strictReplayProtectionOn_txWithoutChainId_chainIdIsConfigured() { + protocolSupportsTxReplayProtection(1337, true); + transactionPool = createTransactionPool(b -> b.strictTransactionReplayProtectionEnabled(true)); + final Transaction tx = createTransactionWithoutChainId(0); + givenTransactionIsValid(tx); + + addAndAssertRemoteTransactionValid(tx); + } + + @Test + public void + addLocalTransaction_strictReplayProtectionOn_txWithoutChainId_chainIdIsNotConfigured() { + protocolDoesNotSupportTxReplayProtection(); + transactionPool = createTransactionPool(b -> b.strictTransactionReplayProtectionEnabled(true)); + final Transaction tx = createTransactionWithoutChainId(0); + givenTransactionIsValid(tx); + + addAndAssertLocalTransactionValid(tx); + } + + @Test + public void + addRemoteTransactions_strictReplayProtectionOn_txWithoutChainId_chainIdIsNotConfigured() { + protocolDoesNotSupportTxReplayProtection(); + transactionPool = createTransactionPool(b -> b.strictTransactionReplayProtectionEnabled(true)); + final Transaction tx = createTransactionWithoutChainId(0); + givenTransactionIsValid(tx); + + addAndAssertRemoteTransactionValid(tx); + } + + @Test + public void shouldIgnoreEIP1559TransactionWhenNotAllowed() { + final Transaction transaction = + createBaseTransaction(1) + .type(TransactionType.EIP1559) + .maxFeePerGas(Optional.of(Wei.of(100L))) + .maxPriorityFeePerGas(Optional.of(Wei.of(50L))) + .gasLimit(10) + .gasPrice(null) + .createTransaction(KEY_PAIR1); + + givenTransactionIsValid(transaction); + + addAndAssertLocalTransactionInvalid(transaction, INVALID_TRANSACTION_FORMAT); + } + + @Test + public void shouldAcceptZeroGasPriceFrontierTransactionsWhenMining() { + when(miningParameters.isMiningEnabled()).thenReturn(true); + + final Transaction transaction = createTransaction(0, Wei.ZERO); + + givenTransactionIsValid(transaction); + + addAndAssertLocalTransactionValid(transaction); + } + + @Test + public void shouldAcceptZeroGasPriceTransactionWhenMinGasPriceIsZero() { + when(miningParameters.getMinTransactionGasPrice()).thenReturn(Wei.ZERO); + + final Transaction transaction = createTransaction(0, Wei.ZERO); + + givenTransactionIsValid(transaction); + + addAndAssertLocalTransactionValid(transaction); + } + + private Transaction createTransactionWithoutChainId(final int nonce) { + return createTransaction(nonce, Optional.empty()); + } + + private void protocolDoesNotSupportTxReplayProtection() { + when(protocolSchedule.getChainId()).thenReturn(Optional.empty()); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsLondonTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsLondonTest.java new file mode 100644 index 0000000000..9be50ea03c --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsLondonTest.java @@ -0,0 +1,294 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.config.StubGenesisConfigOptions; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockBody; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.core.ExecutionContextTestFixture; +import org.hyperledger.besu.ethereum.core.PrivacyParameters; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.core.TransactionTestFixture; +import org.hyperledger.besu.ethereum.eth.transactions.AbstractTransactionsLayeredPendingTransactionsTest; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolScheduleBuilder; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpecAdapters; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.plugin.data.TransactionType; + +import java.math.BigInteger; +import java.util.List; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.junit.Test; + +public class LayeredPendingTransactionsLondonTest + extends AbstractTransactionsLayeredPendingTransactionsTest { + + private static final Wei BASE_FEE_FLOOR = Wei.of(7L); + + @Override + protected PendingTransactions createPendingTransactionsSorter( + final TransactionPoolConfiguration poolConfig, + final BiFunction + transactionReplacementTester) { + + final var txPoolMetrics = new TransactionPoolMetrics(metricsSystem); + return new LayeredPendingTransactions( + poolConfig, + new BaseFeePrioritizedTransactions( + poolConfig, + protocolContext.getBlockchain()::getChainHeadHeader, + new EndLayer(txPoolMetrics), + txPoolMetrics, + transactionReplacementTester, + FeeMarket.london(0L))); + } + + @Override + protected Transaction createTransaction( + final int nonce, final Optional maybeChainId) { + return createBaseTransaction(nonce).chainId(maybeChainId).createTransaction(KEY_PAIR1); + } + + @Override + protected Transaction createTransaction(final int nonce, final Wei maxPrice) { + return createBaseTransaction(nonce) + .maxFeePerGas(Optional.of(maxPrice)) + .maxPriorityFeePerGas(Optional.of(maxPrice.divide(5L))) + .createTransaction(KEY_PAIR1); + } + + @Override + protected TransactionTestFixture createBaseTransaction(final int nonce) { + return new TransactionTestFixture() + .nonce(nonce) + .gasLimit(blockGasLimit) + .gasPrice(null) + .maxFeePerGas(Optional.of(Wei.of(5000L))) + .maxPriorityFeePerGas(Optional.of(Wei.of(1000L))) + .type(TransactionType.EIP1559); + } + + @Override + protected ExecutionContextTestFixture createExecutionContextTestFixture() { + final ProtocolSchedule protocolSchedule = + new ProtocolScheduleBuilder( + new StubGenesisConfigOptions().londonBlock(0L).baseFeePerGas(10L), + BigInteger.valueOf(1), + ProtocolSpecAdapters.create(0, Function.identity()), + new PrivacyParameters(), + false, + EvmConfiguration.DEFAULT) + .createProtocolSchedule(); + final ExecutionContextTestFixture executionContextTestFixture = + ExecutionContextTestFixture.builder().protocolSchedule(protocolSchedule).build(); + + final Block block = + new Block( + new BlockHeaderTestFixture() + .gasLimit( + executionContextTestFixture + .getBlockchain() + .getChainHeadBlock() + .getHeader() + .getGasLimit()) + .difficulty(Difficulty.ONE) + .baseFeePerGas(Wei.of(10L)) + .parentHash(executionContextTestFixture.getBlockchain().getChainHeadHash()) + .number(executionContextTestFixture.getBlockchain().getChainHeadBlockNumber() + 1) + .buildHeader(), + new BlockBody(List.of(), List.of())); + executionContextTestFixture.getBlockchain().appendBlock(block, List.of()); + + return executionContextTestFixture; + } + + @Override + protected FeeMarket getFeeMarket() { + return FeeMarket.london(0L, Optional.of(BASE_FEE_FLOOR)); + } + + @Override + protected Block appendBlock( + final Difficulty difficulty, + final BlockHeader parentBlock, + final Transaction... transactionsToAdd) { + final List transactionList = asList(transactionsToAdd); + final Block block = + new Block( + new BlockHeaderTestFixture() + .baseFeePerGas(Wei.of(10L)) + .gasLimit(parentBlock.getGasLimit()) + .difficulty(difficulty) + .parentHash(parentBlock.getHash()) + .number(parentBlock.getNumber() + 1) + .buildHeader(), + new BlockBody(transactionList, emptyList())); + final List transactionReceipts = + transactionList.stream() + .map(transaction -> new TransactionReceipt(1, 1, emptyList(), Optional.empty())) + .collect(toList()); + blockchain.appendBlock(block, transactionReceipts); + return block; + } + + @Test + public void shouldAcceptZeroGasPriceFrontierTxsWhenMinGasPriceIsZeroAndLondonWithZeroBaseFee() { + when(miningParameters.getMinTransactionGasPrice()).thenReturn(Wei.ZERO); + when(protocolSpec.getFeeMarket()).thenReturn(FeeMarket.london(0, Optional.of(Wei.ZERO))); + whenBlockBaseFeeIs(Wei.ZERO); + + final Transaction frontierTransaction = createFrontierTransaction(0, Wei.ZERO); + + givenTransactionIsValid(frontierTransaction); + addAndAssertLocalTransactionValid(frontierTransaction); + } + + @Test + public void shouldAcceptZeroGasPrice1559TxsWhenMinGasPriceIsZeroAndLondonWithZeroBaseFee() { + when(miningParameters.getMinTransactionGasPrice()).thenReturn(Wei.ZERO); + when(protocolSpec.getFeeMarket()).thenReturn(FeeMarket.london(0, Optional.of(Wei.ZERO))); + whenBlockBaseFeeIs(Wei.ZERO); + + final Transaction transaction = createTransaction(0, Wei.ZERO); + + givenTransactionIsValid(transaction); + addAndAssertLocalTransactionValid(transaction); + } + + @Test + public void shouldAcceptBaseFeeFloorGasPriceFrontierTransactionsWhenMining() { + final Transaction frontierTransaction = createFrontierTransaction(0, BASE_FEE_FLOOR); + + givenTransactionIsValid(frontierTransaction); + + addAndAssertLocalTransactionValid(frontierTransaction); + } + + @Test + public void shouldRejectRemote1559TxsWhenMaxFeePerGasBelowMinGasPrice() { + final Wei genesisBaseFee = Wei.of(100L); + final Wei minGasPrice = Wei.of(200L); + final Wei lastBlockBaseFee = minGasPrice.add(50L); + final Wei txMaxFeePerGas = minGasPrice.subtract(1L); + + assertThat( + add1559TxAndGetPendingTxsCount( + genesisBaseFee, minGasPrice, lastBlockBaseFee, txMaxFeePerGas, false)) + .isEqualTo(0); + } + + @Test + public void shouldAcceptRemote1559TxsWhenMaxFeePerGasIsAtLeastEqualToMinGasPrice() { + final Wei genesisBaseFee = Wei.of(100L); + final Wei minGasPrice = Wei.of(200L); + final Wei lastBlockBaseFee = minGasPrice.add(50L); + final Wei txMaxFeePerGas = minGasPrice; + + assertThat( + add1559TxAndGetPendingTxsCount( + genesisBaseFee, minGasPrice, lastBlockBaseFee, txMaxFeePerGas, false)) + .isEqualTo(1); + } + + @Test + public void shouldRejectLocal1559TxsWhenMaxFeePerGasBelowMinGasPrice() { + final Wei genesisBaseFee = Wei.of(100L); + final Wei minGasPrice = Wei.of(200L); + final Wei lastBlockBaseFee = minGasPrice.add(50L); + final Wei txMaxFeePerGas = minGasPrice.subtract(1L); + + assertThat( + add1559TxAndGetPendingTxsCount( + genesisBaseFee, minGasPrice, lastBlockBaseFee, txMaxFeePerGas, true)) + .isEqualTo(0); + } + + @Test + public void shouldAcceptLocal1559TxsWhenMaxFeePerGasIsAtLeastEqualToMinMinGasPrice() { + final Wei genesisBaseFee = Wei.of(100L); + final Wei minGasPrice = Wei.of(200L); + final Wei lastBlockBaseFee = minGasPrice.add(50L); + final Wei txMaxFeePerGas = minGasPrice; + + assertThat( + add1559TxAndGetPendingTxsCount( + genesisBaseFee, minGasPrice, lastBlockBaseFee, txMaxFeePerGas, true)) + .isEqualTo(1); + } + + private int add1559TxAndGetPendingTxsCount( + final Wei genesisBaseFee, + final Wei minGasPrice, + final Wei lastBlockBaseFee, + final Wei txMaxFeePerGas, + final boolean isLocal) { + when(miningParameters.getMinTransactionGasPrice()).thenReturn(minGasPrice); + when(protocolSpec.getFeeMarket()).thenReturn(FeeMarket.london(0, Optional.of(genesisBaseFee))); + whenBlockBaseFeeIs(lastBlockBaseFee); + + final Transaction transaction = createTransaction(0, txMaxFeePerGas); + + givenTransactionIsValid(transaction); + + if (isLocal) { + transactionPool.addTransactionViaApi(transaction); + } else { + transactionPool.addRemoteTransactions(List.of(transaction)); + } + + return transactions.size(); + } + + private void whenBlockBaseFeeIs(final Wei baseFee) { + final BlockHeader header = + BlockHeaderBuilder.fromHeader(blockchain.getChainHeadHeader()) + .baseFee(baseFee) + .blockHeaderFunctions(new MainnetBlockHeaderFunctions()) + .parentHash(blockchain.getChainHeadHash()) + .buildBlockHeader(); + blockchain.appendBlock(new Block(header, BlockBody.empty()), emptyList()); + } + + private Transaction createFrontierTransaction(final int transactionNumber, final Wei gasPrice) { + return new TransactionTestFixture() + .nonce(transactionNumber) + .gasPrice(gasPrice) + .gasLimit(blockGasLimit) + .type(TransactionType.FRONTIER) + .createTransaction(KEY_PAIR1); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsTest.java new file mode 100644 index 0000000000..f864eb1dc2 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactionsTest.java @@ -0,0 +1,713 @@ +/* + * Copyright Besu contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions.TransactionSelectionResult.COMPLETE_OPERATION; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions.TransactionSelectionResult.CONTINUE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions.TransactionSelectionResult.DELETE_TRANSACTION_AND_CONTINUE; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ADDED; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ALREADY_KNOWN; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.REJECTED_UNDERPRICED_REPLACEMENT; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.DROPPED; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.REPLACED; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionAddedListener; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionDroppedListener; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolReplacementHandler; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.evm.account.Account; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.function.BiFunction; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class LayeredPendingTransactionsTest extends BaseTransactionPoolTest { + + protected static final int MAX_TRANSACTIONS = 5; + protected static final int MAX_CAPACITY_BYTES = 10_000; + protected static final int LIMITED_TRANSACTIONS_BY_SENDER = 4; + protected static final String REMOTE = "remote"; + protected static final String LOCAL = "local"; + protected final PendingTransactionAddedListener listener = + mock(PendingTransactionAddedListener.class); + protected final PendingTransactionDroppedListener droppedListener = + mock(PendingTransactionDroppedListener.class); + + private final TransactionPoolConfiguration poolConf = + ImmutableTransactionPoolConfiguration.builder() + .maxPrioritizedTransactions(MAX_TRANSACTIONS) + .maxFutureBySender(MAX_TRANSACTIONS) + .pendingTransactionsLayerMaxCapacityBytes(MAX_CAPACITY_BYTES) + .build(); + + private final TransactionPoolConfiguration senderLimitedConfig = + ImmutableTransactionPoolConfiguration.builder() + .maxPrioritizedTransactions(MAX_TRANSACTIONS) + .maxFutureBySender(LIMITED_TRANSACTIONS_BY_SENDER) + .pendingTransactionsLayerMaxCapacityBytes(MAX_CAPACITY_BYTES) + .build(); + private LayeredPendingTransactions senderLimitedTransactions; + private LayeredPendingTransactions pendingTransactions; + private CreatedLayers senderLimitedLayers; + private CreatedLayers layers; + private TransactionPoolMetrics txPoolMetrics; + + private static BlockHeader mockBlockHeader() { + final BlockHeader blockHeader = mock(BlockHeader.class); + when(blockHeader.getBaseFee()).thenReturn(Optional.of(Wei.of(100))); + return blockHeader; + } + + private CreatedLayers createLayers(final TransactionPoolConfiguration poolConfig) { + + final BiFunction transactionReplacementTester = + (t1, t2) -> + new TransactionPoolReplacementHandler(poolConf.getPriceBump()) + .shouldReplace(t1, t2, mockBlockHeader()); + + final EvictCollectorLayer evictCollector = new EvictCollectorLayer(txPoolMetrics); + + final SparseTransactions sparseTransactions = + new SparseTransactions( + poolConfig, evictCollector, txPoolMetrics, transactionReplacementTester); + + final ReadyTransactions readyTransactions = + new ReadyTransactions( + poolConfig, sparseTransactions, txPoolMetrics, transactionReplacementTester); + + final BaseFeePrioritizedTransactions prioritizedTransactions = + new BaseFeePrioritizedTransactions( + poolConfig, + LayeredPendingTransactionsTest::mockBlockHeader, + readyTransactions, + txPoolMetrics, + transactionReplacementTester, + FeeMarket.london(0L)); + return new CreatedLayers( + prioritizedTransactions, readyTransactions, sparseTransactions, evictCollector); + } + + @BeforeEach + public void setup() { + + txPoolMetrics = new TransactionPoolMetrics(metricsSystem); + + layers = createLayers(poolConf); + senderLimitedLayers = createLayers(senderLimitedConfig); + + pendingTransactions = new LayeredPendingTransactions(poolConf, layers.prioritizedTransactions); + + senderLimitedTransactions = + new LayeredPendingTransactions( + senderLimitedConfig, senderLimitedLayers.prioritizedTransactions); + } + + @Test + public void returnExclusivelyLocalTransactionsWhenAppropriate() { + final Transaction localTransaction0 = createTransaction(0, KEYS2); + pendingTransactions.addLocalTransaction(localTransaction0, Optional.empty()); + assertThat(pendingTransactions.size()).isEqualTo(1); + + pendingTransactions.addRemoteTransaction(transaction0, Optional.empty()); + assertThat(pendingTransactions.size()).isEqualTo(2); + + pendingTransactions.addRemoteTransaction(transaction1, Optional.empty()); + assertThat(pendingTransactions.size()).isEqualTo(3); + + final List localTransactions = pendingTransactions.getLocalTransactions(); + assertThat(localTransactions.size()).isEqualTo(1); + } + + @Test + public void addRemoteTransactions() { + pendingTransactions.addRemoteTransaction(transaction0, Optional.empty()); + assertThat(pendingTransactions.size()).isEqualTo(1); + + assertThat(getAddedCount(REMOTE, layers.prioritizedTransactions.name())).isEqualTo(1); + + pendingTransactions.addRemoteTransaction(transaction1, Optional.empty()); + assertThat(pendingTransactions.size()).isEqualTo(2); + + assertThat(getAddedCount(REMOTE, layers.prioritizedTransactions.name())).isEqualTo(2); + } + + @Test + public void getNotPresentTransaction() { + assertThat(pendingTransactions.getTransactionByHash(Hash.EMPTY_TRIE_HASH)).isEmpty(); + } + + @Test + public void getTransactionByHash() { + pendingTransactions.addRemoteTransaction(transaction0, Optional.empty()); + assertTransactionPending(pendingTransactions, transaction0); + } + + @Test + public void evictTransactionsWhenSizeLimitExceeded() { + final List firstTxs = new ArrayList<>(MAX_TRANSACTIONS); + + pendingTransactions.subscribeDroppedTransactions(droppedListener); + + for (int i = 0; i < MAX_TRANSACTIONS; i++) { + final Account sender = mock(Account.class); + when(sender.getNonce()).thenReturn((long) i); + final var tx = + createTransaction( + i, + Wei.of((i + 1) * 100L), + (int) poolConf.getPendingTransactionsLayerMaxCapacityBytes() + 1, + SIGNATURE_ALGORITHM.get().generateKeyPair()); + pendingTransactions.addRemoteTransaction(tx, Optional.of(sender)); + firstTxs.add(tx); + assertTransactionPending(pendingTransactions, tx); + } + + assertThat(pendingTransactions.size()).isEqualTo(MAX_TRANSACTIONS); + + final Transaction lastBigTx = + createTransaction( + 0, + Wei.of(100_000L), + (int) poolConf.getPendingTransactionsLayerMaxCapacityBytes(), + SIGNATURE_ALGORITHM.get().generateKeyPair()); + final Account lastSender = mock(Account.class); + when(lastSender.getNonce()).thenReturn(0L); + pendingTransactions.addRemoteTransaction(lastBigTx, Optional.of(lastSender)); + assertTransactionPending(pendingTransactions, lastBigTx); + + assertTransactionNotPending(pendingTransactions, firstTxs.get(0)); + assertThat(getRemovedCount(REMOTE, DROPPED.label(), layers.evictedCollector.name())) + .isEqualTo(1); + assertThat(layers.evictedCollector.getEvictedTransactions()) + .map(PendingTransaction::getTransaction) + .contains(firstTxs.get(0)); + verify(droppedListener).onTransactionDropped(firstTxs.get(0)); + } + + @Test + public void addTransactionForMultipleSenders() { + final var transactionSenderA = createTransaction(0, KEYS1); + final var transactionSenderB = createTransaction(0, KEYS2); + assertThat(pendingTransactions.addRemoteTransaction(transactionSenderA, Optional.empty())) + .isEqualTo(ADDED); + assertTransactionPending(pendingTransactions, transactionSenderA); + assertThat(pendingTransactions.addRemoteTransaction(transactionSenderB, Optional.empty())) + .isEqualTo(ADDED); + assertTransactionPending(pendingTransactions, transactionSenderB); + } + + @Test + public void dropIfTransactionTooFarInFutureForTheSender() { + final var futureTransaction = + createTransaction(poolConf.getTxPoolMaxFutureTransactionByAccount() + 1); + assertThat(pendingTransactions.addRemoteTransaction(futureTransaction, Optional.empty())) + .isEqualTo(NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER); + assertTransactionNotPending(pendingTransactions, futureTransaction); + } + + @Test + public void dropAlreadyConfirmedTransaction() { + final Account sender = mock(Account.class); + when(sender.getNonce()).thenReturn(5L); + + final Transaction oldTransaction = createTransaction(2); + assertThat(pendingTransactions.addRemoteTransaction(oldTransaction, Optional.of(sender))) + .isEqualTo(ALREADY_KNOWN); + assertThat(pendingTransactions.size()).isEqualTo(0); + assertTransactionNotPending(pendingTransactions, oldTransaction); + } + + @Test + public void notifyListenerWhenRemoteTransactionAdded() { + pendingTransactions.subscribePendingTransactions(listener); + + pendingTransactions.addRemoteTransaction(transaction0, Optional.empty()); + + verify(listener).onTransactionAdded(transaction0); + } + + @Test + public void notifyListenerWhenLocalTransactionAdded() { + pendingTransactions.subscribePendingTransactions(listener); + + pendingTransactions.addLocalTransaction(transaction0, Optional.empty()); + + verify(listener).onTransactionAdded(transaction0); + } + + @Test + public void notNotifyListenerAfterUnsubscribe() { + final long id = pendingTransactions.subscribePendingTransactions(listener); + + pendingTransactions.addRemoteTransaction(transaction0, Optional.empty()); + + verify(listener).onTransactionAdded(transaction0); + + pendingTransactions.unsubscribePendingTransactions(id); + + pendingTransactions.addRemoteTransaction(transaction1, Optional.empty()); + + verifyNoMoreInteractions(listener); + } + + @Test + public void selectTransactionsUntilSelectorRequestsNoMore() { + pendingTransactions.addRemoteTransaction(transaction0, Optional.empty()); + pendingTransactions.addRemoteTransaction(transaction1, Optional.empty()); + + final List parsedTransactions = new ArrayList<>(); + pendingTransactions.selectTransactions( + transaction -> { + parsedTransactions.add(transaction); + return COMPLETE_OPERATION; + }); + + assertThat(parsedTransactions.size()).isEqualTo(1); + assertThat(parsedTransactions.get(0)).isEqualTo(transaction0); + } + + @Test + public void selectTransactionsUntilPendingIsEmpty() { + pendingTransactions.addRemoteTransaction(transaction0, Optional.empty()); + pendingTransactions.addRemoteTransaction(transaction1, Optional.empty()); + + final List parsedTransactions = new ArrayList<>(); + pendingTransactions.selectTransactions( + transaction -> { + parsedTransactions.add(transaction); + return CONTINUE; + }); + + assertThat(parsedTransactions.size()).isEqualTo(2); + assertThat(parsedTransactions.get(0)).isEqualTo(transaction0); + assertThat(parsedTransactions.get(1)).isEqualTo(transaction1); + } + + @Test + public void notSelectReplacedTransaction() { + final Transaction transaction1 = createTransaction(0, KEYS1); + final Transaction transaction1b = createTransactionReplacement(transaction1, KEYS1); + + pendingTransactions.addRemoteTransaction(transaction1, Optional.empty()); + pendingTransactions.addRemoteTransaction(transaction1b, Optional.empty()); + + final List parsedTransactions = new ArrayList<>(); + pendingTransactions.selectTransactions( + transaction -> { + parsedTransactions.add(transaction); + return CONTINUE; + }); + + assertThat(parsedTransactions).containsExactly(transaction1b); + } + + @Test + public void selectTransactionsFromSameSenderInNonceOrder() { + final Transaction transaction0 = createTransaction(0, KEYS1); + final Transaction transaction1 = createTransaction(1, KEYS1); + final Transaction transaction2 = createTransaction(2, KEYS1); + + // add out of order + pendingTransactions.addLocalTransaction(transaction2, Optional.empty()); + pendingTransactions.addLocalTransaction(transaction1, Optional.empty()); + pendingTransactions.addLocalTransaction(transaction0, Optional.empty()); + + final List iterationOrder = new ArrayList<>(3); + pendingTransactions.selectTransactions( + transaction -> { + iterationOrder.add(transaction); + return CONTINUE; + }); + + assertThat(iterationOrder).containsExactly(transaction0, transaction1, transaction2); + } + + @Test + public void notForceNonceOrderWhenSendersDiffer() { + final Account sender2 = mock(Account.class); + when(sender2.getNonce()).thenReturn(1L); + + final Transaction transactionSender1 = createTransaction(0, Wei.of(10), KEYS1); + final Transaction transactionSender2 = createTransaction(1, Wei.of(200), KEYS2); + + pendingTransactions.addLocalTransaction(transactionSender1, Optional.empty()); + pendingTransactions.addLocalTransaction(transactionSender2, Optional.of(sender2)); + + final List iterationOrder = new ArrayList<>(2); + pendingTransactions.selectTransactions( + transaction -> { + iterationOrder.add(transaction); + return CONTINUE; + }); + + assertThat(iterationOrder).containsExactly(transactionSender2, transactionSender1); + } + + @Test + public void invalidTransactionIsDeletedFromPendingTransactions() { + pendingTransactions.addRemoteTransaction(transaction0, Optional.empty()); + pendingTransactions.addRemoteTransaction(transaction1, Optional.empty()); + + final List parsedTransactions = new ArrayList<>(2); + pendingTransactions.selectTransactions( + transaction -> { + parsedTransactions.add(transaction); + return DELETE_TRANSACTION_AND_CONTINUE; + }); + + assertThat(parsedTransactions.size()).isEqualTo(2); + assertThat(parsedTransactions.get(0)).isEqualTo(transaction0); + assertThat(parsedTransactions.get(1)).isEqualTo(transaction1); + + assertThat(pendingTransactions.size()).isZero(); + } + + @Test + public void returnEmptyOptionalAsMaximumNonceWhenNoTransactionsPresent() { + assertThat(pendingTransactions.getNextNonceForSender(SENDER1)).isEmpty(); + } + + @Test + public void replaceTransactionWithSameSenderAndNonce() { + final Transaction transaction1 = createTransaction(0, Wei.of(20), KEYS1); + final Transaction transaction1b = createTransactionReplacement(transaction1, KEYS1); + final Transaction transaction2 = createTransaction(1, Wei.of(10), KEYS1); + assertThat(pendingTransactions.addRemoteTransaction(transaction1, Optional.empty())) + .isEqualTo(ADDED); + assertThat(pendingTransactions.addRemoteTransaction(transaction2, Optional.empty())) + .isEqualTo(ADDED); + assertThat( + pendingTransactions + .addRemoteTransaction(transaction1b, Optional.empty()) + .isReplacement()) + .isTrue(); + + assertTransactionNotPending(pendingTransactions, transaction1); + assertTransactionPending(pendingTransactions, transaction1b); + assertTransactionPending(pendingTransactions, transaction2); + assertThat(pendingTransactions.size()).isEqualTo(2); + assertThat(getAddedCount(REMOTE, layers.prioritizedTransactions.name())).isEqualTo(3); + assertThat(getRemovedCount(REMOTE, REPLACED.label(), layers.prioritizedTransactions.name())) + .isEqualTo(1); + } + + @Test + public void replaceTransactionWithSameSenderAndNonce_multipleReplacements() { + final int replacedTxCount = 5; + final List replacedTransactions = new ArrayList<>(replacedTxCount); + Transaction duplicateTx = createTransaction(0, Wei.of(50), KEYS1); + for (int i = 0; i < replacedTxCount; i++) { + replacedTransactions.add(duplicateTx); + pendingTransactions.addRemoteTransaction(duplicateTx, Optional.empty()); + duplicateTx = createTransactionReplacement(duplicateTx, KEYS1); + } + + final Transaction independentTx = createTransaction(1, Wei.ONE, KEYS1); + assertThat(pendingTransactions.addRemoteTransaction(independentTx, Optional.empty())) + .isEqualTo(ADDED); + assertThat( + pendingTransactions.addRemoteTransaction(duplicateTx, Optional.empty()).isReplacement()) + .isTrue(); + + // All txs except the last duplicate should be removed + replacedTransactions.forEach(tx -> assertTransactionNotPending(pendingTransactions, tx)); + assertTransactionPending(pendingTransactions, duplicateTx); + // Tx with distinct nonce should be maintained + assertTransactionPending(pendingTransactions, independentTx); + + assertThat(pendingTransactions.size()).isEqualTo(2); + assertThat(getAddedCount(REMOTE, layers.prioritizedTransactions.name())) + .isEqualTo(replacedTxCount + 2); + assertThat(getRemovedCount(REMOTE, REPLACED.label(), layers.prioritizedTransactions.name())) + .isEqualTo(replacedTxCount); + } + + @Test + public void + replaceTransactionWithSameSenderAndNonce_multipleReplacementsAddedLocallyAndRemotely() { + final int replacedTxCount = 5; + final List replacedTransactions = new ArrayList<>(replacedTxCount); + int remoteDuplicateCount = 0; + Transaction replacingTx = createTransaction(0, KEYS1); + for (int i = 0; i < replacedTxCount; i++) { + replacedTransactions.add(replacingTx); + if (i % 2 == 0) { + pendingTransactions.addRemoteTransaction(replacingTx, Optional.empty()); + remoteDuplicateCount++; + } else { + pendingTransactions.addLocalTransaction(replacingTx, Optional.empty()); + } + replacingTx = createTransactionReplacement(replacingTx, KEYS1); + } + + final Transaction independentTx = createTransaction(1); + assertThat( + pendingTransactions.addLocalTransaction(replacingTx, Optional.empty()).isReplacement()) + .isTrue(); + assertThat(pendingTransactions.addRemoteTransaction(independentTx, Optional.empty())) + .isEqualTo(ADDED); + + // All txs except the last duplicate should be removed + replacedTransactions.forEach(tx -> assertTransactionNotPending(pendingTransactions, tx)); + assertTransactionPending(pendingTransactions, replacingTx); + + // Tx with distinct nonce should be maintained + assertTransactionPending(pendingTransactions, independentTx); + + final int localDuplicateCount = replacedTxCount - remoteDuplicateCount; + assertThat(pendingTransactions.size()).isEqualTo(2); + assertThat(getAddedCount(REMOTE, layers.prioritizedTransactions.name())) + .isEqualTo(remoteDuplicateCount + 1); + assertThat(getAddedCount(LOCAL, layers.prioritizedTransactions.name())) + .isEqualTo(localDuplicateCount + 1); + assertThat(getRemovedCount(REMOTE, REPLACED.label(), layers.prioritizedTransactions.name())) + .isEqualTo(remoteDuplicateCount); + assertThat(getRemovedCount(LOCAL, REPLACED.label(), layers.prioritizedTransactions.name())) + .isEqualTo(localDuplicateCount); + } + + @Test + public void notReplaceTransactionWithSameSenderAndNonceWhenGasPriceIsLower() { + final Transaction transaction1 = createTransaction(0, Wei.of(2)); + final Transaction transaction1b = createTransaction(0, Wei.ONE); + assertThat(pendingTransactions.addRemoteTransaction(transaction1, Optional.empty())) + .isEqualTo(ADDED); + + pendingTransactions.subscribePendingTransactions(listener); + assertThat(pendingTransactions.addRemoteTransaction(transaction1b, Optional.empty())) + .isEqualTo(REJECTED_UNDERPRICED_REPLACEMENT); + + assertTransactionNotPending(pendingTransactions, transaction1b); + assertTransactionPending(pendingTransactions, transaction1); + assertThat(pendingTransactions.size()).isEqualTo(1); + verifyNoInteractions(listener); + } + + @Test + public void trackNextNonceForEachSender() { + // first sender consecutive txs: 0->1->2 + final Account firstSender = mock(Account.class); + when(firstSender.getNonce()).thenReturn(0L); + when(firstSender.getAddress()).thenReturn(SENDER1); + assertNoNextNonceForSender(pendingTransactions, SENDER1); + pendingTransactions.addRemoteTransaction(createTransaction(0, KEYS1), Optional.of(firstSender)); + assertNextNonceForSender(pendingTransactions, SENDER1, 1); + + pendingTransactions.addRemoteTransaction(createTransaction(1, KEYS1), Optional.of(firstSender)); + assertNextNonceForSender(pendingTransactions, SENDER1, 2); + + pendingTransactions.addRemoteTransaction(createTransaction(2, KEYS1), Optional.of(firstSender)); + assertNextNonceForSender(pendingTransactions, SENDER1, 3); + + // second sender not in orders: 3->0->2->1 + final Account secondSender = mock(Account.class); + when(secondSender.getNonce()).thenReturn(0L); + when(secondSender.getAddress()).thenReturn(SENDER2); + assertNoNextNonceForSender(pendingTransactions, SENDER2); + pendingTransactions.addRemoteTransaction( + createTransaction(3, KEYS2), Optional.of(secondSender)); + assertNoNextNonceForSender(pendingTransactions, SENDER2); + + pendingTransactions.addRemoteTransaction( + createTransaction(0, KEYS2), Optional.of(secondSender)); + assertNextNonceForSender(pendingTransactions, SENDER2, 1); + + pendingTransactions.addRemoteTransaction( + createTransaction(2, KEYS2), Optional.of(secondSender)); + assertNextNonceForSender(pendingTransactions, SENDER2, 1); + + // tx 1 will fill the nonce gap and all txs will be ready + pendingTransactions.addRemoteTransaction( + createTransaction(1, KEYS2), Optional.of(secondSender)); + assertNextNonceForSender(pendingTransactions, SENDER2, 4); + } + + @Test + public void correctNonceIsReturned() { + final Account sender = mock(Account.class); + when(sender.getNonce()).thenReturn(1L); + assertThat(pendingTransactions.getNextNonceForSender(transaction2.getSender())).isEmpty(); + // since tx 3 is missing, 4 is sparse, + // note that 0 is already known since sender nonce is 1 + addLocalTransactions(pendingTransactions, sender, 0, 1, 2, 4); + assertThat(pendingTransactions.size()).isEqualTo(3); + assertThat(pendingTransactions.getNextNonceForSender(transaction2.getSender())) + .isPresent() + .hasValue(3); + + // tx 3 arrives and is added, while 4 is moved to ready + addLocalTransactions(pendingTransactions, sender, 3); + assertThat(pendingTransactions.size()).isEqualTo(4); + assertThat(pendingTransactions.getNextNonceForSender(transaction2.getSender())) + .isPresent() + .hasValue(5); + + // when 5 is added, the pool is full, and so 6 and 7 are dropped since too far in future + addLocalTransactions(pendingTransactions, sender, 5, 6, 7); + assertThat(pendingTransactions.size()).isEqualTo(5); + + // assert that transactions are pruned by account from the latest future nonce first + assertThat(pendingTransactions.getNextNonceForSender(transaction2.getSender())) + .isPresent() + .hasValue(6); + } + + @Test + public void correctNonceIsReturnedForSenderLimitedPool() { + final Account sender = mock(Account.class); + when(sender.getNonce()).thenReturn(1L); + + assertThat(senderLimitedTransactions.getNextNonceForSender(transaction2.getSender())).isEmpty(); + // since tx 3 is missing, 4 is sparse, + // note that 0 is already known since sender nonce is 1 + addLocalTransactions(senderLimitedTransactions, sender, 0, 1, 2, 4); + assertThat(senderLimitedTransactions.size()).isEqualTo(3); + assertThat(senderLimitedTransactions.getNextNonceForSender(transaction2.getSender())) + .isPresent() + .hasValue(3); + + // tx 3 arrives and is added, while 4 is moved to ready + addLocalTransactions(senderLimitedTransactions, sender, 3); + assertThat(senderLimitedTransactions.size()).isEqualTo(4); + assertThat(senderLimitedTransactions.getNextNonceForSender(transaction2.getSender())) + .isPresent() + .hasValue(5); + + // for sender max 4 txs are allowed, so 5, 6 and 7 are dropped since too far in future + addLocalTransactions(senderLimitedTransactions, sender, 5, 6, 7); + assertThat(senderLimitedTransactions.size()).isEqualTo(4); + + // assert that we drop txs with future nonce first + assertThat(senderLimitedTransactions.getNextNonceForSender(transaction2.getSender())) + .isPresent() + .hasValue(5); + } + + @Test + public void correctNonceIsReturnedWithRepeatedTransactions() { + assertThat(pendingTransactions.getNextNonceForSender(transaction2.getSender())).isEmpty(); + final Account sender = mock(Account.class); + addLocalTransactions(pendingTransactions, sender, 0, 1, 2, 1, 0, 4); + assertThat(pendingTransactions.getNextNonceForSender(transaction2.getSender())) + .isPresent() + .hasValue(3); + addLocalTransactions(pendingTransactions, sender, 3); + } + + @Test + public void shouldNotIncrementAddedCounterWhenRemoteTransactionAlreadyPresent() { + pendingTransactions.addLocalTransaction(transaction0, Optional.empty()); + assertThat(pendingTransactions.size()).isEqualTo(1); + assertThat(getAddedCount(LOCAL, layers.prioritizedTransactions.name())).isEqualTo(1); + assertThat(getAddedCount(REMOTE, layers.prioritizedTransactions.name())).isZero(); + + assertThat(pendingTransactions.addRemoteTransaction(transaction0, Optional.empty())) + .isEqualTo(ALREADY_KNOWN); + assertThat(pendingTransactions.size()).isEqualTo(1); + assertThat(getAddedCount(LOCAL, layers.prioritizedTransactions.name())).isEqualTo(1); + assertThat(getAddedCount(REMOTE, layers.prioritizedTransactions.name())).isZero(); + } + + @Test + public void shouldNotIncrementAddedCounterWhenLocalTransactionAlreadyPresent() { + pendingTransactions.addRemoteTransaction(transaction0, Optional.empty()); + assertThat(pendingTransactions.size()).isEqualTo(1); + assertThat(getAddedCount(LOCAL, layers.prioritizedTransactions.name())).isZero(); + assertThat(getAddedCount(REMOTE, layers.prioritizedTransactions.name())).isEqualTo(1); + + assertThat(pendingTransactions.addLocalTransaction(transaction0, Optional.empty())) + .isEqualTo(ALREADY_KNOWN); + assertThat(pendingTransactions.size()).isEqualTo(1); + assertThat(getAddedCount(LOCAL, layers.prioritizedTransactions.name())).isZero(); + assertThat(getAddedCount(REMOTE, layers.prioritizedTransactions.name())).isEqualTo(1); + } + + @Test + public void doNothingIfTransactionAlreadyPending() { + final var addedTxs = populateCache(1, 0); + assertThat( + pendingTransactions.addRemoteTransaction( + addedTxs[0].transaction, Optional.of(addedTxs[0].account))) + .isEqualTo(ALREADY_KNOWN); + assertTransactionPending(pendingTransactions, addedTxs[0].transaction); + } + + @Test + public void returnsCorrectNextNonceWhenAddedTransactionsHaveGaps() { + final var addedTxs = populateCache(3, 0, 1); + assertThat(pendingTransactions.getNextNonceForSender(addedTxs[0].transaction.getSender())) + .isPresent() + .hasValue(1); + } + + private TransactionAndAccount[] populateCache(final int numTxs, final long startingNonce) { + return populateCache(numTxs, KEYS1, startingNonce, OptionalLong.empty()); + } + + private TransactionAndAccount[] populateCache( + final int numTxs, final long startingNonce, final long missingNonce) { + return populateCache(numTxs, KEYS1, startingNonce, OptionalLong.of(missingNonce)); + } + + private TransactionAndAccount[] populateCache( + final int numTxs, + final KeyPair keys, + final long startingNonce, + final OptionalLong maybeGapNonce) { + final List addedTransactions = new ArrayList<>(numTxs); + for (int i = 0; i < numTxs; i++) { + final long nonce = startingNonce + i; + if (maybeGapNonce.isEmpty() || maybeGapNonce.getAsLong() != nonce) { + final var transaction = createTransaction(nonce, keys); + final Account sender = mock(Account.class); + when(sender.getNonce()).thenReturn(startingNonce); + final var res = pendingTransactions.addRemoteTransaction(transaction, Optional.of(sender)); + assertTransactionPending(pendingTransactions, transaction); + assertThat(res).isEqualTo(ADDED); + addedTransactions.add(new TransactionAndAccount(transaction, sender)); + } + } + return addedTransactions.toArray(TransactionAndAccount[]::new); + } + + record TransactionAndAccount(Transaction transaction, Account account) {} + + record CreatedLayers( + AbstractPrioritizedTransactions prioritizedTransactions, + ReadyTransactions readyTransactions, + SparseTransactions sparseTransactions, + EvictCollectorLayer evictedCollector) {} +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java new file mode 100644 index 0000000000..1d3843c839 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java @@ -0,0 +1,1331 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.Sender.S1; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.Sender.S2; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.Sender.S3; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.Sender.S4; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.INVALIDATED; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Util; +import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolReplacementHandler; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.evm.account.Account; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.stream.Stream; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class LayersTest extends BaseTransactionPoolTest { + private static final int MAX_PRIO_TRANSACTIONS = 3; + private static final int MAX_FUTURE_FOR_SENDER = 10; + + private final TransactionPoolConfiguration poolConfig = + ImmutableTransactionPoolConfiguration.builder() + .maxPrioritizedTransactions(MAX_PRIO_TRANSACTIONS) + .maxFutureBySender(MAX_FUTURE_FOR_SENDER) + .pendingTransactionsLayerMaxCapacityBytes( + new PendingTransaction.Remote(createEIP1559Transaction(0, KEYS1, 1)).memorySize() * 3) + .build(); + + private final TransactionPoolMetrics txPoolMetrics = new TransactionPoolMetrics(metricsSystem); + + private final EvictCollectorLayer evictCollector = new EvictCollectorLayer(txPoolMetrics); + private final SparseTransactions sparseTransactions = + new SparseTransactions( + poolConfig, evictCollector, txPoolMetrics, this::transactionReplacementTester); + + private final ReadyTransactions readyTransactions = + new ReadyTransactions( + poolConfig, sparseTransactions, txPoolMetrics, this::transactionReplacementTester); + + private final BaseFeePrioritizedTransactions prioritizedTransactions = + new BaseFeePrioritizedTransactions( + poolConfig, + LayersTest::mockBlockHeader, + readyTransactions, + txPoolMetrics, + this::transactionReplacementTester, + FeeMarket.london(0L)); + + private final LayeredPendingTransactions pendingTransactions = + new LayeredPendingTransactions(poolConfig, prioritizedTransactions); + + @AfterEach + void reset() { + pendingTransactions.reset(); + } + + @ParameterizedTest + @MethodSource("providerAddTransactions") + void addTransactions(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerAddTransactionsMultipleSenders") + void addTransactionsMultipleSenders(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerRemoveTransactions") + void removeTransactions(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerInterleavedAddRemoveTransactions") + void interleavedAddRemoveTransactions(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerBlockAdded") + void removeConfirmedTransactions(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerNextNonceForSender") + void nextNonceForSender(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerSelectTransactions") + void selectTransactions(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerReorg") + void reorg(final Scenario scenario) { + assertScenario(scenario); + } + + private void assertScenario(final Scenario scenario) { + scenario.execute( + pendingTransactions, + prioritizedTransactions, + readyTransactions, + sparseTransactions, + evictCollector); + } + + static Stream providerAddTransactions() { + return Stream.of( + Arguments.of( + new Scenario("add first").addForSender(S1, 0).expectedPrioritizedForSender(S1, 0)), + Arguments.of( + new Scenario("add first sparse").addForSender(S1, 1).expectedSparseForSender(S1, 1)), + Arguments.of( + new Scenario("fill prioritized") + .addForSender(S1, 0, 1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill prioritized reverse") + .addForSender(S1, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill prioritized mixed order 1 step by step") + .addForSender(S1, 2) + .expectedSparseForSender(S1, 2) + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .addForSender(S1, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSenders()), + Arguments.of( + new Scenario("fill prioritized mixed order 2") + .addForSender(S1, 0, 2, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("overflow to ready") + .addForSender(S1, 0, 1, 2, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to ready reverse") + .addForSender(S1, 3, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to ready mixed order 1") + .addForSender(S1, 3, 0, 2, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to ready mixed order 2") + .addForSender(S1, 0, 3, 1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to sparse") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse reverse") + .addForSender(S1, 6, 5, 4, 3, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + // 4,5,6 are evicted since max capacity of sparse layer is 3 txs + .expectedReadyForSender(S1, 3) + .expectedDroppedForSender(S1, 4, 5, 6)), + Arguments.of( + new Scenario("overflow to sparse mixed order 1") + .addForSender(S1, 6, 0, 4, 1, 3, 2, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse mixed order 2") + .addForSender(S1, 0, 4, 6, 1, 5, 2, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse mixed order 3") + .addForSender(S1, 0, 1, 2, 3, 5, 6, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6)), + Arguments.of( + new Scenario("nonce gap to sparse 1") + .addForSender(S1, 0, 2) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2)), + Arguments.of( + new Scenario("nonce gap to sparse 2") + .addForSender(S1, 0, 1, 2, 3, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedSparseForSender(S1, 5)), + Arguments.of( + new Scenario("fill sparse 1") + .addForSender(S1, 2, 3, 5) + .expectedSparseForSender(S1, 2, 3, 5)), + Arguments.of( + new Scenario("fill sparse 2") + .addForSender(S1, 5, 3, 2) + .expectedSparseForSender(S1, 5, 3, 2)), + Arguments.of( + new Scenario("overflow sparse 1") + .addForSender(S1, 1, 2, 3, 4) + .expectedSparseForSender(S1, 1, 2, 3) + .expectedDroppedForSender(S1, 4)), + Arguments.of( + new Scenario("overflow sparse 2") + .addForSender(S1, 4, 2, 3, 1) + .expectedSparseForSender(S1, 2, 3, 1) + .expectedDroppedForSender(S1, 4)), + Arguments.of( + new Scenario("overflow sparse 3") + .addForSender(S1, 0, 4, 2, 3, 5) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 4, 2, 3) + .expectedDroppedForSender(S1, 5))); + } + + static Stream providerAddTransactionsMultipleSenders() { + return Stream.of( + Arguments.of( + new Scenario("add first") + .addForSenders(S1, 0, S2, 0) + .expectedPrioritizedForSenders(S2, 0, S1, 0)), + Arguments.of( + new Scenario("add first sparse") + .addForSenders(S1, 1, S2, 2) + .expectedSparseForSenders(S1, 1, S2, 2)), + Arguments.of( + new Scenario("fill prioritized 1") + .addForSender(S1, 0, 1, 2) + .addForSender(S2, 0, 1, 2) + .expectedPrioritizedForSender(S2, 0, 1, 2) + .expectedReadyForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill prioritized 2") + .addForSender(S2, 0, 1, 2) + .addForSender(S1, 0, 1, 2) + .expectedPrioritizedForSender(S2, 0, 1, 2) + .expectedReadyForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill prioritized 3") + .addForSenders(S1, 0, S2, 0, S1, 1, S2, 1, S1, 2, S2, 2) + .expectedPrioritizedForSender(S2, 0, 1, 2) + .expectedReadyForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill prioritized mixed order") + .addForSenders(S1, 2, S2, 1) + .expectedPrioritizedForSenders() + .expectedReadyForSenders() + .expectedSparseForSenders(S1, 2, S2, 1) + .addForSenders(S2, 2, S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedReadyForSenders() + .expectedSparseForSenders(S1, 2, S2, 1, S2, 2) + .addForSenders(S1, 1) + .expectedPrioritizedForSenders(S1, 0, S1, 1, S1, 2) + .expectedReadyForSenders() + .expectedSparseForSenders(S2, 1, S2, 2) + .addForSenders(S2, 0) + // only S2[0] is prioritized because there is no space to try to fill gaps + .expectedPrioritizedForSenders(S2, 0, S1, 0, S1, 1) + .expectedReadyForSender(S1, 2) + .expectedSparseForSenders(S2, 1, S2, 2) + // confirm some S1 txs will promote S2 txs + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 1) + .expectedReadyForSenders(S2, 2, S1, 2) + .expectedSparseForSenders() + .confirmedForSenders(S1, 1) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S2, 2) + .expectedReadyForSender(S1, 2) + .expectedSparseForSenders() + .confirmedForSenders(S2, 1) + .expectedPrioritizedForSenders(S2, 2, S1, 2) + .expectedReadyForSenders() + .expectedSparseForSenders()), + Arguments.of( + new Scenario("overflow to ready 1") + .addForSenders(S1, 0, S1, 1, S2, 0, S2, 1) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to ready 2") + .addForSenders(S2, 0, S2, 1, S1, 0, S1, 1) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to ready 3") + .addForSenders(S1, 0, S2, 0, S2, 1, S1, 1) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to ready reverse 1") + .addForSenders(S1, 1, S1, 0, S2, 1, S2, 0) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to ready reverse 2") + .addForSenders(S1, 1, S2, 1, S2, 0, S1, 0) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to ready mixed") + .addForSenders(S2, 1, S1, 0, S1, 1, S2, 0) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to sparse") + .addForSenders(S1, 0, S1, 1, S2, 0, S2, 1, S3, 0, S3, 1, S3, 2) + .expectedPrioritizedForSender(S3, 0, 1, 2) + .expectedReadyForSenders(S2, 0, S2, 1, S1, 0) + .expectedSparseForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to sparse reverse") + .addForSenders(S3, 2, S3, 1, S3, 0, S2, 1, S2, 0, S1, 1, S1, 0) + .expectedPrioritizedForSender(S3, 0, 1, 2) + .expectedReadyForSenders(S2, 0, S2, 1, S1, 0) + .expectedSparseForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to sparse mixed") + .addForSenders(S2, 0, S3, 2, S1, 1) + .expectedPrioritizedForSender(S2, 0) + .expectedReadyForSenders() + .expectedSparseForSenders(S3, 2, S1, 1) + .addForSenders(S2, 1) + .expectedPrioritizedForSenders(S2, 0, S2, 1) + .expectedReadyForSenders() + .expectedSparseForSenders(S3, 2, S1, 1) + .addForSenders(S3, 0) + .expectedPrioritizedForSenders(S3, 0, S2, 0, S2, 1) + .expectedReadyForSenders() + .expectedSparseForSenders(S3, 2, S1, 1) + .addForSenders(S1, 0) + .expectedPrioritizedForSenders(S3, 0, S2, 0, S2, 1) + .expectedReadyForSenders(S1, 0, S1, 1) + .expectedSparseForSender(S3, 2) + .addForSenders(S3, 1) + // ToDo: only S3[1] is prioritized because there is no space to try to fill gaps + .expectedPrioritizedForSenders(S3, 0, S3, 1, S2, 0) + .expectedReadyForSenders(S2, 1, S1, 0, S1, 1) + .expectedSparseForSender(S3, 2) + .addForSenders(S4, 0, S4, 1, S3, 3) + .expectedPrioritizedForSenders(S4, 0, S4, 1, S3, 0) + .expectedReadyForSenders(S3, 1, S2, 0, S2, 1) + .expectedSparseForSenders(S3, 2, S1, 1, S1, 0) + // ToDo: non optimal discard, worth to improve? + .expectedDroppedForSender(S3, 3)), + Arguments.of( + new Scenario("replacement cross layer") + .addForSenders(S2, 0, S3, 2, S1, 1, S2, 1, S3, 0, S1, 0, S3, 1) + // ToDo: only S3[1] is prioritized because there is no space to try to fill gaps + .expectedPrioritizedForSenders(S3, 0, S3, 1, S2, 0) + .expectedReadyForSenders(S2, 1, S1, 0, S1, 1) + .expectedSparseForSender(S3, 2) + .addForSenders(S3, 2) // added in prioritized, but replacement in sparse + .expectedPrioritizedForSenders(S3, 0, S3, 1, S3, 2) + .expectedReadyForSenders(S2, 0, S2, 1, S1, 0) + .expectedSparseForSender(S1, 1))); + } + + static Stream providerRemoveTransactions() { + return Stream.of( + Arguments.of(new Scenario("remove not existing").removeForSender(S1, 0)), + Arguments.of(new Scenario("add/remove first").addForSender(S1, 0).removeForSender(S1, 0)), + Arguments.of( + new Scenario("add/remove first sparse").addForSender(S1, 1).removeForSender(S1, 1)), + Arguments.of( + new Scenario("fill/remove prioritized 1") + .addForSender(S1, 0, 1, 2) + .removeForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill/remove prioritized 2") + .addForSender(S1, 0, 1, 2) + .removeForSender(S1, 2, 1, 0)), + Arguments.of( + new Scenario("fill/remove prioritized reverse 1") + .addForSender(S1, 2, 1, 0) + .removeForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill/remove prioritized reverse 2") + .addForSender(S1, 2, 1, 0) + .removeForSender(S1, 2, 1, 0)), + Arguments.of( + new Scenario("fill/remove first prioritized") + .addForSender(S1, 0, 1, 2) + .removeForSender(S1, 0) + .expectedSparseForSender(S1, 1, 2)), + Arguments.of( + new Scenario("fill/remove last prioritized") + .addForSender(S1, 0, 1, 2) + .removeForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0, 1)), + Arguments.of( + new Scenario("fill/remove middle prioritized") + .addForSender(S1, 0, 1, 2) + .removeForSender(S1, 1) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2)), + Arguments.of( + new Scenario("overflow to ready then remove 1") + .addForSender(S1, 0, 1, 2, 3) + .removeForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0, 1) + .expectedSparseForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to ready then remove 2") + .addForSender(S1, 0, 1, 2, 3, 4) + .removeForSender(S1, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to ready then remove 3") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .removeForSender(S1, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedSparseForSender(S1, 5)), + Arguments.of( + new Scenario("overflow to ready then remove 4") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .removeForSender(S1, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSender(S1, 4, 5)), + Arguments.of( + new Scenario("overflow to ready then remove 5") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .removeForSender(S1, 0) + .expectedSparseForSender(S1, 1, 2, 3) + .expectedDroppedForSender(S1, 4, 5)), + Arguments.of( + new Scenario("overflow to sparse then remove 1") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .removeForSender(S1, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5)), + Arguments.of( + new Scenario("overflow to sparse then remove 2") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .removeForSender(S1, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedSparseForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse then remove 3") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .removeForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0, 1) + .expectedSparseForSender(S1, 3, 4, 5) + .expectedDroppedForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse then remove 4") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .removeForSender(S1, 7) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 8)), + Arguments.of( + new Scenario("overflow to sparse then remove 5") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .removeForSender(S1, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 7, 8)), + Arguments.of( + new Scenario("overflow to sparse then remove 6") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .removeForSender(S1, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 7, 8)), + Arguments.of( + new Scenario("overflow to sparse then remove 7") + .setAccountNonce(S1, 1) + .addForSender(S1, 1, 2, 3, 4, 5, 8) + .removeForSender(S1, 2) + .expectedPrioritizedForSender(S1, 1) + .expectedSparseForSender(S1, 3, 4, 5) + .expectedDroppedForSender(S1, 8))); + } + + static Stream providerInterleavedAddRemoveTransactions() { + return Stream.of( + Arguments.of( + new Scenario("interleaved add/remove 1") + .addForSender(S1, 0) + .removeForSender(S1, 0) + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0)), + Arguments.of( + new Scenario("interleaved add/remove 2") + .addForSender(S1, 0) + .removeForSender(S1, 0) + .addForSender(S1, 1) + .expectedSparseForSender(S1, 1)), + Arguments.of( + new Scenario("interleaved add/remove 3") + .addForSender(S1, 0) + .removeForSender(S1, 0) + .addForSender(S1, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1))); + } + + static Stream providerBlockAdded() { + return Stream.of( + Arguments.of(new Scenario("confirmed not exist").confirmedForSenders(S1, 0)), + Arguments.of( + new Scenario("confirmed below existing lower nonce") + .addForSender(S1, 1) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1)), + Arguments.of( + new Scenario("confirmed only one existing") + .addForSender(S1, 0) + .confirmedForSenders(S1, 0)), + Arguments.of( + new Scenario("confirmed above existing") + .addForSender(S1, 0) + .confirmedForSenders(S1, 1)), + Arguments.of( + new Scenario("confirmed some existing 1") + .addForSender(S1, 0, 1) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1)), + Arguments.of( + new Scenario("confirmed some existing 2") + .addForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 1) + .expectedPrioritizedForSender(S1, 2)), + Arguments.of( + new Scenario("overflow to ready and confirmed some existing 1") + .addForSender(S1, 0, 1, 2, 3) + .confirmedForSenders(S1, 3)), + Arguments.of( + new Scenario("overflow to ready and confirmed some existing 2") + .addForSender(S1, 0, 1, 2, 3) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2, 3)), + Arguments.of( + new Scenario("overflow to ready and confirmed some existing 3") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .confirmedForSenders(S1, 1) + .expectedPrioritizedForSender(S1, 2, 3, 4) + .expectedReadyForSender(S1, 5)), + Arguments.of( + new Scenario("overflow to ready and confirmed all existing") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .confirmedForSenders(S1, 5)), + Arguments.of( + new Scenario("overflow to ready and confirmed above highest nonce") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .confirmedForSenders(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse and confirmed some existing 1") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2, 3) + .expectedReadyForSender(S1, 4, 5, 6)), + Arguments.of( + new Scenario("overflow to sparse and confirmed some existing 2") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .confirmedForSenders(S1, 2) + .expectedPrioritizedForSender(S1, 3, 4, 5) + .expectedReadyForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse and confirmed some existing 3") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .confirmedForSenders(S1, 3) + .expectedPrioritizedForSender(S1, 4, 5, 6)), + Arguments.of( + new Scenario("overflow to sparse and confirmed some existing 4") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2, 3) + .expectedReadyForSender(S1, 4, 5, 6) + .expectedSparseForSender(S1, 7, 8)), + Arguments.of( + new Scenario("overflow to sparse and confirmed some existing with gap") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 7, 8) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2, 3) + .expectedReadyForSender(S1, 4, 5) + .expectedSparseForSender(S1, 7, 8)), + Arguments.of( + new Scenario("overflow to sparse and confirmed all w/o gap") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .confirmedForSenders(S1, 8)), + Arguments.of( + new Scenario("overflow to sparse and confirmed all with gap") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 7, 8) + .confirmedForSenders(S1, 8)), + Arguments.of( + new Scenario("overflow to sparse and confirmed all before gap 1") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 7, 8) + .confirmedForSenders(S1, 5) + .expectedSparseForSender(S1, 7, 8)), + Arguments.of( + new Scenario("overflow to sparse and confirmed all before gap 2") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 7, 8) + .confirmedForSenders(S1, 6) + .expectedPrioritizedForSender(S1, 7, 8))); + } + + static Stream providerNextNonceForSender() { + return Stream.of( + Arguments.of(new Scenario("not exist").expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("first transaction") + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedNextNonceForSenders(S1, 1)), + Arguments.of( + new Scenario("fill prioritized") + .addForSender(S1, 0, 1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("reverse fill prioritized") + .addForSender(S1, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("reverse fill prioritized 2") + .addForSender(S1, 3, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedNextNonceForSenders(S1, 4)), + Arguments.of( + new Scenario("overflow to ready") + .addForSender(S1, 0, 1, 2, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedNextNonceForSenders(S1, 4)), + Arguments.of( + new Scenario("fill ready") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedNextNonceForSenders(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6) + .expectedNextNonceForSenders(S1, 7)), + Arguments.of( + new Scenario("fill sparse") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedNextNonceForSenders(S1, 9)), + Arguments.of( + new Scenario("drop transaction") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedDroppedForSender(S1, 9) + .expectedNextNonceForSenders(S1, 9)), + Arguments.of( + new Scenario("first with gap") + .addForSender(S1, 1) + .expectedSparseForSender(S1, 1) + .expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("sequence with gap 1") + .addForSender(S1, 0, 2) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .expectedNextNonceForSenders(S1, 1)), + Arguments.of( + new Scenario("sequence with gap 2") + .addForSender(S1, 0, 1, 2, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSender(S1, 4) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("sequence with gap 3") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 7) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 7) + .expectedNextNonceForSenders(S1, 6)), + Arguments.of( + new Scenario("sequence with gap 4") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 8) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 8) + .expectedNextNonceForSenders(S1, 7)), + Arguments.of( + new Scenario("out of order sequence 1") + .addForSender(S1, 2, 0, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("out of order sequence 2") + .addForSender(S1, 2, 0, 4, 3, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedNextNonceForSenders(S1, 5)), + Arguments.of( + new Scenario("out of order sequence with gap 1") + .addForSender(S1, 2, 1) + .expectedSparseForSender(S1, 2, 1) + .expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("out of order sequence with gap 2") + .addForSender(S1, 2, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .expectedNextNonceForSenders(S1, 1)), + Arguments.of( + new Scenario("out of order sequence with gap 3") + .addForSender(S1, 2, 0, 1, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSender(S1, 4) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("out of order sequence with gap 4") + .addForSender(S1, 2, 0, 4, 1, 6, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedSparseForSender(S1, 6) + .expectedNextNonceForSenders(S1, 5)), + Arguments.of( + new Scenario("no gap and confirmed 1") + .addForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("all confirmed 1") + .addForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 2) + .expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("all confirmed 2") + .addForSender(S1, 3, 0, 4, 1, 5, 2) + .confirmedForSenders(S1, 8) + .expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("all confirmed 3") + .addForSender(S1, 3, 0, 4, 1, 2, 5, 6, 7) + .confirmedForSenders(S1, 8) + .expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("all confirmed step by step 1") + .addForSender(S1, 3) + .expectedSparseForSender(S1, 3) + .expectedNextNonceForSenders(S1, null) + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 3) + .expectedNextNonceForSenders(S1, 1) + .addForSender(S1, 4) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 3, 4) + .expectedNextNonceForSenders(S1, 1) + .addForSender(S1, 1) + .expectedPrioritizedForSender(S1, 0, 1) + .expectedSparseForSender(S1, 3, 4) + .expectedNextNonceForSenders(S1, 2) + .addForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedNextNonceForSenders(S1, 5) + .addForSender(S1, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedNextNonceForSenders(S1, 6) + .addForSender(S1, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6) + .expectedNextNonceForSenders(S1, 7) + .addForSender(S1, 7) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7) + .expectedNextNonceForSenders(S1, 8) + .addForSender(S1, 8) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedNextNonceForSenders(S1, 9) + .addForSender(S1, 9) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedDroppedForSender(S1, 9) + .expectedNextNonceForSenders(S1, 9) + .confirmedForSenders(S1, 9) + .expectedPrioritizedForSenders() + .expectedReadyForSenders() + .expectedSparseForSenders() + .expectedNextNonceForSenders(S1, null))); + } + + static Stream providerSelectTransactions() { + return Stream.of( + Arguments.of(new Scenario("no transactions").expectedSelectedTransactions()), + Arguments.of( + new Scenario("first transaction") + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSelectedTransactions(S1, 0)), + Arguments.of( + new Scenario("fill prioritized") + .addForSender(S1, 0, 1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("reverse fill prioritized") + .addForSender(S1, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("reverse fill prioritized 2") + .addForSender(S1, 3, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("overflow to ready") + .addForSender(S1, 0, 1, 2, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("first with gap") + .addForSender(S1, 1) + .expectedSparseForSender(S1, 1) + .expectedSelectedTransactions()), + Arguments.of( + new Scenario("sequence with gap 1") + .addForSender(S1, 0, 2) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .expectedSelectedTransactions(S1, 0)), + Arguments.of( + new Scenario("sequence with gap 2") + .addForSender(S1, 0, 1, 2, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSender(S1, 4) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("out of order sequence 1") + .addForSender(S1, 2, 0, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("out of order sequence 2") + .addForSender(S1, 2, 0, 4, 3, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("out of order sequence with gap 1") + .addForSender(S1, 2, 1) + .expectedSparseForSender(S1, 2, 1) + .expectedSelectedTransactions()), + Arguments.of( + new Scenario("out of order sequence with gap 2") + .addForSender(S1, 2, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .expectedSelectedTransactions(S1, 0)), + Arguments.of( + new Scenario("no gap and confirmed 1") + .addForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2) + .expectedSelectedTransactions(S1, 1, S1, 2)), + Arguments.of( + new Scenario("all confirmed 1") + .addForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 2) + .expectedSelectedTransactions()), + Arguments.of( + new Scenario("all confirmed step by step 1") + .addForSender(S1, 3) + .expectedSparseForSender(S1, 3) + .expectedSelectedTransactions() + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 3) + .expectedSelectedTransactions(S1, 0) + .addForSender(S1, 4) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 3, 4) + .expectedSelectedTransactions(S1, 0) + .addForSender(S1, 1) + .expectedPrioritizedForSender(S1, 0, 1) + .expectedSparseForSender(S1, 3, 4) + .expectedSelectedTransactions(S1, 0, S1, 1) + .addForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .addForSender(S1, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .addForSender(S1, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .addForSender(S1, 7) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .addForSender(S1, 8) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .addForSender(S1, 9) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedDroppedForSender(S1, 9) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .confirmedForSenders(S1, 9) + .expectedPrioritizedForSenders() + .expectedReadyForSenders() + .expectedSparseForSenders() + .expectedSelectedTransactions())); + } + + static Stream providerReorg() { + return Stream.of( + Arguments.of( + new Scenario("reorg") + .setAccountNonce(S1, 0) + .addForSender(S1, 0, 1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 1) + .expectedPrioritizedForSender(S1, 2) + .expectedNextNonceForSenders(S1, 3) + .addForSender(S1, 3) + .expectedPrioritizedForSender(S1, 2, 3) + .setAccountNonce(S1, 0) // rewind nonce due to reorg + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2, 3))); + } + + private static BlockHeader mockBlockHeader() { + final BlockHeader blockHeader = mock(BlockHeader.class); + when(blockHeader.getBaseFee()).thenReturn(Optional.of(Wei.ONE)); + return blockHeader; + } + + private boolean transactionReplacementTester( + final PendingTransaction pt1, final PendingTransaction pt2) { + return transactionReplacementTester(poolConfig, pt1, pt2); + } + + private static boolean transactionReplacementTester( + final TransactionPoolConfiguration poolConfig, + final PendingTransaction pt1, + final PendingTransaction pt2) { + final TransactionPoolReplacementHandler transactionReplacementHandler = + new TransactionPoolReplacementHandler(poolConfig.getPriceBump()); + return transactionReplacementHandler.shouldReplace(pt1, pt2, mockBlockHeader()); + } + + static class Scenario extends BaseTransactionPoolTest { + interface TransactionLayersConsumer { + void accept( + LayeredPendingTransactions pending, + AbstractPrioritizedTransactions prioritized, + ReadyTransactions ready, + SparseTransactions sparse, + EvictCollectorLayer dropped); + } + + final String description; + final List actions = new ArrayList<>(); + List lastExpectedPrioritized = new ArrayList<>(); + List lastExpectedReady = new ArrayList<>(); + List lastExpectedSparse = new ArrayList<>(); + List lastExpectedDropped = new ArrayList<>(); + + final EnumMap nonceBySender = new EnumMap<>(Sender.class); + + { + Arrays.stream(Sender.values()).forEach(e -> nonceBySender.put(e, 0L)); + } + + final EnumMap> txsBySender = new EnumMap<>(Sender.class); + + { + Arrays.stream(Sender.values()).forEach(e -> txsBySender.put(e, new HashMap<>())); + } + + Scenario(final String description) { + this.description = description; + } + + Scenario addForSender(final Sender sender, final long... nonce) { + Arrays.stream(nonce) + .forEach( + n -> { + final var pendingTx = getOrCreate(sender, n); + actions.add( + (pending, prio, ready, sparse, dropped) -> { + final Account mockSender = mock(Account.class); + when(mockSender.getNonce()).thenReturn(nonceBySender.get(sender)); + pending.addTransaction(pendingTx, Optional.of(mockSender)); + }); + }); + return this; + } + + Scenario addForSenders(final Object... args) { + for (int i = 0; i < args.length; i = i + 2) { + final Sender sender = (Sender) args[i]; + final long nonce = (int) args[i + 1]; + addForSender(sender, nonce); + } + return this; + } + + public Scenario confirmedForSenders(final Object... args) { + final Map maxConfirmedNonceBySender = new HashMap<>(); + for (int i = 0; i < args.length; i = i + 2) { + final Sender sender = (Sender) args[i]; + final long nonce = (int) args[i + 1]; + maxConfirmedNonceBySender.put(sender.address, nonce); + } + actions.add( + (pending, prio, ready, sparse, dropped) -> + prio.blockAdded(FeeMarket.london(0L), mockBlockHeader(), maxConfirmedNonceBySender)); + return this; + } + + Scenario setAccountNonce(final Sender sender, final long nonce) { + nonceBySender.put(sender, nonce); + return this; + } + + void execute( + final LayeredPendingTransactions pending, + final AbstractPrioritizedTransactions prioritized, + final ReadyTransactions ready, + final SparseTransactions sparse, + final EvictCollectorLayer dropped) { + actions.forEach(action -> action.accept(pending, prioritized, ready, sparse, dropped)); + assertExpectedPrioritized(prioritized, lastExpectedPrioritized); + assertExpectedReady(ready, lastExpectedReady); + assertExpectedSparse(sparse, lastExpectedSparse); + assertExpectedDropped(dropped, lastExpectedDropped); + } + + private PendingTransaction getOrCreate(final Sender sender, final long nonce) { + return txsBySender + .get(sender) + .computeIfAbsent(nonce, n -> createEIP1559PendingTransactions(sender, n)); + } + + private PendingTransaction get(final Sender sender, final long nonce) { + return txsBySender.get(sender).get(nonce); + } + + private PendingTransaction createEIP1559PendingTransactions( + final Sender sender, final long nonce) { + return createRemotePendingTransaction( + createEIP1559Transaction(nonce, sender.key, sender.gasFeeMultiplier)); + } + + public Scenario expectedPrioritizedForSender(final Sender sender, final long... nonce) { + lastExpectedPrioritized = expectedForSender(sender, nonce); + final var expectedCopy = List.copyOf(lastExpectedPrioritized); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedPrioritized(prio, expectedCopy)); + return this; + } + + public Scenario expectedReadyForSender(final Sender sender, final long... nonce) { + lastExpectedReady = expectedForSender(sender, nonce); + final var expectedCopy = List.copyOf(lastExpectedReady); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedReady(ready, expectedCopy)); + return this; + } + + public Scenario expectedSparseForSender(final Sender sender, final long... nonce) { + lastExpectedSparse = expectedForSender(sender, nonce); + final var expectedCopy = List.copyOf(lastExpectedSparse); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedSparse(sparse, expectedCopy)); + return this; + } + + public Scenario expectedDroppedForSender(final Sender sender, final long... nonce) { + lastExpectedDropped = expectedForSender(sender, nonce); + final var expectedCopy = List.copyOf(lastExpectedDropped); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedDropped(dropped, expectedCopy)); + return this; + } + + public Scenario expectedPrioritizedForSenders( + final Sender sender1, final long nonce1, final Sender sender2, Object... args) { + lastExpectedPrioritized = expectedForSenders(sender1, nonce1, sender2, args); + final var expectedCopy = List.copyOf(lastExpectedPrioritized); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedPrioritized(prio, expectedCopy)); + return this; + } + + public Scenario expectedPrioritizedForSenders() { + lastExpectedPrioritized = List.of(); + final var expectedCopy = List.copyOf(lastExpectedPrioritized); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedPrioritized(prio, expectedCopy)); + return this; + } + + public Scenario expectedReadyForSenders( + final Sender sender1, final long nonce1, final Sender sender2, final Object... args) { + lastExpectedReady = expectedForSenders(sender1, nonce1, sender2, args); + final var expectedCopy = List.copyOf(lastExpectedReady); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedReady(ready, expectedCopy)); + return this; + } + + public Scenario expectedReadyForSenders() { + lastExpectedReady = List.of(); + final var expectedCopy = List.copyOf(lastExpectedReady); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedReady(ready, expectedCopy)); + return this; + } + + public Scenario expectedSparseForSenders( + final Sender sender1, final long nonce1, final Sender sender2, final Object... args) { + lastExpectedSparse = expectedForSenders(sender1, nonce1, sender2, args); + final var expectedCopy = List.copyOf(lastExpectedSparse); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedSparse(sparse, expectedCopy)); + return this; + } + + public Scenario expectedSparseForSenders() { + lastExpectedSparse = List.of(); + final var expectedCopy = List.copyOf(lastExpectedSparse); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedSparse(sparse, expectedCopy)); + return this; + } + + public Scenario expectedDroppedForSenders( + final Sender sender1, final long nonce1, final Sender sender2, final Object... args) { + lastExpectedDropped = expectedForSenders(sender1, nonce1, sender2, args); + final var expectedCopy = List.copyOf(lastExpectedDropped); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedDropped(dropped, expectedCopy)); + return this; + } + + public Scenario expectedDroppedForSenders() { + lastExpectedDropped = List.of(); + final var expectedCopy = List.copyOf(lastExpectedDropped); + actions.add( + (pending, prio, ready, sparse, dropped) -> assertExpectedDropped(dropped, expectedCopy)); + return this; + } + + private void assertExpectedPrioritized( + final AbstractPrioritizedTransactions prioLayer, final List expected) { + assertThat(prioLayer.stream()).describedAs("Prioritized").containsExactlyElementsOf(expected); + } + + private void assertExpectedReady( + final ReadyTransactions readyLayer, final List expected) { + assertThat(readyLayer.stream()).describedAs("Ready").containsExactlyElementsOf(expected); + } + + private void assertExpectedSparse( + final SparseTransactions sparseLayer, final List expected) { + // sparse txs are returned from the most recent to the oldest, so reverse it to make writing + // scenarios easier + final var sortedExpected = new ArrayList<>(expected); + Collections.reverse(sortedExpected); + assertThat(sparseLayer.stream()) + .describedAs("Sparse") + .containsExactlyElementsOf(sortedExpected); + } + + private void assertExpectedDropped( + final EvictCollectorLayer evictCollector, final List expected) { + assertThat(evictCollector.getEvictedTransactions()) + .describedAs("Dropped") + .containsExactlyInAnyOrderElementsOf(expected); + } + + private List expectedForSenders( + final Sender sender1, final long nonce1, final Sender sender2, final Object... args) { + final List expected = new ArrayList<>(); + expected.add(get(sender1, nonce1)); + final List sendersAndNonce = new ArrayList<>(Arrays.asList(args)); + sendersAndNonce.add(0, sender2); + for (int i = 0; i < sendersAndNonce.size(); i = i + 2) { + final Sender sender = (Sender) sendersAndNonce.get(i); + final long nonce = (int) sendersAndNonce.get(i + 1); + expected.add(get(sender, nonce)); + } + return Collections.unmodifiableList(expected); + } + + private List expectedForSender(final Sender sender, final long... nonce) { + return Arrays.stream(nonce).mapToObj(n -> get(sender, n)).toList(); + } + + public Scenario expectedNextNonceForSenders(final Object... args) { + for (int i = 0; i < args.length; i = i + 2) { + final Sender sender = (Sender) args[i]; + final Integer nullableInt = (Integer) args[i + 1]; + final OptionalLong nonce = + nullableInt == null ? OptionalLong.empty() : OptionalLong.of(nullableInt); + actions.add( + (pending, prio, ready, sparse, dropped) -> + assertThat(prio.getNextNonceFor(sender.address)).isEqualTo(nonce)); + } + return this; + } + + public Scenario removeForSender(final Sender sender, final long... nonce) { + Arrays.stream(nonce) + .forEach( + n -> { + final var pendingTx = getOrCreate(sender, n); + actions.add( + (pending, prio, ready, sparse, dropped) -> prio.remove(pendingTx, INVALIDATED)); + }); + return this; + } + + public Scenario expectedSelectedTransactions(final Object... args) { + List expectedSelected = new ArrayList<>(); + for (int i = 0; i < args.length; i = i + 2) { + final Sender sender = (Sender) args[i]; + final long nonce = (int) args[i + 1]; + expectedSelected.add(get(sender, nonce)); + } + actions.add( + (pending, prio, ready, sparse, dropped) -> + assertThat(prio.stream()).containsExactlyElementsOf(expectedSelected)); + return this; + } + + @Override + public String toString() { + return description; + } + } + + enum Sender { + S1(1), + S2(2), + S3(3), + S4(4); + + final KeyPair key; + final Address address; + final int gasFeeMultiplier; + + Sender(final int gasFeeMultiplier) { + this.key = SIGNATURE_ALGORITHM.get().generateKeyPair(); + this.address = Util.publicKeyToAddress(key.getPublicKey()); + this.gasFeeMultiplier = gasFeeMultiplier; + } + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java new file mode 100644 index 0000000000..a4cb760c10 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReplayTest.java @@ -0,0 +1,297 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.TransactionsLayer.RemovalReason.INVALIDATED; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolReplacementHandler; +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; +import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.metrics.StubMetricsSystem; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.zip.GZIPInputStream; + +import com.google.common.base.Splitter; +import kotlin.ranges.LongRange; +import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReplayTest { + private static final Logger LOG = LoggerFactory.getLogger(ReplayTest.class); + private final TransactionPoolConfiguration poolConfig = + ImmutableTransactionPoolConfiguration.builder().build(); + + private final StubMetricsSystem metricsSystem = new StubMetricsSystem(); + private final TransactionPoolMetrics txPoolMetrics = new TransactionPoolMetrics(metricsSystem); + + private final Address senderToLog = + Address.fromHexString("0x1a8ed0d3ad42c9019cc141aace7e5fb6e576b917"); + + private BlockHeader currBlockHeader; + + /** + * Ignored by default since this is useful to debug issues having a dump of txs that could be + * quite big and so could take many minutes to execute. To generate the input file for the test + * enable the LOG_FOR_REPLAY logger by adding these parts to the log4j2 configuration: in the + * Appenders section add + * + * 
{@code
+   * 
+   *    
+   *        %m%n
+   *    
+   *    
+   *        
+   *    
+   * 
+   * }
+ * + * in the Loggers section add + * + * 
{@code
+   * 
+   *    
+   * 
+   * }
+ * + * restart and let it run until you need it, then copy the CSV in the test resource folder. + * + * @throws IOException when fails to read the resource + */ + @Test + @Disabled("Provide a replay file to run the test on demand") + public void replay() throws IOException { + try (BufferedReader br = + new BufferedReader( + new InputStreamReader( + new GZIPInputStream(getClass().getResourceAsStream("/tx.csv.gz")), + StandardCharsets.UTF_8))) { + currBlockHeader = mockBlockHeader(br.readLine()); + final BaseFeeMarket baseFeeMarket = FeeMarket.london(0L); + + final AbstractPrioritizedTransactions prioritizedTransactions = + createLayers(poolConfig, txPoolMetrics, baseFeeMarket); + final LayeredPendingTransactions pendingTransactions = + new LayeredPendingTransactions(poolConfig, prioritizedTransactions); + br.lines() + .forEach( + line -> { + try { + final String[] commaSplit = line.split(","); + final String type = commaSplit[0]; + switch (type) { + case "T": + System.out.println( + "T:" + + commaSplit[1] + + " @ " + + Instant.ofEpochMilli(Long.parseLong(commaSplit[2]))); + processTransaction(commaSplit, pendingTransactions, prioritizedTransactions); + break; + case "B": + System.out.println("B:" + commaSplit[1]); + processBlock(commaSplit, prioritizedTransactions, baseFeeMarket); + break; + case "S": + // ToDo: commented since not always working, needs fix + // System.out.println("S"); + // assertStats(line, pendingTransactions); + break; + case "D": + System.out.println("D:" + commaSplit[1]); + processInvalid(commaSplit, prioritizedTransactions); + break; + default: + throw new IllegalArgumentException("Unexpected first field value " + type); + } + } catch (Throwable throwable) { + fail(line, throwable); + } + }); + } + } + + private BlockHeader mockBlockHeader(final String line) { + final List commaSplit = Splitter.on(',').splitToList(line); + final long number = Long.parseLong(commaSplit.get(0)); + final Wei initBaseFee = Wei.of(new BigInteger(commaSplit.get(1))); + final long gasUsed = Long.parseLong(commaSplit.get(2)); + final long gasLimit = Long.parseLong(commaSplit.get(3)); + + final BlockHeader mockHeader = mock(BlockHeader.class); + when(mockHeader.getNumber()).thenReturn(number); + when(mockHeader.getBaseFee()).thenReturn(Optional.of(initBaseFee)); + when(mockHeader.getGasUsed()).thenReturn(gasUsed); + when(mockHeader.getGasLimit()).thenReturn(gasLimit); + + return mockHeader; + } + + private BaseFeePrioritizedTransactions createLayers( + final TransactionPoolConfiguration poolConfig, + final TransactionPoolMetrics txPoolMetrics, + final BaseFeeMarket baseFeeMarket) { + final EvictCollectorLayer evictCollector = new EvictCollectorLayer(txPoolMetrics); + final SparseTransactions sparseTransactions = + new SparseTransactions( + poolConfig, evictCollector, txPoolMetrics, this::transactionReplacementTester); + + final ReadyTransactions readyTransactions = + new ReadyTransactions( + poolConfig, sparseTransactions, txPoolMetrics, this::transactionReplacementTester); + + return new BaseFeePrioritizedTransactions( + poolConfig, + () -> currBlockHeader, + readyTransactions, + txPoolMetrics, + this::transactionReplacementTester, + baseFeeMarket); + } + + // ToDo: commented since not always working, needs fix + // private void assertStats( + // final String line, final LayeredPendingTransactions pendingTransactions) { + // final String statsString = line.substring(2); + // assertThat(pendingTransactions.logStats()).as(line).endsWith(statsString); + // } + + private void processBlock( + final String[] commaSplit, + final AbstractPrioritizedTransactions prioritizedTransactions, + final FeeMarket feeMarket) { + final Bytes bytes = Bytes.fromHexString(commaSplit[commaSplit.length - 1]); + final RLPInput rlpInput = new BytesValueRLPInput(bytes, false); + final BlockHeader blockHeader = + BlockHeader.readFrom(rlpInput, new MainnetBlockHeaderFunctions()); + + final Map maxNonceBySender = new HashMap<>(); + int i = 3; + if (!commaSplit[i].equals("")) { + while (!commaSplit[i].equals("R")) { + final Address sender = Address.fromHexString(commaSplit[i]); + final long nonce = Long.parseLong(commaSplit[i + 1]); + maxNonceBySender.put(sender, nonce); + i += 2; + } + } else { + ++i; + } + + ++i; + final Map nonceRangeBySender = new HashMap<>(); + if (!commaSplit[i].equals("")) { + for (; i < commaSplit.length - 1; i += 3) { + final Address sender = Address.fromHexString(commaSplit[i]); + final long start = Long.parseLong(commaSplit[i + 1]); + final long end = Long.parseLong(commaSplit[i + 2]); + nonceRangeBySender.put(sender, new LongRange(start, end)); + } + } + + if (maxNonceBySender.containsKey(senderToLog) || nonceRangeBySender.containsKey(senderToLog)) { + LOG.warn( + "B {} M {} R {} Before {}", + blockHeader.getNumber(), + maxNonceBySender.get(senderToLog), + nonceRangeBySender.get(senderToLog), + prioritizedTransactions.logSender(senderToLog)); + } + prioritizedTransactions.blockAdded(feeMarket, blockHeader, maxNonceBySender); + if (maxNonceBySender.containsKey(senderToLog) || nonceRangeBySender.containsKey(senderToLog)) { + LOG.warn("After {}", prioritizedTransactions.logSender(senderToLog)); + } + } + + private void processTransaction( + final String[] commaSplit, + final LayeredPendingTransactions pendingTransactions, + final AbstractPrioritizedTransactions prioritizedTransactions) { + final Bytes rlp = Bytes.fromHexString(commaSplit[commaSplit.length - 1]); + final Transaction tx = Transaction.readFrom(rlp); + final Account mockAccount = mock(Account.class); + final long nonce = Long.parseLong(commaSplit[4]); + when(mockAccount.getNonce()).thenReturn(nonce); + if (tx.getSender().equals(senderToLog)) { + LOG.warn( + "N {} T {}, Before {}", + nonce, + tx.getNonce(), + prioritizedTransactions.logSender(senderToLog)); + } + assertThat(pendingTransactions.addRemoteTransaction(tx, Optional.of(mockAccount))) + .isNotEqualTo(TransactionAddedResult.INTERNAL_ERROR); + if (tx.getSender().equals(senderToLog)) { + LOG.warn("After {}", prioritizedTransactions.logSender(senderToLog)); + } + } + + private void processInvalid( + final String[] commaSplit, final AbstractPrioritizedTransactions prioritizedTransactions) { + final Bytes rlp = Bytes.fromHexString(commaSplit[commaSplit.length - 1]); + final Transaction tx = Transaction.readFrom(rlp); + if (tx.getSender().equals(senderToLog)) { + LOG.warn("D {}, Before {}", tx.getNonce(), prioritizedTransactions.logSender(senderToLog)); + } + prioritizedTransactions.remove(new PendingTransaction.Remote(tx), INVALIDATED); + if (tx.getSender().equals(senderToLog)) { + LOG.warn("After {}", prioritizedTransactions.logSender(senderToLog)); + } + } + + private boolean transactionReplacementTester( + final PendingTransaction pt1, final PendingTransaction pt2) { + return transactionReplacementTester(poolConfig, pt1, pt2); + } + + private boolean transactionReplacementTester( + final TransactionPoolConfiguration poolConfig, + final PendingTransaction pt1, + final PendingTransaction pt2) { + final TransactionPoolReplacementHandler transactionReplacementHandler = + new TransactionPoolReplacementHandler(poolConfig.getPriceBump()); + return transactionReplacementHandler.shouldReplace(pt1, pt2, currBlockHeader); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsTestBase.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsTestBase.java index 6ccbcba6e6..c138779373 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsTestBase.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsTestBase.java @@ -15,12 +15,11 @@ package org.hyperledger.besu.ethereum.eth.transactions.sorter; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions.TransactionSelectionResult.COMPLETE_OPERATION; import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions.TransactionSelectionResult.CONTINUE; import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions.TransactionSelectionResult.DELETE_TRANSACTION_AND_CONTINUE; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus.ADDED; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus.ALREADY_KNOWN; -import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedStatus.REJECTED_UNDERPRICED_REPLACEMENT; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ADDED; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.ALREADY_KNOWN; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionAddedResult.REJECTED_UNDERPRICED_REPLACEMENT; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -38,8 +37,8 @@ import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; import org.hyperledger.besu.ethereum.core.Util; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionAddedListener; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionDroppedListener; -import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactionListener; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.evm.account.Account; @@ -76,7 +75,7 @@ public abstract class AbstractPendingTransactionsTestBase { protected final TestClock clock = new TestClock(); protected final StubMetricsSystem metricsSystem = new StubMetricsSystem(); - protected PendingTransactions transactions = + protected AbstractPendingTransactionsSorter transactions = getPendingTransactions( ImmutableTransactionPoolConfiguration.builder() .txPoolMaxSize(MAX_TRANSACTIONS) @@ -94,13 +93,14 @@ public abstract class AbstractPendingTransactionsTestBase { protected final Transaction transaction1 = createTransaction(2); protected final Transaction transaction2 = createTransaction(1); - protected final PendingTransactionListener listener = mock(PendingTransactionListener.class); + protected final PendingTransactionAddedListener listener = + mock(PendingTransactionAddedListener.class); protected final PendingTransactionDroppedListener droppedListener = mock(PendingTransactionDroppedListener.class); protected static final Address SENDER1 = Util.publicKeyToAddress(KEYS1.getPublicKey()); protected static final Address SENDER2 = Util.publicKeyToAddress(KEYS2.getPublicKey()); - abstract PendingTransactions getPendingTransactions( + abstract AbstractPendingTransactionsSorter getPendingTransactions( final TransactionPoolConfiguration poolConfig, Optional clock); @Test @@ -314,7 +314,7 @@ public abstract class AbstractPendingTransactionsTestBase { transactions.selectTransactions( transaction -> { parsedTransactions.add(transaction); - return COMPLETE_OPERATION; + return PendingTransactions.TransactionSelectionResult.COMPLETE_OPERATION; }); assertThat(parsedTransactions.size()).isEqualTo(1); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/BaseFeePendingTransactionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/BaseFeePendingTransactionsTest.java index a4856d34f7..fc47600e24 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/BaseFeePendingTransactionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/BaseFeePendingTransactionsTest.java @@ -17,7 +17,6 @@ package org.hyperledger.besu.ethereum.eth.transactions.sorter; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.TransactionTestFixture; -import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.plugin.data.TransactionType; import org.hyperledger.besu.testutil.TestClock; @@ -30,7 +29,7 @@ import java.util.Random; public class BaseFeePendingTransactionsTest extends AbstractPendingTransactionsTestBase { @Override - PendingTransactions getPendingTransactions( + AbstractPendingTransactionsSorter getPendingTransactions( final TransactionPoolConfiguration poolConfig, final Optional clock) { return new BaseFeePendingTransactionsSorter( poolConfig, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/GasPricePendingTransactionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/GasPricePendingTransactionsTest.java index 3a2edf524e..6dec438c5c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/GasPricePendingTransactionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/GasPricePendingTransactionsTest.java @@ -14,7 +14,6 @@ */ package org.hyperledger.besu.ethereum.eth.transactions.sorter; -import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.testutil.TestClock; @@ -25,7 +24,7 @@ import java.util.Optional; public class GasPricePendingTransactionsTest extends AbstractPendingTransactionsTestBase { @Override - PendingTransactions getPendingTransactions( + AbstractPendingTransactionsSorter getPendingTransactions( final TransactionPoolConfiguration poolConfig, final Optional clock) { return new BaseFeePendingTransactionsSorter( poolConfig, diff --git a/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPInput.java b/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPInput.java index 184e17219a..342f48de11 100644 --- a/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPInput.java +++ b/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPInput.java @@ -350,7 +350,7 @@ public interface RLPInput { */ default List readList(final Function valueReader) { final int size = enterList(); - final List res = new ArrayList<>(size); + final List res = size == 0 ? List.of() : new ArrayList<>(size); for (int i = 0; i < size; i++) { try { res.add(valueReader.apply(this)); diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 1285868918..ded2ce3592 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -4983,6 +4983,19 @@ + + + + + + + + + + + + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 5c8ed27a3a..111f066606 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -157,6 +157,7 @@ dependencyManagement { } dependency 'org.fusesource.jansi:jansi:2.4.0' + dependency 'org.openjdk.jol:jol-core:0.17' dependency 'tech.pegasys:jc-kzg-4844:0.4.0' dependencySet(group: 'org.hyperledger.besu', version: '0.7.1') {