feat: Gas escalator middleware (#4211)

### Description

Follow up to
https://github.com/hyperlane-xyz/hyperlane-monorepo/pull/3852 and
https://github.com/hyperlane-xyz/hyperlane-monorepo/pull/4098.

This PR brings major fixes to the escalator:
- A revamped escalator implementation, that was forked from ethers-rs
and had various fixes applied to it. The most notable bug caused
"infinite" gas price escalations to occur within a very short timespan,
until the max configured gas price was reached. Full diff here:
950dd3480c...edf703a6e5
- Support for escalating EIP-1559 txs, in addition to the existing
Legacy tx type support
- Memory leak fix. Since each escalator instance spawns a thread and we
create transient providers for each message, this created a memory leak
of escalator tasks that would build up over time - on RC, memory usage
had reached 27gb before we noticed. This PR merges in the changes from
https://github.com/hyperlane-xyz/hyperlane-monorepo/pull/4208 to stop
the leak. Profiling results:
- before the fix (62mb peak usage): ![Screenshot_2024-07-29_at_12 19
26](https://github.com/user-attachments/assets/5b52fecb-6733-481c-96fb-10f505b9bf8a)
- after the fix (16mb peak usage): ![Screenshot_2024-07-29_at_12 19
53](https://github.com/user-attachments/assets/a78b9452-78cb-4b5b-9aa6-fe7f3a6ac683)

### Drive-by changes

<!--
Are there any minor or drive-by changes also included?
-->

### Related issues

- Fixes https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/4108

### Backward compatibility

Yes

### Testing

Manual testing using the unit test setup from ethers-rs, and using it on
RC to observe it in the wild. Prom chain returns a `could not replace
existing tx` error when a tx is resubmitted so the escalator won't be
effective there, but this can be fixed later

---------

Co-authored-by: Trevor Porter <tkporter4@gmail.com>

rust/main/Cargo.lock

@@ -2901,7 +2901,7 @@ dependencies = [
 [[package]]
 name = "ethers"
 version = "1.0.2"
-source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-04-25#361b69b9561e11eb3cf8000a51de1985e2571785"
+source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-12-03-3#edf703a6e515266245b88bb4f1daad24f7c6566c"
 dependencies = [
  "ethers-addressbook",
  "ethers-contract",
@@ -2915,7 +2915,7 @@ dependencies = [
 [[package]]
 name = "ethers-addressbook"
 version = "1.0.2"
-source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-04-25#361b69b9561e11eb3cf8000a51de1985e2571785"
+source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-12-03-3#edf703a6e515266245b88bb4f1daad24f7c6566c"
 dependencies = [
  "ethers-core",
  "once_cell",
@@ -2926,7 +2926,7 @@ dependencies = [
 [[package]]
 name = "ethers-contract"
 version = "1.0.2"
-source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-04-25#361b69b9561e11eb3cf8000a51de1985e2571785"
+source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-12-03-3#edf703a6e515266245b88bb4f1daad24f7c6566c"
 dependencies = [
  "ethers-contract-abigen",
  "ethers-contract-derive",
@@ -2944,7 +2944,7 @@ dependencies = [
 [[package]]
 name = "ethers-contract-abigen"
 version = "1.0.2"
-source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-04-25#361b69b9561e11eb3cf8000a51de1985e2571785"
+source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-12-03-3#edf703a6e515266245b88bb4f1daad24f7c6566c"
 dependencies = [
  "Inflector",
  "cfg-if",
@@ -2968,7 +2968,7 @@ dependencies = [
 [[package]]
 name = "ethers-contract-derive"
 version = "1.0.2"
-source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-04-25#361b69b9561e11eb3cf8000a51de1985e2571785"
+source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-12-03-3#edf703a6e515266245b88bb4f1daad24f7c6566c"
 dependencies = [
  "ethers-contract-abigen",
  "ethers-core",
@@ -2982,7 +2982,7 @@ dependencies = [
 [[package]]
 name = "ethers-core"
 version = "1.0.2"
-source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-04-25#361b69b9561e11eb3cf8000a51de1985e2571785"
+source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-12-03-3#edf703a6e515266245b88bb4f1daad24f7c6566c"
 dependencies = [
  "arrayvec",
  "bytes",
@@ -3012,7 +3012,7 @@ dependencies = [
 [[package]]
 name = "ethers-etherscan"
 version = "1.0.2"
-source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-04-25#361b69b9561e11eb3cf8000a51de1985e2571785"
+source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-12-03-3#edf703a6e515266245b88bb4f1daad24f7c6566c"
 dependencies = [
  "ethers-core",
  "getrandom 0.2.15",
@@ -3028,7 +3028,7 @@ dependencies = [
 [[package]]
 name = "ethers-middleware"
 version = "1.0.2"
-source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-04-25#361b69b9561e11eb3cf8000a51de1985e2571785"
+source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-12-03-3#edf703a6e515266245b88bb4f1daad24f7c6566c"
 dependencies = [
  "async-trait",
  "auto_impl 0.5.0",
@@ -3076,7 +3076,7 @@ dependencies = [
 [[package]]
 name = "ethers-providers"
 version = "1.0.2"
-source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-04-25#361b69b9561e11eb3cf8000a51de1985e2571785"
+source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-12-03-3#edf703a6e515266245b88bb4f1daad24f7c6566c"
 dependencies = [
  "async-trait",
  "auto_impl 1.2.0",
@@ -3112,7 +3112,7 @@ dependencies = [
 [[package]]
 name = "ethers-signers"
 version = "1.0.2"
-source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-04-25#361b69b9561e11eb3cf8000a51de1985e2571785"
+source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2024-12-03-3#edf703a6e515266245b88bb4f1daad24f7c6566c"
 dependencies = [
  "async-trait",
  "coins-bip32 0.7.0",

rust/main/Cargo.toml

@@ -197,27 +197,27 @@ overflow-checks = true
 [workspace.dependencies.ethers]
 features = []
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2024-04-25"
+tag = "2024-12-03-3"
 
 [workspace.dependencies.ethers-contract]
 features = ["legacy"]
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2024-04-25"
+tag = "2024-12-03-3"
 
 [workspace.dependencies.ethers-core]
 features = []
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2024-04-25"
+tag = "2024-12-03-3"
 
 [workspace.dependencies.ethers-providers]
 features = []
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2024-04-25"
+tag = "2024-12-03-3"
 
 [workspace.dependencies.ethers-signers]
 features = ["aws"]
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2024-04-25"
+tag = "2024-12-03-3"
 
 [patch.crates-io.curve25519-dalek]
 branch = "v3.2.2-relax-zeroize"

rust/main/chains/hyperlane-ethereum/src/rpc_clients/trait_builder.rs

@@ -3,6 +3,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use async_trait::async_trait;
+use ethers::middleware::gas_escalator::{Frequency, GasEscalatorMiddleware, GeometricGasPrice};
 use ethers::middleware::gas_oracle::{
     GasCategory, GasOracle, GasOracleMiddleware, Polygon, ProviderOracle,
 };
@@ -10,6 +11,8 @@ use ethers::prelude::{
     Http, JsonRpcClient, Middleware, NonceManagerMiddleware, Provider, Quorum, QuorumProvider,
     SignerMiddleware, WeightedProvider, Ws, WsClientError,
 };
+use ethers::types::Address;
+use ethers_signers::Signer;
 use hyperlane_core::rpc_clients::FallbackProvider;
 use reqwest::{Client, Url};
 use thiserror::Error;
@@ -22,6 +25,7 @@ use ethers_prometheus::middleware::{MiddlewareMetrics, PrometheusMiddlewareConf}
 use hyperlane_core::{
     ChainCommunicationError, ChainResult, ContractLocator, HyperlaneDomain, KnownHyperlaneDomain,
 };
+use tracing::instrument;
 
 use crate::signer::Signers;
 use crate::{ConnectionConf, EthereumFallbackProvider, RetryingProvider, RpcConnectionConf};
@@ -195,13 +199,13 @@ pub trait BuildableWithProvider {
     where
         P: JsonRpcClient + 'static,
     {
-        let provider = wrap_with_gas_oracle(Provider::new(client), locator.domain)?;
-        self.build_with_signer(provider, conn, locator, signer)
+        self.build_with_signer(Provider::new(client), conn, locator, signer)
             .await
     }
 
     /// Wrap the provider creation with a signing provider if signers were
     /// provided, and then create the associated trait.
+    #[instrument(skip(self, provider, conn, locator, signer), fields(domain=locator.domain.name()), level = "debug")]
     async fn build_with_signer<M>(
         &self,
         provider: M,
@@ -213,10 +217,20 @@ pub trait BuildableWithProvider {
         M: Middleware + 'static,
     {
         Ok(if let Some(signer) = signer {
-            let signing_provider = wrap_with_signer(provider, signer)
+            // The signing provider is used for sending txs, which may end up stuck in the mempool due to
+            // gas pricing issues. We first wrap the provider in a signer middleware, to sign any new txs sent by the gas escalator middleware.
+            // We keep nonce manager as the outermost middleware, so that every new tx with a higher gas price reuses the same nonce.
+            let signing_provider = wrap_with_signer(provider, signer.clone())
                 .await
                 .map_err(ChainCommunicationError::from_other)?;
-            self.build_with_provider(signing_provider, conn, locator)
+            let gas_escalator_provider = wrap_with_gas_escalator(signing_provider);
+            let gas_oracle_provider = wrap_with_gas_oracle(gas_escalator_provider, locator.domain)?;
+            let nonce_manager_provider =
+                wrap_with_nonce_manager(gas_oracle_provider, signer.address())
+                    .await
+                    .map_err(ChainCommunicationError::from_other)?;
+
+            self.build_with_provider(nonce_manager_provider, conn, locator)
         } else {
             self.build_with_provider(provider, conn, locator)
         }
@@ -237,15 +251,19 @@ pub trait BuildableWithProvider {
 async fn wrap_with_signer<M: Middleware>(
     provider: M,
     signer: Signers,
-) -> Result<SignerMiddleware<NonceManagerMiddleware<M>, Signers>, M::Error> {
+) -> Result<SignerMiddleware<M, Signers>, M::Error> {
     let provider_chain_id = provider.get_chainid().await?;
     let signer = ethers::signers::Signer::with_chain_id(signer, provider_chain_id.as_u64());
 
-    let address = ethers::prelude::Signer::address(&signer);
-    let provider = NonceManagerMiddleware::new(provider, address);
+    Ok(SignerMiddleware::new(provider, signer))
+}
 
-    let signing_provider = SignerMiddleware::new(provider, signer);
-    Ok(signing_provider)
+async fn wrap_with_nonce_manager<M: Middleware>(
+    provider: M,
+    signer_address: Address,
+) -> Result<NonceManagerMiddleware<M>, M::Error> {
+    let nonce_manager_provider = NonceManagerMiddleware::new(provider, signer_address);
+    Ok(nonce_manager_provider)
 }
 
 fn build_polygon_gas_oracle(chain: ethers_core::types::Chain) -> ChainResult<Box<dyn GasOracle>> {
@@ -277,3 +295,20 @@ where
     };
     Ok(GasOracleMiddleware::new(provider, gas_oracle))
 }
+
+fn wrap_with_gas_escalator<M>(provider: M) -> GasEscalatorMiddleware<M>
+where
+    M: Middleware + 'static,
+{
+    // Increase the gas price by 12.5% every 90 seconds
+    // (These are the default values from ethers doc comments)
+    const COEFFICIENT: f64 = 1.125;
+    const EVERY_SECS: u64 = 90u64;
+    // 550 gwei is the limit we also use for polygon, so we reuse for consistency
+    const MAX_GAS_PRICE: u128 = 550 * 10u128.pow(9);
+    let escalator = GeometricGasPrice::new(COEFFICIENT, EVERY_SECS, MAX_GAS_PRICE.into());
+    // Check the status of sent txs every eth block or so. The alternative is to subscribe to new blocks and check then,
+    // which adds unnecessary load on the provider.
+    const FREQUENCY: Frequency = Frequency::Duration(Duration::from_secs(12).as_millis() as _);
+    GasEscalatorMiddleware::new(provider, escalator, FREQUENCY)
+}

rust/main/chains/hyperlane-ethereum/src/tx.rs

@@ -68,8 +68,7 @@ where
         .cloned()
         .unwrap_or_else(|| NameOrAddress::Address(Default::default()));
 
-    info!(?to, %data, "Dispatching transaction");
-    // We can set the gas higher here!
+    info!(?to, %data, tx=?tx.tx, "Dispatching transaction");
     let dispatch_fut = tx.send();
     let dispatched = dispatch_fut
         .await?

rust/main/ethers-prometheus/src/middleware/mod.rs

@@ -227,7 +227,6 @@ impl<M: Middleware> Middleware for PrometheusMiddleware<M> {
     ) -> Result<PendingTransaction<'_, Self::Provider>, Self::Error> {
         let start = Instant::now();
         let tx: TypedTransaction = tx.into();
-
         let chain = {
             let data = self.conf.read().await;
             chain_name(&data.chain).to_owned()

rust/main/utils/run-locally/src/invariants/termination_invariants.rs

@@ -92,9 +92,12 @@ pub fn termination_invariants_met(
     // EDIT: Having had a quick look, it seems like there are some legitimate reverts happening in the confirm step
     // (`Transaction attempting to process message either reverted or was reorged`)
     // in which case more gas expenditure logs than messages are expected.
+    let gas_expenditure_log_count = log_counts.get(GAS_EXPENDITURE_LOG_MESSAGE).unwrap();
     assert!(
-        log_counts.get(GAS_EXPENDITURE_LOG_MESSAGE).unwrap() >= &total_messages_expected,
-        "Didn't record gas payment for all delivered messages"
+        gas_expenditure_log_count >= &total_messages_expected,
+        "Didn't record gas payment for all delivered messages. Got {} gas payment logs, expected at least {}",
+        gas_expenditure_log_count,
+        total_messages_expected
     );
     // These tests check that we fixed https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/3915, where some logs would not show up
     assert!(