Merge branch 'main' into dan/gas-escalator-middleware

.changeset/many-rice-wave.md

@@ -1,7 +0,0 @@
----
-"@hyperlane-xyz/cli": patch
-"@hyperlane-xyz/helloworld": patch
-"@hyperlane-xyz/infra": patch
----
-
-fix: minor change was breaking in registry export

.changeset/neat-ducks-own.md

@@ -1,5 +0,0 @@
----
-'@hyperlane-xyz/cli': minor
----
-
-Add hyperlane validator address command to retrieve validator address from AWS

.changeset/nice-rivers-own.md

@@ -1,7 +0,0 @@
----
-'@hyperlane-xyz/infra': minor
-'@hyperlane-xyz/cli': minor
-'@hyperlane-xyz/sdk': minor
----
-
-Implement multi collateral warp routes

.changeset/perfect-seahorses-add.md

@@ -1,7 +0,0 @@
----
-'@hyperlane-xyz/cli': minor
-'@hyperlane-xyz/sdk': minor
-'@hyperlane-xyz/core': minor
----
-
-Support xERC20 and xERC20 Lockbox in SDK and CLI

.changeset/sweet-pandas-brush.md

@@ -0,0 +1,5 @@
+---
+"@hyperlane-xyz/core": patch
+---
+
+fix: make XERC20 and XERC20 Lockbox proxy-able

.changeset/witty-vans-return.md

@@ -1,7 +0,0 @@
----
-'@hyperlane-xyz/infra': minor
-'@hyperlane-xyz/utils': minor
-'@hyperlane-xyz/sdk': minor
----
-
-Implement metadata builder fetching from message

.github/workflows/rust-skipped.yml

@@ -9,6 +9,8 @@ on:
     paths-ignore:
       - 'rust/**'
       - .github/workflows/rust.yml
+  # Support for merge queues
+  merge_group:
 
 env:
   CARGO_TERM_COLOR: always
@@ -16,12 +18,10 @@ env:
 jobs:
   test-rs:
     runs-on: ubuntu-latest
-
     steps:
       - run: 'echo "No test required" '
 
   lint-rs:
     runs-on: ubuntu-latest
-
     steps:
       - run: 'echo "No lint required" '

.github/workflows/rust.yml

@@ -6,7 +6,9 @@ on:
     paths:
       - 'rust/**'
       - .github/workflows/rust.yml
-
+      - '!*.md'
+  # Support for merge queues
+  merge_group:
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 

.github/workflows/test-skipped.yml

@@ -0,0 +1,109 @@
+name: test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches:
+      - '*'
+    paths:
+      - '*.md'
+      - '!**/*'
+  merge_group:
+
+concurrency:
+  group: e2e-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref_name != 'main' }}
+
+jobs:
+  yarn-install:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Instant pass
+        run: echo "yarn-install job passed"
+
+  yarn-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Instant pass
+        run: echo "yarn-build job passed"
+
+  checkout-registry:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Instant pass
+        run: echo "checkout-registry job passed"
+
+  lint-prettier:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Instant pass
+        run: echo "lint-prettier job passed"
+
+  yarn-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Instant pass
+        run: echo "yarn-test job passed"
+
+  agent-configs:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: [mainnet3, testnet4]
+    steps:
+      - name: Instant pass
+        run: echo "agent-configs job passed"
+
+  e2e-matrix:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.base_ref == 'main') || github.event_name == 'merge_group'
+    strategy:
+      matrix:
+        e2e-type: [cosmwasm, non-cosmwasm]
+    steps:
+      - name: Instant pass
+        run: echo "e2e-matrix job passed"
+
+  e2e:
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Instant pass
+        run: echo "e2e job passed"
+
+  cli-e2e:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.base_ref == 'main') || github.event_name == 'merge_group'
+    strategy:
+      matrix:
+        include:
+          - test-type: preset_hook_enabled
+          - test-type: configure_hook_enabled
+          - test-type: pi_with_core_chain
+    steps:
+      - name: Instant pass
+        run: echo "cli-e2e job passed"
+
+  env-test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: [mainnet3]
+        chain: [ethereum, arbitrum, optimism, inevm, viction]
+        module: [core, igp]
+        include:
+          - environment: testnet4
+            chain: sepolia
+            module: core
+    steps:
+      - name: Instant pass
+        run: echo "env-test job passed"
+
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Instant pass
+        run: echo "coverage job passed"

.github/workflows/test.yml

@@ -7,6 +7,10 @@ on:
   pull_request:
     branches:
       - '*' # run against all branches
+    paths-ignore:
+      - '*.md'
+  # Support for merge queues
+  merge_group:
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
@@ -224,7 +228,7 @@ jobs:
 
   e2e-matrix:
     runs-on: larger-runner
-    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.base_ref == 'main')
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.base_ref == 'main') || github.event_name == 'merge_group'
     needs: [yarn-build, checkout-registry]
     strategy:
       matrix:
@@ -325,7 +329,7 @@ jobs:
 
   cli-e2e:
     runs-on: larger-runner
-    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.base_ref == 'main')
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.base_ref == 'main') || github.event_name == 'merge_group'
     needs: [yarn-build, checkout-registry]
     strategy:
       matrix:

CODE_OF_CONDUCT.md

@@ -9,10 +9,10 @@ This CoC applies to all members of the Hyperlane Network's community including,
 **Code**
 
 1. Never harass or bully anyone. Not verbally, not physically, not sexually. Harassment will not be tolerated.
-2. Never discrimnate on the basis of personal characteristics or group membership.
+2. Never discriminate on the basis of personal characteristics or group membership.
 3. Treat your fellow contributors with respect, fairness, and professionalism, especially in situations of high pressure.
-4. Seek, offer, and accept objective critism of yours and others work, strive to acknowledge the contributions of others.
-5. Be transparent and honest about your qualifications and any potential conflicts of interest. Transparency is a key tenant of the Hyperlane project and we expect it from all contributors.
+4. Seek, offer, and accept objective criticism  of yours and others work, strive to acknowledge the contributions of others.
+5. Be transparent and honest about your qualifications and any potential conflicts of interest. Transparency is a key tenet of the Hyperlane project and we expect it from all contributors.
 6. Bring an open and curious mind, the Hyperlane project is designed to enable developers to express their curiosity, experiment, and build things we couldn't have imagined ourselves.
 7. Stay on track - Do your best to avoid off-topic discussion and make sure you are posting to the correct channel and repositories. Distractions are costly and it is far too easy for work to go off track.
 8. Step down properly - Think of your fellow contributors when you step down from the project. Contributors of open-source projects come and go. It is crucial that when you leave the project or reduce your contribution significantly you do so in a way that minimizes disruption and keeps continuity in mind. Concretely this means telling your fellow contributors you are leaving and taking the proper steps to enable a smooth transition for other contributors to pick up where you left off.

README.md

@@ -103,3 +103,9 @@ See [`rust/README.md`](rust/README.md)
 - Create a summary of change highlights
 - Create a "breaking changes" section with any changes required
 - Deploy agents with the new image tag (if it makes sense to)
+
+### Releasing packages to NPM
+
+We use [changesets](https://github.com/changesets/changesets) to release to NPM. You can use the `release` script in `package.json` to publish.
+
+For an alpha or beta version, follow the directions [here](https://github.com/changesets/changesets/blob/main/docs/prereleases.md).

funding.json

@@ -0,0 +1,5 @@
+{
+    "opRetro": {
+        "projectId": "0xa47182d330bd0c5c69b1418462f3f742099138f09bff057189cdd19676a6acd1"
+    }
+}

rust/.vscode/extensions.json

@@ -4,7 +4,7 @@
 
   // List of extensions which should be recommended for users of this workspace.
   "recommendations": [
-    "panicbit.cargo",
+    "rust-lang.rust-analyzer",
     "tamasfe.even-better-toml",
     "rust-lang.rust-analyzer",
   ],

rust/Cargo.lock

@@ -7275,7 +7275,9 @@ dependencies = [
  "macro_rules_attribute",
  "maplit",
  "nix 0.26.4",
+ "once_cell",
  "regex",
+ "relayer",
  "ripemd",
  "serde",
  "serde_json",

rust/agents/relayer/Cargo.toml

@@ -38,7 +38,7 @@ tracing-futures.workspace = true
 tracing.workspace = true
 
 hyperlane-core = { path = "../../hyperlane-core", features = ["agent", "async"] }
-hyperlane-base = { path = "../../hyperlane-base" }
+hyperlane-base = { path = "../../hyperlane-base", features = ["test-utils"] }
 hyperlane-ethereum = { path = "../../chains/hyperlane-ethereum" }
 
 [dev-dependencies]

rust/agents/relayer/src/lib.rs

@@ -0,0 +1,10 @@
+mod merkle_tree;
+mod msg;
+mod processor;
+mod prover;
+mod relayer;
+mod server;
+mod settings;
+
+pub use msg::GAS_EXPENDITURE_LOG_MESSAGE;
+pub use relayer::*;

rust/agents/relayer/src/main.rs

@@ -11,15 +11,7 @@ use eyre::Result;
 
 use hyperlane_base::agent_main;
 
-use crate::relayer::Relayer;
-
-mod merkle_tree;
-mod msg;
-mod processor;
-mod prover;
-mod relayer;
-mod server;
-mod settings;
+use relayer::Relayer;
 
 #[tokio::main(flavor = "multi_thread", worker_threads = 20)]
 async fn main() -> Result<()> {

rust/agents/relayer/src/msg/gas_payment/mod.rs

@@ -19,6 +19,8 @@ use crate::{
 
 mod policies;
 
+pub const GAS_EXPENDITURE_LOG_MESSAGE: &str = "Recording gas expenditure for message";
+
 #[async_trait]
 pub trait GasPaymentPolicy: Debug + Send + Sync {
     /// Returns Some(gas_limit) if the policy has approved the transaction or
@@ -132,6 +134,13 @@ impl GasPaymentEnforcer {
     }
 
     pub fn record_tx_outcome(&self, message: &HyperlaneMessage, outcome: TxOutcome) -> Result<()> {
+        // This log is required in E2E, hence the use of a `const`
+        debug!(
+            msg=%message,
+            ?outcome,
+            "{}",
+            GAS_EXPENDITURE_LOG_MESSAGE,
+        );
         self.db.process_gas_expenditure(InterchainGasExpenditure {
             message_id: message.id(),
             gas_used: outcome.gas_used,

rust/agents/relayer/src/msg/mod.rs

@@ -30,5 +30,6 @@ pub(crate) mod metadata;
 pub(crate) mod op_queue;
 pub(crate) mod op_submitter;
 pub(crate) mod pending_message;
-pub(crate) mod pending_operation;
 pub(crate) mod processor;
+
+pub use gas_payment::GAS_EXPENDITURE_LOG_MESSAGE;

rust/agents/relayer/src/msg/op_queue.rs

@@ -1,24 +1,20 @@
 use std::{cmp::Reverse, collections::BinaryHeap, sync::Arc};
 
 use derive_new::new;
-use hyperlane_core::MpmcReceiver;
+use hyperlane_core::{PendingOperation, QueueOperation};
 use prometheus::{IntGauge, IntGaugeVec};
-use tokio::sync::Mutex;
-use tracing::{info, instrument};
+use tokio::sync::{broadcast::Receiver, Mutex};
+use tracing::{debug, info, instrument};
 
 use crate::server::MessageRetryRequest;
 
-use super::pending_operation::PendingOperation;
-
-pub type QueueOperation = Box<dyn PendingOperation>;
-
 /// Queue of generic operations that can be submitted to a destination chain.
 /// Includes logic for maintaining queue metrics by the destination and `app_context` of an operation
 #[derive(Debug, Clone, new)]
 pub struct OpQueue {
     metrics: IntGaugeVec,
     queue_metrics_label: String,
-    retry_rx: MpmcReceiver<MessageRetryRequest>,
+    retry_rx: Arc<Mutex<Receiver<MessageRetryRequest>>>,
     #[new(default)]
     queue: Arc<Mutex<BinaryHeap<Reverse<QueueOperation>>>>,
 }
@@ -41,7 +37,7 @@ impl OpQueue {
     }
 
     /// Pop multiple elements at once from the queue and update metrics
-    #[instrument(skip(self), ret, fields(queue_label=%self.queue_metrics_label), level = "debug")]
+    #[instrument(skip(self), fields(queue_label=%self.queue_metrics_label), level = "debug")]
     pub async fn pop_many(&mut self, limit: usize) -> Vec<QueueOperation> {
         self.process_retry_requests().await;
         let mut queue = self.queue.lock().await;
@@ -55,6 +51,15 @@ impl OpQueue {
                 break;
             }
         }
+        // This function is called very often by the op_submitter tasks, so only log when there are operations to pop
+        // to avoid spamming the logs
+        if !popped.is_empty() {
+            debug!(
+                queue_label = %self.queue_metrics_label,
+                operations = ?popped,
+                "Popped OpQueue operations"
+            );
+        }
         popped
     }
 
@@ -64,7 +69,7 @@ impl OpQueue {
         // The other consideration is whether to put the channel receiver in the OpQueue or in a dedicated task
         // that also holds an Arc to the Mutex. For simplicity, we'll put it in the OpQueue for now.
         let mut message_retry_requests = vec![];
-        while let Ok(message_id) = self.retry_rx.receiver.try_recv() {
+        while let Ok(message_id) = self.retry_rx.lock().await.try_recv() {
             message_retry_requests.push(message_id);
         }
         if message_retry_requests.is_empty() {
@@ -101,15 +106,15 @@ impl OpQueue {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::msg::pending_operation::PendingOperationResult;
     use hyperlane_core::{
-        HyperlaneDomain, HyperlaneMessage, KnownHyperlaneDomain, MpmcChannel, TryBatchAs,
-        TxOutcome, H256,
+        HyperlaneDomain, HyperlaneMessage, KnownHyperlaneDomain, PendingOperationResult,
+        TryBatchAs, TxOutcome, H256, U256,
     };
     use std::{
         collections::VecDeque,
         time::{Duration, Instant},
     };
+    use tokio::sync;
 
     #[derive(Debug, Clone)]
     struct MockPendingOperation {
@@ -174,6 +179,10 @@ mod test {
             todo!()
         }
 
+        fn get_tx_cost_estimate(&self) -> Option<U256> {
+            todo!()
+        }
+
         /// This will be called after the operation has been submitted and is
         /// responsible for checking if the operation has reached a point at
         /// which we consider it safe from reorgs.
@@ -181,6 +190,14 @@ mod test {
             todo!()
         }
 
+        fn set_operation_outcome(
+            &mut self,
+            _submission_outcome: TxOutcome,
+            _submission_estimated_cost: U256,
+        ) {
+            todo!()
+        }
+
         fn next_attempt_after(&self) -> Option<Instant> {
             Some(
                 Instant::now()
@@ -212,13 +229,17 @@ mod test {
     #[tokio::test]
     async fn test_multiple_op_queues_message_id() {
         let (metrics, queue_metrics_label) = dummy_metrics_and_label();
-        let mpmc_channel = MpmcChannel::new(100);
+        let broadcaster = sync::broadcast::Sender::new(100);
         let mut op_queue_1 = OpQueue::new(
             metrics.clone(),
             queue_metrics_label.clone(),
-            mpmc_channel.receiver(),
+            Arc::new(Mutex::new(broadcaster.subscribe())),
+        );
+        let mut op_queue_2 = OpQueue::new(
+            metrics,
+            queue_metrics_label,
+            Arc::new(Mutex::new(broadcaster.subscribe())),
         );
-        let mut op_queue_2 = OpQueue::new(metrics, queue_metrics_label, mpmc_channel.receiver());
 
         // Add some operations to the queue with increasing `next_attempt_after` values
         let destination_domain: HyperlaneDomain = KnownHyperlaneDomain::Injective.into();
@@ -244,11 +265,10 @@ mod test {
         }
 
         // Retry by message ids
-        let mpmc_tx = mpmc_channel.sender();
-        mpmc_tx
+        broadcaster
             .send(MessageRetryRequest::MessageId(op_ids[1]))
             .unwrap();
-        mpmc_tx
+        broadcaster
             .send(MessageRetryRequest::MessageId(op_ids[2]))
             .unwrap();
 
@@ -278,11 +298,11 @@ mod test {
     #[tokio::test]
     async fn test_destination_domain() {
         let (metrics, queue_metrics_label) = dummy_metrics_and_label();
-        let mpmc_channel = MpmcChannel::new(100);
+        let broadcaster = sync::broadcast::Sender::new(100);
         let mut op_queue = OpQueue::new(
             metrics.clone(),
             queue_metrics_label.clone(),
-            mpmc_channel.receiver(),
+            Arc::new(Mutex::new(broadcaster.subscribe())),
         );
 
         // Add some operations to the queue with increasing `next_attempt_after` values
@@ -304,8 +324,7 @@ mod test {
         }
 
         // Retry by domain
-        let mpmc_tx = mpmc_channel.sender();
-        mpmc_tx
+        broadcaster
             .send(MessageRetryRequest::DestinationDomain(
                 destination_domain_2.id(),
             ))

rust/agents/relayer/src/msg/op_submitter.rs

@@ -1,10 +1,14 @@
+use std::sync::Arc;
 use std::time::Duration;
 
 use derive_new::new;
 use futures::future::join_all;
 use futures_util::future::try_join_all;
+use hyperlane_core::total_estimated_cost;
 use prometheus::{IntCounter, IntGaugeVec};
+use tokio::sync::broadcast::Sender;
 use tokio::sync::mpsc;
+use tokio::sync::Mutex;
 use tokio::task::JoinHandle;
 use tokio::time::sleep;
 use tokio_metrics::TaskMonitor;
@@ -14,14 +18,13 @@ use tracing::{info, warn};
 use hyperlane_base::CoreMetrics;
 use hyperlane_core::{
     BatchItem, ChainCommunicationError, ChainResult, HyperlaneDomain, HyperlaneDomainProtocol,
-    HyperlaneMessage, MpmcReceiver, TxOutcome,
+    HyperlaneMessage, PendingOperationResult, QueueOperation, TxOutcome,
 };
 
 use crate::msg::pending_message::CONFIRM_DELAY;
 use crate::server::MessageRetryRequest;
 
-use super::op_queue::{OpQueue, QueueOperation};
-use super::pending_operation::*;
+use super::op_queue::OpQueue;
 
 /// SerialSubmitter accepts operations over a channel. It is responsible for
 /// executing the right strategy to deliver those messages to the destination
@@ -77,7 +80,7 @@ pub struct SerialSubmitter {
     /// Receiver for new messages to submit.
     rx: mpsc::UnboundedReceiver<QueueOperation>,
     /// Receiver for retry requests.
-    retry_rx: MpmcReceiver<MessageRetryRequest>,
+    retry_tx: Sender<MessageRetryRequest>,
     /// Metrics for serial submitter.
     metrics: SerialSubmitterMetrics,
     /// Max batch size for submitting messages
@@ -101,24 +104,24 @@ impl SerialSubmitter {
             domain,
             metrics,
             rx: rx_prepare,
-            retry_rx,
+            retry_tx,
             max_batch_size,
             task_monitor,
         } = self;
         let prepare_queue = OpQueue::new(
             metrics.submitter_queue_length.clone(),
             "prepare_queue".to_string(),
-            retry_rx.clone(),
+            Arc::new(Mutex::new(retry_tx.subscribe())),
         );
         let submit_queue = OpQueue::new(
             metrics.submitter_queue_length.clone(),
             "submit_queue".to_string(),
-            retry_rx.clone(),
+            Arc::new(Mutex::new(retry_tx.subscribe())),
         );
         let confirm_queue = OpQueue::new(
             metrics.submitter_queue_length.clone(),
             "confirm_queue".to_string(),
-            retry_rx,
+            Arc::new(Mutex::new(retry_tx.subscribe())),
         );
 
         let tasks = [
@@ -425,11 +428,10 @@ impl OperationBatch {
     async fn submit(self, confirm_queue: &mut OpQueue, metrics: &SerialSubmitterMetrics) {
         match self.try_submit_as_batch(metrics).await {
             Ok(outcome) => {
-                // TODO: use the `tx_outcome` with the total gas expenditure
-                // We'll need to proportionally set `used_gas` based on the tx_outcome, so it can be updated in the confirm step
-                // which means we need to add a `set_transaction_outcome` fn to `PendingOperation`
                 info!(outcome=?outcome, batch_size=self.operations.len(), batch=?self.operations, "Submitted transaction batch");
+                let total_estimated_cost = total_estimated_cost(&self.operations);
                 for mut op in self.operations {
+                    op.set_operation_outcome(outcome.clone(), total_estimated_cost);
                     op.set_next_attempt_after(CONFIRM_DELAY);
                     confirm_queue.push(op).await;
                 }
@@ -459,8 +461,6 @@ impl OperationBatch {
             return Err(ChainCommunicationError::BatchIsEmpty);
         };
 
-        // We use the estimated gas limit from the prior call to
-        // `process_estimate_costs` to avoid a second gas estimation.
         let outcome = first_item.mailbox.process_batch(&batch).await?;
         metrics.ops_submitted.inc_by(self.operations.len() as u64);
         Ok(outcome)

rust/agents/relayer/src/msg/pending_message.rs

@@ -9,8 +9,9 @@ use derive_new::new;
 use eyre::Result;
 use hyperlane_base::{db::HyperlaneRocksDB, CoreMetrics};
 use hyperlane_core::{
-    BatchItem, ChainCommunicationError, ChainResult, HyperlaneChain, HyperlaneDomain,
-    HyperlaneMessage, Mailbox, MessageSubmissionData, TryBatchAs, TxOutcome, H256, U256,
+    gas_used_by_operation, make_op_try, BatchItem, ChainCommunicationError, ChainResult,
+    HyperlaneChain, HyperlaneDomain, HyperlaneMessage, Mailbox, MessageSubmissionData,
+    PendingOperation, PendingOperationResult, TryBatchAs, TxOutcome, H256, U256,
 };
 use prometheus::{IntCounter, IntGauge};
 use tracing::{debug, error, info, instrument, trace, warn};
@@ -18,7 +19,6 @@ use tracing::{debug, error, info, instrument, trace, warn};
 use super::{
     gas_payment::GasPaymentEnforcer,
     metadata::{BaseMetadataBuilder, MessageMetadataBuilder, MetadataBuilder},
-    pending_operation::*,
 };
 
 pub const CONFIRM_DELAY: Duration = if cfg!(any(test, feature = "test-utils")) {
@@ -259,7 +259,7 @@ impl PendingOperation for PendingMessage {
 
         let state = self
             .submission_data
-            .take()
+            .clone()
             .expect("Pending message must be prepared before it can be submitted");
 
         // We use the estimated gas limit from the prior call to
@@ -271,7 +271,7 @@ impl PendingOperation for PendingMessage {
             .await;
         match tx_outcome {
             Ok(outcome) => {
-                self.set_submission_outcome(outcome);
+                self.set_operation_outcome(outcome, state.gas_limit);
             }
             Err(e) => {
                 error!(error=?e, "Error when processing message");
@@ -283,6 +283,10 @@ impl PendingOperation for PendingMessage {
         self.submission_outcome = Some(outcome);
     }
 
+    fn get_tx_cost_estimate(&self) -> Option<U256> {
+        self.submission_data.as_ref().map(|d| d.gas_limit)
+    }
+
     async fn confirm(&mut self) -> PendingOperationResult {
         make_op_try!(|| {
             // Provider error; just try again later
@@ -313,15 +317,6 @@ impl PendingOperation for PendingMessage {
             );
             PendingOperationResult::Success
         } else {
-            if let Some(outcome) = &self.submission_outcome {
-                if let Err(e) = self
-                    .ctx
-                    .origin_gas_payment_enforcer
-                    .record_tx_outcome(&self.message, outcome.clone())
-                {
-                    error!(error=?e, "Error when recording tx outcome");
-                }
-            }
             warn!(
                 tx_outcome=?self.submission_outcome,
                 message_id=?self.message.id(),
@@ -331,6 +326,50 @@ impl PendingOperation for PendingMessage {
         }
     }
 
+    fn set_operation_outcome(
+        &mut self,
+        submission_outcome: TxOutcome,
+        submission_estimated_cost: U256,
+    ) {
+        let Some(operation_estimate) = self.get_tx_cost_estimate() else {
+            warn!("Cannot set operation outcome without a cost estimate set previously");
+            return;
+        };
+        // calculate the gas used by the operation
+        let gas_used_by_operation = match gas_used_by_operation(
+            &submission_outcome,
+            submission_estimated_cost,
+            operation_estimate,
+        ) {
+            Ok(gas_used_by_operation) => gas_used_by_operation,
+            Err(e) => {
+                warn!(error = %e, "Error when calculating gas used by operation, falling back to charging the full cost of the tx. Are gas estimates enabled for this chain?");
+                submission_outcome.gas_used
+            }
+        };
+        let operation_outcome = TxOutcome {
+            gas_used: gas_used_by_operation,
+            ..submission_outcome
+        };
+        // record it in the db, to subtract from the sender's igp allowance
+        if let Err(e) = self
+            .ctx
+            .origin_gas_payment_enforcer
+            .record_tx_outcome(&self.message, operation_outcome.clone())
+        {
+            error!(error=?e, "Error when recording tx outcome");
+        }
+        // set the outcome in `Self` as well, for later logging
+        self.set_submission_outcome(operation_outcome);
+        debug!(
+            actual_gas_for_message = ?gas_used_by_operation,
+            message_gas_estimate = ?operation_estimate,
+            submission_gas_estimate = ?submission_estimated_cost,
+            message = ?self.message,
+            "Gas used by message submission"
+        );
+    }
+
     fn next_attempt_after(&self) -> Option<Instant> {
         self.next_attempt_after
     }
@@ -343,7 +382,6 @@ impl PendingOperation for PendingMessage {
         self.reset_attempts();
     }
 
-    #[cfg(test)]
     fn set_retries(&mut self, retries: u32) {
         self.set_retries(retries);
     }

rust/agents/relayer/src/msg/processor.rs

@@ -13,12 +13,12 @@ use hyperlane_base::{
     db::{HyperlaneRocksDB, ProcessMessage},
     CoreMetrics,
 };
-use hyperlane_core::{HyperlaneDomain, HyperlaneMessage};
+use hyperlane_core::{HyperlaneDomain, HyperlaneMessage, QueueOperation};
 use prometheus::IntGauge;
 use tokio::sync::mpsc::UnboundedSender;
 use tracing::{debug, instrument, trace};
 
-use super::{metadata::AppContextClassifier, op_queue::QueueOperation, pending_message::*};
+use super::{metadata::AppContextClassifier, pending_message::*};
 use crate::{processor::ProcessorExt, settings::matching_list::MatchingList};
 
 /// Finds unprocessed messages from an origin and submits then through a channel

rust/agents/relayer/src/relayer.rs

@@ -13,13 +13,15 @@ use hyperlane_base::{
     metrics::{AgentMetrics, MetricsUpdater},
     settings::ChainConf,
     BaseAgent, ChainMetrics, ContractSyncMetrics, ContractSyncer, CoreMetrics, HyperlaneAgentCore,
+    SyncOptions,
 };
 use hyperlane_core::{
-    HyperlaneDomain, HyperlaneMessage, InterchainGasPayment, MerkleTreeInsertion, MpmcChannel,
-    MpmcReceiver, U256,
+    HyperlaneDomain, HyperlaneMessage, InterchainGasPayment, MerkleTreeInsertion, QueueOperation,
+    H512, U256,
 };
 use tokio::{
     sync::{
+        broadcast::{Receiver, Sender},
         mpsc::{self, UnboundedReceiver, UnboundedSender},
         RwLock,
     },
@@ -33,7 +35,6 @@ use crate::{
     msg::{
         gas_payment::GasPaymentEnforcer,
         metadata::{BaseMetadataBuilder, IsmAwareAppContextClassifier},
-        op_queue::QueueOperation,
         op_submitter::{SerialSubmitter, SerialSubmitterMetrics},
         pending_message::{MessageContext, MessageSubmissionMetrics},
         processor::{MessageProcessor, MessageProcessorMetrics},
@@ -134,7 +135,7 @@ impl BaseAgent for Relayer {
 
         let contract_sync_metrics = Arc::new(ContractSyncMetrics::new(&core_metrics));
 
-        let message_syncs = settings
+        let message_syncs: HashMap<_, Arc<dyn ContractSyncer<HyperlaneMessage>>> = settings
             .contract_syncs::<HyperlaneMessage, _>(
                 settings.origin_chains.iter(),
                 &core_metrics,
@@ -305,8 +306,8 @@ impl BaseAgent for Relayer {
         }
 
         // run server
-        let mpmc_channel = MpmcChannel::<MessageRetryRequest>::new(ENDPOINT_MESSAGES_QUEUE_SIZE);
-        let custom_routes = relayer_server::routes(mpmc_channel.sender());
+        let sender = Sender::<MessageRetryRequest>::new(ENDPOINT_MESSAGES_QUEUE_SIZE);
+        let custom_routes = relayer_server::routes(sender.clone());
 
         let server = self
             .core
@@ -328,7 +329,7 @@ impl BaseAgent for Relayer {
                 self.run_destination_submitter(
                     dest_domain,
                     receive_channel,
-                    mpmc_channel.receiver(),
+                    sender.clone(),
                     // Default to submitting one message at a time if there is no batch config
                     self.core.settings.chains[dest_domain.name()]
                         .connection
@@ -352,14 +353,26 @@ impl BaseAgent for Relayer {
         }
 
         for origin in &self.origin_chains {
+            let maybe_broadcaster = self
+                .message_syncs
+                .get(origin)
+                .and_then(|sync| sync.get_broadcaster());
             tasks.push(self.run_message_sync(origin, task_monitor.clone()).await);
             tasks.push(
-                self.run_interchain_gas_payment_sync(origin, task_monitor.clone())
-                    .await,
+                self.run_interchain_gas_payment_sync(
+                    origin,
+                    maybe_broadcaster.clone().map(|b| b.subscribe()),
+                    task_monitor.clone(),
+                )
+                .await,
             );
             tasks.push(
-                self.run_merkle_tree_hook_syncs(origin, task_monitor.clone())
-                    .await,
+                self.run_merkle_tree_hook_syncs(
+                    origin,
+                    maybe_broadcaster.map(|b| b.subscribe()),
+                    task_monitor.clone(),
+                )
+                .await,
             );
         }
 
@@ -394,7 +407,7 @@ impl Relayer {
         tokio::spawn(TaskMonitor::instrument(&task_monitor, async move {
             contract_sync
                 .clone()
-                .sync("dispatched_messages", cursor)
+                .sync("dispatched_messages", cursor.into())
                 .await
         }))
         .instrument(info_span!("MessageSync"))
@@ -403,6 +416,7 @@ impl Relayer {
     async fn run_interchain_gas_payment_sync(
         &self,
         origin: &HyperlaneDomain,
+        tx_id_receiver: Option<Receiver<H512>>,
         task_monitor: TaskMonitor,
     ) -> Instrumented<JoinHandle<()>> {
         let index_settings = self.as_ref().settings.chains[origin.name()].index_settings();
@@ -413,7 +427,13 @@ impl Relayer {
             .clone();
         let cursor = contract_sync.cursor(index_settings).await;
         tokio::spawn(TaskMonitor::instrument(&task_monitor, async move {
-            contract_sync.clone().sync("gas_payments", cursor).await
+            contract_sync
+                .clone()
+                .sync(
+                    "gas_payments",
+                    SyncOptions::new(Some(cursor), tx_id_receiver),
+                )
+                .await
         }))
         .instrument(info_span!("IgpSync"))
     }
@@ -421,13 +441,20 @@ impl Relayer {
     async fn run_merkle_tree_hook_syncs(
         &self,
         origin: &HyperlaneDomain,
+        tx_id_receiver: Option<Receiver<H512>>,
         task_monitor: TaskMonitor,
     ) -> Instrumented<JoinHandle<()>> {
         let index_settings = self.as_ref().settings.chains[origin.name()].index.clone();
         let contract_sync = self.merkle_tree_hook_syncs.get(origin).unwrap().clone();
         let cursor = contract_sync.cursor(index_settings).await;
         tokio::spawn(TaskMonitor::instrument(&task_monitor, async move {
-            contract_sync.clone().sync("merkle_tree_hook", cursor).await
+            contract_sync
+                .clone()
+                .sync(
+                    "merkle_tree_hook",
+                    SyncOptions::new(Some(cursor), tx_id_receiver),
+                )
+                .await
         }))
         .instrument(info_span!("MerkleTreeHookSync"))
     }
@@ -498,7 +525,7 @@ impl Relayer {
         &self,
         destination: &HyperlaneDomain,
         receiver: UnboundedReceiver<QueueOperation>,
-        retry_receiver_channel: MpmcReceiver<MessageRetryRequest>,
+        retry_receiver_channel: Sender<MessageRetryRequest>,
         batch_size: u32,
         task_monitor: TaskMonitor,
     ) -> Instrumented<JoinHandle<()>> {

rust/agents/relayer/src/server.rs

@@ -3,13 +3,11 @@ use axum::{
     routing, Router,
 };
 use derive_new::new;
-use hyperlane_core::{ChainCommunicationError, H256};
+use hyperlane_core::{ChainCommunicationError, QueueOperation, H256};
 use serde::Deserialize;
 use std::str::FromStr;
 use tokio::sync::broadcast::Sender;
 
-use crate::msg::op_queue::QueueOperation;
-
 const MESSAGE_RETRY_API_BASE: &str = "/message_retry";
 pub const ENDPOINT_MESSAGES_QUEUE_SIZE: usize = 1_000;
 
@@ -109,12 +107,12 @@ mod tests {
     use super::*;
     use axum::http::StatusCode;
     use ethers::utils::hex::ToHex;
-    use hyperlane_core::{MpmcChannel, MpmcReceiver};
     use std::net::SocketAddr;
+    use tokio::sync::broadcast::{Receiver, Sender};
 
-    fn setup_test_server() -> (SocketAddr, MpmcReceiver<MessageRetryRequest>) {
-        let mpmc_channel = MpmcChannel::<MessageRetryRequest>::new(ENDPOINT_MESSAGES_QUEUE_SIZE);
-        let message_retry_api = MessageRetryApi::new(mpmc_channel.sender());
+    fn setup_test_server() -> (SocketAddr, Receiver<MessageRetryRequest>) {
+        let broadcast_tx = Sender::<MessageRetryRequest>::new(ENDPOINT_MESSAGES_QUEUE_SIZE);
+        let message_retry_api = MessageRetryApi::new(broadcast_tx.clone());
         let (path, retry_router) = message_retry_api.get_route();
         let app = Router::new().nest(path, retry_router);
 
@@ -124,7 +122,7 @@ mod tests {
         let addr = server.local_addr();
         tokio::spawn(server);
 
-        (addr, mpmc_channel.receiver())
+        (addr, broadcast_tx.subscribe())
     }
 
     #[tokio::test]
@@ -148,7 +146,7 @@ mod tests {
         assert_eq!(response.status(), StatusCode::OK);
 
         assert_eq!(
-            rx.receiver.try_recv().unwrap(),
+            rx.try_recv().unwrap(),
             MessageRetryRequest::MessageId(message_id)
         );
     }
@@ -172,7 +170,7 @@ mod tests {
         assert_eq!(response.status(), StatusCode::OK);
 
         assert_eq!(
-            rx.receiver.try_recv().unwrap(),
+            rx.try_recv().unwrap(),
             MessageRetryRequest::DestinationDomain(destination_domain)
         );
     }

rust/agents/relayer/src/settings/matching_list.rs

@@ -1,4 +1,4 @@
-//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape
+//! The correct settings shape is defined in the TypeScript SDK metadata. While the exact shape
 //! and validations it defines are not applied here, we should mirror them.
 //! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK.
 
@@ -267,13 +267,13 @@ impl<'a> From<&'a HyperlaneMessage> for MatchInfo<'a> {
 
 impl MatchingList {
     /// Check if a message matches any of the rules.
-    /// - `default`: What to return if the the matching list is empty.
+    /// - `default`: What to return if the matching list is empty.
     pub fn msg_matches(&self, msg: &HyperlaneMessage, default: bool) -> bool {
         self.matches(msg.into(), default)
     }
 
     /// Check if a message matches any of the rules.
-    /// - `default`: What to return if the the matching list is empty.
+    /// - `default`: What to return if the matching list is empty.
     fn matches(&self, info: MatchInfo, default: bool) -> bool {
         if let Some(rules) = &self.0 {
             matches_any_rule(rules.iter(), info)

rust/agents/relayer/src/settings/mod.rs

@@ -1,6 +1,6 @@
 //! Relayer configuration
 //!
-//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape
+//! The correct settings shape is defined in the TypeScript SDK metadata. While the exact shape
 //! and validations it defines are not applied here, we should mirror them.
 //! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK.
 

rust/agents/scraper/src/agent.rs

@@ -5,10 +5,13 @@ use derive_more::AsRef;
 use futures::future::try_join_all;
 use hyperlane_base::{
     metrics::AgentMetrics, settings::IndexSettings, BaseAgent, ChainMetrics, ContractSyncMetrics,
-    ContractSyncer, CoreMetrics, HyperlaneAgentCore, MetricsUpdater,
+    ContractSyncer, CoreMetrics, HyperlaneAgentCore, MetricsUpdater, SyncOptions,
+};
+use hyperlane_core::{Delivery, HyperlaneDomain, HyperlaneMessage, InterchainGasPayment, H512};
+use tokio::{
+    sync::broadcast::{Receiver, Sender},
+    task::JoinHandle,
 };
-use hyperlane_core::{Delivery, HyperlaneDomain, HyperlaneMessage, InterchainGasPayment};
-use tokio::task::JoinHandle;
 use tracing::{info_span, instrument::Instrumented, trace, Instrument};
 
 use crate::{chain_scraper::HyperlaneSqlDb, db::ScraperDb, settings::ScraperSettings};
@@ -135,16 +138,16 @@ impl Scraper {
         let domain = scraper.domain.clone();
 
         let mut tasks = Vec::with_capacity(2);
-        tasks.push(
-            self.build_message_indexer(
+        let (message_indexer, maybe_broadcaster) = self
+            .build_message_indexer(
                 domain.clone(),
                 self.core_metrics.clone(),
                 self.contract_sync_metrics.clone(),
                 db.clone(),
                 index_settings.clone(),
             )
-            .await,
-        );
+            .await;
+        tasks.push(message_indexer);
         tasks.push(
             self.build_delivery_indexer(
                 domain.clone(),
@@ -152,6 +155,7 @@ impl Scraper {
                 self.contract_sync_metrics.clone(),
                 db.clone(),
                 index_settings.clone(),
+                maybe_broadcaster.clone().map(|b| b.subscribe()),
             )
             .await,
         );
@@ -162,6 +166,7 @@ impl Scraper {
                 self.contract_sync_metrics.clone(),
                 db,
                 index_settings.clone(),
+                maybe_broadcaster.map(|b| b.subscribe()),
             )
             .await,
         );
@@ -182,7 +187,7 @@ impl Scraper {
         contract_sync_metrics: Arc<ContractSyncMetrics>,
         db: HyperlaneSqlDb,
         index_settings: IndexSettings,
-    ) -> Instrumented<JoinHandle<()>> {
+    ) -> (Instrumented<JoinHandle<()>>, Option<Sender<H512>>) {
         let sync = self
             .as_ref()
             .settings
@@ -195,9 +200,12 @@ impl Scraper {
             .await
             .unwrap();
         let cursor = sync.cursor(index_settings.clone()).await;
-        tokio::spawn(async move { sync.sync("message_dispatch", cursor).await }).instrument(
-            info_span!("ChainContractSync", chain=%domain.name(), event="message_dispatch"),
-        )
+        let maybe_broadcaser = sync.get_broadcaster();
+        let task = tokio::spawn(async move { sync.sync("message_dispatch", cursor.into()).await })
+            .instrument(
+                info_span!("ChainContractSync", chain=%domain.name(), event="message_dispatch"),
+            );
+        (task, maybe_broadcaser)
     }
 
     async fn build_delivery_indexer(
@@ -207,6 +215,7 @@ impl Scraper {
         contract_sync_metrics: Arc<ContractSyncMetrics>,
         db: HyperlaneSqlDb,
         index_settings: IndexSettings,
+        tx_id_receiver: Option<Receiver<H512>>,
     ) -> Instrumented<JoinHandle<()>> {
         let sync = self
             .as_ref()
@@ -222,8 +231,11 @@ impl Scraper {
 
         let label = "message_delivery";
         let cursor = sync.cursor(index_settings.clone()).await;
-        tokio::spawn(async move { sync.sync(label, cursor).await })
-            .instrument(info_span!("ChainContractSync", chain=%domain.name(), event=label))
+        tokio::spawn(async move {
+            sync.sync(label, SyncOptions::new(Some(cursor), tx_id_receiver))
+                .await
+        })
+        .instrument(info_span!("ChainContractSync", chain=%domain.name(), event=label))
     }
 
     async fn build_interchain_gas_payment_indexer(
@@ -233,6 +245,7 @@ impl Scraper {
         contract_sync_metrics: Arc<ContractSyncMetrics>,
         db: HyperlaneSqlDb,
         index_settings: IndexSettings,
+        tx_id_receiver: Option<Receiver<H512>>,
     ) -> Instrumented<JoinHandle<()>> {
         let sync = self
             .as_ref()
@@ -248,7 +261,10 @@ impl Scraper {
 
         let label = "gas_payment";
         let cursor = sync.cursor(index_settings.clone()).await;
-        tokio::spawn(async move { sync.sync(label, cursor).await })
-            .instrument(info_span!("ChainContractSync", chain=%domain.name(), event=label))
+        tokio::spawn(async move {
+            sync.sync(label, SyncOptions::new(Some(cursor), tx_id_receiver))
+                .await
+        })
+        .instrument(info_span!("ChainContractSync", chain=%domain.name(), event=label))
     }
 }

rust/agents/scraper/src/settings.rs

@@ -1,6 +1,6 @@
 //! Scraper configuration.
 //!
-//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape
+//! The correct settings shape is defined in the TypeScript SDK metadata. While the exact shape
 //! and validations it defines are not applied here, we should mirror them.
 //! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK.
 

rust/agents/validator/src/settings.rs

@@ -1,6 +1,6 @@
 //! Validator configuration.
 //!
-//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape
+//! The correct settings shape is defined in the TypeScript SDK metadata. While the exact shape
 //! and validations it defines are not applied here, we should mirror them.
 //! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK.
 

rust/agents/validator/src/validator.rs

@@ -210,7 +210,10 @@ impl Validator {
         let contract_sync = self.merkle_tree_hook_sync.clone();
         let cursor = contract_sync.cursor(index_settings).await;
         tokio::spawn(async move {
-            contract_sync.clone().sync("merkle_tree_hook", cursor).await;
+            contract_sync
+                .clone()
+                .sync("merkle_tree_hook", cursor.into())
+                .await;
         })
         .instrument(info_span!("MerkleTreeHookSyncer"))
     }

rust/chains/hyperlane-cosmos/src/interchain_gas.rs

@@ -202,7 +202,7 @@ impl CosmosInterchainGasPaymasterIndexer {
 
 #[async_trait]
 impl Indexer<InterchainGasPayment> for CosmosInterchainGasPaymasterIndexer {
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<InterchainGasPayment>, LogMeta)>> {

rust/chains/hyperlane-cosmos/src/mailbox.rs

@@ -350,7 +350,7 @@ impl CosmosMailboxIndexer {
 
 #[async_trait]
 impl Indexer<HyperlaneMessage> for CosmosMailboxIndexer {
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<HyperlaneMessage>, LogMeta)>> {
@@ -397,7 +397,7 @@ impl Indexer<HyperlaneMessage> for CosmosMailboxIndexer {
 
 #[async_trait]
 impl Indexer<H256> for CosmosMailboxIndexer {
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<H256>, LogMeta)>> {

rust/chains/hyperlane-cosmos/src/merkle_tree_hook.rs

@@ -283,7 +283,7 @@ impl CosmosMerkleTreeHookIndexer {
 #[async_trait]
 impl Indexer<MerkleTreeInsertion> for CosmosMerkleTreeHookIndexer {
     /// Fetch list of logs between `range` of blocks
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<MerkleTreeInsertion>, LogMeta)>> {

rust/chains/hyperlane-ethereum/src/contracts/interchain_gas.rs

@@ -10,12 +10,14 @@ use ethers::prelude::Middleware;
 use hyperlane_core::{
     ChainCommunicationError, ChainResult, ContractLocator, HyperlaneAbi, HyperlaneChain,
     HyperlaneContract, HyperlaneDomain, HyperlaneProvider, Indexed, Indexer,
-    InterchainGasPaymaster, InterchainGasPayment, LogMeta, SequenceAwareIndexer, H160, H256,
+    InterchainGasPaymaster, InterchainGasPayment, LogMeta, SequenceAwareIndexer, H160, H256, H512,
 };
 use tracing::instrument;
 
+use super::utils::fetch_raw_logs_and_log_meta;
 use crate::interfaces::i_interchain_gas_paymaster::{
-    IInterchainGasPaymaster as EthereumInterchainGasPaymasterInternal, IINTERCHAINGASPAYMASTER_ABI,
+    GasPaymentFilter, IInterchainGasPaymaster as EthereumInterchainGasPaymasterInternal,
+    IINTERCHAINGASPAYMASTER_ABI,
 };
 use crate::{BuildableWithProvider, ConnectionConf, EthereumProvider};
 
@@ -86,7 +88,7 @@ where
 {
     /// Note: This call may return duplicates depending on the provider used
     #[instrument(err, skip(self))]
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<InterchainGasPayment>, LogMeta)>> {
@@ -124,6 +126,32 @@ where
             .as_u32()
             .saturating_sub(self.reorg_period))
     }
+
+    async fn fetch_logs_by_tx_hash(
+        &self,
+        tx_hash: H512,
+    ) -> ChainResult<Vec<(Indexed<InterchainGasPayment>, LogMeta)>> {
+        let logs = fetch_raw_logs_and_log_meta::<GasPaymentFilter, M>(
+            tx_hash,
+            self.provider.clone(),
+            self.contract.address(),
+        )
+        .await?
+        .into_iter()
+        .map(|(log, log_meta)| {
+            (
+                Indexed::new(InterchainGasPayment {
+                    message_id: H256::from(log.message_id),
+                    destination: log.destination_domain,
+                    payment: log.payment.into(),
+                    gas_amount: log.gas_amount.into(),
+                }),
+                log_meta,
+            )
+        })
+        .collect();
+        Ok(logs)
+    }
 }
 
 #[async_trait]

rust/chains/hyperlane-ethereum/src/contracts/mailbox.rs

@@ -11,6 +11,7 @@ use ethers::abi::{AbiEncode, Detokenize};
 use ethers::prelude::Middleware;
 use ethers_contract::builders::ContractCall;
 use futures_util::future::join_all;
+use hyperlane_core::H512;
 use tracing::instrument;
 
 use hyperlane_core::{
@@ -25,10 +26,12 @@ use crate::interfaces::arbitrum_node_interface::ArbitrumNodeInterface;
 use crate::interfaces::i_mailbox::{
     IMailbox as EthereumMailboxInternal, ProcessCall, IMAILBOX_ABI,
 };
+use crate::interfaces::mailbox::DispatchFilter;
 use crate::tx::{call_with_lag, fill_tx_gas_params, report_tx};
 use crate::{BuildableWithProvider, ConnectionConf, EthereumProvider, TransactionOverrides};
 
 use super::multicall::{self, build_multicall};
+use super::utils::fetch_raw_logs_and_log_meta;
 
 impl<M> std::fmt::Display for EthereumMailboxInternal<M>
 where
@@ -134,7 +137,7 @@ where
 
     /// Note: This call may return duplicates depending on the provider used
     #[instrument(err, skip(self))]
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<HyperlaneMessage>, LogMeta)>> {
@@ -157,6 +160,27 @@ where
         events.sort_by(|a, b| a.0.inner().nonce.cmp(&b.0.inner().nonce));
         Ok(events)
     }
+
+    async fn fetch_logs_by_tx_hash(
+        &self,
+        tx_hash: H512,
+    ) -> ChainResult<Vec<(Indexed<HyperlaneMessage>, LogMeta)>> {
+        let logs = fetch_raw_logs_and_log_meta::<DispatchFilter, M>(
+            tx_hash,
+            self.provider.clone(),
+            self.contract.address(),
+        )
+        .await?
+        .into_iter()
+        .map(|(log, log_meta)| {
+            (
+                HyperlaneMessage::from(log.message.to_vec()).into(),
+                log_meta,
+            )
+        })
+        .collect();
+        Ok(logs)
+    }
 }
 
 #[async_trait]
@@ -183,7 +207,7 @@ where
 
     /// Note: This call may return duplicates depending on the provider used
     #[instrument(err, skip(self))]
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<H256>, LogMeta)>> {

rust/chains/hyperlane-ethereum/src/contracts/merkle_tree_hook.rs

@@ -11,13 +11,17 @@ use tracing::instrument;
 use hyperlane_core::{
     ChainCommunicationError, ChainResult, Checkpoint, ContractLocator, HyperlaneChain,
     HyperlaneContract, HyperlaneDomain, HyperlaneProvider, Indexed, Indexer, LogMeta,
-    MerkleTreeHook, MerkleTreeInsertion, SequenceAwareIndexer, H256,
+    MerkleTreeHook, MerkleTreeInsertion, SequenceAwareIndexer, H256, H512,
 };
 
-use crate::interfaces::merkle_tree_hook::{MerkleTreeHook as MerkleTreeHookContract, Tree};
+use crate::interfaces::merkle_tree_hook::{
+    InsertedIntoTreeFilter, MerkleTreeHook as MerkleTreeHookContract, Tree,
+};
 use crate::tx::call_with_lag;
 use crate::{BuildableWithProvider, ConnectionConf, EthereumProvider};
 
+use super::utils::fetch_raw_logs_and_log_meta;
+
 // We don't need the reverse of this impl, so it's ok to disable the clippy lint
 #[allow(clippy::from_over_into)]
 impl Into<IncrementalMerkle> for Tree {
@@ -108,7 +112,7 @@ where
 {
     /// Note: This call may return duplicates depending on the provider used
     #[instrument(err, skip(self))]
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<MerkleTreeInsertion>, LogMeta)>> {
@@ -142,6 +146,27 @@ where
             .as_u32()
             .saturating_sub(self.reorg_period))
     }
+
+    async fn fetch_logs_by_tx_hash(
+        &self,
+        tx_hash: H512,
+    ) -> ChainResult<Vec<(Indexed<MerkleTreeInsertion>, LogMeta)>> {
+        let logs = fetch_raw_logs_and_log_meta::<InsertedIntoTreeFilter, M>(
+            tx_hash,
+            self.provider.clone(),
+            self.contract.address(),
+        )
+        .await?
+        .into_iter()
+        .map(|(log, log_meta)| {
+            (
+                MerkleTreeInsertion::new(log.index, H256::from(log.message_id)).into(),
+                log_meta,
+            )
+        })
+        .collect();
+        Ok(logs)
+    }
 }
 
 #[async_trait]

rust/chains/hyperlane-ethereum/src/contracts/mod.rs

@@ -1,11 +1,8 @@
 pub use {interchain_gas::*, mailbox::*, merkle_tree_hook::*, validator_announce::*};
 
 mod interchain_gas;
-
 mod mailbox;
-
 mod merkle_tree_hook;
-
 mod multicall;
-
+mod utils;
 mod validator_announce;

rust/chains/hyperlane-ethereum/src/contracts/utils.rs

@@ -0,0 +1,48 @@
+use std::sync::Arc;
+
+use ethers::{
+    abi::RawLog,
+    providers::Middleware,
+    types::{H160 as EthersH160, H256 as EthersH256},
+};
+use ethers_contract::{ContractError, EthEvent, LogMeta as EthersLogMeta};
+use hyperlane_core::{ChainResult, LogMeta, H512};
+use tracing::warn;
+
+pub async fn fetch_raw_logs_and_log_meta<T: EthEvent, M>(
+    tx_hash: H512,
+    provider: Arc<M>,
+    contract_address: EthersH160,
+) -> ChainResult<Vec<(T, LogMeta)>>
+where
+    M: Middleware + 'static,
+{
+    let ethers_tx_hash: EthersH256 = tx_hash.into();
+    let receipt = provider
+        .get_transaction_receipt(ethers_tx_hash)
+        .await
+        .map_err(|err| ContractError::<M>::MiddlewareError(err))?;
+    let Some(receipt) = receipt else {
+        warn!(%tx_hash, "No receipt found for tx hash");
+        return Ok(vec![]);
+    };
+
+    let logs: Vec<(T, LogMeta)> = receipt
+        .logs
+        .into_iter()
+        .filter_map(|log| {
+            // Filter out logs that aren't emitted by this contract
+            if log.address != contract_address {
+                return None;
+            }
+            let raw_log = RawLog {
+                topics: log.topics.clone(),
+                data: log.data.to_vec(),
+            };
+            let log_meta: EthersLogMeta = (&log).into();
+            let event_filter = T::decode_log(&raw_log).ok();
+            event_filter.map(|log| (log, log_meta.into()))
+        })
+        .collect();
+    Ok(logs)
+}

rust/chains/hyperlane-fuel/src/interchain_gas.rs

@@ -35,7 +35,7 @@ pub struct FuelInterchainGasPaymasterIndexer {}
 
 #[async_trait]
 impl Indexer<InterchainGasPayment> for FuelInterchainGasPaymasterIndexer {
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<InterchainGasPayment>, LogMeta)>> {

rust/chains/hyperlane-fuel/src/mailbox.rs

@@ -126,7 +126,7 @@ pub struct FuelMailboxIndexer {}
 
 #[async_trait]
 impl Indexer<HyperlaneMessage> for FuelMailboxIndexer {
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<HyperlaneMessage>, LogMeta)>> {
@@ -140,7 +140,7 @@ impl Indexer<HyperlaneMessage> for FuelMailboxIndexer {
 
 #[async_trait]
 impl Indexer<H256> for FuelMailboxIndexer {
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<H256>, LogMeta)>> {

rust/chains/hyperlane-sealevel/src/interchain_gas.rs

@@ -246,7 +246,7 @@ impl SealevelInterchainGasPaymasterIndexer {
 #[async_trait]
 impl Indexer<InterchainGasPayment> for SealevelInterchainGasPaymasterIndexer {
     #[instrument(err, skip(self))]
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<InterchainGasPayment>, LogMeta)>> {

rust/chains/hyperlane-sealevel/src/mailbox.rs

@@ -646,7 +646,7 @@ impl SequenceAwareIndexer<HyperlaneMessage> for SealevelMailboxIndexer {
 
 #[async_trait]
 impl Indexer<HyperlaneMessage> for SealevelMailboxIndexer {
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<HyperlaneMessage>, LogMeta)>> {
@@ -670,7 +670,7 @@ impl Indexer<HyperlaneMessage> for SealevelMailboxIndexer {
 
 #[async_trait]
 impl Indexer<H256> for SealevelMailboxIndexer {
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         _range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<H256>, LogMeta)>> {

rust/chains/hyperlane-sealevel/src/merkle_tree_hook.rs

@@ -83,11 +83,11 @@ pub struct SealevelMerkleTreeHookIndexer(SealevelMailboxIndexer);
 
 #[async_trait]
 impl Indexer<MerkleTreeInsertion> for SealevelMerkleTreeHookIndexer {
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<MerkleTreeInsertion>, LogMeta)>> {
-        let messages = Indexer::<HyperlaneMessage>::fetch_logs(&self.0, range).await?;
+        let messages = Indexer::<HyperlaneMessage>::fetch_logs_in_range(&self.0, range).await?;
         let merkle_tree_insertions = messages
             .into_iter()
             .map(|(m, meta)| (message_to_merkle_tree_insertion(m.inner()).into(), meta))

rust/config/mainnet_config.json

@@ -634,13 +634,19 @@
     },
     "injective": {
       "bech32Prefix": "inj",
+      "blockExplorers": [
+      ],
       "blocks": {
+        "confirmations": 1,
+        "estimateBlockTime": 1,
         "reorgPeriod": 10
       },
       "canonicalAsset": "inj",
       "chainId": "injective-1",
       "contractAddressBytes": 20,
-      "domainId": "6909546",
+      "displayName": "Injective",
+      "domainId": 6909546,
+      "gasCurrencyCoinGeckoId": "injective-protocol",
       "gasPrice": {
         "amount": "700000000",
         "denom": "inj"
@@ -658,12 +664,24 @@
       "mailbox": "0x0f7fb53961d70687e352aa55cb329ca76edc0c19",
       "merkleTreeHook": "0x568ad3638447f07def384969f4ea39fae3802962",
       "name": "injective",
+      "nativeToken": {
+        "decimals": 18,
+        "denom": "inj",
+        "name": "Injective",
+        "symbol": "INJ"
+      },
       "protocol": "cosmos",
+      "restUrls": [
+        {
+          "http": "https://sentry.lcd.injective.network:443"
+        }
+      ],
       "rpcUrls": [
         {
-          "http": "https://injective-rpc.polkachu.com"
+          "http": "https://sentry.tm.injective.network:443"
         }
       ],
+      "slip44": 118,
       "validatorAnnounce": "0x1fb225b2fcfbe75e614a1d627de97ff372242eed"
     },
     "mantapacific": {
@@ -837,13 +855,25 @@
     },
     "neutron": {
       "bech32Prefix": "neutron",
+      "blockExplorers": [
+        {
+          "apiUrl": "https://www.mintscan.io/neutron",
+          "family": "other",
+          "name": "Mintscan",
+          "url": "https://www.mintscan.io/neutron"
+        }
+      ],
       "blocks": {
+        "confirmations": 1,
+        "estimateBlockTime": 3,
         "reorgPeriod": 1
       },
       "canonicalAsset": "untrn",
       "chainId": "neutron-1",
       "contractAddressBytes": 32,
-      "domainId": "1853125230",
+      "displayName": "Neutron",
+      "domainId": 1853125230,
+      "gasCurrencyCoinGeckoId": "neutron-3",
       "gasPrice": {
         "amount": "0.0053",
         "denom": "untrn"
@@ -858,10 +888,22 @@
         "from": 4000000
       },
       "interchainGasPaymaster": "0x504ee9ac43ec5814e00c7d21869a90ec52becb489636bdf893b7df9d606b5d67",
+      "isTestnet": false,
       "mailbox": "0x848426d50eb2104d5c6381ec63757930b1c14659c40db8b8081e516e7c5238fc",
       "merkleTreeHook": "0xcd30a0001cc1f436c41ef764a712ebabc5a144140e3fd03eafe64a9a24e4e27c",
       "name": "neutron",
+      "nativeToken": {
+        "decimals": 6,
+        "denom": "untrn",
+        "name": "Neutron",
+        "symbol": "NTRN"
+      },
       "protocol": "cosmos",
+      "restUrls": [
+        {
+          "http": "https://rest-lb.neutron.org"
+        }
+      ],
       "rpcUrls": [
         {
           "http": "https://rpc-kralum.neutron-1.neutron.org"
@@ -872,6 +914,7 @@
         "prefix": "neutron",
         "type": "cosmosKey"
       },
+      "slip44": 118,
       "validatorAnnounce": "0xf3aa0d652226e21ae35cd9035c492ae41725edc9036edf0d6a48701b153b90a0"
     },
     "optimism": {
@@ -998,8 +1041,8 @@
       "name": "polygon",
       "nativeToken": {
         "decimals": 18,
-        "name": "Ether",
-        "symbol": "ETH"
+        "name": "Matic",
+        "symbol": "MATIC"
       },
       "pausableHook": "0x748040afB89B8FdBb992799808215419d36A0930",
       "pausableIsm": "0x6741e91fFDC31c7786E3684427c628dad06299B0",

rust/config/testnet_config.json

@@ -207,6 +207,56 @@
       "timelockController": "0x0000000000000000000000000000000000000000",
       "validatorAnnounce": "0x4f7179A691F8a684f56cF7Fed65171877d30739a"
     },
+    "holesky": {
+      "blockExplorers": [
+        {
+          "apiUrl": "https://api-holesky.etherscan.io/api",
+          "family": "etherscan",
+          "name": "Etherscan",
+          "url": "https://holesky.etherscan.io"
+        }
+      ],
+      "blocks": {
+        "confirmations": 1,
+        "estimateBlockTime": 13,
+        "reorgPeriod": 2
+      },
+      "chainId": 17000,
+      "displayName": "Holesky",
+      "domainId": 17000,
+      "domainRoutingIsmFactory": "0xDDcFEcF17586D08A5740B7D91735fcCE3dfe3eeD",
+      "fallbackRoutingHook": "0x07009DA2249c388aD0f416a235AfE90D784e1aAc",
+      "index": {
+        "from": 1543015
+      },
+      "interchainGasPaymaster": "0x5CBf4e70448Ed46c2616b04e9ebc72D29FF0cfA9",
+      "interchainSecurityModule": "0x751f2b684EeBb916dB777767CCb8fd793C8b2956",
+      "isTestnet": true,
+      "mailbox": "0x46f7C5D896bbeC89bE1B19e4485e59b4Be49e9Cc",
+      "merkleTreeHook": "0x98AAE089CaD930C64a76dD2247a2aC5773a4B8cE",
+      "name": "holesky",
+      "nativeToken": {
+        "decimals": 18,
+        "name": "Ether",
+        "symbol": "ETH"
+      },
+      "pausableHook": "0xF7561c34f17A32D5620583A3397C304e7038a7F6",
+      "protocol": "ethereum",
+      "protocolFee": "0x6b1bb4ce664Bb4164AEB4d3D2E7DE7450DD8084C",
+      "proxyAdmin": "0x33dB966328Ea213b0f76eF96CA368AB37779F065",
+      "rpcUrls": [
+        {
+          "http": "https://ethereum-holesky-rpc.publicnode.com"
+        }
+      ],
+      "staticAggregationHookFactory": "0x589C201a07c26b4725A4A829d772f24423da480B",
+      "staticAggregationIsmFactory": "0x54148470292C24345fb828B003461a9444414517",
+      "staticMerkleRootMultisigIsmFactory": "0xC2E36cd6e32e194EE11f15D9273B64461A4D49A2",
+      "staticMessageIdMultisigIsmFactory": "0x6966b0E55883d49BFB24539356a2f8A673E02039",
+      "storageGasOracle": "0x2b2a158B4059C840c7aC67399B153bb567D06303",
+      "testRecipient": "0x86fb9F1c124fB20ff130C41a79a432F770f67AFD",
+      "validatorAnnounce": "0xAb9B273366D794B7F80B4378bc8Aaca75C6178E2"
+    },
     "plumetestnet": {
       "aggregationHook": "0x31dF0EEE7Dc7565665468698a0da221225619a1B",
       "blockExplorers": [

rust/hyperlane-base/src/contract_sync/cursors/mod.rs

@@ -13,8 +13,18 @@ pub enum CursorType {
     RateLimited,
 }
 
+// H256 * 1M = 32MB per origin chain worst case
+// With one such channel per origin chain.
+const TX_ID_CHANNEL_CAPACITY: Option<usize> = Some(1_000_000);
+
 pub trait Indexable {
+    /// Returns the configured cursor type of this type for the given domain, (e.g. `SequenceAware` or `RateLimited`)
     fn indexing_cursor(domain: HyperlaneDomainProtocol) -> CursorType;
+    /// Indexing tasks may have channels open between them to share information that improves reliability (such as the txid where a message event was indexed).
+    /// By default this method is None, and it should return a channel capacity if this indexing task is to broadcast anything to other tasks.
+    fn broadcast_channel_size() -> Option<usize> {
+        None
+    }
 }
 
 impl Indexable for HyperlaneMessage {
@@ -26,6 +36,11 @@ impl Indexable for HyperlaneMessage {
             HyperlaneDomainProtocol::Cosmos => CursorType::SequenceAware,
         }
     }
+
+    // Only broadcast txids from the message indexing task
+    fn broadcast_channel_size() -> Option<usize> {
+        TX_ID_CHANNEL_CAPACITY
+    }
 }
 
 impl Indexable for InterchainGasPayment {

rust/hyperlane-base/src/contract_sync/cursors/rate_limited.rs

@@ -216,6 +216,16 @@ where
     }
 }
 
+impl<T> Debug for RateLimitedContractSyncCursor<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RateLimitedContractSyncCursor")
+            .field("tip", &self.tip)
+            .field("last_tip_update", &self.last_tip_update)
+            .field("sync_state", &self.sync_state)
+            .finish()
+    }
+}
+
 #[cfg(test)]
 pub(crate) mod test {
     use super::*;
@@ -234,7 +244,7 @@ pub(crate) mod test {
 
         #[async_trait]
         impl Indexer<()> for Indexer {
-            async fn fetch_logs(&self, range: RangeInclusive<u32>) -> ChainResult<Vec<(hyperlane_core::Indexed<()> , LogMeta)>>;
+            async fn fetch_logs_in_range(&self, range: RangeInclusive<u32>) -> ChainResult<Vec<(hyperlane_core::Indexed<()> , LogMeta)>>;
             async fn get_finalized_block_number(&self) -> ChainResult<u32>;
         }
     }

rust/hyperlane-base/src/contract_sync/cursors/sequence_aware/backward.rs

@@ -9,10 +9,13 @@ use hyperlane_core::{
     HyperlaneSequenceAwareIndexerStoreReader, IndexMode, Indexed, LogMeta, SequenceIndexed,
 };
 use itertools::Itertools;
+use tokio::time::sleep;
 use tracing::{debug, instrument, warn};
 
 use super::{LastIndexedSnapshot, TargetSnapshot};
 
+const MAX_BACKWARD_SYNC_BLOCKING_TIME: Duration = Duration::from_secs(5);
+
 /// A sequence-aware cursor that syncs backward until there are no earlier logs to index.
 pub(crate) struct BackwardSequenceAwareSyncCursor<T> {
     /// The max chunk size to query for logs.
@@ -32,6 +35,17 @@ pub(crate) struct BackwardSequenceAwareSyncCursor<T> {
     index_mode: IndexMode,
 }
 
+impl<T> Debug for BackwardSequenceAwareSyncCursor<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("BackwardSequenceAwareSyncCursor")
+            .field("chunk_size", &self.chunk_size)
+            .field("last_indexed_snapshot", &self.last_indexed_snapshot)
+            .field("current_indexing_snapshot", &self.current_indexing_snapshot)
+            .field("index_mode", &self.index_mode)
+            .finish()
+    }
+}
+
 impl<T: Debug> BackwardSequenceAwareSyncCursor<T> {
     #[instrument(
         skip(db),
@@ -68,7 +82,11 @@ impl<T: Debug> BackwardSequenceAwareSyncCursor<T> {
     #[instrument(ret)]
     pub async fn get_next_range(&mut self) -> Result<Option<RangeInclusive<u32>>> {
         // Skip any already indexed logs.
-        self.skip_indexed().await?;
+        tokio::select! {
+            res = self.skip_indexed() => res?,
+            // return early to allow the forward cursor to also make progress
+            _ = sleep(MAX_BACKWARD_SYNC_BLOCKING_TIME) => { return Ok(None); }
+        };
 
         // If `self.current_indexing_snapshot` is None, we are synced and there are no more ranges to query.
         // Otherwise, we query the next range, searching for logs prior to and including the current indexing snapshot.
@@ -309,17 +327,6 @@ impl<T: Debug> BackwardSequenceAwareSyncCursor<T> {
     }
 }
 
-impl<T: Debug> Debug for BackwardSequenceAwareSyncCursor<T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("BackwardSequenceAwareSyncCursor")
-            .field("chunk_size", &self.chunk_size)
-            .field("current_indexing_snapshot", &self.current_indexing_snapshot)
-            .field("last_indexed_snapshot", &self.last_indexed_snapshot)
-            .field("index_mode", &self.index_mode)
-            .finish()
-    }
-}
-
 #[async_trait]
 impl<T: Send + Sync + Clone + Debug + 'static> ContractSyncCursor<T>
     for BackwardSequenceAwareSyncCursor<T>

rust/hyperlane-base/src/contract_sync/cursors/sequence_aware/forward.rs

@@ -41,6 +41,18 @@ pub(crate) struct ForwardSequenceAwareSyncCursor<T> {
     index_mode: IndexMode,
 }
 
+impl<T> Debug for ForwardSequenceAwareSyncCursor<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ForwardSequenceAwareSyncCursor")
+            .field("chunk_size", &self.chunk_size)
+            .field("last_indexed_snapshot", &self.last_indexed_snapshot)
+            .field("current_indexing_snapshot", &self.current_indexing_snapshot)
+            .field("target_snapshot", &self.target_snapshot)
+            .field("index_mode", &self.index_mode)
+            .finish()
+    }
+}
+
 impl<T: Debug> ForwardSequenceAwareSyncCursor<T> {
     #[instrument(
         skip(db, latest_sequence_querier),
@@ -391,18 +403,6 @@ impl<T: Debug> ForwardSequenceAwareSyncCursor<T> {
     }
 }
 
-impl<T: Debug> Debug for ForwardSequenceAwareSyncCursor<T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("ForwardSequenceAwareSyncCursor")
-            .field("chunk_size", &self.chunk_size)
-            .field("current_indexing_snapshot", &self.current_indexing_snapshot)
-            .field("last_indexed_snapshot", &self.last_indexed_snapshot)
-            .field("target_snapshot", &self.target_snapshot)
-            .field("index_mode", &self.index_mode)
-            .finish()
-    }
-}
-
 #[async_trait]
 impl<T: Send + Sync + Clone + Debug + 'static> ContractSyncCursor<T>
     for ForwardSequenceAwareSyncCursor<T>
@@ -493,7 +493,7 @@ pub(crate) mod test {
     where
         T: Sequenced + Debug,
     {
-        async fn fetch_logs(
+        async fn fetch_logs_in_range(
             &self,
             _range: RangeInclusive<u32>,
         ) -> ChainResult<Vec<(Indexed<T>, LogMeta)>> {

rust/hyperlane-base/src/contract_sync/cursors/sequence_aware/mod.rs

@@ -62,6 +62,7 @@ pub enum SyncDirection {
 
 /// A cursor that prefers to sync forward, but will sync backward if there is nothing to
 /// sync forward.
+#[derive(Debug)]
 pub(crate) struct ForwardBackwardSequenceAwareSyncCursor<T> {
     forward: ForwardSequenceAwareSyncCursor<T>,
     backward: BackwardSequenceAwareSyncCursor<T>,

rust/hyperlane-base/src/contract_sync/mod.rs

@@ -10,9 +10,13 @@ use hyperlane_core::{
     HyperlaneSequenceAwareIndexerStore, HyperlaneWatermarkedLogStore, Indexer,
     SequenceAwareIndexer,
 };
+use hyperlane_core::{Indexed, LogMeta, H512};
 pub use metrics::ContractSyncMetrics;
+use prometheus::core::{AtomicI64, AtomicU64, GenericCounter, GenericGauge};
+use tokio::sync::broadcast::error::TryRecvError;
+use tokio::sync::broadcast::{Receiver as BroadcastReceiver, Sender as BroadcastSender};
 use tokio::time::sleep;
-use tracing::{debug, info, warn};
+use tracing::{debug, info, instrument, trace, warn};
 
 use crate::settings::IndexSettings;
 
@@ -27,17 +31,33 @@ const SLEEP_DURATION: Duration = Duration::from_secs(5);
 /// Entity that drives the syncing of an agent's db with on-chain data.
 /// Extracts chain-specific data (emitted checkpoints, messages, etc) from an
 /// `indexer` and fills the agent's db with this data.
-#[derive(Debug, new, Clone)]
-pub struct ContractSync<T, D: HyperlaneLogStore<T>, I: Indexer<T>> {
+#[derive(Debug)]
+pub struct ContractSync<T: Indexable, D: HyperlaneLogStore<T>, I: Indexer<T>> {
     domain: HyperlaneDomain,
     db: D,
     indexer: I,
     metrics: ContractSyncMetrics,
+    broadcast_sender: Option<BroadcastSender<H512>>,
     _phantom: PhantomData<T>,
 }
 
+impl<T: Indexable, D: HyperlaneLogStore<T>, I: Indexer<T>> ContractSync<T, D, I> {
+    /// Create a new ContractSync
+    pub fn new(domain: HyperlaneDomain, db: D, indexer: I, metrics: ContractSyncMetrics) -> Self {
+        Self {
+            domain,
+            db,
+            indexer,
+            metrics,
+            broadcast_sender: T::broadcast_channel_size().map(BroadcastSender::new),
+            _phantom: PhantomData,
+        }
+    }
+}
+
 impl<T, D, I> ContractSync<T, D, I>
 where
+    T: Indexable + Debug + Send + Sync + Clone + Eq + Hash + 'static,
     D: HyperlaneLogStore<T>,
     I: Indexer<T> + 'static,
 {
@@ -45,82 +65,161 @@ where
     pub fn domain(&self) -> &HyperlaneDomain {
         &self.domain
     }
-}
 
-impl<T, D, I> ContractSync<T, D, I>
-where
-    T: Debug + Send + Sync + Clone + Eq + Hash + 'static,
-    D: HyperlaneLogStore<T>,
-    I: Indexer<T> + 'static,
-{
+    fn get_broadcaster(&self) -> Option<BroadcastSender<H512>> {
+        self.broadcast_sender.clone()
+    }
+
     /// Sync logs and write them to the LogStore
-    #[tracing::instrument(name = "ContractSync", fields(domain=self.domain().name()), skip(self, cursor))]
-    pub async fn sync(&self, label: &'static str, mut cursor: Box<dyn ContractSyncCursor<T>>) {
+    #[instrument(name = "ContractSync", fields(domain=self.domain().name()), skip(self, opts))]
+    pub async fn sync(&self, label: &'static str, mut opts: SyncOptions<T>) {
         let chain_name = self.domain.as_ref();
-        let indexed_height = self
+        let indexed_height_metric = self
             .metrics
             .indexed_height
             .with_label_values(&[label, chain_name]);
-        let stored_logs = self
+        let stored_logs_metric = self
             .metrics
             .stored_events
             .with_label_values(&[label, chain_name]);
 
         loop {
-            indexed_height.set(cursor.latest_queried_block() as i64);
+            if let Some(rx) = opts.tx_id_receiver.as_mut() {
+                self.fetch_logs_from_receiver(rx, &stored_logs_metric).await;
+            }
+            if let Some(cursor) = opts.cursor.as_mut() {
+                self.fetch_logs_with_cursor(cursor, &stored_logs_metric, &indexed_height_metric)
+                    .await;
+            }
+        }
+    }
 
-            let (action, eta) = match cursor.next_action().await {
-                Ok((action, eta)) => (action, eta),
-                Err(err) => {
-                    warn!(?err, "Error getting next action");
-                    sleep(SLEEP_DURATION).await;
-                    continue;
-                }
-            };
-            let sleep_duration = match action {
-                // Use `loop` but always break - this allows for returning a value
-                // from the loop (the sleep duration)
-                #[allow(clippy::never_loop)]
-                CursorAction::Query(range) => loop {
-                    debug!(?range, "Looking for for events in index range");
-
-                    let logs = match self.indexer.fetch_logs(range.clone()).await {
+    #[instrument(fields(domain=self.domain().name()), skip(self, recv, stored_logs_metric))]
+    async fn fetch_logs_from_receiver(
+        &self,
+        recv: &mut BroadcastReceiver<H512>,
+        stored_logs_metric: &GenericCounter<AtomicU64>,
+    ) {
+        loop {
+            match recv.try_recv() {
+                Ok(tx_id) => {
+                    let logs = match self.indexer.fetch_logs_by_tx_hash(tx_id).await {
                         Ok(logs) => logs,
                         Err(err) => {
-                            warn!(?err, "Error fetching logs");
-                            break SLEEP_DURATION;
+                            warn!(?err, ?tx_id, "Error fetching logs for tx id");
+                            continue;
                         }
                     };
-                    let deduped_logs = HashSet::<_>::from_iter(logs);
-                    let logs = Vec::from_iter(deduped_logs);
-
+                    let logs = self.dedupe_and_store_logs(logs, stored_logs_metric).await;
+                    let num_logs = logs.len() as u64;
                     info!(
-                        ?range,
-                        num_logs = logs.len(),
-                        estimated_time_to_sync = fmt_sync_time(eta),
-                        "Found log(s) in index range"
+                        num_logs,
+                        ?tx_id,
+                        sequences = ?logs.iter().map(|(log, _)| log.sequence).collect::<Vec<_>>(),
+                        "Found log(s) for tx id"
                     );
-                    // Store deliveries
-                    let stored = match self.db.store_logs(&logs).await {
-                        Ok(stored) => stored,
-                        Err(err) => {
-                            warn!(?err, "Error storing logs in db");
-                            break SLEEP_DURATION;
-                        }
-                    };
-                    // Report amount of deliveries stored into db
-                    stored_logs.inc_by(stored as u64);
-                    // Update cursor
-                    if let Err(err) = cursor.update(logs, range).await {
-                        warn!(?err, "Error updating cursor");
+                }
+                Err(TryRecvError::Empty) => {
+                    trace!("No txid received");
+                    break;
+                }
+                Err(err) => {
+                    warn!(?err, "Error receiving txid from channel");
+                    break;
+                }
+            }
+        }
+    }
+
+    #[instrument(fields(domain=self.domain().name()), skip(self, stored_logs_metric, indexed_height_metric))]
+    async fn fetch_logs_with_cursor(
+        &self,
+        cursor: &mut Box<dyn ContractSyncCursor<T>>,
+        stored_logs_metric: &GenericCounter<AtomicU64>,
+        indexed_height_metric: &GenericGauge<AtomicI64>,
+    ) {
+        indexed_height_metric.set(cursor.latest_queried_block() as i64);
+        let (action, eta) = match cursor.next_action().await {
+            Ok((action, eta)) => (action, eta),
+            Err(err) => {
+                warn!(?err, "Error getting next action");
+                sleep(SLEEP_DURATION).await;
+                return;
+            }
+        };
+        let sleep_duration = match action {
+            // Use `loop` but always break - this allows for returning a value
+            // from the loop (the sleep duration)
+            #[allow(clippy::never_loop)]
+            CursorAction::Query(range) => loop {
+                debug!(?range, "Looking for events in index range");
+
+                let logs = match self.indexer.fetch_logs_in_range(range.clone()).await {
+                    Ok(logs) => logs,
+                    Err(err) => {
+                        warn!(?err, ?range, "Error fetching logs in range");
                         break SLEEP_DURATION;
-                    };
-                    break Default::default();
-                },
-                CursorAction::Sleep(duration) => duration,
-            };
-            sleep(sleep_duration).await;
+                    }
+                };
+
+                let logs = self.dedupe_and_store_logs(logs, stored_logs_metric).await;
+                let logs_found = logs.len() as u64;
+                info!(
+                    ?range,
+                    num_logs = logs_found,
+                    estimated_time_to_sync = fmt_sync_time(eta),
+                    sequences = ?logs.iter().map(|(log, _)| log.sequence).collect::<Vec<_>>(),
+                    cursor = ?cursor,
+                    "Found log(s) in index range"
+                );
+
+                if let Some(tx) = self.broadcast_sender.as_ref() {
+                    logs.iter().for_each(|(_, meta)| {
+                        if let Err(err) = tx.send(meta.transaction_id) {
+                            trace!(?err, "Error sending txid to receiver");
+                        }
+                    });
+                }
+
+                // Update cursor
+                if let Err(err) = cursor.update(logs, range).await {
+                    warn!(?err, "Error updating cursor");
+                    break SLEEP_DURATION;
+                };
+                break Default::default();
+            },
+            CursorAction::Sleep(duration) => duration,
+        };
+        sleep(sleep_duration).await
+    }
+
+    async fn dedupe_and_store_logs(
+        &self,
+        logs: Vec<(Indexed<T>, LogMeta)>,
+        stored_logs_metric: &GenericCounter<AtomicU64>,
+    ) -> Vec<(Indexed<T>, LogMeta)> {
+        let deduped_logs = HashSet::<_>::from_iter(logs);
+        let logs = Vec::from_iter(deduped_logs);
+
+        // Store deliveries
+        let stored = match self.db.store_logs(&logs).await {
+            Ok(stored) => stored,
+            Err(err) => {
+                warn!(?err, "Error storing logs in db");
+                Default::default()
+            }
+        };
+        if stored > 0 {
+            debug!(
+                domain = self.domain.as_ref(),
+                count = stored,
+                sequences = ?logs.iter().map(|(log, _)| log.sequence).collect::<Vec<_>>(),
+                "Stored logs in db",
+            );
         }
+        // Report amount of deliveries stored into db
+        stored_logs_metric.inc_by(stored as u64);
+        logs
     }
 }
 
@@ -141,16 +240,38 @@ pub trait ContractSyncer<T>: Send + Sync {
     async fn cursor(&self, index_settings: IndexSettings) -> Box<dyn ContractSyncCursor<T>>;
 
     /// Syncs events from the indexer using the provided cursor
-    async fn sync(&self, label: &'static str, cursor: Box<dyn ContractSyncCursor<T>>);
+    async fn sync(&self, label: &'static str, opts: SyncOptions<T>);
 
     /// The domain of this syncer
     fn domain(&self) -> &HyperlaneDomain;
+
+    /// If this syncer is also a broadcaster, return the channel to receive txids
+    fn get_broadcaster(&self) -> Option<BroadcastSender<H512>>;
+}
+
+#[derive(new)]
+/// Options for syncing events
+pub struct SyncOptions<T> {
+    // Keep as optional fields for now to run them simultaneously.
+    // Might want to refactor into an enum later, where we either index with a cursor or rely on receiving
+    // txids from a channel to other indexing tasks
+    cursor: Option<Box<dyn ContractSyncCursor<T>>>,
+    tx_id_receiver: Option<BroadcastReceiver<H512>>,
+}
+
+impl<T> From<Box<dyn ContractSyncCursor<T>>> for SyncOptions<T> {
+    fn from(cursor: Box<dyn ContractSyncCursor<T>>) -> Self {
+        Self {
+            cursor: Some(cursor),
+            tx_id_receiver: None,
+        }
+    }
 }
 
 #[async_trait]
 impl<T> ContractSyncer<T> for WatermarkContractSync<T>
 where
-    T: Debug + Send + Sync + Clone + Eq + Hash + 'static,
+    T: Indexable + Debug + Send + Sync + Clone + Eq + Hash + 'static,
 {
     /// Returns a new cursor to be used for syncing events from the indexer based on time
     async fn cursor(&self, index_settings: IndexSettings) -> Box<dyn ContractSyncCursor<T>> {
@@ -172,13 +293,17 @@ where
         )
     }
 
-    async fn sync(&self, label: &'static str, cursor: Box<dyn ContractSyncCursor<T>>) {
-        ContractSync::sync(self, label, cursor).await;
+    async fn sync(&self, label: &'static str, opts: SyncOptions<T>) {
+        ContractSync::sync(self, label, opts).await
     }
 
     fn domain(&self) -> &HyperlaneDomain {
         ContractSync::domain(self)
     }
+
+    fn get_broadcaster(&self) -> Option<BroadcastSender<H512>> {
+        ContractSync::get_broadcaster(self)
+    }
 }
 
 /// Log store for sequence aware cursors
@@ -191,7 +316,7 @@ pub type SequencedDataContractSync<T> =
 #[async_trait]
 impl<T> ContractSyncer<T> for SequencedDataContractSync<T>
 where
-    T: Send + Sync + Debug + Clone + Eq + Hash + 'static,
+    T: Indexable + Send + Sync + Debug + Clone + Eq + Hash + 'static,
 {
     /// Returns a new cursor to be used for syncing dispatched messages from the indexer
     async fn cursor(&self, index_settings: IndexSettings) -> Box<dyn ContractSyncCursor<T>> {
@@ -207,11 +332,15 @@ where
         )
     }
 
-    async fn sync(&self, label: &'static str, cursor: Box<dyn ContractSyncCursor<T>>) {
-        ContractSync::sync(self, label, cursor).await;
+    async fn sync(&self, label: &'static str, opts: SyncOptions<T>) {
+        ContractSync::sync(self, label, opts).await;
     }
 
     fn domain(&self) -> &HyperlaneDomain {
         ContractSync::domain(self)
     }
+
+    fn get_broadcaster(&self) -> Option<BroadcastSender<H512>> {
+        ContractSync::get_broadcaster(self)
+    }
 }

rust/hyperlane-base/src/db/rocks/hyperlane_db.rs

@@ -242,10 +242,10 @@ impl HyperlaneRocksDB {
         &self,
         event: InterchainGasExpenditure,
     ) -> DbResult<()> {
-        let existing_payment = self.retrieve_gas_expenditure_by_message_id(event.message_id)?;
-        let total = existing_payment + event;
+        let existing_expenditure = self.retrieve_gas_expenditure_by_message_id(event.message_id)?;
+        let total = existing_expenditure + event;
 
-        debug!(?event, new_total_gas_payment=?total, "Storing gas payment");
+        debug!(?event, new_total_gas_expenditure=?total, "Storing gas expenditure");
         self.store_interchain_gas_expenditure_data_by_message_id(
             &total.message_id,
             &InterchainGasExpenditureData {

rust/hyperlane-base/src/settings/base.rs

@@ -160,7 +160,7 @@ impl Settings {
         db: Arc<D>,
     ) -> eyre::Result<Arc<SequencedDataContractSync<T>>>
     where
-        T: Debug,
+        T: Indexable + Debug,
         SequenceIndexer<T>: TryFromWithMetrics<ChainConf>,
         D: HyperlaneLogStore<T> + HyperlaneSequenceAwareIndexerStoreReader<T> + 'static,
     {
@@ -184,7 +184,7 @@ impl Settings {
         db: Arc<D>,
     ) -> eyre::Result<Arc<WatermarkContractSync<T>>>
     where
-        T: Debug,
+        T: Indexable + Debug,
         SequenceIndexer<T>: TryFromWithMetrics<ChainConf>,
         D: HyperlaneLogStore<T> + HyperlaneWatermarkedLogStore<T> + 'static,
     {

rust/hyperlane-base/src/settings/mod.rs

@@ -1,6 +1,6 @@
 //! Common settings and configuration for Hyperlane agents
 //!
-//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape
+//! The correct settings shape is defined in the TypeScript SDK metadata. While the exact shape
 //! and validations it defines are not applied here, we should mirror them.
 //! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK.
 //!

rust/hyperlane-base/src/settings/parser/mod.rs

@@ -1,6 +1,6 @@
 //! This module is responsible for parsing the agent's settings.
 //!
-//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape
+//! The correct settings shape is defined in the TypeScript SDK metadata. While the exact shape
 //! and validations it defines are not applied here, we should mirror them.
 //! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK.
 

rust/hyperlane-core/Cargo.toml

@@ -49,7 +49,7 @@ uint.workspace = true
 tokio = { workspace = true, features = ["rt", "time"] }
 
 [features]
-default = []
+default = ["strum"]
 float = []
 test-utils = ["dep:config"]
 agent = ["ethers", "strum"]

rust/hyperlane-core/src/chain.rs

@@ -51,6 +51,7 @@ impl<'a> std::fmt::Display for ContractLocator<'a> {
 pub enum KnownHyperlaneDomain {
     Ethereum = 1,
     Sepolia = 11155111,
+    Holesky = 17000,
 
     Polygon = 137,
 
@@ -218,7 +219,7 @@ impl KnownHyperlaneDomain {
                 Moonbeam, Gnosis, MantaPacific, Neutron, Injective, InEvm
             ],
             Testnet: [
-                Alfajores, MoonbaseAlpha, Sepolia, ScrollSepolia, Chiado, PlumeTestnet, Fuji, BinanceSmartChainTestnet
+                Alfajores, MoonbaseAlpha, Sepolia, ScrollSepolia, Chiado, PlumeTestnet, Fuji, BinanceSmartChainTestnet, Holesky
             ],
             LocalTestChain: [Test1, Test2, Test3, FuelTest1, SealevelTest1, SealevelTest2, CosmosTest99990, CosmosTest99991],
         })
@@ -229,7 +230,7 @@ impl KnownHyperlaneDomain {
 
         many_to_one!(match self {
             HyperlaneDomainProtocol::Ethereum: [
-                Ethereum, Sepolia, Polygon, Avalanche, Fuji, Arbitrum,
+                Ethereum, Sepolia, Holesky, Polygon, Avalanche, Fuji, Arbitrum,
                 Optimism, BinanceSmartChain, BinanceSmartChainTestnet, Celo, Gnosis,
                 Alfajores, Moonbeam, InEvm, MoonbaseAlpha, ScrollSepolia,
                 Chiado, MantaPacific, PlumeTestnet, Test1, Test2, Test3
@@ -246,7 +247,7 @@ impl KnownHyperlaneDomain {
         many_to_one!(match self {
             HyperlaneDomainTechnicalStack::ArbitrumNitro: [Arbitrum, PlumeTestnet],
             HyperlaneDomainTechnicalStack::Other: [
-                Ethereum, Sepolia, Polygon, Avalanche, Fuji, Optimism,
+                Ethereum, Sepolia, Holesky, Polygon, Avalanche, Fuji, Optimism,
                 BinanceSmartChain, BinanceSmartChainTestnet, Celo, Gnosis, Alfajores, Moonbeam, MoonbaseAlpha,
                 ScrollSepolia, Chiado, MantaPacific, Neutron, Injective, InEvm,
                 Test1, Test2, Test3, FuelTest1, SealevelTest1, SealevelTest2, CosmosTest99990, CosmosTest99991

rust/hyperlane-core/src/traits/cursor.rs

@@ -1,4 +1,8 @@
-use std::{fmt, ops::RangeInclusive, time::Duration};
+use std::{
+    fmt::{self, Debug},
+    ops::RangeInclusive,
+    time::Duration,
+};
 
 use async_trait::async_trait;
 use auto_impl::auto_impl;
@@ -9,7 +13,7 @@ use crate::{Indexed, LogMeta};
 /// A cursor governs event indexing for a contract.
 #[async_trait]
 #[auto_impl(Box)]
-pub trait ContractSyncCursor<T>: Send + Sync + 'static {
+pub trait ContractSyncCursor<T>: Debug + Send + Sync + 'static {
     /// The next block range that should be queried.
     /// This method should be tolerant to being called multiple times in a row
     /// without any updates in between.

rust/hyperlane-core/src/traits/indexer.rs

@@ -11,7 +11,7 @@ use async_trait::async_trait;
 use auto_impl::auto_impl;
 use serde::Deserialize;
 
-use crate::{ChainResult, Indexed, LogMeta};
+use crate::{ChainResult, Indexed, LogMeta, H512};
 
 /// Indexing mode.
 #[derive(Copy, Debug, Default, Deserialize, Clone)]
@@ -29,13 +29,21 @@ pub enum IndexMode {
 #[auto_impl(&, Box, Arc,)]
 pub trait Indexer<T: Sized>: Send + Sync + Debug {
     /// Fetch list of logs between blocks `from` and `to`, inclusive.
-    async fn fetch_logs(
+    async fn fetch_logs_in_range(
         &self,
         range: RangeInclusive<u32>,
     ) -> ChainResult<Vec<(Indexed<T>, LogMeta)>>;
 
     /// Get the chain's latest block number that has reached finality
     async fn get_finalized_block_number(&self) -> ChainResult<u32>;
+
+    /// Fetch list of logs emitted in a transaction with the given hash.
+    async fn fetch_logs_by_tx_hash(
+        &self,
+        _tx_hash: H512,
+    ) -> ChainResult<Vec<(Indexed<T>, LogMeta)>> {
+        Ok(vec![])
+    }
 }
 
 /// Interface for indexing data in sequence.

rust/hyperlane-core/src/traits/mod.rs

@@ -10,6 +10,7 @@ pub use interchain_security_module::*;
 pub use mailbox::*;
 pub use merkle_tree_hook::*;
 pub use multisig_ism::*;
+pub use pending_operation::*;
 pub use provider::*;
 pub use routing_ism::*;
 pub use signing::*;
@@ -29,6 +30,7 @@ mod interchain_security_module;
 mod mailbox;
 mod merkle_tree_hook;
 mod multisig_ism;
+mod pending_operation;
 mod provider;
 mod routing_ism;
 mod signing;

rust/hyperlane-core/src/traits/pending_operation.rs

@@ -4,10 +4,16 @@ use std::{
     time::{Duration, Instant},
 };
 
+use crate::{
+    ChainResult, FixedPointNumber, HyperlaneDomain, HyperlaneMessage, TryBatchAs, TxOutcome, H256,
+    U256,
+};
 use async_trait::async_trait;
-use hyperlane_core::{HyperlaneDomain, HyperlaneMessage, TryBatchAs, TxOutcome, H256};
+use num::CheckedDiv;
+use tracing::warn;
 
-use super::op_queue::QueueOperation;
+/// Boxed operation that can be stored in an operation queue
+pub type QueueOperation = Box<dyn PendingOperation>;
 
 /// A pending operation that will be run by the submitter and cause a
 /// transaction to be sent.
@@ -67,11 +73,21 @@ pub trait PendingOperation: Send + Sync + Debug + TryBatchAs<HyperlaneMessage> {
     /// Set the outcome of the `submit` call
     fn set_submission_outcome(&mut self, outcome: TxOutcome);
 
+    /// Get the estimated the cost of the `submit` call
+    fn get_tx_cost_estimate(&self) -> Option<U256>;
+
     /// This will be called after the operation has been submitted and is
     /// responsible for checking if the operation has reached a point at
     /// which we consider it safe from reorgs.
     async fn confirm(&mut self) -> PendingOperationResult;
 
+    /// Record the outcome of the operation
+    fn set_operation_outcome(
+        &mut self,
+        submission_outcome: TxOutcome,
+        submission_estimated_cost: U256,
+    );
+
     /// Get the earliest instant at which this should next be attempted.
     ///
     /// This is only used for sorting, the functions are responsible for
@@ -85,11 +101,41 @@ pub trait PendingOperation: Send + Sync + Debug + TryBatchAs<HyperlaneMessage> {
     /// retried immediately.
     fn reset_attempts(&mut self);
 
-    #[cfg(test)]
     /// Set the number of times this operation has been retried.
+    #[cfg(any(test, feature = "test-utils"))]
     fn set_retries(&mut self, retries: u32);
 }
 
+/// Utility fn to calculate the total estimated cost of an operation batch
+pub fn total_estimated_cost(ops: &[Box<dyn PendingOperation>]) -> U256 {
+    ops.iter()
+        .fold(U256::zero(), |acc, op| match op.get_tx_cost_estimate() {
+            Some(cost_estimate) => acc.saturating_add(cost_estimate),
+            None => {
+                warn!(operation=?op, "No cost estimate available for operation, defaulting to 0");
+                acc
+            }
+        })
+}
+
+/// Calculate the gas used by an operation (either in a batch or single-submission), by looking at the total cost of the tx,
+/// and the estimated cost of the operation compared to the sum of the estimates of all operations in the batch.
+/// When using this for single-submission rather than a batch,
+/// the `tx_estimated_cost` should be the same as the `tx_estimated_cost`
+pub fn gas_used_by_operation(
+    tx_outcome: &TxOutcome,
+    tx_estimated_cost: U256,
+    operation_estimated_cost: U256,
+) -> ChainResult<U256> {
+    let gas_used_by_tx = FixedPointNumber::try_from(tx_outcome.gas_used)?;
+    let operation_gas_estimate = FixedPointNumber::try_from(operation_estimated_cost)?;
+    let tx_gas_estimate = FixedPointNumber::try_from(tx_estimated_cost)?;
+    let gas_used_by_operation = (gas_used_by_tx * operation_gas_estimate)
+        .checked_div(&tx_gas_estimate)
+        .ok_or(eyre::eyre!("Division by zero"))?;
+    gas_used_by_operation.try_into()
+}
+
 impl Display for QueueOperation {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
@@ -138,6 +184,7 @@ impl Ord for QueueOperation {
     }
 }
 
+/// Possible outcomes of performing an action on a pending operation (such as `prepare`, `submit` or `confirm`).
 #[derive(Debug)]
 pub enum PendingOperationResult {
     /// Promote to the next step
@@ -153,6 +200,7 @@ pub enum PendingOperationResult {
 }
 
 /// create a `op_try!` macro for the `on_retry` handler.
+#[macro_export]
 macro_rules! make_op_try {
     ($on_retry:expr) => {
         /// Handle a result and either return early with retry or a critical failure on
@@ -181,5 +229,3 @@ macro_rules! make_op_try {
                         }
     };
 }
-
-pub(super) use make_op_try;

rust/hyperlane-core/src/types/channel.rs

@@ -1,50 +0,0 @@
-use derive_new::new;
-use tokio::sync::broadcast::{Receiver, Sender};
-
-/// Multi-producer, multi-consumer channel
-pub struct MpmcChannel<T> {
-    sender: Sender<T>,
-    receiver: MpmcReceiver<T>,
-}
-
-impl<T: Clone> MpmcChannel<T> {
-    /// Creates a new `MpmcChannel` with the specified capacity.
-    ///
-    /// # Arguments
-    ///
-    /// * `capacity` - The maximum number of messages that can be buffered in the channel.
-    pub fn new(capacity: usize) -> Self {
-        let (sender, receiver) = tokio::sync::broadcast::channel(capacity);
-        Self {
-            sender: sender.clone(),
-            receiver: MpmcReceiver::new(sender, receiver),
-        }
-    }
-
-    /// Returns a clone of the sender end of the channel.
-    pub fn sender(&self) -> Sender<T> {
-        self.sender.clone()
-    }
-
-    /// Returns a clone of the receiver end of the channel.
-    pub fn receiver(&self) -> MpmcReceiver<T> {
-        self.receiver.clone()
-    }
-}
-
-/// Clonable receiving end of a multi-producer, multi-consumer channel
-#[derive(Debug, new)]
-pub struct MpmcReceiver<T> {
-    sender: Sender<T>,
-    /// The receiving end of the channel.
-    pub receiver: Receiver<T>,
-}
-
-impl<T> Clone for MpmcReceiver<T> {
-    fn clone(&self) -> Self {
-        Self {
-            sender: self.sender.clone(),
-            receiver: self.sender.subscribe(),
-        }
-    }
-}

rust/hyperlane-core/src/types/mod.rs

@@ -8,8 +8,6 @@ pub use self::primitive_types::*;
 pub use ::primitive_types as ethers_core_types;
 pub use announcement::*;
 pub use chain_data::*;
-#[cfg(feature = "async")]
-pub use channel::*;
 pub use checkpoint::*;
 pub use indexing::*;
 pub use log_metadata::*;
@@ -21,8 +19,6 @@ use crate::{Decode, Encode, HyperlaneProtocolError};
 
 mod announcement;
 mod chain_data;
-#[cfg(feature = "async")]
-mod channel;
 mod checkpoint;
 mod indexing;
 mod log_metadata;

rust/hyperlane-core/src/types/primitive_types.rs

@@ -3,11 +3,15 @@
 #![allow(clippy::assign_op_pattern)]
 #![allow(clippy::reversed_empty_ranges)]
 
-use std::{ops::Mul, str::FromStr};
+use std::{
+    ops::{Div, Mul},
+    str::FromStr,
+};
 
 use bigdecimal::{BigDecimal, RoundingMode};
 use borsh::{BorshDeserialize, BorshSerialize};
 use fixed_hash::impl_fixed_hash_conversions;
+use num::CheckedDiv;
 use num_traits::Zero;
 use uint::construct_uint;
 
@@ -421,6 +425,27 @@ where
     }
 }
 
+impl<T> Div<T> for FixedPointNumber
+where
+    T: Into<FixedPointNumber>,
+{
+    type Output = FixedPointNumber;
+
+    fn div(self, rhs: T) -> Self::Output {
+        let rhs = rhs.into();
+        Self(self.0 / rhs.0)
+    }
+}
+
+impl CheckedDiv for FixedPointNumber {
+    fn checked_div(&self, v: &Self) -> Option<Self> {
+        if v.0.is_zero() {
+            return None;
+        }
+        Some(Self(self.0.clone() / v.0.clone()))
+    }
+}
+
 impl FromStr for FixedPointNumber {
     type Err = ChainCommunicationError;
 

rust/utils/backtrace-oneline/src/lib.rs

@@ -118,7 +118,7 @@ impl BacktraceFrameFmt<'_, '_, '_> {
             symbol.name(),
             // TODO: this isn't great that we don't end up printing anything
             // with non-utf8 filenames. Thankfully almost everything is utf8 so
-            // this shouldn't be too too bad.
+            // this shouldn't be too bad.
             symbol
                 .filename()
                 .and_then(|p| Some(BytesOrWideString::Bytes(p.to_str()?.as_bytes()))),

rust/utils/run-locally/Cargo.toml

@@ -28,11 +28,13 @@ ethers-contract.workspace = true
 tokio.workspace = true
 maplit.workspace = true
 nix = { workspace = true, features = ["signal"], default-features = false }
+once_cell.workspace = true
 tempfile.workspace = true
 ureq = { workspace = true, default-features = false }
 which.workspace = true
 macro_rules_attribute.workspace = true
 regex.workspace = true
+relayer = { path = "../../agents/relayer"}
 hyperlane-cosmwasm-interface.workspace = true
 cosmwasm-schema.workspace = true
 

rust/utils/run-locally/src/config.rs

@@ -6,6 +6,7 @@ pub struct Config {
     pub ci_mode: bool,
     pub ci_mode_timeout: u64,
     pub kathy_messages: u64,
+    pub sealevel_enabled: bool,
     // TODO: Include count of sealevel messages in a field separate from `kathy_messages`?
 }
 
@@ -26,6 +27,9 @@ impl Config {
                     .map(|r| r.parse::<u64>().unwrap());
                 r.unwrap_or(16)
             },
+            sealevel_enabled: env::var("SEALEVEL_ENABLED")
+                .map(|k| k.parse::<bool>().unwrap())
+                .unwrap_or(true),
         })
     }
 }

rust/utils/run-locally/src/cosmos/cli.rs

@@ -152,7 +152,7 @@ impl OsmosisCLI {
             .arg("grpc.address", &endpoint.grpc_addr) // default is 0.0.0.0:9090
             .arg("rpc.pprof_laddr", pprof_addr) // default is localhost:6060
             .arg("log_level", "panic")
-            .spawn("COSMOS");
+            .spawn("COSMOS", None);
 
         endpoint.wait_for_node();
 

rust/utils/run-locally/src/cosmos/mod.rs

@@ -271,7 +271,7 @@ fn launch_cosmos_validator(
         .hyp_env("SIGNER_SIGNER_TYPE", "hexKey")
         .hyp_env("SIGNER_KEY", agent_config.signer.key)
         .hyp_env("TRACING_LEVEL", if debug { "debug" } else { "info" })
-        .spawn("VAL");
+        .spawn("VAL", None);
 
     validator
 }
@@ -299,7 +299,7 @@ fn launch_cosmos_relayer(
         .hyp_env("TRACING_LEVEL", if debug { "debug" } else { "info" })
         .hyp_env("GASPAYMENTENFORCEMENT", "[{\"type\": \"none\"}]")
         .hyp_env("METRICSPORT", metrics.to_string())
-        .spawn("RLY");
+        .spawn("RLY", None);
 
     relayer
 }

rust/utils/run-locally/src/ethereum/mod.rs

@@ -36,7 +36,7 @@ pub fn start_anvil(config: Arc<Config>) -> AgentHandles {
     }
     log!("Launching anvil...");
     let anvil_args = Program::new("anvil").flag("silent").filter_logs(|_| false); // for now do not keep any of the anvil logs
-    let anvil = anvil_args.spawn("ETH");
+    let anvil = anvil_args.spawn("ETH", None);
 
     sleep(Duration::from_secs(10));
 

rust/utils/run-locally/src/invariants.rs

@@ -1,14 +1,15 @@
-// use std::path::Path;
-
+use std::fs::File;
 use std::path::Path;
 
 use crate::config::Config;
 use crate::metrics::agent_balance_sum;
+use crate::utils::get_matching_lines;
 use maplit::hashmap;
+use relayer::GAS_EXPENDITURE_LOG_MESSAGE;
 
 use crate::logging::log;
 use crate::solana::solana_termination_invariants_met;
-use crate::{fetch_metric, ZERO_MERKLE_INSERTION_KATHY_MESSAGES};
+use crate::{fetch_metric, AGENT_LOGGING_DIR, ZERO_MERKLE_INSERTION_KATHY_MESSAGES};
 
 // This number should be even, so the messages can be split into two equal halves
 // sent before and after the relayer spins up, to avoid rounding errors.
@@ -19,11 +20,16 @@ pub const SOL_MESSAGES_EXPECTED: u32 = 20;
 pub fn termination_invariants_met(
     config: &Config,
     starting_relayer_balance: f64,
-    solana_cli_tools_path: &Path,
-    solana_config_path: &Path,
+    solana_cli_tools_path: Option<&Path>,
+    solana_config_path: Option<&Path>,
 ) -> eyre::Result<bool> {
     let eth_messages_expected = (config.kathy_messages / 2) as u32 * 2;
-    let total_messages_expected = eth_messages_expected + SOL_MESSAGES_EXPECTED;
+    let sol_messages_expected = if config.sealevel_enabled {
+        SOL_MESSAGES_EXPECTED
+    } else {
+        0
+    };
+    let total_messages_expected = eth_messages_expected + sol_messages_expected;
 
     let lengths = fetch_metric("9092", "hyperlane_submitter_queue_length", &hashmap! {})?;
     assert!(!lengths.is_empty(), "Could not find queue length metric");
@@ -55,6 +61,19 @@ pub fn termination_invariants_met(
     .iter()
     .sum::<u32>();
 
+    let log_file_path = AGENT_LOGGING_DIR.join("RLY-output.log");
+    let relayer_logfile = File::open(log_file_path)?;
+    let gas_expenditure_log_count =
+        get_matching_lines(&relayer_logfile, GAS_EXPENDITURE_LOG_MESSAGE)
+            .unwrap()
+            .len();
+
+    // Zero insertion messages don't reach `submit` stage where gas is spent, so we only expect these logs for the other messages.
+    assert_eq!(
+        gas_expenditure_log_count as u32, total_messages_expected,
+        "Didn't record gas payment for all delivered messages"
+    );
+
     let gas_payment_sealevel_events_count = fetch_metric(
         "9092",
         "hyperlane_contract_sync_stored_events",
@@ -76,9 +95,13 @@ pub fn termination_invariants_met(
         return Ok(false);
     }
 
-    if !solana_termination_invariants_met(solana_cli_tools_path, solana_config_path) {
-        log!("Solana termination invariants not met");
-        return Ok(false);
+    if let Some((solana_cli_tools_path, solana_config_path)) =
+        solana_cli_tools_path.zip(solana_config_path)
+    {
+        if !solana_termination_invariants_met(solana_cli_tools_path, solana_config_path) {
+            log!("Solana termination invariants not met");
+            return Ok(false);
+        }
     }
 
     let dispatched_messages_scraped = fetch_metric(

rust/utils/run-locally/src/main.rs

@@ -11,12 +11,17 @@
 //! the end conditions are met, the test is a failure. Defaults to 10 min.
 //! - `E2E_KATHY_MESSAGES`: Number of kathy messages to dispatch. Defaults to 16 if CI mode is enabled.
 //! else false.
+//! - `SEALEVEL_ENABLED`: true/false, enables sealevel testing. Defaults to true.
 
 use std::{
-    fs,
+    collections::HashMap,
+    fs::{self, File},
     path::Path,
     process::{Child, ExitCode},
-    sync::atomic::{AtomicBool, Ordering},
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, Mutex,
+    },
     thread::sleep,
     time::{Duration, Instant},
 };
@@ -24,6 +29,7 @@ use std::{
 use ethers_contract::MULTICALL_ADDRESS;
 use logging::log;
 pub use metrics::fetch_metric;
+use once_cell::sync::Lazy;
 use program::Program;
 use tempfile::tempdir;
 
@@ -46,6 +52,12 @@ mod program;
 mod solana;
 mod utils;
 
+pub static AGENT_LOGGING_DIR: Lazy<&Path> = Lazy::new(|| {
+    let dir = Path::new("/tmp/test_logs");
+    fs::create_dir_all(dir).unwrap();
+    dir
+});
+
 /// These private keys are from hardhat/anvil's testing accounts.
 const RELAYER_KEYS: &[&str] = &[
     // test1
@@ -61,17 +73,18 @@ const RELAYER_KEYS: &[&str] = &[
 ];
 /// These private keys are from hardhat/anvil's testing accounts.
 /// These must be consistent with the ISM config for the test.
-const VALIDATOR_KEYS: &[&str] = &[
+const ETH_VALIDATOR_KEYS: &[&str] = &[
     // eth
     "0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a",
     "0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba",
     "0x92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e",
+];
+
+const SEALEVEL_VALIDATOR_KEYS: &[&str] = &[
     // sealevel
     "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d",
 ];
 
-const VALIDATOR_ORIGIN_CHAINS: &[&str] = &["test1", "test2", "test3", "sealeveltest1"];
-
 const AGENT_BIN_PATH: &str = "target/debug";
 const INFRA_PATH: &str = "../typescript/infra";
 const MONOREPO_ROOT_PATH: &str = "../";
@@ -87,14 +100,15 @@ static SHUTDOWN: AtomicBool = AtomicBool::new(false);
 /// cleanup purposes at this time.
 #[derive(Default)]
 struct State {
-    agents: Vec<(String, Child)>,
+    #[allow(clippy::type_complexity)]
+    agents: HashMap<String, (Child, Option<Arc<Mutex<File>>>)>,
     watchers: Vec<Box<dyn TaskHandle<Output = ()>>>,
     data: Vec<Box<dyn ArbitraryData>>,
 }
 
 impl State {
     fn push_agent(&mut self, handles: AgentHandles) {
-        self.agents.push((handles.0, handles.1));
+        self.agents.insert(handles.0, (handles.1, handles.5));
         self.watchers.push(handles.2);
         self.watchers.push(handles.3);
         self.data.push(handles.4);
@@ -105,9 +119,7 @@ impl Drop for State {
     fn drop(&mut self) {
         SHUTDOWN.store(true, Ordering::Relaxed);
         log!("Signaling children to stop...");
-        // stop children in reverse order
-        self.agents.reverse();
-        for (name, mut agent) in self.agents.drain(..) {
+        for (name, (mut agent, _)) in self.agents.drain() {
             log!("Stopping child {}", name);
             stop_child(&mut agent);
         }
@@ -122,6 +134,7 @@ impl Drop for State {
             drop(data)
         }
         fs::remove_dir_all(SOLANA_CHECKPOINT_LOCATION).unwrap_or_default();
+        fs::remove_dir_all::<&Path>(AGENT_LOGGING_DIR.as_ref()).unwrap_or_default();
     }
 }
 
@@ -133,20 +146,27 @@ fn main() -> ExitCode {
     })
     .unwrap();
 
-    assert_eq!(VALIDATOR_ORIGIN_CHAINS.len(), VALIDATOR_KEYS.len());
-    const VALIDATOR_COUNT: usize = VALIDATOR_KEYS.len();
-
     let config = Config::load();
-
-    let solana_checkpoint_path = Path::new(SOLANA_CHECKPOINT_LOCATION);
-    fs::remove_dir_all(solana_checkpoint_path).unwrap_or_default();
-    let checkpoints_dirs: Vec<DynPath> = (0..VALIDATOR_COUNT - 1)
+    let mut validator_origin_chains = ["test1", "test2", "test3"].to_vec();
+    let mut validator_keys = ETH_VALIDATOR_KEYS.to_vec();
+    let mut validator_count: usize = validator_keys.len();
+    let mut checkpoints_dirs: Vec<DynPath> = (0..validator_count)
         .map(|_| Box::new(tempdir().unwrap()) as DynPath)
-        .chain([Box::new(solana_checkpoint_path) as DynPath])
         .collect();
+    if config.sealevel_enabled {
+        validator_origin_chains.push("sealeveltest1");
+        let mut sealevel_keys = SEALEVEL_VALIDATOR_KEYS.to_vec();
+        validator_keys.append(&mut sealevel_keys);
+        let solana_checkpoint_path = Path::new(SOLANA_CHECKPOINT_LOCATION);
+        fs::remove_dir_all(solana_checkpoint_path).unwrap_or_default();
+        checkpoints_dirs.push(Box::new(solana_checkpoint_path) as DynPath);
+        validator_count += 1;
+    }
+    assert_eq!(validator_origin_chains.len(), validator_keys.len());
+
     let rocks_db_dir = tempdir().unwrap();
     let relayer_db = concat_path(&rocks_db_dir, "relayer");
-    let validator_dbs = (0..VALIDATOR_COUNT)
+    let validator_dbs = (0..validator_count)
         .map(|i| concat_path(&rocks_db_dir, format!("validator{i}")))
         .collect::<Vec<_>>();
 
@@ -200,15 +220,6 @@ fn main() -> ExitCode {
             r#"[{
                 "type": "minimum",
                 "payment": "1",
-                "matchingList": [
-                    {
-                        "originDomain": ["13375","13376"],
-                        "destinationDomain": ["13375","13376"]
-                    }
-                ]
-            },
-            {
-                "type": "none"
             }]"#,
         )
         .arg(
@@ -216,11 +227,15 @@ fn main() -> ExitCode {
             "http://127.0.0.1:8545,http://127.0.0.1:8545,http://127.0.0.1:8545",
         )
         // default is used for TEST3
-        .arg("defaultSigner.key", RELAYER_KEYS[2])
-        .arg(
+        .arg("defaultSigner.key", RELAYER_KEYS[2]);
+    let relayer_env = if config.sealevel_enabled {
+        relayer_env.arg(
             "relayChains",
             "test1,test2,test3,sealeveltest1,sealeveltest2",
-        );
+        )
+    } else {
+        relayer_env.arg("relayChains", "test1,test2,test3")
+    };
 
     let base_validator_env = common_agent_env
         .clone()
@@ -242,14 +257,14 @@ fn main() -> ExitCode {
         .hyp_env("INTERVAL", "5")
         .hyp_env("CHECKPOINTSYNCER_TYPE", "localStorage");
 
-    let validator_envs = (0..VALIDATOR_COUNT)
+    let validator_envs = (0..validator_count)
         .map(|i| {
             base_validator_env
                 .clone()
                 .hyp_env("METRICSPORT", (9094 + i).to_string())
                 .hyp_env("DB", validator_dbs[i].to_str().unwrap())
-                .hyp_env("ORIGINCHAINNAME", VALIDATOR_ORIGIN_CHAINS[i])
-                .hyp_env("VALIDATOR_KEY", VALIDATOR_KEYS[i])
+                .hyp_env("ORIGINCHAINNAME", validator_origin_chains[i])
+                .hyp_env("VALIDATOR_KEY", validator_keys[i])
                 .hyp_env(
                     "CHECKPOINTSYNCER_PATH",
                     (*checkpoints_dirs[i]).as_ref().to_str().unwrap(),
@@ -283,7 +298,7 @@ fn main() -> ExitCode {
             .join(", ")
     );
     log!("Relayer DB in {}", relayer_db.display());
-    (0..VALIDATOR_COUNT).for_each(|i| {
+    (0..validator_count).for_each(|i| {
         log!("Validator {} DB in {}", i + 1, validator_dbs[i].display());
     });
 
@@ -291,9 +306,14 @@ fn main() -> ExitCode {
     // Ready to run...
     //
 
-    let (solana_path, solana_path_tempdir) = install_solana_cli_tools().join();
-    state.data.push(Box::new(solana_path_tempdir));
-    let solana_program_builder = build_solana_programs(solana_path.clone());
+    let solana_paths = if config.sealevel_enabled {
+        let (solana_path, solana_path_tempdir) = install_solana_cli_tools().join();
+        state.data.push(Box::new(solana_path_tempdir));
+        let solana_program_builder = build_solana_programs(solana_path.clone());
+        Some((solana_program_builder.join(), solana_path))
+    } else {
+        None
+    };
 
     // this task takes a long time in the CI so run it in parallel
     log!("Building rust...");
@@ -303,15 +323,18 @@ fn main() -> ExitCode {
         .arg("bin", "relayer")
         .arg("bin", "validator")
         .arg("bin", "scraper")
-        .arg("bin", "init-db")
-        .arg("bin", "hyperlane-sealevel-client")
+        .arg("bin", "init-db");
+    let build_rust = if config.sealevel_enabled {
+        build_rust.arg("bin", "hyperlane-sealevel-client")
+    } else {
+        build_rust
+    };
+    let build_rust = build_rust
         .filter_logs(|l| !l.contains("workspace-inheritance"))
         .run();
 
     let start_anvil = start_anvil(config.clone());
 
-    let solana_program_path = solana_program_builder.join();
-
     log!("Running postgres db...");
     let postgres = Program::new("docker")
         .cmd("run")
@@ -320,24 +343,31 @@ fn main() -> ExitCode {
         .arg("env", "POSTGRES_PASSWORD=47221c18c610")
         .arg("publish", "5432:5432")
         .cmd("postgres:14")
-        .spawn("SQL");
+        .spawn("SQL", None);
     state.push_agent(postgres);
 
     build_rust.join();
 
     let solana_ledger_dir = tempdir().unwrap();
-    let start_solana_validator = start_solana_test_validator(
-        solana_path.clone(),
-        solana_program_path,
-        solana_ledger_dir.as_ref().to_path_buf(),
-    );
+    let solana_config_path = if let Some((solana_program_path, solana_path)) = solana_paths.clone()
+    {
+        let start_solana_validator = start_solana_test_validator(
+            solana_path.clone(),
+            solana_program_path,
+            solana_ledger_dir.as_ref().to_path_buf(),
+        );
+
+        let (solana_config_path, solana_validator) = start_solana_validator.join();
+        state.push_agent(solana_validator);
+        Some(solana_config_path)
+    } else {
+        None
+    };
 
-    let (solana_config_path, solana_validator) = start_solana_validator.join();
-    state.push_agent(solana_validator);
     state.push_agent(start_anvil.join());
 
     // spawn 1st validator before any messages have been sent to test empty mailbox
-    state.push_agent(validator_envs.first().unwrap().clone().spawn("VL1"));
+    state.push_agent(validator_envs.first().unwrap().clone().spawn("VL1", None));
 
     sleep(Duration::from_secs(5));
 
@@ -345,7 +375,7 @@ fn main() -> ExitCode {
     Program::new(concat_path(AGENT_BIN_PATH, "init-db"))
         .run()
         .join();
-    state.push_agent(scraper_env.spawn("SCR"));
+    state.push_agent(scraper_env.spawn("SCR", None));
 
     // Send half the kathy messages before starting the rest of the agents
     let kathy_env_single_insertion = Program::new("yarn")
@@ -378,22 +408,35 @@ fn main() -> ExitCode {
         .arg("required-hook", "merkleTreeHook");
     kathy_env_double_insertion.clone().run().join();
 
-    // Send some sealevel messages before spinning up the agents, to test the backward indexing cursor
-    for _i in 0..(SOL_MESSAGES_EXPECTED / 2) {
-        initiate_solana_hyperlane_transfer(solana_path.clone(), solana_config_path.clone()).join();
+    if let Some((solana_config_path, (_, solana_path))) =
+        solana_config_path.clone().zip(solana_paths.clone())
+    {
+        // Send some sealevel messages before spinning up the agents, to test the backward indexing cursor
+        for _i in 0..(SOL_MESSAGES_EXPECTED / 2) {
+            initiate_solana_hyperlane_transfer(solana_path.clone(), solana_config_path.clone())
+                .join();
+        }
     }
 
     // spawn the rest of the validators
     for (i, validator_env) in validator_envs.into_iter().enumerate().skip(1) {
-        let validator = validator_env.spawn(make_static(format!("VL{}", 1 + i)));
+        let validator = validator_env.spawn(
+            make_static(format!("VL{}", 1 + i)),
+            Some(AGENT_LOGGING_DIR.as_ref()),
+        );
         state.push_agent(validator);
     }
 
-    state.push_agent(relayer_env.spawn("RLY"));
+    state.push_agent(relayer_env.spawn("RLY", Some(&AGENT_LOGGING_DIR)));
 
-    // Send some sealevel messages after spinning up the relayer, to test the forward indexing cursor
-    for _i in 0..(SOL_MESSAGES_EXPECTED / 2) {
-        initiate_solana_hyperlane_transfer(solana_path.clone(), solana_config_path.clone()).join();
+    if let Some((solana_config_path, (_, solana_path))) =
+        solana_config_path.clone().zip(solana_paths.clone())
+    {
+        // Send some sealevel messages before spinning up the agents, to test the backward indexing cursor
+        for _i in 0..(SOL_MESSAGES_EXPECTED / 2) {
+            initiate_solana_hyperlane_transfer(solana_path.clone(), solana_config_path.clone())
+                .join();
+        }
     }
 
     log!("Setup complete! Agents running in background...");
@@ -402,7 +445,11 @@ fn main() -> ExitCode {
     // Send half the kathy messages after the relayer comes up
     kathy_env_double_insertion.clone().run().join();
     kathy_env_zero_insertion.clone().run().join();
-    state.push_agent(kathy_env_single_insertion.flag("mineforever").spawn("KTY"));
+    state.push_agent(
+        kathy_env_single_insertion
+            .flag("mineforever")
+            .spawn("KTY", None),
+    );
 
     let loop_start = Instant::now();
     // give things a chance to fully start.
@@ -412,12 +459,14 @@ fn main() -> ExitCode {
     while !SHUTDOWN.load(Ordering::Relaxed) {
         if config.ci_mode {
             // for CI we have to look for the end condition.
-            // if termination_invariants_met(&config, starting_relayer_balance)
             if termination_invariants_met(
                 &config,
                 starting_relayer_balance,
-                &solana_path,
-                &solana_config_path,
+                solana_paths
+                    .clone()
+                    .map(|(_, solana_path)| solana_path)
+                    .as_deref(),
+                solana_config_path.as_deref(),
             )
             .unwrap_or(false)
             {
@@ -432,7 +481,7 @@ fn main() -> ExitCode {
         }
 
         // verify long-running tasks are still running
-        for (name, child) in state.agents.iter_mut() {
+        for (name, (child, _)) in state.agents.iter_mut() {
             if let Some(status) = child.try_wait().unwrap() {
                 if !status.success() {
                     log!(

rust/utils/run-locally/src/program.rs

@@ -2,14 +2,14 @@ use std::{
     collections::BTreeMap,
     ffi::OsStr,
     fmt::{Debug, Display, Formatter},
-    io::{BufRead, BufReader, Read},
+    fs::{File, OpenOptions},
+    io::{BufRead, BufReader, Read, Write},
     path::{Path, PathBuf},
     process::{Command, Stdio},
     sync::{
         atomic::{AtomicBool, Ordering},
-        mpsc,
-        mpsc::Sender,
-        Arc,
+        mpsc::{self, Sender},
+        Arc, Mutex,
     },
     thread::{sleep, spawn},
     time::Duration,
@@ -240,8 +240,18 @@ impl Program {
         })
     }
 
-    pub fn spawn(self, log_prefix: &'static str) -> AgentHandles {
+    pub fn spawn(self, log_prefix: &'static str, logs_dir: Option<&Path>) -> AgentHandles {
         let mut command = self.create_command();
+        let log_file = logs_dir.map(|logs_dir| {
+            let log_file_name = format!("{}-output.log", log_prefix);
+            let log_file_path = logs_dir.join(log_file_name);
+            let log_file = OpenOptions::new()
+                .append(true)
+                .create(true)
+                .open(log_file_path)
+                .expect("Failed to create a log file");
+            Arc::new(Mutex::new(log_file))
+        });
         command.stdout(Stdio::piped()).stderr(Stdio::piped());
 
         log!("Spawning {}...", &self);
@@ -250,17 +260,35 @@ impl Program {
             .unwrap_or_else(|e| panic!("Failed to start {:?} with error: {e}", &self));
         let child_stdout = child.stdout.take().unwrap();
         let filter = self.get_filter();
-        let stdout =
-            spawn(move || prefix_log(child_stdout, log_prefix, &RUN_LOG_WATCHERS, filter, None));
+        let cloned_log_file = log_file.clone();
+        let stdout = spawn(move || {
+            prefix_log(
+                child_stdout,
+                log_prefix,
+                &RUN_LOG_WATCHERS,
+                filter,
+                cloned_log_file,
+                None,
+            )
+        });
         let child_stderr = child.stderr.take().unwrap();
-        let stderr =
-            spawn(move || prefix_log(child_stderr, log_prefix, &RUN_LOG_WATCHERS, filter, None));
+        let stderr = spawn(move || {
+            prefix_log(
+                child_stderr,
+                log_prefix,
+                &RUN_LOG_WATCHERS,
+                filter,
+                None,
+                None,
+            )
+        });
         (
             log_prefix.to_owned(),
             child,
             Box::new(SimpleTaskHandle(stdout)),
             Box::new(SimpleTaskHandle(stderr)),
             self.get_memory(),
+            log_file.clone(),
         )
     }
 
@@ -281,13 +309,13 @@ impl Program {
             let stdout = child.stdout.take().unwrap();
             let name = self.get_bin_name();
             let running = running.clone();
-            spawn(move || prefix_log(stdout, &name, &running, filter, stdout_ch_tx))
+            spawn(move || prefix_log(stdout, &name, &running, filter, None, stdout_ch_tx))
         };
         let stderr = {
             let stderr = child.stderr.take().unwrap();
             let name = self.get_bin_name();
             let running = running.clone();
-            spawn(move || prefix_log(stderr, &name, &running, filter, None))
+            spawn(move || prefix_log(stderr, &name, &running, filter, None, None))
         };
 
         let status = loop {
@@ -321,6 +349,7 @@ fn prefix_log(
     prefix: &str,
     run_log_watcher: &AtomicBool,
     filter: Option<LogFilter>,
+    file: Option<Arc<Mutex<File>>>,
     channel: Option<Sender<String>>,
 ) {
     let mut reader = BufReader::new(output).lines();
@@ -340,6 +369,10 @@ fn prefix_log(
                 }
             }
             println!("<{prefix}> {line}");
+            if let Some(file) = &file {
+                let mut writer = file.lock().expect("Failed to acquire lock for log file");
+                writeln!(writer, "{}", line).unwrap_or(());
+            }
             if let Some(channel) = &channel {
                 // ignore send errors
                 channel.send(line).unwrap_or(());

rust/utils/run-locally/src/solana.rs

@@ -202,7 +202,7 @@ pub fn start_solana_test_validator(
             concat_path(&solana_programs_path, lib).to_str().unwrap(),
         );
     }
-    let validator = args.spawn("SOL");
+    let validator = args.spawn("SOL", None);
     sleep(Duration::from_secs(5));
 
     log!("Deploying the hyperlane programs to solana");

rust/utils/run-locally/src/utils.rs

@@ -1,5 +1,8 @@
+use std::fs::File;
+use std::io::{self, BufRead};
 use std::path::{Path, PathBuf};
 use std::process::Child;
+use std::sync::{Arc, Mutex};
 use std::thread::JoinHandle;
 
 use nix::libc::pid_t;
@@ -54,6 +57,8 @@ pub type AgentHandles = (
     Box<dyn TaskHandle<Output = ()>>,
     // data to drop once program exits
     Box<dyn ArbitraryData>,
+    // file with stdout logs
+    Option<Arc<Mutex<File>>>,
 );
 pub type LogFilter = fn(&str) -> bool;
 
@@ -112,3 +117,16 @@ pub fn stop_child(child: &mut Child) {
         }
     };
 }
+
+pub fn get_matching_lines(file: &File, search_string: &str) -> io::Result<Vec<String>> {
+    let reader = io::BufReader::new(file);
+
+    // Read lines and collect those that contain the search string
+    let matching_lines: Vec<String> = reader
+        .lines()
+        .map_while(Result::ok)
+        .filter(|line| line.contains(search_string))
+        .collect();
+
+    Ok(matching_lines)
+}

solidity/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @hyperlane-xyz/core
 
+## 3.13.0
+
+### Minor Changes
+
+- babe816f8: Support xERC20 and xERC20 Lockbox in SDK and CLI
+- b440d98be: Added support for registering/deregistering from the Hyperlane AVS
+
+### Patch Changes
+
+- Updated dependencies [0cf692e73]
+  - @hyperlane-xyz/utils@3.13.0
+
 ## 3.12.0
 
 ### Patch Changes

solidity/contracts/avs/ECDSAStakeRegistry.sol

@@ -44,13 +44,14 @@ contract ECDSAStakeRegistry is
         __ECDSAStakeRegistry_init(_serviceManager, _thresholdWeight, _quorum);
     }
 
-    /// @notice Registers a new operator using a provided signature
+    /// @notice Registers a new operator using a provided signature and signing key
     /// @param _operatorSignature Contains the operator's signature, salt, and expiry
+    /// @param _signingKey The signing key to add to the operator's history
     function registerOperatorWithSignature(
-        address _operator,
-        ISignatureUtils.SignatureWithSaltAndExpiry memory _operatorSignature
+        ISignatureUtils.SignatureWithSaltAndExpiry memory _operatorSignature,
+        address _signingKey
     ) external {
-        _registerOperatorWithSig(_operator, _operatorSignature);
+        _registerOperatorWithSig(msg.sender, _operatorSignature, _signingKey);
     }
 
     /// @notice Deregisters an existing operator
@@ -58,6 +59,18 @@ contract ECDSAStakeRegistry is
         _deregisterOperator(msg.sender);
     }
 
+    /**
+     * @notice Updates the signing key for an operator
+     * @dev Only callable by the operator themselves
+     * @param _newSigningKey The new signing key to set for the operator
+     */
+    function updateOperatorSigningKey(address _newSigningKey) external {
+        if (!_operatorRegistered[msg.sender]) {
+            revert OperatorNotRegistered();
+        }
+        _updateOperatorSigningKey(msg.sender, _newSigningKey);
+    }
+
     /**
      * @notice Updates the StakeRegistry's view of one or more operators' stakes adding a new entry in their history of stake checkpoints,
      * @dev Queries stakes from the Eigenlayer core DelegationManager contract
@@ -107,18 +120,18 @@ contract ECDSAStakeRegistry is
 
     /// @notice Verifies if the provided signature data is valid for the given data hash.
     /// @param _dataHash The hash of the data that was signed.
-    /// @param _signatureData Encoded signature data consisting of an array of signers, an array of signatures, and a reference block number.
+    /// @param _signatureData Encoded signature data consisting of an array of operators, an array of signatures, and a reference block number.
     /// @return The function selector that indicates the signature is valid according to ERC1271 standard.
     function isValidSignature(
         bytes32 _dataHash,
         bytes memory _signatureData
     ) external view returns (bytes4) {
         (
-            address[] memory signers,
+            address[] memory operators,
             bytes[] memory signatures,
             uint32 referenceBlock
         ) = abi.decode(_signatureData, (address[], bytes[], uint32));
-        _checkSignatures(_dataHash, signers, signatures, referenceBlock);
+        _checkSignatures(_dataHash, operators, signatures, referenceBlock);
         return IERC1271Upgradeable.isValidSignature.selector;
     }
 
@@ -128,6 +141,37 @@ contract ECDSAStakeRegistry is
         return _quorum;
     }
 
+    /**
+     * @notice Retrieves the latest signing key for a given operator.
+     * @param _operator The address of the operator.
+     * @return The latest signing key of the operator.
+     */
+    function getLastestOperatorSigningKey(
+        address _operator
+    ) external view returns (address) {
+        return address(uint160(_operatorSigningKeyHistory[_operator].latest()));
+    }
+
+    /**
+     * @notice Retrieves the latest signing key for a given operator at a specific block number.
+     * @param _operator The address of the operator.
+     * @param _blockNumber The block number to get the operator's signing key.
+     * @return The signing key of the operator at the given block.
+     */
+    function getOperatorSigningKeyAtBlock(
+        address _operator,
+        uint256 _blockNumber
+    ) external view returns (address) {
+        return
+            address(
+                uint160(
+                    _operatorSigningKeyHistory[_operator].getAtBlock(
+                        _blockNumber
+                    )
+                )
+            );
+    }
+
     /// @notice Retrieves the last recorded weight for a given operator.
     /// @param _operator The address of the operator.
     /// @return uint256 - The latest weight of the operator.
@@ -313,9 +357,11 @@ contract ECDSAStakeRegistry is
 
     /// @dev registers an operator through a provided signature
     /// @param _operatorSignature Contains the operator's signature, salt, and expiry
+    /// @param _signingKey The signing key to add to the operator's history
     function _registerOperatorWithSig(
         address _operator,
-        ISignatureUtils.SignatureWithSaltAndExpiry memory _operatorSignature
+        ISignatureUtils.SignatureWithSaltAndExpiry memory _operatorSignature,
+        address _signingKey
     ) internal virtual {
         if (_operatorRegistered[_operator]) {
             revert OperatorAlreadyRegistered();
@@ -324,6 +370,7 @@ contract ECDSAStakeRegistry is
         _operatorRegistered[_operator] = true;
         int256 delta = _updateOperatorWeight(_operator);
         _updateTotalWeight(delta);
+        _updateOperatorSigningKey(_operator, _signingKey);
         IServiceManager(_serviceManager).registerOperatorToAVS(
             _operator,
             _operatorSignature
@@ -331,6 +378,28 @@ contract ECDSAStakeRegistry is
         emit OperatorRegistered(_operator, _serviceManager);
     }
 
+    /// @dev Internal function to update an operator's signing key
+    /// @param _operator The address of the operator to update the signing key for
+    /// @param _newSigningKey The new signing key to set for the operator
+    function _updateOperatorSigningKey(
+        address _operator,
+        address _newSigningKey
+    ) internal {
+        address oldSigningKey = address(
+            uint160(_operatorSigningKeyHistory[_operator].latest())
+        );
+        if (_newSigningKey == oldSigningKey) {
+            return;
+        }
+        _operatorSigningKeyHistory[_operator].push(uint160(_newSigningKey));
+        emit SigningKeyUpdate(
+            _operator,
+            block.number,
+            _newSigningKey,
+            oldSigningKey
+        );
+    }
+
     /// @notice Updates the weight of an operator and returns the previous and current weights.
     /// @param _operator The address of the operator to update the weight of.
     function _updateOperatorWeight(
@@ -401,30 +470,33 @@ contract ECDSAStakeRegistry is
     /**
      * @notice Common logic to verify a batch of ECDSA signatures against a hash, using either last stake weight or at a specific block.
      * @param _dataHash The hash of the data the signers endorsed.
-     * @param _signers A collection of addresses that endorsed the data hash.
+     * @param _operators A collection of addresses that endorsed the data hash.
      * @param _signatures A collection of signatures matching the signers.
      * @param _referenceBlock The block number for evaluating stake weight; use max uint32 for latest weight.
      */
     function _checkSignatures(
         bytes32 _dataHash,
-        address[] memory _signers,
+        address[] memory _operators,
         bytes[] memory _signatures,
         uint32 _referenceBlock
     ) internal view {
-        uint256 signersLength = _signers.length;
-        address lastSigner;
+        uint256 signersLength = _operators.length;
+        address currentOperator;
+        address lastOperator;
+        address signer;
         uint256 signedWeight;
 
         _validateSignaturesLength(signersLength, _signatures.length);
         for (uint256 i; i < signersLength; i++) {
-            address currentSigner = _signers[i];
+            currentOperator = _operators[i];
+            signer = _getOperatorSigningKey(currentOperator, _referenceBlock);
 
-            _validateSortedSigners(lastSigner, currentSigner);
-            _validateSignature(currentSigner, _dataHash, _signatures[i]);
+            _validateSortedSigners(lastOperator, currentOperator);
+            _validateSignature(signer, _dataHash, _signatures[i]);
 
-            lastSigner = currentSigner;
+            lastOperator = currentOperator;
             uint256 operatorWeight = _getOperatorWeight(
-                currentSigner,
+                currentOperator,
                 _referenceBlock
             );
             signedWeight += operatorWeight;
@@ -474,6 +546,27 @@ contract ECDSAStakeRegistry is
         }
     }
 
+    /// @notice Retrieves the operator weight for a signer, either at the last checkpoint or a specified block.
+    /// @param _operator The operator to query their signing key history for
+    /// @param _referenceBlock The block number to query the operator's weight at, or the maximum uint32 value for the last checkpoint.
+    /// @return The weight of the operator.
+    function _getOperatorSigningKey(
+        address _operator,
+        uint32 _referenceBlock
+    ) internal view returns (address) {
+        if (_referenceBlock >= block.number) {
+            revert InvalidReferenceBlock();
+        }
+        return
+            address(
+                uint160(
+                    _operatorSigningKeyHistory[_operator].getAtBlock(
+                        _referenceBlock
+                    )
+                )
+            );
+    }
+
     /// @notice Retrieves the operator weight for a signer, either at the last checkpoint or a specified block.
     /// @param _signer The address of the signer whose weight is returned.
     /// @param _referenceBlock The block number to query the operator's weight at, or the maximum uint32 value for the last checkpoint.
@@ -482,11 +575,10 @@ contract ECDSAStakeRegistry is
         address _signer,
         uint32 _referenceBlock
     ) internal view returns (uint256) {
-        if (_referenceBlock == type(uint32).max) {
-            return _operatorWeightHistory[_signer].latest();
-        } else {
-            return _operatorWeightHistory[_signer].getAtBlock(_referenceBlock);
+        if (_referenceBlock >= block.number) {
+            revert InvalidReferenceBlock();
         }
+        return _operatorWeightHistory[_signer].getAtBlock(_referenceBlock);
     }
 
     /// @notice Retrieve the total stake weight at a specific block or the latest if not specified.
@@ -496,11 +588,10 @@ contract ECDSAStakeRegistry is
     function _getTotalWeight(
         uint32 _referenceBlock
     ) internal view returns (uint256) {
-        if (_referenceBlock == type(uint32).max) {
-            return _totalWeightHistory.latest();
-        } else {
-            return _totalWeightHistory.getAtBlock(_referenceBlock);
+        if (_referenceBlock >= block.number) {
+            revert InvalidReferenceBlock();
         }
+        return _totalWeightHistory.getAtBlock(_referenceBlock);
     }
 
     /// @notice Retrieves the threshold stake for a given reference block.
@@ -510,11 +601,10 @@ contract ECDSAStakeRegistry is
     function _getThresholdStake(
         uint32 _referenceBlock
     ) internal view returns (uint256) {
-        if (_referenceBlock == type(uint32).max) {
-            return _thresholdWeightHistory.latest();
-        } else {
-            return _thresholdWeightHistory.getAtBlock(_referenceBlock);
+        if (_referenceBlock >= block.number) {
+            revert InvalidReferenceBlock();
         }
+        return _thresholdWeightHistory.getAtBlock(_referenceBlock);
     }
 
     /// @notice Validates that the cumulative stake of signed messages meets or exceeds the required threshold.

solidity/contracts/avs/ECDSAStakeRegistryStorage.sol

@@ -30,6 +30,10 @@ abstract contract ECDSAStakeRegistryStorage is
     /// @notice Defines the duration after which the stake's weight expires.
     uint256 internal _stakeExpiry;
 
+    /// @notice Maps an operator to their signing key history using checkpoints
+    mapping(address => CheckpointsUpgradeable.History)
+        internal _operatorSigningKeyHistory;
+
     /// @notice Tracks the total stake history over time using checkpoints
     CheckpointsUpgradeable.History internal _totalWeightHistory;
 
@@ -51,5 +55,5 @@ abstract contract ECDSAStakeRegistryStorage is
     // slither-disable-next-line shadowing-state
     /// @dev Reserves storage slots for future upgrades
     // solhint-disable-next-line
-    uint256[42] private __gap;
+    uint256[40] private __gap;
 }

solidity/contracts/interfaces/avs/vendored/IECDSAStakeRegistryEventsAndErrors.sol

@@ -12,8 +12,6 @@ struct Quorum {
     StrategyParams[] strategies; // An array of strategy parameters to define the quorum
 }
 
-/// part of mock interfaces for vendoring necessary Eigenlayer contracts for the hyperlane AVS
-/// @author Layr Labs, Inc.
 interface IECDSAStakeRegistryEventsAndErrors {
     /// @notice Emitted when the system registers an operator
     /// @param _operator The address of the registered operator
@@ -61,7 +59,19 @@ interface IECDSAStakeRegistryEventsAndErrors {
     /// @notice Emits when setting a new threshold weight.
     event ThresholdWeightUpdated(uint256 _thresholdWeight);
 
+    /// @notice Emitted when an operator's signing key is updated
+    /// @param operator The address of the operator whose signing key was updated
+    /// @param updateBlock The block number at which the signing key was updated
+    /// @param newSigningKey The operator's signing key after the update
+    /// @param oldSigningKey The operator's signing key before the update
+    event SigningKeyUpdate(
+        address indexed operator,
+        uint256 indexed updateBlock,
+        address indexed newSigningKey,
+        address oldSigningKey
+    );
     /// @notice Indicates when the lengths of the signers array and signatures array do not match.
+
     error LengthMismatch();
 
     /// @notice Indicates encountering an invalid length for the signers or signatures array.
@@ -76,6 +86,9 @@ interface IECDSAStakeRegistryEventsAndErrors {
     /// @notice Thrown when missing operators in an update
     error MustUpdateAllOperators();
 
+    /// @notice Reference blocks must be for blocks that have already been confirmed
+    error InvalidReferenceBlock();
+
     /// @notice Indicates operator weights were out of sync and the signed weight exceed the total
     error InvalidSignedWeight();
 

solidity/contracts/test/ERC20Test.sol

@@ -3,6 +3,7 @@ pragma solidity >=0.8.0;
 
 import "@openzeppelin/contracts/token/ERC20/ERC20.sol";
 
+import "../token/interfaces/IXERC20Lockbox.sol";
 import "../token/interfaces/IXERC20.sol";
 import "../token/interfaces/IFiatToken.sol";
 
@@ -66,15 +67,50 @@ contract XERC20Test is ERC20Test, IXERC20 {
         _burn(account, amount);
     }
 
-    function setLimits(
-        address _bridge,
-        uint256 _mintingLimit,
-        uint256 _burningLimit
-    ) external {
-        require(false);
+    function setLimits(address, uint256, uint256) external pure {
+        assert(false);
     }
 
-    function owner() external returns (address) {
+    function owner() external pure returns (address) {
         return address(0x0);
     }
 }
+
+contract XERC20LockboxTest is IXERC20Lockbox {
+    IXERC20 public immutable XERC20;
+    IERC20 public immutable ERC20;
+
+    constructor(
+        string memory name,
+        string memory symbol,
+        uint256 totalSupply,
+        uint8 __decimals
+    ) {
+        ERC20Test erc20 = new ERC20Test(name, symbol, totalSupply, __decimals);
+        erc20.transfer(msg.sender, totalSupply);
+        ERC20 = erc20;
+        XERC20 = new XERC20Test(name, symbol, 0, __decimals);
+    }
+
+    function depositTo(address _user, uint256 _amount) public {
+        ERC20.transferFrom(msg.sender, address(this), _amount);
+        XERC20.mint(_user, _amount);
+    }
+
+    function deposit(uint256 _amount) external {
+        depositTo(msg.sender, _amount);
+    }
+
+    function depositNativeTo(address) external payable {
+        assert(false);
+    }
+
+    function withdrawTo(address _user, uint256 _amount) public {
+        XERC20.burn(msg.sender, _amount);
+        ERC20Test(address(ERC20)).mintTo(_user, _amount);
+    }
+
+    function withdraw(uint256 _amount) external {
+        withdrawTo(msg.sender, _amount);
+    }
+}

solidity/contracts/token/README.md

@@ -6,7 +6,7 @@ For instructions on deploying Warp Routes, see [the deployment documentation](ht
 
 ## Warp Route Architecture
 
-A Warp Route is a collection of [`TokenRouter`](./contracts/libs/TokenRouter.sol) contracts deployed across a set of Hyperlane chains. These contracts leverage the `Router` pattern to implement access control and routing logic for remote token transfers. These contracts send and receive [`Messages`](./contracts/libs/Message.sol) which encode payloads containing a transfer `amount` and `recipient` address.
+A Warp Route is a collection of [`TokenRouter`](./libs/TokenRouter.sol) contracts deployed across a set of Hyperlane chains. These contracts leverage the `Router` pattern to implement access control and routing logic for remote token transfers. These contracts send and receive [`Messages`](./libs/TokenMessage.sol) which encode payloads containing a transfer `amount` and `recipient` address.
 
 ```mermaid
 %%{ init: {
@@ -39,7 +39,7 @@ graph LR
         Mailbox_G[(Mailbox)]
     end
 
-    HYP_E -. "router" .- HYP_P -. "router" .- HYP_G
+    HYP_E -. "TokenMessage" .- HYP_P -. "TokenMessage" .- HYP_G
 
 ```
 

solidity/contracts/token/extensions/HypXERC20Lockbox.sol

@@ -17,18 +17,39 @@ contract HypXERC20Lockbox is HypERC20Collateral {
     ) HypERC20Collateral(address(IXERC20Lockbox(_lockbox).ERC20()), _mailbox) {
         lockbox = IXERC20Lockbox(_lockbox);
         xERC20 = lockbox.XERC20();
+        approveLockbox();
+    }
 
-        // grant infinite approvals to lockbox
+    /**
+     * @notice Approve the lockbox to spend the wrapped token and xERC20
+     * @dev This function is idempotent and need not be access controlled
+     */
+    function approveLockbox() public {
         require(
-            IERC20(wrappedToken).approve(_lockbox, MAX_INT),
+            IERC20(wrappedToken).approve(address(lockbox), MAX_INT),
             "erc20 lockbox approve failed"
         );
         require(
-            xERC20.approve(_lockbox, MAX_INT),
+            xERC20.approve(address(lockbox), MAX_INT),
             "xerc20 lockbox approve failed"
         );
     }
 
+    /**
+     * @notice Initialize the contract
+     * @param _hook The address of the hook contract
+     * @param _ism The address of the interchain security module
+     * @param _owner The address of the owner
+     */
+    function initialize(
+        address _hook,
+        address _ism,
+        address _owner
+    ) public override initializer {
+        approveLockbox();
+        _MailboxClient_initialize(_hook, _ism, _owner);
+    }
+
     function _transferFromSender(
         uint256 _amount
     ) internal override returns (bytes memory) {

solidity/coverage.sh

@@ -14,7 +14,7 @@ fi
 lcov --version
 
 # exclude FastTokenRouter until https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/2806
-EXCLUDE="*test* *mock* *node_modules* *FastHyp*"
+EXCLUDE="*test* *mock* *node_modules* *script* *FastHyp*"
 lcov \
     --rc lcov_branch_coverage=1 \
     --remove lcov.info $EXCLUDE \

solidity/foundry.toml

@@ -14,7 +14,11 @@ fs_permissions = [
     { access = "read", path = "./script/avs/"},
     { access = "write", path = "./fixtures" }
 ]
-ignored_warnings_from = ['fx-portal']
+ignored_warnings_from = [
+    'lib',
+    'test',
+    'contracts/test'
+]
 
 [profile.ci]
 verbosity = 4

solidity/lib/forge-std

@@ -1 +1 @@
-Subproject commit e8a047e3f40f13fa37af6fe14e6e06283d9a060e
+Subproject commit 52715a217dc51d0de15877878ab8213f6cbbbab5

solidity/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@hyperlane-xyz/core",
   "description": "Core solidity contracts for Hyperlane",
-  "version": "3.12.2",
+  "version": "3.13.0",
   "dependencies": {
     "@eth-optimism/contracts": "^0.6.0",
-    "@hyperlane-xyz/utils": "3.12.2",
+    "@hyperlane-xyz/utils": "3.13.0",
     "@layerzerolabs/lz-evm-oapp-v2": "2.0.2",
     "@openzeppelin/contracts": "^4.9.3",
     "@openzeppelin/contracts-upgradeable": "^v4.9.3",

solidity/script/avs/DeployAVS.s.sol

@@ -12,6 +12,7 @@ import {ProxyAdmin} from "../../contracts/upgrade/ProxyAdmin.sol";
 import {TransparentUpgradeableProxy} from "../../contracts/upgrade/TransparentUpgradeableProxy.sol";
 import {ECDSAStakeRegistry} from "../../contracts/avs/ECDSAStakeRegistry.sol";
 import {Quorum, StrategyParams} from "../../contracts/interfaces/avs/vendored/IECDSAStakeRegistryEventsAndErrors.sol";
+import {ECDSAServiceManagerBase} from "../../contracts/avs/ECDSAServiceManagerBase.sol";
 import {HyperlaneServiceManager} from "../../contracts/avs/HyperlaneServiceManager.sol";
 
 import {TestPaymentCoordinator} from "../../contracts/test/avs/TestPaymentCoordinator.sol";
@@ -42,6 +43,11 @@ contract DeployAVS is Script {
         );
         string memory json = vm.readFile(path);
 
+        proxyAdmin = ProxyAdmin(
+            json.readAddress(
+                string(abi.encodePacked(".", targetEnv, ".proxyAdmin"))
+            )
+        );
         avsDirectory = IAVSDirectory(
             json.readAddress(
                 string(abi.encodePacked(".", targetEnv, ".avsDirectory"))
@@ -88,15 +94,14 @@ contract DeployAVS is Script {
         }
     }
 
-    function run(string memory network) external {
+    function run(string memory network, string memory metadataUri) external {
         deployerPrivateKey = vm.envUint("DEPLOYER_PRIVATE_KEY");
+        address deployerAddress = vm.addr(deployerPrivateKey);
 
         _loadEigenlayerAddresses(network);
 
         vm.startBroadcast(deployerPrivateKey);
 
-        proxyAdmin = new ProxyAdmin();
-
         ECDSAStakeRegistry stakeRegistryImpl = new ECDSAStakeRegistry(
             delegationManager
         );
@@ -118,7 +123,7 @@ contract DeployAVS is Script {
             address(proxyAdmin),
             abi.encodeWithSelector(
                 HyperlaneServiceManager.initialize.selector,
-                msg.sender
+                address(deployerAddress)
             )
         );
 
@@ -131,7 +136,24 @@ contract DeployAVS is Script {
                 quorum
             )
         );
+
+        HyperlaneServiceManager hsm = HyperlaneServiceManager(
+            address(hsmProxy)
+        );
+
         require(success, "Failed to initialize ECDSAStakeRegistry");
+        require(
+            ECDSAStakeRegistry(address(stakeRegistryProxy)).owner() ==
+                address(deployerAddress),
+            "Owner of ECDSAStakeRegistry is not the deployer"
+        );
+        require(
+            HyperlaneServiceManager(address(hsmProxy)).owner() ==
+                address(deployerAddress),
+            "Owner of HyperlaneServiceManager is not the deployer"
+        );
+
+        hsm.updateAVSMetadataURI(metadataUri);
 
         console.log(
             "ECDSAStakeRegistry Implementation: ",

solidity/script/avs/eigenlayer_addresses.json

@@ -1,5 +1,6 @@
 {
   "ethereum": {
+    "proxyAdmin": "0x75EE15Ee1B4A75Fa3e2fDF5DF3253c25599cc659",
     "delegationManager": "0x39053D51B77DC0d36036Fc1fCc8Cb819df8Ef37A",
     "avsDirectory": "0x135DDa560e946695d6f155dACaFC6f1F25C1F5AF",
     "paymentCoordinator": "",
@@ -19,6 +20,7 @@
     ]
   },
   "holesky": {
+    "proxyAdmin": "0x33dB966328Ea213b0f76eF96CA368AB37779F065",
     "delegationManager": "0xA44151489861Fe9e3055d95adC98FbD462B948e7",
     "avsDirectory": "0x055733000064333CaDDbC92763c58BF0192fFeBf",
     "paymentCoordinator": "",

solidity/script/xerc20/.env.blast

@@ -0,0 +1,4 @@
+export ROUTER_ADDRESS=0xA34ceDf9068C5deE726C67A4e1DCfCc2D6E2A7fD
+export ERC20_ADDRESS=0x2416092f143378750bb29b79eD961ab195CcEea5
+export XERC20_ADDRESS=0x2416092f143378750bb29b79eD961ab195CcEea5
+export RPC_URL="https://rpc.blast.io"

solidity/script/xerc20/.env.ethereum

@@ -0,0 +1,5 @@
+export ROUTER_ADDRESS=0x8dfbEA2582F41c8C4Eb25252BbA392fd3c09449A
+export ADMIN_ADDRESS=0xa5B0D537CeBE97f087Dc5FE5732d70719caaEc1D
+export ERC20_ADDRESS=0xbf5495Efe5DB9ce00f80364C8B423567e58d2110
+export XERC20_ADDRESS=0x2416092f143378750bb29b79eD961ab195CcEea5
+export RPC_URL="https://eth.merkle.io"

solidity/script/xerc20/ApproveLockbox.s.sol

@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0
+pragma solidity >=0.8.0;
+
+import "forge-std/Script.sol";
+
+import {AnvilRPC} from "test/AnvilRPC.sol";
+import {TypeCasts} from "contracts/libs/TypeCasts.sol";
+
+import {ITransparentUpgradeableProxy} from "@openzeppelin/contracts/proxy/transparent/TransparentUpgradeableProxy.sol";
+
+import {ProxyAdmin} from "contracts/upgrade/ProxyAdmin.sol";
+
+import {HypXERC20Lockbox} from "contracts/token/extensions/HypXERC20Lockbox.sol";
+import {IXERC20Lockbox} from "contracts/token/interfaces/IXERC20Lockbox.sol";
+import {IXERC20} from "contracts/token/interfaces/IXERC20.sol";
+import {IERC20} from "contracts/token/interfaces/IXERC20.sol";
+
+// source .env.<CHAIN>
+// forge script ApproveLockbox.s.sol --broadcast --rpc-url localhost:XXXX
+contract ApproveLockbox is Script {
+    address router = vm.envAddress("ROUTER_ADDRESS");
+    address admin = vm.envAddress("ADMIN_ADDRESS");
+    uint256 deployerPrivateKey = vm.envUint("PRIVATE_KEY");
+
+    ITransparentUpgradeableProxy proxy = ITransparentUpgradeableProxy(router);
+    HypXERC20Lockbox old = HypXERC20Lockbox(router);
+    address lockbox = address(old.lockbox());
+    address mailbox = address(old.mailbox());
+    ProxyAdmin proxyAdmin = ProxyAdmin(admin);
+
+    function run() external {
+        assert(proxyAdmin.getProxyAdmin(proxy) == admin);
+
+        vm.startBroadcast(deployerPrivateKey);
+        HypXERC20Lockbox logic = new HypXERC20Lockbox(lockbox, mailbox);
+        proxyAdmin.upgradeAndCall(
+            proxy,
+            address(logic),
+            abi.encodeCall(HypXERC20Lockbox.approveLockbox, ())
+        );
+        vm.stopBroadcast();
+
+        vm.expectRevert("Initializable: contract is already initialized");
+        HypXERC20Lockbox(address(proxy)).initialize(
+            address(0),
+            address(0),
+            mailbox
+        );
+    }
+}

solidity/script/xerc20/GrantLimits.s.sol

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0
+pragma solidity >=0.8.0;
+
+import "forge-std/Script.sol";
+
+import {AnvilRPC} from "test/AnvilRPC.sol";
+
+import {IXERC20Lockbox} from "contracts/token/interfaces/IXERC20Lockbox.sol";
+import {IXERC20} from "contracts/token/interfaces/IXERC20.sol";
+import {IERC20} from "contracts/token/interfaces/IXERC20.sol";
+
+// source .env.<CHAIN>
+// anvil --fork-url $RPC_URL --port XXXX
+// forge script GrantLimits.s.sol --broadcast --unlocked --rpc-url localhost:XXXX
+contract GrantLimits is Script {
+    address tester = 0xa7ECcdb9Be08178f896c26b7BbD8C3D4E844d9Ba;
+    uint256 amount = 1 gwei;
+
+    address router = vm.envAddress("ROUTER_ADDRESS");
+    IERC20 erc20 = IERC20(vm.envAddress("ERC20_ADDRESS"));
+    IXERC20 xerc20 = IXERC20(vm.envAddress("XERC20_ADDRESS"));
+
+    function runFrom(address account) internal {
+        AnvilRPC.setBalance(account, 1 ether);
+        AnvilRPC.impersonateAccount(account);
+        vm.broadcast(account);
+    }
+
+    function run() external {
+        address owner = xerc20.owner();
+        runFrom(owner);
+        xerc20.setLimits(router, amount, amount);
+
+        runFrom(address(erc20));
+        erc20.transfer(tester, amount);
+    }
+}

solidity/script/xerc20/ezETH.s.sol

@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0
+pragma solidity >=0.8.0;
+
+import "forge-std/Script.sol";
+
+import {IXERC20Lockbox} from "../../contracts/token/interfaces/IXERC20Lockbox.sol";
+import {IXERC20} from "../../contracts/token/interfaces/IXERC20.sol";
+import {IERC20} from "../../contracts/token/interfaces/IXERC20.sol";
+import {HypXERC20Lockbox} from "../../contracts/token/extensions/HypXERC20Lockbox.sol";
+import {HypERC20Collateral} from "../../contracts/token/HypERC20Collateral.sol";
+import {HypXERC20} from "../../contracts/token/extensions/HypXERC20.sol";
+import {TransparentUpgradeableProxy} from "../../contracts/upgrade/TransparentUpgradeableProxy.sol";
+import {ProxyAdmin} from "../../contracts/upgrade/ProxyAdmin.sol";
+
+import {TypeCasts} from "../../contracts/libs/TypeCasts.sol";
+import {TokenMessage} from "../../contracts/token/libs/TokenMessage.sol";
+
+contract ezETH is Script {
+    using TypeCasts for address;
+
+    string ETHEREUM_RPC_URL = vm.envString("ETHEREUM_RPC_URL");
+    string BLAST_RPC_URL = vm.envString("BLAST_RPC_URL");
+
+    uint256 ethereumFork;
+    uint32 ethereumDomainId = 1;
+    address ethereumMailbox = 0xc005dc82818d67AF737725bD4bf75435d065D239;
+    address ethereumLockbox = 0xC8140dA31E6bCa19b287cC35531c2212763C2059;
+
+    uint256 blastFork;
+    uint32 blastDomainId = 81457;
+    address blastXERC20 = 0x2416092f143378750bb29b79eD961ab195CcEea5;
+    address blastMailbox = 0x3a867fCfFeC2B790970eeBDC9023E75B0a172aa7;
+
+    uint256 amount = 100;
+
+    function setUp() public {
+        ethereumFork = vm.createFork(ETHEREUM_RPC_URL);
+        blastFork = vm.createFork(BLAST_RPC_URL);
+    }
+
+    function run() external {
+        address deployer = address(this);
+        bytes32 recipient = deployer.addressToBytes32();
+        bytes memory tokenMessage = TokenMessage.format(recipient, amount, "");
+        vm.selectFork(ethereumFork);
+        HypXERC20Lockbox hypXERC20Lockbox = new HypXERC20Lockbox(
+            ethereumLockbox,
+            ethereumMailbox
+        );
+        ProxyAdmin ethAdmin = new ProxyAdmin();
+        TransparentUpgradeableProxy ethProxy = new TransparentUpgradeableProxy(
+            address(hypXERC20Lockbox),
+            address(ethAdmin),
+            abi.encodeCall(
+                HypXERC20Lockbox.initialize,
+                (address(0), address(0), deployer)
+            )
+        );
+        hypXERC20Lockbox = HypXERC20Lockbox(address(ethProxy));
+
+        vm.selectFork(blastFork);
+        HypXERC20 hypXERC20 = new HypXERC20(blastXERC20, blastMailbox);
+        ProxyAdmin blastAdmin = new ProxyAdmin();
+        TransparentUpgradeableProxy blastProxy = new TransparentUpgradeableProxy(
+                address(hypXERC20),
+                address(blastAdmin),
+                abi.encodeCall(
+                    HypERC20Collateral.initialize,
+                    (address(0), address(0), deployer)
+                )
+            );
+        hypXERC20 = HypXERC20(address(blastProxy));
+        hypXERC20.enrollRemoteRouter(
+            ethereumDomainId,
+            address(hypXERC20Lockbox).addressToBytes32()
+        );
+
+        // grant `amount` mint and burn limit to warp route
+        vm.prank(IXERC20(blastXERC20).owner());
+        IXERC20(blastXERC20).setLimits(address(hypXERC20), amount, amount);
+
+        // test sending `amount` on warp route
+        vm.prank(0x7BE481D464CAD7ad99500CE8A637599eB8d0FCDB); // ezETH whale
+        IXERC20(blastXERC20).transfer(address(this), amount);
+        IXERC20(blastXERC20).approve(address(hypXERC20), amount);
+        uint256 value = hypXERC20.quoteGasPayment(ethereumDomainId);
+        hypXERC20.transferRemote{value: value}(
+            ethereumDomainId,
+            recipient,
+            amount
+        );
+
+        // test receiving `amount` on warp route
+        vm.prank(blastMailbox);
+        hypXERC20.handle(
+            ethereumDomainId,
+            address(hypXERC20Lockbox).addressToBytes32(),
+            tokenMessage
+        );
+
+        vm.selectFork(ethereumFork);
+        hypXERC20Lockbox.enrollRemoteRouter(
+            blastDomainId,
+            address(hypXERC20).addressToBytes32()
+        );
+
+        // grant `amount` mint and burn limit to warp route
+        IXERC20 ethereumXERC20 = hypXERC20Lockbox.xERC20();
+        vm.prank(ethereumXERC20.owner());
+        ethereumXERC20.setLimits(address(hypXERC20Lockbox), amount, amount);
+
+        // test sending `amount` on warp route
+        IERC20 erc20 = IXERC20Lockbox(ethereumLockbox).ERC20();
+        vm.prank(ethereumLockbox);
+        erc20.transfer(address(this), amount);
+        erc20.approve(address(hypXERC20Lockbox), amount);
+        hypXERC20Lockbox.transferRemote(blastDomainId, recipient, amount);
+
+        // test receiving `amount` on warp route
+        vm.prank(ethereumMailbox);
+        hypXERC20Lockbox.handle(
+            blastDomainId,
+            address(hypXERC20).addressToBytes32(),
+            tokenMessage
+        );
+    }
+}

solidity/test/AnvilRPC.sol

@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0
+pragma solidity >=0.8.0;
+
+import "forge-std/Vm.sol";
+import {Strings} from "@openzeppelin/contracts/utils/Strings.sol";
+
+// see https://book.getfoundry.sh/reference/anvil/#supported-rpc-methods
+library AnvilRPC {
+    using Strings for address;
+    using Strings for uint256;
+
+    using AnvilRPC for string;
+    using AnvilRPC for string[1];
+    using AnvilRPC for string[2];
+    using AnvilRPC for string[3];
+
+    Vm private constant vm =
+        Vm(address(uint160(uint256(keccak256("hevm cheat code")))));
+
+    string private constant OPEN_ARRAY = "[";
+    string private constant CLOSE_ARRAY = "]";
+    string private constant COMMA = ",";
+    string private constant EMPTY_ARRAY = "[]";
+
+    function escaped(
+        string memory value
+    ) internal pure returns (string memory) {
+        return string.concat(ESCAPED_QUOTE, value, ESCAPED_QUOTE);
+    }
+
+    function toString(
+        string[1] memory values
+    ) internal pure returns (string memory) {
+        return string.concat(OPEN_ARRAY, values[0], CLOSE_ARRAY);
+    }
+
+    function toString(
+        string[2] memory values
+    ) internal pure returns (string memory) {
+        return
+            string.concat(OPEN_ARRAY, values[0], COMMA, values[1], CLOSE_ARRAY);
+    }
+
+    function toString(
+        string[3] memory values
+    ) internal pure returns (string memory) {
+        return
+            string.concat(
+                OPEN_ARRAY,
+                values[0],
+                COMMA,
+                values[1],
+                COMMA,
+                values[2],
+                CLOSE_ARRAY
+            );
+    }
+
+    function impersonateAccount(address account) internal {
+        vm.rpc(
+            "anvil_impersonateAccount",
+            [account.toHexString().escaped()].toString()
+        );
+    }
+
+    function setBalance(address account, uint256 balance) internal {
+        vm.rpc(
+            "anvil_setBalance",
+            [account.toHexString().escaped(), balance.toString()].toString()
+        );
+    }
+
+    function setCode(address account, bytes memory code) internal {
+        vm.rpc(
+            "anvil_setCode",
+            [account.toHexString().escaped(), string(code).escaped()].toString()
+        );
+    }
+
+    function setStorageAt(
+        address account,
+        uint256 slot,
+        uint256 value
+    ) internal {
+        vm.rpc(
+            "anvil_setStorageAt",
+            [
+                account.toHexString().escaped(),
+                slot.toHexString(),
+                value.toHexString()
+            ].toString()
+        );
+    }
+
+    function resetFork(string memory rpcUrl) internal {
+        string memory obj = string.concat(
+            // solhint-disable-next-line quotes
+            '{"forking":{"jsonRpcUrl":',
+            string(rpcUrl).escaped(),
+            "}}"
+        );
+        vm.rpc("anvil_reset", [obj].toString());
+    }
+}
+
+// here to prevent syntax highlighting from breaking
+string constant ESCAPED_QUOTE = '"';

solidity/test/avs/HyperlaneServiceManager.t.sol

@@ -29,6 +29,7 @@ contract HyperlaneServiceManagerTest is EigenlayerBase {
     // Operator info
     uint256 operatorPrivateKey = 0xdeadbeef;
     address operator;
+    address avsSigningKey = address(0xc0ffee);
 
     bytes32 emptySalt;
     uint256 maxExpiry = type(uint256).max;
@@ -97,9 +98,11 @@ contract HyperlaneServiceManagerTest is EigenlayerBase {
                 emptySalt,
                 maxExpiry
             );
+
+        vm.prank(operator);
         _ecdsaStakeRegistry.registerOperatorWithSignature(
-            operator,
-            operatorSignature
+            operatorSignature,
+            avsSigningKey
         );
 
         // assert
@@ -122,12 +125,13 @@ contract HyperlaneServiceManagerTest is EigenlayerBase {
                 maxExpiry
             );
 
+        vm.prank(operator);
         vm.expectRevert(
             "EIP1271SignatureUtils.checkSignature_EIP1271: signature not from signer"
         );
         _ecdsaStakeRegistry.registerOperatorWithSignature(
-            operator,
-            operatorSignature
+            operatorSignature,
+            avsSigningKey
         );
 
         // assert
@@ -409,9 +413,10 @@ contract HyperlaneServiceManagerTest is EigenlayerBase {
                 maxExpiry
             );
 
+        vm.prank(operator);
         _ecdsaStakeRegistry.registerOperatorWithSignature(
-            operator,
-            operatorSignature
+            operatorSignature,
+            avsSigningKey
         );
     }
 

solidity/test/token/HypERC20.t.sol

@@ -19,13 +19,14 @@ import {TransparentUpgradeableProxy} from "@openzeppelin/contracts/proxy/transpa
 import {Mailbox} from "../../contracts/Mailbox.sol";
 import {TypeCasts} from "../../contracts/libs/TypeCasts.sol";
 import {TestMailbox} from "../../contracts/test/TestMailbox.sol";
-import {XERC20Test, FiatTokenTest, ERC20Test} from "../../contracts/test/ERC20Test.sol";
+import {XERC20LockboxTest, XERC20Test, FiatTokenTest, ERC20Test} from "../../contracts/test/ERC20Test.sol";
 import {TestPostDispatchHook} from "../../contracts/test/TestPostDispatchHook.sol";
 import {TestInterchainGasPaymaster} from "../../contracts/test/TestInterchainGasPaymaster.sol";
 import {GasRouter} from "../../contracts/client/GasRouter.sol";
 
 import {HypERC20} from "../../contracts/token/HypERC20.sol";
 import {HypERC20Collateral} from "../../contracts/token/HypERC20Collateral.sol";
+import {HypXERC20Lockbox} from "../../contracts/token/extensions/HypXERC20Lockbox.sol";
 import {IXERC20} from "../../contracts/token/interfaces/IXERC20.sol";
 import {IFiatToken} from "../../contracts/token/interfaces/IFiatToken.sol";
 import {HypXERC20} from "../../contracts/token/extensions/HypXERC20.sol";
@@ -442,6 +443,80 @@ contract HypXERC20Test is HypTokenTest {
     }
 }
 
+contract HypXERC20LockboxTest is HypTokenTest {
+    using TypeCasts for address;
+    HypXERC20Lockbox internal xerc20Lockbox;
+
+    function setUp() public override {
+        super.setUp();
+
+        XERC20LockboxTest lockbox = new XERC20LockboxTest(
+            NAME,
+            SYMBOL,
+            TOTAL_SUPPLY,
+            DECIMALS
+        );
+        primaryToken = ERC20Test(address(lockbox.ERC20()));
+
+        localToken = new HypXERC20Lockbox(
+            address(lockbox),
+            address(localMailbox)
+        );
+        xerc20Lockbox = HypXERC20Lockbox(address(localToken));
+
+        xerc20Lockbox.enrollRemoteRouter(
+            DESTINATION,
+            address(remoteToken).addressToBytes32()
+        );
+
+        primaryToken.transfer(ALICE, 1000e18);
+
+        _enrollRemoteTokenRouter();
+    }
+
+    uint256 constant MAX_INT = 2 ** 256 - 1;
+
+    function testApproval() public {
+        assertEq(
+            xerc20Lockbox.xERC20().allowance(
+                address(localToken),
+                address(xerc20Lockbox.lockbox())
+            ),
+            MAX_INT
+        );
+        assertEq(
+            xerc20Lockbox.wrappedToken().allowance(
+                address(localToken),
+                address(xerc20Lockbox.lockbox())
+            ),
+            MAX_INT
+        );
+    }
+
+    function testRemoteTransfer() public {
+        uint256 balanceBefore = localToken.balanceOf(ALICE);
+
+        vm.prank(ALICE);
+        primaryToken.approve(address(localToken), TRANSFER_AMT);
+        vm.expectCall(
+            address(xerc20Lockbox.xERC20()),
+            abi.encodeCall(IXERC20.burn, (address(localToken), TRANSFER_AMT))
+        );
+        _performRemoteTransferWithEmit(REQUIRED_VALUE, TRANSFER_AMT, 0);
+        assertEq(localToken.balanceOf(ALICE), balanceBefore - TRANSFER_AMT);
+    }
+
+    function testHandle() public {
+        uint256 balanceBefore = localToken.balanceOf(ALICE);
+        vm.expectCall(
+            address(xerc20Lockbox.xERC20()),
+            abi.encodeCall(IXERC20.mint, (address(localToken), TRANSFER_AMT))
+        );
+        _handleLocalTransfer(TRANSFER_AMT);
+        assertEq(localToken.balanceOf(ALICE), balanceBefore + TRANSFER_AMT);
+    }
+}
+
 contract HypFiatTokenTest is HypTokenTest {
     using TypeCasts for address;
     HypFiatToken internal fiatToken;

tools/grafana/easy-relayer-dashboard-external.json

@@ -0,0 +1,436 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 66,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "grafanacloud-prom"
+      },
+      "description": "There shouldn't be abrupt changes, especially for a specific pair",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 78,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "grafanacloud-prom"
+          },
+          "editorMode": "code",
+          "expr": "sum by (origin,remote)(round(increase(hyperlane_messages_processed_count[5m])))",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "{{hyperlane_deployment}}: {{origin}}->{{remote}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Messages Processed",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "grafanacloud-prom"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "grafanacloud-prom"
+          },
+          "editorMode": "code",
+          "expr": "sum by (remote, queue_name)(\n    hyperlane_submitter_queue_length{queue_name=\"prepare_queue\"}\n)",
+          "interval": "",
+          "legendFormat": "{{hyperlane_deployment }} - {{remote}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Prepare queues (all)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "grafanacloud-prom"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 18
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "grafanacloud-prom"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "sum by(remote, queue_name) (hyperlane_submitter_queue_length{queue_name=\"submit_queue\"})",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "interval": "",
+          "legendFormat": "{{remote}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Submit Queues",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "grafanacloud-prom"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 26
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "grafanacloud-prom"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "sum by(remote, queue_name) (avg_over_time(hyperlane_submitter_queue_length{queue_name=\"confirm_queue\"}[20m]))",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "interval": "",
+          "legendFormat": "{{remote}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Confirm Queues",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "1m",
+  "schemaVersion": 39,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-7d",
+    "to": "now"
+  },
+  "timeRangeUpdatedDuringEditOrView": false,
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Easy Dashboard (External Sharing Template)",
+  "uid": "afdf6ada6uzvgga",
+  "version": 5,
+  "weekStart": ""
+}

typescript/ccip-server/CHANGELOG.md

@@ -1,5 +1,7 @@
 # @hyperlane-xyz/ccip-server
 
+## 3.13.0
+
 ## 3.12.0
 
 ## 3.11.1