From cdbaf9ef0206480230f3ee25550580a96ad71d2a Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Tue, 2 Jan 2024 17:10:25 +0000 Subject: [PATCH] fix: dedup eth logs (#3108) ### Description Deduplicates ethereum logs by collecting them into a HashSet before returning a Vec. I couldn't find a way to deduplicate using an ethers config, but since `fetch_logs` is only called once in hyperlane-base, I'm doing the deduplication there "globally". I also noted in doc comments that duplicates may be returned. ### Related issues - Fixes https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/3070 ### Backward compatibility Yes ### Testing Will rely on e2e passing but haven't added unit tests for this scenario --- rust/chains/hyperlane-ethereum/src/interchain_gas.rs | 1 + rust/chains/hyperlane-ethereum/src/mailbox.rs | 2 ++ rust/chains/hyperlane-ethereum/src/merkle_tree_hook.rs | 1 + rust/hyperlane-base/src/contract_sync/mod.rs | 6 ++++-- rust/hyperlane-core/src/types/log_metadata.rs | 2 +- rust/hyperlane-core/src/types/merkle_tree.rs | 2 +- rust/hyperlane-core/src/types/message.rs | 2 +- rust/hyperlane-core/src/types/mod.rs | 2 +- 8 files changed, 12 insertions(+), 6 deletions(-) diff --git a/rust/chains/hyperlane-ethereum/src/interchain_gas.rs b/rust/chains/hyperlane-ethereum/src/interchain_gas.rs index 24b7ae502..b4a93fd54 100644 --- a/rust/chains/hyperlane-ethereum/src/interchain_gas.rs +++ b/rust/chains/hyperlane-ethereum/src/interchain_gas.rs @@ -84,6 +84,7 @@ impl Indexer for EthereumInterchainGasPaymasterIndexer< where M: Middleware + 'static, { + /// Note: This call may return duplicates depending on the provider used #[instrument(err, skip(self))] async fn fetch_logs( &self, diff --git a/rust/chains/hyperlane-ethereum/src/mailbox.rs b/rust/chains/hyperlane-ethereum/src/mailbox.rs index 4c566abbd..02c5bb51d 100644 --- a/rust/chains/hyperlane-ethereum/src/mailbox.rs +++ b/rust/chains/hyperlane-ethereum/src/mailbox.rs @@ -125,6 +125,7 @@ where self.get_finalized_block_number().await } + /// Note: This call may return duplicates depending on the provider used #[instrument(err, skip(self))] async fn fetch_logs( &self, @@ -168,6 +169,7 @@ where self.get_finalized_block_number().await } + /// Note: This call may return duplicates depending on the provider used #[instrument(err, skip(self))] async fn fetch_logs(&self, range: RangeInclusive) -> ChainResult> { Ok(self diff --git a/rust/chains/hyperlane-ethereum/src/merkle_tree_hook.rs b/rust/chains/hyperlane-ethereum/src/merkle_tree_hook.rs index 1beffed5b..90b0f7331 100644 --- a/rust/chains/hyperlane-ethereum/src/merkle_tree_hook.rs +++ b/rust/chains/hyperlane-ethereum/src/merkle_tree_hook.rs @@ -105,6 +105,7 @@ impl Indexer for EthereumMerkleTreeHookIndexer where M: Middleware + 'static, { + /// Note: This call may return duplicates depending on the provider used #[instrument(err, skip(self))] async fn fetch_logs( &self, diff --git a/rust/hyperlane-base/src/contract_sync/mod.rs b/rust/hyperlane-base/src/contract_sync/mod.rs index 3968ad9f5..219f06c90 100644 --- a/rust/hyperlane-base/src/contract_sync/mod.rs +++ b/rust/hyperlane-base/src/contract_sync/mod.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, marker::PhantomData, sync::Arc}; +use std::{collections::HashSet, fmt::Debug, hash::Hash, marker::PhantomData, sync::Arc}; use cursor::*; use derive_new::new; @@ -31,7 +31,7 @@ pub struct ContractSync, I: Indexer> { impl ContractSync where - T: Debug + Send + Sync + Clone + 'static, + T: Debug + Send + Sync + Clone + Eq + Hash + 'static, D: HyperlaneLogStore + 'static, I: Indexer + Clone + 'static, { @@ -67,6 +67,8 @@ where debug!(?range, "Looking for for events in index range"); let logs = self.indexer.fetch_logs(range.clone()).await?; + let deduped_logs = HashSet::<_>::from_iter(logs); + let logs = Vec::from_iter(deduped_logs); info!( ?range, diff --git a/rust/hyperlane-core/src/types/log_metadata.rs b/rust/hyperlane-core/src/types/log_metadata.rs index bdb4b043f..e85824dab 100644 --- a/rust/hyperlane-core/src/types/log_metadata.rs +++ b/rust/hyperlane-core/src/types/log_metadata.rs @@ -10,7 +10,7 @@ use crate::{H256, H512, U256}; /// A close clone of the Ethereum `LogMeta`, this is designed to be a more /// generic metadata that we can use for other blockchains later. Some changes /// may be required in the future. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default, Hash)] pub struct LogMeta { /// Address from which this log originated pub address: H256, diff --git a/rust/hyperlane-core/src/types/merkle_tree.rs b/rust/hyperlane-core/src/types/merkle_tree.rs index 96ea78ff1..5d4e20a9f 100644 --- a/rust/hyperlane-core/src/types/merkle_tree.rs +++ b/rust/hyperlane-core/src/types/merkle_tree.rs @@ -4,7 +4,7 @@ use std::io::{Read, Write}; use crate::{Decode, Encode, HyperlaneProtocolError, Sequenced, H256}; /// Merkle Tree Hook insertion event -#[derive(Debug, Copy, Clone, new, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, new, Eq, PartialEq, Hash)] pub struct MerkleTreeInsertion { leaf_index: u32, message_id: H256, diff --git a/rust/hyperlane-core/src/types/message.rs b/rust/hyperlane-core/src/types/message.rs index badabd815..99fb2fef4 100644 --- a/rust/hyperlane-core/src/types/message.rs +++ b/rust/hyperlane-core/src/types/message.rs @@ -21,7 +21,7 @@ impl From<&HyperlaneMessage> for RawHyperlaneMessage { } /// A full Hyperlane message between chains -#[derive(Clone, Eq, PartialEq)] +#[derive(Clone, Eq, PartialEq, Hash)] pub struct HyperlaneMessage { /// 1 Hyperlane version number pub version: u8, diff --git a/rust/hyperlane-core/src/types/mod.rs b/rust/hyperlane-core/src/types/mod.rs index 58d466abc..9d56cb7f0 100644 --- a/rust/hyperlane-core/src/types/mod.rs +++ b/rust/hyperlane-core/src/types/mod.rs @@ -115,7 +115,7 @@ pub struct GasPaymentKey { } /// A payment of a message's gas costs. -#[derive(Debug, Copy, Clone, Default, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Default, Eq, PartialEq, Hash)] pub struct InterchainGasPayment { /// Id of the message pub message_id: H256,