diff --git a/rust/chains/abacus-ethereum/src/retrying.rs b/rust/chains/abacus-ethereum/src/retrying.rs index a791c29c9..7dbc8fcd9 100644 --- a/rust/chains/abacus-ethereum/src/retrying.rs +++ b/rust/chains/abacus-ethereum/src/retrying.rs @@ -160,7 +160,7 @@ where impl JsonRpcClient for RetryingProvider { type Error = RetryingProviderError; - #[instrument(level = "error", skip_all, fields(method = %method))] + #[instrument(level = "error", skip_all, fields(method = %method, provider_host = %self.inner.url().host_str().unwrap_or("unknown")))] async fn request(&self, method: &str, params: T) -> Result where T: Debug + Serialize + Send + Sync, @@ -190,7 +190,7 @@ impl JsonRpcClient for RetryingProvider { } } Err(HttpClientError::SerdeJson { err, text }) => { - info!(attempt, next_backoff_ms, error = %err, "SerdeJson error in http provider"); + info!(attempt, next_backoff_ms, error = %err, text = text, "SerdeJson error in http provider"); HandleMethod::Retry(HttpClientError::SerdeJson { err, text }) } }) diff --git a/rust/chains/abacus-ethereum/src/trait_builder.rs b/rust/chains/abacus-ethereum/src/trait_builder.rs index f553818ff..eb209d874 100644 --- a/rust/chains/abacus-ethereum/src/trait_builder.rs +++ b/rust/chains/abacus-ethereum/src/trait_builder.rs @@ -47,8 +47,17 @@ pub trait MakeableWithProvider { let mut builder = QuorumProvider::builder().quorum(Quorum::Majority); for url in urls.split(',') { let http_provider: Http = url.parse()?; + // Wrap the inner providers as RetryingProviders rather than the QuorumProvider. + // We've observed issues where the QuorumProvider will first get the latest block + // number and then submit an RPC at that block height, sometimes resulting in the + // second RPC getting serviced by a node that isn't aware of the requested block + // height yet. Retrying at the QuorumProvider level will result in both those RPCs + // being retried, while retrying at the inner provider level will result in only the + // second RPC being retried (the one with the error), which is the desired behavior. + let retrying_provider = + RetryingProvider::new(http_provider, Some(3), Some(1000)); let metrics_provider = self.wrap_rpc_with_metrics( - http_provider, + retrying_provider, Url::parse(url)?, &rpc_metrics, &middleware_metrics, @@ -57,8 +66,7 @@ pub trait MakeableWithProvider { builder = builder.add_provider(weighted_provider); } let quorum_provider = builder.build(); - let retrying = RetryingProvider::new(quorum_provider, Some(3), Some(1000)); - self.wrap_with_metrics(retrying, locator, signer, middleware_metrics) + self.wrap_with_metrics(quorum_provider, locator, signer, middleware_metrics) .await? } Connection::Http { url } => { diff --git a/typescript/infra/config/environments/testnet2/agent.ts b/typescript/infra/config/environments/testnet2/agent.ts index f0ecc9b22..d79abe12b 100644 --- a/typescript/infra/config/environments/testnet2/agent.ts +++ b/typescript/infra/config/environments/testnet2/agent.ts @@ -26,7 +26,7 @@ export const abacus: AgentConfig = { context: Contexts.Abacus, docker: { repo: 'gcr.io/abacus-labs-dev/abacus-agent', - tag: 'sha-0d76398', + tag: 'sha-ebfd1c1', }, aws: { region: 'us-east-1', @@ -37,7 +37,7 @@ export const abacus: AgentConfig = { gelato: { enabledChains: ['alfajores', 'mumbai', 'goerli'], }, - connectionType: ConnectionType.Http, + connectionType: ConnectionType.HttpQuorum, validator: { default: { interval: 5, @@ -124,7 +124,7 @@ export const releaseCandidate: AgentConfig = { context: Contexts.ReleaseCandidate, docker: { repo: 'gcr.io/abacus-labs-dev/abacus-agent', - tag: 'sha-0d76398', + tag: 'sha-ebfd1c1', }, aws: { region: 'us-east-1',