Wrap inner providers with RetryingProvider when using QuorumProvider, not the QuorumProvider itself (#1099)

* Retry inner providers when using QuorumProvider, not the QuorumProvider itself

* small log change for serde issues, test quorumprovider

* Add provider_host to retrying provider span

* Deploy

rust/chains/abacus-ethereum/src/retrying.rs

@@ -160,7 +160,7 @@ where
 impl JsonRpcClient for RetryingProvider<Http> {
     type Error = RetryingProviderError<Http>;
 
-    #[instrument(level = "error", skip_all, fields(method = %method))]
+    #[instrument(level = "error", skip_all, fields(method = %method, provider_host = %self.inner.url().host_str().unwrap_or("unknown")))]
     async fn request<T, R>(&self, method: &str, params: T) -> Result<R, Self::Error>
     where
         T: Debug + Serialize + Send + Sync,
@@ -190,7 +190,7 @@ impl JsonRpcClient for RetryingProvider<Http> {
                 }
             }
             Err(HttpClientError::SerdeJson { err, text }) => {
-                info!(attempt, next_backoff_ms, error = %err, "SerdeJson error in http provider");
+                info!(attempt, next_backoff_ms, error = %err, text = text,  "SerdeJson error in http provider");
                 HandleMethod::Retry(HttpClientError::SerdeJson { err, text })
             }
         })

rust/chains/abacus-ethereum/src/trait_builder.rs

@@ -47,8 +47,17 @@ pub trait MakeableWithProvider {
                 let mut builder = QuorumProvider::builder().quorum(Quorum::Majority);
                 for url in urls.split(',') {
                     let http_provider: Http = url.parse()?;
+                    // Wrap the inner providers as RetryingProviders rather than the QuorumProvider.
+                    // We've observed issues where the QuorumProvider will first get the latest block
+                    // number and then submit an RPC at that block height, sometimes resulting in the
+                    // second RPC getting serviced by a node that isn't aware of the requested block
+                    // height yet. Retrying at the QuorumProvider level will result in both those RPCs
+                    // being retried, while retrying at the inner provider level will result in only the
+                    // second RPC being retried (the one with the error), which is the desired behavior.
+                    let retrying_provider =
+                        RetryingProvider::new(http_provider, Some(3), Some(1000));
                     let metrics_provider = self.wrap_rpc_with_metrics(
-                        http_provider,
+                        retrying_provider,
                         Url::parse(url)?,
                         &rpc_metrics,
                         &middleware_metrics,
@@ -57,8 +66,7 @@ pub trait MakeableWithProvider {
                     builder = builder.add_provider(weighted_provider);
                 }
                 let quorum_provider = builder.build();
-                let retrying = RetryingProvider::new(quorum_provider, Some(3), Some(1000));
+                self.wrap_with_metrics(quorum_provider, locator, signer, middleware_metrics)
-                self.wrap_with_metrics(retrying, locator, signer, middleware_metrics)
                     .await?
             }
             Connection::Http { url } => {

typescript/infra/config/environments/testnet2/agent.ts

@@ -26,7 +26,7 @@ export const abacus: AgentConfig<TestnetChains> = {
   context: Contexts.Abacus,
   docker: {
     repo: 'gcr.io/abacus-labs-dev/abacus-agent',
-    tag: 'sha-0d76398',
+    tag: 'sha-ebfd1c1',
   },
   aws: {
     region: 'us-east-1',
@@ -37,7 +37,7 @@ export const abacus: AgentConfig<TestnetChains> = {
   gelato: {
     enabledChains: ['alfajores', 'mumbai', 'goerli'],
   },
-  connectionType: ConnectionType.Http,
+  connectionType: ConnectionType.HttpQuorum,
   validator: {
     default: {
       interval: 5,
@@ -124,7 +124,7 @@ export const releaseCandidate: AgentConfig<TestnetChains> = {
   context: Contexts.ReleaseCandidate,
   docker: {
     repo: 'gcr.io/abacus-labs-dev/abacus-agent',
-    tag: 'sha-0d76398',
+    tag: 'sha-ebfd1c1',
   },
   aws: {
     region: 'us-east-1',