Improve error surfacing in SmartProvider

src/features/debugger/debugMessage.ts

@@ -124,7 +124,7 @@ async function debugMessageDelivery(
     );
     return { gasEstimate: deliveryGasEst.toString() };
   } catch (err: any) {
-    logger.info('Estimate gas call failed');
+    logger.info('Estimate gas call failed:', err);
     const errorReason = extractReasonString(err);
     logger.debug(errorReason);
 

src/features/messages/pi-queries/fetchPiChainMessages.test.ts

@@ -56,6 +56,9 @@ const goerliMessage: Message = {
     maxFeePerGas: 0,
     maxPriorityPerGas: 0,
   },
+  numPayments: 1,
+  totalGasAmount: '209736',
+  totalPayment: '1635940800000000',
   isPiMsg: true,
 };
 

src/features/messages/pi-queries/fetchPiChainMessages.ts

@@ -82,12 +82,18 @@ export async function fetchMessagesFromPiChain(
     .map((l) => logToMessage(multiProvider, l, chainConfig))
     .filter((m): m is Message => !!m);
 
-  const messagesWithGasPayments: Message[] = [];
-  // Avoiding parallelism here out of caution for RPC rate limits
-  for (const m of messages) {
-    messagesWithGasPayments.push(await tryFetchIgpGasPayments(m, chainConfig, multiProvider));
+  // Fetch IGP gas payments for each message if it's a small set
+  if (messages.length < 5) {
+    const messagesWithGasPayments: Message[] = [];
+    // Avoiding parallelism here out of caution for RPC rate limits
+    for (const m of messages) {
+      messagesWithGasPayments.push(await tryFetchIgpGasPayments(m, chainConfig, multiProvider));
+    }
+    return messagesWithGasPayments;
+  } else {
+    // Otherwise skip IGP gas fetching
+    return messages;
   }
-  return messagesWithGasPayments;
 }
 
 async function fetchLogsForAddress(

src/features/providers/SmartMultiProvider.ts

@@ -17,7 +17,7 @@ export class SmartMultiProvider extends MultiProvider {
     logger.debug('SmartMultiProvider constructed');
   }
   // Override to use SmartProvider instead of FallbackProvider
-  tryGetProvider(chainNameOrId: ChainName | number): HyperlaneSmartProvider | null {
+  override tryGetProvider(chainNameOrId: ChainName | number): HyperlaneSmartProvider | null {
     const metadata = this.tryGetChainMetadata(chainNameOrId);
     if (!metadata) return null;
     const { name, publicRpcUrls, blockExplorers } = metadata;

src/features/providers/SmartProvider.ts

@@ -2,17 +2,20 @@ import { providers } from 'ethers';
 
 import { ChainMetadata, ExplorerFamily } from '@hyperlane-xyz/sdk';
 
-import { logAndThrow } from '../../utils/errors';
 import { logger } from '../../utils/logger';
-import { sleep } from '../../utils/timeout';
+import { timeout } from '../../utils/timeout';
 
 import { HyperlaneEtherscanProvider } from './HyperlaneEtherscanProvider';
 import { HyperlaneJsonRpcProvider } from './HyperlaneJsonRpcProvider';
 import { IProviderMethods, ProviderMethod } from './ProviderMethods';
-import { ChainMetadataWithRpcConnectionInfo } from './types';
+import {
+  ChainMetadataWithRpcConnectionInfo,
+  ProviderPerformResult,
+  ProviderStatus,
+  ProviderTimeoutResult,
+} from './types';
 
 const PROVIDER_STAGGER_DELAY_MS = 1000; // 1 seconds
-const PROVIDER_TIMEOUT_MARKER = '__PROVIDER_TIMEOUT__';
 
 type HyperlaneProvider = HyperlaneEtherscanProvider | HyperlaneJsonRpcProvider;
 
@@ -83,48 +86,83 @@ export class HyperlaneSmartProvider extends providers.BaseProvider implements IP
 
     let pIndex = 0;
     const maxPIndex = supportedProviders.length - 1;
-    const providerResultPromises: Promise<any>[] = [];
+    const providerResultPromises: Promise<ProviderPerformResult>[] = [];
+    const providerResultErrors: unknown[] = [];
     // TODO consider implementing quorum and/or retry logic here similar to FallbackProvider/RetryProvider
     while (true) {
       if (pIndex <= maxPIndex) {
         // Trigger the next provider in line
         const provider = supportedProviders[pIndex];
         const providerUrl = provider.getBaseUrl();
+        const isLastProvider = pIndex === maxPIndex;
 
         // Skip the explorer provider if it's currently in a cooldown period
         if (
           this.isExplorerProvider(provider) &&
           provider.getQueryWaitTime() > 0 &&
-          pIndex < maxPIndex &&
+          !isLastProvider &&
           method !== ProviderMethod.GetLogs // never skip GetLogs
         ) {
           pIndex += 1;
           continue;
         }
 
-        const resultPromise = performWithLogging(provider, providerUrl, method, params, reqId);
-        providerResultPromises.push(resultPromise);
-        const timeoutPromise = sleep(PROVIDER_STAGGER_DELAY_MS, PROVIDER_TIMEOUT_MARKER);
-        const result = await Promise.any([resultPromise, timeoutPromise]);
+        const resultPromise = wrapProviderPerform(provider, providerUrl, method, params, reqId);
+        const timeoutPromise = timeout<ProviderTimeoutResult>(PROVIDER_STAGGER_DELAY_MS, {
+          status: ProviderStatus.Timeout,
+        });
+        const result = await Promise.race([resultPromise, timeoutPromise]);
 
-        if (result === PROVIDER_TIMEOUT_MARKER) {
+        if (result.status === ProviderStatus.Success) {
+          return result.value;
+        } else if (result.status === ProviderStatus.Timeout) {
           logger.warn(
-            `Slow response from provider using ${providerUrl}. Triggering next provider if available`,
+            `Slow response from provider using ${providerUrl}.${
+              !isLastProvider ? ' Triggering next provider.' : ''
+            }`,
           );
+          providerResultPromises.push(resultPromise);
+          pIndex += 1;
+        } else if (result.status === ProviderStatus.Error) {
+          logger.warn(
+            `Error from provider using ${providerUrl}.${
+              !isLastProvider ? ' Triggering next provider.' : ''
+            }`,
+          );
+          providerResultErrors.push(result.error);
           pIndex += 1;
         } else {
-          // Result looks good
-          return result;
+          throw new Error('Unexpected result from provider');
         }
-      } else {
-        // All providers already triggered, wait for one to complete
-        const timeoutPromise = sleep(PROVIDER_STAGGER_DELAY_MS * 20, PROVIDER_TIMEOUT_MARKER);
-        const result = await Promise.any([...providerResultPromises, timeoutPromise]);
-        if (result === PROVIDER_TIMEOUT_MARKER) {
-          logAndThrow(`All providers failed or timed out for method ${method}`, result);
+      } else if (providerResultPromises.length > 0) {
+        // All providers already triggered, wait for one to complete or all to fail/timeout
+        const timeoutPromise = timeout<ProviderTimeoutResult>(PROVIDER_STAGGER_DELAY_MS * 20, {
+          status: ProviderStatus.Timeout,
+        });
+        const resultPromise = waitForProviderSuccess(providerResultPromises);
+        const result = await Promise.race([resultPromise, timeoutPromise]);
+
+        if (result.status === ProviderStatus.Success) {
+          return result.value;
+        } else if (result.status === ProviderStatus.Timeout) {
+          throwCombinedProviderErrors(
+            providerResultErrors,
+            `All providers timed out for method ${method}`,
+          );
+        } else if (result.status === ProviderStatus.Error) {
+          throwCombinedProviderErrors(
+            [result.error, ...providerResultErrors],
+            `All providers failed for method ${method}`,
+          );
         } else {
-          return result;
+          throw new Error('Unexpected result from provider');
         }
+      } else {
+        // All providers have already failed, all hope is lost
+        throwCombinedProviderErrors(
+          providerResultErrors,
+          `All providers failed for method ${method}`,
+        );
       }
     }
   }
@@ -134,20 +172,59 @@ export class HyperlaneSmartProvider extends providers.BaseProvider implements IP
   }
 }
 
-function performWithLogging(
+// Warp for additional logging and error handling
+async function wrapProviderPerform(
   provider: HyperlaneProvider,
   providerUrl: string,
   method: string,
   params: any,
   reqId: number,
-): Promise<any> {
+): Promise<ProviderPerformResult> {
   try {
     logger.debug(`Provider using ${providerUrl} performing method ${method} for reqId ${reqId}`);
-    return provider.perform(method, params, reqId);
+    const result = await provider.perform(method, params, reqId);
+    return { status: ProviderStatus.Success, value: result };
   } catch (error) {
     logger.error(`Error performing ${method} on provider ${providerUrl} for reqId ${reqId}`, error);
-    throw new Error(`Error performing ${method} with ${providerUrl} for reqId ${reqId}`);
+    return { status: ProviderStatus.Error, error };
+  }
+}
+
+async function waitForProviderSuccess(
+  _resultPromises: Promise<ProviderPerformResult>[],
+): Promise<ProviderPerformResult> {
+  // A hack to remove the promise from the array when it resolves
+  const resolvedPromiseIndexes = new Set<number>();
+  const resultPromises = _resultPromises.map((p, i) =>
+    p.then((r) => {
+      resolvedPromiseIndexes.add(i);
+      return r;
+    }),
+  );
+  const combinedErrors: unknown[] = [];
+  for (let i = 0; i < resultPromises.length; i += 1) {
+    const promises = resultPromises.filter((_, i) => !resolvedPromiseIndexes.has(i));
+    const result = await Promise.race(promises);
+    if (result.status === ProviderStatus.Success) {
+      return result;
+    } else if (result.status === ProviderStatus.Error) {
+      combinedErrors.push(result.error);
+    } else {
+      return { status: ProviderStatus.Error, error: new Error('Unexpected result from provider') };
+    }
   }
+  return {
+    status: ProviderStatus.Error,
+    // TODO combine errors
+    error: combinedErrors.length ? combinedErrors[0] : new Error('Unknown error from provider'),
+  };
+}
+
+function throwCombinedProviderErrors(errors: unknown[], fallbackMsg: string): void {
+  logger.error(fallbackMsg);
+  // TODO inspect the errors in some clever way to choose which to throw
+  if (errors.length > 0) throw errors[0];
+  else throw new Error(fallbackMsg);
 }
 
 function chainMetadataToProviderNetwork(chainMetadata: ChainMetadata): providers.Network {

src/features/providers/types.ts

@@ -9,3 +9,32 @@ export type RpcConfigWithConnectionInfo = ChainMetadata['publicRpcUrls'][number]
 export interface ChainMetadataWithRpcConnectionInfo extends ChainMetadata {
   publicRpcUrls: RpcConfigWithConnectionInfo[];
 }
+
+export enum ProviderStatus {
+  Success = 'success',
+  Error = 'error',
+  Timeout = 'timeout',
+}
+
+export interface ProviderPerformResultBase {
+  status: ProviderStatus;
+}
+
+export interface ProviderSuccessResult extends ProviderPerformResultBase {
+  status: ProviderStatus.Success;
+  value: any;
+}
+
+export interface ProviderErrorResult extends ProviderPerformResultBase {
+  status: ProviderStatus.Error;
+  error: unknown;
+}
+
+export interface ProviderTimeoutResult extends ProviderPerformResultBase {
+  status: ProviderStatus.Timeout;
+}
+
+export type ProviderPerformResult =
+  | ProviderSuccessResult
+  | ProviderErrorResult
+  | ProviderTimeoutResult;

src/store.ts

@@ -25,7 +25,7 @@ export const useStore = create<AppState>()(
       setChainConfigs: (configs: ChainMap<ChainConfig>) => {
         set(() => ({ chainConfigsV2: configs, multiProvider: buildSmartProvider(configs) }));
       },
-      multiProvider: new MultiProvider(),
+      multiProvider: buildSmartProvider({}),
       setMultiProvider: (mp: MultiProvider) => set(() => ({ multiProvider: mp })),
       bannerClassName: '',
       setBanner: (className: string) => set(() => ({ bannerClassName: className })),

src/utils/timeout.ts

@@ -40,10 +40,14 @@ export async function fetchWithTimeout(
   return response;
 }
 
-export function sleep(milliseconds: number, resolveValue: any = true) {
+export function timeout<T>(milliseconds: number, resolveValue: T): Promise<T> {
   return new Promise((resolve) => setTimeout(() => resolve(resolveValue), milliseconds));
 }
 
+export function sleep(milliseconds: number): Promise<true> {
+  return timeout(milliseconds, true);
+}
+
 export const PROMISE_TIMEOUT = '__promise_timeout__';
 
 export async function promiseTimeout<T>(promise: Promise<T>, milliseconds: number) {