Fixed key funder (#1725)

* key funder fixes

* Minor tweaks

* Updated configs

* Update funding to deploy to rc

* Deploy

* Better error handling

* Don't include empty addresses and log errors

* Deploy kathy to rc

* Deploy key funder

typescript/infra/config/environments/mainnet2/funding.ts

@@ -8,13 +8,12 @@ import { environment } from './chains';
 export const keyFunderConfig: KeyFunderConfig = {
   docker: {
     repo: 'gcr.io/abacus-labs-dev/hyperlane-monorepo',
-    // TODO: Use an image built off of main
+    tag: 'sha-3ff9c8b',
-    tag: 'sha-d1ad862',
   },
   // We're currently using the same deployer key as mainnet.
   // To minimize nonce clobbering we offset the key funder cron
   // schedule by 30 minutes.
-  cronSchedule: '15 * * * *', // Every hour at the 15 minute mark
+  cronSchedule: '15 * * * *', // Every hour at the 15-minute mark
   namespace: environment,
   prometheusPushGateway:
     'http://prometheus-pushgateway.monitoring.svc.cluster.local:9091',

typescript/infra/config/environments/mainnet2/helloworld.ts

@@ -33,7 +33,7 @@ export const releaseCandidate: HelloWorldConfig<MainnetChains> = {
   kathy: {
     docker: {
       repo: 'gcr.io/abacus-labs-dev/hyperlane-monorepo',
-      tag: 'sha-0477ee1',
+      tag: 'sha-19dc142',
     },
     chainsToSkip: [],
     runEnv: environment,

typescript/infra/config/environments/testnet3/funding.ts

@@ -8,12 +8,12 @@ import { environment } from './chains';
 export const keyFunderConfig: KeyFunderConfig = {
   docker: {
     repo: 'gcr.io/abacus-labs-dev/hyperlane-monorepo',
-    tag: 'sha-6ee34e4',
+    tag: 'sha-3ff9c8b',
   },
   // We're currently using the same deployer key as testnet2.
   // To minimize nonce clobbering we offset the key funder cron
   // schedule by 30 minutes.
-  cronSchedule: '15 * * * *', // Every hour at the 15 minute mark
+  cronSchedule: '15 * * * *', // Every hour at the 15-minute mark
   namespace: environment,
   prometheusPushGateway:
     'http://prometheus-pushgateway.monitoring.svc.cluster.local:9091',

typescript/infra/config/environments/testnet3/helloworld.ts

@@ -32,7 +32,7 @@ export const releaseCandidate: HelloWorldConfig<TestnetChains> = {
   kathy: {
     docker: {
       repo: 'gcr.io/abacus-labs-dev/hyperlane-monorepo',
-      tag: 'sha-0477ee1',
+      tag: 'sha-6ee34e4',
     },
     chainsToSkip: [],
     runEnv: environment,

typescript/infra/scripts/funding/fund-keys-from-deployer.ts

@@ -177,7 +177,6 @@ async function main() {
       ),
     );
   } else {
-    contextFunders = [];
     const contexts = Object.keys(argv.contextsAndRoles) as Contexts[];
     contextFunders = await Promise.all(
       contexts.map((context) =>
@@ -192,10 +191,7 @@ async function main() {
 
   let failureOccurred = false;
   for (const funder of contextFunders) {
-    const failure = await funder.fund();
+    failureOccurred ||= await funder.fund();
-    if (failure) {
-      failureOccurred = true;
-    }
   }
 
   await submitMetrics(metricsRegister, 'key-funder');
@@ -296,18 +292,25 @@ class ContextFunder {
   async fund(): Promise<boolean> {
     let failureOccurred = false;
 
-    const chainKeys = this.getChainKeys();
+    const promises = Object.entries(this.getChainKeys()).map(
-    await Promise.all(
+      async ([chain, keys]) => {
-      Object.entries(chainKeys).map(async ([chain, keys]) => {
         if (keys.length > 0) {
           await this.bridgeIfL2(chain as ChainName);
         }
-        keys.forEach(async (key) => {
+        for (const key of keys) {
           const failure = await this.attemptToFundKey(key, chain as ChainName);
-          failureOccurred = failureOccurred || failure;
+          failureOccurred ||= failure;
-        });
+        }
-      }),
+      },
     );
+
+    try {
+      await Promise.all(promises);
+    } catch (e) {
+      error('Unhandled error when funding key', { error: format(e) });
+      failureOccurred = true;
+    }
+
     return failureOccurred;
   }
 
@@ -381,7 +384,6 @@ class ContextFunder {
       // on L1 gas.
       const bridgeAmount = await this.getFundingAmount(
         chainConnection,
-        chain,
         funderAddress,
         desiredBalanceEther.mul(10),
       );
@@ -393,7 +395,6 @@ class ContextFunder {
 
   private async getFundingAmount(
     chainConnection: ChainConnection,
-    chain: ChainName,
     address: string,
     desiredBalance: BigNumber,
   ): Promise<BigNumber> {
@@ -419,7 +420,6 @@ class ContextFunder {
     );
     const fundingAmount = await this.getFundingAmount(
       chainConnection,
-      chain,
       key.address,
       desiredBalanceEther,
     );

typescript/infra/scripts/get-key-addresses.ts

@@ -6,23 +6,24 @@ async function main() {
   const agentConfig = await getContextAgentConfig();
 
   const keys = getAllCloudAgentKeys(agentConfig);
-
+  const keyInfoPromises = keys.map(async (key) => {
-  const keyInfos = await Promise.all(
-    keys.map(async (key) => {
     let address = '';
     try {
       await key.fetch();
       address = key.address;
     } catch (e) {
       // Swallow error
+      console.error('Error getting address', { key: key.identifier });
     }
     return {
       identifier: key.identifier,
       address,
     };
-    }),
+  });
+  const keyInfos = (await Promise.all(keyInfoPromises)).filter(
+    // remove any keys we could not get an address for
+    ({ address }) => !!address,
   );
-
   console.log(JSON.stringify(keyInfos, null, 2));
 }
 

typescript/infra/src/utils/metrics.ts

@@ -1,5 +1,8 @@
 import http from 'http';
 import { Pushgateway, Registry } from 'prom-client';
+import { format } from 'util';
+
+import { error, log } from '@hyperlane-xyz/utils';
 
 function getPushGateway(register: Registry): Pushgateway | null {
   const gatewayAddr = process.env['PROMETHEUS_PUSH_GATEWAY'];
@@ -22,19 +25,22 @@ export async function submitMetrics(
   if (!gateway) return;
 
   let resp;
+  try {
     if (options?.appendMode) {
       resp = (await gateway.pushAdd({ jobName })).resp;
     } else {
       resp = (await gateway.push({ jobName })).resp;
     }
+  } catch (e) {
+    error('Error when pushing metrics', { error: format(e) });
+    return;
+  }
 
   const statusCode =
     typeof resp == 'object' && resp != null && 'statusCode' in resp
       ? (resp as any).statusCode
       : 'unknown';
-  console.log(
+  log('Prometheus metrics pushed to PushGateway', { statusCode });
-    `Prometheus metrics pushed to PushGateway with status ${statusCode}`,
-  );
 }
 
 /**