Graceful handling of errors with key funder's IGP claim or L2 bridging (#1935)

### Description

Noticed that if an IGP claim fails, it's not gracefully handled

### Drive-by changes

none

### Related issues

n/a

### Backward compatibility

_Are these changes backward compatible?_

Yes

_Are there any infrastructure implications, e.g. changes that would
prohibit deploying older commits using this infra tooling?_

None


### Testing

_What kind of testing have these changes undergone?_

Ran locally

typescript/infra/scripts/announce-validators.ts

@@ -53,21 +53,28 @@ async function main() {
     multiProvider,
   );
 
-  const announcements = [];
+  const announcements: {
+    storageLocation: string;
+    announcement: any;
+  }[] = [];
   const chains: ChainName[] = [];
   if (location) {
     chains.push(chain!);
     if (location.startsWith('s3://')) {
       const validator = await S3Validator.fromStorageLocation(location);
-      announcements.push(await validator.getAnnouncement());
+      announcements.push({
+        storageLocation: validator.storageLocation(),
+        announcement: await validator.getAnnouncement(),
+      });
     } else if (location.startsWith('file://')) {
       const announcementFilepath = path.join(
         location.substring(7),
         'announcement.json',
       );
-      announcements.push(
+      announcements.push({
-        JSON.parse(readFileSync(announcementFilepath, 'utf-8')),
+        storageLocation: location,
-      );
+        announcement: JSON.parse(readFileSync(announcementFilepath, 'utf-8')),
+      });
     } else {
       throw new Error(`Unknown location type %{location}`);
     }
@@ -94,10 +101,10 @@ async function main() {
                 validatorBaseConfig.checkpointSyncer.bucket,
                 validatorBaseConfig.checkpointSyncer.region,
               );
-              announcements.push([
+              announcements.push({
-                validatorBaseConfig.name,
+                storageLocation: validator.storageLocation(),
-                await validator.getAnnouncement(),
+                announcement: await validator.getAnnouncement(),
-              ]);
+              });
               chains.push(chain);
             }
           }
@@ -107,9 +114,9 @@ async function main() {
   }
 
   for (let i = 0; i < announcements.length; i++) {
-    const [name, announcement] = announcements[i];
+    const { storageLocation, announcement } = announcements[i];
     if (!announcement) {
-      console.info(`No announcement for ${name}`);
+      console.info(`No announcement for storageLocation ${storageLocation}`);
     }
     const chain = chains[i];
     const contracts = core.getContracts(chain);

typescript/infra/scripts/funding/fund-keys-from-deployer.ts

@@ -376,12 +376,21 @@ class ContextFunder {
       async ([chain, keys]) => {
         if (keys.length > 0) {
           if (!this.skipIgpClaim) {
-            await this.attemptToClaimFromIgp(chain as ChainName);
+            failureOccurred ||= await gracefullyHandleError(
+              () => this.attemptToClaimFromIgp(chain),
+              chain,
+              'Error claiming from IGP',
+            );
           }
-          await this.bridgeIfL2(chain as ChainName);
+
+          failureOccurred ||= await gracefullyHandleError(
+            () => this.bridgeIfL2(chain),
+            chain,
+            'Error bridging to L2',
+          );
         }
         for (const key of keys) {
-          const failure = await this.attemptToFundKey(key, chain as ChainName);
+          const failure = await this.attemptToFundKey(key, chain);
           failureOccurred ||= failure;
         }
       },
@@ -492,15 +501,20 @@ class ContextFunder {
     const igpBalance = await provider.getBalance(igp.address);
 
     log('Checking IGP balance', {
+      chain,
       igpBalance: ethers.utils.formatEther(igpBalance),
       igpClaimThreshold: ethers.utils.formatEther(igpClaimThreshold),
     });
 
     if (igpBalance.gt(igpClaimThreshold)) {
-      log('IGP balance exceeds claim threshold, claiming');
+      log('IGP balance exceeds claim threshold, claiming', {
+        chain,
+      });
       await this.multiProvider.handleTx(chain, igp.contract.claim());
     } else {
-      log('IGP balance does not exceed claim threshold, skipping');
+      log('IGP balance does not exceed claim threshold, skipping', {
+        chain,
+      });
     }
   }
 
@@ -744,6 +758,24 @@ function parseContextAndRoles(str: string): ContextAndRoles {
   };
 }
 
+// Returns whether an error occurred
+async function gracefullyHandleError(
+  fn: () => Promise<void>,
+  chain: ChainName,
+  errorMessage: string,
+): Promise<boolean> {
+  try {
+    await fn();
+    return false;
+  } catch (err) {
+    error(errorMessage, {
+      chain,
+      error: format(err),
+    });
+  }
+  return true;
+}
+
 main().catch((err) => {
   error('Error occurred in main', {
     // JSON.stringifying an Error returns '{}'.

typescript/infra/src/agents/aws/s3.ts

@@ -14,6 +14,7 @@ export interface S3Receipt<T = unknown> {
 export class S3Wrapper {
   private readonly client: S3Client;
   readonly bucket: string;
+  readonly region: string;
 
   static fromBucketUrl(bucketUrl: string): S3Wrapper {
     const match = bucketUrl.match(S3_BUCKET_REGEX);
@@ -23,6 +24,7 @@ export class S3Wrapper {
 
   constructor(bucket: string, region: string) {
     this.bucket = bucket;
+    this.region = region;
     this.client = new S3Client({ region });
   }
 

typescript/infra/src/agents/aws/validator.ts

@@ -39,6 +39,7 @@ const checkpointKey = (checkpointIndex: number) =>
   `checkpoint_${checkpointIndex}.json`;
 const LATEST_KEY = 'checkpoint_latest_index.json';
 const ANNOUNCEMENT_KEY = 'announcement.json';
+const LOCATION_PREFIX = 's3://';
 
 /**
  * Extension of BaseValidator that includes AWS S3 utilities.
@@ -60,9 +61,8 @@ export class S3Validator extends BaseValidator {
   static async fromStorageLocation(
     storageLocation: string,
   ): Promise<S3Validator> {
-    const prefix = 's3://';
+    if (storageLocation.startsWith(LOCATION_PREFIX)) {
-    if (storageLocation.startsWith(prefix)) {
+      const suffix = storageLocation.slice(LOCATION_PREFIX.length);
-      const suffix = storageLocation.slice(prefix.length);
       const pieces = suffix.split('/');
       if (pieces.length == 2) {
         const s3Bucket = new S3Wrapper(pieces[0], pieces[1]);
@@ -172,6 +172,10 @@ export class S3Validator extends BaseValidator {
     return checkpointMetrics.slice(-1 * count);
   }
 
+  storageLocation(): string {
+    return `${LOCATION_PREFIX}/${this.s3Bucket.bucket}/${this.s3Bucket.region}`;
+  }
+
   private async getCheckpointReceipt(
     index: number,
   ): Promise<CheckpointReceipt | undefined> {