From 355168d66ca7efe6f2461d1e84024c657b832897 Mon Sep 17 00:00:00 2001 From: Trevor Porter Date: Wed, 15 Mar 2023 13:02:24 +0000 Subject: [PATCH] Graceful handling of errors with key funder's IGP claim or L2 bridging (#1935) ### Description Noticed that if an IGP claim fails, it's not gracefully handled ### Drive-by changes none ### Related issues n/a ### Backward compatibility _Are these changes backward compatible?_ Yes _Are there any infrastructure implications, e.g. changes that would prohibit deploying older commits using this infra tooling?_ None ### Testing _What kind of testing have these changes undergone?_ Ran locally --- .../infra/scripts/announce-validators.ts | 29 ++++++++----- .../funding/fund-keys-from-deployer.ts | 42 ++++++++++++++++--- typescript/infra/src/agents/aws/s3.ts | 2 + typescript/infra/src/agents/aws/validator.ts | 10 +++-- 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/typescript/infra/scripts/announce-validators.ts b/typescript/infra/scripts/announce-validators.ts index c2148ef8a..de9afb3b3 100644 --- a/typescript/infra/scripts/announce-validators.ts +++ b/typescript/infra/scripts/announce-validators.ts @@ -53,21 +53,28 @@ async function main() { multiProvider, ); - const announcements = []; + const announcements: { + storageLocation: string; + announcement: any; + }[] = []; const chains: ChainName[] = []; if (location) { chains.push(chain!); if (location.startsWith('s3://')) { const validator = await S3Validator.fromStorageLocation(location); - announcements.push(await validator.getAnnouncement()); + announcements.push({ + storageLocation: validator.storageLocation(), + announcement: await validator.getAnnouncement(), + }); } else if (location.startsWith('file://')) { const announcementFilepath = path.join( location.substring(7), 'announcement.json', ); - announcements.push( - JSON.parse(readFileSync(announcementFilepath, 'utf-8')), - ); + announcements.push({ + storageLocation: location, + announcement: JSON.parse(readFileSync(announcementFilepath, 'utf-8')), + }); } else { throw new Error(`Unknown location type %{location}`); } @@ -94,10 +101,10 @@ async function main() { validatorBaseConfig.checkpointSyncer.bucket, validatorBaseConfig.checkpointSyncer.region, ); - announcements.push([ - validatorBaseConfig.name, - await validator.getAnnouncement(), - ]); + announcements.push({ + storageLocation: validator.storageLocation(), + announcement: await validator.getAnnouncement(), + }); chains.push(chain); } } @@ -107,9 +114,9 @@ async function main() { } for (let i = 0; i < announcements.length; i++) { - const [name, announcement] = announcements[i]; + const { storageLocation, announcement } = announcements[i]; if (!announcement) { - console.info(`No announcement for ${name}`); + console.info(`No announcement for storageLocation ${storageLocation}`); } const chain = chains[i]; const contracts = core.getContracts(chain); diff --git a/typescript/infra/scripts/funding/fund-keys-from-deployer.ts b/typescript/infra/scripts/funding/fund-keys-from-deployer.ts index 704200788..03982df36 100644 --- a/typescript/infra/scripts/funding/fund-keys-from-deployer.ts +++ b/typescript/infra/scripts/funding/fund-keys-from-deployer.ts @@ -376,12 +376,21 @@ class ContextFunder { async ([chain, keys]) => { if (keys.length > 0) { if (!this.skipIgpClaim) { - await this.attemptToClaimFromIgp(chain as ChainName); + failureOccurred ||= await gracefullyHandleError( + () => this.attemptToClaimFromIgp(chain), + chain, + 'Error claiming from IGP', + ); } - await this.bridgeIfL2(chain as ChainName); + + failureOccurred ||= await gracefullyHandleError( + () => this.bridgeIfL2(chain), + chain, + 'Error bridging to L2', + ); } for (const key of keys) { - const failure = await this.attemptToFundKey(key, chain as ChainName); + const failure = await this.attemptToFundKey(key, chain); failureOccurred ||= failure; } }, @@ -492,15 +501,20 @@ class ContextFunder { const igpBalance = await provider.getBalance(igp.address); log('Checking IGP balance', { + chain, igpBalance: ethers.utils.formatEther(igpBalance), igpClaimThreshold: ethers.utils.formatEther(igpClaimThreshold), }); if (igpBalance.gt(igpClaimThreshold)) { - log('IGP balance exceeds claim threshold, claiming'); + log('IGP balance exceeds claim threshold, claiming', { + chain, + }); await this.multiProvider.handleTx(chain, igp.contract.claim()); } else { - log('IGP balance does not exceed claim threshold, skipping'); + log('IGP balance does not exceed claim threshold, skipping', { + chain, + }); } } @@ -744,6 +758,24 @@ function parseContextAndRoles(str: string): ContextAndRoles { }; } +// Returns whether an error occurred +async function gracefullyHandleError( + fn: () => Promise, + chain: ChainName, + errorMessage: string, +): Promise { + try { + await fn(); + return false; + } catch (err) { + error(errorMessage, { + chain, + error: format(err), + }); + } + return true; +} + main().catch((err) => { error('Error occurred in main', { // JSON.stringifying an Error returns '{}'. diff --git a/typescript/infra/src/agents/aws/s3.ts b/typescript/infra/src/agents/aws/s3.ts index cf62fd9c9..09f73fd4a 100644 --- a/typescript/infra/src/agents/aws/s3.ts +++ b/typescript/infra/src/agents/aws/s3.ts @@ -14,6 +14,7 @@ export interface S3Receipt { export class S3Wrapper { private readonly client: S3Client; readonly bucket: string; + readonly region: string; static fromBucketUrl(bucketUrl: string): S3Wrapper { const match = bucketUrl.match(S3_BUCKET_REGEX); @@ -23,6 +24,7 @@ export class S3Wrapper { constructor(bucket: string, region: string) { this.bucket = bucket; + this.region = region; this.client = new S3Client({ region }); } diff --git a/typescript/infra/src/agents/aws/validator.ts b/typescript/infra/src/agents/aws/validator.ts index 0476ae959..65ac6bf0e 100644 --- a/typescript/infra/src/agents/aws/validator.ts +++ b/typescript/infra/src/agents/aws/validator.ts @@ -39,6 +39,7 @@ const checkpointKey = (checkpointIndex: number) => `checkpoint_${checkpointIndex}.json`; const LATEST_KEY = 'checkpoint_latest_index.json'; const ANNOUNCEMENT_KEY = 'announcement.json'; +const LOCATION_PREFIX = 's3://'; /** * Extension of BaseValidator that includes AWS S3 utilities. @@ -60,9 +61,8 @@ export class S3Validator extends BaseValidator { static async fromStorageLocation( storageLocation: string, ): Promise { - const prefix = 's3://'; - if (storageLocation.startsWith(prefix)) { - const suffix = storageLocation.slice(prefix.length); + if (storageLocation.startsWith(LOCATION_PREFIX)) { + const suffix = storageLocation.slice(LOCATION_PREFIX.length); const pieces = suffix.split('/'); if (pieces.length == 2) { const s3Bucket = new S3Wrapper(pieces[0], pieces[1]); @@ -172,6 +172,10 @@ export class S3Validator extends BaseValidator { return checkpointMetrics.slice(-1 * count); } + storageLocation(): string { + return `${LOCATION_PREFIX}/${this.s3Bucket.bucket}/${this.s3Bucket.region}`; + } + private async getCheckpointReceipt( index: number, ): Promise {