diff --git a/typescript/infra/scripts/verify-validator.ts b/typescript/infra/scripts/verify-validator.ts index fca4d257a..135862894 100644 --- a/typescript/infra/scripts/verify-validator.ts +++ b/typescript/infra/scripts/verify-validator.ts @@ -1,6 +1,8 @@ import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; import yargs from 'yargs'; +import { mean, median, stdDev, streamToString } from '../src/utils/utils'; + const MAX_MISSING_CHECKPOINTS = 10; interface Checkpoint { @@ -48,22 +50,6 @@ function isValidHashStr(s: string): boolean { return !!s.match(/^0x[0-9a-f]{1,64}$/im); } -function getArgs() { - return yargs(process.argv.slice(2)) - .alias('a', 'address') - .describe('a', 'address of the validator to inspect') - .demandOption('a') - .string('a') - .alias('p', 'prospective') - .describe('p', 'S3 bucket of the prospective validator') - .demandOption('p') - .string('p') - .alias('c', 'control') - .describe('c', 'S3 bucket of the the known (control) validator') - .demandOption('c') - .string('c').argv; -} - class S3Wrapper { private readonly client: S3Client; readonly region: string; @@ -104,201 +90,304 @@ class S3Wrapper { } } -async function main() { - const { - a: _validatorAddress, - p: prospectiveBucket, - c: controlBucket, - } = await getArgs(); +class Validator { + private readonly controlS3BucketClientWrapper: S3Wrapper; + private readonly prospectiveS3BucketClientWrapper: S3Wrapper; - const cClient = new S3Wrapper(controlBucket); - const pClient = new S3Wrapper(prospectiveBucket); + // accumulators for stats + /** Checkpoints the prospective validator has that the control validator does not */ + private extraCheckpoints!: number[]; + /** Checkpoints the prospective validator does not have that the control validator does have */ + private missingCheckpoints!: number[]; + /** Checkpoints the prospective validator has but for which we detected an issue */ + private invalidCheckpoints!: number[]; + /** The difference in modification times on the s3 objects between the control and validator + * buckets. (validator time - control time). + */ + private modTimeDeltasS!: number[]; + /** The checkpoints which were, as far as this validation logic is concerned, present and valid */ + private validCheckpoints!: number[]; + /** The number of checkpoints that the control had that the validator did not have in a row. + * (Not necessarily consecutive indexes) */ + private missingInARow!: number; + /** Index of the last index we found an entry for from the prospective validator */ + private lastNonMissingCheckpointIndex!: number; - const [{ obj: controlLatestCheckpoint }, { obj: prospectiveLastCheckpoint }] = - await Promise.all([ - cClient.getS3Obj('checkpoint_latest_index.json').catch((err) => { - console.error( - "Failed to get control validator's latest checkpoint.", - err, - ); - process.exit(1); - }), - pClient.getS3Obj('checkpoint_latest_index.json').catch((err) => { - console.error( - "Failed to get prospective validator's latest checkpoint.", - err, - ); - process.exit(1); - }), - ]); + constructor( + public readonly validatorAddress: string, + public readonly controlS3BucketAddress: string, + public readonly prospectiveS3BucketAddress: string, + ) { + this.controlS3BucketClientWrapper = new S3Wrapper( + this.controlS3BucketAddress, + ); + this.prospectiveS3BucketClientWrapper = new S3Wrapper( + this.prospectiveS3BucketAddress, + ); + } - if ( - !isLatestCheckpoint(controlLatestCheckpoint) || - !isLatestCheckpoint(prospectiveLastCheckpoint) - ) - process.exit(1); + initStatsState() { + this.extraCheckpoints = []; + this.missingCheckpoints = []; + this.invalidCheckpoints = []; + this.modTimeDeltasS = []; + this.validCheckpoints = []; + this.missingInARow = 0; + this.lastNonMissingCheckpointIndex = Infinity; + } - console.log(`Latest Index`); - console.log(`control: ${controlLatestCheckpoint}`); - console.log(`prospective: ${prospectiveLastCheckpoint}\n`); - - let extraCheckpoints = []; - const missingCheckpoints = []; - let invalidCheckpoints = []; - const modTimeDeltasS = []; - const fullyCorrectCheckpoints = []; - let missingInARow = 0; - let lastNonMissingCheckpointIndex = Infinity; - for ( - let i = Math.max(controlLatestCheckpoint, prospectiveLastCheckpoint); - i >= 0; - --i - ) { - if (missingInARow == MAX_MISSING_CHECKPOINTS) { - missingCheckpoints.length -= MAX_MISSING_CHECKPOINTS; - invalidCheckpoints = invalidCheckpoints.filter( - (j) => j < lastNonMissingCheckpointIndex, - ); - extraCheckpoints = extraCheckpoints.filter( - (j) => j < lastNonMissingCheckpointIndex, - ); - break; - } + /** + * Validate that the control and prospective validators are in agreement. Will throw an error on + * any critical failures and will log stats to the console as it goes. + * + * If we want to make this callable from outside the script later I would suggest making this + * return a stats object or something. + */ + async validate(): Promise { + this.initStatsState(); - const key = `checkpoint_${i}.json`; + const { controlLatestCheckpoint, prospectiveLastCheckpoint } = + await this.getLatestCheckpoints(); - let control: Checkpoint | null; - let controlLastMod: Date | null; - try { - const t = await cClient.getS3Obj(key); - if (isCheckpoint(t.obj)) { - if (t.obj.checkpoint.index != i) { - console.log(`${i}: Control index is invalid`, t); - process.exit(1); - } - [control, controlLastMod] = [t.obj, t.modified]; - } else { - console.log(`${i}: Invalid control checkpoint`, t); - process.exit(1); + const maxCheckpointIndex = Math.max( + controlLatestCheckpoint, + prospectiveLastCheckpoint, + ); + for ( + let checkpointIndex = maxCheckpointIndex; + checkpointIndex >= 0; + --checkpointIndex + ) { + if (this.missingInARow == MAX_MISSING_CHECKPOINTS) { + this.missingCheckpoints.length -= MAX_MISSING_CHECKPOINTS; + this.extraCheckpoints = this.extraCheckpoints.filter( + (j) => j < this.lastNonMissingCheckpointIndex, + ); + break; } - } catch (err) { - control = controlLastMod = null; + + await this.validateCheckpointIndex(checkpointIndex); } - let prospective: Checkpoint; - let prospectiveLastMod: Date; - try { - const t = await pClient.getS3Obj(key); - if (isCheckpoint(t.obj)) { - [prospective, prospectiveLastMod] = [t.obj, t.modified]; - lastNonMissingCheckpointIndex = i; - } else { - console.log(`${i}: Invalid prospective checkpoint`, t.obj); - invalidCheckpoints.push(i); - continue; - } - if (!control) { - extraCheckpoints.push(i); - } - missingInARow = 0; - } catch (err) { - if (control) { - missingCheckpoints.push(i); - missingInARow++; - } - continue; + console.log( + `Fully correct checkpoints (${this.validCheckpoints.length}): ${this.validCheckpoints}\n`, + ); + if (this.extraCheckpoints.length) + console.log( + `Extra checkpoints (${this.extraCheckpoints.length}): ${this.extraCheckpoints}\n`, + ); + if (this.missingCheckpoints.length) + console.log( + `Missing checkpoints (${this.missingCheckpoints.length}): ${this.missingCheckpoints}\n`, + ); + if (this.invalidCheckpoints.length) + console.log( + `Invalid checkpoints (${this.invalidCheckpoints.length}): ${this.invalidCheckpoints}\n`, + ); + + if (this.modTimeDeltasS.length > 1) { + // Drop the time of the first one since it is probably way off + this.modTimeDeltasS.length--; + console.log( + `Time deltas (∆ < 0 -> prospective came earlier than the control)`, + ); + console.log(this.modTimeDeltasS); + console.log(`Median: ${median(this.modTimeDeltasS)}s`); + console.log(`Mean: ${mean(this.modTimeDeltasS)}s`); + console.log(`Stdev: ${stdDev(this.modTimeDeltasS)}s`); } + } + + private async validateCheckpointIndex( + checkpointIndex: number, + ): Promise { + const { control, controlLastMod } = (await this.getControlCheckpoint( + checkpointIndex, + )) ?? { control: null, controlLastMod: null }; + + const getProspectiveCheckpointResult = await this.getProspectiveCheckpoint( + checkpointIndex, + !!control, + ); + if (!getProspectiveCheckpointResult) return; + const { prospective, prospectiveLastMod } = getProspectiveCheckpointResult; console.assert( - prospective.checkpoint.index == i, - `${i}: checkpoint indexes do not match`, + prospective.checkpoint.index == checkpointIndex, + `${checkpointIndex}: checkpoint indexes do not match`, ); // TODO: verify signature if (!control) { - continue; + return; } // compare against the control console.assert( control.checkpoint.outbox_domain == prospective.checkpoint.outbox_domain, - `${i}: outbox_domains do not match`, + `${checkpointIndex}: outbox_domains do not match`, ); console.assert( control.checkpoint.root == prospective.checkpoint.root, - `${i}: checkpoint roots do not match`, + `${checkpointIndex}: checkpoint roots do not match`, ); const diffS = (prospectiveLastMod.valueOf() - controlLastMod!.valueOf()) / 1000; if (Math.abs(diffS) > 10) { - console.log(`${i}: Modification times differ by ${diffS}s`); + console.log(`${checkpointIndex}: Modification times differ by ${diffS}s`); } - modTimeDeltasS.push(diffS); - fullyCorrectCheckpoints.push(i); + this.modTimeDeltasS.push(diffS); + this.validCheckpoints.push(checkpointIndex); } - console.log( - `Fully correct checkpoints (${fullyCorrectCheckpoints.length}): ${fullyCorrectCheckpoints}\n`, - ); - if (extraCheckpoints.length) - console.log( - `Extra checkpoints (${extraCheckpoints.length}): ${extraCheckpoints}\n`, - ); - if (missingCheckpoints.length) - console.log( - `Missing checkpoints (${missingCheckpoints.length}): ${missingCheckpoints}\n`, - ); - if (invalidCheckpoints.length) - console.log( - `Invalid checkpoints (${invalidCheckpoints.length}): ${invalidCheckpoints}\n`, - ); + private async getLatestCheckpoints(): Promise<{ + controlLatestCheckpoint: number; + prospectiveLastCheckpoint: number; + }> { + const [ + { obj: controlLatestCheckpoint }, + { obj: prospectiveLastCheckpoint }, + ] = await Promise.all([ + this.controlS3BucketClientWrapper + .getS3Obj('checkpoint_latest_index.json') + .catch((err) => { + console.error( + "Failed to get control validator's latest checkpoint.", + err, + ); + process.exit(1); + }), + this.prospectiveS3BucketClientWrapper + .getS3Obj('checkpoint_latest_index.json') + .catch((err) => { + console.error( + "Failed to get prospective validator's latest checkpoint.", + err, + ); + process.exit(1); + }), + ]); - if (modTimeDeltasS.length > 1) { - // Drop the time of the first one since it is probably way off - modTimeDeltasS.length--; - console.log( - `Time deltas (∆ < 0 -> prospective came earlier than the control)`, - ); - console.log(modTimeDeltasS); - console.log(`Median: ${median(modTimeDeltasS)}s`); - console.log(`Mean: ${mean(modTimeDeltasS)}s`); - console.log(`Stdev: ${stdDev(modTimeDeltasS)}s`); + if ( + !isLatestCheckpoint(controlLatestCheckpoint) || + !isLatestCheckpoint(prospectiveLastCheckpoint) + ) + throw new Error('Invalid latest checkpoint data'); + + console.log(`Latest Index`); + console.log(`control: ${controlLatestCheckpoint}`); + console.log(`prospective: ${prospectiveLastCheckpoint}\n`); + return { controlLatestCheckpoint, prospectiveLastCheckpoint }; } -} -function median(a: number[]): number { - a = [...a]; // clone - a.sort((a, b) => a - b); - if (a.length <= 0) { - return 0; - } else if (a.length % 2 == 0) { - return (a[a.length / 2] + a[a.length / 2 - 1]) / 2; - } else { - return a[(a.length - 1) / 2]; + private async getControlCheckpoint( + checkpointIndex: number, + ): Promise<{ control: Checkpoint; controlLastMod: Date } | null> { + let control: Checkpoint, controlLastMod: Date, unrecoverableError; + try { + const s3Object = await this.controlS3BucketClientWrapper.getS3Obj( + this.checkpointKey(checkpointIndex), + ); + if (isCheckpoint(s3Object.obj)) { + if (s3Object.obj.checkpoint.index != checkpointIndex) { + console.log(`${checkpointIndex}: Control index is invalid`, s3Object); + process.exit(1); + } + [control, controlLastMod] = [s3Object.obj, s3Object.modified]; + } else { + console.log(`${checkpointIndex}: Invalid control checkpoint`, s3Object); + unrecoverableError = new Error('Invalid control checkpoint.'); + } + } catch (err) { + return null; + } + if (unrecoverableError) throw unrecoverableError; + return { control: control!, controlLastMod: controlLastMod! }; + } + + private async getProspectiveCheckpoint( + checkpointIndex: number, + controlFoundForIndex: boolean, + ): Promise<{ prospective: Checkpoint; prospectiveLastMod: Date } | null> { + let prospective: Checkpoint, prospectiveLastMod: Date; + try { + const s3Object = await this.prospectiveS3BucketClientWrapper.getS3Obj( + this.checkpointKey(checkpointIndex), + ); + if (isCheckpoint(s3Object.obj)) { + [prospective, prospectiveLastMod] = [s3Object.obj, s3Object.modified]; + this.lastNonMissingCheckpointIndex = checkpointIndex; + } else { + console.log( + `${checkpointIndex}: Invalid prospective checkpoint`, + s3Object.obj, + ); + this.invalidCheckpoints.push(checkpointIndex); + return null; + } + if (!controlFoundForIndex) { + this.extraCheckpoints.push(checkpointIndex); + } + this.missingInARow = 0; + } catch (err) { + if (controlFoundForIndex) { + this.missingCheckpoints.push(checkpointIndex); + this.missingInARow++; + } + return null; + } + + return { + prospective: prospective!, + prospectiveLastMod: prospectiveLastMod!, + }; + } + + private checkpointKey(checkpointIndex: number): string { + return `checkpoint_${checkpointIndex}.json`; } } -function mean(a: number[]): number { - return a.reduce((acc, i) => acc + i, 0) / a.length; +//// +// Bootstrapper +//// +function getArgs() { + return yargs(process.argv.slice(2)) + .alias('a', 'address') + .describe('a', 'address of the validator to inspect') + .demandOption('a') + .string('a') + .alias('p', 'prospective') + .describe('p', 'S3 bucket of the prospective validator') + .demandOption('p') + .string('p') + .alias('c', 'control') + .describe('c', 'S3 bucket of the the known (control) validator') + .demandOption('c') + .string('c').argv; } -function stdDev(a: number[]): number { - return Math.sqrt( - a.map((i) => i * i).reduce((acc, i) => acc + i, 0) / a.length, +async function main() { + const { + a: validatorAddress, + p: prospectiveBucket, + c: controlBucket, + } = await getArgs(); + + const validator = new Validator( + validatorAddress, + prospectiveBucket, + controlBucket, ); -} -function streamToString(stream: NodeJS.ReadableStream): Promise { - return new Promise((resolve, reject) => { - const chunks: string[] = []; - stream - .setEncoding('utf8') - .on('data', (chunk) => chunks.push(chunk)) - .on('error', (err) => reject(err)) - .on('end', () => resolve(String.prototype.concat(...chunks))); - }); + try { + await validator.validate(); + } catch (err) { + console.error(err); + process.exit(1); + } } main().catch(console.error); diff --git a/typescript/infra/src/utils/utils.ts b/typescript/infra/src/utils/utils.ts index 7407e4dd7..331c634fa 100644 --- a/typescript/infra/src/utils/utils.ts +++ b/typescript/infra/src/utils/utils.ts @@ -205,3 +205,38 @@ export function assertContext(contextStr: string): Contexts { }`, ); } + +export function median(a: number[]): number { + a = [...a]; // clone + a.sort((a, b) => a - b); + if (a.length <= 0) { + return 0; + } else if (a.length % 2 == 0) { + return (a[a.length / 2] + a[a.length / 2 - 1]) / 2; + } else { + return a[(a.length - 1) / 2]; + } +} + +export function mean(a: number[]): number { + return a.reduce((acc, i) => acc + i, 0) / a.length; +} + +export function stdDev(a: number[]): number { + const xbar = mean(a); + return Math.sqrt( + a.map((x) => Math.pow(x - xbar, 2)).reduce((acc, i) => acc + i, 0) / + a.length, + ); +} + +export function streamToString(stream: NodeJS.ReadableStream): Promise { + return new Promise((resolve, reject) => { + const chunks: string[] = []; + stream + .setEncoding('utf8') + .on('data', (chunk) => chunks.push(chunk)) + .on('error', (err) => reject(err)) + .on('end', () => resolve(String.prototype.concat(...chunks))); + }); +}