CipherLabs
/
hyperlane-monorepo
mirror of https://github.com/hyperlane-xyz/hyperlane-monorepo
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactoring validator verification loop for clarity (#773)

Browse Source 
* Refactoring loop for clarity

* fix process name

* remove `t` var name

* move calculation outside for loop

* don't bother filtering invalid checkpoints

* update const name

* Added docs

* Fix overloaded var names

* change var name

* move utils to utils

* Fix stdev calc
pull/789/head
Mattie Conover 2 years ago committed by GitHub
parent b195edb35a
commit 47f4f10f14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 292 additions and 168 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 401
      typescript/infra/scripts/verify-validator.ts
  2. 35
      typescript/infra/src/utils/utils.ts

401
typescript/infra/scripts/verify-validator.ts
Unescape View File

@ -1,6 +1,8 @@
import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
import yargs from 'yargs'; import yargs from 'yargs';
import { mean, median, stdDev, streamToString } from '../src/utils/utils';
const MAX_MISSING_CHECKPOINTS = 10; const MAX_MISSING_CHECKPOINTS = 10;
interface Checkpoint { interface Checkpoint {
@ -48,22 +50,6 @@ function isValidHashStr(s: string): boolean {
return !!s.match(/^0x[0-9a-f]{1,64}$/im); return !!s.match(/^0x[0-9a-f]{1,64}$/im);
} }
function getArgs() {
return yargs(process.argv.slice(2))
.alias('a', 'address')
.describe('a', 'address of the validator to inspect')
.demandOption('a')
.string('a')
.alias('p', 'prospective')
.describe('p', 'S3 bucket of the prospective validator')
.demandOption('p')
.string('p')
.alias('c', 'control')
.describe('c', 'S3 bucket of the the known (control) validator')
.demandOption('c')
.string('c').argv;
}
class S3Wrapper { class S3Wrapper {
private readonly client: S3Client; private readonly client: S3Client;
readonly region: string; readonly region: string;
@ -104,201 +90,304 @@ class S3Wrapper {
} }
} }
async function main() { class Validator {
const { private readonly controlS3BucketClientWrapper: S3Wrapper;
a: _validatorAddress, private readonly prospectiveS3BucketClientWrapper: S3Wrapper;
p: prospectiveBucket,
c: controlBucket,
} = await getArgs();
const cClient = new S3Wrapper(controlBucket); // accumulators for stats
const pClient = new S3Wrapper(prospectiveBucket); /** Checkpoints the prospective validator has that the control validator does not */
private extraCheckpoints!: number[];
/** Checkpoints the prospective validator does not have that the control validator does have */
private missingCheckpoints!: number[];
/** Checkpoints the prospective validator has but for which we detected an issue */
private invalidCheckpoints!: number[];
/** The difference in modification times on the s3 objects between the control and validator
* buckets. (validator time - control time).
*/
private modTimeDeltasS!: number[];
/** The checkpoints which were, as far as this validation logic is concerned, present and valid */
private validCheckpoints!: number[];
/** The number of checkpoints that the control had that the validator did not have in a row.
* (Not necessarily consecutive indexes) */
private missingInARow!: number;
/** Index of the last index we found an entry for from the prospective validator */
private lastNonMissingCheckpointIndex!: number;
const [{ obj: controlLatestCheckpoint }, { obj: prospectiveLastCheckpoint }] = constructor(
await Promise.all([ public readonly validatorAddress: string,
cClient.getS3Obj<number>('checkpoint_latest_index.json').catch((err) => { public readonly controlS3BucketAddress: string,
console.error( public readonly prospectiveS3BucketAddress: string,
"Failed to get control validator's latest checkpoint.", ) {
err, this.controlS3BucketClientWrapper = new S3Wrapper(
this.controlS3BucketAddress,
); );
process.exit(1); this.prospectiveS3BucketClientWrapper = new S3Wrapper(
}), this.prospectiveS3BucketAddress,
pClient.getS3Obj<number>('checkpoint_latest_index.json').catch((err) => {
console.error(
"Failed to get prospective validator's latest checkpoint.",
err,
); );
process.exit(1); }
}),
]);
if ( initStatsState() {
!isLatestCheckpoint(controlLatestCheckpoint) || this.extraCheckpoints = [];
!isLatestCheckpoint(prospectiveLastCheckpoint) this.missingCheckpoints = [];
) this.invalidCheckpoints = [];
process.exit(1); this.modTimeDeltasS = [];
this.validCheckpoints = [];
this.missingInARow = 0;
this.lastNonMissingCheckpointIndex = Infinity;
}
console.log(`Latest Index`); /**
console.log(`control: ${controlLatestCheckpoint}`); * Validate that the control and prospective validators are in agreement. Will throw an error on
console.log(`prospective: ${prospectiveLastCheckpoint}\n`); * any critical failures and will log stats to the console as it goes.
*
* If we want to make this callable from outside the script later I would suggest making this
* return a stats object or something.
*/
async validate(): Promise<void> {
this.initStatsState();
const { controlLatestCheckpoint, prospectiveLastCheckpoint } =
await this.getLatestCheckpoints();
let extraCheckpoints = []; const maxCheckpointIndex = Math.max(
const missingCheckpoints = []; controlLatestCheckpoint,
let invalidCheckpoints = []; prospectiveLastCheckpoint,
const modTimeDeltasS = []; );
const fullyCorrectCheckpoints = [];
let missingInARow = 0;
let lastNonMissingCheckpointIndex = Infinity;
for ( for (
let i = Math.max(controlLatestCheckpoint, prospectiveLastCheckpoint); let checkpointIndex = maxCheckpointIndex;
i >= 0; checkpointIndex >= 0;
--i --checkpointIndex
) { ) {
if (missingInARow == MAX_MISSING_CHECKPOINTS) { if (this.missingInARow == MAX_MISSING_CHECKPOINTS) {
missingCheckpoints.length -= MAX_MISSING_CHECKPOINTS; this.missingCheckpoints.length -= MAX_MISSING_CHECKPOINTS;
invalidCheckpoints = invalidCheckpoints.filter( this.extraCheckpoints = this.extraCheckpoints.filter(
(j) => j < lastNonMissingCheckpointIndex, (j) => j < this.lastNonMissingCheckpointIndex,
);
extraCheckpoints = extraCheckpoints.filter(
(j) => j < lastNonMissingCheckpointIndex,
); );
break; break;
} }
const key = `checkpoint_${i}.json`; await this.validateCheckpointIndex(checkpointIndex);
let control: Checkpoint | null;
let controlLastMod: Date | null;
try {
const t = await cClient.getS3Obj(key);
if (isCheckpoint(t.obj)) {
if (t.obj.checkpoint.index != i) {
console.log(`${i}: Control index is invalid`, t);
process.exit(1);
}
[control, controlLastMod] = [t.obj, t.modified];
} else {
console.log(`${i}: Invalid control checkpoint`, t);
process.exit(1);
}
} catch (err) {
control = controlLastMod = null;
} }
let prospective: Checkpoint; console.log(
let prospectiveLastMod: Date; `Fully correct checkpoints (${this.validCheckpoints.length}): ${this.validCheckpoints}\n`,
try { );
const t = await pClient.getS3Obj(key); if (this.extraCheckpoints.length)
if (isCheckpoint(t.obj)) { console.log(
[prospective, prospectiveLastMod] = [t.obj, t.modified]; `Extra checkpoints (${this.extraCheckpoints.length}): ${this.extraCheckpoints}\n`,
lastNonMissingCheckpointIndex = i; );
} else { if (this.missingCheckpoints.length)
console.log(`${i}: Invalid prospective checkpoint`, t.obj); console.log(
invalidCheckpoints.push(i); `Missing checkpoints (${this.missingCheckpoints.length}): ${this.missingCheckpoints}\n`,
continue; );
} if (this.invalidCheckpoints.length)
if (!control) { console.log(
extraCheckpoints.push(i); `Invalid checkpoints (${this.invalidCheckpoints.length}): ${this.invalidCheckpoints}\n`,
} );
missingInARow = 0;
} catch (err) { if (this.modTimeDeltasS.length > 1) {
if (control) { // Drop the time of the first one since it is probably way off
missingCheckpoints.push(i); this.modTimeDeltasS.length--;
missingInARow++; console.log(
`Time deltas (∆ < 0 -> prospective came earlier than the control)`,
);
console.log(this.modTimeDeltasS);
console.log(`Median: ${median(this.modTimeDeltasS)}s`);
console.log(`Mean: ${mean(this.modTimeDeltasS)}s`);
console.log(`Stdev: ${stdDev(this.modTimeDeltasS)}s`);
} }
continue;
} }
private async validateCheckpointIndex(
checkpointIndex: number,
): Promise<void> {
const { control, controlLastMod } = (await this.getControlCheckpoint(
checkpointIndex,
)) ?? { control: null, controlLastMod: null };
const getProspectiveCheckpointResult = await this.getProspectiveCheckpoint(
checkpointIndex,
!!control,
);
if (!getProspectiveCheckpointResult) return;
const { prospective, prospectiveLastMod } = getProspectiveCheckpointResult;
console.assert( console.assert(
prospective.checkpoint.index == i, prospective.checkpoint.index == checkpointIndex,
`${i}: checkpoint indexes do not match`, `${checkpointIndex}: checkpoint indexes do not match`,
); );
// TODO: verify signature // TODO: verify signature
if (!control) { if (!control) {
continue; return;
} }
// compare against the control // compare against the control
console.assert( console.assert(
control.checkpoint.outbox_domain == prospective.checkpoint.outbox_domain, control.checkpoint.outbox_domain == prospective.checkpoint.outbox_domain,
`${i}: outbox_domains do not match`, `${checkpointIndex}: outbox_domains do not match`,
); );
console.assert( console.assert(
control.checkpoint.root == prospective.checkpoint.root, control.checkpoint.root == prospective.checkpoint.root,
`${i}: checkpoint roots do not match`, `${checkpointIndex}: checkpoint roots do not match`,
); );
const diffS = const diffS =
(prospectiveLastMod.valueOf() - controlLastMod!.valueOf()) / 1000; (prospectiveLastMod.valueOf() - controlLastMod!.valueOf()) / 1000;
if (Math.abs(diffS) > 10) { if (Math.abs(diffS) > 10) {
console.log(`${i}: Modification times differ by ${diffS}s`); console.log(`${checkpointIndex}: Modification times differ by ${diffS}s`);
} }
modTimeDeltasS.push(diffS); this.modTimeDeltasS.push(diffS);
fullyCorrectCheckpoints.push(i); this.validCheckpoints.push(checkpointIndex);
} }
console.log( private async getLatestCheckpoints(): Promise<{
`Fully correct checkpoints (${fullyCorrectCheckpoints.length}): ${fullyCorrectCheckpoints}\n`, controlLatestCheckpoint: number;
); prospectiveLastCheckpoint: number;
if (extraCheckpoints.length) }> {
console.log( const [
`Extra checkpoints (${extraCheckpoints.length}): ${extraCheckpoints}\n`, { obj: controlLatestCheckpoint },
{ obj: prospectiveLastCheckpoint },
] = await Promise.all([
this.controlS3BucketClientWrapper
.getS3Obj<number>('checkpoint_latest_index.json')
.catch((err) => {
console.error(
"Failed to get control validator's latest checkpoint.",
err,
); );
if (missingCheckpoints.length) process.exit(1);
console.log( }),
`Missing checkpoints (${missingCheckpoints.length}): ${missingCheckpoints}\n`, this.prospectiveS3BucketClientWrapper
.getS3Obj<number>('checkpoint_latest_index.json')
.catch((err) => {
console.error(
"Failed to get prospective validator's latest checkpoint.",
err,
); );
if (invalidCheckpoints.length) process.exit(1);
console.log( }),
`Invalid checkpoints (${invalidCheckpoints.length}): ${invalidCheckpoints}\n`, ]);
if (
!isLatestCheckpoint(controlLatestCheckpoint) ||
!isLatestCheckpoint(prospectiveLastCheckpoint)
)
throw new Error('Invalid latest checkpoint data');
console.log(`Latest Index`);
console.log(`control: ${controlLatestCheckpoint}`);
console.log(`prospective: ${prospectiveLastCheckpoint}\n`);
return { controlLatestCheckpoint, prospectiveLastCheckpoint };
}
private async getControlCheckpoint(
checkpointIndex: number,
): Promise<{ control: Checkpoint; controlLastMod: Date } | null> {
let control: Checkpoint, controlLastMod: Date, unrecoverableError;
try {
const s3Object = await this.controlS3BucketClientWrapper.getS3Obj(
this.checkpointKey(checkpointIndex),
); );
if (isCheckpoint(s3Object.obj)) {
if (s3Object.obj.checkpoint.index != checkpointIndex) {
console.log(`${checkpointIndex}: Control index is invalid`, s3Object);
process.exit(1);
}
[control, controlLastMod] = [s3Object.obj, s3Object.modified];
} else {
console.log(`${checkpointIndex}: Invalid control checkpoint`, s3Object);
unrecoverableError = new Error('Invalid control checkpoint.');
}
} catch (err) {
return null;
}
if (unrecoverableError) throw unrecoverableError;
return { control: control!, controlLastMod: controlLastMod! };
}
if (modTimeDeltasS.length > 1) { private async getProspectiveCheckpoint(
// Drop the time of the first one since it is probably way off checkpointIndex: number,
modTimeDeltasS.length--; controlFoundForIndex: boolean,
): Promise<{ prospective: Checkpoint; prospectiveLastMod: Date } | null> {
let prospective: Checkpoint, prospectiveLastMod: Date;
try {
const s3Object = await this.prospectiveS3BucketClientWrapper.getS3Obj(
this.checkpointKey(checkpointIndex),
);
if (isCheckpoint(s3Object.obj)) {
[prospective, prospectiveLastMod] = [s3Object.obj, s3Object.modified];
this.lastNonMissingCheckpointIndex = checkpointIndex;
} else {
console.log( console.log(
`Time deltas (∆ < 0 -> prospective came earlier than the control)`, `${checkpointIndex}: Invalid prospective checkpoint`,
s3Object.obj,
); );
console.log(modTimeDeltasS); this.invalidCheckpoints.push(checkpointIndex);
console.log(`Median: ${median(modTimeDeltasS)}s`); return null;
console.log(`Mean: ${mean(modTimeDeltasS)}s`); }
console.log(`Stdev: ${stdDev(modTimeDeltasS)}s`); if (!controlFoundForIndex) {
this.extraCheckpoints.push(checkpointIndex);
}
this.missingInARow = 0;
} catch (err) {
if (controlFoundForIndex) {
this.missingCheckpoints.push(checkpointIndex);
this.missingInARow++;
}
return null;
} }
}
function median(a: number[]): number { return {
a = [...a]; // clone prospective: prospective!,
a.sort((a, b) => a - b); prospectiveLastMod: prospectiveLastMod!,
if (a.length <= 0) { };
return 0; }
} else if (a.length % 2 == 0) {
return (a[a.length / 2] + a[a.length / 2 - 1]) / 2; private checkpointKey(checkpointIndex: number): string {
} else { return `checkpoint_${checkpointIndex}.json`;
return a[(a.length - 1) / 2];
} }
} }
function mean(a: number[]): number { ////
return a.reduce((acc, i) => acc + i, 0) / a.length; // Bootstrapper
////
function getArgs() {
return yargs(process.argv.slice(2))
.alias('a', 'address')
.describe('a', 'address of the validator to inspect')
.demandOption('a')
.string('a')
.alias('p', 'prospective')
.describe('p', 'S3 bucket of the prospective validator')
.demandOption('p')
.string('p')
.alias('c', 'control')
.describe('c', 'S3 bucket of the the known (control) validator')
.demandOption('c')
.string('c').argv;
} }
function stdDev(a: number[]): number { async function main() {
return Math.sqrt( const {
a.map((i) => i * i).reduce((acc, i) => acc + i, 0) / a.length, a: validatorAddress,
p: prospectiveBucket,
c: controlBucket,
} = await getArgs();
const validator = new Validator(
validatorAddress,
prospectiveBucket,
controlBucket,
); );
}
function streamToString(stream: NodeJS.ReadableStream): Promise<string> { try {
return new Promise((resolve, reject) => { await validator.validate();
const chunks: string[] = []; } catch (err) {
stream console.error(err);
.setEncoding('utf8') process.exit(1);
.on('data', (chunk) => chunks.push(chunk)) }
.on('error', (err) => reject(err))
.on('end', () => resolve(String.prototype.concat(...chunks)));
});
} }
main().catch(console.error); main().catch(console.error);

35
typescript/infra/src/utils/utils.ts
Unescape View File

@ -205,3 +205,38 @@ export function assertContext(contextStr: string): Contexts {
}`, }`,
); );
} }
export function median(a: number[]): number {
a = [...a]; // clone
a.sort((a, b) => a - b);
if (a.length <= 0) {
return 0;
} else if (a.length % 2 == 0) {
return (a[a.length / 2] + a[a.length / 2 - 1]) / 2;
} else {
return a[(a.length - 1) / 2];
}
}
export function mean(a: number[]): number {
return a.reduce((acc, i) => acc + i, 0) / a.length;
}
export function stdDev(a: number[]): number {
const xbar = mean(a);
return Math.sqrt(
a.map((x) => Math.pow(x - xbar, 2)).reduce((acc, i) => acc + i, 0) /
a.length,
);
}
export function streamToString(stream: NodeJS.ReadableStream): Promise<string> {
return new Promise((resolve, reject) => {
const chunks: string[] = [];
stream
.setEncoding('utf8')
.on('data', (chunk) => chunks.push(chunk))
.on('error', (err) => reject(err))
.on('end', () => resolve(String.prototype.concat(...chunks)));
});
}

Write Preview
Loading…
Cancel
Save