CipherLabs
/
hyperlane-monorepo
mirror of https://github.com/hyperlane-xyz/hyperlane-monorepo
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactoring validator verification loop for clarity (#773)

Browse Source 
* Refactoring loop for clarity

* fix process name

* remove `t` var name

* move calculation outside for loop

* don't bother filtering invalid checkpoints

* update const name

* Added docs

* Fix overloaded var names

* change var name

* move utils to utils

* Fix stdev calc
pull/789/head
Mattie Conover 2 years ago committed by GitHub
parent b195edb35a
commit 47f4f10f14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 292 additions and 168 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 399
      typescript/infra/scripts/verify-validator.ts
  2. 35
      typescript/infra/src/utils/utils.ts

399
typescript/infra/scripts/verify-validator.ts
Unescape View File

@ -1,6 +1,8 @@
import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
import yargs from 'yargs';
import { mean, median, stdDev, streamToString } from '../src/utils/utils';
const MAX_MISSING_CHECKPOINTS = 10;
interface Checkpoint {
@ -48,22 +50,6 @@ function isValidHashStr(s: string): boolean {
return !!s.match(/^0x[0-9a-f]{1,64}$/im);
}
function getArgs() {
return yargs(process.argv.slice(2))
.alias('a', 'address')
.describe('a', 'address of the validator to inspect')
.demandOption('a')
.string('a')
.alias('p', 'prospective')
.describe('p', 'S3 bucket of the prospective validator')
.demandOption('p')
.string('p')
.alias('c', 'control')
.describe('c', 'S3 bucket of the the known (control) validator')
.demandOption('c')
.string('c').argv;
}
class S3Wrapper {
private readonly client: S3Client;
readonly region: string;
@ -104,201 +90,304 @@ class S3Wrapper {
}
}
async function main() {
const {
a: _validatorAddress,
p: prospectiveBucket,
c: controlBucket,
} = await getArgs();
class Validator {
private readonly controlS3BucketClientWrapper: S3Wrapper;
private readonly prospectiveS3BucketClientWrapper: S3Wrapper;
const cClient = new S3Wrapper(controlBucket);
const pClient = new S3Wrapper(prospectiveBucket);
// accumulators for stats
/** Checkpoints the prospective validator has that the control validator does not */
private extraCheckpoints!: number[];
/** Checkpoints the prospective validator does not have that the control validator does have */
private missingCheckpoints!: number[];
/** Checkpoints the prospective validator has but for which we detected an issue */
private invalidCheckpoints!: number[];
/** The difference in modification times on the s3 objects between the control and validator
* buckets. (validator time - control time).
*/
private modTimeDeltasS!: number[];
/** The checkpoints which were, as far as this validation logic is concerned, present and valid */
private validCheckpoints!: number[];
/** The number of checkpoints that the control had that the validator did not have in a row.
* (Not necessarily consecutive indexes) */
private missingInARow!: number;
/** Index of the last index we found an entry for from the prospective validator */
private lastNonMissingCheckpointIndex!: number;
const [{ obj: controlLatestCheckpoint }, { obj: prospectiveLastCheckpoint }] =
await Promise.all([
cClient.getS3Obj<number>('checkpoint_latest_index.json').catch((err) => {
console.error(
"Failed to get control validator's latest checkpoint.",
err,
constructor(
public readonly validatorAddress: string,
public readonly controlS3BucketAddress: string,
public readonly prospectiveS3BucketAddress: string,
) {
this.controlS3BucketClientWrapper = new S3Wrapper(
this.controlS3BucketAddress,
);
process.exit(1);
}),
pClient.getS3Obj<number>('checkpoint_latest_index.json').catch((err) => {
console.error(
"Failed to get prospective validator's latest checkpoint.",
err,
this.prospectiveS3BucketClientWrapper = new S3Wrapper(
this.prospectiveS3BucketAddress,
);
process.exit(1);
}),
]);
}
if (
!isLatestCheckpoint(controlLatestCheckpoint) ||
!isLatestCheckpoint(prospectiveLastCheckpoint)
)
process.exit(1);
initStatsState() {
this.extraCheckpoints = [];
this.missingCheckpoints = [];
this.invalidCheckpoints = [];
this.modTimeDeltasS = [];
this.validCheckpoints = [];
this.missingInARow = 0;
this.lastNonMissingCheckpointIndex = Infinity;
}
console.log(`Latest Index`);
console.log(`control: ${controlLatestCheckpoint}`);
console.log(`prospective: ${prospectiveLastCheckpoint}\n`);
/**
* Validate that the control and prospective validators are in agreement. Will throw an error on
* any critical failures and will log stats to the console as it goes.
*
* If we want to make this callable from outside the script later I would suggest making this
* return a stats object or something.
*/
async validate(): Promise<void> {
this.initStatsState();
const { controlLatestCheckpoint, prospectiveLastCheckpoint } =
await this.getLatestCheckpoints();
let extraCheckpoints = [];
const missingCheckpoints = [];
let invalidCheckpoints = [];
const modTimeDeltasS = [];
const fullyCorrectCheckpoints = [];
let missingInARow = 0;
let lastNonMissingCheckpointIndex = Infinity;
const maxCheckpointIndex = Math.max(
controlLatestCheckpoint,
prospectiveLastCheckpoint,
);
for (
let i = Math.max(controlLatestCheckpoint, prospectiveLastCheckpoint);
i >= 0;
--i
let checkpointIndex = maxCheckpointIndex;
checkpointIndex >= 0;
--checkpointIndex
) {
if (missingInARow == MAX_MISSING_CHECKPOINTS) {
missingCheckpoints.length -= MAX_MISSING_CHECKPOINTS;
invalidCheckpoints = invalidCheckpoints.filter(
(j) => j < lastNonMissingCheckpointIndex,
);
extraCheckpoints = extraCheckpoints.filter(
(j) => j < lastNonMissingCheckpointIndex,
if (this.missingInARow == MAX_MISSING_CHECKPOINTS) {
this.missingCheckpoints.length -= MAX_MISSING_CHECKPOINTS;
this.extraCheckpoints = this.extraCheckpoints.filter(
(j) => j < this.lastNonMissingCheckpointIndex,
);
break;
}
const key = `checkpoint_${i}.json`;
let control: Checkpoint | null;
let controlLastMod: Date | null;
try {
const t = await cClient.getS3Obj(key);
if (isCheckpoint(t.obj)) {
if (t.obj.checkpoint.index != i) {
console.log(`${i}: Control index is invalid`, t);
process.exit(1);
}
[control, controlLastMod] = [t.obj, t.modified];
} else {
console.log(`${i}: Invalid control checkpoint`, t);
process.exit(1);
}
} catch (err) {
control = controlLastMod = null;
await this.validateCheckpointIndex(checkpointIndex);
}
let prospective: Checkpoint;
let prospectiveLastMod: Date;
try {
const t = await pClient.getS3Obj(key);
if (isCheckpoint(t.obj)) {
[prospective, prospectiveLastMod] = [t.obj, t.modified];
lastNonMissingCheckpointIndex = i;
} else {
console.log(`${i}: Invalid prospective checkpoint`, t.obj);
invalidCheckpoints.push(i);
continue;
}
if (!control) {
extraCheckpoints.push(i);
}
missingInARow = 0;
} catch (err) {
if (control) {
missingCheckpoints.push(i);
missingInARow++;
console.log(
`Fully correct checkpoints (${this.validCheckpoints.length}): ${this.validCheckpoints}\n`,
);
if (this.extraCheckpoints.length)
console.log(
`Extra checkpoints (${this.extraCheckpoints.length}): ${this.extraCheckpoints}\n`,
);
if (this.missingCheckpoints.length)
console.log(
`Missing checkpoints (${this.missingCheckpoints.length}): ${this.missingCheckpoints}\n`,
);
if (this.invalidCheckpoints.length)
console.log(
`Invalid checkpoints (${this.invalidCheckpoints.length}): ${this.invalidCheckpoints}\n`,
);
if (this.modTimeDeltasS.length > 1) {
// Drop the time of the first one since it is probably way off
this.modTimeDeltasS.length--;
console.log(
`Time deltas (∆ < 0 -> prospective came earlier than the control)`,
);
console.log(this.modTimeDeltasS);
console.log(`Median: ${median(this.modTimeDeltasS)}s`);
console.log(`Mean: ${mean(this.modTimeDeltasS)}s`);
console.log(`Stdev: ${stdDev(this.modTimeDeltasS)}s`);
}
continue;
}
private async validateCheckpointIndex(
checkpointIndex: number,
): Promise<void> {
const { control, controlLastMod } = (await this.getControlCheckpoint(
checkpointIndex,
)) ?? { control: null, controlLastMod: null };
const getProspectiveCheckpointResult = await this.getProspectiveCheckpoint(
checkpointIndex,
!!control,
);
if (!getProspectiveCheckpointResult) return;
const { prospective, prospectiveLastMod } = getProspectiveCheckpointResult;
console.assert(
prospective.checkpoint.index == i,
`${i}: checkpoint indexes do not match`,
prospective.checkpoint.index == checkpointIndex,
`${checkpointIndex}: checkpoint indexes do not match`,
);
// TODO: verify signature
if (!control) {
continue;
return;
}
// compare against the control
console.assert(
control.checkpoint.outbox_domain == prospective.checkpoint.outbox_domain,
`${i}: outbox_domains do not match`,
`${checkpointIndex}: outbox_domains do not match`,
);
console.assert(
control.checkpoint.root == prospective.checkpoint.root,
`${i}: checkpoint roots do not match`,
`${checkpointIndex}: checkpoint roots do not match`,
);
const diffS =
(prospectiveLastMod.valueOf() - controlLastMod!.valueOf()) / 1000;
if (Math.abs(diffS) > 10) {
console.log(`${i}: Modification times differ by ${diffS}s`);
console.log(`${checkpointIndex}: Modification times differ by ${diffS}s`);
}
modTimeDeltasS.push(diffS);
fullyCorrectCheckpoints.push(i);
this.modTimeDeltasS.push(diffS);
this.validCheckpoints.push(checkpointIndex);
}
console.log(
`Fully correct checkpoints (${fullyCorrectCheckpoints.length}): ${fullyCorrectCheckpoints}\n`,
);
if (extraCheckpoints.length)
console.log(
`Extra checkpoints (${extraCheckpoints.length}): ${extraCheckpoints}\n`,
private async getLatestCheckpoints(): Promise<{
controlLatestCheckpoint: number;
prospectiveLastCheckpoint: number;
}> {
const [
{ obj: controlLatestCheckpoint },
{ obj: prospectiveLastCheckpoint },
] = await Promise.all([
this.controlS3BucketClientWrapper
.getS3Obj<number>('checkpoint_latest_index.json')
.catch((err) => {
console.error(
"Failed to get control validator's latest checkpoint.",
err,
);
if (missingCheckpoints.length)
console.log(
`Missing checkpoints (${missingCheckpoints.length}): ${missingCheckpoints}\n`,
process.exit(1);
}),
this.prospectiveS3BucketClientWrapper
.getS3Obj<number>('checkpoint_latest_index.json')
.catch((err) => {
console.error(
"Failed to get prospective validator's latest checkpoint.",
err,
);
if (invalidCheckpoints.length)
console.log(
`Invalid checkpoints (${invalidCheckpoints.length}): ${invalidCheckpoints}\n`,
process.exit(1);
}),
]);
if (
!isLatestCheckpoint(controlLatestCheckpoint) ||
!isLatestCheckpoint(prospectiveLastCheckpoint)
)
throw new Error('Invalid latest checkpoint data');
console.log(`Latest Index`);
console.log(`control: ${controlLatestCheckpoint}`);
console.log(`prospective: ${prospectiveLastCheckpoint}\n`);
return { controlLatestCheckpoint, prospectiveLastCheckpoint };
}
private async getControlCheckpoint(
checkpointIndex: number,
): Promise<{ control: Checkpoint; controlLastMod: Date } | null> {
let control: Checkpoint, controlLastMod: Date, unrecoverableError;
try {
const s3Object = await this.controlS3BucketClientWrapper.getS3Obj(
this.checkpointKey(checkpointIndex),
);
if (isCheckpoint(s3Object.obj)) {
if (s3Object.obj.checkpoint.index != checkpointIndex) {
console.log(`${checkpointIndex}: Control index is invalid`, s3Object);
process.exit(1);
}
[control, controlLastMod] = [s3Object.obj, s3Object.modified];
} else {
console.log(`${checkpointIndex}: Invalid control checkpoint`, s3Object);
unrecoverableError = new Error('Invalid control checkpoint.');
}
} catch (err) {
return null;
}
if (unrecoverableError) throw unrecoverableError;
return { control: control!, controlLastMod: controlLastMod! };
}
if (modTimeDeltasS.length > 1) {
// Drop the time of the first one since it is probably way off
modTimeDeltasS.length--;
private async getProspectiveCheckpoint(
checkpointIndex: number,
controlFoundForIndex: boolean,
): Promise<{ prospective: Checkpoint; prospectiveLastMod: Date } | null> {
let prospective: Checkpoint, prospectiveLastMod: Date;
try {
const s3Object = await this.prospectiveS3BucketClientWrapper.getS3Obj(
this.checkpointKey(checkpointIndex),
);
if (isCheckpoint(s3Object.obj)) {
[prospective, prospectiveLastMod] = [s3Object.obj, s3Object.modified];
this.lastNonMissingCheckpointIndex = checkpointIndex;
} else {
console.log(
`Time deltas (∆ < 0 -> prospective came earlier than the control)`,
`${checkpointIndex}: Invalid prospective checkpoint`,
s3Object.obj,
);
console.log(modTimeDeltasS);
console.log(`Median: ${median(modTimeDeltasS)}s`);
console.log(`Mean: ${mean(modTimeDeltasS)}s`);
console.log(`Stdev: ${stdDev(modTimeDeltasS)}s`);
this.invalidCheckpoints.push(checkpointIndex);
return null;
}
if (!controlFoundForIndex) {
this.extraCheckpoints.push(checkpointIndex);
}
this.missingInARow = 0;
} catch (err) {
if (controlFoundForIndex) {
this.missingCheckpoints.push(checkpointIndex);
this.missingInARow++;
}
return null;
}
function median(a: number[]): number {
a = [...a]; // clone
a.sort((a, b) => a - b);
if (a.length <= 0) {
return 0;
} else if (a.length % 2 == 0) {
return (a[a.length / 2] + a[a.length / 2 - 1]) / 2;
} else {
return a[(a.length - 1) / 2];
return {
prospective: prospective!,
prospectiveLastMod: prospectiveLastMod!,
};
}
private checkpointKey(checkpointIndex: number): string {
return `checkpoint_${checkpointIndex}.json`;
}
}
function mean(a: number[]): number {
return a.reduce((acc, i) => acc + i, 0) / a.length;
////
// Bootstrapper
////
function getArgs() {
return yargs(process.argv.slice(2))
.alias('a', 'address')
.describe('a', 'address of the validator to inspect')
.demandOption('a')
.string('a')
.alias('p', 'prospective')
.describe('p', 'S3 bucket of the prospective validator')
.demandOption('p')
.string('p')
.alias('c', 'control')
.describe('c', 'S3 bucket of the the known (control) validator')
.demandOption('c')
.string('c').argv;
}
function stdDev(a: number[]): number {
return Math.sqrt(
a.map((i) => i * i).reduce((acc, i) => acc + i, 0) / a.length,
async function main() {
const {
a: validatorAddress,
p: prospectiveBucket,
c: controlBucket,
} = await getArgs();
const validator = new Validator(
validatorAddress,
prospectiveBucket,
controlBucket,
);
}
function streamToString(stream: NodeJS.ReadableStream): Promise<string> {
return new Promise((resolve, reject) => {
const chunks: string[] = [];
stream
.setEncoding('utf8')
.on('data', (chunk) => chunks.push(chunk))
.on('error', (err) => reject(err))
.on('end', () => resolve(String.prototype.concat(...chunks)));
});
try {
await validator.validate();
} catch (err) {
console.error(err);
process.exit(1);
}
}
main().catch(console.error);

35
typescript/infra/src/utils/utils.ts
Unescape View File

@ -205,3 +205,38 @@ export function assertContext(contextStr: string): Contexts {
}`,
);
}
export function median(a: number[]): number {
a = [...a]; // clone
a.sort((a, b) => a - b);
if (a.length <= 0) {
return 0;
} else if (a.length % 2 == 0) {
return (a[a.length / 2] + a[a.length / 2 - 1]) / 2;
} else {
return a[(a.length - 1) / 2];
}
}
export function mean(a: number[]): number {
return a.reduce((acc, i) => acc + i, 0) / a.length;
}
export function stdDev(a: number[]): number {
const xbar = mean(a);
return Math.sqrt(
a.map((x) => Math.pow(x - xbar, 2)).reduce((acc, i) => acc + i, 0) /
a.length,
);
}
export function streamToString(stream: NodeJS.ReadableStream): Promise<string> {
return new Promise((resolve, reject) => {
const chunks: string[] = [];
stream
.setEncoding('utf8')
.on('data', (chunk) => chunks.push(chunk))
.on('error', (err) => reject(err))
.on('end', () => resolve(String.prototype.concat(...chunks)));
});
}

Write Preview
Loading…
Cancel
Save