Refactoring validator verification loop for clarity (#773)

* Refactoring loop for clarity

* fix process name

* remove `t` var name

* move calculation outside for loop

* don't bother filtering invalid checkpoints

* update const name

* Added docs

* Fix overloaded var names

* change var name

* move utils to utils

* Fix stdev calc

typescript/infra/scripts/verify-validator.ts

@@ -1,6 +1,8 @@
 import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
 import yargs from 'yargs';
 
+import { mean, median, stdDev, streamToString } from '../src/utils/utils';
+
 const MAX_MISSING_CHECKPOINTS = 10;
 
 interface Checkpoint {
@@ -48,22 +50,6 @@ function isValidHashStr(s: string): boolean {
   return !!s.match(/^0x[0-9a-f]{1,64}$/im);
 }
 
-function getArgs() {
-  return yargs(process.argv.slice(2))
-    .alias('a', 'address')
-    .describe('a', 'address of the validator to inspect')
-    .demandOption('a')
-    .string('a')
-    .alias('p', 'prospective')
-    .describe('p', 'S3 bucket of the prospective validator')
-    .demandOption('p')
-    .string('p')
-    .alias('c', 'control')
-    .describe('c', 'S3 bucket of the the known (control) validator')
-    .demandOption('c')
-    .string('c').argv;
-}
-
 class S3Wrapper {
   private readonly client: S3Client;
   readonly region: string;
@@ -104,201 +90,304 @@ class S3Wrapper {
   }
 }
 
-async function main() {
-  const {
-    a: _validatorAddress,
-    p: prospectiveBucket,
-    c: controlBucket,
-  } = await getArgs();
+class Validator {
+  private readonly controlS3BucketClientWrapper: S3Wrapper;
+  private readonly prospectiveS3BucketClientWrapper: S3Wrapper;
 
-  const cClient = new S3Wrapper(controlBucket);
-  const pClient = new S3Wrapper(prospectiveBucket);
+  // accumulators for stats
+  /** Checkpoints the prospective validator has that the control validator does not */
+  private extraCheckpoints!: number[];
+  /** Checkpoints the prospective validator does not have that the control validator does have */
+  private missingCheckpoints!: number[];
+  /** Checkpoints the prospective validator has but for which we detected an issue */
+  private invalidCheckpoints!: number[];
+  /** The difference in modification times on the s3 objects between the control and validator
+   * buckets. (validator time - control time).
+   */
+  private modTimeDeltasS!: number[];
+  /** The checkpoints which were, as far as this validation logic is concerned, present and valid */
+  private validCheckpoints!: number[];
+  /** The number of checkpoints that the control had that the validator did not have in a row.
+   * (Not necessarily consecutive indexes) */
+  private missingInARow!: number;
+  /** Index of the last index we found an entry for from the prospective validator */
+  private lastNonMissingCheckpointIndex!: number;
 
-  const [{ obj: controlLatestCheckpoint }, { obj: prospectiveLastCheckpoint }] =
-    await Promise.all([
-      cClient.getS3Obj<number>('checkpoint_latest_index.json').catch((err) => {
-        console.error(
-          "Failed to get control validator's latest checkpoint.",
-          err,
+  constructor(
+    public readonly validatorAddress: string,
+    public readonly controlS3BucketAddress: string,
+    public readonly prospectiveS3BucketAddress: string,
+  ) {
+    this.controlS3BucketClientWrapper = new S3Wrapper(
+      this.controlS3BucketAddress,
     );
-        process.exit(1);
-      }),
-      pClient.getS3Obj<number>('checkpoint_latest_index.json').catch((err) => {
-        console.error(
-          "Failed to get prospective validator's latest checkpoint.",
-          err,
+    this.prospectiveS3BucketClientWrapper = new S3Wrapper(
+      this.prospectiveS3BucketAddress,
     );
-        process.exit(1);
-      }),
-    ]);
+  }
 
-  if (
-    !isLatestCheckpoint(controlLatestCheckpoint) ||
-    !isLatestCheckpoint(prospectiveLastCheckpoint)
-  )
-    process.exit(1);
+  initStatsState() {
+    this.extraCheckpoints = [];
+    this.missingCheckpoints = [];
+    this.invalidCheckpoints = [];
+    this.modTimeDeltasS = [];
+    this.validCheckpoints = [];
+    this.missingInARow = 0;
+    this.lastNonMissingCheckpointIndex = Infinity;
+  }
 
-  console.log(`Latest Index`);
-  console.log(`control: ${controlLatestCheckpoint}`);
-  console.log(`prospective: ${prospectiveLastCheckpoint}\n`);
+  /**
+   * Validate that the control and prospective validators are in agreement. Will throw an error on
+   * any critical failures and will log stats to the console as it goes.
+   *
+   * If we want to make this callable from outside the script later I would suggest making this
+   * return a stats object or something.
+   */
+  async validate(): Promise<void> {
+    this.initStatsState();
+
+    const { controlLatestCheckpoint, prospectiveLastCheckpoint } =
+      await this.getLatestCheckpoints();
 
-  let extraCheckpoints = [];
-  const missingCheckpoints = [];
-  let invalidCheckpoints = [];
-  const modTimeDeltasS = [];
-  const fullyCorrectCheckpoints = [];
-  let missingInARow = 0;
-  let lastNonMissingCheckpointIndex = Infinity;
+    const maxCheckpointIndex = Math.max(
+      controlLatestCheckpoint,
+      prospectiveLastCheckpoint,
+    );
     for (
-    let i = Math.max(controlLatestCheckpoint, prospectiveLastCheckpoint);
-    i >= 0;
-    --i
+      let checkpointIndex = maxCheckpointIndex;
+      checkpointIndex >= 0;
+      --checkpointIndex
     ) {
-    if (missingInARow == MAX_MISSING_CHECKPOINTS) {
-      missingCheckpoints.length -= MAX_MISSING_CHECKPOINTS;
-      invalidCheckpoints = invalidCheckpoints.filter(
-        (j) => j < lastNonMissingCheckpointIndex,
-      );
-      extraCheckpoints = extraCheckpoints.filter(
-        (j) => j < lastNonMissingCheckpointIndex,
+      if (this.missingInARow == MAX_MISSING_CHECKPOINTS) {
+        this.missingCheckpoints.length -= MAX_MISSING_CHECKPOINTS;
+        this.extraCheckpoints = this.extraCheckpoints.filter(
+          (j) => j < this.lastNonMissingCheckpointIndex,
         );
         break;
       }
 
-    const key = `checkpoint_${i}.json`;
-
-    let control: Checkpoint | null;
-    let controlLastMod: Date | null;
-    try {
-      const t = await cClient.getS3Obj(key);
-      if (isCheckpoint(t.obj)) {
-        if (t.obj.checkpoint.index != i) {
-          console.log(`${i}: Control index is invalid`, t);
-          process.exit(1);
-        }
-        [control, controlLastMod] = [t.obj, t.modified];
-      } else {
-        console.log(`${i}: Invalid control checkpoint`, t);
-        process.exit(1);
-      }
-    } catch (err) {
-      control = controlLastMod = null;
+      await this.validateCheckpointIndex(checkpointIndex);
     }
 
-    let prospective: Checkpoint;
-    let prospectiveLastMod: Date;
-    try {
-      const t = await pClient.getS3Obj(key);
-      if (isCheckpoint(t.obj)) {
-        [prospective, prospectiveLastMod] = [t.obj, t.modified];
-        lastNonMissingCheckpointIndex = i;
-      } else {
-        console.log(`${i}: Invalid prospective checkpoint`, t.obj);
-        invalidCheckpoints.push(i);
-        continue;
-      }
-      if (!control) {
-        extraCheckpoints.push(i);
-      }
-      missingInARow = 0;
-    } catch (err) {
-      if (control) {
-        missingCheckpoints.push(i);
-        missingInARow++;
+    console.log(
+      `Fully correct checkpoints (${this.validCheckpoints.length}): ${this.validCheckpoints}\n`,
+    );
+    if (this.extraCheckpoints.length)
+      console.log(
+        `Extra checkpoints (${this.extraCheckpoints.length}): ${this.extraCheckpoints}\n`,
+      );
+    if (this.missingCheckpoints.length)
+      console.log(
+        `Missing checkpoints (${this.missingCheckpoints.length}): ${this.missingCheckpoints}\n`,
+      );
+    if (this.invalidCheckpoints.length)
+      console.log(
+        `Invalid checkpoints (${this.invalidCheckpoints.length}): ${this.invalidCheckpoints}\n`,
+      );
+
+    if (this.modTimeDeltasS.length > 1) {
+      // Drop the time of the first one since it is probably way off
+      this.modTimeDeltasS.length--;
+      console.log(
+        `Time deltas (∆ < 0 -> prospective came earlier than the control)`,
+      );
+      console.log(this.modTimeDeltasS);
+      console.log(`Median: ${median(this.modTimeDeltasS)}s`);
+      console.log(`Mean:   ${mean(this.modTimeDeltasS)}s`);
+      console.log(`Stdev:  ${stdDev(this.modTimeDeltasS)}s`);
     }
-      continue;
   }
 
+  private async validateCheckpointIndex(
+    checkpointIndex: number,
+  ): Promise<void> {
+    const { control, controlLastMod } = (await this.getControlCheckpoint(
+      checkpointIndex,
+    )) ?? { control: null, controlLastMod: null };
+
+    const getProspectiveCheckpointResult = await this.getProspectiveCheckpoint(
+      checkpointIndex,
+      !!control,
+    );
+    if (!getProspectiveCheckpointResult) return;
+    const { prospective, prospectiveLastMod } = getProspectiveCheckpointResult;
+
     console.assert(
-      prospective.checkpoint.index == i,
-      `${i}: checkpoint indexes do not match`,
+      prospective.checkpoint.index == checkpointIndex,
+      `${checkpointIndex}: checkpoint indexes do not match`,
     );
 
     // TODO: verify signature
 
     if (!control) {
-      continue;
+      return;
     }
 
     // compare against the control
     console.assert(
       control.checkpoint.outbox_domain == prospective.checkpoint.outbox_domain,
-      `${i}: outbox_domains do not match`,
+      `${checkpointIndex}: outbox_domains do not match`,
     );
     console.assert(
       control.checkpoint.root == prospective.checkpoint.root,
-      `${i}: checkpoint roots do not match`,
+      `${checkpointIndex}: checkpoint roots do not match`,
     );
 
     const diffS =
       (prospectiveLastMod.valueOf() - controlLastMod!.valueOf()) / 1000;
     if (Math.abs(diffS) > 10) {
-      console.log(`${i}: Modification times differ by ${diffS}s`);
+      console.log(`${checkpointIndex}: Modification times differ by ${diffS}s`);
     }
-    modTimeDeltasS.push(diffS);
-    fullyCorrectCheckpoints.push(i);
+    this.modTimeDeltasS.push(diffS);
+    this.validCheckpoints.push(checkpointIndex);
   }
 
-  console.log(
-    `Fully correct checkpoints (${fullyCorrectCheckpoints.length}): ${fullyCorrectCheckpoints}\n`,
-  );
-  if (extraCheckpoints.length)
-    console.log(
-      `Extra checkpoints (${extraCheckpoints.length}): ${extraCheckpoints}\n`,
+  private async getLatestCheckpoints(): Promise<{
+    controlLatestCheckpoint: number;
+    prospectiveLastCheckpoint: number;
+  }> {
+    const [
+      { obj: controlLatestCheckpoint },
+      { obj: prospectiveLastCheckpoint },
+    ] = await Promise.all([
+      this.controlS3BucketClientWrapper
+        .getS3Obj<number>('checkpoint_latest_index.json')
+        .catch((err) => {
+          console.error(
+            "Failed to get control validator's latest checkpoint.",
+            err,
           );
-  if (missingCheckpoints.length)
-    console.log(
-      `Missing checkpoints (${missingCheckpoints.length}): ${missingCheckpoints}\n`,
+          process.exit(1);
+        }),
+      this.prospectiveS3BucketClientWrapper
+        .getS3Obj<number>('checkpoint_latest_index.json')
+        .catch((err) => {
+          console.error(
+            "Failed to get prospective validator's latest checkpoint.",
+            err,
           );
-  if (invalidCheckpoints.length)
-    console.log(
-      `Invalid checkpoints (${invalidCheckpoints.length}): ${invalidCheckpoints}\n`,
+          process.exit(1);
+        }),
+    ]);
+
+    if (
+      !isLatestCheckpoint(controlLatestCheckpoint) ||
+      !isLatestCheckpoint(prospectiveLastCheckpoint)
+    )
+      throw new Error('Invalid latest checkpoint data');
+
+    console.log(`Latest Index`);
+    console.log(`control: ${controlLatestCheckpoint}`);
+    console.log(`prospective: ${prospectiveLastCheckpoint}\n`);
+    return { controlLatestCheckpoint, prospectiveLastCheckpoint };
+  }
+
+  private async getControlCheckpoint(
+    checkpointIndex: number,
+  ): Promise<{ control: Checkpoint; controlLastMod: Date } | null> {
+    let control: Checkpoint, controlLastMod: Date, unrecoverableError;
+    try {
+      const s3Object = await this.controlS3BucketClientWrapper.getS3Obj(
+        this.checkpointKey(checkpointIndex),
       );
+      if (isCheckpoint(s3Object.obj)) {
+        if (s3Object.obj.checkpoint.index != checkpointIndex) {
+          console.log(`${checkpointIndex}: Control index is invalid`, s3Object);
+          process.exit(1);
+        }
+        [control, controlLastMod] = [s3Object.obj, s3Object.modified];
+      } else {
+        console.log(`${checkpointIndex}: Invalid control checkpoint`, s3Object);
+        unrecoverableError = new Error('Invalid control checkpoint.');
+      }
+    } catch (err) {
+      return null;
+    }
+    if (unrecoverableError) throw unrecoverableError;
+    return { control: control!, controlLastMod: controlLastMod! };
+  }
 
-  if (modTimeDeltasS.length > 1) {
-    // Drop the time of the first one since it is probably way off
-    modTimeDeltasS.length--;
+  private async getProspectiveCheckpoint(
+    checkpointIndex: number,
+    controlFoundForIndex: boolean,
+  ): Promise<{ prospective: Checkpoint; prospectiveLastMod: Date } | null> {
+    let prospective: Checkpoint, prospectiveLastMod: Date;
+    try {
+      const s3Object = await this.prospectiveS3BucketClientWrapper.getS3Obj(
+        this.checkpointKey(checkpointIndex),
+      );
+      if (isCheckpoint(s3Object.obj)) {
+        [prospective, prospectiveLastMod] = [s3Object.obj, s3Object.modified];
+        this.lastNonMissingCheckpointIndex = checkpointIndex;
+      } else {
         console.log(
-      `Time deltas (∆ < 0 -> prospective came earlier than the control)`,
+          `${checkpointIndex}: Invalid prospective checkpoint`,
+          s3Object.obj,
         );
-    console.log(modTimeDeltasS);
-    console.log(`Median: ${median(modTimeDeltasS)}s`);
-    console.log(`Mean:   ${mean(modTimeDeltasS)}s`);
-    console.log(`Stdev:  ${stdDev(modTimeDeltasS)}s`);
+        this.invalidCheckpoints.push(checkpointIndex);
+        return null;
+      }
+      if (!controlFoundForIndex) {
+        this.extraCheckpoints.push(checkpointIndex);
+      }
+      this.missingInARow = 0;
+    } catch (err) {
+      if (controlFoundForIndex) {
+        this.missingCheckpoints.push(checkpointIndex);
+        this.missingInARow++;
+      }
+      return null;
     }
-}
 
-function median(a: number[]): number {
-  a = [...a]; // clone
-  a.sort((a, b) => a - b);
-  if (a.length <= 0) {
-    return 0;
-  } else if (a.length % 2 == 0) {
-    return (a[a.length / 2] + a[a.length / 2 - 1]) / 2;
-  } else {
-    return a[(a.length - 1) / 2];
+    return {
+      prospective: prospective!,
+      prospectiveLastMod: prospectiveLastMod!,
+    };
+  }
+
+  private checkpointKey(checkpointIndex: number): string {
+    return `checkpoint_${checkpointIndex}.json`;
   }
 }
 
-function mean(a: number[]): number {
-  return a.reduce((acc, i) => acc + i, 0) / a.length;
+////
+// Bootstrapper
+////
+function getArgs() {
+  return yargs(process.argv.slice(2))
+    .alias('a', 'address')
+    .describe('a', 'address of the validator to inspect')
+    .demandOption('a')
+    .string('a')
+    .alias('p', 'prospective')
+    .describe('p', 'S3 bucket of the prospective validator')
+    .demandOption('p')
+    .string('p')
+    .alias('c', 'control')
+    .describe('c', 'S3 bucket of the the known (control) validator')
+    .demandOption('c')
+    .string('c').argv;
 }
 
-function stdDev(a: number[]): number {
-  return Math.sqrt(
-    a.map((i) => i * i).reduce((acc, i) => acc + i, 0) / a.length,
+async function main() {
+  const {
+    a: validatorAddress,
+    p: prospectiveBucket,
+    c: controlBucket,
+  } = await getArgs();
+
+  const validator = new Validator(
+    validatorAddress,
+    prospectiveBucket,
+    controlBucket,
   );
-}
 
-function streamToString(stream: NodeJS.ReadableStream): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const chunks: string[] = [];
-    stream
-      .setEncoding('utf8')
-      .on('data', (chunk) => chunks.push(chunk))
-      .on('error', (err) => reject(err))
-      .on('end', () => resolve(String.prototype.concat(...chunks)));
-  });
+  try {
+    await validator.validate();
+  } catch (err) {
+    console.error(err);
+    process.exit(1);
+  }
 }
 
 main().catch(console.error);

typescript/infra/src/utils/utils.ts

@@ -205,3 +205,38 @@ export function assertContext(contextStr: string): Contexts {
     }`,
   );
 }
+
+export function median(a: number[]): number {
+  a = [...a]; // clone
+  a.sort((a, b) => a - b);
+  if (a.length <= 0) {
+    return 0;
+  } else if (a.length % 2 == 0) {
+    return (a[a.length / 2] + a[a.length / 2 - 1]) / 2;
+  } else {
+    return a[(a.length - 1) / 2];
+  }
+}
+
+export function mean(a: number[]): number {
+  return a.reduce((acc, i) => acc + i, 0) / a.length;
+}
+
+export function stdDev(a: number[]): number {
+  const xbar = mean(a);
+  return Math.sqrt(
+    a.map((x) => Math.pow(x - xbar, 2)).reduce((acc, i) => acc + i, 0) /
+      a.length,
+  );
+}
+
+export function streamToString(stream: NodeJS.ReadableStream): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const chunks: string[] = [];
+    stream
+      .setEncoding('utf8')
+      .on('data', (chunk) => chunks.push(chunk))
+      .on('error', (err) => reject(err))
+      .on('end', () => resolve(String.prototype.concat(...chunks)));
+  });
+}