Running locally fixes and docs (#410)

* Update readme with instructions on how to run locally * Better error message * Fix kathy task * Update rust configs * Update envs * Move to rust dir * Update rust readme
3 years ago · 37af9c15a0
parent d7014fd9d6
commit 37af9c15a0
12 changed files with 263 additions and 256 deletions
--- a/README.md
+++ b/README.md
@ -92,6 +92,9 @@ If you are using [VSCode](https://code.visualstudio.com/), you can launch the [m
 - install [`rustup`](https://rustup.rs)
 - see `rust/README.md`

+### Running Agents Locally
+See [this guide](rust/running-locally.md) for how to run the agents locally and perform a full end-to-end test.
+
 #### Building Agent Images

 There exists a docker build for the agent binaries. These docker images are used for deploying the agents in a production environment.
--- a/rust/README.md
+++ b/rust/README.md
@ -4,8 +4,6 @@

 - install `rustup`
  - [link here](https://rustup.rs/)
- setup pre-commit hooks: `cp ../pre-commit.sh ../.git/hooks/pre-commit`
-  - Note: To bypass pre-commit hooks, pass `--no-verify` after commit message

 Note: You should be running >= version `1.60.0` of the rustc compiler, you can see that version with this command and should see similar output:

@ -101,6 +99,7 @@ We use the tokio async runtime environment. Please see the docs
  - interfaces to the ethereum contracts
 - `agents`
  - each of the off-chain agents implemented thus far
+
 ### High-level guide to building an agent

 - `cargo new $AGENT_NAME`
@ -122,3 +121,5 @@ We use the tokio async runtime environment. Please see the docs
 - make a `config` folder and a toml file
  - Make sure to include your own settings from above

+### Running Locally
+See the guide [here](./running-locally.md) to run the agents locally.
--- a/rust/config/test/alfajores_config.json
+++ b/rust/config/test/alfajores_config.json
@ -1,62 +0,0 @@
-{
-  "environment": "test",
-  "signers": {},
-  "inboxes": {
-    "kovan": {
-      "domain": "3000",
-      "name": "kovan",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB",
-        "validatorManager": "0x67d269191c92Caf3cD7723F116c85e6E9bf55933"
-      }
-    },
-    "mumbai": {
-      "domain": "80001",
-      "name": "mumbai",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0xb7278A61aa25c888815aFC32Ad3cC52fF24fE575",
-        "validatorManager": "0x4c5859f0F772848b2D91F1D83E2Fe57935348029"
-      }
-    },
-    "fuji": {
-      "domain": "43113",
-      "name": "fuji",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0xD8a5a9b31c3C0232E196d518E89Fd8bF83AcAd43",
-        "validatorManager": "0x4C4a2f8c81640e47606d3fd77B353E87Ba015584"
-      }
-    }
-  },
-  "outbox": {
-    "addresses": {
-      "outbox": "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9"
-    },
-    "domain": "1000",
-    "name": "alfajores",
-    "rpcStyle": "ethereum",
-    "connection": {
-      "type": "http",
-      "url": ""
-    }
-  },
-  "tracing": {
-    "level": "debug",
-    "fmt": "json"
-  },
-  "db": "db_path"
-}
--- a/rust/config/test/fuji_config.json
+++ b/rust/config/test/fuji_config.json
@ -1,62 +0,0 @@
-{
-  "environment": "test",
-  "signers": {},
-  "inboxes": {
-    "alfajores": {
-      "domain": "1000",
-      "name": "alfajores",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0x68B1D87F95878fE05B998F19b66F4baba5De1aed",
-        "validatorManager": "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE"
-      }
-    },
-    "kovan": {
-      "domain": "3000",
-      "name": "kovan",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0x95401dc811bb5740090279Ba06cfA8fcF6113778",
-        "validatorManager": "0xf5059a5D33d5853360D16C683c16e67980206f36"
-      }
-    },
-    "mumbai": {
-      "domain": "80001",
-      "name": "mumbai",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0xc351628EB244ec633d5f21fBD6621e1a683B1181",
-        "validatorManager": "0x7bc06c482DEAd17c0e297aFbC32f6e63d3846650"
-      }
-    }
-  },
-  "outbox": {
-    "addresses": {
-      "outbox": "0x922D6956C99E12DFeB3224DEA977D0939758A1Fe"
-    },
-    "domain": "43113",
-    "name": "fuji",
-    "rpcStyle": "ethereum",
-    "connection": {
-      "type": "http",
-      "url": ""
-    }
-  },
-  "tracing": {
-    "level": "debug",
-    "fmt": "json"
-  },
-  "db": "db_path"
-}
--- a/rust/config/test/kovan_config.json
+++ b/rust/config/test/kovan_config.json
@ -1,62 +0,0 @@
-{
-  "environment": "test",
-  "signers": {},
-  "inboxes": {
-    "alfajores": {
-      "domain": "1000",
-      "name": "alfajores",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0xA51c1fc2f0D1a1b8494Ed1FE312d7C3a78Ed91C0",
-        "validatorManager": "0x8A791620dd6260079BF849Dc5567aDC3F2FdC318"
-      }
-    },
-    "mumbai": {
-      "domain": "80001",
-      "name": "mumbai",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0x2bdCC0de6bE1f7D2ee689a0342D76F52E8EFABa3",
-        "validatorManager": "0x82e01223d51Eb87e16A03E24687EDF0F294da6f1"
-      }
-    },
-    "fuji": {
-      "domain": "43113",
-      "name": "fuji",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0x36b58F5C1969B7b6591D752ea6F5486D069010AB",
-        "validatorManager": "0x51A1ceB83B83F1985a81C295d1fF28Afef186E02"
-      }
-    }
-  },
-  "outbox": {
-    "addresses": {
-      "outbox": "0xa85233C63b9Ee964Add6F2cffe00Fd84eb32338f"
-    },
-    "domain": "3000",
-    "name": "kovan",
-    "rpcStyle": "ethereum",
-    "connection": {
-      "type": "http",
-      "url": ""
-    }
-  },
-  "tracing": {
-    "level": "debug",
-    "fmt": "json"
-  },
-  "db": "db_path"
-}
--- a/rust/config/test/mumbai_config.json
+++ b/rust/config/test/mumbai_config.json
@ -1,62 +0,0 @@
-{
-  "environment": "test",
-  "signers": {},
-  "inboxes": {
-    "alfajores": {
-      "domain": "1000",
-      "name": "alfajores",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0x0B306BF915C4d645ff596e518fAf3F9669b97016",
-        "validatorManager": "0x9A676e781A523b5d0C0e43731313A708CB607508"
-      }
-    },
-    "kovan": {
-      "domain": "3000",
-      "name": "kovan",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0x1613beB3B2C4f22Ee086B2b38C1476A3cE7f78E8",
-        "validatorManager": "0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9"
-      }
-    },
-    "fuji": {
-      "domain": "43113",
-      "name": "fuji",
-      "rpcStyle": "ethereum",
-      "connection": {
-        "type": "http",
-        "url": ""
-      },
-      "addresses": {
-        "inbox": "0x202CCe504e04bEd6fC0521238dDf04Bc9E8E15aB",
-        "validatorManager": "0x0355B7B8cb128fA5692729Ab3AAa199C1753f726"
-      }
-    }
-  },
-  "outbox": {
-    "addresses": {
-      "outbox": "0x8f86403A4DE0BB5791fa46B8e795C547942fE4Cf"
-    },
-    "domain": "80001",
-    "name": "mumbai",
-    "rpcStyle": "ethereum",
-    "connection": {
-      "type": "http",
-      "url": ""
-    }
-  },
-  "tracing": {
-    "level": "debug",
-    "fmt": "json"
-  },
-  "db": "db_path"
-}
--- a/rust/config/test/test1_config.json
+++ b/rust/config/test/test1_config.json
@ -0,0 +1,49 @@
+{
+  "environment": "test",
+  "signers": {},
+  "inboxes": {
+    "test2": {
+      "domain": "13372",
+      "name": "test2",
+      "rpcStyle": "ethereum",
+      "connection": {
+        "type": "http",
+        "url": ""
+      },
+      "addresses": {
+        "inbox": "0xA51c1fc2f0D1a1b8494Ed1FE312d7C3a78Ed91C0",
+        "validatorManager": "0x8A791620dd6260079BF849Dc5567aDC3F2FdC318"
+      }
+    },
+    "test3": {
+      "domain": "13373",
+      "name": "test3",
+      "rpcStyle": "ethereum",
+      "connection": {
+        "type": "http",
+        "url": ""
+      },
+      "addresses": {
+        "inbox": "0x9A676e781A523b5d0C0e43731313A708CB607508",
+        "validatorManager": "0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82"
+      }
+    }
+  },
+  "outbox": {
+    "addresses": {
+      "outbox": "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9"
+    },
+    "domain": "13371",
+    "name": "test1",
+    "rpcStyle": "ethereum",
+    "connection": {
+      "type": "http",
+      "url": ""
+    }
+  },
+  "tracing": {
+    "level": "debug",
+    "fmt": "json"
+  },
+  "db": "db_path"
+}
--- a/rust/config/test/test2_config.json
+++ b/rust/config/test/test2_config.json
@ -0,0 +1,49 @@
+{
+  "environment": "test",
+  "signers": {},
+  "inboxes": {
+    "test1": {
+      "domain": "13371",
+      "name": "test1",
+      "rpcStyle": "ethereum",
+      "connection": {
+        "type": "http",
+        "url": ""
+      },
+      "addresses": {
+        "inbox": "0x67d269191c92Caf3cD7723F116c85e6E9bf55933",
+        "validatorManager": "0x7a2088a1bFc9d81c55368AE168C2C02570cB814F"
+      }
+    },
+    "test3": {
+      "domain": "13373",
+      "name": "test3",
+      "rpcStyle": "ethereum",
+      "connection": {
+        "type": "http",
+        "url": ""
+      },
+      "addresses": {
+        "inbox": "0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690",
+        "validatorManager": "0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E"
+      }
+    }
+  },
+  "outbox": {
+    "addresses": {
+      "outbox": "0x59b670e9fA9D0A427751Af201D676719a970857b"
+    },
+    "domain": "13372",
+    "name": "test2",
+    "rpcStyle": "ethereum",
+    "connection": {
+      "type": "http",
+      "url": ""
+    }
+  },
+  "tracing": {
+    "level": "debug",
+    "fmt": "json"
+  },
+  "db": "db_path"
+}
--- a/rust/config/test/test3_config.json
+++ b/rust/config/test/test3_config.json
@ -0,0 +1,49 @@
+{
+  "environment": "test",
+  "signers": {},
+  "inboxes": {
+    "test1": {
+      "domain": "13371",
+      "name": "test1",
+      "rpcStyle": "ethereum",
+      "connection": {
+        "type": "http",
+        "url": ""
+      },
+      "addresses": {
+        "inbox": "0x5eb3Bc0a489C5A8288765d2336659EbCA68FCd00",
+        "validatorManager": "0x0E801D84Fa97b50751Dbf25036d067dCf18858bF"
+      }
+    },
+    "test2": {
+      "domain": "13372",
+      "name": "test2",
+      "rpcStyle": "ethereum",
+      "connection": {
+        "type": "http",
+        "url": ""
+      },
+      "addresses": {
+        "inbox": "0x809d550fca64d94Bd9F66E60752A544199cfAC3D",
+        "validatorManager": "0x36C02dA8a0983159322a80FFE9F24b1acfF8B570"
+      }
+    }
+  },
+  "outbox": {
+    "addresses": {
+      "outbox": "0x95401dc811bb5740090279Ba06cfA8fcF6113778"
+    },
+    "domain": "13373",
+    "name": "test3",
+    "rpcStyle": "ethereum",
+    "connection": {
+      "type": "http",
+      "url": ""
+    }
+  },
+  "tracing": {
+    "level": "debug",
+    "fmt": "json"
+  },
+  "db": "db_path"
+}
--- a/rust/running-locally.md
+++ b/rust/running-locally.md
@ -0,0 +1,97 @@
+## Setup
+Example configuration files
+
+`rust/local.checkpointer.env`
+```shell
+OPT_BASE_OUTBOX_CONNECTION_URL=http://localhost:8545
+OPT_BASE_INBOXES_TEST2_CONNECTION_URL=http://localhost:8545
+OPT_BASE_INBOXES_TEST3_CONNECTION_URL=http://localhost:8545
+
+BASE_CONFIG=test1_config.json
+RUN_ENV=test
+OPT_BASE_METRICS=9090
+OPT_BASE_TRACING_FMT=pretty
+OPT_BASE_TRACING_LEVEL=info
+OPT_BASE_DB=/tmp/local-checkpointer-test1-db
+OPT_BASE_SIGNERS_TEST1_KEY=8166f546bab6da521a8369cab06c5d2b9e46670292d85c875ee9ec20e84ffb61
+OPT_BASE_SIGNERS_TEST1_TYPE=hexKey
+OPT_BASE_SIGNERS_TEST2_KEY=f214f2b2cd398c806f84e317254e0f0b801d0643303237d97a22a48e01628897
+OPT_BASE_SIGNERS_TEST2_TYPE=hexKey
+OPT_BASE_SIGNERS_TEST3_KEY=701b615bbdfb9de65240bc28bd21bbc0d996645a3dd57e7b12bc2bdf6f192c82
+OPT_BASE_SIGNERS_TEST3_TYPE=hexKey
+OPT_CHECKPOINTER_POLLINGINTERVAL=5
+OPT_CHECKPOINTER_CREATIONLATENCY=5
+
+```
+
+`rust/local.relayer.env`
+```shell
+OPT_BASE_OUTBOX_CONNECTION_URL=http://localhost:8545
+OPT_BASE_INBOXES_TEST2_CONNECTION_URL=http://localhost:8545
+OPT_BASE_INBOXES_TEST3_CONNECTION_URL=http://localhost:8545
+
+BASE_CONFIG=test1_config.json
+RUN_ENV=test
+OPT_BASE_METRICS=9092
+OPT_BASE_TRACING_FMT=pretty
+OPT_BASE_TRACING_LEVEL=info
+OPT_BASE_DB=/tmp/local-relayer-test1-db
+OPT_BASE_SIGNERS_TEST1_KEY=8166f546bab6da521a8369cab06c5d2b9e46670292d85c875ee9ec20e84ffb61
+OPT_BASE_SIGNERS_TEST1_TYPE=hexKey
+OPT_BASE_SIGNERS_TEST2_KEY=f214f2b2cd398c806f84e317254e0f0b801d0643303237d97a22a48e01628897
+OPT_BASE_SIGNERS_TEST2_TYPE=hexKey
+OPT_BASE_SIGNERS_TEST3_KEY=701b615bbdfb9de65240bc28bd21bbc0d996645a3dd57e7b12bc2bdf6f192c82
+OPT_BASE_SIGNERS_TEST3_TYPE=hexKey
+OPT_RELAYER_POLLINGINTERVAL=5
+OPT_RELAYER_SUBMISSIONLATENCY=5
+OPT_RELAYER_MAXRETRIES=5
+OPT_RELAYER_RELAYERMESSAGEPROCESSING=false
+OPT_RELAYER_MULTISIGCHECKPOINTSYNCER_THRESHOLD=1
+OPT_RELAYER_MULTISIGCHECKPOINTSYNCER_CHECKPOINTSYNCERS_0x70997970c51812dc3a010c7d01b50e0d17dc79c8_TYPE=localStorage
+OPT_RELAYER_MULTISIGCHECKPOINTSYNCER_CHECKPOINTSYNCERS_0x70997970c51812dc3a010c7d01b50e0d17dc79c8_PATH=/tmp/validatorsignatures
+
+```
+
+`rust/local.validator.env`
+```shell
+OPT_BASE_OUTBOX_CONNECTION_URL=http://127.0.0.1:8545
+OPT_BASE_INBOXES_TEST2_CONNECTION_URL=http://127.0.0.1:8545
+OPT_BASE_INBOXES_TEST3_CONNECTION_URL=http://127.0.0.1:8545
+
+BASE_CONFIG=test1_config.json
+RUN_ENV=test
+OPT_BASE_METRICS=9091
+OPT_BASE_TRACING_LEVEL=info
+OPT_BASE_DB=/tmp/local-validator-test1-db
+OPT_BASE_VALIDATOR_KEY=59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d
+OPT_BASE_VALIDATOR_TYPE=hexKey
+OPT_VALIDATOR_REORGPERIOD=0
+OPT_VALIDATOR_INTERVAL=5
+OPT_VALIDATOR_CHECKPOINTSYNCER_THRESHOLD=1
+OPT_VALIDATOR_CHECKPOINTSYNCER_TYPE=localStorage
+OPT_VALIDATOR_CHECKPOINTSYNCER_PATH=/tmp/validatorsignatures
+```
+
+## Running
+In `typescript/infra` to start the ethereum node with
+```shell
+yarn hardhat node
+```
+
+and then deploy (in a new shell) the abacus contracts
+```shell
+yarn abacus
+```
+
+In `rust` then start the agents (you will need one shell for each)
+```shell
+env $(xargs <local.checkpointer.env) cargo run --bin checkpointer
+env $(xargs <local.relayer.env) cargo run --bin relayer
+env $(xargs <local.validator.env) cargo run --bin validator
+```
+
+Then to test, start `kathy` which will generate messages.
+In `typescript/infra`
+```shell
+yarn kathy
+```
--- a/typescript/infra/hardhat.config.ts
+++ b/typescript/infra/hardhat.config.ts
@ -5,7 +5,11 @@ import { HardhatRuntimeEnvironment } from 'hardhat/types';

 import { BadRandomRecipient__factory } from '@abacus-network/core';
 import { utils as deployUtils } from '@abacus-network/deploy';
-import { AbacusCore, ChainName } from '@abacus-network/sdk';
+import {
+  AbacusCore,
+  ChainName,
+  ChainNameToDomainId,
+} from '@abacus-network/sdk';
 import { utils } from '@abacus-network/utils';

 import {
@ -101,7 +105,7 @@ task('kathy', 'Dispatches random abacus messages').setAction(
    );
    const core = AbacusCore.fromEnvironment(environment, multiProvider);

-    const randomElement = (list: any[]) =>
+    const randomElement = <T>(list: T[]) =>
      list[Math.floor(Math.random() * list.length)];

    // Deploy a recipient
@ -112,18 +116,21 @@ task('kathy', 'Dispatches random abacus messages').setAction(
    // Generate artificial traffic
    while (true) {
      const local = core.networks()[0];
-      const remote = randomElement(core.remotes(local));
+      const remote: ChainName = randomElement(core.remotes(local));
+      const remoteId = ChainNameToDomainId[remote];
      const coreContracts = core.getContracts(local);
      const outbox = coreContracts.outbox.outbox;
      // Send a batch of messages to the remote domain to test
      // the relayer submitting only greedily
      for (let i = 0; i < 10; i++) {
        await outbox.dispatch(
-          remote,
+          remoteId,
          utils.addressToBytes32(recipient.address),
          '0x1234',
        );
-        await outbox.checkpoint();
+        if ((await outbox.count()).gt(1)) {
+          await outbox.checkpoint();
+        }
        console.log(
          `send to ${recipient.address} on ${remote} at index ${
            (await outbox.count()).toNumber() - 1
--- a/typescript/infra/scripts/deploy-agents.ts
+++ b/typescript/infra/scripts/deploy-agents.ts
@ -12,7 +12,7 @@ async function deploy() {
  const currentKubeContext = await getCurrentKubernetesContext();
  if (!currentKubeContext.endsWith(`-${config.agent.runEnv}`)) {
    console.error(
-      `Attempting to deploy ${config.agent.runEnv} to ${currentKubeContext}`,
+      `Cowardly refusing to deploy ${config.agent.runEnv} to ${currentKubeContext}; are you sure you have the right k8s context active?`,
    );
    process.exit(1);
  }