QuorumProvider testing in RC (#1018)

* Update Cargo.lock * wip * Cleanup * Deploy to all chains * Fix build, still only write http connectionType in config * Use QuorumProvider for the outbox too * Use newer img
2 years ago · 893d8efc46
parent 3e60d9aa29
commit 893d8efc46
9 changed files with 59 additions and 10 deletions
--- a/rust/helm/abacus-agent/templates/configmap.yaml
+++ b/rust/helm/abacus-agent/templates/configmap.yaml
@ -21,6 +21,9 @@ data:
  {{- if .disabled }}
  ABC_BASE_INBOXES_{{ .name | upper }}_DISABLED: "true"
  {{- end }}
+  {{- if .connectionType }}
+  ABC_BASE_INBOXES_{{ .name | upper }}_CONNECTION_TYPE: {{ .connectionType }}
+  {{- end }}
  {{- end }}
  {{- if .Values.abacus.tracing.uri }}
  ABC_BASE_TRACING_JAEGER_COLLECTOR_URI: {{ .Values.abacus.tracing.uri }}
--- a/rust/helm/abacus-agent/templates/external-secret.yaml
+++ b/rust/helm/abacus-agent/templates/external-secret.yaml
@ -21,7 +21,11 @@ spec:
          {{- include "abacus-agent.labels" . | nindent 10 }}
      data:
        {{/* RPC URLs */}}
-        ABC_BASE_OUTBOX_CONNECTION_URL: {{ print "'{{ .home_rpc | toString }}'" }}
+        {{- if eq .Values.abacus.outboxChain.connectionType "httpQuorum" }}
+        ABC_BASE_OUTBOX_CONNECTION_URLS: {{ print "'{{ .outbox_rpcs | fromJson | join \",\" }}'" }}
+        {{- else }}
+        ABC_BASE_OUTBOX_CONNECTION_URL: {{ print "'{{ .outbox_rpc | toString }}'" }}
+        {{- end }}
 {{/*
   * For each network, create an environment variable with the RPC endpoint.
   * The templating of external-secrets will use the data section below to know how
@ -29,21 +33,37 @@ spec:
   */}}
        {{- range .Values.abacus.inboxChains }}
        {{- if not .disabled }}
+        {{- if eq .connectionType "httpQuorum" }}
+        ABC_BASE_INBOXES_{{ .name | upper }}_CONNECTION_URLS: {{ printf "'{{ .%s_rpcs | fromJson | join \",\" }}'" .name }}
+        {{- else }}
        ABC_BASE_INBOXES_{{ .name | upper }}_CONNECTION_URL: {{ printf "'{{ .%s_rpc | toString }}'" .name }}
        {{- end }}
        {{- end }}
+        {{- end }}
  data:
-  - secretKey: home_rpc
+  {{- if eq .Values.abacus.outboxChain.connectionType "httpQuorum" }}
+  - secretKey: outbox_rpcs
+    remoteRef:
+      key: {{ printf "%s-rpc-endpoints-%s" .Values.abacus.runEnv .Values.abacus.outboxChain.name }}
+  {{- else }}
+  - secretKey: outbox_rpc
    remoteRef:
      key: {{ printf "%s-rpc-endpoint-%s" .Values.abacus.runEnv .Values.abacus.outboxChain.name }}
+  {{- end }}
 {{/*
   * For each network, load the secret in GCP secret manager with the form: environment-rpc-endpoint-network,
   * and associate it with the secret key networkname_rpc.
   */}}
  {{- range .Values.abacus.inboxChains }}
  {{- if not .disabled }}
+  {{- if eq .connectionType "httpQuorum" }}
+  - secretKey: {{ printf "%s_rpcs" .name }}
+    remoteRef:
+      key: {{ printf "%s-rpc-endpoints-%s" $.Values.abacus.runEnv .name }}
+  {{- else }}
  - secretKey: {{ printf "%s_rpc" .name }}
    remoteRef:
      key: {{ printf "%s-rpc-endpoint-%s" $.Values.abacus.runEnv .name }}
  {{- end }}
  {{- end }}
+  {{- end }}
--- a/rust/utils/run-locally/src/main.rs
+++ b/rust/utils/run-locally/src/main.rs
@ -141,8 +141,11 @@ fn main() -> ExitCode {

    let relayer_env = hashmap! {
        "ABC_BASE_OUTBOX_CONNECTION_URLS" => "http://127.0.0.1:8545,http://127.0.0.1:8545,http://127.0.0.1:8545",
+        "ABC_BASE_OUTBOX_CONNECTION_TYPE" => "httpQuorum",
        "ABC_BASE_INBOXES_TEST2_CONNECTION_URL" => "http://localhost:8545",
+        "ABC_BASE_INBOXES_TEST2_CONNECTION_URL" => "http",
        "ABC_BASE_INBOXES_TEST3_CONNECTION_URL" => "http://localhost:8545",
+        "ABC_BASE_INBOXES_TEST3_CONNECTION_URL" => "http",
        "BASE_CONFIG" => "test1_config.json",
        "RUN_ENV" => "test",
        "ABC_BASE_METRICS" => "9092",
@ -164,9 +167,12 @@ fn main() -> ExitCode {
    };

    let validator_env = hashmap! {
-        "ABC_BASE_OUTBOX_CONNECTION_URLS" => "http://localhost:8545,http://localhost:8545,http://localhost:8545",
+        "ABC_BASE_OUTBOX_CONNECTION_URLS" => "http://127.0.0.1:8545,http://127.0.0.1:8545,http://127.0.0.1:8545",
+        "ABC_BASE_OUTBOX_CONNECTION_TYPE" => "httpQuorum",
        "ABC_BASE_INBOXES_TEST2_CONNECTION_URL" => "http://localhost:8545",
+        "ABC_BASE_INBOXES_TEST2_CONNECTION_URL" => "http",
        "ABC_BASE_INBOXES_TEST3_CONNECTION_URL" => "http://localhost:8545",
+        "ABC_BASE_INBOXES_TEST3_CONNECTION_URL" => "http",
        "BASE_CONFIG" => "test1_config.json",
        "RUN_ENV" => "test",
        "ABC_BASE_METRICS" => "9091",
--- a/typescript/infra/config/environments/mainnet/agent.ts
+++ b/typescript/infra/config/environments/mainnet/agent.ts
@ -1,5 +1,6 @@
 import { ALL_KEY_ROLES, KEY_ROLE_ENUM } from '../../../src/agents/roles';
 import { AgentConfig } from '../../../src/config';
+import { ConnectionType } from '../../../src/config/agent';
 import { Contexts } from '../../contexts';
 import { helloworldMatchingList } from '../../utils';

@ -27,6 +28,7 @@ export const abacus: AgentConfig<MainnetChains> = {
  environmentChainNames: chainNames,
  contextChainNames: chainNames,
  validatorSets: validators,
+  connectionType: ConnectionType.Http,
  validator: {
    default: {
      interval: 5,
@ -81,6 +83,7 @@ export const releaseCandidate: AgentConfig<MainnetChains> = {
  environmentChainNames: chainNames,
  contextChainNames: chainNames,
  validatorSets: validators,
+  connectionType: ConnectionType.Http,
  relayer: {
    default: {
      signedCheckpointPollingInterval: 5,
--- a/typescript/infra/config/environments/test/agent.ts
+++ b/typescript/infra/config/environments/test/agent.ts
@ -1,5 +1,6 @@
 import { ALL_KEY_ROLES } from '../../../src/agents/roles';
 import { AgentConfig } from '../../../src/config';
+import { ConnectionType } from '../../../src/config/agent';
 import { Contexts } from '../../contexts';

 import { TestChains, chainNames } from './chains';
@ -17,6 +18,7 @@ export const abacus: AgentConfig<TestChains> = {
  environmentChainNames: chainNames,
  contextChainNames: chainNames,
  validatorSets: validators,
+  connectionType: ConnectionType.Http,
  validator: {
    default: {
      interval: 5,
--- a/typescript/infra/config/environments/testnet2/agent.ts
+++ b/typescript/infra/config/environments/testnet2/agent.ts
@ -1,5 +1,6 @@
 import { ALL_KEY_ROLES, KEY_ROLE_ENUM } from '../../../src/agents/roles';
 import { AgentConfig } from '../../../src/config';
+import { ConnectionType } from '../../../src/config/agent';
 import { Contexts } from '../../contexts';
 import {
  MATCHING_LIST_ALL_WILDCARDS,
@ -30,6 +31,7 @@ export const abacus: AgentConfig<TestnetChains> = {
  environmentChainNames: chainNames,
  contextChainNames: chainNames,
  validatorSets: validators,
+  connectionType: ConnectionType.Http,
  validator: {
    default: {
      interval: 5,
@ -84,6 +86,7 @@ export const flowcarbon: AgentConfig<TestnetChains> = {
  environmentChainNames: chainNames,
  contextChainNames: ['alfajores', 'kovan'],
  validatorSets: validators,
+  connectionType: ConnectionType.Http,
  relayer: {
    default: {
      signedCheckpointPollingInterval: 5,
@ -102,7 +105,7 @@ export const releaseCandidate: AgentConfig<TestnetChains> = {
  context: Contexts.ReleaseCandidate,
  docker: {
    repo: 'gcr.io/abacus-labs-dev/abacus-agent',
-    tag: 'sha-33b82dc',
+    tag: 'sha-3e60d9a',
  },
  aws: {
    region: 'us-east-1',
@ -110,6 +113,7 @@ export const releaseCandidate: AgentConfig<TestnetChains> = {
  environmentChainNames: chainNames,
  contextChainNames: chainNames,
  validatorSets: validators,
+  connectionType: ConnectionType.HttpQuorum,
  relayer: {
    default: {
      signedCheckpointPollingInterval: 5,
--- a/typescript/infra/src/agents/index.ts
+++ b/typescript/infra/src/agents/index.ts
@ -32,6 +32,7 @@ async function helmValuesForChain<Chain extends ChainName>(
      baseConfig: `${chainName}_config.json`,
      outboxChain: {
        name: chainName,
+        connectionType: agentConfig.connectionType,
      },
      aws: !!agentConfig.aws,
      inboxChains: agentConfig.environmentChainNames
@ -40,6 +41,7 @@ async function helmValuesForChain<Chain extends ChainName>(
          return {
            name: remoteChainName,
            disabled: !agentConfig.contextChainNames.includes(remoteChainName),
+            connectionType: agentConfig.connectionType,
          };
        }),
      validator: {
--- a/typescript/infra/src/config/agent.ts
+++ b/typescript/infra/src/config/agent.ts
@ -184,6 +184,8 @@ export interface AgentConfig<Chain extends ChainName> {
  runEnv: string;
  context: Contexts;
  docker: DockerConfig;
+  quorumProvider?: boolean;
+  connectionType: ConnectionType;
  index?: IndexingConfig;
  aws?: AwsConfig;
  // Names of all chains in the environment
@ -202,13 +204,19 @@ export type RustSigner = {
  type: string; // TODO
 };

+export enum ConnectionType {
+  Http = 'http',
+  Ws = 'ws',
+  HttpQuorum = 'httpQuorum',
+}
+
 export type RustConnection =
  | {
-      type: 'http';
+      type: ConnectionType.Http;
      url: string;
    }
-  | { type: 'ws'; url: string }
-  | { type: 'httpQuorum'; urls: string };
+  | { type: ConnectionType.Ws; url: string }
+  | { type: ConnectionType.HttpQuorum; urls: string };

 export type RustContractBlock<T> = {
  addresses: T;
--- a/typescript/infra/src/core/deploy.ts
+++ b/typescript/infra/src/core/deploy.ts
@ -6,6 +6,7 @@ import {
 } from '@abacus-network/sdk';

 import { DeployEnvironment, RustConfig } from '../config';
+import { ConnectionType } from '../config/agent';
 import { writeJSON } from '../utils/utils';

 export class AbacusCoreInfraDeployer<
@ -35,8 +36,8 @@ export class AbacusCoreInfraDeployer<
          rpcStyle: 'ethereum',
          finalityBlocks: outboxMetadata.finalityBlocks.toString(),
          connection: {
-            type: 'httpQuorum',
-            urls: '',
+            type: ConnectionType.Http,
+            url: '',
          },
        },
        tracing: {
@ -66,7 +67,7 @@ export class AbacusCoreInfraDeployer<
          rpcStyle: 'ethereum',
          finalityBlocks: metadata.finalityBlocks.toString(),
          connection: {
-            type: 'http',
+            type: ConnectionType.Http,
            url: '',
          },
          addresses: {