Validator observability infra (#3116)

### Description

- Configures the relayers to start tracking the latest checkpoints of
the manta TIA WR, arbitrum TIA WR, and helloworld sets (and, implicitly,
the default ISM sets too)
- deployed all agent roles & networks with the new image
- created alerts and added to the nexus dashboard

One thing that's a bit of a bummer is the metrics are only updated
whenever we try to construct metadata for a message. Naturally the
validators will poll & sign checkpoints slightly out of sync, so pretty
frequently the relayer attempts to deliver a message where say 4/6
signatures are needed, and 2 of the validators happen to just have not
polled & signed the checkpoint yet. The message is then successfully
delivered, and the 2 remaining validators probably sign the checkpoint
in a matter of seconds, but the metrics aren't updated to reflect this
unless there's another message whose metadata is being constructed. This
just means that the graph is a bit ugly when things are working. But
when things aren't working, we'll still be able to clearly see which
validators are behind.

We may want to consider changing the metrics if this proves confusing.
Some ideas:
1. Move to a separate task that occasionally polls the latest
checkpoints of the configured app contexts
2. Also track the threshold. This way we can construct alerts &
dashboards based off the threshold and not be overly concerned if there
are some stragglers

### Drive-by changes

added `bytesToBytes32` so that we can construct matching lists, which
the agents expect to be 0x-prefixed, using a router address config that
may include protocol-specific address formats

### Related issues

Fixes #3109 

### Backward compatibility

yes

### Testing

deployed

.changeset/shaggy-seals-arrive.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/utils': patch
+---
+
+addressToBytes32 changed to work for all protocol types

typescript/infra/config/environments/mainnet3/agent.ts

@@ -6,12 +6,18 @@ import {
 } from '@hyperlane-xyz/sdk';
 
 import { RootAgentConfig, allAgentChainNames } from '../../../src/config';
-import { GasPaymentEnforcementConfig } from '../../../src/config/agent/relayer';
+import {
+  GasPaymentEnforcementConfig,
+  routerMatchingList,
+} from '../../../src/config/agent/relayer';
 import { ALL_KEY_ROLES, Role } from '../../../src/roles';
 import { Contexts } from '../../contexts';
 
 import { agentChainNames, environment } from './chains';
+import { helloWorld } from './helloworld';
 import { validatorChainConfig } from './validators';
+import arbitrumTIAAddresses from './warp/arbitrum-TIA-addresses.json';
+import mantaTIAAddresses from './warp/manta-TIA-addresses.json';
 
 // const releaseCandidateHelloworldMatchingList = routerMatchingList(
 //   helloWorld[Contexts.ReleaseCandidate].addresses,
@@ -43,14 +49,22 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '86b7f98-20231207-153805',
+      tag: '8ccfdb7-20240103-084118',
     },
     gasPaymentEnforcement,
+    metricAppContexts: [
+      {
+        name: 'helloworld',
+        matchingList: routerMatchingList(
+          helloWorld[Contexts.Hyperlane].addresses,
+        ),
+      },
+    ],
   },
   validators: {
     docker: {
       repo,
-      tag: '86b7f98-20231207-153805',
+      tag: '8ccfdb7-20240103-084118',
     },
     rpcConsensusType: RpcConsensusType.Quorum,
     chains: validatorChainConfig(Contexts.Hyperlane),
@@ -59,7 +73,7 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '86b7f98-20231207-153805',
+      tag: '8ccfdb7-20240103-084118',
     },
   },
 };
@@ -72,7 +86,7 @@ const releaseCandidate: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '86b7f98-20231207-153805',
+      tag: '8ccfdb7-20240103-084118',
     },
     // whitelist: releaseCandidateHelloworldMatchingList,
     gasPaymentEnforcement,
@@ -84,7 +98,7 @@ const releaseCandidate: RootAgentConfig = {
   validators: {
     docker: {
       repo,
-      tag: '86b7f98-20231207-153805',
+      tag: '8ccfdb7-20240103-084118',
     },
     rpcConsensusType: RpcConsensusType.Quorum,
     chains: validatorChainConfig(Contexts.ReleaseCandidate),
@@ -108,7 +122,7 @@ const neutron: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '67585a2-20231220-223937',
+      tag: '8ccfdb7-20240103-084118',
     },
     gasPaymentEnforcement: [
       {
@@ -130,6 +144,16 @@ const neutron: RootAgentConfig = {
       },
       ...gasPaymentEnforcement,
     ],
+    metricAppContexts: [
+      {
+        name: 'manta_tia',
+        matchingList: routerMatchingList(mantaTIAAddresses),
+      },
+      {
+        name: 'arbitrum_tia',
+        matchingList: routerMatchingList(arbitrumTIAAddresses),
+      },
+    ],
   },
 };
 

typescript/infra/config/environments/mainnet3/warp/arbitrum-TIA-addresses.json

@@ -0,0 +1,8 @@
+{
+  "neutron": {
+    "router": "neutron1jyyjd3x0jhgswgm6nnctxvzla8ypx50tew3ayxxwkrjfxhvje6kqzvzudq"
+  },
+  "arbitrum": {
+    "router": "0xd56734d7f9979dd94fae3d67c7e928234e71cd4c"
+  }
+}

typescript/infra/config/environments/mainnet3/warp/manta-TIA-addresses.json

@@ -0,0 +1,8 @@
+{
+  "neutron": {
+    "router": "neutron1ch7x3xgpnj62weyes8vfada35zff6z59kt2psqhnx9gjnt2ttqdqtva3pa"
+  },
+  "mantapacific": {
+    "router": "0x6fae4d9935e2fcb11fc79a64e917fb2bf14dafaa"
+  }
+}

typescript/infra/config/environments/testnet4/agent.ts

@@ -50,7 +50,7 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '86b7f98-20231207-153805',
+      tag: '8ccfdb7-20240103-084118',
     },
     blacklist: [
       ...releaseCandidateHelloworldMatchingList,
@@ -62,12 +62,20 @@ const hyperlane: RootAgentConfig = {
       },
     ],
     gasPaymentEnforcement,
+    metricAppContexts: [
+      {
+        name: 'helloworld',
+        matchingList: routerMatchingList(
+          helloWorld[Contexts.Hyperlane].addresses,
+        ),
+      },
+    ],
   },
   validators: {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '86b7f98-20231207-153805',
+      tag: '8ccfdb7-20240103-084118',
     },
     chains: validatorChainConfig(Contexts.Hyperlane),
   },
@@ -75,7 +83,7 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '86b7f98-20231207-153805',
+      tag: '8ccfdb7-20240103-084118',
     },
   },
 };
@@ -88,7 +96,7 @@ const releaseCandidate: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '86b7f98-20231207-153805',
+      tag: '8ccfdb7-20240103-084118',
     },
     whitelist: [...releaseCandidateHelloworldMatchingList],
     gasPaymentEnforcement,
@@ -101,7 +109,7 @@ const releaseCandidate: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '86b7f98-20231207-153805',
+      tag: '8ccfdb7-20240103-084118',
     },
     chains: validatorChainConfig(Contexts.ReleaseCandidate),
   },

typescript/infra/scripts/agents/utils.ts

@@ -16,8 +16,6 @@ import {
   withContext,
 } from '../utils';
 
-type GetConfigsArgv = NonNullable<Parameters<typeof getConfigsBasedOnArgs>[0]>;
-
 export class AgentCli {
   roles!: Role[];
   envConfig!: EnvironmentConfig;

typescript/infra/src/config/agent/relayer.ts

@@ -10,7 +10,7 @@ import {
   chainMetadata,
   getDomainId,
 } from '@hyperlane-xyz/sdk';
-import { ProtocolType } from '@hyperlane-xyz/utils';
+import { ProtocolType, addressToBytes32 } from '@hyperlane-xyz/utils';
 
 import { AgentAwsUser } from '../../agents/aws';
 import { Role } from '../../roles';
@@ -25,6 +25,11 @@ import {
 
 export { GasPaymentEnforcement as GasPaymentEnforcementConfig } from '@hyperlane-xyz/sdk';
 
+export interface MetricAppContext {
+  name: string;
+  matchingList: MatchingList;
+}
+
 // Incomplete basic relayer agent config
 export interface BaseRelayerConfig {
   gasPaymentEnforcement: GasPaymentEnforcement[];
@@ -32,6 +37,7 @@ export interface BaseRelayerConfig {
   blacklist?: MatchingList;
   transactionGasLimit?: BigNumberish;
   skipTransactionGasLimitFor?: string[];
+  metricAppContexts?: MetricAppContext[];
 }
 
 // Full relayer-specific agent config for a single chain
@@ -83,6 +89,11 @@ export class RelayerConfigHelper extends AgentConfigHelper<RelayerConfig> {
       relayerConfig.skipTransactionGasLimitFor =
         baseConfig.skipTransactionGasLimitFor.join(',');
     }
+    if (baseConfig.metricAppContexts) {
+      relayerConfig.metricAppContexts = JSON.stringify(
+        baseConfig.metricAppContexts,
+      );
+    }
 
     return relayerConfig;
   }
@@ -159,9 +170,9 @@ export function routerMatchingList(
 
       matchingList.push({
         originDomain: getDomainId(chainMetadata[source]),
-        senderAddress: routers[source].router,
+        senderAddress: addressToBytes32(routers[source].router),
         destinationDomain: getDomainId(chainMetadata[destination]),
-        recipientAddress: routers[destination].router,
+        recipientAddress: addressToBytes32(routers[destination].router),
       });
     }
   }

typescript/utils/src/addresses.ts

@@ -7,6 +7,8 @@ import { Address, HexString, ProtocolType } from './types';
 const EVM_ADDRESS_REGEX = /^0x[a-fA-F0-9]{40}$/;
 const SEALEVEL_ADDRESS_REGEX = /^[a-zA-Z0-9]{36,44}$/;
 
+const HEX_BYTES32_REGEX = /^0x[a-fA-F0-9]{64}$/;
+
 // https://github.com/cosmos/cosmos-sdk/blob/84c33215658131d87daf3c629e909e12ed9370fa/types/coin.go#L601C17-L601C44
 const COSMOS_DENOM_PATTERN = `[a-zA-Z][a-zA-Z0-9]{2,127}`;
 // https://en.bitcoin.it/wiki/BIP_0173
@@ -233,8 +235,7 @@ export function capitalizeAddress(address: Address) {
   else return address.toUpperCase();
 }
 
-// For EVM addresses only, kept for backwards compatibility and convenience
+export function addressToBytes32Evm(address: Address): string {
-export function addressToBytes32(address: Address): string {
   return ethersUtils
     .hexZeroPad(ethersUtils.hexStripZeros(address), 32)
     .toLowerCase();
@@ -246,7 +247,7 @@ export function bytes32ToAddress(bytes32: HexString): Address {
 }
 
 export function addressToBytesEvm(address: Address): Uint8Array {
-  const addrBytes32 = addressToBytes32(address);
+  const addrBytes32 = addressToBytes32Evm(address);
   return Buffer.from(strip0x(addrBytes32), 'hex');
 }
 
@@ -258,7 +259,10 @@ export function addressToBytesCosmos(address: Address): Uint8Array {
   return fromBech32(address).data;
 }
 
-export function addressToBytes(address: Address, protocol?: ProtocolType) {
+export function addressToBytes(
+  address: Address,
+  protocol?: ProtocolType,
+): Uint8Array {
   return routeAddressUtil(
     {
       [ProtocolType.Ethereum]: addressToBytesEvm,
@@ -280,6 +284,29 @@ export function addressToByteHexString(
   );
 }
 
+export function addressToBytes32(
+  address: Address,
+  protocol?: ProtocolType,
+): string {
+  // If the address is already bytes32, just return, avoiding a regression
+  // where an already bytes32 address cannot be categorized as a protocol address.
+  if (HEX_BYTES32_REGEX.test(ensure0x(address))) return ensure0x(address);
+
+  const bytes = addressToBytes(address, protocol);
+  return bytesToBytes32(bytes);
+}
+
+export function bytesToBytes32(bytes: Uint8Array): string {
+  if (bytes.length > 32) {
+    throw new Error('bytes must be 32 bytes or less');
+  }
+  // This 0x-prefixes the hex string
+  return ethersUtils.hexZeroPad(
+    ensure0x(Buffer.from(bytes).toString('hex')),
+    32,
+  );
+}
+
 export function bytesToAddressEvm(bytes: Uint8Array): Address {
   return bytes32ToAddress(Buffer.from(bytes).toString('hex'));
 }