Submit relayer funds metric (#714)

* Submit relayer funds metric

* update package

* fix num conversion

* add env and context to metric labels

* Add prometheus push gateway addr to config

typescript/infra/config/environments/mainnet/funding.ts

@@ -9,4 +9,5 @@ export const relayerFunderConfig: RelayerFunderConfig = {
   },
   cronSchedule: '*/10 * * * *', // Every 10 minutes
   namespace: environment,
+  prometheusPushGateway: 'http://pushgateway.monitoring.svc.cluster.local:9091',
 };

typescript/infra/config/environments/testnet2/funding.ts

@@ -9,4 +9,5 @@ export const relayerFunderConfig: RelayerFunderConfig = {
   },
   cronSchedule: '*/10 * * * *', // Every 10 minutes
   namespace: environment,
+  prometheusPushGateway: 'http://pushgateway.monitoring.svc.cluster.local:9091',
 };

typescript/infra/helm/relayer-funder/templates/cron-job.yaml

@@ -24,6 +24,9 @@ spec:
             - {{ .Values.abacus.runEnv }}
             - -f
             - /addresses-secret/addresses.json
+            env:
+            - name: PROMETHEUS_PUSH_GATEWAY
+              value: {{ .Values.infra.prometheusPushGateway }}
             envFrom:
             - secretRef:
                 name: relayer-funder-env-var-secret

typescript/infra/package.json

@@ -17,6 +17,7 @@
     "asn1.js": "5.4.1",
     "chai": "^4.3.4",
     "dotenv": "^10.0.0",
+    "prom-client": "^14.0.1",
     "yargs": "^17.4.1"
   },
   "devDependencies": {

typescript/infra/scripts/funding/fund-relayers-from-deployer.ts

@@ -1,7 +1,12 @@
 import { Console } from 'console';
 import { ethers } from 'ethers';
+import { Gauge, Pushgateway, Registry } from 'prom-client';
 
-import { ChainConnection, CompleteChainMap } from '@abacus-network/sdk';
+import {
+  ChainConnection,
+  ChainName,
+  CompleteChainMap,
+} from '@abacus-network/sdk';
 
 import { AgentKey, ReadOnlyAgentKey } from '../../src/agents/agent';
 import { getRelayerKeys } from '../../src/agents/key-utils';
@@ -9,6 +14,36 @@ import { KEY_ROLE_ENUM } from '../../src/agents/roles';
 import { readJSONAtPath } from '../../src/utils/utils';
 import { assertEnvironment, getArgs, getCoreEnvironmentConfig } from '../utils';
 
+const constMetricLabels = {
+  // this needs to get set in main because of async reasons
+  abacus_deployment: '',
+  abacus_context: 'abacus',
+};
+
+const metricsRegister = new Registry();
+const walletBalanceGauge = new Gauge({
+  // Mirror the rust/ethers-prometheus `wallet_balance` gauge metric.
+  name: 'abacus_wallet_balance',
+  help: 'Current balance of eth and other tokens in the `tokens` map for the wallet addresses in the `wallets` set',
+  registers: [metricsRegister],
+  labelNames: [
+    'chain',
+    'wallet_address',
+    'wallet_name',
+    'token_address',
+    'token_symbol',
+    'token_name',
+    ...(Object.keys(constMetricLabels) as (keyof typeof constMetricLabels)[]),
+  ],
+});
+metricsRegister.registerMetric(walletBalanceGauge);
+
+interface FunderBalance {
+  chain: ChainName;
+  address?: string;
+  balance: number;
+}
+
 // Min delta is 1/10 of the desired balance
 const MIN_DELTA_NUMERATOR = ethers.BigNumber.from(1);
 const MIN_DELTA_DENOMINATOR = ethers.BigNumber.from(10);
@@ -105,6 +140,7 @@ async function main() {
     .string('f').argv;
 
   const environment = assertEnvironment(argv.e as string);
+  constMetricLabels.abacus_deployment = environment;
   const config = getCoreEnvironmentConfig(environment);
   const multiProvider = await config.getMultiProvider();
 
@@ -113,16 +149,18 @@ async function main() {
     : getRelayerKeys(config.agent);
 
   const chains = relayerKeys.map((key) => key.chainName!);
+  const balances: FunderBalance[] = [];
 
   for (const chain of chains) {
     const chainConnection = multiProvider.getChainConnection(chain);
 
     const desiredBalance = desiredBalancePerChain[chain];
+    const funderAddress = await chainConnection.getAddress();
 
     console.group({
       chain,
       funder: {
-        address: await chainConnection.getAddress(),
+        address: funderAddress,
         balance: ethers.utils.formatEther(
           await chainConnection.signer!.getBalance(),
         ),
@@ -137,10 +175,19 @@ async function main() {
       await relayerKey.fetch();
       await fundRelayer(chainConnection, relayerKey, desiredBalance);
     }
+    balances.push({
+      chain,
+      address: funderAddress,
+      balance: parseFloat(
+        ethers.utils.formatEther(await chainConnection.signer!.getBalance()),
+      ),
+    });
 
     console.groupEnd();
     console.log('\n');
   }
+
+  await submitFunderBalanceMetrics(balances);
 }
 
 function getRelayerKeysFromSerializedAddressFile(path: string): AgentKey[] {
@@ -158,4 +205,49 @@ function getRelayerKeysFromSerializedAddressFile(path: string): AgentKey[] {
     .filter((key: AgentKey) => key.role === KEY_ROLE_ENUM.Relayer);
 }
 
+function getPushGateway(): Pushgateway | null {
+  const gatewayAddr = process.env['PROMETHEUS_PUSH_GATEWAY'];
+  if (gatewayAddr) {
+    return new Pushgateway(gatewayAddr, [], metricsRegister);
+  } else {
+    console.warn(
+      'Prometheus push gateway address was not defined; not publishing metrics.',
+    );
+    return null;
+  }
+}
+
+function submitFunderBalanceMetrics(balances: FunderBalance[]) {
+  const gateway = getPushGateway();
+  if (!gateway) return;
+
+  for (const { chain, address, balance } of balances) {
+    walletBalanceGauge
+      .labels({
+        chain,
+        wallet_address: address ?? 'unknown',
+        wallet_name: 'relayer-funder',
+        token_symbol: 'Native',
+        token_name: 'Native',
+        ...constMetricLabels,
+      })
+      .set(balance);
+  }
+
+  gateway
+    .push({ jobName: 'relayer_funder' })
+    .then(({ resp, body }) => {
+      const statusCode =
+        typeof resp == 'object' && resp != null && 'statusCode' in resp
+          ? (resp as any).statusCode
+          : 'unknown';
+      console.debug(
+        `Prometheus push resulted with status ${statusCode} and body ${body}`,
+      );
+    })
+    .catch((err) => {
+      console.error(`Error pushing metrics: ${err}`);
+    });
+}
+
 main().catch(console.error);

typescript/infra/src/config/funding.ts

@@ -4,4 +4,5 @@ export interface RelayerFunderConfig {
   docker: DockerConfig;
   cronSchedule: string;
   namespace: string;
+  prometheusPushGateway: string;
 }

typescript/infra/src/funding/deploy-relayer-funder.ts

@@ -35,6 +35,9 @@ function getRelayerFunderHelmValues<Chain extends ChainName>(
       repository: relayerFunderConfig.docker.repo,
       tag: relayerFunderConfig.docker.tag,
     },
+    infra: {
+      prometheusPushGateway: relayerFunderConfig.prometheusPushGateway,
+    },
   };
   return helmifyValues(values);
 }

yarn.lock

@@ -106,6 +106,7 @@ __metadata:
     ethers: ^5.6.8
     hardhat: ^2.8.4
     prettier: ^2.4.1
+    prom-client: ^14.0.1
     ts-node: ^10.8.0
     typescript: ^4.7.2
     yargs: ^17.4.1
@@ -5151,6 +5152,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"bintrees@npm:1.0.2":
+  version: 1.0.2
+  resolution: "bintrees@npm:1.0.2"
+  checksum: 56a52b7d3634e30002b1eda740d2517a22fa8e9e2eb088e919f37c030a0ed86e364ab59e472fc770fc8751308054bb1c892979d150e11d9e11ac33bcc1b5d16e
+  languageName: node
+  linkType: hard
+
 "bip39@npm:2.5.0":
   version: 2.5.0
   resolution: "bip39@npm:2.5.0"
@@ -12740,6 +12748,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"prom-client@npm:^14.0.1":
+  version: 14.0.1
+  resolution: "prom-client@npm:14.0.1"
+  dependencies:
+    tdigest: ^0.1.1
+  checksum: 864c19b7086eda8fae652385bc8b8aeb155f85922e58672d07a64918a603341e120e65e08f9d77ccab546518dc18930284da8743c2aac3c968f626d7063d6bba
+  languageName: node
+  linkType: hard
+
 "promise-inflight@npm:^1.0.1":
   version: 1.0.1
   resolution: "promise-inflight@npm:1.0.1"
@@ -14812,6 +14829,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tdigest@npm:^0.1.1":
+  version: 0.1.2
+  resolution: "tdigest@npm:0.1.2"
+  dependencies:
+    bintrees: 1.0.2
+  checksum: 44de8246752b6f8c2924685f969fd3d94c36949f22b0907e99bef2b2220726dd8467f4730ea96b06040b9aa2587c0866049640039d1b956952dfa962bc2075a3
+  languageName: node
+  linkType: hard
+
 "test-value@npm:^2.1.0":
   version: 2.1.0
   resolution: "test-value@npm:2.1.0"