chore: set agent and environment specific resource requests (#4027)

### Description

- based off https://abacusworks.grafana.net/d/FSR9YWr7k, updated
resource requests for all agents. Because mainnet3 sees so many more
messages and more chains, the resource needs there are higher
- allows for resources to be defined in infra - previously we'd just use
the default value in the values.yaml of the helm charts. Also makes the
resources specific to the type of agent, which is nice because relayers
require much higher spec than validators
- I haven't deployed this yet, will do so when I get a review

### Drive-by changes

<!--
Are there any minor or drive-by changes also included?
-->

### Related issues

- Fixes https://github.com/hyperlane-xyz/issues/issues/1288

### Backward compatibility

<!--
Are these changes backward compatible? Are there any infrastructure
implications, e.g. changes that would prohibit deploying older commits
using this infra tooling?

Yes/No
-->

### Testing

<!--
What kind of testing have these changes undergone?

None/Manual/Unit Tests
-->

typescript/infra/config/environments/mainnet3/agent.ts

@@ -219,6 +219,28 @@ const metricAppContexts = [
   },
 ];
 
+// Resource requests are based on observed usage found in https://abacusworks.grafana.net/d/FSR9YWr7k
+const relayerResources = {
+  requests: {
+    cpu: '3000m',
+    memory: '8Gi',
+  },
+};
+
+const validatorResources = {
+  requests: {
+    cpu: '250m',
+    memory: '256Mi',
+  },
+};
+
+const scraperResources = {
+  requests: {
+    cpu: '100m',
+    memory: '4Gi',
+  },
+};
+
 const hyperlane: RootAgentConfig = {
   ...contextBase,
   context: Contexts.Hyperlane,
@@ -232,21 +254,24 @@ const hyperlane: RootAgentConfig = {
     },
     gasPaymentEnforcement: gasPaymentEnforcement,
     metricAppContexts,
+    resources: relayerResources,
   },
   validators: {
     docker: {
       repo,
-      tag: '3bb9d0a-20240619-130157',
+      tag: '0d12ff3-20240620-173353',
     },
     rpcConsensusType: RpcConsensusType.Quorum,
     chains: validatorChainConfig(Contexts.Hyperlane),
+    resources: validatorResources,
   },
   scraper: {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '59451d6-20240612-171611',
+      tag: '0d12ff3-20240620-173353',
     },
+    resources: scraperResources,
   },
 };
 
@@ -259,21 +284,23 @@ const releaseCandidate: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '3bb9d0a-20240619-130157',
+      tag: '0d12ff3-20240620-173353',
     },
     // We're temporarily (ab)using the RC relayer as a way to increase
     // message throughput.
     // whitelist: releaseCandidateHelloworldMatchingList,
     gasPaymentEnforcement,
     metricAppContexts,
+    resources: relayerResources,
   },
   validators: {
     docker: {
       repo,
-      tag: '3bb9d0a-20240619-130157',
+      tag: '0d12ff3-20240620-173353',
     },
     rpcConsensusType: RpcConsensusType.Quorum,
     chains: validatorChainConfig(Contexts.ReleaseCandidate),
+    resources: validatorResources,
   },
 };
 
@@ -290,7 +317,7 @@ const neutron: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: 'c9c5d37-20240510-014327',
+      tag: '0d12ff3-20240620-173353',
     },
     gasPaymentEnforcement: [
       {
@@ -317,6 +344,7 @@ const neutron: RootAgentConfig = {
         matchingList: routerMatchingList(arbitrumNeutronEclipAddresses),
       },
     ],
+    resources: relayerResources,
   },
 };
 

typescript/infra/config/environments/testnet4/agent.ts

@@ -89,6 +89,28 @@ const gasPaymentEnforcement: GasPaymentEnforcement[] = [
   },
 ];
 
+// Resource requests are based on observed usage found in https://abacusworks.grafana.net/d/FSR9YWr7k
+const relayerResources = {
+  requests: {
+    cpu: '1000m',
+    memory: '4Gi',
+  },
+};
+
+const validatorResources = {
+  requests: {
+    cpu: '250m',
+    memory: '256Mi',
+  },
+};
+
+const scraperResources = {
+  requests: {
+    cpu: '100m',
+    memory: '1Gi',
+  },
+};
+
 const hyperlane: RootAgentConfig = {
   ...contextBase,
   contextChainNames: hyperlaneContextAgentChainNames,
@@ -98,7 +120,7 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '3bb9d0a-20240619-130157',
+      tag: '0d12ff3-20240620-173353',
     },
     blacklist: [
       ...releaseCandidateHelloworldMatchingList,
@@ -122,21 +144,24 @@ const hyperlane: RootAgentConfig = {
         matchingList: routerMatchingList(plumetestnetSepoliaAddresses),
       },
     ],
+    resources: relayerResources,
   },
   validators: {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '3bb9d0a-20240619-130157',
+      tag: '0d12ff3-20240620-173353',
     },
     chains: validatorChainConfig(Contexts.Hyperlane),
+    resources: validatorResources,
   },
   scraper: {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: 'c9c5d37-20240510-014327',
+      tag: '0d12ff3-20240620-173353',
     },
+    resources: scraperResources,
   },
 };
 
@@ -149,19 +174,21 @@ const releaseCandidate: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '3bb9d0a-20240619-130157',
+      tag: '0d12ff3-20240620-173353',
     },
     whitelist: [...releaseCandidateHelloworldMatchingList],
     gasPaymentEnforcement,
     transactionGasLimit: 750000,
+    resources: relayerResources,
   },
   validators: {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '3bb9d0a-20240619-130157',
+      tag: '0d12ff3-20240620-173353',
     },
     chains: validatorChainConfig(Contexts.ReleaseCandidate),
+    resources: validatorResources,
   },
 };
 

typescript/infra/src/agents/index.ts

@@ -10,6 +10,7 @@ import {
   AgentContextConfig,
   DockerConfig,
   HelmRootAgentValues,
+  KubernetesResources,
   RootAgentConfig,
 } from '../config/agent/agent.js';
 import { RelayerConfigHelper } from '../config/agent/relayer.js';
@@ -151,7 +152,7 @@ export abstract class AgentHelmManager {
       return RpcConsensusType.Single;
     }
 
-    return this.config.rpcConsensusType;
+    return this.config.agentRoleConfig.rpcConsensusType;
   }
 
   async doesAgentReleaseExist() {
@@ -169,7 +170,11 @@ export abstract class AgentHelmManager {
   }
 
   dockerImage(): DockerConfig {
-    return this.config.docker;
+    return this.config.agentRoleConfig.docker;
+  }
+
+  kubernetesResources(): KubernetesResources | undefined {
+    return this.config.agentRoleConfig.resources;
   }
 }
 
@@ -216,6 +221,7 @@ export class RelayerHelmManager extends OmniscientAgentHelmManager {
       enabled: true,
       aws: this.config.requiresAwsCredentials,
       config: await this.config.buildConfig(),
+      resources: this.kubernetesResources(),
     };
 
     const signers = await this.config.signers();
@@ -244,6 +250,7 @@ export class ScraperHelmManager extends OmniscientAgentHelmManager {
     values.hyperlane.scraper = {
       enabled: true,
       config: await this.config.buildConfig(),
+      resources: this.kubernetesResources(),
     };
     // scraper never requires aws credentials
     values.hyperlane.aws = false;
@@ -279,6 +286,7 @@ export class ValidatorHelmManager extends MultichainAgentHelmManager {
         originChainName: cfg.originChainName,
         interval: cfg.interval,
       })),
+      resources: this.kubernetesResources(),
     };
 
     // The name of the helm release for agents is `hyperlane-agent`.

typescript/infra/src/config/agent/agent.ts

@@ -89,8 +89,12 @@ export interface AgentContextConfig extends AgentEnvConfig {
 
 // incomplete common agent configuration for a role
 interface AgentRoleConfig {
+  // K8s-specific
   docker: DockerConfig;
   chainDockerOverrides?: Record<ChainName, Partial<DockerConfig>>;
+  resources?: KubernetesResources;
+
+  // Agent-specific
   rpcConsensusType: RpcConsensusType;
   index?: IndexingConfig;
 }
@@ -119,6 +123,16 @@ export interface DockerConfig {
   tag: string;
 }
 
+export interface KubernetesResources {
+  requests?: KubernetesComputeResources;
+  limits?: KubernetesComputeResources;
+}
+
+export interface KubernetesComputeResources {
+  cpu: string;
+  memory: string;
+}
+
 export class RootAgentConfigHelper implements AgentContextConfig {
   readonly rawConfig: RootAgentConfig;
 
@@ -154,21 +168,14 @@ export class RootAgentConfigHelper implements AgentContextConfig {
   }
 }
 
-export abstract class AgentConfigHelper<R = unknown>
-  extends RootAgentConfigHelper
-  implements AgentRoleConfig
-{
-  rpcConsensusType: RpcConsensusType;
-  docker: DockerConfig;
-  chainDockerOverrides?: Record<ChainName, Partial<DockerConfig>>;
-  index?: IndexingConfig;
-
-  protected constructor(root: RootAgentConfig, agent: AgentRoleConfig) {
+export abstract class AgentConfigHelper<
+  R = unknown,
+> extends RootAgentConfigHelper {
+  protected constructor(
+    root: RootAgentConfig,
+    readonly agentRoleConfig: AgentRoleConfig,
+  ) {
     super(root);
-    this.rpcConsensusType = agent.rpcConsensusType;
-    this.docker = agent.docker;
-    this.chainDockerOverrides = agent.chainDockerOverrides;
-    this.index = agent.index;
   }
 
   // role this config is for
@@ -178,13 +185,13 @@ export abstract class AgentConfigHelper<R = unknown>
 
   // If the provided chain has an override, return the override, otherwise return the default.
   dockerImageForChain(chainName: ChainName): DockerConfig {
-    if (this.chainDockerOverrides?.[chainName]) {
+    if (this.agentRoleConfig.chainDockerOverrides?.[chainName]) {
       return {
-        ...this.docker,
-        ...this.chainDockerOverrides[chainName],
+        ...this.agentRoleConfig.docker,
+        ...this.agentRoleConfig.chainDockerOverrides[chainName],
       };
     }
-    return this.docker;
+    return this.agentRoleConfig.docker;
   }
 }
 

typescript/infra/src/config/infrastructure.ts

@@ -1,3 +1,5 @@
+import { KubernetesResources } from './agent/agent.js';
+
 export interface HelmImageValues {
   repository: string;
   tag: string;
@@ -6,6 +8,7 @@ export interface HelmImageValues {
 // This encompasses things like storage and resources for stateful sets.
 export interface HelmStatefulSetValues {
   enabled: boolean;
+  resources?: KubernetesResources;
 }
 
 interface KubernetesConfig {