From 832351d0d022c38ad32fcb05b68cdf95f5b1d7e6 Mon Sep 17 00:00:00 2001 From: Trevor Porter Date: Fri, 21 Jun 2024 13:10:42 +0100 Subject: [PATCH] chore: set agent and environment specific resource requests (#4027) ### Description - based off https://abacusworks.grafana.net/d/FSR9YWr7k, updated resource requests for all agents. Because mainnet3 sees so many more messages and more chains, the resource needs there are higher - allows for resources to be defined in infra - previously we'd just use the default value in the values.yaml of the helm charts. Also makes the resources specific to the type of agent, which is nice because relayers require much higher spec than validators - I haven't deployed this yet, will do so when I get a review ### Drive-by changes ### Related issues - Fixes https://github.com/hyperlane-xyz/issues/issues/1288 ### Backward compatibility ### Testing --- .../config/environments/mainnet3/agent.ts | 38 +++++++++++++--- .../config/environments/testnet4/agent.ts | 37 +++++++++++++--- typescript/infra/src/agents/index.ts | 12 +++++- typescript/infra/src/config/agent/agent.ts | 43 +++++++++++-------- typescript/infra/src/config/infrastructure.ts | 3 ++ 5 files changed, 103 insertions(+), 30 deletions(-) diff --git a/typescript/infra/config/environments/mainnet3/agent.ts b/typescript/infra/config/environments/mainnet3/agent.ts index 1d24baa48..8df43bdb6 100644 --- a/typescript/infra/config/environments/mainnet3/agent.ts +++ b/typescript/infra/config/environments/mainnet3/agent.ts @@ -219,6 +219,28 @@ const metricAppContexts = [ }, ]; +// Resource requests are based on observed usage found in https://abacusworks.grafana.net/d/FSR9YWr7k +const relayerResources = { + requests: { + cpu: '3000m', + memory: '8Gi', + }, +}; + +const validatorResources = { + requests: { + cpu: '250m', + memory: '256Mi', + }, +}; + +const scraperResources = { + requests: { + cpu: '100m', + memory: '4Gi', + }, +}; + const hyperlane: RootAgentConfig = { ...contextBase, context: Contexts.Hyperlane, @@ -232,21 +254,24 @@ const hyperlane: RootAgentConfig = { }, gasPaymentEnforcement: gasPaymentEnforcement, metricAppContexts, + resources: relayerResources, }, validators: { docker: { repo, - tag: '3bb9d0a-20240619-130157', + tag: '0d12ff3-20240620-173353', }, rpcConsensusType: RpcConsensusType.Quorum, chains: validatorChainConfig(Contexts.Hyperlane), + resources: validatorResources, }, scraper: { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: '59451d6-20240612-171611', + tag: '0d12ff3-20240620-173353', }, + resources: scraperResources, }, }; @@ -259,21 +284,23 @@ const releaseCandidate: RootAgentConfig = { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: '3bb9d0a-20240619-130157', + tag: '0d12ff3-20240620-173353', }, // We're temporarily (ab)using the RC relayer as a way to increase // message throughput. // whitelist: releaseCandidateHelloworldMatchingList, gasPaymentEnforcement, metricAppContexts, + resources: relayerResources, }, validators: { docker: { repo, - tag: '3bb9d0a-20240619-130157', + tag: '0d12ff3-20240620-173353', }, rpcConsensusType: RpcConsensusType.Quorum, chains: validatorChainConfig(Contexts.ReleaseCandidate), + resources: validatorResources, }, }; @@ -290,7 +317,7 @@ const neutron: RootAgentConfig = { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: 'c9c5d37-20240510-014327', + tag: '0d12ff3-20240620-173353', }, gasPaymentEnforcement: [ { @@ -317,6 +344,7 @@ const neutron: RootAgentConfig = { matchingList: routerMatchingList(arbitrumNeutronEclipAddresses), }, ], + resources: relayerResources, }, }; diff --git a/typescript/infra/config/environments/testnet4/agent.ts b/typescript/infra/config/environments/testnet4/agent.ts index c21678f4d..77a23f3b8 100644 --- a/typescript/infra/config/environments/testnet4/agent.ts +++ b/typescript/infra/config/environments/testnet4/agent.ts @@ -89,6 +89,28 @@ const gasPaymentEnforcement: GasPaymentEnforcement[] = [ }, ]; +// Resource requests are based on observed usage found in https://abacusworks.grafana.net/d/FSR9YWr7k +const relayerResources = { + requests: { + cpu: '1000m', + memory: '4Gi', + }, +}; + +const validatorResources = { + requests: { + cpu: '250m', + memory: '256Mi', + }, +}; + +const scraperResources = { + requests: { + cpu: '100m', + memory: '1Gi', + }, +}; + const hyperlane: RootAgentConfig = { ...contextBase, contextChainNames: hyperlaneContextAgentChainNames, @@ -98,7 +120,7 @@ const hyperlane: RootAgentConfig = { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: '3bb9d0a-20240619-130157', + tag: '0d12ff3-20240620-173353', }, blacklist: [ ...releaseCandidateHelloworldMatchingList, @@ -122,21 +144,24 @@ const hyperlane: RootAgentConfig = { matchingList: routerMatchingList(plumetestnetSepoliaAddresses), }, ], + resources: relayerResources, }, validators: { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: '3bb9d0a-20240619-130157', + tag: '0d12ff3-20240620-173353', }, chains: validatorChainConfig(Contexts.Hyperlane), + resources: validatorResources, }, scraper: { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: 'c9c5d37-20240510-014327', + tag: '0d12ff3-20240620-173353', }, + resources: scraperResources, }, }; @@ -149,19 +174,21 @@ const releaseCandidate: RootAgentConfig = { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: '3bb9d0a-20240619-130157', + tag: '0d12ff3-20240620-173353', }, whitelist: [...releaseCandidateHelloworldMatchingList], gasPaymentEnforcement, transactionGasLimit: 750000, + resources: relayerResources, }, validators: { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: '3bb9d0a-20240619-130157', + tag: '0d12ff3-20240620-173353', }, chains: validatorChainConfig(Contexts.ReleaseCandidate), + resources: validatorResources, }, }; diff --git a/typescript/infra/src/agents/index.ts b/typescript/infra/src/agents/index.ts index c959bca65..c1486e83a 100644 --- a/typescript/infra/src/agents/index.ts +++ b/typescript/infra/src/agents/index.ts @@ -10,6 +10,7 @@ import { AgentContextConfig, DockerConfig, HelmRootAgentValues, + KubernetesResources, RootAgentConfig, } from '../config/agent/agent.js'; import { RelayerConfigHelper } from '../config/agent/relayer.js'; @@ -151,7 +152,7 @@ export abstract class AgentHelmManager { return RpcConsensusType.Single; } - return this.config.rpcConsensusType; + return this.config.agentRoleConfig.rpcConsensusType; } async doesAgentReleaseExist() { @@ -169,7 +170,11 @@ export abstract class AgentHelmManager { } dockerImage(): DockerConfig { - return this.config.docker; + return this.config.agentRoleConfig.docker; + } + + kubernetesResources(): KubernetesResources | undefined { + return this.config.agentRoleConfig.resources; } } @@ -216,6 +221,7 @@ export class RelayerHelmManager extends OmniscientAgentHelmManager { enabled: true, aws: this.config.requiresAwsCredentials, config: await this.config.buildConfig(), + resources: this.kubernetesResources(), }; const signers = await this.config.signers(); @@ -244,6 +250,7 @@ export class ScraperHelmManager extends OmniscientAgentHelmManager { values.hyperlane.scraper = { enabled: true, config: await this.config.buildConfig(), + resources: this.kubernetesResources(), }; // scraper never requires aws credentials values.hyperlane.aws = false; @@ -279,6 +286,7 @@ export class ValidatorHelmManager extends MultichainAgentHelmManager { originChainName: cfg.originChainName, interval: cfg.interval, })), + resources: this.kubernetesResources(), }; // The name of the helm release for agents is `hyperlane-agent`. diff --git a/typescript/infra/src/config/agent/agent.ts b/typescript/infra/src/config/agent/agent.ts index 1ccd1d45b..f21bcb19f 100644 --- a/typescript/infra/src/config/agent/agent.ts +++ b/typescript/infra/src/config/agent/agent.ts @@ -89,8 +89,12 @@ export interface AgentContextConfig extends AgentEnvConfig { // incomplete common agent configuration for a role interface AgentRoleConfig { + // K8s-specific docker: DockerConfig; chainDockerOverrides?: Record>; + resources?: KubernetesResources; + + // Agent-specific rpcConsensusType: RpcConsensusType; index?: IndexingConfig; } @@ -119,6 +123,16 @@ export interface DockerConfig { tag: string; } +export interface KubernetesResources { + requests?: KubernetesComputeResources; + limits?: KubernetesComputeResources; +} + +export interface KubernetesComputeResources { + cpu: string; + memory: string; +} + export class RootAgentConfigHelper implements AgentContextConfig { readonly rawConfig: RootAgentConfig; @@ -154,21 +168,14 @@ export class RootAgentConfigHelper implements AgentContextConfig { } } -export abstract class AgentConfigHelper - extends RootAgentConfigHelper - implements AgentRoleConfig -{ - rpcConsensusType: RpcConsensusType; - docker: DockerConfig; - chainDockerOverrides?: Record>; - index?: IndexingConfig; - - protected constructor(root: RootAgentConfig, agent: AgentRoleConfig) { +export abstract class AgentConfigHelper< + R = unknown, +> extends RootAgentConfigHelper { + protected constructor( + root: RootAgentConfig, + readonly agentRoleConfig: AgentRoleConfig, + ) { super(root); - this.rpcConsensusType = agent.rpcConsensusType; - this.docker = agent.docker; - this.chainDockerOverrides = agent.chainDockerOverrides; - this.index = agent.index; } // role this config is for @@ -178,13 +185,13 @@ export abstract class AgentConfigHelper // If the provided chain has an override, return the override, otherwise return the default. dockerImageForChain(chainName: ChainName): DockerConfig { - if (this.chainDockerOverrides?.[chainName]) { + if (this.agentRoleConfig.chainDockerOverrides?.[chainName]) { return { - ...this.docker, - ...this.chainDockerOverrides[chainName], + ...this.agentRoleConfig.docker, + ...this.agentRoleConfig.chainDockerOverrides[chainName], }; } - return this.docker; + return this.agentRoleConfig.docker; } } diff --git a/typescript/infra/src/config/infrastructure.ts b/typescript/infra/src/config/infrastructure.ts index 23b898f43..9fe1d461f 100644 --- a/typescript/infra/src/config/infrastructure.ts +++ b/typescript/infra/src/config/infrastructure.ts @@ -1,3 +1,5 @@ +import { KubernetesResources } from './agent/agent.js'; + export interface HelmImageValues { repository: string; tag: string; @@ -6,6 +8,7 @@ export interface HelmImageValues { // This encompasses things like storage and resources for stateful sets. export interface HelmStatefulSetValues { enabled: boolean; + resources?: KubernetesResources; } interface KubernetesConfig {