Scraper infrastructure (#1145)
* gitignore * wip * more wip * Mostly works * Update scraper config values * Prevent deploying to wrong context * throw error instead of exit * remove dry-run from scraper deploy * must be non-zero volume size * Add image tag for scraper deploy * Fix env config for scraper * minor cleanup * Minor cleaning * No volume Co-authored-by: Mattie Conover <git@mconover.dev>deploy-create2-factory-moonbeam
Trevor Porter
2 years ago
committed by
GitHub
parent
1fc65f3b7f
commit
8f9f7add72
@ -0,0 +1,6 @@ |
||||
dependencies: |
||||
- name: agent-common |
||||
repository: file://../agent-common |
||||
version: 0.1.0 |
||||
digest: sha256:9077bf1e5fe081300f9d30a01fa5b196a61ec4f2f802b7b8f67dffc84d425dc0 |
||||
generated: "2022-10-07T14:58:15.423221-04:00" |
||||
@ -1,101 +0,0 @@ |
||||
{{/* |
||||
Expand the name of the chart. |
||||
*/}} |
||||
{{- define "abacus-agent.name" -}} |
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Create a default fully qualified app name. |
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). |
||||
If release name contains chart name it will be used as a full name. |
||||
*/}} |
||||
{{- define "abacus-agent.fullname" -}} |
||||
{{- if .Values.fullnameOverride }} |
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} |
||||
{{- else }} |
||||
{{- $name := default .Chart.Name .Values.nameOverride }} |
||||
{{- if contains $name .Release.Name }} |
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }} |
||||
{{- else }} |
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} |
||||
{{- end }} |
||||
{{- end }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Create chart name and version as used by the chart label. |
||||
*/}} |
||||
{{- define "abacus-agent.chart" -}} |
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Common labels |
||||
*/}} |
||||
{{- define "abacus-agent.labels" -}} |
||||
helm.sh/chart: {{ include "abacus-agent.chart" . }} |
||||
abacus/deployment: {{ .Values.abacus.runEnv | quote }} |
||||
abacus/context: {{ .Values.abacus.context | quote }} |
||||
{{ include "abacus-agent.selectorLabels" . }} |
||||
{{- if .Chart.AppVersion }} |
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} |
||||
{{- end }} |
||||
app.kubernetes.io/managed-by: {{ .Release.Service }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Selector labels |
||||
*/}} |
||||
{{- define "abacus-agent.selectorLabels" -}} |
||||
app.kubernetes.io/name: {{ include "abacus-agent.name" . }} |
||||
app.kubernetes.io/instance: {{ .Release.Name }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Create the name of the service account to use |
||||
*/}} |
||||
{{- define "abacus-agent.serviceAccountName" -}} |
||||
{{- if .Values.serviceAccount.create }} |
||||
{{- default (include "abacus-agent.fullname" .) .Values.serviceAccount.name }} |
||||
{{- else }} |
||||
{{- default "default" .Values.serviceAccount.name }} |
||||
{{- end }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
The name of the ClusterSecretStore |
||||
*/}} |
||||
{{- define "abacus-agent.cluster-secret-store.name" -}} |
||||
{{- default "external-secrets-gcp-cluster-secret-store" .Values.externalSecrets.clusterSecretStore }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Recursively converts a config object into environment variables than can |
||||
be parsed by rust. For example, a config of { foo: { bar: { baz: 420 }, boo: 421 } } will |
||||
be: HYP_FOO_BAR_BAZ=420 and HYP_FOO_BOO=421 |
||||
Env vars can be formatted in FOO="BAR" format if .format is "dot_env", |
||||
FOO: "BAR" format if .format is "config_map", or otherwise |
||||
they will be formatted as spec YAML-friendly environment variables |
||||
*/}} |
||||
{{- define "abacus-agent.config-env-vars" -}} |
||||
{{- range $key, $value := .config }} |
||||
{{- $key_name := printf "%s%s" (default "" $.key_name_prefix) $key }} |
||||
{{- if typeIs "map[string]interface {}" $value }} |
||||
{{- include "abacus-agent.config-env-vars" (dict "config" $value "agent_name" $.agent_name "format" $.format "key_name_prefix" (printf "%s_" $key_name)) }} |
||||
{{- else }} |
||||
{{- include "abacus-agent.config-env-var" (dict "agent_name" $.agent_name "key" $key_name "value" $value "format" $.format ) }} |
||||
{{- end }} |
||||
{{- end }} |
||||
{{- end }} |
||||
|
||||
{{- define "abacus-agent.config-env-var" }} |
||||
{{- if (eq .format "dot_env") }} |
||||
HYP_{{ .agent_name | upper }}_{{ .key | upper }}={{ .value | quote }} |
||||
{{- else if (eq .format "config_map") }} |
||||
HYP_{{ .agent_name | upper }}_{{ .key | upper }}: {{ .value | quote }} |
||||
{{- else }} |
||||
- name: HYP_{{ .agent_name | upper }}_{{ .key | upper }} |
||||
value: {{ .value | quote }} |
||||
{{- end }} |
||||
{{- end }} |
||||
@ -0,0 +1,6 @@ |
||||
apiVersion: v2 |
||||
name: agent-common |
||||
description: Common helper templates for agents |
||||
type: library |
||||
version: 0.1.0 |
||||
appVersion: "0.1.0" |
||||
@ -0,0 +1,103 @@ |
||||
{{/* |
||||
Expand the name of the chart. |
||||
*/}} |
||||
{{- define "agent-common.name" -}} |
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Create a default fully qualified app name. |
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). |
||||
If release name contains chart name it will be used as a full name. |
||||
*/}} |
||||
{{- define "agent-common.fullname" -}} |
||||
{{- if .Values.fullnameOverride }} |
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} |
||||
{{- else }} |
||||
{{- $name := default .Chart.Name .Values.nameOverride }} |
||||
{{- if contains $name .Release.Name }} |
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }} |
||||
{{- else }} |
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} |
||||
{{- end }} |
||||
{{- end }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Create chart name and version as used by the chart label. |
||||
*/}} |
||||
{{- define "agent-common.chart" -}} |
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Common labels |
||||
*/}} |
||||
{{- define "agent-common.labels" -}} |
||||
helm.sh/chart: {{ include "agent-common.chart" . }} |
||||
app.kubernetes.io/component: agent-common |
||||
abacus/deployment: {{ .Values.abacus.runEnv | quote }} |
||||
abacus/context: {{ .Values.abacus.context | quote }} |
||||
{{ include "agent-common.selectorLabels" . }} |
||||
{{- if .Chart.AppVersion }} |
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} |
||||
{{- end }} |
||||
app.kubernetes.io/managed-by: {{ .Release.Service }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Selector labels |
||||
*/}} |
||||
{{- define "agent-common.selectorLabels" -}} |
||||
app.kubernetes.io/name: {{ include "agent-common.name" . }} |
||||
app.kubernetes.io/instance: {{ .Release.Name }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Create the name of the service account to use |
||||
*/}} |
||||
{{- define "agent-common.serviceAccountName" -}} |
||||
{{- if .Values.serviceAccount.create }} |
||||
{{- default (include "agent-common.fullname" .) .Values.serviceAccount.name }} |
||||
{{- else }} |
||||
{{- default "default" .Values.serviceAccount.name }} |
||||
{{- end }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
The name of the ClusterSecretStore |
||||
*/}} |
||||
{{- define "agent-common.cluster-secret-store.name" -}} |
||||
{{- default "external-secrets-gcp-cluster-secret-store" .Values.externalSecrets.clusterSecretStore }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Recursively converts a config object into environment variables than can |
||||
be parsed by rust. For example, a config of { foo: { bar: { baz: 420 }, boo: 421 } } will |
||||
be: HYP_FOO_BAR_BAZ=420 and HYP_FOO_BOO=421 |
||||
Env vars can be formatted in FOO="BAR" format if .format is "dot_env", |
||||
FOO: "BAR" format if .format is "config_map", or otherwise |
||||
they will be formatted as spec YAML-friendly environment variables |
||||
*/}} |
||||
{{- define "agent-common.config-env-vars" -}} |
||||
{{- range $key, $value := .config }} |
||||
{{- $key_name := printf "%s%s" (default "" $.key_name_prefix) $key }} |
||||
{{- if typeIs "map[string]interface {}" $value }} |
||||
{{- include "agent-common.config-env-vars" (dict "config" $value "agent_name" $.agent_name "format" $.format "key_name_prefix" (printf "%s_" $key_name)) }} |
||||
{{- else }} |
||||
{{- include "agent-common.config-env-var" (dict "agent_name" $.agent_name "key" $key_name "value" $value "format" $.format ) }} |
||||
{{- end }} |
||||
{{- end }} |
||||
{{- end }} |
||||
|
||||
{{- define "agent-common.config-env-var" }} |
||||
{{- if (eq .format "dot_env") }} |
||||
HYP_{{ .agent_name | upper }}_{{ .key | upper }}={{ .value | quote }} |
||||
{{- else if (eq .format "config_map") }} |
||||
HYP_{{ .agent_name | upper }}_{{ .key | upper }}: {{ .value | quote }} |
||||
{{- else }} |
||||
- name: HYP_{{ .agent_name | upper }}_{{ .key | upper }} |
||||
value: {{ .value | quote }} |
||||
{{- end }} |
||||
{{- end }} |
||||
|
||||
@ -0,0 +1,6 @@ |
||||
dependencies: |
||||
- name: agent-common |
||||
repository: file://../agent-common |
||||
version: 0.1.0 |
||||
digest: sha256:9077bf1e5fe081300f9d30a01fa5b196a61ec4f2f802b7b8f67dffc84d425dc0 |
||||
generated: "2022-10-07T15:37:35.347695-04:00" |
||||
@ -0,0 +1,10 @@ |
||||
apiVersion: v2 |
||||
name: scraper |
||||
description: A Helm Chart that encapsulates the deployment of the Abacus Rust Agent(s) |
||||
type: application |
||||
version: 0.1.0 |
||||
appVersion: "0.1.0" |
||||
dependencies: |
||||
- name: agent-common |
||||
version: 0.1.0 |
||||
repository: file://../agent-common |
||||
@ -0,0 +1,45 @@ |
||||
{{/* |
||||
Expand the name of the chart. |
||||
*/}} |
||||
{{- define "scraper.name" -}} |
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Create a default fully qualified app name. |
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). |
||||
If release name contains chart name it will be used as a full name. |
||||
*/}} |
||||
{{- define "scraper.fullname" -}} |
||||
{{- if .Values.fullnameOverride }} |
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} |
||||
{{- else }} |
||||
{{- $name := default .Chart.Name .Values.nameOverride }} |
||||
{{- if contains $name .Release.Name }} |
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }} |
||||
{{- else }} |
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} |
||||
{{- end }} |
||||
{{- end }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Create chart name and version as used by the chart label. |
||||
*/}} |
||||
{{- define "scraper.chart" -}} |
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} |
||||
{{- end }} |
||||
|
||||
{{/* |
||||
Common labels |
||||
*/}} |
||||
{{- define "scraper.labels" -}} |
||||
helm.sh/chart: {{ include "scraper.chart" . }} |
||||
abacus/deployment: {{ .Values.abacus.runEnv | quote }} |
||||
abacus/context: {{ .Values.abacus.context | quote }} |
||||
{{ include "scraper.selectorLabels" . }} |
||||
{{- if .Chart.AppVersion }} |
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} |
||||
{{- end }} |
||||
app.kubernetes.io/managed-by: {{ .Release.Service }} |
||||
{{- end }} |
||||
@ -0,0 +1,36 @@ |
||||
apiVersion: v1 |
||||
kind: ConfigMap |
||||
metadata: |
||||
name: {{ include "agent-common.fullname" . }} |
||||
labels: |
||||
{{- include "agent-common.labels" . | nindent 4 }} |
||||
data: |
||||
RUN_ENV: {{ .Values.abacus.runEnv | quote }} |
||||
BASE_CONFIG: {{ .Values.abacus.baseConfig }} |
||||
RUST_BACKTRACE: {{ .Values.abacus.rustBacktrace }} |
||||
HYP_BASE_DB: {{ .Values.abacus.dbPath }} |
||||
HYP_BASE_TRACING_FMT: {{ .Values.abacus.tracing.format }} |
||||
HYP_BASE_TRACING_LEVEL: {{ .Values.abacus.tracing.level }} |
||||
{{- range .Values.abacus.inboxChains }} |
||||
{{- include "agent-common.config-env-vars" (dict "config" . "agent_name" "base" "key_name_prefix" (printf "INBOXES_%s_" (.name | upper)) "format" "config_map") | indent 2 }} |
||||
{{- end }} |
||||
|
||||
{{- range .Values.abacus.outboxChains }} |
||||
{{- if not .disabled }} |
||||
HYP_SCRAPER_OUTBOXES_{{ .name | upper }}_DISABLED: "false" |
||||
HYP_SCRAPER_OUTBOXES_{{ .name | upper }}_CONNECTION_TYPE: {{ .connection.type }} |
||||
{{- if eq .connection.type "httpQuorum" }} |
||||
HYP_SCRAPER_OUTBOXES_{{ .name | upper }}_CONNECTION_URLS: "" |
||||
{{- else }} |
||||
HYP_SCRAPER_OUTBOXES_{{ .name | upper }}_CONNECTION_URL: "" |
||||
{{- end }} |
||||
HYP_SCRAPER_OUTBOXES_{{ .name | upper }}_NAME: {{ .name | lower }} |
||||
{{- else}} |
||||
HYP_SCRAPER_OUTBOXES_{{ .name | upper }}_DISABLED: "true" |
||||
{{- end }} |
||||
{{- end }} |
||||
|
||||
{{- if .Values.abacus.tracing.uri }} |
||||
HYP_BASE_TRACING_JAEGER_COLLECTOR_URI: {{ .Values.abacus.tracing.uri }} |
||||
{{- end }} |
||||
HYP_BASE_METRICS: {{ .Values.abacus.metrics.port | quote }} |
||||
@ -0,0 +1,74 @@ |
||||
apiVersion: external-secrets.io/v1beta1 |
||||
kind: ExternalSecret |
||||
metadata: |
||||
name: {{ include "agent-common.fullname" . }}-external-secret |
||||
labels: |
||||
{{- include "agent-common.labels" . | nindent 4 }} |
||||
annotations: |
||||
update-on-redeploy: "{{ now }}" |
||||
spec: |
||||
secretStoreRef: |
||||
name: {{ include "agent-common.cluster-secret-store.name" . }} |
||||
kind: ClusterSecretStore |
||||
refreshInterval: "1h" |
||||
# The secret that will be created |
||||
target: |
||||
name: {{ include "agent-common.fullname" . }}-secret |
||||
template: |
||||
type: Opaque |
||||
metadata: |
||||
labels: |
||||
{{- include "agent-common.labels" . | nindent 10 }} |
||||
data: |
||||
HYP_BASE_DB: {{ print "'{{ .db | toString }}'" }} |
||||
{{/* RPC URLs */}} |
||||
{{- /* |
||||
* For each Outbox network, create an environment variable with the RPC endpoint. |
||||
* The templating of external-secrets will use the data section below to know how |
||||
* to replace the correct value in the created secret. |
||||
*/}} |
||||
{{- range .Values.abacus.outboxChains }} |
||||
{{- if not .disabled }} |
||||
{{- if eq .connection.type "httpQuorum" }} |
||||
HYP_SCRAPER_OUTBOXES_{{ .name | upper }}_CONNECTION_URLS: {{ printf "'{{ .%s_rpcs | fromJson | join \",\" }}'" .name }} |
||||
{{- else }} |
||||
HYP_SCRAPER_OUTBOXES_{{ .name | upper }}_CONNECTION_URL: {{ printf "'{{ .%s_rpc | toString }}'" .name }} |
||||
{{- end }} |
||||
{{- end }} |
||||
{{- end }} |
||||
{{- /* |
||||
* For each Inbox network, create an environment variable with the RPC endpoint. |
||||
* The templating of external-secrets will use the data section below to know how |
||||
* to replace the correct value in the created secret. |
||||
*/}} |
||||
{{- range .Values.abacus.inboxChains }} |
||||
{{- if not .disabled }} |
||||
{{- if eq .connection.type "httpQuorum" }} |
||||
HYP_BASE_INBOXES_{{ .name | upper }}_CONNECTION_URLS: {{ printf "'{{ .%s_rpcs | fromJson | join \",\" }}'" .name }} |
||||
{{- else }} |
||||
HYP_BASE_INBOXES_{{ .name | upper }}_CONNECTION_URL: {{ printf "'{{ .%s_rpc | toString }}'" .name }} |
||||
{{- end }} |
||||
{{- end }} |
||||
{{- end }} |
||||
data: |
||||
- secretKey: db |
||||
remoteRef: |
||||
key: {{ printf "%s-%s-scraper-db" .Values.abacus.context .Values.abacus.runEnv }} |
||||
{{- /* |
||||
* For each network, load the secret in GCP secret manager with the form: environment-rpc-endpoint-network, |
||||
* and associate it with the secret key networkname_rpc. |
||||
* We assume that the set of Outbox chains and Inbox chains are the same. |
||||
*/}} |
||||
{{- range .Values.abacus.inboxChains }} |
||||
{{- if not .disabled }} |
||||
{{- if eq .connection.type "httpQuorum" }} |
||||
- secretKey: {{ printf "%s_rpcs" .name }} |
||||
remoteRef: |
||||
key: {{ printf "%s-rpc-endpoints-%s" $.Values.abacus.runEnv .name }} |
||||
{{- else }} |
||||
- secretKey: {{ printf "%s_rpc" .name }} |
||||
remoteRef: |
||||
key: {{ printf "%s-rpc-endpoint-%s" $.Values.abacus.runEnv .name }} |
||||
{{- end }} |
||||
{{- end }} |
||||
{{- end }} |
||||
@ -0,0 +1,81 @@ |
||||
{{- if .Values.abacus.scraper.enabled }} |
||||
apiVersion: apps/v1 |
||||
kind: StatefulSet |
||||
metadata: |
||||
name: {{ include "agent-common.fullname" . }} |
||||
labels: |
||||
{{- include "agent-common.labels" . | nindent 4 }} |
||||
app.kubernetes.io/component: scraper |
||||
spec: |
||||
selector: |
||||
matchLabels: |
||||
{{- include "agent-common.selectorLabels" . | nindent 6 }} |
||||
app.kubernetes.io/component: scraper |
||||
replicas: 1 |
||||
serviceName: {{ include "agent-common.fullname" . }} |
||||
template: |
||||
metadata: |
||||
annotations: |
||||
checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} |
||||
{{- with .Values.podAnnotations }} |
||||
{{- toYaml . | nindent 8 }} |
||||
{{- end }} |
||||
{{- with .Values.abacus.scraper.podAnnotations }} |
||||
{{ toYaml . | nindent 8 }} |
||||
{{- end }} |
||||
labels: |
||||
{{- include "agent-common.labels" . | nindent 8 }} |
||||
app.kubernetes.io/component: scraper |
||||
{{- with .Values.podCommonLabels }} |
||||
{{ toYaml . | nindent 8 }} |
||||
{{- end }} |
||||
{{- with .Values.abacus.scraper.podLabels }} |
||||
{{ toYaml . | nindent 8 }} |
||||
{{- end }} |
||||
spec: |
||||
{{- with .Values.imagePullSecrets }} |
||||
imagePullSecrets: |
||||
{{- toYaml . | nindent 8 }} |
||||
{{- end }} |
||||
terminationGracePeriodSeconds: 10 |
||||
securityContext: |
||||
{{- toYaml .Values.podSecurityContext | nindent 8 }} |
||||
containers: |
||||
- name: agent |
||||
securityContext: |
||||
{{- toYaml .Values.securityContext | nindent 10 }} |
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" |
||||
imagePullPolicy: {{ .Values.image.pullPolicy }} |
||||
command: ["./scraper"] |
||||
env: |
||||
{{- include "agent-common.config-env-vars" (dict "config" .Values.abacus.scraper.config "agent_name" "scraper") | indent 10 }} |
||||
{{- range .Values.abacus.scraper.signers }} |
||||
{{- include "agent-common.config-env-vars" (dict "config" .keyConfig "agent_name" "base" "key_name_prefix" (printf "SIGNERS_%s_" (.name | upper))) | indent 10 }} |
||||
{{- end }} |
||||
{{- if .Values.abacus.tracing.uri }} |
||||
- name: HYP_BASE_TRACING_JAEGER_NAME |
||||
value: {{ include "agent-common.fullname" . }}-scraper |
||||
{{- end }} |
||||
envFrom: |
||||
- configMapRef: |
||||
name: {{ include "agent-common.fullname" . }} |
||||
- secretRef: |
||||
name: {{ include "agent-common.fullname" . }}-secret |
||||
resources: |
||||
{{- toYaml .Values.abacus.scraper.resources | nindent 10 }} |
||||
ports: |
||||
- name: metrics |
||||
containerPort: {{ .Values.abacus.metrics.port }} |
||||
{{- with .Values.nodeSelector }} |
||||
nodeSelector: |
||||
{{- toYaml . | nindent 8 }} |
||||
{{- end }} |
||||
{{- with .Values.affinity }} |
||||
affinity: |
||||
{{- toYaml . | nindent 8 }} |
||||
{{- end }} |
||||
{{- with .Values.tolerations }} |
||||
tolerations: |
||||
{{- toYaml . | nindent 8 }} |
||||
{{- end }} |
||||
{{- end }} |
||||
@ -0,0 +1,92 @@ |
||||
image: |
||||
repository: gcr.io/clabs-abacus/abacus-agent |
||||
pullPolicy: Always |
||||
tag: 'latest' |
||||
|
||||
imagePullSecrets: [] |
||||
nameOverride: '' |
||||
fullnameOverride: '' |
||||
|
||||
serviceAccount: |
||||
create: true |
||||
annotations: {} |
||||
name: '' |
||||
|
||||
externalSecrets: |
||||
clusterSecretStore: |
||||
|
||||
podAnnotations: {} |
||||
podCommonLabels: {} |
||||
|
||||
storage: |
||||
storageClass: 'standard' |
||||
accessModes: ReadWriteOnce |
||||
|
||||
# -- Abacus Overrides |
||||
# By Default, Abacus Agents load the config baked into the Docker Image |
||||
# Pass values here in order to override the values in the config |
||||
# Note: For successful operation, one _must_ pass signer keys as |
||||
# they are not baked into the image for security reasons. |
||||
abacus: |
||||
# Sets the config folder to use |
||||
runEnv: 'default' |
||||
context: 'abacus' |
||||
# Sets the base config to be used (switch between Homes) |
||||
baseConfig: 'base.json' |
||||
# Set the SQL db connection info |
||||
dbPath: external-secret |
||||
rustBacktrace: full |
||||
tracing: |
||||
# Set the log formatting |
||||
format: json |
||||
# Set the log level |
||||
level: info |
||||
uri: '' |
||||
metrics: |
||||
port: 9090 |
||||
|
||||
inboxChains: |
||||
|
||||
scraper: |
||||
enabled: true |
||||
podAnnotations: |
||||
prometheus.io/port: '9090' |
||||
prometheus.io/scrape: 'true' |
||||
podLabels: {} |
||||
storage: |
||||
size: 10Mi |
||||
snapshot: |
||||
enabled: false |
||||
name: '' |
||||
resources: |
||||
requests: |
||||
cpu: 250m |
||||
memory: 256Mi |
||||
signers: |
||||
config: |
||||
|
||||
podSecurityContext: |
||||
fsGroup: 2000 |
||||
|
||||
securityContext: |
||||
{} |
||||
# capabilities: |
||||
# drop: |
||||
# - ALL |
||||
# readOnlyRootFilesystem: true |
||||
# runAsNonRoot: true |
||||
# runAsUser: 1000 |
||||
|
||||
resources: |
||||
# limits: |
||||
# cpu: 100m |
||||
# memory: 128Mi |
||||
# requests: |
||||
# cpu: 250m |
||||
# memory: 1Gi |
||||
|
||||
nodeSelector: {} |
||||
|
||||
tolerations: [] |
||||
|
||||
affinity: {} |
||||
@ -0,0 +1,30 @@ |
||||
import { Contexts } from '../config/contexts'; |
||||
import { runScraperHelmCommand } from '../src/scraper/deploy'; |
||||
import { HelmCommand } from '../src/utils/helm'; |
||||
|
||||
import { |
||||
assertCorrectKubeContext, |
||||
getContextAgentConfig, |
||||
getCoreEnvironmentConfig, |
||||
getEnvironment, |
||||
} from './utils'; |
||||
|
||||
async function deploy() { |
||||
const environment = await getEnvironment(); |
||||
const config = getCoreEnvironmentConfig(environment); |
||||
|
||||
const agentConfig = await getContextAgentConfig(config); |
||||
if (agentConfig.context != Contexts.Abacus) { |
||||
// scraper scrapes everything so deploying for multiple contexts might cause unintentional
|
||||
// conflicts
|
||||
throw new Error( |
||||
`Scraper only supports the '${Contexts.Abacus}' context at this time`, |
||||
); |
||||
} |
||||
|
||||
await assertCorrectKubeContext(config); |
||||
|
||||
await runScraperHelmCommand(HelmCommand.InstallOrUpgrade, agentConfig); |
||||
} |
||||
|
||||
deploy().then(console.log).catch(console.error); |
||||
@ -0,0 +1,108 @@ |
||||
import { ChainName } from '@hyperlane-xyz/sdk'; |
||||
|
||||
import { AgentConfig } from '../config'; |
||||
import { ConnectionType } from '../config/agent'; |
||||
import { |
||||
HelmCommand, |
||||
buildHelmChartDependencies, |
||||
helmifyValues, |
||||
} from '../utils/helm'; |
||||
import { execCmd } from '../utils/utils'; |
||||
|
||||
const helmChartPath = '../../rust/helm/scraper/'; |
||||
|
||||
export async function runScraperHelmCommand<Chain extends ChainName>( |
||||
action: HelmCommand, |
||||
agentConfig: AgentConfig<Chain>, |
||||
) { |
||||
const values = await scraperHelmValues(agentConfig); |
||||
|
||||
const extraPipe = |
||||
action === HelmCommand.UpgradeDiff |
||||
? ` | kubectl diff -n ${agentConfig.namespace} --field-manager="Go-http-client" -f - || true` |
||||
: ''; |
||||
|
||||
const helmReleaseName = getScraperHelmReleaseName(agentConfig); |
||||
|
||||
if (action === HelmCommand.InstallOrUpgrade) { |
||||
// Delete secrets to avoid them being stale
|
||||
try { |
||||
await execCmd( |
||||
`kubectl delete secrets --namespace ${agentConfig.namespace} --selector app.kubernetes.io/instance=${helmReleaseName}`, |
||||
{}, |
||||
false, |
||||
false, |
||||
); |
||||
} catch (e) { |
||||
console.error(e); |
||||
} |
||||
} |
||||
|
||||
// Build the chart dependencies
|
||||
await buildHelmChartDependencies(helmChartPath); |
||||
|
||||
await execCmd( |
||||
`helm ${action} ${helmReleaseName} ${helmChartPath} --create-namespace --namespace ${ |
||||
agentConfig.namespace |
||||
} ${values.join(' ')} ${extraPipe}`,
|
||||
{}, |
||||
false, |
||||
true, |
||||
); |
||||
} |
||||
|
||||
async function scraperHelmValues<Chain extends ChainName>( |
||||
agentConfig: AgentConfig<Chain>, |
||||
) { |
||||
// By default, if a context only enables a subset of chains, the
|
||||
// connection url (or urls, when HttpQuorum is used) are not fetched
|
||||
// from GCP secret manager. For Http/Ws, the `url` param is expected,
|
||||
// which is set by default to "" in the agent json configs. For HttpQuorum,
|
||||
// no default is present in those configs, so we make sure to pass in urls
|
||||
// as "" to avoid startup configuration issues.
|
||||
let baseConnectionConfig: Record<string, string> = { |
||||
type: agentConfig.connectionType, |
||||
}; |
||||
if (baseConnectionConfig.type == ConnectionType.HttpQuorum) { |
||||
baseConnectionConfig = { |
||||
...baseConnectionConfig, |
||||
urls: '', |
||||
}; |
||||
} else { |
||||
baseConnectionConfig = { |
||||
...baseConnectionConfig, |
||||
url: '', |
||||
}; |
||||
} |
||||
|
||||
const chains = agentConfig.environmentChainNames.map((name) => ({ |
||||
name, |
||||
disabled: !agentConfig.contextChainNames.includes(name), |
||||
connection: baseConnectionConfig, |
||||
})); |
||||
|
||||
const valueDict = { |
||||
abacus: { |
||||
baseConfig: 'scraper_config.json', |
||||
runEnv: agentConfig.environment, |
||||
context: agentConfig.context, |
||||
// Expects an array
|
||||
inboxChains: chains, |
||||
outboxChains: chains, |
||||
scraper: { |
||||
config: {}, |
||||
}, |
||||
}, |
||||
image: { |
||||
repository: agentConfig.docker.repo, |
||||
tag: agentConfig.docker.tag, |
||||
}, |
||||
}; |
||||
return helmifyValues(valueDict); |
||||
} |
||||
|
||||
function getScraperHelmReleaseName<Chain extends ChainName>( |
||||
agentConfig: AgentConfig<Chain>, |
||||
) { |
||||
return `${agentConfig.context}-scraper`; |
||||
} |
||||
Loading…
Reference in new issue