feat: Add github proxy (#4418)
### Description This PR adds github-proxy CloudFlare Worker project which will attach an api key to github requests to increase read capacity. It is mostly a passthrough proxy that has a simple allowlist. This project is created and deployed using CloudFlare's [cloudflare create CLI](https://developers.cloudflare.com/workers/get-started/guide/) It is deployed using `yarn deploy` (aka wrangler) ### Drive-by changes - `yarn up chai@4.5.0` - `yarn up typescript@5.5.2` - `yarn up yaml@2.4.5` ### Related issues Fixes: https://github.com/hyperlane-xyz/hyperlane-registry/issues/163 ### Backward compatibility Yes ### Testing Manual/Unit Tests - Use a script to ddos github, then try cli command `hyperlane warp read` - Unit tests for the Worker, and GithubRegistry changespull/4482/head
parent
1ab8751afb
commit
0e2f94ba10
@ -0,0 +1,5 @@ |
|||||||
|
--- |
||||||
|
'github-proxy': major |
||||||
|
--- |
||||||
|
|
||||||
|
Add github proxy to reduce github API load |
@ -0,0 +1,54 @@ |
|||||||
|
import { expect } from 'chai'; |
||||||
|
|
||||||
|
import { WarpRouteDeployConfig } from '@hyperlane-xyz/sdk'; |
||||||
|
|
||||||
|
import { readYamlOrJson, writeYamlOrJson } from '../utils/files.js'; |
||||||
|
|
||||||
|
import { |
||||||
|
ANVIL_KEY, |
||||||
|
REGISTRY_PATH, |
||||||
|
deployOrUseExistingCore, |
||||||
|
} from './commands/helpers.js'; |
||||||
|
import { hyperlaneWarpDeploy, readWarpConfig } from './commands/warp.js'; |
||||||
|
|
||||||
|
const CHAIN_NAME_2 = 'anvil2'; |
||||||
|
|
||||||
|
const EXAMPLES_PATH = './examples'; |
||||||
|
const CORE_CONFIG_PATH = `${EXAMPLES_PATH}/core-config.yaml`; |
||||||
|
const WARP_CONFIG_PATH_EXAMPLE = `${EXAMPLES_PATH}/warp-route-deployment.yaml`; |
||||||
|
|
||||||
|
const TEMP_PATH = '/tmp'; // /temp gets removed at the end of all-test.sh
|
||||||
|
const WARP_CONFIG_PATH_2 = `${TEMP_PATH}/anvil2/warp-route-deployment-anvil2.yaml`; |
||||||
|
const WARP_CORE_CONFIG_PATH_2 = `${REGISTRY_PATH}/deployments/warp_routes/ETH/anvil2-config.yaml`; |
||||||
|
|
||||||
|
const TEST_TIMEOUT = 60_000; // Long timeout since these tests can take a while
|
||||||
|
describe('WarpRead e2e tests', async function () { |
||||||
|
let anvil2Config: WarpRouteDeployConfig; |
||||||
|
this.timeout(TEST_TIMEOUT); |
||||||
|
before(async function () { |
||||||
|
await deployOrUseExistingCore(CHAIN_NAME_2, CORE_CONFIG_PATH, ANVIL_KEY); |
||||||
|
|
||||||
|
// Create a new warp config using the example
|
||||||
|
const exampleWarpConfig: WarpRouteDeployConfig = readYamlOrJson( |
||||||
|
WARP_CONFIG_PATH_EXAMPLE, |
||||||
|
); |
||||||
|
anvil2Config = { anvil2: { ...exampleWarpConfig.anvil1 } }; |
||||||
|
writeYamlOrJson(WARP_CONFIG_PATH_2, anvil2Config); |
||||||
|
}); |
||||||
|
|
||||||
|
beforeEach(async function () { |
||||||
|
await hyperlaneWarpDeploy(WARP_CONFIG_PATH_2); |
||||||
|
}); |
||||||
|
|
||||||
|
it('should be able to read a warp route', async function () { |
||||||
|
const warpConfigPath = `${TEMP_PATH}/warp-route-deployment-2.yaml`; |
||||||
|
const warpConfig = await readWarpConfig( |
||||||
|
CHAIN_NAME_2, |
||||||
|
WARP_CORE_CONFIG_PATH_2, |
||||||
|
warpConfigPath, |
||||||
|
); |
||||||
|
expect(warpConfig[CHAIN_NAME_2].type).to.be.equal( |
||||||
|
anvil2Config[CHAIN_NAME_2].type, |
||||||
|
); |
||||||
|
}); |
||||||
|
}); |
@ -0,0 +1 @@ |
|||||||
|
GITHUB_API_KEY= |
@ -0,0 +1,172 @@ |
|||||||
|
# Logs |
||||||
|
|
||||||
|
logs |
||||||
|
_.log |
||||||
|
npm-debug.log_ |
||||||
|
yarn-debug.log* |
||||||
|
yarn-error.log* |
||||||
|
lerna-debug.log* |
||||||
|
.pnpm-debug.log* |
||||||
|
|
||||||
|
# Diagnostic reports (https://nodejs.org/api/report.html) |
||||||
|
|
||||||
|
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json |
||||||
|
|
||||||
|
# Runtime data |
||||||
|
|
||||||
|
pids |
||||||
|
_.pid |
||||||
|
_.seed |
||||||
|
\*.pid.lock |
||||||
|
|
||||||
|
# Directory for instrumented libs generated by jscoverage/JSCover |
||||||
|
|
||||||
|
lib-cov |
||||||
|
|
||||||
|
# Coverage directory used by tools like istanbul |
||||||
|
|
||||||
|
coverage |
||||||
|
\*.lcov |
||||||
|
|
||||||
|
# nyc test coverage |
||||||
|
|
||||||
|
.nyc_output |
||||||
|
|
||||||
|
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) |
||||||
|
|
||||||
|
.grunt |
||||||
|
|
||||||
|
# Bower dependency directory (https://bower.io/) |
||||||
|
|
||||||
|
bower_components |
||||||
|
|
||||||
|
# node-waf configuration |
||||||
|
|
||||||
|
.lock-wscript |
||||||
|
|
||||||
|
# Compiled binary addons (https://nodejs.org/api/addons.html) |
||||||
|
|
||||||
|
build/Release |
||||||
|
|
||||||
|
# Dependency directories |
||||||
|
|
||||||
|
node_modules/ |
||||||
|
jspm_packages/ |
||||||
|
|
||||||
|
# Snowpack dependency directory (https://snowpack.dev/) |
||||||
|
|
||||||
|
web_modules/ |
||||||
|
|
||||||
|
# TypeScript cache |
||||||
|
|
||||||
|
\*.tsbuildinfo |
||||||
|
|
||||||
|
# Optional npm cache directory |
||||||
|
|
||||||
|
.npm |
||||||
|
|
||||||
|
# Optional eslint cache |
||||||
|
|
||||||
|
.eslintcache |
||||||
|
|
||||||
|
# Optional stylelint cache |
||||||
|
|
||||||
|
.stylelintcache |
||||||
|
|
||||||
|
# Microbundle cache |
||||||
|
|
||||||
|
.rpt2_cache/ |
||||||
|
.rts2_cache_cjs/ |
||||||
|
.rts2_cache_es/ |
||||||
|
.rts2_cache_umd/ |
||||||
|
|
||||||
|
# Optional REPL history |
||||||
|
|
||||||
|
.node_repl_history |
||||||
|
|
||||||
|
# Output of 'npm pack' |
||||||
|
|
||||||
|
\*.tgz |
||||||
|
|
||||||
|
# Yarn Integrity file |
||||||
|
|
||||||
|
.yarn-integrity |
||||||
|
|
||||||
|
# dotenv environment variable files |
||||||
|
|
||||||
|
.env |
||||||
|
.env.development.local |
||||||
|
.env.test.local |
||||||
|
.env.production.local |
||||||
|
.env.local |
||||||
|
|
||||||
|
# parcel-bundler cache (https://parceljs.org/) |
||||||
|
|
||||||
|
.cache |
||||||
|
.parcel-cache |
||||||
|
|
||||||
|
# Next.js build output |
||||||
|
|
||||||
|
.next |
||||||
|
out |
||||||
|
|
||||||
|
# Nuxt.js build / generate output |
||||||
|
|
||||||
|
.nuxt |
||||||
|
dist |
||||||
|
|
||||||
|
# Gatsby files |
||||||
|
|
||||||
|
.cache/ |
||||||
|
|
||||||
|
# Comment in the public line in if your project uses Gatsby and not Next.js |
||||||
|
|
||||||
|
# https://nextjs.org/blog/next-9-1#public-directory-support |
||||||
|
|
||||||
|
# public |
||||||
|
|
||||||
|
# vuepress build output |
||||||
|
|
||||||
|
.vuepress/dist |
||||||
|
|
||||||
|
# vuepress v2.x temp and cache directory |
||||||
|
|
||||||
|
.temp |
||||||
|
.cache |
||||||
|
|
||||||
|
# Docusaurus cache and generated files |
||||||
|
|
||||||
|
.docusaurus |
||||||
|
|
||||||
|
# Serverless directories |
||||||
|
|
||||||
|
.serverless/ |
||||||
|
|
||||||
|
# FuseBox cache |
||||||
|
|
||||||
|
.fusebox/ |
||||||
|
|
||||||
|
# DynamoDB Local files |
||||||
|
|
||||||
|
.dynamodb/ |
||||||
|
|
||||||
|
# TernJS port file |
||||||
|
|
||||||
|
.tern-port |
||||||
|
|
||||||
|
# Stores VSCode versions used for testing VSCode extensions |
||||||
|
|
||||||
|
.vscode-test |
||||||
|
|
||||||
|
# yarn v2 |
||||||
|
|
||||||
|
.yarn/cache |
||||||
|
.yarn/unplugged |
||||||
|
.yarn/build-state.yml |
||||||
|
.yarn/install-state.gz |
||||||
|
.pnp.\* |
||||||
|
|
||||||
|
# wrangler project |
||||||
|
|
||||||
|
.dev.vars |
||||||
|
.wrangler/ |
@ -0,0 +1,35 @@ |
|||||||
|
{ |
||||||
|
"name": "@hyperlane-xyz/github-proxy", |
||||||
|
"description": "Github proxy that adds the API key to requests", |
||||||
|
"version": "5.1.0", |
||||||
|
"scripts": { |
||||||
|
"deploy": "wrangler deploy", |
||||||
|
"deploy:staging": "wrangler deploy --env staging", |
||||||
|
"deploy:key": "wrangler secret put GITHUB_API_KEY", |
||||||
|
"dev": "wrangler dev", |
||||||
|
"start": "wrangler dev", |
||||||
|
"test": "vitest", |
||||||
|
"prettier": "prettier --write ./src ./test", |
||||||
|
"cf-typegen": "wrangler types" |
||||||
|
}, |
||||||
|
"type": "module", |
||||||
|
"homepage": "https://www.hyperlane.xyz", |
||||||
|
"repository": "https://github.com/hyperlane-xyz/hyperlane-monorepo", |
||||||
|
"keywords": [ |
||||||
|
"Hyperlane", |
||||||
|
"Github", |
||||||
|
"Proxy", |
||||||
|
"Typescript" |
||||||
|
], |
||||||
|
"license": "Apache-2.0", |
||||||
|
"devDependencies": { |
||||||
|
"@cloudflare/vitest-pool-workers": "^0.4.5", |
||||||
|
"@cloudflare/workers-types": "^4.20240821.1", |
||||||
|
"@faker-js/faker": "^8.4.1", |
||||||
|
"chai": "4.5.0", |
||||||
|
"prettier": "^2.8.8", |
||||||
|
"typescript": "5.3.3", |
||||||
|
"vitest": "1.4.0", |
||||||
|
"wrangler": "^3.74.0" |
||||||
|
} |
||||||
|
} |
@ -0,0 +1 @@ |
|||||||
|
export const DISALLOWED_URL_MSG = 'Origin not allowed'; |
@ -0,0 +1,26 @@ |
|||||||
|
import { DISALLOWED_URL_MSG } from './errors.js'; |
||||||
|
|
||||||
|
const GITHUB_API_ALLOWLIST = [ |
||||||
|
'/repos/hyperlane-xyz/hyperlane-registry/git/trees/main', |
||||||
|
]; |
||||||
|
const GITPUB_API_HOST = 'https://api.github.com'; |
||||||
|
export default { |
||||||
|
async fetch(request, env, _ctx): Promise<Response> { |
||||||
|
const apiUrlPath = new URL(request.url).pathname; |
||||||
|
const isAllowed = GITHUB_API_ALLOWLIST.includes(apiUrlPath); |
||||||
|
if (!isAllowed) { |
||||||
|
return new Response(DISALLOWED_URL_MSG, { status: 401 }); |
||||||
|
} |
||||||
|
|
||||||
|
const apiUrl = new URL(`${GITPUB_API_HOST}${apiUrlPath}`); |
||||||
|
return fetch(apiUrl, { |
||||||
|
method: 'GET', |
||||||
|
headers: { |
||||||
|
'Content-Type': 'application/json', |
||||||
|
'User-Agent': 'Hyperlane-Github-Proxy', |
||||||
|
'X-GitHub-Api-Version': '2022-11-28', |
||||||
|
Authorization: `Bearer ${env.GITHUB_API_KEY}`, |
||||||
|
}, |
||||||
|
}); |
||||||
|
}, |
||||||
|
} satisfies ExportedHandler<Env>; |
@ -0,0 +1,34 @@ |
|||||||
|
import { faker } from '@faker-js/faker'; |
||||||
|
import { SELF } from 'cloudflare:test'; |
||||||
|
import { describe, expect, it } from 'vitest'; |
||||||
|
|
||||||
|
import { DISALLOWED_URL_MSG } from '../src/errors.js'; |
||||||
|
|
||||||
|
describe('Hello World worker', () => { |
||||||
|
it('returns empty response if pathname provided is not a valid api url', async () => { |
||||||
|
const results = await SELF.fetch('https://example.com/favicon.ico'); |
||||||
|
|
||||||
|
expect(results.status).toBe(401); |
||||||
|
expect(await results.text()).toBe(DISALLOWED_URL_MSG); |
||||||
|
}); |
||||||
|
|
||||||
|
it('returns empty response if origin is not on allowlist', async () => { |
||||||
|
const results = await SELF.fetch( |
||||||
|
'https://example.com/repos/custom-hyperlane-xyz/hyperlane-registry/git/trees/main?recursive=true', |
||||||
|
); |
||||||
|
|
||||||
|
expect(results.status).toBe(401); |
||||||
|
expect(await results.text()).toBe(DISALLOWED_URL_MSG); |
||||||
|
}); |
||||||
|
|
||||||
|
it('returns empty response if origin is not on allowlist (with faker 200 tests)', async () => { |
||||||
|
for (let i = 0; i < 200; i++) { |
||||||
|
const results = await SELF.fetch( |
||||||
|
`https://example.com/${faker.internet.url}`, |
||||||
|
); |
||||||
|
|
||||||
|
expect(results.status).toBe(401); |
||||||
|
expect(await results.text()).toBe(DISALLOWED_URL_MSG); |
||||||
|
} |
||||||
|
}); |
||||||
|
}); |
@ -0,0 +1,11 @@ |
|||||||
|
{ |
||||||
|
"extends": "../tsconfig.json", |
||||||
|
"compilerOptions": { |
||||||
|
"types": [ |
||||||
|
"@cloudflare/workers-types/experimental", |
||||||
|
"@cloudflare/vitest-pool-workers" |
||||||
|
] |
||||||
|
}, |
||||||
|
"include": ["./**/*.ts", "../src/env.d.ts"], |
||||||
|
"exclude": [] |
||||||
|
} |
@ -0,0 +1,15 @@ |
|||||||
|
{ |
||||||
|
"extends": "../../tsconfig.json", |
||||||
|
"compilerOptions": { |
||||||
|
"types": [ |
||||||
|
"@cloudflare/workers-types" |
||||||
|
] /* Specify type package names to be included without being referenced in a source file. */, |
||||||
|
|
||||||
|
"noEmit": true /* Disable emitting files from a compilation. */, |
||||||
|
|
||||||
|
"isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */, |
||||||
|
"allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */ |
||||||
|
}, |
||||||
|
"exclude": ["test"], |
||||||
|
"include": ["worker-configuration.d.ts", "src/**/*.ts"] |
||||||
|
} |
@ -0,0 +1,11 @@ |
|||||||
|
import { defineWorkersConfig } from '@cloudflare/vitest-pool-workers/config'; |
||||||
|
|
||||||
|
export default defineWorkersConfig({ |
||||||
|
test: { |
||||||
|
poolOptions: { |
||||||
|
workers: { |
||||||
|
wrangler: { configPath: './wrangler.toml' }, |
||||||
|
}, |
||||||
|
}, |
||||||
|
}, |
||||||
|
}); |
@ -0,0 +1,6 @@ |
|||||||
|
// Generated by Wrangler on Tue Sep 03 2024 19:01:13 GMT-0400 (Eastern Daylight Time)
|
||||||
|
// by running `wrangler types`
|
||||||
|
|
||||||
|
interface Env { |
||||||
|
GITHUB_API_KEY: string; |
||||||
|
} |
@ -0,0 +1,24 @@ |
|||||||
|
#:schema node_modules/wrangler/config-schema.json |
||||||
|
name = "github-proxy-prod" |
||||||
|
main = "src/index.ts" |
||||||
|
compatibility_date = "2024-08-21" |
||||||
|
compatibility_flags = ["nodejs_compat"] |
||||||
|
workers_dev = false |
||||||
|
routes = [ |
||||||
|
{ pattern = "proxy.hyperlane.xyz", custom_domain = true } |
||||||
|
] |
||||||
|
|
||||||
|
# CPU limit - 10ms keeps us within the free tier. As of 9/6/2024, the median CPU time is ~1ms |
||||||
|
[limits] |
||||||
|
cpu_ms = 10 |
||||||
|
|
||||||
|
# Automatically place your workloads in an optimal location to minimize latency. |
||||||
|
# If you are running back-end logic in a Worker, running it closer to your back-end infrastructure |
||||||
|
# rather than the end user may result in better performance. |
||||||
|
# Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement |
||||||
|
[placement] |
||||||
|
mode = "smart" |
||||||
|
|
||||||
|
[env.staging] |
||||||
|
name = "github-proxy-staging" |
||||||
|
workers_dev = true |
Loading…
Reference in new issue