feat: Add github proxy (#4418)
### Description This PR adds github-proxy CloudFlare Worker project which will attach an api key to github requests to increase read capacity. It is mostly a passthrough proxy that has a simple allowlist. This project is created and deployed using CloudFlare's [cloudflare create CLI](https://developers.cloudflare.com/workers/get-started/guide/) It is deployed using `yarn deploy` (aka wrangler) ### Drive-by changes - `yarn up chai@4.5.0` - `yarn up typescript@5.5.2` - `yarn up yaml@2.4.5` ### Related issues Fixes: https://github.com/hyperlane-xyz/hyperlane-registry/issues/163 ### Backward compatibility Yes ### Testing Manual/Unit Tests - Use a script to ddos github, then try cli command `hyperlane warp read` - Unit tests for the Worker, and GithubRegistry changespull/4482/head
Lee
2 months ago
committed by
GitHub
parent
1ab8751afb
commit
0e2f94ba10
@ -0,0 +1,5 @@ |
||||
--- |
||||
'github-proxy': major |
||||
--- |
||||
|
||||
Add github proxy to reduce github API load |
||||
@ -0,0 +1,54 @@ |
||||
import { expect } from 'chai'; |
||||
|
||||
import { WarpRouteDeployConfig } from '@hyperlane-xyz/sdk'; |
||||
|
||||
import { readYamlOrJson, writeYamlOrJson } from '../utils/files.js'; |
||||
|
||||
import { |
||||
ANVIL_KEY, |
||||
REGISTRY_PATH, |
||||
deployOrUseExistingCore, |
||||
} from './commands/helpers.js'; |
||||
import { hyperlaneWarpDeploy, readWarpConfig } from './commands/warp.js'; |
||||
|
||||
const CHAIN_NAME_2 = 'anvil2'; |
||||
|
||||
const EXAMPLES_PATH = './examples'; |
||||
const CORE_CONFIG_PATH = `${EXAMPLES_PATH}/core-config.yaml`; |
||||
const WARP_CONFIG_PATH_EXAMPLE = `${EXAMPLES_PATH}/warp-route-deployment.yaml`; |
||||
|
||||
const TEMP_PATH = '/tmp'; // /temp gets removed at the end of all-test.sh
|
||||
const WARP_CONFIG_PATH_2 = `${TEMP_PATH}/anvil2/warp-route-deployment-anvil2.yaml`; |
||||
const WARP_CORE_CONFIG_PATH_2 = `${REGISTRY_PATH}/deployments/warp_routes/ETH/anvil2-config.yaml`; |
||||
|
||||
const TEST_TIMEOUT = 60_000; // Long timeout since these tests can take a while
|
||||
describe('WarpRead e2e tests', async function () { |
||||
let anvil2Config: WarpRouteDeployConfig; |
||||
this.timeout(TEST_TIMEOUT); |
||||
before(async function () { |
||||
await deployOrUseExistingCore(CHAIN_NAME_2, CORE_CONFIG_PATH, ANVIL_KEY); |
||||
|
||||
// Create a new warp config using the example
|
||||
const exampleWarpConfig: WarpRouteDeployConfig = readYamlOrJson( |
||||
WARP_CONFIG_PATH_EXAMPLE, |
||||
); |
||||
anvil2Config = { anvil2: { ...exampleWarpConfig.anvil1 } }; |
||||
writeYamlOrJson(WARP_CONFIG_PATH_2, anvil2Config); |
||||
}); |
||||
|
||||
beforeEach(async function () { |
||||
await hyperlaneWarpDeploy(WARP_CONFIG_PATH_2); |
||||
}); |
||||
|
||||
it('should be able to read a warp route', async function () { |
||||
const warpConfigPath = `${TEMP_PATH}/warp-route-deployment-2.yaml`; |
||||
const warpConfig = await readWarpConfig( |
||||
CHAIN_NAME_2, |
||||
WARP_CORE_CONFIG_PATH_2, |
||||
warpConfigPath, |
||||
); |
||||
expect(warpConfig[CHAIN_NAME_2].type).to.be.equal( |
||||
anvil2Config[CHAIN_NAME_2].type, |
||||
); |
||||
}); |
||||
}); |
||||
@ -0,0 +1 @@ |
||||
GITHUB_API_KEY= |
||||
@ -0,0 +1,172 @@ |
||||
# Logs |
||||
|
||||
logs |
||||
_.log |
||||
npm-debug.log_ |
||||
yarn-debug.log* |
||||
yarn-error.log* |
||||
lerna-debug.log* |
||||
.pnpm-debug.log* |
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html) |
||||
|
||||
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json |
||||
|
||||
# Runtime data |
||||
|
||||
pids |
||||
_.pid |
||||
_.seed |
||||
\*.pid.lock |
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover |
||||
|
||||
lib-cov |
||||
|
||||
# Coverage directory used by tools like istanbul |
||||
|
||||
coverage |
||||
\*.lcov |
||||
|
||||
# nyc test coverage |
||||
|
||||
.nyc_output |
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) |
||||
|
||||
.grunt |
||||
|
||||
# Bower dependency directory (https://bower.io/) |
||||
|
||||
bower_components |
||||
|
||||
# node-waf configuration |
||||
|
||||
.lock-wscript |
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html) |
||||
|
||||
build/Release |
||||
|
||||
# Dependency directories |
||||
|
||||
node_modules/ |
||||
jspm_packages/ |
||||
|
||||
# Snowpack dependency directory (https://snowpack.dev/) |
||||
|
||||
web_modules/ |
||||
|
||||
# TypeScript cache |
||||
|
||||
\*.tsbuildinfo |
||||
|
||||
# Optional npm cache directory |
||||
|
||||
.npm |
||||
|
||||
# Optional eslint cache |
||||
|
||||
.eslintcache |
||||
|
||||
# Optional stylelint cache |
||||
|
||||
.stylelintcache |
||||
|
||||
# Microbundle cache |
||||
|
||||
.rpt2_cache/ |
||||
.rts2_cache_cjs/ |
||||
.rts2_cache_es/ |
||||
.rts2_cache_umd/ |
||||
|
||||
# Optional REPL history |
||||
|
||||
.node_repl_history |
||||
|
||||
# Output of 'npm pack' |
||||
|
||||
\*.tgz |
||||
|
||||
# Yarn Integrity file |
||||
|
||||
.yarn-integrity |
||||
|
||||
# dotenv environment variable files |
||||
|
||||
.env |
||||
.env.development.local |
||||
.env.test.local |
||||
.env.production.local |
||||
.env.local |
||||
|
||||
# parcel-bundler cache (https://parceljs.org/) |
||||
|
||||
.cache |
||||
.parcel-cache |
||||
|
||||
# Next.js build output |
||||
|
||||
.next |
||||
out |
||||
|
||||
# Nuxt.js build / generate output |
||||
|
||||
.nuxt |
||||
dist |
||||
|
||||
# Gatsby files |
||||
|
||||
.cache/ |
||||
|
||||
# Comment in the public line in if your project uses Gatsby and not Next.js |
||||
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support |
||||
|
||||
# public |
||||
|
||||
# vuepress build output |
||||
|
||||
.vuepress/dist |
||||
|
||||
# vuepress v2.x temp and cache directory |
||||
|
||||
.temp |
||||
.cache |
||||
|
||||
# Docusaurus cache and generated files |
||||
|
||||
.docusaurus |
||||
|
||||
# Serverless directories |
||||
|
||||
.serverless/ |
||||
|
||||
# FuseBox cache |
||||
|
||||
.fusebox/ |
||||
|
||||
# DynamoDB Local files |
||||
|
||||
.dynamodb/ |
||||
|
||||
# TernJS port file |
||||
|
||||
.tern-port |
||||
|
||||
# Stores VSCode versions used for testing VSCode extensions |
||||
|
||||
.vscode-test |
||||
|
||||
# yarn v2 |
||||
|
||||
.yarn/cache |
||||
.yarn/unplugged |
||||
.yarn/build-state.yml |
||||
.yarn/install-state.gz |
||||
.pnp.\* |
||||
|
||||
# wrangler project |
||||
|
||||
.dev.vars |
||||
.wrangler/ |
||||
@ -0,0 +1,35 @@ |
||||
{ |
||||
"name": "@hyperlane-xyz/github-proxy", |
||||
"description": "Github proxy that adds the API key to requests", |
||||
"version": "5.1.0", |
||||
"scripts": { |
||||
"deploy": "wrangler deploy", |
||||
"deploy:staging": "wrangler deploy --env staging", |
||||
"deploy:key": "wrangler secret put GITHUB_API_KEY", |
||||
"dev": "wrangler dev", |
||||
"start": "wrangler dev", |
||||
"test": "vitest", |
||||
"prettier": "prettier --write ./src ./test", |
||||
"cf-typegen": "wrangler types" |
||||
}, |
||||
"type": "module", |
||||
"homepage": "https://www.hyperlane.xyz", |
||||
"repository": "https://github.com/hyperlane-xyz/hyperlane-monorepo", |
||||
"keywords": [ |
||||
"Hyperlane", |
||||
"Github", |
||||
"Proxy", |
||||
"Typescript" |
||||
], |
||||
"license": "Apache-2.0", |
||||
"devDependencies": { |
||||
"@cloudflare/vitest-pool-workers": "^0.4.5", |
||||
"@cloudflare/workers-types": "^4.20240821.1", |
||||
"@faker-js/faker": "^8.4.1", |
||||
"chai": "4.5.0", |
||||
"prettier": "^2.8.8", |
||||
"typescript": "5.3.3", |
||||
"vitest": "1.4.0", |
||||
"wrangler": "^3.74.0" |
||||
} |
||||
} |
||||
@ -0,0 +1 @@ |
||||
export const DISALLOWED_URL_MSG = 'Origin not allowed'; |
||||
@ -0,0 +1,26 @@ |
||||
import { DISALLOWED_URL_MSG } from './errors.js'; |
||||
|
||||
const GITHUB_API_ALLOWLIST = [ |
||||
'/repos/hyperlane-xyz/hyperlane-registry/git/trees/main', |
||||
]; |
||||
const GITPUB_API_HOST = 'https://api.github.com'; |
||||
export default { |
||||
async fetch(request, env, _ctx): Promise<Response> { |
||||
const apiUrlPath = new URL(request.url).pathname; |
||||
const isAllowed = GITHUB_API_ALLOWLIST.includes(apiUrlPath); |
||||
if (!isAllowed) { |
||||
return new Response(DISALLOWED_URL_MSG, { status: 401 }); |
||||
} |
||||
|
||||
const apiUrl = new URL(`${GITPUB_API_HOST}${apiUrlPath}`); |
||||
return fetch(apiUrl, { |
||||
method: 'GET', |
||||
headers: { |
||||
'Content-Type': 'application/json', |
||||
'User-Agent': 'Hyperlane-Github-Proxy', |
||||
'X-GitHub-Api-Version': '2022-11-28', |
||||
Authorization: `Bearer ${env.GITHUB_API_KEY}`, |
||||
}, |
||||
}); |
||||
}, |
||||
} satisfies ExportedHandler<Env>; |
||||
@ -0,0 +1,34 @@ |
||||
import { faker } from '@faker-js/faker'; |
||||
import { SELF } from 'cloudflare:test'; |
||||
import { describe, expect, it } from 'vitest'; |
||||
|
||||
import { DISALLOWED_URL_MSG } from '../src/errors.js'; |
||||
|
||||
describe('Hello World worker', () => { |
||||
it('returns empty response if pathname provided is not a valid api url', async () => { |
||||
const results = await SELF.fetch('https://example.com/favicon.ico'); |
||||
|
||||
expect(results.status).toBe(401); |
||||
expect(await results.text()).toBe(DISALLOWED_URL_MSG); |
||||
}); |
||||
|
||||
it('returns empty response if origin is not on allowlist', async () => { |
||||
const results = await SELF.fetch( |
||||
'https://example.com/repos/custom-hyperlane-xyz/hyperlane-registry/git/trees/main?recursive=true', |
||||
); |
||||
|
||||
expect(results.status).toBe(401); |
||||
expect(await results.text()).toBe(DISALLOWED_URL_MSG); |
||||
}); |
||||
|
||||
it('returns empty response if origin is not on allowlist (with faker 200 tests)', async () => { |
||||
for (let i = 0; i < 200; i++) { |
||||
const results = await SELF.fetch( |
||||
`https://example.com/${faker.internet.url}`, |
||||
); |
||||
|
||||
expect(results.status).toBe(401); |
||||
expect(await results.text()).toBe(DISALLOWED_URL_MSG); |
||||
} |
||||
}); |
||||
}); |
||||
@ -0,0 +1,11 @@ |
||||
{ |
||||
"extends": "../tsconfig.json", |
||||
"compilerOptions": { |
||||
"types": [ |
||||
"@cloudflare/workers-types/experimental", |
||||
"@cloudflare/vitest-pool-workers" |
||||
] |
||||
}, |
||||
"include": ["./**/*.ts", "../src/env.d.ts"], |
||||
"exclude": [] |
||||
} |
||||
@ -0,0 +1,15 @@ |
||||
{ |
||||
"extends": "../../tsconfig.json", |
||||
"compilerOptions": { |
||||
"types": [ |
||||
"@cloudflare/workers-types" |
||||
] /* Specify type package names to be included without being referenced in a source file. */, |
||||
|
||||
"noEmit": true /* Disable emitting files from a compilation. */, |
||||
|
||||
"isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */, |
||||
"allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */ |
||||
}, |
||||
"exclude": ["test"], |
||||
"include": ["worker-configuration.d.ts", "src/**/*.ts"] |
||||
} |
||||
@ -0,0 +1,11 @@ |
||||
import { defineWorkersConfig } from '@cloudflare/vitest-pool-workers/config'; |
||||
|
||||
export default defineWorkersConfig({ |
||||
test: { |
||||
poolOptions: { |
||||
workers: { |
||||
wrangler: { configPath: './wrangler.toml' }, |
||||
}, |
||||
}, |
||||
}, |
||||
}); |
||||
@ -0,0 +1,6 @@ |
||||
// Generated by Wrangler on Tue Sep 03 2024 19:01:13 GMT-0400 (Eastern Daylight Time)
|
||||
// by running `wrangler types`
|
||||
|
||||
interface Env { |
||||
GITHUB_API_KEY: string; |
||||
} |
||||
@ -0,0 +1,24 @@ |
||||
#:schema node_modules/wrangler/config-schema.json |
||||
name = "github-proxy-prod" |
||||
main = "src/index.ts" |
||||
compatibility_date = "2024-08-21" |
||||
compatibility_flags = ["nodejs_compat"] |
||||
workers_dev = false |
||||
routes = [ |
||||
{ pattern = "proxy.hyperlane.xyz", custom_domain = true } |
||||
] |
||||
|
||||
# CPU limit - 10ms keeps us within the free tier. As of 9/6/2024, the median CPU time is ~1ms |
||||
[limits] |
||||
cpu_ms = 10 |
||||
|
||||
# Automatically place your workloads in an optimal location to minimize latency. |
||||
# If you are running back-end logic in a Worker, running it closer to your back-end infrastructure |
||||
# rather than the end user may result in better performance. |
||||
# Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement |
||||
[placement] |
||||
mode = "smart" |
||||
|
||||
[env.staging] |
||||
name = "github-proxy-staging" |
||||
workers_dev = true |
||||
Loading…
Reference in new issue