Get scraped chains list from DB instead of registry (#106)

- Update Next and Hyperlane deps
- Query scraped chains from domains DB table

package.json

@@ -6,9 +6,9 @@
   "dependencies": {
     "@headlessui/react": "^1.7.17",
     "@hyperlane-xyz/registry": "2.5.0",
-    "@hyperlane-xyz/sdk": "3.13.0",
-    "@hyperlane-xyz/utils": "3.13.0",
-    "@hyperlane-xyz/widgets": "4.1.0",
+    "@hyperlane-xyz/sdk": "5.1.0",
+    "@hyperlane-xyz/utils": "5.1.0",
+    "@hyperlane-xyz/widgets": "5.1.0",
     "@metamask/jazzicon": "https://github.com/jmrossy/jazzicon#7a8df28974b4e81129bfbe3cab76308b889032a6",
     "@tanstack/react-query": "^5.35.5",
     "bignumber.js": "^9.1.2",
@@ -16,7 +16,7 @@
     "ethers": "^5.7.2",
     "formik": "^2.2.9",
     "graphql": "^16.6.0",
-    "next": "^13.4.19",
+    "next": "^13.5.6",
     "nextjs-cors": "^2.1.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
@@ -44,7 +44,7 @@
     "prettier": "^2.8.4",
     "tailwindcss": "^3.3.3",
     "ts-node": "^10.9.1",
-    "typescript": "^5.1.6"
+    "typescript": "^5.5.4"
   },
   "homepage": "https://www.hyperlane.xyz",
   "license": "Apache-2.0",

src/components/search/SearchFilterBar.tsx

@@ -5,11 +5,12 @@ import { useMemo, useState } from 'react';
 import { ChainMetadata } from '@hyperlane-xyz/sdk';
 import { arrayToObject } from '@hyperlane-xyz/utils';
 
+import { useScrapedChains } from '../../features/chains/queries/useScrapedChains';
 import {
   getChainDisplayName,
   isEvmChain,
   isPiChain,
-  isUnscrapedEvmChain,
+  isUnscrapedDbChain,
 } from '../../features/chains/utils';
 import GearIcon from '../../images/icons/gear.svg';
 import { useMultiProvider } from '../../store';
@@ -87,22 +88,23 @@ function ChainMultiSelector({
   onChangeValue: (value: string | null) => void;
   position?: string;
 }) {
+  const { scrapedChains } = useScrapedChains();
   const multiProvider = useMultiProvider();
   const { chains, mainnets, testnets } = useMemo(() => {
     const chains = Object.values(multiProvider.metadata);
     // Filtering to EVM is necessary to prevent errors until cosmos support is added
     // https://github.com/hyperlane-xyz/hyperlane-explorer/issues/61
-    const coreEvmChains = chains.filter(
+    const scrapedEvmChains = chains.filter(
       (c) =>
         isEvmChain(multiProvider, c.chainId) &&
-        !isPiChain(multiProvider, c.chainId) &&
-        !isUnscrapedEvmChain(multiProvider, c.chainId),
+        !isPiChain(multiProvider, scrapedChains, c.chainId) &&
+        !isUnscrapedDbChain(multiProvider, c.chainId),
     );
-    const mainnets = coreEvmChains.filter((c) => !c.isTestnet);
-    const testnets = coreEvmChains.filter((c) => !!c.isTestnet);
+    const mainnets = scrapedEvmChains.filter((c) => !c.isTestnet);
+    const testnets = scrapedEvmChains.filter((c) => !!c.isTestnet);
     // Return only evmChains because of graphql only accept query non-evm chains (with bigint type not string)
-    return { chains: coreEvmChains, mainnets, testnets };
-  }, [multiProvider]);
+    return { chains: scrapedEvmChains, mainnets, testnets };
+  }, [multiProvider, scrapedChains]);
 
   // Need local state as buffer before user hits apply
   const [checkedChains, setCheckedChains] = useState(

src/consts/config.ts

@@ -1,5 +1,3 @@
-import { CoreChain } from '@hyperlane-xyz/registry';
-
 const isDevMode = process?.env?.NODE_ENV === 'development';
 const version = process?.env?.NEXT_PUBLIC_VERSION ?? null;
 const explorerApiKeys = JSON.parse(process?.env?.EXPLORER_API_KEYS || '{}');
@@ -20,4 +18,4 @@ export const config: Config = Object.freeze({
 
 // Based on https://github.com/hyperlane-xyz/hyperlane-monorepo/blob/main/typescript/infra/config/environments/mainnet3/agent.ts
 // Based on https://github.com/hyperlane-xyz/hyperlane-monorepo/blob/main/typescript/infra/config/environments/testnet4/agent.ts
-export const unscrapedEvmChains = [CoreChain.proteustestnet, CoreChain.sei, CoreChain.viction];
+export const unscrapedChainsInDb = ['proteustestnet', 'sei', 'viction'];

src/features/api/getMessages.ts

@@ -9,7 +9,7 @@ import { MessagesQueryResult } from '../messages/queries/fragments';
 import { parseMessageQueryResult } from '../messages/queries/parse';
 
 import { ApiHandlerResult, ApiMessage, toApiMessage } from './types';
-import { failureResult, getMultiProvider, successResult } from './utils';
+import { failureResult, getMultiProvider, getScrapedChains, successResult } from './utils';
 
 export async function handler(
   req: NextApiRequest,
@@ -27,7 +27,9 @@ export async function handler(
   const result = await client.query<MessagesQueryResult>(query, variables).toPromise();
 
   const multiProvider = await getMultiProvider();
-  const messages = parseMessageQueryResult(multiProvider, result.data);
+  const scrapedChains = await getScrapedChains(client);
+
+  const messages = parseMessageQueryResult(multiProvider, scrapedChains, result.data);
   return successResult(messages.map(toApiMessage));
 }
 

src/features/api/getStatus.ts

@@ -10,7 +10,7 @@ import { parseMessageStubResult } from '../messages/queries/parse';
 
 import { parseQueryParams } from './getMessages';
 import { ApiHandlerResult } from './types';
-import { failureResult, getMultiProvider, successResult } from './utils';
+import { failureResult, getMultiProvider, getScrapedChains, successResult } from './utils';
 
 interface MessageStatusResult {
   id: string;
@@ -34,7 +34,9 @@ export async function handler(
   const result = await client.query<MessagesStubQueryResult>(query, variables).toPromise();
 
   const multiProvider = await getMultiProvider();
-  const messages = parseMessageStubResult(multiProvider, result.data);
+  const scrapedChains = await getScrapedChains(client);
+
+  const messages = parseMessageStubResult(multiProvider, scrapedChains, result.data);
 
   return successResult(messages.map((m) => ({ id: m.msgId, status: m.status })));
 }

src/features/api/searchMessages.ts

@@ -9,7 +9,7 @@ import { MessagesQueryResult } from '../messages/queries/fragments';
 import { parseMessageQueryResult } from '../messages/queries/parse';
 
 import { ApiHandlerResult, ApiMessage, toApiMessage } from './types';
-import { failureResult, getMultiProvider, successResult } from './utils';
+import { failureResult, getMultiProvider, getScrapedChains, successResult } from './utils';
 
 const SEARCH_QUERY_PARAM_NAME = 'query';
 
@@ -33,7 +33,9 @@ export async function handler(
   const result = await client.query<MessagesQueryResult>(query, variables).toPromise();
 
   const multiProvider = await getMultiProvider();
-  const messages = parseMessageQueryResult(multiProvider, result.data);
+  const scrapedChains = await getScrapedChains(client);
+
+  const messages = parseMessageQueryResult(multiProvider, scrapedChains, result.data);
 
   return successResult(messages.map(toApiMessage));
 }

src/features/api/utils.ts

@@ -1,6 +1,11 @@
+import { Client } from '@urql/core';
+
 import { GithubRegistry } from '@hyperlane-xyz/registry';
 import { MultiProvider } from '@hyperlane-xyz/sdk';
 
+import { logger } from '../../utils/logger';
+import { DOMAINS_QUERY, DomainsEntry } from '../chains/queries/fragments';
+
 export function successResult<R>(data: R): { success: true; data: R } {
   return { success: true, data };
 }
@@ -15,3 +20,9 @@ export async function getMultiProvider(): Promise<MultiProvider> {
   const chainMetadata = await registry.getMetadata();
   return new MultiProvider(chainMetadata);
 }
+
+export async function getScrapedChains(client: Client): Promise<Array<DomainsEntry>> {
+  logger.debug('Fetching list of scraped chains');
+  const result = await client.query<{ domain: Array<DomainsEntry> }>(DOMAINS_QUERY, {}).toPromise();
+  return result.data?.domain || [];
+}

src/features/chains/queries/fragments.ts

@@ -0,0 +1,21 @@
+export const DOMAINS_QUERY = `
+query @cached {
+  domain {
+    id
+    native_token
+    name
+    is_test_net
+    is_deprecated
+    chain_id
+  }
+}
+`;
+
+export interface DomainsEntry {
+  id: number; // domainId
+  native_token: string;
+  name: string;
+  is_test_net: boolean;
+  is_deprecated: boolean;
+  chain_id: string | number;
+}

src/features/chains/queries/useScrapedChains.ts

@@ -0,0 +1,30 @@
+import { useEffect } from 'react';
+import { useQuery } from 'urql';
+
+import { useStore } from '../../../store';
+
+import { DOMAINS_QUERY, DomainsEntry } from './fragments';
+
+export function useScrapedChains() {
+  const { scrapedChains, setScrapedChains } = useStore((s) => ({
+    scrapedChains: s.scrapedChains,
+    setScrapedChains: s.setScrapedChains,
+  }));
+
+  const [result] = useQuery<{ domain: Array<DomainsEntry> }>({
+    query: DOMAINS_QUERY,
+    pause: !!scrapedChains?.length,
+  });
+  const { data, fetching: isFetching, error } = result;
+
+  useEffect(() => {
+    if (!data) return;
+    setScrapedChains(data.domain);
+  }, [data, error, setScrapedChains]);
+
+  return {
+    scrapedChains,
+    isFetching,
+    isError: !!error,
+  };
+}

src/features/chains/utils.ts

@@ -1,11 +1,12 @@
-import { CoreChain, CoreChains, IRegistry } from '@hyperlane-xyz/registry';
+import { IRegistry } from '@hyperlane-xyz/registry';
 import { ChainMap, MultiProvider } from '@hyperlane-xyz/sdk';
 import { ProtocolType, toTitleCase } from '@hyperlane-xyz/utils';
 
-import { unscrapedEvmChains } from '../../consts/config';
+import { unscrapedChainsInDb } from '../../consts/config';
 import { Environment } from '../../consts/environments';
 
 import { ChainConfig } from './chainConfig';
+import { DomainsEntry } from './queries/fragments';
 
 export async function getMailboxAddress(
   chainName: string,
@@ -35,9 +36,15 @@ export function getChainEnvironment(multiProvider: MultiProvider, chainIdOrName:
   return isTestnet ? Environment.Testnet : Environment.Mainnet;
 }
 
-export function isPiChain(multiProvider: MultiProvider, chainIdOrName: number | string) {
+// Is a 'Permisionless Interop' chain (i.e. one not deployed and scraped by Abacus Works)
+export function isPiChain(
+  multiProvider: MultiProvider,
+  scrapedChains: DomainsEntry[],
+  chainIdOrName: number | string,
+) {
   const chainName = multiProvider.tryGetChainName(chainIdOrName);
-  return !chainName || !CoreChains.includes(chainName as CoreChain);
+  // Note: .trim() because one chain name in the DB has a trailing \n char for some reason
+  return !chainName || !scrapedChains.find((chain) => chain.name.trim() === chainName);
 }
 
 export function isEvmChain(multiProvider: MultiProvider, chainIdOrName: number | string) {
@@ -45,8 +52,8 @@ export function isEvmChain(multiProvider: MultiProvider, chainIdOrName: number |
   return protocol === ProtocolType.Ethereum;
 }
 
-// TODO: Remove once we fetch CoreChains dynamically from the DB https://github.com/hyperlane-xyz/hyperlane-explorer/issues/74
-export function isUnscrapedEvmChain(multiProvider: MultiProvider, chainIdOrName: number | string) {
+// TODO: Remove once all chains in the DB are scraped
+export function isUnscrapedDbChain(multiProvider: MultiProvider, chainIdOrName: number | string) {
   const chainName = multiProvider.tryGetChainName(chainIdOrName);
-  return chainName && unscrapedEvmChains.includes(chainName as CoreChain);
+  return chainName && unscrapedChainsInDb.includes(chainName);
 }

src/features/messages/pi-queries/usePiChainMessageQuery.ts

@@ -8,6 +8,7 @@ import { useReadyMultiProvider, useRegistry } from '../../../store';
 import { Message } from '../../../types';
 import { logger } from '../../../utils/logger';
 import { ChainConfig } from '../../chains/chainConfig';
+import { useScrapedChains } from '../../chains/queries/useScrapedChains';
 import { isEvmChain, isPiChain } from '../../chains/utils';
 import { isValidSearchQuery } from '../queries/useMessageQuery';
 
@@ -30,8 +31,10 @@ export function usePiChainMessageSearchQuery({
   piQueryType?: PiQueryType;
   pause: boolean;
 }) {
+  const { scrapedChains } = useScrapedChains();
   const multiProvider = useReadyMultiProvider();
   const registry = useRegistry();
+
   const { isLoading, isError, data } = useQuery({
     queryKey: [
       'usePiChainMessageSearchQuery',
@@ -51,7 +54,9 @@ export function usePiChainMessageSearchQuery({
       const query = { input: ensure0x(sanitizedInput) };
       const allChains = Object.values(multiProvider.metadata);
       const piChains = allChains.filter(
-        (c) => isEvmChain(multiProvider, c.chainId) && isPiChain(multiProvider, c.chainId),
+        (c) =>
+          isEvmChain(multiProvider, c.chainId) &&
+          isPiChain(multiProvider, scrapedChains, c.chainId),
       );
       try {
         const results = await Promise.allSettled(

src/features/messages/queries/parse.ts

@@ -3,6 +3,7 @@ import { MultiProvider } from '@hyperlane-xyz/sdk';
 import { Message, MessageStatus, MessageStub } from '../../../types';
 import { logger } from '../../../utils/logger';
 import { tryUtf8DecodeBytes } from '../../../utils/string';
+import { DomainsEntry } from '../../chains/queries/fragments';
 import { isPiChain } from '../../chains/utils';
 
 import { postgresByteaToString } from './encoding';
@@ -22,29 +23,35 @@ import {
 
 export function parseMessageStubResult(
   multiProvider: MultiProvider,
+  scrapedChains: DomainsEntry[],
   data: MessagesStubQueryResult | undefined,
 ): MessageStub[] {
   if (!data || !Object.keys(data).length) return [];
   return Object.values(data)
     .flat()
-    .map((m) => parseMessageStub(multiProvider, m))
+    .map((m) => parseMessageStub(multiProvider, scrapedChains, m))
     .filter((m): m is MessageStub => !!m)
     .sort((a, b) => b.origin.timestamp - a.origin.timestamp);
 }
 
 export function parseMessageQueryResult(
   multiProvider: MultiProvider,
+  scrapedChains: DomainsEntry[],
   data: MessagesQueryResult | undefined,
 ): Message[] {
   if (!data || !Object.keys(data).length) return [];
   return Object.values(data)
     .flat()
-    .map((m) => parseMessage(multiProvider, m))
+    .map((m) => parseMessage(multiProvider, scrapedChains, m))
     .filter((m): m is Message => !!m)
     .sort((a, b) => b.origin.timestamp - a.origin.timestamp);
 }
 
-function parseMessageStub(multiProvider: MultiProvider, m: MessageStubEntry): MessageStub | null {
+function parseMessageStub(
+  multiProvider: MultiProvider,
+  scrapedChains: DomainsEntry[],
+  m: MessageStubEntry,
+): MessageStub | null {
   try {
     const destinationDomainId = m.destination_domain_id;
     let destinationChainId =
@@ -54,7 +61,8 @@ function parseMessageStub(multiProvider: MultiProvider, m: MessageStubEntry): Me
       destinationChainId = destinationDomainId;
     }
     const isPiMsg =
-      isPiChain(multiProvider, m.origin_chain_id) || isPiChain(multiProvider, destinationChainId);
+      isPiChain(multiProvider, scrapedChains, m.origin_chain_id) ||
+      isPiChain(multiProvider, scrapedChains, destinationChainId);
 
     return {
       status: getMessageStatus(m),
@@ -87,9 +95,13 @@ function parseMessageStub(multiProvider: MultiProvider, m: MessageStubEntry): Me
   }
 }
 
-function parseMessage(multiProvider: MultiProvider, m: MessageEntry): Message | null {
+function parseMessage(
+  multiProvider: MultiProvider,
+  scrapedChains: DomainsEntry[],
+  m: MessageEntry,
+): Message | null {
   try {
-    const stub = parseMessageStub(multiProvider, m);
+    const stub = parseMessageStub(multiProvider, scrapedChains, m);
     if (!stub) throw new Error('Message stub required');
 
     const body = postgresByteaToString(m.message_body ?? '');

src/features/messages/queries/useMessageQuery.ts

@@ -6,6 +6,7 @@ import { isAddressEvm, isValidTransactionHashEvm } from '@hyperlane-xyz/utils';
 import { useMultiProvider } from '../../../store';
 import { MessageStatus } from '../../../types';
 import { useInterval } from '../../../utils/useInterval';
+import { useScrapedChains } from '../../chains/queries/useScrapedChains';
 
 import { MessageIdentifierType, buildMessageQuery, buildMessageSearchQuery } from './build';
 import { MessagesQueryResult, MessagesStubQueryResult } from './fragments';
@@ -29,6 +30,8 @@ export function useMessageSearchQuery(
   startTimeFilter: number | null,
   endTimeFilter: number | null,
 ) {
+  const { scrapedChains } = useScrapedChains();
+
   const hasInput = !!sanitizedInput;
   const isValidInput = hasInput ? isValidSearchQuery(sanitizedInput, true) : true;
 
@@ -54,8 +57,8 @@ export function useMessageSearchQuery(
   // Parse results
   const multiProvider = useMultiProvider();
   const messageList = useMemo(
-    () => parseMessageStubResult(multiProvider, data),
-    [multiProvider, data],
+    () => parseMessageStubResult(multiProvider, scrapedChains, data),
+    [multiProvider, scrapedChains, data],
   );
   const isMessagesFound = messageList.length > 0;
 
@@ -77,6 +80,8 @@ export function useMessageSearchQuery(
 }
 
 export function useMessageQuery({ messageId, pause }: { messageId: string; pause: boolean }) {
+  const { scrapedChains } = useScrapedChains();
+
   // Assemble GraphQL Query
   const { query, variables } = buildMessageQuery(MessageIdentifierType.Id, messageId, 1);
 
@@ -90,8 +95,8 @@ export function useMessageQuery({ messageId, pause }: { messageId: string; pause
   // Parse results
   const multiProvider = useMultiProvider();
   const messageList = useMemo(
-    () => parseMessageQueryResult(multiProvider, data),
-    [multiProvider, data],
+    () => parseMessageQueryResult(multiProvider, scrapedChains, data),
+    [multiProvider, scrapedChains, data],
   );
   const isMessageFound = messageList.length > 0;
   const message = isMessageFound ? messageList[0] : null;

src/store.ts

@@ -5,6 +5,7 @@ import { GithubRegistry, IRegistry } from '@hyperlane-xyz/registry';
 import { ChainMap, MultiProvider } from '@hyperlane-xyz/sdk';
 
 import { ChainConfig } from './features/chains/chainConfig';
+import { DomainsEntry } from './features/chains/queries/fragments';
 import { logger } from './utils/logger';
 
 // Increment this when persist state has breaking changes
@@ -13,6 +14,8 @@ const PERSIST_STATE_VERSION = 1;
 // Keeping everything here for now as state is simple
 // Will refactor into slices as necessary
 interface AppState {
+  scrapedChains: Array<DomainsEntry>;
+  setScrapedChains: (chains: Array<DomainsEntry>) => void;
   chainConfigs: ChainMap<ChainConfig>;
   setChainConfigs: (configs: ChainMap<ChainConfig>) => void;
   multiProvider: MultiProvider;
@@ -26,6 +29,8 @@ interface AppState {
 export const useStore = create<AppState>()(
   persist(
     (set, get) => ({
+      scrapedChains: [],
+      setScrapedChains: (chains: Array<DomainsEntry>) => set({ scrapedChains: chains }),
       chainConfigs: {},
       setChainConfigs: async (configs: ChainMap<ChainConfig>) => {
         const multiProvider = await buildMultiProvider(get().registry, configs);