extract-urls.ts

// save as extract-urls.ts
// Usage: node extract-urls.ts <extension_id> <version>
import fs from 'fs';
import path from 'path';
import AdmZip from 'adm-zip';

// Known genesis hashes
const clusters = {
  mainnet: '5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp',
  testnet: '4uhcVJyU9pJkvQyS88uRDiswHXSCkY3z',
  devnet: 'EtWTRABZaYq6iMfeYKouRu166VU2xqa1'
};

const excludedHostnames = ['github.com'];

async function downloadExtension(extId: string, chromeVersion: string) {
  const url = `https://clients2.google.com/service/update2/crx?response=redirect&prodversion=${chromeVersion}&acceptformat=crx2,crx3&x=id%3D${extId}%26uc`;

  console.log('[*] Downloading:', url);

  const res = await fetch(url);
  if (!res.ok) throw new Error('Failed to download extension');

  return Buffer.from(await res.arrayBuffer());
}

const checkSolanaRpc = async (url: string, timeout = 3000) => {
  try {
    const res = await fetch(url, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        jsonrpc: '2.0',
        id: 1,
        method: 'getGenesisHash'
      }),
      signal: AbortSignal.timeout(timeout)
    });

    if (!res.ok) return false;

    const data = await res.json();
    const hash = data.result;

    for (const [cluster, knownHash] of Object.entries(clusters)) {
      if (hash.startsWith(knownHash)) {
        return cluster;
      }
    }
  } catch {}
  return false;
};

function stripCrxHeader(buffer: Buffer) {
  // CRX file starts with magic number "Cr24" (43 72 32 34)
  if (buffer.readUInt32LE(0) !== 0x34327243) {
    throw new Error('Invalid CRX file');
  }

  const version = buffer.readUInt32LE(4);
  let zipStartOffset = 0;

  if (version === 2) {
    const pubKeyLength = buffer.readUInt32LE(8);
    const sigLength = buffer.readUInt32LE(12);
    zipStartOffset = 16 + pubKeyLength + sigLength;
  } else if (version === 3) {
    const headerSize = buffer.readUInt32LE(8);
    zipStartOffset = 12 + headerSize;
  } else {
    throw new Error('Unsupported CRX version: ' + version);
  }

  return buffer.subarray(zipStartOffset);
}

async function extractValidUrls(text: string): Promise<string[]> {
  const urlRegex = /\bhttps?:\/\/[^\s"'()<>{}\x00]+/gi;
  const matches = text.match(urlRegex) || [];

  return matches
    .map((url) => {
      try {
        const parsed = new URL(url);
        if (
          !parsed.hostname.includes('.') ||
          excludedHostnames.includes(parsed.hostname) ||
          /[*$]/.test(url)
        )
          return null;
        // Normalize URL and remove trailing slash
        return parsed.toString().replace(/\/$/, '');
      } catch {
        return null;
      }
    })
    .filter((u) => u !== null);
}

async function extractUrlsFromExtension(crxPath: string) {
  const crxBuffer = fs.readFileSync(crxPath);
  const zipBuffer = stripCrxHeader(crxBuffer); // remove CRX header

  const zip = new AdmZip(zipBuffer);
  const entries = zip.getEntries();

  const urls = new Set<string>();

  // Step1: Rough match of http/https URLs
  for (const entry of entries) {
    if (entry.isDirectory) continue;
    const content = entry.getData().toString('utf8');
    const matches = await extractValidUrls(content);
    for (const url of matches) urls.add(url);
  }

  const uniqUrls = [...urls];

  // Step 2: Check Solana RPC endpoints in parallel
  const checkPromises = uniqUrls.map(async (url) => {
    const start = performance.now();
    const cluster = await checkSolanaRpc(url);
    if (cluster) {
      const elapsed = parseFloat((performance.now() - start).toFixed(2));
      console.log(`\n[+] Found URL(${cluster}):`, url, 'time:', elapsed, 'ms');
      return [url, elapsed, cluster];
    }
    return null;
  });

  const results = await Promise.allSettled(checkPromises);

  // Step 3: Filter only working RPC URLs
  return results
    .filter((r) => r.status === 'fulfilled' && r.value)
    .map((r: any) => r.value);
}

(async () => {
  const extId = process.argv[2];
  const chromeVersion = process.argv[3] || '139.0.7258.128';

  if (!extId) {
    console.log('Usage: node extract-urls.js <extension_id> <version>');
    process.exit(1);
  }

  const cacheDir = path.resolve('cache');
  if (!fs.existsSync(cacheDir)) {
    fs.mkdirSync(cacheDir);
  }

  const cacheFile = path.join(cacheDir, `${extId}.crx`);

  try {
    if (!fs.existsSync(cacheFile)) {
      console.log('[*] No cache found, downloading...');
      const buffer = await downloadExtension(extId, chromeVersion);
      // @ts-ignore
      fs.writeFileSync(cacheFile, buffer);
      console.log('[*] Saved to cache:', cacheFile);
    } else {
      console.log('[*] Using cached file:', cacheFile);
    }

    const urls = await extractUrlsFromExtension(cacheFile);
    const grouped: Record<string, [string, number, string][]> = urls.reduce(
      (acc, [url, time, cluster]) => {
        if (!acc[cluster]) acc[cluster] = [];
        acc[cluster].push([url, time, cluster]);
        return acc;
      },
      {}
    );

    console.log('\n[+] Extracted URLs:');
    for (const cluster in grouped) {
      console.log('\nCluster:', cluster);
      grouped[cluster]
        .sort((a, b) => a[1] - b[1])
        .forEach((u) => console.log('\t', u[0], u[1], 'ms'));
    }
  } catch (err) {
    console.error('Error:', err);
  }
})();