crawler puppeeter.ts

import puppeteer, { Browser, Page } from 'puppeteer';

class Crawler {
  private browser: Browser;
  constructor(private readonly userAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36') {}

  async init() {
    // Launch the browser and open a new blank page
    this.browser = await puppeteer.launch({ headless: true });
  }

  async load(address: string) {
    const page = await this.browser.newPage();
    page.setUserAgent(this.userAgent);
    page.setRequestInterception(true);
    const url = `https://solscan.io/account/${address}#transfers`;

    // ignore images,css,font
    page.on('request', (request) => {
      if (['image', 'stylesheet', 'font'].indexOf(request.resourceType()) !== -1) {
        request.abort();
      } else {
        request.continue();
      }
    });

    return new Promise(async (resolve) => {
      page.on('requestfinished', async (req) => {
        if (req.url() !== `https://api-v2.solscan.io/v2/account/transfer?address=${address}&page=1&page_size=10&remove_spam=false&exclude_amount_zero=false`) return;
        const response = req.response();
        try {
          const data = await response.json();
          resolve(data);
        } catch {}
      });

      // Navigate the page to a URL.
      await page.goto(url);
      page.close();
    });

    // await page.evaluate()

    // Locate the paging to craw next data:
    // await page.locator('.next-button').click();
  }

  async destroy() {
    await this.browser.close();
  }
}

(async () => {
  const crawler = new Crawler();
  await crawler.init();

  const data = await crawler.load('HGPezSRSzZNXiBhzEXPw1gwCqsdbW7Psy5TjeyB78x8j');
  console.log(data);
})();