puppeteer_scrape_cdn.ts

import Puppeteer from 'puppeteer';
import { mapSeries } from 'bluebird';
import fs from 'fs';

const url = 'https://cdnjs.com/libraries';

async function getNewPages(browser: Puppeteer.Browser, page: Puppeteer.Page) {
  console.log(`Getting libraries...`);
  const urls = await page.$$eval('.title a', (els) =>
    els.map((el) => (el as HTMLAnchorElement).href),
  );
  return await mapSeries(urls, async function (url) {
    console.log(`Opening ${url}...`);
    const page = await browser.newPage();
    await page.goto(url);
    return page;
  });
}

(async function () {
  const browser = await Puppeteer.launch();

  const page = await browser.newPage();
  await page.goto(url);

  const pages = await getNewPages(browser, page);

  const scriptStrs: string[] = await mapSeries(pages, async (page) => {
    console.log(`Getting all scripts from ${page.url()}`);
    return await page.evaluate(function () {
      const els = [...document.querySelectorAll<HTMLSpanElement>('.asset')];
      return els
        .map((el) => `<script src="${el.innerText.trim()}"></script>`)
        .filter((str) =>
          ['.scss', '.map', '.d.ts', '.woff', '.tiff'].every(
            (ext) => !str.includes(ext),
          ),
        )
        .join('\n');
    });
  });

  console.log('Writing script strings...');
  const writeStream = fs.createWriteStream('./shell_tests/scripts', {
    encoding: 'utf8',
  });
  writeStream.write(scriptStrs.join('\n'));
  writeStream.close();

  await browser.close();
})();