Cloudflare: Curl vs node-libcurl vs axios vs node-fetch

test.js

const logger = console;

const main = async () => {
  logger.info("Testing curl exec");
  await testCurlExec();
  logger.info("Testing node-libcurl");
  await testNodeLibCurl();
  logger.info("Testing axios");
  await testAxios();
  logger.info("Testing node-fetch");
  await testNodeFetch();
};

const isGood = data => {
  return data.indexOf(`<script src="/static/js/main.`) > -1;
};

const testCurlExec = async () => {
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  
  try {
    let command =
      "curl " +
      '--header "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" ' +
      '--header "Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3" ' +
      '--header "Connection: close" ' +
      '--header "Te: trailers" ' +
      '--header "Upgrade-Insecure-Requests: 1" ' +
      '--user-agent "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0" ' +
      "https://all-access.wax.io/";

    const { stdout } = await exec(command);
    // logger.debug(stdout);
    logger.info(isGood(stdout) ? "Page good" : "Page bad");
  } catch (err) {
    logger.error("Can't execute curl: " + err);
  }
};

const testNodeLibCurl = async () => {
  try {
    const { curly } = require("node-libcurl");

    const res = await curly.get("https://all-access.wax.io/", {
      userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0",
      httpHeader: [
        "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
        "Upgrade-Insecure-Requests: 1",
        "Te: trailers",
        "Upgrade-Insecure-Requests: 1",
        "Connection: close",
      ],
      // acceptEncoding: "gzip, deflate", // doesn't matter
      sslVerifyPeer: false, // if true also doesn't work
    });

    // logger.debug(res.data); // error code: 1020
    logger.info(isGood(res.data) ? "Page good" : "Page bad");
  } catch (err) {
    logger.error("Curly error: " + err);
  }
};

const testAxios = async () => {
  const axios = require("axios");

  try {
    let res = await axios({
      method: "GET",
      url: "https://all-access.wax.io/",
      headers: {
        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Language": "ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
        "Upgrade-Insecure-Requests": "1",
        Te: "trailers",
        "Upgrade-Insecure-Requests": "1",
        Connection: "close",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0",
      },
    });

    // logger.debug(res.data); // error code: 1020
    logger.info(isGood(res.data) ? "Page good" : "Page bad");
  } catch (err) {
    logger.error("axios error: " + err);
  }
};

const testNodeFetch = async () => {
  const fetch = require("node-fetch");

  try {
    let res = await fetch("https://all-access.wax.io/", {
      method: "GET",
      headers: {
        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Language": "ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
        "Upgrade-Insecure-Requests": "1",
        Te: "trailers",
        "Upgrade-Insecure-Requests": "1",
        Connection: "close",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0",
      },
    });

    let text = await res.text();
    // logger.debug(text); // error code: 1020
    logger.info(isGood(text) ? "Page good" : "Page bad");
  } catch (err) {
    logger.error("node-fetch: " + err);
  }
};

main();

thanks for the reply. I found that Puppeteer works right through CF so I've implemented a solution that uses it