fetch.js

"use strict";

const pRetry = require("p-retry");

const _logger = require("ingo-app-log").get("fetch");

const fetch = require("node-fetch");

const uuid = require("uuid").v4;

const Stream = require("stream");

const LRU = require("lru-cache");
const { URL } = require("url");

const AGENT_CACHE = new LRU({ max: 50 });
let HttpsAgent;
let HttpAgent;

const USER_AGENT = `INGO/0.0.1 fetch`;

const getAgent = function getAgent(url, opts) {
  const parsedUrl = url instanceof URL ? url : new URL(typeof url === "string" ? url : url.url);
  const isHttps = parsedUrl.protocol === "https:";
  const origin = parsedUrl.origin;

  const key = origin;

  if (opts.agent != null) {
    // `agent: false` has special behavior!
    return opts.agent;
  }

  if (AGENT_CACHE.peek(key)) {
    return AGENT_CACHE.get(key);
  }

  if (isHttps && !HttpsAgent) {
    HttpsAgent = require("agentkeepalive").HttpsAgent;
  } else if (!isHttps && !HttpAgent) {
    HttpAgent = require("agentkeepalive");
  }

  // If opts.timeout is zero, set the agentTimeout to zero as well. A timeout
  // of zero disables the timeout behavior (OS limits still apply). Else, if
  // opts.timeout is a non-zero value, set it to timeout + 1, to ensure that
  // the node-fetch-npm timeout will always fire first, giving us more
  // consistent errors.
  const agentTimeout = opts.timeout === 0 ? 0 : opts.timeout + 1;

  const agent = isHttps
    ? new HttpsAgent({
        maxSockets: opts.maxSockets || 15,
        ca: opts.ca,
        cert: opts.cert,
        key: opts.key,
        localAddress: opts.localAddress,
        rejectUnauthorized: opts.strictSSL,
        timeout: agentTimeout
      })
    : new HttpAgent({
        maxSockets: opts.maxSockets || 15,
        localAddress: opts.localAddress,
        timeout: agentTimeout
      });
  AGENT_CACHE.set(key, agent);
  return agent;
};

const _fetch = async (url, opts = {}) => {
  const id = uuid();
  const logOpts = { method: opts.method || "GET", body: opts.body, headers: opts.headers };
  const logger = _logger.with({ id, url, opts: logOpts }); //XXX it is not the best idea to dump body and headers
  logger.trace("fetch request", typeof url === "string" ? url : url.toString(), "opts", logOpts);

  opts.timeout = opts.timeout == null ? 60 * 5 * 1000 : opts.timeout;

  const agent = getAgent(url, opts);

  const headers = Object.assign(
    {
      connection: agent ? "keep-alive" : "close",
      "user-agent": USER_AGENT
    },
    opts.headers || {}
  );

  opts.headers = new fetch.Headers(headers);
  opts.agent = agent;

  const isStream = opts.body instanceof Stream;

  const responseBody = await pRetry(
    async () => {
      const res = await fetch(url, opts);

      // assume all requests are immutable (thus do not rely on cookies for example)
      const isRetriable =
        !isStream &&
        (res.status === 408 || // Request Timeout
        res.status === 420 || // Enhance Your Calm (usually Twitter rate-limit)
        res.status === 429 || // Too Many Requests ("standard" rate-limiting)
          res.status >= 500); // Assume server errors are momentary hiccups

      if (isRetriable) {
        if (res.status >= 500) {
          logger.debug("response status", res.status);
          logger.debug("respose headers");
          for (const [name, value] of res.headers.entries()) {
            logger.debug("H", name, "=>", value);
          }
          try {
            logger.debug("response body", await res.json());
          } catch (err) {
            logger.error("could not decode response body", err);
          }
        }
        throw new Error(`Could not fetch ${url} status ${res.status}`);
      } else if (res.status >= 400) {
        const text = await res.text();
        throw new pRetry.AbortError(text || res.statusText);
      }

      const contentType = res.headers.get("content-type") || "text/plain";

      const result = contentType.includes("text/") ? await res.text() : await res.json();

      return result;
    },
    {
      retries: opts.retries == null ? 3 : opts.retries,
      onFailedAttempt(err) {
        logger.warn(`fetch attempt ${err.attemptNumber} (${err.retriesLeft} left)`, err);
      }
    }
  );

  return responseBody;
};

module.exports = {
  fetch: _fetch
};