spacedrive/scripts/utils/fetch.mjs

import * as fs from "node:fs/promises";
import { dirname, join as joinPath } from "node:path";
import { env } from "node:process";
import { fileURLToPath } from "node:url";

import { getSystemProxy } from "os-proxy-config";
import { Agent, fetch, Headers, ProxyAgent } from "undici";

const CONNECT_TIMEOUT = 5 * 60 * 1000;
const __debug = env.NODE_ENV === "debug";
const __offline = env.OFFLINE === "true";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const cacheDir = joinPath(__dirname, ".tmp");

/** @type {Agent.Options} */
const agentOpts = {
  allowH2: !!env.HTTP2,
  connect: { timeout: CONNECT_TIMEOUT },
  connectTimeout: CONNECT_TIMEOUT,
  autoSelectFamily: true,
};

const { proxyUrl } = (await getSystemProxy()) ?? {};
const dispatcher = proxyUrl
  ? new ProxyAgent({
      ...agentOpts,
      proxyTls: { timeout: CONNECT_TIMEOUT },
      requestTls: { timeout: CONNECT_TIMEOUT },
      uri: proxyUrl,
    })
  : new Agent(agentOpts);

await fs.mkdir(cacheDir, { recursive: true, mode: 0o751 });

/**
 * @param {string} resource
 * @param {Headers} [headers]
 * @returns {Promise<null | {data: Buffer, header: [string, string] | undefined}>}
 */
async function getCache(resource, headers) {
  /** @type {Buffer | undefined} */
  let data;
  /** @type {[string, string] | undefined} */
  let header;

  // Don't cache in CI
  if (env.CI === "true" || env.NO_CACHE === "true") return null;

  if (headers)
    resource += Array.from(headers.entries())
      .filter(
        ([name]) => name !== "If-None-Match" && name !== "If-Modified-Since"
      )
      .flat()
      .join(":");
  try {
    const cache = JSON.parse(
      await fs.readFile(
        joinPath(cacheDir, Buffer.from(resource).toString("base64url")),
        {
          encoding: "utf8",
        }
      )
    );
    if (cache && typeof cache === "object") {
      if (cache.etag && typeof cache.etag === "string") {
        header = ["If-None-Match", cache.etag];
      } else if (
        cache.modifiedSince &&
        typeof cache.modifiedSince === "string"
      ) {
        header = ["If-Modified-Since", cache.modifiedSince];
      }

      if (cache.data && typeof cache.data === "string")
        data = Buffer.from(cache.data, "base64");
    }
  } catch (error) {
    if (__debug) {
      console.warn(`CACHE MISS: ${resource}`);
      console.error(error);
    }
  }

  return data ? { data, header } : null;
}

/**
 * @param {import('undici').Response} response
 * @param {string} resource
 * @param {Buffer} [cachedData]
 * @param {Headers} [headers]
 * @returns {Promise<Buffer>}
 */
async function setCache(response, resource, cachedData, headers) {
  const data = Buffer.from(await response.arrayBuffer());

  // Don't cache in CI
  if (env.CI === "true") return data;

  const etag = response.headers.get("ETag") || undefined;
  const modifiedSince = response.headers.get("Last-Modified") || undefined;
  if (headers)
    resource += Array.from(headers.entries())
      .filter(
        ([name]) => name !== "If-None-Match" && name !== "If-Modified-Since"
      )
      .flat()
      .join(":");

  if (response.status === 304 || (response.ok && data.length === 0)) {
    // Cache hit
    if (!cachedData) throw new Error("Empty cache hit ????");
    return cachedData;
  }

  try {
    await fs.writeFile(
      joinPath(cacheDir, Buffer.from(resource).toString("base64url")),
      JSON.stringify({
        etag,
        modifiedSince,
        data: data.toString("base64"),
      }),
      { mode: 0o640, flag: "w+" }
    );
  } catch (error) {
    if (__debug) {
      console.warn(`CACHE WRITE FAIL: ${resource}`);
      console.error(error);
    }
  }

  return data;
}

/**
 * @param {URL | string} resource
 * @param {Headers?} [headers]
 * @param {boolean} [preferCache]
 * @returns {Promise<Buffer>}
 */
export async function get(resource, headers, preferCache) {
  if (headers == null) headers = new Headers();
  if (resource instanceof URL) resource = resource.toString();

  const cache = await getCache(resource, headers);
  if (__offline) {
    if (cache?.data == null)
      throw new Error(
        `OFFLINE MODE: Cache for request ${resource} doesn't exist`
      );
    return cache.data;
  }
  if (preferCache && cache?.data != null) return cache.data;

  if (cache?.header) headers.append(...cache.header);

  if (__debug)
    console.log(`Downloading ${resource} ${cache?.data ? " (cached)" : ""}...`);
  const response = await fetch(resource, { dispatcher, headers });

  if (!response.ok) {
    if (cache?.data) {
      if (__debug)
        console.warn(
          `CACHE HIT due to fail: ${resource} ${response.statusText}`
        );
      return cache.data;
    }
    throw new Error(response.statusText);
  }

  return await setCache(response, resource, cache?.data, headers);
}