import * as fs from "node:fs/promises"; import { dirname, join as joinPath } from "node:path"; import { env } from "node:process"; import { fileURLToPath } from "node:url"; import { getSystemProxy } from "os-proxy-config"; import { Agent, fetch, Headers, ProxyAgent } from "undici"; const CONNECT_TIMEOUT = 5 * 60 * 1000; const __debug = env.NODE_ENV === "debug"; const __offline = env.OFFLINE === "true"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const cacheDir = joinPath(__dirname, ".tmp"); /** @type {Agent.Options} */ const agentOpts = { allowH2: !!env.HTTP2, connect: { timeout: CONNECT_TIMEOUT }, connectTimeout: CONNECT_TIMEOUT, autoSelectFamily: true, }; const { proxyUrl } = (await getSystemProxy()) ?? {}; const dispatcher = proxyUrl ? new ProxyAgent({ ...agentOpts, proxyTls: { timeout: CONNECT_TIMEOUT }, requestTls: { timeout: CONNECT_TIMEOUT }, uri: proxyUrl, }) : new Agent(agentOpts); await fs.mkdir(cacheDir, { recursive: true, mode: 0o751 }); /** * @param {string} resource * @param {Headers} [headers] * @returns {Promise} */ async function getCache(resource, headers) { /** @type {Buffer | undefined} */ let data; /** @type {[string, string] | undefined} */ let header; // Don't cache in CI if (env.CI === "true" || env.NO_CACHE === "true") return null; if (headers) resource += Array.from(headers.entries()) .filter( ([name]) => name !== "If-None-Match" && name !== "If-Modified-Since" ) .flat() .join(":"); try { const cache = JSON.parse( await fs.readFile( joinPath(cacheDir, Buffer.from(resource).toString("base64url")), { encoding: "utf8", } ) ); if (cache && typeof cache === "object") { if (cache.etag && typeof cache.etag === "string") { header = ["If-None-Match", cache.etag]; } else if ( cache.modifiedSince && typeof cache.modifiedSince === "string" ) { header = ["If-Modified-Since", cache.modifiedSince]; } if (cache.data && typeof cache.data === "string") data = Buffer.from(cache.data, "base64"); } } catch (error) { if (__debug) { console.warn(`CACHE MISS: ${resource}`); console.error(error); } } return data ? { data, header } : null; } /** * @param {import('undici').Response} response * @param {string} resource * @param {Buffer} [cachedData] * @param {Headers} [headers] * @returns {Promise} */ async function setCache(response, resource, cachedData, headers) { const data = Buffer.from(await response.arrayBuffer()); // Don't cache in CI if (env.CI === "true") return data; const etag = response.headers.get("ETag") || undefined; const modifiedSince = response.headers.get("Last-Modified") || undefined; if (headers) resource += Array.from(headers.entries()) .filter( ([name]) => name !== "If-None-Match" && name !== "If-Modified-Since" ) .flat() .join(":"); if (response.status === 304 || (response.ok && data.length === 0)) { // Cache hit if (!cachedData) throw new Error("Empty cache hit ????"); return cachedData; } try { await fs.writeFile( joinPath(cacheDir, Buffer.from(resource).toString("base64url")), JSON.stringify({ etag, modifiedSince, data: data.toString("base64"), }), { mode: 0o640, flag: "w+" } ); } catch (error) { if (__debug) { console.warn(`CACHE WRITE FAIL: ${resource}`); console.error(error); } } return data; } /** * @param {URL | string} resource * @param {Headers?} [headers] * @param {boolean} [preferCache] * @returns {Promise} */ export async function get(resource, headers, preferCache) { if (headers == null) headers = new Headers(); if (resource instanceof URL) resource = resource.toString(); const cache = await getCache(resource, headers); if (__offline) { if (cache?.data == null) throw new Error( `OFFLINE MODE: Cache for request ${resource} doesn't exist` ); return cache.data; } if (preferCache && cache?.data != null) return cache.data; if (cache?.header) headers.append(...cache.header); if (__debug) console.log(`Downloading ${resource} ${cache?.data ? " (cached)" : ""}...`); const response = await fetch(resource, { dispatcher, headers }); if (!response.ok) { if (cache?.data) { if (__debug) console.warn( `CACHE HIT due to fail: ${resource} ${response.statusText}` ); return cache.data; } throw new Error(response.statusText); } return await setCache(response, resource, cache?.data, headers); }