custom cache action

.github/actions/rust-cache/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+/target
+!dist

.github/actions/rust-cache/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+publish = false
+name = "rust-cache"
+version = "0.1.0"
+authors = ["Arpad Borsos <arpad.borsos@googlemail.com>"]
+edition = "2018"
+
+[dev-dependencies]
+reqwest = "0.11.0"
+actix-web = { git = "https://github.com/actix/actix-web.git", rev = "bd26083f333ecf63e3eb444748250364ce124f5e" }

.github/actions/rust-cache/action.yml

@@ -0,0 +1,30 @@
+name: "Rust Cache"
+description: "A GitHub Action that implements smart caching for rust/cargo projects with sensible defaults."
+author: "Arpad Borsos <arpad.borsos@googlemail.com>"
+inputs:
+  key:
+    description: 'An explicit key for restoring and saving the cache'
+    required: true
+  restore-keys:
+    description: 'An ordered list of keys to use for restoring the cache if no cache hit occurred for key'
+    required: false
+  working-directory:
+    description: "The working directory this action should operate in"
+    required: false
+  target-dir:
+    description: "The target dir that should be cleaned and persisted, defaults to `./target`"
+    required: false
+  cache-on-failure:
+    description: "Cache even if the build fails. Defaults to false"
+    required: false
+outputs:
+  cache-hit:
+    description: "A boolean value that indicates an exact match was found"
+runs:
+  using: "node12"
+  main: "dist/restore/index.js"
+  post: "dist/save/index.js"
+  post-if: "success() || env.CACHE_ON_FAILURE == 'true'"
+branding:
+  icon: "archive"
+  color: "gray-dark"

.github/actions/rust-cache/package.json

@@ -0,0 +1,38 @@
+{
+  "private": true,
+  "name": "rust-cache",
+  "version": "1.4.0",
+  "description": "A GitHub Action that implements smart caching for rust/cargo projects with sensible defaults.",
+  "keywords": [
+    "actions",
+    "rust",
+    "cache"
+  ],
+  "author": "Arpad Borsos <arpad.borsos@googlemail.com>",
+  "license": "LGPL-3.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Swatinem/rust-cache.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Swatinem/rust-cache/issues"
+  },
+  "funding": {
+    "url": "https://github.com/sponsors/Swatinem"
+  },
+  "homepage": "https://github.com/Swatinem/rust-cache#readme",
+  "dependencies": {
+    "@actions/cache": "^2.0.2",
+    "@actions/core": "^1.6.0",
+    "@actions/exec": "^1.1.1",
+    "@actions/glob": "^0.2.1",
+    "@actions/io": "^1.1.2"
+  },
+  "devDependencies": {
+    "@vercel/ncc": "^0.33.3",
+    "typescript": "4.6.3"
+  },
+  "scripts": {
+    "prepare": "ncc build --target es2020 -o dist/restore src/restore.ts && ncc build --target es2020 -o dist/save src/save.ts"
+  }
+}

.github/actions/rust-cache/src/common.ts

@@ -0,0 +1,281 @@
+import * as core from "@actions/core";
+import * as exec from "@actions/exec";
+import * as glob from "@actions/glob";
+import * as io from "@actions/io";
+import crypto from "crypto";
+import fs from "fs";
+import os from "os";
+import path from "path";
+
+process.on("uncaughtException", (e) => {
+  core.info(`[warning] ${e.message}`);
+  if (e.stack) {
+    core.info(e.stack);
+  }
+});
+
+const cwd = core.getInput("working-directory");
+// TODO: this could be read from .cargo config file directly
+const targetDir = core.getInput("target-dir") || "./target";
+if (cwd) {
+  process.chdir(cwd);
+}
+
+export const stateBins = "RUST_CACHE_BINS";
+export const stateKey = "RUST_CACHE_KEY";
+const stateHash = "RUST_CACHE_HASH";
+
+const home = os.homedir();
+const cargoHome = process.env.CARGO_HOME || path.join(home, ".cargo");
+export const paths = {
+  cargoHome,
+  index: path.join(cargoHome, "registry/index"),
+  cache: path.join(cargoHome, "registry/cache"),
+  git: path.join(cargoHome, "git"),
+  target: targetDir,
+};
+
+interface CacheConfig {
+  paths: Array<string>;
+  key: string;
+  restoreKeys: Array<string>;
+}
+
+const RefKey = "GITHUB_REF";
+
+export function isValidEvent(): boolean {
+  return RefKey in process.env && Boolean(process.env[RefKey]);
+}
+
+export async function getCacheConfig(): Promise<CacheConfig> {
+  let lockHash = core.getState(stateHash);
+  if (!lockHash) {
+    lockHash = await getLockfileHash();
+    core.saveState(stateHash, lockHash);
+  }
+
+  return {
+    paths: [
+      path.join(cargoHome, "bin"),
+      path.join(cargoHome, ".crates2.json"),
+      path.join(cargoHome, ".crates.toml"),
+      paths.git,
+      paths.cache,
+      paths.index,
+      paths.target,
+    ],
+    key: core.getInput("key"),
+    restoreKeys: getInputAsArray("restore-keys"),
+  };
+}
+
+export async function getCargoBins(): Promise<Set<string>> {
+  try {
+    const {
+      installs,
+    }: { installs: { [key: string]: { bins: Array<string> } } } = JSON.parse(
+      await fs.promises.readFile(
+        path.join(paths.cargoHome, ".crates2.json"),
+        "utf8"
+      )
+    );
+    const bins = new Set<string>();
+    for (const pkg of Object.values(installs)) {
+      for (const bin of pkg.bins) {
+        bins.add(bin);
+      }
+    }
+    return bins;
+  } catch {
+    return new Set<string>();
+  }
+}
+
+async function getRustKey(): Promise<string> {
+  const rustc = await getRustVersion();
+  return `${rustc.release}-${rustc.host}-${rustc["commit-hash"].slice(0, 12)}`;
+}
+
+interface RustVersion {
+  host: string;
+  release: string;
+  "commit-hash": string;
+}
+
+async function getRustVersion(): Promise<RustVersion> {
+  const stdout = await getCmdOutput("rustc", ["-vV"]);
+  let splits = stdout
+    .split(/[\n\r]+/)
+    .filter(Boolean)
+    .map((s) => s.split(":").map((s) => s.trim()))
+    .filter((s) => s.length === 2);
+  return Object.fromEntries(splits);
+}
+
+export async function getCmdOutput(
+  cmd: string,
+  args: Array<string> = [],
+  options: exec.ExecOptions = {}
+): Promise<string> {
+  let stdout = "";
+  await exec.exec(cmd, args, {
+    silent: true,
+    listeners: {
+      stdout(data) {
+        stdout += data.toString();
+      },
+    },
+    ...options,
+  });
+  return stdout;
+}
+
+async function getLockfileHash(): Promise<string> {
+  const globber = await glob.create(
+    "**/Cargo.toml\n**/Cargo.lock\nrust-toolchain\nrust-toolchain.toml",
+    {
+      followSymbolicLinks: false,
+    }
+  );
+  const files = await globber.glob();
+  files.sort((a, b) => a.localeCompare(b));
+
+  const hasher = crypto.createHash("sha1");
+  for (const file of files) {
+    for await (const chunk of fs.createReadStream(file)) {
+      hasher.update(chunk);
+    }
+  }
+  return hasher.digest("hex").slice(0, 20);
+}
+
+export interface PackageDefinition {
+  name: string;
+  version: string;
+  path: string;
+  targets: Array<string>;
+}
+
+export type Packages = Array<PackageDefinition>;
+
+interface Meta {
+  packages: Array<{
+    name: string;
+    version: string;
+    manifest_path: string;
+    targets: Array<{ kind: Array<string>; name: string }>;
+  }>;
+}
+
+export async function getPackages(): Promise<Packages> {
+  const cwd = process.cwd();
+  const meta: Meta = JSON.parse(
+    await getCmdOutput("cargo", [
+      "metadata",
+      "--all-features",
+      "--format-version",
+      "1",
+    ])
+  );
+
+  return (
+    meta.packages
+      // .filter((p) => !p.manifest_path.startsWith(cwd))
+      .map((p) => {
+        const targets = p.targets
+          .filter((t) => t.kind[0] === "lib")
+          .map((t) => t.name);
+        return {
+          name: p.name,
+          version: p.version,
+          targets,
+          path: path.dirname(p.manifest_path),
+        };
+      })
+  );
+}
+
+export async function cleanTarget(packages: Packages) {
+  await fs.promises.unlink(path.join(targetDir, "./.rustc_info.json"));
+
+  await cleanProfileTarget(packages, "debug");
+  await cleanProfileTarget(packages, "release");
+}
+
+async function cleanProfileTarget(packages: Packages, profile: string) {
+  try {
+    await fs.promises.access(path.join(targetDir, profile));
+  } catch {
+    return;
+  }
+
+  await io.rmRF(path.join(targetDir, profile, "./examples"));
+  await io.rmRF(path.join(targetDir, profile, "./incremental"));
+
+  let dir: fs.Dir;
+  // remove all *files* from the profile directory
+  dir = await fs.promises.opendir(path.join(targetDir, profile));
+  for await (const dirent of dir) {
+    if (dirent.isFile()) {
+      await rm(dir.path, dirent);
+    }
+  }
+
+  const keepPkg = new Set(packages.map((p) => p.name));
+  await rmExcept(path.join(targetDir, profile, "./build"), keepPkg);
+  await rmExcept(path.join(targetDir, profile, "./.fingerprint"), keepPkg);
+
+  const keepDeps = new Set(
+    packages.flatMap((p) => {
+      const names = [];
+      for (const n of [p.name, ...p.targets]) {
+        const name = n.replace(/-/g, "_");
+        names.push(name, `lib${name}`);
+      }
+      return names;
+    })
+  );
+  await rmExcept(path.join(targetDir, profile, "./deps"), keepDeps);
+}
+
+const oneWeek = 7 * 24 * 3600 * 1000;
+
+export async function rmExcept(dirName: string, keepPrefix: Set<string>) {
+  const dir = await fs.promises.opendir(dirName);
+  for await (const dirent of dir) {
+    let name = dirent.name;
+    const idx = name.lastIndexOf("-");
+    if (idx !== -1) {
+      name = name.slice(0, idx);
+    }
+    const fileName = path.join(dir.path, dirent.name);
+    const { mtime } = await fs.promises.stat(fileName);
+    // we don’t really know
+    if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > oneWeek) {
+      await rm(dir.path, dirent);
+    }
+  }
+}
+
+export async function rm(parent: string, dirent: fs.Dirent) {
+  try {
+    const fileName = path.join(parent, dirent.name);
+    core.debug(`deleting "${fileName}"`);
+    if (dirent.isFile()) {
+      await fs.promises.unlink(fileName);
+    } else if (dirent.isDirectory()) {
+      await io.rmRF(fileName);
+    }
+  } catch {}
+}
+
+export function getInputAsArray(
+  name: string,
+  options?: core.InputOptions
+): string[] {
+  return core
+    .getInput(name, options)
+    .split("\n")
+    .map((s) => s.trim())
+    .filter((x) => x !== "");
+}

.github/actions/rust-cache/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}

.github/actions/rust-cache/src/restore.ts

@@ -0,0 +1,56 @@
+import * as cache from "@actions/cache";
+import * as core from "@actions/core";
+import { cleanTarget, getCacheConfig, getCargoBins, getPackages, stateBins, stateKey } from "./common";
+
+async function run() {
+  if (!cache.isFeatureAvailable()) {
+    setCacheHitOutput(false);
+    return;
+  }
+
+  try {
+    var cacheOnFailure = core.getInput("cache-on-failure").toLowerCase();
+    if (cacheOnFailure !== "true") {
+      cacheOnFailure = "false";
+    }
+    core.exportVariable("CACHE_ON_FAILURE", cacheOnFailure);
+    core.exportVariable("CARGO_INCREMENTAL", 0);
+
+    const { paths, key, restoreKeys } = await getCacheConfig();
+
+    const bins = await getCargoBins();
+    core.saveState(stateBins, JSON.stringify([...bins]));
+
+    core.info(`Restoring paths:\n    ${paths.join("\n    ")}`);
+    core.info(`In directory:\n    ${process.cwd()}`);
+    core.info(`Using keys:\n    ${[key, ...restoreKeys].join("\n    ")}`);
+    const restoreKey = await cache.restoreCache(paths, key, restoreKeys);
+    if (restoreKey) {
+      core.info(`Restored from cache key "${restoreKey}".`);
+      core.saveState(stateKey, restoreKey);
+
+      if (restoreKey !== key) {
+        // pre-clean the target directory on cache mismatch
+        const packages = await getPackages();
+
+        await cleanTarget(packages);
+      }
+
+      setCacheHitOutput(restoreKey === key);
+    } else {
+      core.info("No cache found.");
+
+      setCacheHitOutput(false);
+    }
+  } catch (e) {
+    setCacheHitOutput(false);
+
+    core.info(`[warning] ${(e as any).message}`);
+  }
+}
+
+function setCacheHitOutput(cacheHit: boolean): void {
+  core.setOutput("cache-hit", cacheHit.toString());
+}
+
+run();

.github/actions/rust-cache/src/save.ts

@@ -0,0 +1,161 @@
+import * as cache from "@actions/cache";
+import * as core from "@actions/core";
+import * as exec from "@actions/exec";
+import * as glob from "@actions/glob";
+import * as io from "@actions/io";
+import fs from "fs";
+import path from "path";
+import {
+  cleanTarget,
+  getCacheConfig,
+  getCargoBins,
+  getPackages,
+  Packages,
+  paths,
+  rm,
+  stateBins,
+  stateKey,
+} from "./common";
+
+async function run() {
+  if (!cache.isFeatureAvailable()) {
+    return;
+  }
+
+  try {
+    const { paths: savePaths, key } = await getCacheConfig();
+
+    if (core.getState(stateKey) === key) {
+      core.info(`Cache up-to-date.`);
+      return;
+    }
+
+    // TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
+    await macOsWorkaround();
+
+    const registryName = await getRegistryName();
+    const packages = await getPackages();
+
+    try {
+      await cleanRegistry(registryName, packages);
+    } catch {}
+
+    try {
+      await cleanBin();
+    } catch {}
+
+    try {
+      await cleanGit(packages);
+    } catch {}
+
+    try {
+      await cleanTarget(packages);
+    } catch {}
+
+    core.info(`Saving paths:\n    ${savePaths.join("\n    ")}`);
+    core.info(`In directory:\n    ${process.cwd()}`);
+    core.info(`Using key:\n    ${key}`);
+    await cache.saveCache(savePaths, key);
+  } catch (e) {
+    core.info(`[warning] ${(e as any).message}`);
+  }
+}
+
+run();
+
+async function getRegistryName(): Promise<string> {
+  const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
+  const files = await globber.glob();
+  if (files.length > 1) {
+    core.warning(`got multiple registries: "${files.join('", "')}"`);
+  }
+
+  const first = files.shift()!;
+  return path.basename(path.dirname(first));
+}
+
+async function cleanBin() {
+  const bins = await getCargoBins();
+  const oldBins = JSON.parse(core.getState(stateBins));
+
+  for (const bin of oldBins) {
+    bins.delete(bin);
+  }
+
+  const dir = await fs.promises.opendir(path.join(paths.cargoHome, "bin"));
+  for await (const dirent of dir) {
+    if (dirent.isFile() && !bins.has(dirent.name)) {
+      await rm(dir.path, dirent);
+    }
+  }
+}
+
+async function cleanRegistry(registryName: string, packages: Packages) {
+  await io.rmRF(path.join(paths.index, registryName, ".cache"));
+
+  const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
+
+  const dir = await fs.promises.opendir(path.join(paths.cache, registryName));
+  for await (const dirent of dir) {
+    if (dirent.isFile() && !pkgSet.has(dirent.name)) {
+      await rm(dir.path, dirent);
+    }
+  }
+}
+
+async function cleanGit(packages: Packages) {
+  const coPath = path.join(paths.git, "checkouts");
+  const dbPath = path.join(paths.git, "db");
+  const repos = new Map<string, Set<string>>();
+  for (const p of packages) {
+    if (!p.path.startsWith(coPath)) {
+      continue;
+    }
+    const [repo, ref] = p.path.slice(coPath.length + 1).split(path.sep);
+    const refs = repos.get(repo);
+    if (refs) {
+      refs.add(ref);
+    } else {
+      repos.set(repo, new Set([ref]));
+    }
+  }
+
+  // we have to keep both the clone, and the checkout, removing either will
+  // trigger a rebuild
+
+  let dir: fs.Dir;
+  // clean the db
+  dir = await fs.promises.opendir(dbPath);
+  for await (const dirent of dir) {
+    if (!repos.has(dirent.name)) {
+      await rm(dir.path, dirent);
+    }
+  }
+
+  // clean the checkouts
+  dir = await fs.promises.opendir(coPath);
+  for await (const dirent of dir) {
+    const refs = repos.get(dirent.name);
+    if (!refs) {
+      await rm(dir.path, dirent);
+      continue;
+    }
+    if (!dirent.isDirectory()) {
+      continue;
+    }
+    const refsDir = await fs.promises.opendir(path.join(dir.path, dirent.name));
+    for await (const dirent of refsDir) {
+      if (!refs.has(dirent.name)) {
+        await rm(refsDir.path, dirent);
+      }
+    }
+  }
+}
+
+async function macOsWorkaround() {
+  try {
+    // Workaround for https://github.com/actions/cache/issues/403
+    // Also see https://github.com/rust-lang/cargo/issues/8603
+    await exec.exec("sudo", ["/usr/sbin/purge"], { silent: true });
+  } catch {}
+}

.github/actions/rust-cache/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "noEmitOnError": false,
+    "diagnostics": true,
+    "lib": ["esnext"],
+
+    "target": "es2020",
+
+    "resolveJsonModule": true,
+    "moduleResolution": "node",
+    "module": "esnext",
+    "esModuleInterop": true,
+
+    "strict": true,
+    "skipLibCheck": true,
+    // "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noImplicitReturns": true
+  },
+  "exclude": ["dist"]
+}

.github/workflows/ci.yaml

@@ -87,9 +87,11 @@ jobs:
           components: rustfmt, rust-src
 
       - name: Cache Rust Dependencies
-        uses: Brendonovich/rust-cache@v1
+        uses: ./.github/actions/rust-cache
         with:
-          sharedKey: all-${{ matrix.platform }}
+          key: rust-${{ matrix.platform }}-${{ github.run_id }}
+          restore-keys: |
+            rust-${{ matrix.platform }}
 
       - name: Generate Prisma client
         working-directory: core
@@ -128,9 +130,11 @@ jobs:
           components: rustfmt, rust-src
 
       - name: Cache Rust Dependencies
-        uses: Brendonovich/rust-cache@v1
+        uses: ./.github/actions/rust-cache
         with:
-          sharedKey: all-${{ matrix.platform }}
+          key: rust-${{ matrix.platform }}-${{ github.run_id }}
+          restore-keys: |
+            rust-${{ matrix.platform }}
 
       - name: Install pnpm
         uses: pnpm/action-setup@v2.2.1
@@ -176,9 +180,11 @@ jobs:
           components: rustfmt, rust-src
 
       - name: Cache Rust Dependencies
-        uses: Brendonovich/rust-cache@v1
+        uses: ./.github/actions/rust-cache
         with:
-          sharedKey: all-${{ matrix.platform }}
+          key: rust-${{ matrix.platform }}-${{ github.run_id }}
+          restore-keys: |
+            rust-${{ matrix.platform }}
       
       - name: Publish docker image
         uses: ./.github/actions/publish-server

pnpm-workspace.yaml

@@ -2,3 +2,4 @@ packages:
   - 'packages/*'
   - 'apps/*'
   - 'core'
+  - '.github/actions/*'