fix(ci): only rebuild the full backend matrix on breaking backend.proto edits (#11192)

backend/backend.proto is consumed by every language, so its SHARED_BUILD_INPUTS
rule could only ever be always/always: 417 Linux plus 56 Darwin builds. It fires
on ~1.3% of commits (10 of 767 over six months), which made it the single
largest CI cost driver in the repo.

On 2026-07-29 the queue reached 2178 jobs against 8 concurrent runners. Four
runs totalling 935 of those jobs were triggered by nothing but a proto edit. The
largest, 378 jobs on master, came from PR #11158, whose entire proto diff was
six lines adding `bool cache_prompt = 8;` to one message. No backend that does
not read that field behaves any differently for it.

Make the rule content-aware. changed-backends.js resolves backend.proto at the
base revision (the contents-API pattern already used for backend-matrix.yml) and
hands both texts to protoChangeIsAdditive(), which compares them structurally so
a comment reflow, reindent or field reorder does not read as a change. An
additive-only edit (new field with an unused number, new message, new enum
value, new RPC) suppresses the rule and rebuilds nothing; a removed, renumbered,
retyped or renamed field, a dropped RPC or a changed option still rebuilds
everything, as does an unresolvable base revision.

Every other matched rule is untouched, so a PR that edits the proto and
scripts/build/ is still a full rebuild, and the weekly full-matrix cron remains
the backstop for stale wheels.

Verified against all ten proto commits of the preceding six months: the nine
with a resolvable parent all classify as additive, and controls covering a
retyped-and-renumbered field, a deleted RPC, identical revisions and a
reindent-plus-comment-reflow all classify correctly.


Assisted-by: Claude:opus-5 [claude-code]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

scripts/lib/backend-filter.mjs

@@ -178,6 +178,126 @@ const GO_BACKEND_PKG_PREFIXES = [
   "pkg/utils/",
 ];
 
+export const BACKEND_PROTO_FILE = "backend/backend.proto";
+const PROTO_RULE_ID = "backend-proto";
+
+// Split a .proto into a map of symbol -> fingerprint so two revisions can be
+// compared structurally instead of textually. A comment reflow, a reindent or a
+// reordered field must not read as a change, and a renumbered or retyped field
+// must.
+//
+// The scanner is deliberately syntax-light: it tracks brace depth to build a
+// container path and records one entry per declaration (`message`, `enum`,
+// `service`, `oneof`, `rpc`) and one per statement (fields, enum values,
+// `option`, `reserved`). It never needs to understand types, only to notice
+// when the text describing one stops being identical.
+function protoSymbols(text) {
+  const symbols = new Map();
+  const stack = [];
+  const norm = s => s.trim().replace(/\s+/g, " ");
+
+  // A declaration is identified by its kind and name, so that changing its body
+  // shows up as changed members rather than as a wholesale replacement.
+  const declKey = header => {
+    const rpc = header.match(/^rpc\s+([A-Za-z_]\w*)/);
+    if (rpc) return `rpc ${rpc[1]}`;
+    const decl = header.match(/^(message|enum|service|oneof|extend)\s+([A-Za-z_]\w*)/);
+    if (decl) return `${decl[1]} ${decl[2]}`;
+    return header;
+  };
+
+  // `bool cache_prompt = 8` is identified by `cache_prompt`, so renumbering or
+  // retyping it changes the fingerprint under a stable key, while renaming it
+  // reads as a removal plus an addition. Statements with no `=` (`reserved 4;`)
+  // are their own identity.
+  const stmtKey = stmt => {
+    const eq = stmt.indexOf("=");
+    if (eq === -1) return stmt;
+    const lhs = norm(stmt.slice(0, eq)).split(" ");
+    return lhs[lhs.length - 1] || stmt;
+  };
+
+  let buf = "";
+  let i = 0;
+  while (i < text.length) {
+    const c = text[i];
+
+    // String literals first: `option go_package = "github.com/..."` contains a
+    // `//` that is not a comment.
+    if (c === '"' || c === "'") {
+      buf += c;
+      i++;
+      while (i < text.length) {
+        if (text[i] === "\\") {
+          buf += text.slice(i, i + 2);
+          i += 2;
+          continue;
+        }
+        buf += text[i];
+        i++;
+        if (text[i - 1] === c) break;
+      }
+      continue;
+    }
+    if (c === "/" && text[i + 1] === "/") {
+      while (i < text.length && text[i] !== "\n") i++;
+      continue;
+    }
+    if (c === "/" && text[i + 1] === "*") {
+      i += 2;
+      while (i < text.length && !(text[i] === "*" && text[i + 1] === "/")) i++;
+      i += 2;
+      continue;
+    }
+    if (c === "{") {
+      const header = norm(buf);
+      buf = "";
+      i++;
+      const key = header ? declKey(header) : "";
+      if (header) symbols.set(`${stack.join("/")}|${key}`, header);
+      stack.push(key);
+      continue;
+    }
+    if (c === "}") {
+      stack.pop();
+      buf = "";
+      i++;
+      continue;
+    }
+    if (c === ";") {
+      const stmt = norm(buf);
+      buf = "";
+      i++;
+      if (stmt) symbols.set(`${stack.join("/")}|${stmtKey(stmt)}`, stmt);
+      continue;
+    }
+    buf += c;
+    i++;
+  }
+  return symbols;
+}
+
+// True when every symbol the previous revision declared survives unchanged into
+// the current one. New symbols are free: a backend that never references a new
+// field, message or RPC produces the same behavior with or without it, so its
+// image does not need rebuilding.
+//
+// Anything else (a removed, renumbered, retyped or renamed field, a dropped
+// RPC, a changed option) is treated as breaking and rebuilds the full matrix.
+// Either text being unresolvable is breaking too, matching the run-all posture
+// changed-backends.js takes for a diff it cannot compute.
+export function protoChangeIsAdditive(previousText, currentText) {
+  if (typeof previousText !== "string" || typeof currentText !== "string") {
+    return false;
+  }
+  const before = protoSymbols(previousText);
+  const after = protoSymbols(currentText);
+  for (const [key, fingerprint] of before) {
+    if (after.get(key) !== fingerprint) return false;
+  }
+  return true;
+}
+
 // Shared build inputs: files that end up in, or decide the contents of, images
 // belonging to backends whose own directory they do not live under. The
 // per-backend prefix match in filterMatrix() structurally cannot see these, so
@@ -200,7 +320,14 @@ export const SHARED_BUILD_INPUTS = [
     // backends regenerate their stubs from it via `make protogen-go`, the C++
     // CMakeLists compile backend.pb.cc from it, and the Rust crate Makefile
     // copies it in before build.rs runs.
-    matches: file => file === "backend/backend.proto",
+    //
+    // Which is why this rule is always/always, and why it is the single most
+    // expensive entry in the table: 417 Linux plus 56 Darwin builds. It fires
+    // on 1.3% of commits (10 of 767 over six months), and every one of those
+    // ten was purely additive. filterMatrix() suppresses this rule for an
+    // additive-only diff, so `id` exists to identify it there.
+    id: PROTO_RULE_ID,
+    matches: file => file === BACKEND_PROTO_FILE,
     linux: always,
     darwin: always,
   },
@@ -362,8 +489,19 @@ export function filterMatrix({
   includesDarwin,
   changedFiles,
   previousMatrix,
+  protoRevisions,
 }) {
-  const sharedRules = matchedSharedRules(changedFiles);
+  // An additive-only backend.proto edit invalidates no existing image, so drop
+  // its always/always rule. Every other matched rule still applies: a PR that
+  // touches the proto and scripts/build/ is still a full rebuild.
+  const protoAdditiveOnly =
+    changedFiles.includes(BACKEND_PROTO_FILE) &&
+    !!protoRevisions &&
+    protoChangeIsAdditive(protoRevisions.previous, protoRevisions.current);
+
+  const sharedRules = matchedSharedRules(changedFiles).filter(
+    rule => !(rule.id === PROTO_RULE_ID && protoAdditiveOnly)
+  );
 
   const matrixFileChanged = changedFiles.includes(BACKEND_MATRIX_FILE);
   // The matrix file changed but we could not resolve what it used to say (API

scripts/lib/backend-filter_test.mjs

@@ -385,3 +385,146 @@ test("an unavailable previous matrix conservatively rebuilds everything", () =>
   assert.equal(filtered.length, includes.length);
   assert.equal(filteredDarwin.length, includesDarwin.length);
 });
+
+// --- backend.proto: additive changes must not rebuild the world -------------
+//
+// backend/backend.proto is consumed by every language, so the SHARED_BUILD_INPUTS
+// rule for it is always/always: a full 417-entry Linux matrix plus all 56 Darwin
+// entries. It fires on 1.3% of commits, and in practice every one of those has
+// been purely additive (a new field with an unused number, a new message, a new
+// RPC). Adding `bool cache_prompt = 8;` (PR #11158) cannot change the behavior of
+// a backend that never reads it, yet it rebuilt all 473 images.
+//
+// So the rule becomes content-aware rather than path-aware, using the same shape
+// as previousMatrix above: changed-backends.js resolves the base revision and
+// hands the two texts in, and everything here stays pure.
+
+const protoWith = body => `
+syntax = "proto3";
+
+package backend;
+
+service Backend {
+  rpc Health(HealthMessage) returns (Reply) {}
+  rpc Predict(PredictOptions) returns (Reply) {}
+}
+
+message HealthMessage {}
+
+message PredictOptions {
+${body}
+}
+`;
+
+const BASE_FIELDS = `  string Prompt = 1;
+  int32 Tokens = 2;
+  bool UseTokenizerTemplate = 3;`;
+
+const runProto = (previous, current) =>
+  filterMatrix({
+    includes,
+    includesDarwin,
+    changedFiles: ["backend/backend.proto"],
+    protoRevisions: { previous, current },
+  });
+
+test("an added proto field rebuilds nothing", () => {
+  // The real PR #11158 diff: six added lines, one new field, unused number.
+  const { filtered, filteredDarwin, changedBackends } = runProto(
+    protoWith(BASE_FIELDS),
+    protoWith(`${BASE_FIELDS}\n  bool cache_prompt = 8;`)
+  );
+
+  assert.deepEqual(filtered, []);
+  assert.deepEqual(filteredDarwin, []);
+  assert.equal(changedBackends.size, 0);
+});
+
+test("an added proto message and RPC rebuild nothing", () => {
+  const current = protoWith(BASE_FIELDS).replace(
+    "message HealthMessage {}",
+    "message HealthMessage {}\n\nmessage ScoreRequest {\n  string Text = 1;\n}"
+  ).replace(
+    "  rpc Predict(PredictOptions) returns (Reply) {}",
+    "  rpc Predict(PredictOptions) returns (Reply) {}\n  rpc Score(ScoreRequest) returns (Reply) {}"
+  );
+
+  const { filtered, filteredDarwin } = runProto(protoWith(BASE_FIELDS), current);
+
+  assert.deepEqual(filtered, []);
+  assert.deepEqual(filteredDarwin, []);
+});
+
+test("a removed proto field rebuilds every backend on every OS", () => {
+  const { filtered, filteredDarwin } = runProto(
+    protoWith(BASE_FIELDS),
+    protoWith(`  string Prompt = 1;\n  bool UseTokenizerTemplate = 3;`)
+  );
+
+  assert.equal(filtered.length, includes.length);
+  assert.equal(filteredDarwin.length, includesDarwin.length);
+});
+
+test("a renumbered proto field rebuilds every backend on every OS", () => {
+  // Wire-incompatible: an old backend reading field 2 gets nothing.
+  const { filtered, filteredDarwin } = runProto(
+    protoWith(BASE_FIELDS),
+    protoWith(`  string Prompt = 1;\n  int32 Tokens = 9;\n  bool UseTokenizerTemplate = 3;`)
+  );
+
+  assert.equal(filtered.length, includes.length);
+  assert.equal(filteredDarwin.length, includesDarwin.length);
+});
+
+test("a retyped proto field rebuilds every backend on every OS", () => {
+  const { filtered, filteredDarwin } = runProto(
+    protoWith(BASE_FIELDS),
+    protoWith(`  string Prompt = 1;\n  int64 Tokens = 2;\n  bool UseTokenizerTemplate = 3;`)
+  );
+
+  assert.equal(filtered.length, includes.length);
+  assert.equal(filteredDarwin.length, includesDarwin.length);
+});
+
+test("a renamed proto field rebuilds every backend on every OS", () => {
+  // Same number and type, but every generated accessor changes name.
+  const { filtered, filteredDarwin } = runProto(
+    protoWith(BASE_FIELDS),
+    protoWith(`  string Prompt = 1;\n  int32 MaxTokens = 2;\n  bool UseTokenizerTemplate = 3;`)
+  );
+
+  assert.equal(filtered.length, includes.length);
+  assert.equal(filteredDarwin.length, includesDarwin.length);
+});
+
+test("a removed proto RPC rebuilds every backend on every OS", () => {
+  const { filtered, filteredDarwin } = runProto(
+    protoWith(BASE_FIELDS),
+    protoWith(BASE_FIELDS).replace(
+      "  rpc Predict(PredictOptions) returns (Reply) {}\n",
+      ""
+    )
+  );
+
+  assert.equal(filtered.length, includes.length);
+  assert.equal(filteredDarwin.length, includesDarwin.length);
+});
+
+test("a comment-only proto change rebuilds nothing", () => {
+  const { filtered, filteredDarwin } = runProto(
+    protoWith(BASE_FIELDS),
+    protoWith(`  string Prompt = 1;\n  // how many tokens to emit\n  int32 Tokens = 2;\n  bool UseTokenizerTemplate = 3;`)
+  );
+
+  assert.deepEqual(filtered, []);
+  assert.deepEqual(filteredDarwin, []);
+});
+
+test("unresolvable proto revisions conservatively rebuild everything", () => {
+  // Same posture as the previousMatrix fallback: if we cannot resolve what the
+  // proto used to say, we must not claim the change was additive.
+  const { filtered, filteredDarwin } = runProto(null, protoWith(BASE_FIELDS));
+
+  assert.equal(filtered.length, includes.length);
+  assert.equal(filteredDarwin.length, includesDarwin.length);
+});