chore(ci): Scope tests extras backend tests (#9170)

Signed-off-by: Richard Palethorpe <io@richiejp.com>

.github/workflows/test-extra.yml

@@ -14,6 +14,37 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      run-all: ${{ steps.detect.outputs.run-all }}
+      transformers: ${{ steps.detect.outputs.transformers }}
+      rerankers: ${{ steps.detect.outputs.rerankers }}
+      diffusers: ${{ steps.detect.outputs.diffusers }}
+      coqui: ${{ steps.detect.outputs.coqui }}
+      moonshine: ${{ steps.detect.outputs.moonshine }}
+      pocket-tts: ${{ steps.detect.outputs.pocket-tts }}
+      qwen-tts: ${{ steps.detect.outputs.qwen-tts }}
+      qwen-asr: ${{ steps.detect.outputs.qwen-asr }}
+      nemo: ${{ steps.detect.outputs.nemo }}
+      voxcpm: ${{ steps.detect.outputs.voxcpm }}
+      llama-cpp-quantization: ${{ steps.detect.outputs.llama-cpp-quantization }}
+      acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }}
+      voxtral: ${{ steps.detect.outputs.voxtral }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+      - name: Install dependencies
+        run: bun add js-yaml @octokit/core
+      - name: Detect changed backends
+        id: detect
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_EVENT_PATH: ${{ github.event_path }}
+        run: bun run scripts/changed-backends.js
+
   # Requires CUDA
   # tests-chatterbox-tts:
   #   runs-on: ubuntu-latest
@@ -37,6 +68,8 @@ jobs:
   #          make --jobs=5 --output-sync=target -C backend/python/chatterbox
   #          make --jobs=5 --output-sync=target -C backend/python/chatterbox test
   tests-transformers:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.transformers == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -58,6 +91,8 @@ jobs:
            make --jobs=5 --output-sync=target -C backend/python/transformers
            make --jobs=5 --output-sync=target -C backend/python/transformers test
   tests-rerankers:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.rerankers == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -80,6 +115,8 @@ jobs:
            make --jobs=5 --output-sync=target -C backend/python/rerankers test
 
   tests-diffusers:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.diffusers == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -229,6 +266,8 @@ jobs:
   #          make --jobs=5 --output-sync=target -C backend/python/vllm test
 
   tests-coqui:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.coqui == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -248,6 +287,8 @@ jobs:
           make --jobs=5 --output-sync=target -C backend/python/coqui
           make --jobs=5 --output-sync=target -C backend/python/coqui test
   tests-moonshine:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.moonshine == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -267,6 +308,8 @@ jobs:
           make --jobs=5 --output-sync=target -C backend/python/moonshine
           make --jobs=5 --output-sync=target -C backend/python/moonshine test
   tests-pocket-tts:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.pocket-tts == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -286,6 +329,8 @@ jobs:
           make --jobs=5 --output-sync=target -C backend/python/pocket-tts
           make --jobs=5 --output-sync=target -C backend/python/pocket-tts test
   tests-qwen-tts:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.qwen-tts == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -327,6 +372,8 @@ jobs:
   #         make --jobs=5 --output-sync=target -C backend/python/fish-speech
   #         make --jobs=5 --output-sync=target -C backend/python/fish-speech test
   tests-qwen-asr:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.qwen-asr == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -346,6 +393,8 @@ jobs:
           make --jobs=5 --output-sync=target -C backend/python/qwen-asr
           make --jobs=5 --output-sync=target -C backend/python/qwen-asr test
   tests-nemo:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.nemo == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -365,6 +414,8 @@ jobs:
           make --jobs=5 --output-sync=target -C backend/python/nemo
           make --jobs=5 --output-sync=target -C backend/python/nemo test
   tests-voxcpm:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.voxcpm == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -384,6 +435,8 @@ jobs:
           make --jobs=5 --output-sync=target -C backend/python/voxcpm
           make --jobs=5 --output-sync=target -C backend/python/voxcpm test
   tests-llama-cpp-quantization:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.llama-cpp-quantization == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -411,6 +464,8 @@ jobs:
         run: |
           make --jobs=5 --output-sync=target -C backend/python/llama-cpp-quantization test
   tests-acestep-cpp:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.acestep-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone
@@ -441,6 +496,8 @@ jobs:
         run: |
           make --jobs=5 --output-sync=target -C backend/go/acestep-cpp test
   tests-voxtral:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.voxtral == 'true' || needs.detect-changes.outputs.run-all == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Clone

scripts/changed-backends.js

@@ -10,40 +10,9 @@ const backendJobsDarwin = jobs["backend-jobs-darwin"];
 const includes = backendJobs.strategy.matrix.include;
 const includesDarwin = backendJobsDarwin.strategy.matrix.include;
 
-// Set up Octokit for PR changed files
-const token = process.env.GITHUB_TOKEN;
-const octokit = new Octokit({ auth: token });
-
 const eventPath = process.env.GITHUB_EVENT_PATH;
 const event = JSON.parse(fs.readFileSync(eventPath, "utf8"));
 
-let prNumber, repo, owner;
-if (event.pull_request) {
-  prNumber = event.pull_request.number;
-  repo = event.repository.name;
-  owner = event.repository.owner.login;
-} else {
-  throw new Error("This workflow must be triggered by a pull_request event.");
-}
-
-async function getChangedFiles() {
-  let files = [];
-  let page = 1;
-  while (true) {
-    const res = await octokit.request('GET /repos/{owner}/{repo}/pulls/{pull_number}/files', {
-      owner,
-      repo,
-      pull_number: prNumber,
-      per_page: 100,
-      page
-    });
-    files = files.concat(res.data.map(f => f.filename));
-    if (res.data.length < 100) break;
-    page++;
-  }
-  return files;
-}
-
 // Infer backend path
 function inferBackendPath(item) {
   if (item.dockerfile.endsWith("python")) {
@@ -66,6 +35,65 @@ function inferBackendPathDarwin(item) {
   return `backend/${item.lang}/${item.backend}/`;
 }
 
+// Build a deduplicated map of backend name -> path prefix from all matrix entries
+function getAllBackendPaths() {
+  const paths = new Map();
+  for (const item of includes) {
+    const p = inferBackendPath(item);
+    if (p && !paths.has(item.backend)) {
+      paths.set(item.backend, p);
+    }
+  }
+  for (const item of includesDarwin) {
+    const p = inferBackendPathDarwin(item);
+    if (p && !paths.has(item.backend)) {
+      paths.set(item.backend, p);
+    }
+  }
+  return paths;
+}
+
+const allBackendPaths = getAllBackendPaths();
+
+// Non-PR events: output run-all=true and all backends as true
+if (!event.pull_request) {
+  fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=true\n`);
+  fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends=true\n`);
+  fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=true\n`);
+  fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix=${JSON.stringify({ include: includes })}\n`);
+  fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: includesDarwin })}\n`);
+  for (const backend of allBackendPaths.keys()) {
+    fs.appendFileSync(process.env.GITHUB_OUTPUT, `${backend}=true\n`);
+  }
+  process.exit(0);
+}
+
+// PR context
+const prNumber = event.pull_request.number;
+const repo = event.repository.name;
+const owner = event.repository.owner.login;
+
+const token = process.env.GITHUB_TOKEN;
+const octokit = new Octokit({ auth: token });
+
+async function getChangedFiles() {
+  let files = [];
+  let page = 1;
+  while (true) {
+    const res = await octokit.request('GET /repos/{owner}/{repo}/pulls/{pull_number}/files', {
+      owner,
+      repo,
+      pull_number: prNumber,
+      per_page: 100,
+      page
+    });
+    files = files.concat(res.data.map(f => f.filename));
+    if (res.data.length < 100) break;
+    page++;
+  }
+  return files;
+}
+
 (async () => {
   const changedFiles = await getChangedFiles();
 
@@ -90,8 +118,15 @@ function inferBackendPathDarwin(item) {
   console.log("Has backends?:", hasBackends);
   console.log("Has Darwin backends?:", hasBackendsDarwin);
 
+  fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=false\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends=${hasBackends}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=${hasBackendsDarwin}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix=${JSON.stringify({ include: filtered })}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: filteredDarwin })}\n`);
+
+  // Per-backend boolean outputs
+  for (const [backend, pathPrefix] of allBackendPaths) {
+    const changed = changedFiles.some(file => file.startsWith(pathPrefix));
+    fs.appendFileSync(process.env.GITHUB_OUTPUT, `${backend}=${changed ? 'true' : 'false'}\n`);
+  }
 })();