From 621c612b2d6169da9fcfe52e70853e1cdda67dda Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Mon, 11 May 2026 22:46:02 +0200 Subject: [PATCH] ci(bump-deps): register ds4 + move version pin into the Makefile (#9761) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci(bump-deps): register ds4 + move version pin into the Makefile The initial ds4 PR (#9758) put the upstream commit pin in backend/cpp/ds4/prepare.sh as a shell variable. The auto-bump bot at .github/bump_deps.sh greps for ^$VAR?= in a Makefile, so DS4_VERSION was invisible to it - other backends (llama-cpp, ik-llama-cpp, turboquant, voxtral, etc.) all pin in their Makefile. This change: - Moves DS4_VERSION?= and DS4_REPO?= to the top of backend/cpp/ds4/Makefile. - Inlines the git init/fetch/checkout recipe into the 'ds4:' target (matches llama-cpp's 'llama.cpp:' target pattern). Directory acts as the target so make only re-clones when missing. - Deletes the now-redundant prepare.sh. - Adds antirez/ds4 + DS4_VERSION + main + backend/cpp/ds4/Makefile to the .github/workflows/bump_deps.yaml matrix so the daily bot opens PRs against this pin. - Updates .agents/ds4-backend.md to point at the Makefile. Verified: $ grep -m1 '^DS4_VERSION?=' backend/cpp/ds4/Makefile DS4_VERSION?=ae302c2fa18cc6d9aefc021d0f27ae03c9ad2fc0 $ make -C backend/cpp/ds4 ds4 # clones into ds4/ at the pin $ make -C backend/cpp/ds4 ds4 # no-op on second invocation make: 'ds4' is up to date. Signed-off-by: Ettore Di Giacinto * ci: route backend/cpp/ds4/ changes through changed-backends.js scripts/changed-backends.js:inferBackendPath has an explicit branch per cpp dockerfile suffix (ik-llama-cpp, turboquant, llama-cpp). Without a matching branch the function returns null, the backend never lands in the path map, and PR change-detection cannot map "backend/cpp/ds4/X changed" -> "rebuild ds4 image". This is why PR #9761 produced zero ds4 jobs even though it directly edits backend/cpp/ds4/Makefile. Adds the missing branch (Dockerfile.ds4 -> backend/cpp/ds4/), placed before the llama-cpp branch (since both share the .cpp ancestry but ds4 is more specific - same ordering rule documented in .agents/adding-backends.md). Verified with a local Node simulation of the script against this PR's diff: the path map now contains 'ds4 -> backend/cpp/ds4/' and a 'backend/cpp/ds4/Makefile' change correctly triggers the ds4 backend in the rebuild set. Signed-off-by: Ettore Di Giacinto * docs(adding-backends): harden the two gotchas that bit ds4 Both omissions are silent at the time you ADD a backend - the failure mode only appears later (the bump bot stays silent forever, or the path filter shows up on the next PR that touches your backend with zero CI jobs and looks broken for unrelated reasons). Expanding the `scripts/changed-backends.js` paragraph from a one-liner to a fully worked example, and adding a new sibling paragraph for the `bump_deps.yaml` + Makefile-pin contract. Both call out the specific mistakes from the ds4 timeline (#9758 → #9761) so future contributors can pattern-match on the cause. Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- .agents/adding-backends.md | 50 +++++++++++++++++++++++++++++++- .agents/ds4-backend.md | 8 +++-- .github/workflows/bump_deps.yaml | 4 +++ backend/cpp/ds4/Makefile | 49 ++++++++++++++++++++----------- backend/cpp/ds4/prepare.sh | 24 --------------- scripts/changed-backends.js | 3 ++ 6 files changed, 94 insertions(+), 44 deletions(-) delete mode 100755 backend/cpp/ds4/prepare.sh diff --git a/.agents/adding-backends.md b/.agents/adding-backends.md index 627bf5b0d..60c84ba84 100644 --- a/.agents/adding-backends.md +++ b/.agents/adding-backends.md @@ -34,7 +34,55 @@ The build matrix is data-only YAML at `.github/backend-matrix.yml` (not inside ` **Without an entry here no image is ever built or pushed, and the gallery entry in `backend/index.yaml` will point at a tag that does not exist.** The `dockerfile:` field must point at `./backend/Dockerfile.` matching the language bucket from step 1 (e.g. `Dockerfile.python`, `Dockerfile.golang`, `Dockerfile.rust`). The `tag-suffix` must match the `uri:` in the corresponding `backend/index.yaml` image entry exactly. -If you add a new language bucket, `scripts/changed-backends.js` also needs a branch in `inferBackendPath` so PR change-detection routes file edits correctly. +**`scripts/changed-backends.js` registration — REQUIRED for any new dockerfile suffix.** This is the single most common omission, because it has no effect on the PR that adds the backend (when no prior path filter could catch it anyway) — it only breaks the *next* PR that touches your backend's directory, which then gets zero CI jobs and looks broken for unrelated reasons. Edit `scripts/changed-backends.js:inferBackendPath` and add a branch BEFORE the more-generic suffixes: + +```js +if (item.dockerfile.endsWith("")) { + return `backend/cpp//`; // or backend/python|go|rust/... +} +``` + +The `endsWith()` test is against the matrix entry's `dockerfile:` value (e.g. `./backend/Dockerfile.ds4` → `endsWith("ds4")`). Specificity order matters here just like it does for importers: more-specific suffixes go BEFORE more-generic ones (e.g. `ds4` before `llama-cpp` even though both end with letters, because some upstream might one day call itself `super-ds4-llama-cpp`). Verify locally before pushing: + +```bash +# Confirm your dockerfile suffix is unique enough +node -e " +const yaml = require('js-yaml'); const fs = require('fs'); +const m = yaml.load(fs.readFileSync('.github/backend-matrix.yml','utf8')); +for (const e of m.include.filter(e => e.backend === '')) { + console.log(e.dockerfile, '->', e.dockerfile.endsWith('')); +}" +``` + +A quick way to find the right insertion point: `grep -n 'item.dockerfile.endsWith' scripts/changed-backends.js`. + +**`bump_deps.yaml` registration — REQUIRED for any backend pinning an upstream commit.** If your backend's Makefile has a `*_VERSION?=` pin to a third-party repo, the daily auto-bump bot at `.github/workflows/bump_deps.yaml` won't notice it unless you register the backend in its matrix. The bot runs `.github/bump_deps.sh` which `grep`s for `^$VAR?=` in the Makefile you list — so the pin MUST live in the Makefile (not in a separate shell script). The bump for ds4 (#9761) had to walk this back because the original landed the pin in `prepare.sh`, which the bot can't see. Pattern (for `antirez/ds4`): + +```yaml +# .github/workflows/bump_deps.yaml +matrix: + include: + - repository: "antirez/ds4" + variable: "DS4_VERSION" + branch: "main" + file: "backend/cpp/ds4/Makefile" +``` + +And the corresponding Makefile shape (mirror `backend/cpp/llama-cpp/Makefile`): + +```makefile +DS4_VERSION?=ae302c2fa18cc6d9aefc021d0f27ae03c9ad2fc0 +DS4_REPO?=https://github.com/antirez/ds4 +... +ds4: + mkdir -p ds4 + cd ds4 && git init -q && \ + git remote add origin $(DS4_REPO) && \ + git fetch --depth 1 origin $(DS4_VERSION) && \ + git checkout FETCH_HEAD +``` + +If you have a `prepare.sh` doing the clone, delete it — the recipe belongs in the Makefile target so `make purge && make` works as a clean-and-rebuild and so the bump bot finds the pin. **Placement in file:** - CPU builds: Add after other CPU builds (e.g., after `cpu-chatterbox`) diff --git a/.agents/ds4-backend.md b/.agents/ds4-backend.md index 691ec88cb..5ac70ee7f 100644 --- a/.agents/ds4-backend.md +++ b/.agents/ds4-backend.md @@ -6,8 +6,12 @@ LocalAI wraps the engine's C API (`ds4/ds4.h`) with a fresh C++ gRPC server at ## Pin -`backend/cpp/ds4/prepare.sh` clones `antirez/ds4` at `DS4_VERSION`. Bump that -commit to follow upstream. +`backend/cpp/ds4/Makefile` pins `DS4_VERSION?=` at the top. The `ds4` +target in the Makefile clones `antirez/ds4` at that commit (mirroring the +llama-cpp / ik-llama-cpp / turboquant pattern). The bump-deps bot +(`.github/workflows/bump_deps.yaml`) finds this pin via grep and opens a +daily PR to update it. To bump manually: edit the `DS4_VERSION?=` line, +then `make purge && make` (or rely on CI's clean build). ## Wire shape diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index 4028177bb..c8b95fb03 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -22,6 +22,10 @@ jobs: variable: "TURBOQUANT_VERSION" branch: "feature/turboquant-kv-cache" file: "backend/cpp/turboquant/Makefile" + - repository: "antirez/ds4" + variable: "DS4_VERSION" + branch: "main" + file: "backend/cpp/ds4/Makefile" - repository: "ggml-org/whisper.cpp" variable: "WHISPER_CPP_VERSION" branch: "master" diff --git a/backend/cpp/ds4/Makefile b/backend/cpp/ds4/Makefile index b702713d6..6c80b9847 100644 --- a/backend/cpp/ds4/Makefile +++ b/backend/cpp/ds4/Makefile @@ -1,7 +1,14 @@ # ds4 backend Makefile. -CURDIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -DS4_DIR := $(CURDIR)ds4 -BUILD_DIR := $(CURDIR)build +# +# Upstream pin lives below as DS4_VERSION?= so the bump-deps bot +# (.github/bump_deps.sh) can find and update it - matches the +# llama-cpp / ik-llama-cpp / turboquant convention. + +DS4_VERSION?=ae302c2fa18cc6d9aefc021d0f27ae03c9ad2fc0 +DS4_REPO?=https://github.com/antirez/ds4 + +CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +BUILD_DIR := build BUILD_TYPE ?= NATIVE ?= false @@ -27,37 +34,45 @@ ifneq ($(NATIVE),true) CMAKE_ARGS += -DDS4_NATIVE=OFF endif -.PHONY: prepare grpc-server package clean purge test all +.PHONY: grpc-server package clean purge test all all: grpc-server -prepare: - bash $(CURDIR)prepare.sh +# Clone the upstream ds4 source at the pinned commit. Directory acts as the +# target so make only re-clones when missing. After a DS4_VERSION bump, +# run 'make purge && make' to refetch (or rely on CI's clean build). +ds4: + mkdir -p ds4 + cd ds4 && \ + git init -q && \ + git remote add origin $(DS4_REPO) && \ + git fetch --depth 1 origin $(DS4_VERSION) && \ + git checkout FETCH_HEAD # Build ds4's engine object files via its own Makefile, which already encodes # the right per-platform compile flags (Objective-C/Metal on Darwin, nvcc on Linux+CUDA). -$(DS4_DIR)/ds4.o: prepare +ds4/ds4.o: ds4 ifeq ($(BUILD_TYPE),cublas) - +$(MAKE) -C $(DS4_DIR) ds4.o ds4_cuda.o + +$(MAKE) -C ds4 ds4.o ds4_cuda.o else ifeq ($(UNAME_S),Darwin) - +$(MAKE) -C $(DS4_DIR) ds4.o ds4_metal.o + +$(MAKE) -C ds4 ds4.o ds4_metal.o else - +$(MAKE) -C $(DS4_DIR) ds4_cpu.o + +$(MAKE) -C ds4 ds4_cpu.o endif -grpc-server: $(DS4_DIR)/ds4.o +grpc-server: ds4/ds4.o mkdir -p $(BUILD_DIR) - cd $(BUILD_DIR) && cmake $(CMAKE_ARGS) -DDS4_DIR=$(DS4_DIR) $(CURDIR) && cmake --build . --config Release -j $(JOBS) - cp $(BUILD_DIR)/grpc-server $(CURDIR)grpc-server + cd $(BUILD_DIR) && cmake $(CMAKE_ARGS) $(CURRENT_MAKEFILE_DIR) && cmake --build . --config Release -j $(JOBS) + cp $(BUILD_DIR)/grpc-server grpc-server package: grpc-server - bash $(CURDIR)package.sh + bash package.sh test: @echo "ds4 backend: e2e coverage at tests/e2e-backends/ (BACKEND_BINARY mode)" clean: - rm -rf $(BUILD_DIR) $(CURDIR)grpc-server $(CURDIR)package - if [ -d $(DS4_DIR) ]; then $(MAKE) -C $(DS4_DIR) clean; fi + rm -rf $(BUILD_DIR) grpc-server package + if [ -d ds4 ]; then $(MAKE) -C ds4 clean; fi purge: clean - rm -rf $(DS4_DIR) + rm -rf ds4 diff --git a/backend/cpp/ds4/prepare.sh b/backend/cpp/ds4/prepare.sh deleted file mode 100755 index ec76e6fe2..000000000 --- a/backend/cpp/ds4/prepare.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Clones the upstream ds4 source at the pinned commit into ./ds4/. Idempotent. -set -e - -DS4_REPO="${DS4_REPO:-https://github.com/antirez/ds4}" -DS4_VERSION="${DS4_VERSION:-ae302c2fa18cc6d9aefc021d0f27ae03c9ad2fc0}" - -if [ -d ds4/.git ]; then - current=$(git -C ds4 rev-parse HEAD 2>/dev/null || echo none) - if [ "$current" = "$DS4_VERSION" ]; then - echo "ds4 already at $DS4_VERSION" - exit 0 - fi - git -C ds4 fetch --depth 1 origin "$DS4_VERSION" - git -C ds4 checkout "$DS4_VERSION" - exit 0 -fi - -mkdir -p ds4 -cd ds4 -git init -q -git remote add origin "$DS4_REPO" -git fetch --depth 1 origin "$DS4_VERSION" -git checkout FETCH_HEAD diff --git a/scripts/changed-backends.js b/scripts/changed-backends.js index d5dc309d3..4b3ad1fc9 100644 --- a/scripts/changed-backends.js +++ b/scripts/changed-backends.js @@ -32,6 +32,9 @@ function inferBackendPath(item) { // via a thin wrapper Makefile. Changes to either dir should retrigger it. return `backend/cpp/turboquant/`; } + if (item.dockerfile.endsWith("ds4")) { + return `backend/cpp/ds4/`; + } if (item.dockerfile.endsWith("llama-cpp")) { return `backend/cpp/llama-cpp/`; }