From 7e8542ba32f6668cfd0372d285ec7f516f0fbcf8 Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:19:17 +0200 Subject: [PATCH] fix(tests): make e2e backend model downloads resumable and stall-based (#10766) The vibevoice transcription e2e hangs until the go test timeout when the HF CDN is slow: the ASR Q4_K model is >10 GB, downloadFile capped every curl attempt at --max-time 600 (needs a sustained ~17 MB/s to fit), and curl's --retry restarts from byte zero, so no attempt ever makes forward progress. This killed the job twice on PR #10764 and previously forced skipping it on release tags (#10567). Replace the wall-clock cap with stall detection (--speed-limit 1 MiB/s over --speed-time 120s) and resume from the bytes already on disk with -C -, retrying from Go because curl does not re-evaluate the resume offset on its internal retries. Resume against the HF Xet CDN was verified by killing a transfer mid-flight and confirming the next invocation appended (114 MB -> 235 MB, GGUF magic intact). Also parameterize the suite timeout (BACKEND_TEST_TIMEOUT, default 30m) and raise it to 120m for the vibevoice transcription wrapper: a 10 GB download plus 25 specs does not fit in 30m even on a good day, and the job-level GHA timeout there is already 150m. Assisted-by: Claude:claude-fable-5 [Claude Code] Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- Makefile | 6 ++++- tests/e2e-backends/backend_test.go | 38 ++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 35e5d2251..c6f38b1d0 100644 --- a/Makefile +++ b/Makefile @@ -645,6 +645,9 @@ test-extra: prepare-test-extra ## suite against it. ## BACKEND_TEST_MODEL_URL?=https://huggingface.co/Qwen/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q8_0.gguf +## Suite timeout for `go test`. Wrappers whose model download alone can eat +## most of the default (multi-GB models on a slow HF CDN day) override this. +BACKEND_TEST_TIMEOUT?=30m ## Generic target — runs the suite against whatever BACKEND_IMAGE points at. ## Depends on protogen-go so pkg/grpc/proto is generated before `go test`. @@ -672,7 +675,7 @@ test-extra-backend: protogen-go BACKEND_TEST_FACE_IMAGE_3_URL="$$BACKEND_TEST_FACE_IMAGE_3_URL" \ BACKEND_TEST_FACE_IMAGE_3_FILE="$$BACKEND_TEST_FACE_IMAGE_3_FILE" \ BACKEND_TEST_VERIFY_DISTANCE_CEILING="$$BACKEND_TEST_VERIFY_DISTANCE_CEILING" \ - go test -v -timeout 30m ./tests/e2e-backends/... + go test -v -timeout $(BACKEND_TEST_TIMEOUT) ./tests/e2e-backends/... ## Convenience wrappers: build the image, then exercise it. test-extra-backend-llama-cpp: docker-build-llama-cpp @@ -1012,6 +1015,7 @@ test-extra-backend-vibevoice-cpp-tts: docker-build-vibevoice-cpp ## post-image disk budget. test-extra-backend-vibevoice-cpp-transcription: docker-build-vibevoice-cpp BACKEND_IMAGE=local-ai-backend:vibevoice-cpp \ + BACKEND_TEST_TIMEOUT=120m \ BACKEND_TEST_MODEL_URL='https://huggingface.co/mudler/vibevoice.cpp-models/resolve/main/vibevoice-asr-q4_k.gguf#vibevoice-asr-q4_k.gguf' \ BACKEND_TEST_EXTRA_FILES='https://huggingface.co/mudler/vibevoice.cpp-models/resolve/main/tokenizer.gguf#tokenizer.gguf' \ BACKEND_TEST_AUDIO_URL=https://github.com/ggml-org/whisper.cpp/raw/master/samples/jfk.wav \ diff --git a/tests/e2e-backends/backend_test.go b/tests/e2e-backends/backend_test.go index c60077fd7..7ac3ad410 100644 --- a/tests/e2e-backends/backend_test.go +++ b/tests/e2e-backends/backend_test.go @@ -1453,20 +1453,34 @@ func extractImage(image, dest string) { // downloadFile fetches url into dest using curl -L. Used for CI convenience; // local runs can use BACKEND_TEST_MODEL_FILE to skip downloading. -// Retry flags guard against transient CI network hiccups (github.com in -// particular has been flaky from GHA runners, timing out TCP connects). +// Model files can be huge (the vibevoice ASR e2e model is >10 GB), so a +// wall-clock cap per attempt (--max-time) is the wrong guard: on a slow CDN +// day no attempt can ever finish, and curl's --retry restarts from byte +// zero, so repeated attempts make no forward progress (this hung the +// vibevoice transcription job until the suite timeout). Instead, abort only +// on a real stall (--speed-limit/--speed-time) and resume from the bytes +// already on disk (-C -), with the retry loop out here because curl does +// not re-evaluate the resume offset on its internal retries. func downloadFile(url, dest string) { GinkgoHelper() - cmd := exec.Command("curl", "-sSfL", - "--connect-timeout", "30", - "--max-time", "600", - "--retry", "5", - "--retry-delay", "5", - "--retry-all-errors", - "-o", dest, url) - cmd.Stdout = GinkgoWriter - cmd.Stderr = GinkgoWriter - Expect(cmd.Run()).To(Succeed(), "failed to download %s", url) + var err error + for attempt := 1; attempt <= 6; attempt++ { + if attempt > 1 { + time.Sleep(5 * time.Second) + } + cmd := exec.Command("curl", "-sSfL", + "--connect-timeout", "30", + "-C", "-", + "--speed-limit", "1048576", + "--speed-time", "120", + "-o", dest, url) + cmd.Stdout = GinkgoWriter + cmd.Stderr = GinkgoWriter + if err = cmd.Run(); err == nil { + break + } + } + Expect(err).NotTo(HaveOccurred(), "failed to download %s", url) fi, err := os.Stat(dest) Expect(err).NotTo(HaveOccurred()) Expect(fi.Size()).To(BeNumerically(">", 1024), "downloaded file is suspiciously small")