diff --git a/Makefile b/Makefile index 35e5d2251..c6f38b1d0 100644 --- a/Makefile +++ b/Makefile @@ -645,6 +645,9 @@ test-extra: prepare-test-extra ## suite against it. ## BACKEND_TEST_MODEL_URL?=https://huggingface.co/Qwen/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q8_0.gguf +## Suite timeout for `go test`. Wrappers whose model download alone can eat +## most of the default (multi-GB models on a slow HF CDN day) override this. +BACKEND_TEST_TIMEOUT?=30m ## Generic target — runs the suite against whatever BACKEND_IMAGE points at. ## Depends on protogen-go so pkg/grpc/proto is generated before `go test`. @@ -672,7 +675,7 @@ test-extra-backend: protogen-go BACKEND_TEST_FACE_IMAGE_3_URL="$$BACKEND_TEST_FACE_IMAGE_3_URL" \ BACKEND_TEST_FACE_IMAGE_3_FILE="$$BACKEND_TEST_FACE_IMAGE_3_FILE" \ BACKEND_TEST_VERIFY_DISTANCE_CEILING="$$BACKEND_TEST_VERIFY_DISTANCE_CEILING" \ - go test -v -timeout 30m ./tests/e2e-backends/... + go test -v -timeout $(BACKEND_TEST_TIMEOUT) ./tests/e2e-backends/... ## Convenience wrappers: build the image, then exercise it. test-extra-backend-llama-cpp: docker-build-llama-cpp @@ -1012,6 +1015,7 @@ test-extra-backend-vibevoice-cpp-tts: docker-build-vibevoice-cpp ## post-image disk budget. test-extra-backend-vibevoice-cpp-transcription: docker-build-vibevoice-cpp BACKEND_IMAGE=local-ai-backend:vibevoice-cpp \ + BACKEND_TEST_TIMEOUT=120m \ BACKEND_TEST_MODEL_URL='https://huggingface.co/mudler/vibevoice.cpp-models/resolve/main/vibevoice-asr-q4_k.gguf#vibevoice-asr-q4_k.gguf' \ BACKEND_TEST_EXTRA_FILES='https://huggingface.co/mudler/vibevoice.cpp-models/resolve/main/tokenizer.gguf#tokenizer.gguf' \ BACKEND_TEST_AUDIO_URL=https://github.com/ggml-org/whisper.cpp/raw/master/samples/jfk.wav \ diff --git a/tests/e2e-backends/backend_test.go b/tests/e2e-backends/backend_test.go index c60077fd7..7ac3ad410 100644 --- a/tests/e2e-backends/backend_test.go +++ b/tests/e2e-backends/backend_test.go @@ -1453,20 +1453,34 @@ func extractImage(image, dest string) { // downloadFile fetches url into dest using curl -L. Used for CI convenience; // local runs can use BACKEND_TEST_MODEL_FILE to skip downloading. -// Retry flags guard against transient CI network hiccups (github.com in -// particular has been flaky from GHA runners, timing out TCP connects). +// Model files can be huge (the vibevoice ASR e2e model is >10 GB), so a +// wall-clock cap per attempt (--max-time) is the wrong guard: on a slow CDN +// day no attempt can ever finish, and curl's --retry restarts from byte +// zero, so repeated attempts make no forward progress (this hung the +// vibevoice transcription job until the suite timeout). Instead, abort only +// on a real stall (--speed-limit/--speed-time) and resume from the bytes +// already on disk (-C -), with the retry loop out here because curl does +// not re-evaluate the resume offset on its internal retries. func downloadFile(url, dest string) { GinkgoHelper() - cmd := exec.Command("curl", "-sSfL", - "--connect-timeout", "30", - "--max-time", "600", - "--retry", "5", - "--retry-delay", "5", - "--retry-all-errors", - "-o", dest, url) - cmd.Stdout = GinkgoWriter - cmd.Stderr = GinkgoWriter - Expect(cmd.Run()).To(Succeed(), "failed to download %s", url) + var err error + for attempt := 1; attempt <= 6; attempt++ { + if attempt > 1 { + time.Sleep(5 * time.Second) + } + cmd := exec.Command("curl", "-sSfL", + "--connect-timeout", "30", + "-C", "-", + "--speed-limit", "1048576", + "--speed-time", "120", + "-o", dest, url) + cmd.Stdout = GinkgoWriter + cmd.Stderr = GinkgoWriter + if err = cmd.Run(); err == nil { + break + } + } + Expect(err).NotTo(HaveOccurred(), "failed to download %s", url) fi, err := os.Stat(dest) Expect(err).NotTo(HaveOccurred()) Expect(fi.Size()).To(BeNumerically(">", 1024), "downloaded file is suspiciously small")