test: parallelize coverage without remote fixtures

Assisted-by: Codex:gpt-5 [apply_patch] [exec_command]
Signed-off-by: Richard Palethorpe <io@richiejp.com>

.agents/building-and-testing.md

@@ -22,6 +22,7 @@ The core Go suites (`./pkg`, `./core`, plus the in-process integration suite `./
 
 - `make test-coverage` — runs the suites with `covermode=atomic` instrumentation and writes a merged profile to `coverage/coverage.out`. Uses the same prerequisites as `make test`.
   - Verbose Ginkgo output is written to `coverage/logs/<root>.log`, with the prior run retained as `<root>.log.previous`. The terminal prints one status line per root and a short failure extract. If any suite fails, no merged profile is produced and the percentage ratchet is explicitly not run. A lock under `coverage/` rejects concurrent runs, which would otherwise corrupt their shared profiles and logs.
+  - Suites run in parallel by default and each recursive root invocation has a five-minute budget. Override auto-detected parallelism with `COVERAGE_PROCS`; tune diagnostics with `COVERAGE_SUITE_TIMEOUT` and `COVERAGE_PROGRESS_AFTER`. A timeout is a performance failure to investigate, not a reason to raise the committed default.
   - **`--coverpkg` (`COVERAGE_COVERPKG = core/...,pkg/...`):** coverage is attributed to the core+pkg packages, not just the package under test. This is what lets the in-process `tests/e2e` suite (which drives the real HTTP server over loopback via `application.New`) credit the `core/http/endpoints/...` handlers it exercises — folding it in roughly doubled endpoint coverage (e.g. `endpoints/openai` 13.6% → 52%). The denominator is therefore *all* of `core`+`pkg` (minus generated proto, dropped via `COVERAGE_EXCLUDE_RE`), so the number isn't comparable to a plain per-package figure.
   - **Integration suites (`COVERAGE_E2E_ROOTS = ./tests/e2e`)** run non-recursively (excludes `tests/e2e/distributed`, which needs containers) with `--label-filter=!real-models` (those need a downloaded model) against the mock backend built by `prepare-test`. `tests/integration` is deliberately excluded — it needs `make backends/local-store`, which the coverage CI job doesn't build.
   - **Flake note:** folding integration tests into a *strict* gate means a hard e2e failure (or a spec that silently stops running) can fail the coverage gate, not just the test. `--flake-attempts` absorbs transient retryable failures; covermode=atomic keeps line coverage deterministic otherwise.

Makefile

@@ -99,6 +99,9 @@ COVERAGE_COVERPKG?=github.com/mudler/LocalAI/core/...,github.com/mudler/LocalAI/
 ## the coverage CI job doesn't do.
 COVERAGE_E2E_ROOTS?=./tests/e2e
 COVERAGE_E2E_LABELS?=!real-models
+COVERAGE_PROCS?=0
+COVERAGE_SUITE_TIMEOUT?=5m
+COVERAGE_PROGRESS_AFTER?=30s
 ## Drop generated protobuf from the denominator (it has no tests by design).
 COVERAGE_EXCLUDE_RE?=grpc/proto/.*[.]pb[.]go
 
@@ -232,6 +235,9 @@ test-coverage: prepare-test
 	COVERAGE_COVERPKG="$(COVERAGE_COVERPKG)" \
 	COVERAGE_E2E_ROOTS="$(COVERAGE_E2E_ROOTS)" \
 	COVERAGE_E2E_LABELS="$(COVERAGE_E2E_LABELS)" \
+	COVERAGE_PROCS="$(COVERAGE_PROCS)" \
+	COVERAGE_SUITE_TIMEOUT="$(COVERAGE_SUITE_TIMEOUT)" \
+	COVERAGE_PROGRESS_AFTER="$(COVERAGE_PROGRESS_AFTER)" \
 	COVERAGE_EXCLUDE_RE='$(COVERAGE_EXCLUDE_RE)' \
 	OPUS_SHIM_LIBRARY=$(abspath ./pkg/opus/shim/libopusshim.so) \
 	scripts/run-coverage.sh $(COVERAGE_DIR) $(COVERAGE_PROFILE) $(TEST_FLAKES) $(COVERAGE_ROOTS)

core/http/app_test.go

@@ -297,9 +297,7 @@ func getRequest(url string, header http.Header) (error, int, []byte) {
 	return nil, resp.StatusCode, body
 }
 
-const bertEmbeddingsURL = `https://gist.githubusercontent.com/mudler/0a080b166b87640e8644b09c2aee6e3b/raw/f0e8c26bb72edc16d9fbafbfd6638072126ff225/bert-embeddings-gallery.yaml`
-
-var _ = Describe("API test", func() {
+var _ = Describe("API test", Serial, func() {
 
 	var app *echo.Echo
 	var client *openai.Client
@@ -308,6 +306,7 @@ var _ = Describe("API test", func() {
 	var cancel context.CancelFunc
 	var tmpdir string
 	var modelDir string
+	var bertEmbeddingsURL string
 	// localAIApp captures the Application so AfterEach can synchronously
 	// stop the spawned gRPC backend processes. application.New cancels
 	// them asynchronously on context cancel, which races with test-binary
@@ -332,6 +331,17 @@ var _ = Describe("API test", func() {
 			modelDir = filepath.Join(tmpdir, "models")
 			err = os.Mkdir(modelDir, 0750)
 			Expect(err).ToNot(HaveOccurred())
+			fixtureDir := filepath.Join(modelDir, ".fixtures")
+			err = os.Mkdir(fixtureDir, 0750)
+			Expect(err).ToNot(HaveOccurred())
+			galleryFixturePath := filepath.Join(fixtureDir, "bert-embeddings-gallery.yaml")
+			err = os.WriteFile(galleryFixturePath, []byte("name: bert\nconfig_file: |\n  name: bert\n  backend: embeddings\n  usage: You can test this model with curl like this\n  parameters:\n    model: bert\n"), 0600)
+			Expect(err).ToNot(HaveOccurred())
+			bertEmbeddingsURL = "file://" + galleryFixturePath
+			// Additional files are cache inputs, not behavior under test here. Seed the
+			// destination so model application never reaches the public network.
+			err = os.WriteFile(filepath.Join(modelDir, "foo.yaml"), []byte("fixture: true\n"), 0600)
+			Expect(err).ToNot(HaveOccurred())
 
 			c, cancel = context.WithCancel(context.Background())
 
@@ -511,7 +521,7 @@ var _ = Describe("API test", func() {
 					fmt.Println(response)
 					resp = response
 					return response["processed"].(bool)
-				}, "360s", "10s").Should(Equal(true))
+				}, "30s", "50ms").Should(Equal(true))
 				Expect(resp["message"]).ToNot(ContainSubstring("error"))
 
 				dat, err := os.ReadFile(filepath.Join(modelDir, "bert2.yaml"))
@@ -556,7 +566,7 @@ var _ = Describe("API test", func() {
 				Eventually(func() bool {
 					response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid)
 					return response["processed"].(bool)
-				}, "360s", "10s").Should(Equal(true))
+				}, "30s", "50ms").Should(Equal(true))
 
 				dat, err := os.ReadFile(filepath.Join(modelDir, "bert.yaml"))
 				Expect(err).ToNot(HaveOccurred())
@@ -580,7 +590,7 @@ var _ = Describe("API test", func() {
 				Eventually(func() bool {
 					response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid)
 					return response["processed"].(bool)
-				}, "360s", "10s").Should(Equal(true))
+				}, "30s", "50ms").Should(Equal(true))
 
 				dat, err := os.ReadFile(filepath.Join(modelDir, "bert.yaml"))
 				Expect(err).ToNot(HaveOccurred())

core/http/openresponses_test.go

@@ -28,7 +28,7 @@ import (
 // the registered model is "Qwen3-VL-2B-Instruct-Q4_K_M", not the repo name.
 const testModel = "Qwen3-VL-2B-Instruct-Q4_K_M"
 
-var _ = Describe("Open Responses API", func() {
+var _ = Describe("Open Responses API", Serial, func() {
 	var app *echo.Echo
 	var localApp *application.Application
 	var localModelDir string

core/startup/model_preload.go

@@ -75,13 +75,18 @@ func InstallModelsWithOptions(ctx context.Context, galleryService *galleryop.Gal
 			}
 
 			var status *galleryop.OpStatus
-			// wait for op to finish
+			poll := time.NewTicker(50 * time.Millisecond)
+			defer poll.Stop()
 			for {
 				status = galleryService.GetStatus(uuid.String())
 				if status != nil && status.Processed {
 					break
 				}
-				time.Sleep(1 * time.Second)
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				case <-poll.C:
+				}
 			}
 
 			if status.Error != nil {

core/startup/model_preload_test.go

@@ -3,6 +3,8 @@ package startup_test
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 
@@ -39,7 +41,11 @@ var _ = Describe("Preload test", func() {
 
 	Context("Preloading from strings", func() {
 		It("loads from embedded full-urls", func() {
-			url := "https://raw.githubusercontent.com/mudler/LocalAI-examples/main/configurations/phi-2.yaml"
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				_, _ = w.Write([]byte("name: phi-2\nbackend: llama-cpp\nparameters:\n  model: phi-2.gguf\n"))
+			}))
+			defer server.Close()
+			url := server.URL + "/phi-2.yaml"
 			fileName := fmt.Sprintf("%s.yaml", "phi-2")
 
 			galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{
@@ -59,7 +65,11 @@ var _ = Describe("Preload test", func() {
 			Expect(string(content)).To(ContainSubstring("name: phi-2"))
 		})
 		It("downloads from urls", func() {
-			url := "huggingface://TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/tinyllama-1.1b-chat-v0.3.Q2_K.gguf"
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				_, _ = w.Write([]byte("tiny local GGUF fixture"))
+			}))
+			defer server.Close()
+			url := server.URL + "/tinyllama-1.1b-chat-v0.3.Q2_K.gguf"
 			fileName := fmt.Sprintf("%s.gguf", "tinyllama-1.1b-chat-v0.3.Q2_K")
 
 			galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{

pkg/downloader/cancel_test.go

@@ -59,9 +59,7 @@ var _ = Describe("Download cancellation", func() {
 	}
 
 	BeforeEach(func() {
-		dir, err := os.Getwd()
-		Expect(err).ToNot(HaveOccurred())
-		filePath = dir + "/cancel_model"
+		filePath = GinkgoT().TempDir() + "/cancel_model"
 	})
 
 	AfterEach(func() {
@@ -112,7 +110,7 @@ var _ = Describe("Download cancellation", func() {
 		Expect(err).To(HaveOccurred())
 		Expect(errors.Is(err, context.Canceled)).To(BeTrue())
 
-		Expect(filePath + ".partial").ToNot(BeAnExistingFile(),
+		Expect(filePath+".partial").ToNot(BeAnExistingFile(),
 			"a deliberate user cancel must not leave a dangling .partial behind")
 	})
 

pkg/downloader/stall_test.go

@@ -18,9 +18,7 @@ var _ = Describe("Download stall timeout", func() {
 	var savedTimeout time.Duration
 
 	BeforeEach(func() {
-		dir, err := os.Getwd()
-		Expect(err).ToNot(HaveOccurred())
-		filePath = dir + "/stall_model"
+		filePath = GinkgoT().TempDir() + "/stall_model"
 		savedTimeout = DownloadStallTimeout
 	})
 

pkg/downloader/uri_test.go

@@ -263,9 +263,7 @@ var _ = Describe("Download Test", func() {
 		_, err = _mockDataSha.Write(mockData)
 		Expect(err).ToNot(HaveOccurred())
 		mockDataSha = fmt.Sprintf("%x", _mockDataSha.Sum(nil))
-		dir, err := os.Getwd()
-		filePath = dir + "/my_supercool_model"
-		Expect(err).NotTo(HaveOccurred())
+		filePath = GinkgoT().TempDir() + "/my_supercool_model"
 	})
 
 	Context("URI DownloadFile", func() {

pkg/model/loader_test.go

@@ -65,8 +65,7 @@ var _ = Describe("ModelLoader", func() {
 
 	BeforeEach(func() {
 		// Setup the model loader with a test directory
-		modelPath = "/tmp/test_model_path"
-		os.Mkdir(modelPath, 0755)
+		modelPath = GinkgoT().TempDir()
 
 		systemState, err := system.GetSystemState(
 			system.WithModelPath(modelPath),
@@ -75,11 +74,6 @@ var _ = Describe("ModelLoader", func() {
 		modelLoader = model.NewModelLoader(systemState)
 	})
 
-	AfterEach(func() {
-		// Cleanup test directory
-		os.RemoveAll(modelPath)
-	})
-
 	Context("NewModelLoader", func() {
 		It("should create a new ModelLoader with an empty model map", func() {
 			Expect(modelLoader).ToNot(BeNil())

pkg/oci/blob.go

@@ -16,6 +16,10 @@ import (
 )
 
 func FetchImageBlob(ctx context.Context, r, reference, dst string, statusReader func(ocispec.Descriptor) io.Writer) error {
+	return fetchImageBlob(ctx, r, reference, dst, statusReader, false)
+}
+
+func fetchImageBlob(ctx context.Context, r, reference, dst string, statusReader func(ocispec.Descriptor) io.Writer, plainHTTP bool) error {
 	// 0. Create a file store for the output
 	fs, err := os.Create(dst)
 	if err != nil {
@@ -29,6 +33,7 @@ func FetchImageBlob(ctx context.Context, r, reference, dst string, statusReader
 		return fmt.Errorf("failed to create repository: %v", err)
 	}
 	repo.SkipReferrersGC = true
+	repo.PlainHTTP = plainHTTP
 
 	// Identify LocalAI to the registry. This mirrors oras' auth.DefaultClient
 	// (same retry policy) but advertises a LocalAI User-Agent instead of the

pkg/oci/blob_test.go

@@ -1,10 +1,14 @@
-package oci_test
+package oci
 
 import (
 	"context"
+	"crypto/sha256"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"strings"
 
-	. "github.com/mudler/LocalAI/pkg/oci" // Update with your module path
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -12,11 +16,22 @@ import (
 var _ = Describe("OCI", func() {
 	Context("pulling images", func() {
 		It("should fetch blobs correctly", func() {
+			payload := []byte("local OCI blob fixture")
+			digest := fmt.Sprintf("sha256:%x", sha256.Sum256(payload))
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Docker-Distribution-API-Version", "registry/2.0")
+				w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
+				if r.Method != http.MethodHead {
+					_, _ = w.Write(payload)
+				}
+			}))
+			defer server.Close()
 			f, err := os.CreateTemp("", "ollama")
 			Expect(err).NotTo(HaveOccurred())
 			defer os.RemoveAll(f.Name())
-			err = FetchImageBlob(context.TODO(), "registry.ollama.ai/library/gemma", "sha256:c1864a5eb19305c40519da12cc543519e48a0697ecd30e15d5ac228644957d12", f.Name(), nil)
+			err = fetchImageBlob(context.Background(), strings.TrimPrefix(server.URL, "http://")+"/library/gemma", digest, f.Name(), nil, true)
 			Expect(err).NotTo(HaveOccurred())
+			Expect(os.ReadFile(f.Name())).To(Equal(payload))
 		})
 	})
 })

pkg/oci/ollama.go

@@ -35,6 +35,10 @@ type LayerDetail struct {
 }
 
 func OllamaModelManifest(image string) (*Manifest, error) {
+	return ollamaModelManifest("https", "registry.ollama.ai", image)
+}
+
+func ollamaModelManifest(scheme, registry, image string) (*Manifest, error) {
 	// parse the repository and tag from `image`. `image` should be for e.g. gemma:2b, or foobar/gemma:2b
 
 	// if there is a : in the image, then split it
@@ -42,7 +46,7 @@ func OllamaModelManifest(image string) (*Manifest, error) {
 	tag, repository, image := ParseImageParts(image)
 
 	// get e.g. https://registry.ollama.ai/v2/library/llama3/manifests/latest
-	req, err := http.NewRequest("GET", "https://registry.ollama.ai/v2/"+repository+"/"+image+"/manifests/"+tag, nil)
+	req, err := http.NewRequest("GET", scheme+"://"+registry+"/v2/"+repository+"/"+image+"/manifests/"+tag, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -65,7 +69,11 @@ func OllamaModelManifest(image string) (*Manifest, error) {
 }
 
 func OllamaModelBlob(image string) (string, error) {
-	manifest, err := OllamaModelManifest(image)
+	return ollamaModelBlob("https", "registry.ollama.ai", image)
+}
+
+func ollamaModelBlob(scheme, registry, image string) (string, error) {
+	manifest, err := ollamaModelManifest(scheme, registry, image)
 	if err != nil {
 		return "", err
 	}
@@ -81,12 +89,16 @@ func OllamaModelBlob(image string) (string, error) {
 }
 
 func OllamaFetchModel(ctx context.Context, image string, output string, statusWriter func(ocispec.Descriptor) io.Writer) error {
+	return ollamaFetchModel(ctx, "https", "registry.ollama.ai", image, output, statusWriter)
+}
+
+func ollamaFetchModel(ctx context.Context, scheme, registry, image string, output string, statusWriter func(ocispec.Descriptor) io.Writer) error {
 	_, repository, imageNoTag := ParseImageParts(image)
 
-	blobID, err := OllamaModelBlob(image)
+	blobID, err := ollamaModelBlob(scheme, registry, image)
 	if err != nil {
 		return err
 	}
 
-	return FetchImageBlob(ctx, fmt.Sprintf("registry.ollama.ai/%s/%s", repository, imageNoTag), blobID, output, statusWriter)
+	return fetchImageBlob(ctx, fmt.Sprintf("%s/%s/%s", registry, repository, imageNoTag), blobID, output, statusWriter, scheme == "http")
 }

pkg/oci/ollama_test.go

@@ -1,10 +1,15 @@
-package oci_test
+package oci
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"strings"
 
-	. "github.com/mudler/LocalAI/pkg/oci" // Update with your module path
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -12,11 +17,26 @@ import (
 var _ = Describe("OCI", func() {
 	Context("ollama", func() {
 		It("pulls model files", func() {
+			payload := []byte("local Ollama model fixture")
+			digest := fmt.Sprintf("sha256:%x", sha256.Sum256(payload))
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Docker-Distribution-API-Version", "registry/2.0")
+				if strings.Contains(r.URL.Path, "/manifests/") {
+					_ = json.NewEncoder(w).Encode(Manifest{SchemaVersion: 2, Layers: []LayerDetail{{Digest: digest, MediaType: "application/vnd.ollama.image.model", Size: len(payload)}}})
+					return
+				}
+				w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
+				if r.Method != http.MethodHead {
+					_, _ = w.Write(payload)
+				}
+			}))
+			defer server.Close()
 			f, err := os.CreateTemp("", "ollama")
 			Expect(err).NotTo(HaveOccurred())
 			defer os.RemoveAll(f.Name())
-			err = OllamaFetchModel(context.TODO(), "gemma:2b", f.Name(), nil)
+			err = ollamaFetchModel(context.Background(), "http", strings.TrimPrefix(server.URL, "http://"), "gemma:2b", f.Name(), nil)
 			Expect(err).NotTo(HaveOccurred())
+			Expect(os.ReadFile(f.Name())).To(Equal(payload))
 		})
 	})
 })

scripts/run-coverage.sh

@@ -21,6 +21,9 @@
 #                        "!real-models" (those specs need a downloaded model).
 #   COVERAGE_EXCLUDE_RE  egrep pattern of profile lines to drop before merging,
 #                        e.g. generated protobuf (grpc/proto/.*\.pb\.go).
+#   COVERAGE_PROCS       parallel Ginkgo processes; 0 lets Ginkgo detect CPUs.
+#   COVERAGE_SUITE_TIMEOUT maximum duration of each recursive root (default 5m).
+#   COVERAGE_PROGRESS_AFTER emit diagnostics when a spec is slow (default 30s).
 #
 # Verbose Ginkgo output is retained in OUTPUT_DIR/logs. The previous run's log
 # for each root is kept with a .previous suffix, so a noisy failure remains
@@ -66,6 +69,14 @@ rm -f "$out_dir"/cover-*.out
 rm -f "$merged"
 fail=0
 
+procs="${COVERAGE_PROCS:-0}"
+suite_timeout="${COVERAGE_SUITE_TIMEOUT:-5m}"
+progress_after="${COVERAGE_PROGRESS_AFTER:-30s}"
+parallel_flags="-p --keep-going --timeout=$suite_timeout --poll-progress-after=$progress_after --poll-progress-interval=10s"
+if [ "$procs" -gt 0 ] 2>/dev/null; then
+	parallel_flags="$parallel_flags --procs=$procs --compilers=$procs"
+fi
+
 # Common optional flags go into "$@"; unquoted ${VAR:+...} would word-split a
 # --tags value that contains a space. The unit roots were captured above, so
 # overwriting the positional parameters here is safe.
@@ -112,7 +123,9 @@ for root in $unit_roots; do
 	log="$log_dir/$(log_name "$root")"
 	rotate_log "$log"
 	echo "run-coverage: testing $root (full output: $log)"
-	go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v -r "$@" \
+	# parallel_flags is intentionally word-split: it contains CLI arguments only.
+	# shellcheck disable=SC2086
+	go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --flake-attempts "$flakes" -v -r "$@" \
 		--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \
 		&& echo "run-coverage: PASS — $root" \
 		|| { fail=1; report_failure "$root" "$log"; }
@@ -125,13 +138,15 @@ for root in ${COVERAGE_E2E_ROOTS:-}; do
 	rotate_log "$log"
 	echo "run-coverage: testing $root (full output: $log)"
 	if [ -n "${COVERAGE_E2E_LABELS:-}" ]; then
-		go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \
+		# shellcheck disable=SC2086
+		go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --flake-attempts "$flakes" -v "$@" \
 			--label-filter="$COVERAGE_E2E_LABELS" \
 			--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \
 			&& echo "run-coverage: PASS — $root" \
 			|| { fail=1; report_failure "$root" "$log"; }
 	else
-		go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \
+		# shellcheck disable=SC2086
+		go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --flake-attempts "$flakes" -v "$@" \
 			--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \
 			&& echo "run-coverage: PASS — $root" \
 			|| { fail=1; report_failure "$root" "$log"; }