test: enforce offline resource replay

Assisted-by: Codex:gpt-5
Signed-off-by: Richard Palethorpe <io@richiejp.com>

.github/workflows/external-probes.yml

@@ -0,0 +1,38 @@
+---
+name: external compatibility probes
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '23 4 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  external-probe-huggingface-xet:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.26.x'
+          cache: false
+      - name: Probe Hugging Face Xet compatibility
+        run: LOCALAI_HF_XET_SMOKE=1 go test ./pkg/huggingface-api -ginkgo.focus='pinned public Xet fixture' -count=1
+
+  external-probe-sigstore:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.26.x'
+          cache: false
+      - name: Probe public Sigstore compatibility
+        env:
+          LOCALAI_COSIGN_LIVE: '1'
+          LOCALAI_COSIGN_LIVE_IMAGE: ${{ vars.LOCALAI_COSIGN_LIVE_IMAGE }}
+          LOCALAI_COSIGN_LIVE_ISSUER: ${{ vars.LOCALAI_COSIGN_LIVE_ISSUER }}
+          LOCALAI_COSIGN_LIVE_IDENTITY_REGEX: ${{ vars.LOCALAI_COSIGN_LIVE_IDENTITY_REGEX }}
+        run: go test ./pkg/oci/cosignverify -ginkgo.focus='VerifyImage' -count=1

.github/workflows/test.yml

@@ -53,13 +53,29 @@ jobs:
           node-version: '22'
       - name: Build React UI
         run: make react-ui
+      - name: Record and pack declared test resources
+        run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-test-resources TARGET=default
+      - name: Transfer local test-resource bundle
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-resources-default-${{ github.run_id }}
+          path: |
+            .cache/test-resources/bundles/default.tar
+            test-resources/manifests/lock.json
+      - name: Clear recorded resource cache
+        run: rm -rf .cache/test-resources
+      - name: Restore local test-resource bundle
+        uses: actions/download-artifact@v4
+        with:
+          name: test-resources-default-${{ github.run_id }}
+          path: .
       # Runs the core suite with coverage and fails if total coverage dropped
       # below the committed baseline (coverage-baseline.txt). The gate is
       # strict — any decrease fails. Raise the baseline with
       # `make test-coverage-baseline` and commit it when coverage rises.
       - name: Test (with coverage gate)
         run: |
-          PATH="$PATH:/root/go/bin" make --jobs 5 --output-sync=target test-coverage-check
+          LOCALAI_TEST_KERNEL_ENFORCE=1 PATH="$PATH:/root/go/bin" make --jobs 5 --output-sync=target test-coverage-check
       - name: Upload coverage report
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
@@ -113,7 +129,7 @@ jobs:
           # Used to run the newer GNUMake version from brew that supports --output-sync
           export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH"
           PATH="$PATH:$HOME/go/bin" make protogen-go
-          PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target test
+          PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target TARGET=default-darwin test
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3.23

.github/workflows/tests-aio.yml

@@ -76,7 +76,9 @@ jobs:
           PATH="$PATH:$HOME/go/bin" make protogen-go
       - name: Test
         run: |
-            PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e e2e-aio
+            PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e
+            LOCALAI_TEST_RESOURCES_ONLINE=1 PATH="$PATH:$HOME/go/bin" make update-test-resources TARGET=aio
+            LOCALAI_BACKEND_DIR="$GITHUB_WORKSPACE/backends" LOCALAI_MODELS_DIR="$GITHUB_WORKSPACE/tests/e2e-aio/models" LOCALAI_IMAGE_TAG=tests LOCALAI_IMAGE=local-ai PATH="$PATH:$HOME/go/bin" make run-e2e-aio
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3.23

Makefile

@@ -215,11 +215,11 @@ update-test-resources:
 	$(GOCMD) run ./cmd/test-resources update "$(TARGET)" "$(TEST_RESOURCE_MANIFESTS)" "$(TEST_RESOURCE_CACHE)"
 
 test: TARGET=default
-test: test-resources test-network-lint prepare-test
+test: test-network-lint prepare-test
 	@echo 'Running tests'
 	export GO_TAGS="debug"
 	OPUS_SHIM_LIBRARY=$(abspath ./pkg/opus/shim/libopusshim.so) \
-	$(OFFLINE_RUN) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) --fail-fast -v -r $(TEST_PATHS)
+	$(OFFLINE_RUN) $(TARGET) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) --fail-fast -v -r $(TEST_PATHS)
 
 ## Compiles and runs the standalone C++ unit tests for the backends (pure
 ## helpers that depend only on the stdlib + nlohmann/json, no full backend
@@ -247,7 +247,7 @@ test-ci-scripts:
 ## --fail-fast so a single failure doesn't truncate the coverage number, and
 ## uses covermode=atomic so the result is deterministic. Prints the total.
 test-coverage: TARGET=default
-test-coverage: test-resources test-network-lint prepare-test
+test-coverage: test-network-lint prepare-test
 	@echo 'Running tests with coverage (test failures stop before the percentage ratchet)'
 	GINKGO_TAGS="$(COVERAGE_TAGS)" \
 	COVERAGE_COVERPKG="$(COVERAGE_COVERPKG)" \
@@ -258,7 +258,7 @@ test-coverage: test-resources test-network-lint prepare-test
 	COVERAGE_PROGRESS_AFTER="$(COVERAGE_PROGRESS_AFTER)" \
 	COVERAGE_EXCLUDE_RE='$(COVERAGE_EXCLUDE_RE)' \
 	OPUS_SHIM_LIBRARY=$(abspath ./pkg/opus/shim/libopusshim.so) \
-	$(OFFLINE_RUN) scripts/run-coverage.sh $(COVERAGE_DIR) $(COVERAGE_PROFILE) $(TEST_FLAKES) $(COVERAGE_ROOTS)
+	$(OFFLINE_RUN) $(TARGET) scripts/run-coverage.sh $(COVERAGE_DIR) $(COVERAGE_PROFILE) $(TEST_FLAKES) $(COVERAGE_ROOTS)
 	@$(GOCMD) tool cover -html=$(COVERAGE_PROFILE) -o $(COVERAGE_DIR)/coverage.html
 	@$(GOCMD) tool cover -func=$(COVERAGE_PROFILE) | tail -n1
 
@@ -340,17 +340,17 @@ e2e-aio:
 	$(MAKE) run-e2e-aio
 
 run-e2e-aio: TARGET=aio
-run-e2e-aio: test-resources protogen-go
+run-e2e-aio: protogen-go
 	@echo 'Running e2e AIO tests'
-	$(OFFLINE_RUN) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio
+	$(OFFLINE_RUN) $(TARGET) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio
 
 # Distributed architecture e2e (PostgreSQL + NATS via testcontainers).
 # Includes NatsJWT specs (JWT-enabled NATS). Requires Docker.
 # VLLMMultinode is excluded here; use test-e2e-vllm-multinode for that.
 test-e2e-distributed: TARGET=distributed-e2e
-test-e2e-distributed: test-resources protogen-go
+test-e2e-distributed: protogen-go
 	@echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)'
-	$(OFFLINE_RUN) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e/distributed
+	$(OFFLINE_RUN) $(TARGET) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e/distributed
 
 # vLLM multi-node DP smoke (CPU). Builds local-ai:tests and the
 # cpu-vllm backend from the current working tree, then drives a

cmd/test-resources/main.go

@@ -4,16 +4,18 @@ package main
 
 import (
 	"crypto/sha256"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 
+	"github.com/mudler/LocalAI/core/services/cloudproxy/mitm"
 	"github.com/mudler/LocalAI/internal/testresources"
 	"github.com/mudler/LocalAI/pkg/httpclient"
 )
@@ -26,8 +28,11 @@ func main() {
 }
 
 func run(args []string) error {
+	if len(args) >= 6 && args[0] == "run" && args[4] == "--" {
+		return runOffline(args[1], args[2], args[3], args[5:])
+	}
 	if len(args) != 4 {
-		return errors.New("usage: test-resources <prepare|update> TARGET MANIFEST_DIR CACHE_DIR")
+		return errors.New("usage: test-resources <prepare|update> TARGET MANIFEST_DIR CACHE_DIR | test-resources run TARGET MANIFEST_DIR CACHE_DIR -- COMMAND")
 	}
 	target, manifestDir, cacheDir := args[1], args[2], args[3]
 	if args[0] == "update" {
@@ -36,6 +41,10 @@ func run(args []string) error {
 	if args[0] != "prepare" {
 		return errors.New("usage: test-resources <prepare|update> TARGET MANIFEST_DIR CACHE_DIR")
 	}
+	return prepare(target, manifestDir, cacheDir)
+}
+
+func prepare(target, manifestDir, cacheDir string) error {
 	manifest, err := testresources.LoadManifest(filepath.Join(manifestDir, target+".json"))
 	if err != nil {
 		return fmt.Errorf("%w; run `make update-test-resources TARGET=%s`", err, target)
@@ -47,34 +56,51 @@ func run(args []string) error {
 	if err != nil {
 		return err
 	}
-	if _, ok := lock.Bundles[target]; !ok {
+	locked, ok := lock.Bundles[target]
+	if !ok {
 		return fmt.Errorf("cache bundle is not locked for target %q; run `make update-test-resources TARGET=%s`", target, target)
 	}
+	if digest, ok := strings.CutPrefix(locked, "sha256:"); ok {
+		bundlePath := filepath.Join(cacheDir, "bundles", target+".tar")
+		if err := testresources.RestoreBundle(cacheDir, bundlePath, digest); err != nil {
+			return preparationError(target, err)
+		}
+	}
 	materialized := filepath.Join(cacheDir, "materialized", target)
 	if err := os.MkdirAll(materialized, 0o755); err != nil {
 		return err
 	}
-	index := map[string]string{}
+	index, err := testresources.LoadHTTPIndex(cacheDir)
+	if err != nil {
+		return preparationError(target, err)
+	}
 	for _, resource := range manifest.HTTP {
-		path, err := testresources.VerifyBlob(cacheDir, resource.SHA256)
+		_, err := testresources.VerifyBlob(cacheDir, resource.SHA256)
 		if err != nil {
 			return preparationError(target, err)
 		}
-		index[resource.Method+" "+resource.URL] = path
+		entry, ok := index[testresources.RequestKey(resource.Method, resource.URL, resource.Headers())]
+		if !ok || entry.Digest != resource.SHA256 {
+			return preparationError(target, fmt.Errorf("HTTP cache entry missing or mismatched: %s %s", resource.Method, resource.URL))
+		}
 	}
 	for _, resource := range manifest.Files {
 		path, err := testresources.VerifyBlob(cacheDir, resource.SHA256)
 		if err != nil {
 			return preparationError(target, err)
 		}
+		environmentPath := path
 		if resource.Destination != "" {
 			destination := filepath.Join(materialized, resource.Destination)
 			if err := copyFile(path, destination); err != nil {
 				return err
 			}
+			environmentPath = destination
 		}
 		if resource.Environment != "" {
-			index["env:"+resource.Environment] = path
+			if err := os.Setenv(resource.Environment, environmentPath); err != nil {
+				return err
+			}
 		}
 	}
 	for _, resource := range manifest.Images {
@@ -88,7 +114,7 @@ func run(args []string) error {
 			return fmt.Errorf("load declared image %s: %w", resource.Reference, err)
 		}
 	}
-	return writeIndex(filepath.Join(cacheDir, "index.json"), index)
+	return nil
 }
 
 func update(target, manifestDir, cacheDir string) error {
@@ -99,20 +125,21 @@ func update(target, manifestDir, cacheDir string) error {
 	if err != nil {
 		return err
 	}
-	client := httpclient.New(httpclient.WithFollowRedirects())
+	client := httpclient.New()
+	client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }
+	index, err := testresources.LoadHTTPIndex(cacheDir)
+	if err != nil {
+		return err
+	}
 	for _, resource := range manifest.HTTP {
-		if resource.Method != http.MethodGet && resource.Method != http.MethodHead {
-			return fmt.Errorf("recording HTTP method %s requires the replay proxy recorder", resource.Method)
-		}
-		if resource.Method == http.MethodHead {
-			if err := storeVerified(strings.NewReader(""), resource.SHA256, cacheDir); err != nil {
-				return err
-			}
-			continue
-		}
-		if err := fetch(client, resource.URL, resource.SHA256, cacheDir); err != nil {
+		entry, err := fetchHTTP(client, resource, cacheDir)
+		if err != nil {
 			return err
 		}
+		index[testresources.RequestKey(resource.Method, resource.URL, resource.Headers())] = entry
+	}
+	if err := testresources.WriteHTTPIndex(cacheDir, index); err != nil {
+		return err
 	}
 	for _, resource := range manifest.Files {
 		if err := fetch(client, resource.URL, resource.SHA256, cacheDir); err != nil {
@@ -124,7 +151,36 @@ func update(target, manifestDir, cacheDir string) error {
 			return err
 		}
 	}
-	return nil
+	bundlePath := filepath.Join(cacheDir, "bundles", target+".tar")
+	digest, err := testresources.PackBundle(cacheDir, bundlePath, manifest)
+	if err != nil {
+		return err
+	}
+	lockPath := filepath.Join(manifestDir, "lock.json")
+	lock, err := testresources.LoadLock(lockPath)
+	if err != nil {
+		return err
+	}
+	lock.Bundles[target] = "sha256:" + digest
+	return testresources.WriteLock(lockPath, lock)
+}
+
+func fetchHTTP(client *http.Client, resource testresources.HTTP, cacheDir string) (testresources.HTTPEntry, error) {
+	request, err := http.NewRequest(resource.Method, resource.URL, nil)
+	if err != nil {
+		return testresources.HTTPEntry{}, err
+	}
+	request.Header = resource.Headers()
+	response, err := client.Do(request)
+	if err != nil {
+		return testresources.HTTPEntry{}, fmt.Errorf("fetch %s: %w", resource.URL, err)
+	}
+	defer func() { _ = response.Body.Close() }()
+	size, err := storeVerified(response.Body, resource.SHA256, cacheDir)
+	if err != nil {
+		return testresources.HTTPEntry{}, err
+	}
+	return testresources.HTTPEntry{Digest: resource.SHA256, Size: size, Status: response.StatusCode, Header: testresources.SanitizeHeaders(response.Header)}, nil
 }
 
 func fetch(client *http.Client, rawURL, expected, cacheDir string) error {
@@ -143,32 +199,35 @@ func fetch(client *http.Client, rawURL, expected, cacheDir string) error {
 		}
 		return fmt.Errorf("fetch %s: status %s", rawURL, response.Status)
 	}
-	storeErr := storeVerified(response.Body, expected, cacheDir)
+	_, storeErr := storeVerified(response.Body, expected, cacheDir)
 	return errors.Join(storeErr, response.Body.Close())
 }
 
-func storeVerified(reader io.Reader, expected, cacheDir string) error {
+func storeVerified(reader io.Reader, expected, cacheDir string) (int64, error) {
 	directory := filepath.Join(cacheDir, "blobs", "sha256")
 	if err := os.MkdirAll(directory, 0o755); err != nil {
-		return err
+		return 0, err
 	}
 	temporary, err := os.CreateTemp(directory, ".record-*")
 	if err != nil {
-		return err
+		return 0, err
 	}
 	temporaryName := temporary.Name()
 	defer func() { _ = os.Remove(temporaryName) }()
 	hash := sha256.New()
-	_, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader)
+	size, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader)
 	closeErr := temporary.Close()
 	if err := errors.Join(copyErr, closeErr); err != nil {
-		return err
+		return 0, err
 	}
 	actual := fmt.Sprintf("%x", hash.Sum(nil))
 	if actual != expected {
-		return fmt.Errorf("resource digest mismatch: expected sha256:%s, got sha256:%s", expected, actual)
+		return 0, fmt.Errorf("resource digest mismatch: expected sha256:%s, got sha256:%s", expected, actual)
 	}
-	return os.Rename(temporaryName, testresources.BlobPath(cacheDir, expected))
+	if err := os.Rename(temporaryName, testresources.BlobPath(cacheDir, expected)); err != nil {
+		return 0, err
+	}
+	return size, nil
 }
 
 func pullAndPack(reference, expected, cacheDir string) error {
@@ -186,7 +245,7 @@ func pullAndPack(reference, expected, cacheDir string) error {
 	if err := cmd.Start(); err != nil {
 		return err
 	}
-	storeErr := storeVerified(stdout, expected, cacheDir)
+	_, storeErr := storeVerified(stdout, expected, cacheDir)
 	waitErr := cmd.Wait()
 	return errors.Join(storeErr, waitErr)
 }
@@ -212,11 +271,81 @@ func copyFile(source, destination string) error {
 	return errors.Join(copyErr, in.Close(), out.Close())
 }
 
-func writeIndex(path string, index map[string]string) error {
-	data, err := json.MarshalIndent(index, "", "  ")
+func runOffline(target, manifestDir, cacheDir string, command []string) error {
+	manifest, err := testresources.LoadManifest(filepath.Join(manifestDir, target+".json"))
 	if err != nil {
 		return err
 	}
-	data = append(data, '\n')
-	return os.WriteFile(path, data, 0o644)
+	if err := prepare(target, manifestDir, cacheDir); err != nil {
+		return err
+	}
+	dockerNetwork := ""
+	if runtime.GOOS == "linux" && (len(manifest.Images) > 0 || target == "aio") {
+		dockerNetwork = fmt.Sprintf("localai-test-%d", os.Getpid())
+		create := exec.Command("docker", "network", "create", "--internal", dockerNetwork)
+		create.Stdout, create.Stderr = io.Discard, os.Stderr
+		if err := create.Run(); err != nil {
+			return fmt.Errorf("create internal test Docker network: %w", err)
+		}
+		defer func() { _ = exec.Command("docker", "network", "rm", dockerNetwork).Run() }()
+	}
+	index, err := testresources.LoadHTTPIndex(cacheDir)
+	if err != nil {
+		return err
+	}
+	hosts := make([]string, 0, len(manifest.HTTP))
+	seen := map[string]bool{}
+	for _, resource := range manifest.HTTP {
+		parsed, err := url.Parse(resource.URL)
+		if err != nil {
+			return err
+		}
+		if parsed.Hostname() != "" && !seen[parsed.Hostname()] {
+			hosts = append(hosts, parsed.Hostname())
+			seen[parsed.Hostname()] = true
+		}
+	}
+	caDir := filepath.Join(cacheDir, "ca")
+	ca, err := mitm.LoadOrCreateCA(caDir)
+	if err != nil {
+		return err
+	}
+	server, err := mitm.NewServer(mitm.Config{
+		Addr: "127.0.0.1:0", CA: ca, InterceptHosts: hosts, AllowPlainHTTP: true, InterceptAll: true,
+		Handler: func(w http.ResponseWriter, r *http.Request, _ string) {
+			key := testresources.RequestKey(r.Method, r.URL.String(), r.Header)
+			entry, ok := index[key]
+			if !ok {
+				http.Error(w, "undeclared test HTTP request: "+key, http.StatusGatewayTimeout)
+				return
+			}
+			if err := testresources.ReplayResponse(w, cacheDir, entry); err != nil {
+				http.Error(w, err.Error(), http.StatusBadGateway)
+			}
+		},
+	})
+	if err != nil {
+		return err
+	}
+	if err := server.Start(); err != nil {
+		return err
+	}
+	defer server.Stop()
+	proxyURL := "http://" + server.Addr()
+	caPath := filepath.Join(caDir, "ca.crt")
+	env := append(os.Environ(),
+		"LOCALAI_TEST_OFFLINE=1", "HTTP_PROXY="+proxyURL, "HTTPS_PROXY="+proxyURL,
+		"ALL_PROXY="+proxyURL, "http_proxy="+proxyURL, "https_proxy="+proxyURL,
+		"all_proxy="+proxyURL, "SSL_CERT_FILE="+caPath, "CURL_CA_BUNDLE="+caPath,
+		"REQUESTS_CA_BUNDLE="+caPath, "GIT_SSL_CAINFO="+caPath, "NODE_EXTRA_CA_CERTS="+caPath,
+		"NO_PROXY=localhost,127.0.0.0/8,::1,172.16.0.0/12,192.168.0.0/16",
+		"no_proxy=localhost,127.0.0.0/8,::1,172.16.0.0/12,192.168.0.0/16",
+		"TESTCONTAINERS_RYUK_DISABLED=true",
+	)
+	if dockerNetwork != "" {
+		env = append(env, "LOCALAI_TEST_DOCKER_NETWORK="+dockerNetwork)
+	}
+	cmd := exec.Command(command[0], command[1:]...)
+	cmd.Env, cmd.Stdin, cmd.Stdout, cmd.Stderr = env, os.Stdin, os.Stdout, os.Stderr
+	return cmd.Run()
 }

core/services/cloudproxy/mitm/proxy.go

@@ -23,15 +23,17 @@ import (
 // in its intercept allowlist; non-allowlisted hosts get a plain
 // TCP CONNECT tunnel.
 type Server struct {
-	addr            string
-	ca              *CA
-	interceptHosts  map[string]bool
-	handler         InterceptHandler
-	connectTimeout  time.Duration
-	dialTimeout     time.Duration
-	upstreamTLS     *tls.Config
-	events          pii.EventStore
-	eventSeq        atomic.Uint64
+	addr           string
+	ca             *CA
+	interceptHosts map[string]bool
+	handler        InterceptHandler
+	connectTimeout time.Duration
+	dialTimeout    time.Duration
+	upstreamTLS    *tls.Config
+	events         pii.EventStore
+	eventSeq       atomic.Uint64
+	allowPlainHTTP bool
+	interceptAll   bool
 
 	listener net.Listener
 	srv      *http.Server
@@ -51,6 +53,12 @@ type Config struct {
 	CA             *CA
 	InterceptHosts []string
 	Handler        InterceptHandler
+	// AllowPlainHTTP is used by the deterministic test-resource proxy.
+	// Production listeners leave it false and continue to require CONNECT.
+	AllowPlainHTTP bool
+	// InterceptAll prevents undeclared HTTPS hosts from being tunnelled by
+	// strict test-resource replay. Production listeners use the host allowlist.
+	InterceptAll bool
 	// EventStore optionally receives a proxy_connect event for every
 	// CONNECT, recording the destination host and whether the proxy
 	// intercepted or tunneled it. nil disables connect-event recording.
@@ -73,6 +81,8 @@ func NewServer(cfg Config) (*Server, error) {
 		ca:             cfg.CA,
 		interceptHosts: hosts,
 		handler:        cfg.Handler,
+		allowPlainHTTP: cfg.AllowPlainHTTP,
+		interceptAll:   cfg.InterceptAll,
 		connectTimeout: 30 * time.Second,
 		dialTimeout:    15 * time.Second,
 		upstreamTLS:    &tls.Config{NextProtos: []string{"http/1.1"}},
@@ -126,6 +136,10 @@ func (s *Server) Stop() {
 
 func (s *Server) handle(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodConnect {
+		if s.allowPlainHTTP && r.URL != nil && r.URL.IsAbs() {
+			s.handler(w, r, r.URL.Host)
+			return
+		}
 		http.Error(w, "this proxy only supports HTTPS via CONNECT", http.StatusMethodNotAllowed)
 		return
 	}
@@ -168,6 +182,9 @@ func (s *Server) recordConnectEvent(host string, intercepted bool) {
 // shouldIntercept reports whether host is in the allowlist. An
 // empty allowlist tunnels everything.
 func (s *Server) shouldIntercept(host string) bool {
+	if s.interceptAll {
+		return true
+	}
 	if len(s.interceptHosts) == 0 {
 		return false
 	}

core/services/testutil/testdb.go

@@ -2,11 +2,14 @@ package testutil
 
 import (
 	"context"
+	"fmt"
 	"runtime"
 	"time"
 
+	"github.com/mudler/LocalAI/internal/testfixtures"
 	"github.com/testcontainers/testcontainers-go"
 	tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres"
+	tcnetwork "github.com/testcontainers/testcontainers-go/network"
 	"github.com/testcontainers/testcontainers-go/wait"
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
@@ -23,17 +26,22 @@ func SetupTestDB() *gorm.DB {
 		Skip("testcontainers requires Docker, not available on macOS CI")
 	}
 	ctx := context.Background()
-	pgC, err := tcpostgres.Run(ctx, "postgres:16",
+	Expect(testfixtures.RequireImage(ctx, testfixtures.Postgres16, "default")).To(Succeed())
+	testNetwork, err := testfixtures.DockerNetwork()
+	Expect(err).NotTo(HaveOccurred())
+	pgC, err := tcpostgres.Run(ctx, testfixtures.Postgres16,
 		tcpostgres.WithDatabase("testdb"),
 		tcpostgres.WithUsername("test"),
 		tcpostgres.WithPassword("test"),
 		testcontainers.WithWaitStrategyAndDeadline(60*time.Second,
 			wait.ForLog("database system is ready to accept connections").WithOccurrence(2)),
+		tcnetwork.WithNetworkName([]string{"postgres"}, testNetwork),
 	)
 	Expect(err).ToNot(HaveOccurred())
 	DeferCleanup(func() { pgC.Terminate(context.Background()) })
-	connStr, err := pgC.ConnectionString(ctx, "sslmode=disable")
+	endpoint, err := testfixtures.ContainerEndpoint(ctx, pgC, "5432")
 	Expect(err).ToNot(HaveOccurred())
+	connStr := fmt.Sprintf("postgres://test:test@%s/testdb?sslmode=disable", endpoint)
 	db, err := gorm.Open(postgres.Open(connStr), &gorm.Config{
 		Logger: logger.Default.LogMode(logger.Silent),
 	})

docs/content/development/offline-tests.md

@@ -21,12 +21,34 @@ cache from pinned declarations only by explicitly enabling online mode:
 LOCALAI_TEST_RESOURCES_ONLINE=1 make update-test-resources TARGET=default
 ```
 
-Ordinary test recipes execute through `scripts/run-test-offline.sh`. It denies
-public HTTP(S) through a closed proxy while allowing loopback and private
-Docker networks. Linux CI may additionally place this runner in a restricted
-network namespace; macOS relies on the proxy, declared resources, and guarded
-Go transports because it has no equivalent portable kernel-level subprocess
-filter.
+The update command records declared responses, files, and digest-pinned images,
+then writes a deterministic local bundle at
+`.cache/test-resources/bundles/<target>.tar`. Its SHA-256 is written to the
+lock file. Until registry publication is enabled, CI transfers this tar as a
+workflow artifact and verifies it after deleting the recording cache.
+
+HTTP declarations may include `request_headers`. `Range` participates in the
+cache key, and authorization values participate only through a SHA-256 value;
+credentials are never written verbatim to the cache index. Redirect responses
+are recorded without following them, so every hop needed by a test must be
+declared explicitly.
+
+Ordinary test recipes execute through `scripts/run-test-offline.sh`. Its
+supervised replay proxy terminates HTTP and HTTPS and returns an immediate
+error containing the method and URL for undeclared requests. Linux CI also
+runs the command in a cgroup with public IPv4 and IPv6 rejected; macOS relies
+on replay, declared resources, guarded Go transports, and static lint because
+kernel-level subprocess enforcement is Linux-only.
+
+Testcontainer images must be registry-digest pinned and loaded during
+preparation. Container helpers check that an image exists before startup and
+attach services to internal-only Docker networks, preventing testcontainers
+from silently pulling a missing tag.
+
+The default Linux and macOS suites use separate resource targets because
+Docker archives are platform-specific. Backend and hardware resources remain
+separate targets so ordinary contributors do not acquire large model fixtures
+that their test command does not use.
 
 Real third-party compatibility checks belong in separately named
 `external-probe-*` scheduled workflows and must not be part of deterministic

internal/testfixtures/images.go

@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: MIT
+
+// Package testfixtures centralizes immutable resources shared by test suites.
+package testfixtures
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+
+	"github.com/moby/moby/client"
+	"github.com/testcontainers/testcontainers-go"
+)
+
+const (
+	Postgres16       = "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20"
+	Postgres16Alpine = "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777"
+	NATS2Alpine      = "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0"
+)
+
+// RequireImage fails before testcontainers can fall back to a registry pull.
+func RequireImage(ctx context.Context, reference, target string) error {
+	docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return err
+	}
+	defer func() { _ = docker.Close() }()
+	if _, err := docker.ImageInspect(ctx, reference); err != nil {
+		return fmt.Errorf("required offline test image %s is not loaded; run `make test-resources TARGET=%s`: %w", reference, target, err)
+	}
+	return nil
+}
+
+func DockerNetwork() (string, error) {
+	name := os.Getenv("LOCALAI_TEST_DOCKER_NETWORK") //nolint:forbidigo
+	if name == "" {
+		return "", errors.New("offline test Docker network is not configured; run the test through scripts/run-test-offline.sh")
+	}
+	return name, nil
+}
+
+// ContainerEndpoint returns an address reachable from the Linux test host
+// without publishing a port from the internal-only Docker network.
+func ContainerEndpoint(ctx context.Context, container testcontainers.Container, port string) (string, error) {
+	ip, err := container.ContainerIP(ctx)
+	if err != nil {
+		return "", err
+	}
+	if ip == "" {
+		return "", errors.New("offline test container has no private network address")
+	}
+	return net.JoinHostPort(ip, port), nil
+}

internal/testresources/bundle.go

@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: MIT
+
+package testresources
+
+import (
+	"archive/tar"
+	"bytes"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+func PackBundle(cacheDir, output string, manifest Manifest) (string, error) {
+	index, err := LoadHTTPIndex(cacheDir)
+	if err != nil {
+		return "", err
+	}
+	targetIndex := map[string]HTTPEntry{}
+	digests := map[string]bool{}
+	for _, resource := range manifest.HTTP {
+		key := RequestKey(resource.Method, resource.URL, resource.Headers())
+		entry, ok := index[key]
+		if !ok {
+			return "", fmt.Errorf("cannot pack missing HTTP entry %s", key)
+		}
+		targetIndex[key], digests[resource.SHA256] = entry, true
+	}
+	for _, resource := range manifest.Files {
+		digests[resource.SHA256] = true
+	}
+	for _, resource := range manifest.Images {
+		digests[resource.SHA256] = true
+	}
+	if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil {
+		return "", err
+	}
+	tmp, err := os.CreateTemp(filepath.Dir(output), "bundle-*.tmp")
+	if err != nil {
+		return "", err
+	}
+	name := tmp.Name()
+	defer func() { _ = os.Remove(name) }()
+	hash := sha256.New()
+	tw := tar.NewWriter(io.MultiWriter(tmp, hash))
+	indexData, err := json.Marshal(targetIndex)
+	if err == nil {
+		err = writeTarBytes(tw, "http-index.json", indexData)
+	}
+	ordered := make([]string, 0, len(digests))
+	for digest := range digests {
+		ordered = append(ordered, digest)
+	}
+	sort.Strings(ordered)
+	for _, digest := range ordered {
+		if err != nil {
+			break
+		}
+		path, verifyErr := VerifyBlob(cacheDir, digest)
+		if verifyErr != nil {
+			err = verifyErr
+			break
+		}
+		var data []byte
+		data, err = os.ReadFile(path)
+		if err == nil {
+			err = writeTarBytes(tw, filepath.ToSlash(filepath.Join("blobs", "sha256", digest)), data)
+		}
+	}
+	err = errors.Join(err, tw.Close(), tmp.Close())
+	if err != nil {
+		return "", err
+	}
+	if err := os.Rename(name, output); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%x", hash.Sum(nil)), nil
+}
+
+func RestoreBundle(cacheDir, bundle, expected string) error {
+	data, err := os.ReadFile(bundle)
+	if err != nil {
+		return err
+	}
+	actual := fmt.Sprintf("%x", sha256.Sum256(data))
+	if actual != expected {
+		return fmt.Errorf("test resource bundle checksum mismatch: expected %s, got %s", expected, actual)
+	}
+	tr := tar.NewReader(bytes.NewReader(data))
+	recorded := map[string]HTTPEntry{}
+	for {
+		header, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		name := filepath.Clean(filepath.FromSlash(header.Name))
+		if filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) {
+			return fmt.Errorf("unsafe bundle path %q", header.Name)
+		}
+		body, err := io.ReadAll(tr)
+		if err != nil {
+			return err
+		}
+		if name == "http-index.json" {
+			if err := json.Unmarshal(body, &recorded); err != nil {
+				return err
+			}
+			continue
+		}
+		destination := filepath.Join(cacheDir, name)
+		if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {
+			return err
+		}
+		if err := os.WriteFile(destination, body, 0o644); err != nil {
+			return err
+		}
+	}
+	index, err := LoadHTTPIndex(cacheDir)
+	if err != nil {
+		return err
+	}
+	for key, entry := range recorded {
+		index[key] = entry
+	}
+	return WriteHTTPIndex(cacheDir, index)
+}
+
+func writeTarBytes(tw *tar.Writer, name string, data []byte) error {
+	header := &tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), ModTime: time.Unix(0, 0).UTC()}
+	if err := tw.WriteHeader(header); err != nil {
+		return err
+	}
+	_, err := tw.Write(data)
+	return err
+}

internal/testresources/httpcache.go

@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: MIT
+
+package testresources
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+var hopHeaders = map[string]bool{
+	"Connection": true, "Proxy-Connection": true, "Keep-Alive": true,
+	"Transfer-Encoding": true, "Content-Length": true, "Te": true,
+	"Trailer": true, "Upgrade": true, "Proxy-Authenticate": true,
+	"Proxy-Authorization": true,
+}
+
+func LoadHTTPIndex(cacheDir string) (map[string]HTTPEntry, error) {
+	index := map[string]HTTPEntry{}
+	data, err := os.ReadFile(filepath.Join(cacheDir, "index.json"))
+	if errors.Is(err, os.ErrNotExist) {
+		return index, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read HTTP cache index: %w", err)
+	}
+	if err := json.Unmarshal(data, &index); err != nil {
+		return nil, fmt.Errorf("parse HTTP cache index: %w", err)
+	}
+	return index, nil
+}
+
+func WriteHTTPIndex(cacheDir string, index map[string]HTTPEntry) error {
+	data, err := json.MarshalIndent(index, "", "  ")
+	if err != nil {
+		return err
+	}
+	data = append(data, '\n')
+	if err := os.MkdirAll(cacheDir, 0o755); err != nil {
+		return err
+	}
+	tmp, err := os.CreateTemp(cacheDir, "index-*.tmp")
+	if err != nil {
+		return err
+	}
+	name := tmp.Name()
+	defer func() { _ = os.Remove(name) }()
+	if _, err := tmp.Write(data); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(name, filepath.Join(cacheDir, "index.json"))
+}
+
+func SanitizeHeaders(header http.Header) http.Header {
+	out := header.Clone()
+	for name := range hopHeaders {
+		out.Del(name)
+	}
+	return out
+}
+
+func ReplayResponse(w http.ResponseWriter, cacheDir string, entry HTTPEntry) error {
+	path, err := VerifyBlob(cacheDir, entry.Digest)
+	if err != nil {
+		return err
+	}
+	for name, values := range entry.Header {
+		for _, value := range values {
+			w.Header().Add(name, value)
+		}
+	}
+	w.Header().Set("Content-Length", fmt.Sprint(entry.Size))
+	w.WriteHeader(entry.Status)
+	if entry.Size == 0 {
+		return nil
+	}
+	body, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = body.Close() }()
+	_, err = io.Copy(w, body)
+	return err
+}

internal/testresources/resources.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -25,9 +26,17 @@ type Manifest struct {
 }
 
 type HTTP struct {
-	Method string `json:"method"`
-	URL    string `json:"url"`
-	SHA256 string `json:"sha256"`
+	Method         string            `json:"method"`
+	URL            string            `json:"url"`
+	SHA256         string            `json:"sha256"`
+	RequestHeaders map[string]string `json:"request_headers,omitempty"`
+}
+
+type HTTPEntry struct {
+	Digest string      `json:"digest"`
+	Size   int64       `json:"size"`
+	Status int         `json:"status"`
+	Header http.Header `json:"header"`
 }
 
 type File struct {
@@ -69,6 +78,15 @@ func LoadLock(path string) (Lock, error) {
 	return lock, nil
 }
 
+func WriteLock(path string, lock Lock) error {
+	data, err := json.MarshalIndent(lock, "", "  ")
+	if err != nil {
+		return err
+	}
+	data = append(data, '\n')
+	return os.WriteFile(path, data, 0o644)
+}
+
 func decode(path string, value any) error {
 	data, err := os.ReadFile(path)
 	if err != nil {
@@ -117,6 +135,33 @@ func BlobPath(cacheDir, digest string) string {
 	return filepath.Join(cacheDir, "blobs", "sha256", digest)
 }
 
+func RequestKey(method, rawURL string, headers ...http.Header) string {
+	key := strings.ToUpper(method) + " " + rawURL
+	if len(headers) == 0 {
+		return key
+	}
+	for _, name := range []string{"Authorization", "Range"} {
+		value := headers[0].Get(name)
+		if value == "" {
+			continue
+		}
+		if name == "Authorization" {
+			digest := sha256.Sum256([]byte(value))
+			value = "sha256:" + hex.EncodeToString(digest[:])
+		}
+		key += "\n" + strings.ToLower(name) + ":" + value
+	}
+	return key
+}
+
+func (resource HTTP) Headers() http.Header {
+	header := make(http.Header, len(resource.RequestHeaders))
+	for name, value := range resource.RequestHeaders {
+		header.Set(name, value)
+	}
+	return header
+}
+
 func VerifyBlob(cacheDir, digest string) (string, error) {
 	path := BlobPath(cacheDir, digest)
 	data, err := os.ReadFile(path)

internal/testresources/resources_test.go

@@ -5,6 +5,8 @@ package testresources_test
 import (
 	"crypto/sha256"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 
@@ -46,4 +48,65 @@ var _ = Describe("Declared test resources", func() {
 		Expect(os.WriteFile(path, content, 0o644)).To(Succeed())
 		Expect(testresources.VerifyBlob(cache, digest)).To(Equal(path))
 	})
+
+	It("persists response metadata and replays a verified body", func() {
+		cache := GinkgoT().TempDir()
+		content := []byte("cached response")
+		digest := fmt.Sprintf("%x", sha256.Sum256(content))
+		path := testresources.BlobPath(cache, digest)
+		Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed())
+		Expect(os.WriteFile(path, content, 0o644)).To(Succeed())
+		index := map[string]testresources.HTTPEntry{
+			"GET https://example.invalid/data": {
+				Digest: digest, Size: int64(len(content)), Status: http.StatusPartialContent,
+				Header: http.Header{"Content-Range": {"bytes 0-14/15"}},
+			},
+		}
+		Expect(testresources.WriteHTTPIndex(cache, index)).To(Succeed())
+		loaded, err := testresources.LoadHTTPIndex(cache)
+		Expect(err).NotTo(HaveOccurred())
+		recorder := httptest.NewRecorder()
+		Expect(testresources.ReplayResponse(recorder, cache, loaded["GET https://example.invalid/data"])).To(Succeed())
+		Expect(recorder.Code).To(Equal(http.StatusPartialContent))
+		Expect(recorder.Body.Bytes()).To(Equal(content))
+		Expect(recorder.Header().Get("Content-Range")).To(Equal("bytes 0-14/15"))
+	})
+
+	It("sanitizes connection-specific response headers", func() {
+		header := http.Header{"Transfer-Encoding": {"chunked"}, "Authorization": {"secret"}, "X-Fixture": {"yes"}}
+		clean := testresources.SanitizeHeaders(header)
+		Expect(clean).NotTo(HaveKey("Transfer-Encoding"))
+		Expect(clean).To(HaveKeyWithValue("Authorization", []string{"secret"}))
+		Expect(clean).To(HaveKeyWithValue("X-Fixture", []string{"yes"}))
+	})
+
+	It("keys range and authorization variants without storing credentials", func() {
+		header := http.Header{"Authorization": {"Bearer secret"}, "Range": {"bytes=4-"}}
+		key := testresources.RequestKey(http.MethodGet, "https://example.invalid/model", header)
+		Expect(key).To(ContainSubstring("range:bytes=4-"))
+		Expect(key).To(ContainSubstring("authorization:sha256:"))
+		Expect(key).NotTo(ContainSubstring("Bearer secret"))
+		Expect(key).NotTo(Equal(testresources.RequestKey(http.MethodGet, "https://example.invalid/model")))
+	})
+
+	It("packs deterministically and restores a target cache", func() {
+		cache := GinkgoT().TempDir()
+		content := []byte("bundle fixture")
+		digest := fmt.Sprintf("%x", sha256.Sum256(content))
+		path := testresources.BlobPath(cache, digest)
+		Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed())
+		Expect(os.WriteFile(path, content, 0o644)).To(Succeed())
+		manifest := testresources.Manifest{Version: 1, Target: "fixture", Files: []testresources.File{{URL: "https://example.invalid/file", SHA256: digest, Destination: "file"}}}
+		first := filepath.Join(GinkgoT().TempDir(), "first.tar")
+		second := filepath.Join(GinkgoT().TempDir(), "second.tar")
+		firstDigest, err := testresources.PackBundle(cache, first, manifest)
+		Expect(err).NotTo(HaveOccurred())
+		secondDigest, err := testresources.PackBundle(cache, second, manifest)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(secondDigest).To(Equal(firstDigest))
+
+		restored := GinkgoT().TempDir()
+		Expect(testresources.RestoreBundle(restored, first, firstDigest)).To(Succeed())
+		Expect(os.ReadFile(testresources.BlobPath(restored, digest))).To(Equal(content))
+	})
 })

pkg/httpclient/client.go

@@ -28,8 +28,11 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"time"
+
+	"github.com/mudler/LocalAI/pkg/testnetwork"
 )
 
 const (
@@ -105,12 +108,20 @@ func sameOrigin(a, b *url.URL) bool {
 // (e.g. a credential-injecting RoundTripper) should base it on this rather than
 // http.DefaultTransport so the TLS floor and timeouts are preserved.
 func HardenedTransport() *http.Transport {
+	dialContext := (&net.Dialer{
+		Timeout:   dialTimeout,
+		KeepAlive: dialKeepAlive,
+	}).DialContext
+	// This is set only by the test-resource supervisor before it starts the
+	// child process; production configuration does not cross this boundary.
+	if os.Getenv("LOCALAI_TEST_OFFLINE") == "1" { //nolint:forbidigo
+		guard := testnetwork.LocalGuard()
+		guard.Dial = dialContext
+		dialContext = guard.DialContext
+	}
 	return &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   dialTimeout,
-			KeepAlive: dialKeepAlive,
-		}).DialContext,
+		Proxy:                 http.ProxyFromEnvironment,
+		DialContext:           dialContext,
 		ForceAttemptHTTP2:     true,
 		MaxIdleConns:          maxIdleConns,
 		IdleConnTimeout:       idleConnTimeout,

pkg/huggingface-api/client_test.go

@@ -336,8 +336,12 @@ var _ = Describe("HuggingFace API Client", func() {
 
 	Context("when handling network errors", func() {
 		It("should handle connection failures gracefully", func() {
-			// Use an invalid URL to simulate connection failure
-			client.SetBaseURL("http://invalid-url-that-does-not-exist")
+			// A closed loopback listener produces a deterministic connection
+			// failure without relying on DNS or public network access.
+			closedServer := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+			closedURL := closedServer.URL
+			closedServer.Close()
+			client.SetBaseURL(closedURL)
 
 			params := hfapi.SearchParams{
 				Sort:      "lastModified",

pkg/oci/cosignverify/verify.go

@@ -33,6 +33,8 @@ import (
 	"github.com/sigstore/sigstore-go/pkg/root"
 	"github.com/sigstore/sigstore-go/pkg/tuf"
 	"github.com/sigstore/sigstore-go/pkg/verify"
+
+	"github.com/mudler/LocalAI/pkg/httpclient"
 )
 
 // Policy is the verification policy a backend image must satisfy.
@@ -288,7 +290,7 @@ func enforceNotBefore(result *verify.VerificationResult, cutoff time.Time) error
 func (v *Verifier) remoteOptions(ctx context.Context) []remote.Option {
 	t := v.transport
 	if t == nil {
-		t = http.DefaultTransport
+		t = httpclient.HardenedTransport()
 	}
 	// Match the retry policy used elsewhere in pkg/oci so transient
 	// registry hiccups don't fail verification.

scripts/run-test-linux-offline.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+if [[ $(uname -s) != Linux ]]; then
+  echo 'kernel-level test egress enforcement is Linux-only' >&2
+  exit 2
+fi
+if [[ $# -lt 2 ]]; then
+  echo "usage: $0 TARGET COMMAND [ARG...]" >&2
+  exit 2
+fi
+
+root=$(cd "$(dirname "$0")/.." && pwd)
+group="localai-test-$$"
+cgroup="/sys/fs/cgroup/$group"
+parent_cgroup="/sys/fs/cgroup$(awk -F: '$1 == "0" {print $3}' /proc/self/cgroup)"
+
+sudo mkdir "$cgroup"
+cleanup() {
+  echo $$ | sudo tee "$parent_cgroup/cgroup.procs" >/dev/null 2>&1 || true
+  sudo iptables -D OUTPUT -m cgroup --path "$group" -j REJECT 2>/dev/null || true
+  sudo iptables -D OUTPUT -m cgroup --path "$group" -d 192.168.0.0/16 -j ACCEPT 2>/dev/null || true
+  sudo iptables -D OUTPUT -m cgroup --path "$group" -d 172.16.0.0/12 -j ACCEPT 2>/dev/null || true
+  sudo iptables -D OUTPUT -m cgroup --path "$group" -d 10.0.0.0/8 -j ACCEPT 2>/dev/null || true
+  sudo iptables -D OUTPUT -m cgroup --path "$group" -d 127.0.0.0/8 -j ACCEPT 2>/dev/null || true
+  sudo ip6tables -D OUTPUT -m cgroup --path "$group" -d ::1/128 -j ACCEPT 2>/dev/null || true
+  sudo ip6tables -D OUTPUT -m cgroup --path "$group" -j REJECT 2>/dev/null || true
+  sudo rmdir "$cgroup" 2>/dev/null || true
+}
+trap cleanup EXIT INT TERM
+
+sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -j REJECT
+sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -d 192.168.0.0/16 -j ACCEPT
+sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -d 172.16.0.0/12 -j ACCEPT
+sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -d 10.0.0.0/8 -j ACCEPT
+sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -d 127.0.0.0/8 -j ACCEPT
+sudo ip6tables -I OUTPUT 1 -m cgroup --path "$group" -j REJECT
+sudo ip6tables -I OUTPUT 1 -m cgroup --path "$group" -d ::1/128 -j ACCEPT
+echo $$ | sudo tee "$cgroup/cgroup.procs" >/dev/null
+
+LOCALAI_TEST_KERNEL_ACTIVE=1 "$root/scripts/run-test-offline.sh" "$@"

scripts/run-test-offline.sh

@@ -2,21 +2,18 @@
 # SPDX-License-Identifier: MIT
 set -euo pipefail
 
-if [[ $# -lt 1 ]]; then
-  echo "usage: $0 COMMAND [ARG...]" >&2
+if [[ $# -lt 2 ]]; then
+  echo "usage: $0 TARGET COMMAND [ARG...]" >&2
   exit 2
 fi
 
-# A closed loopback proxy fails accidental HTTP(S) immediately while keeping
-# existing loopback fixtures and isolated container networks reachable.
-export HTTP_PROXY="http://127.0.0.1:1"
-export HTTPS_PROXY="$HTTP_PROXY"
-export ALL_PROXY="$HTTP_PROXY"
-export http_proxy="$HTTP_PROXY"
-export https_proxy="$HTTP_PROXY"
-export all_proxy="$HTTP_PROXY"
-export NO_PROXY="localhost,127.0.0.0/8,::1,172.16.0.0/12,192.168.0.0/16"
-export no_proxy="$NO_PROXY"
-export TESTCONTAINERS_RYUK_DISABLED=true
+target=$1
+shift
+root=$(cd "$(dirname "$0")/.." && pwd)
 
-exec "$@"
+if [[ ${LOCALAI_TEST_KERNEL_ENFORCE:-0} == 1 && ${LOCALAI_TEST_KERNEL_ACTIVE:-0} != 1 ]]; then
+  exec "$root/scripts/run-test-linux-offline.sh" "$target" "$@"
+fi
+
+exec go run "$root/cmd/test-resources" run "$target" \
+  "$root/test-resources/manifests" "${TEST_RESOURCE_CACHE:-$root/.cache/test-resources}" -- "$@"

scripts/test-network-lint.sh

@@ -2,9 +2,32 @@
 # SPDX-License-Identifier: MIT
 set -euo pipefail
 
-# Enforce the policy on additions while the existing loopback-only test client
-# call sites are migrated. The normal lint baseline must not make unrelated
-# changes responsible for historical debt.
+# The full-tree fingerprint makes this effective on a clean CI checkout (where
+# a worktree-only diff would always be empty). Most existing direct clients are
+# loopback fixtures; changing the inventory requires an intentional baseline
+# update after review.
+expected_inventory=2885a428cdab55eea357dae3ec47b3d44f9999b59542d06cd3d792cf491c76b3
+inventory=$(
+  {
+    rg --no-heading --no-line-number --glob '*_test.go' \
+      '(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)")' \
+      pkg core tests backend || true
+    rg --no-heading --no-line-number --glob '*.sh' '(curl|wget)[[:space:]]' tests backend || true
+  } | LC_ALL=C sort
+)
+if command -v sha256sum >/dev/null 2>&1; then
+  actual_inventory=$(printf '%s\n' "$inventory" | sha256sum | awk '{print $1}')
+else
+  actual_inventory=$(printf '%s\n' "$inventory" | shasum -a 256 | awk '{print $1}')
+fi
+if [[ $actual_inventory != "$expected_inventory" ]]; then
+  echo 'Test network mechanism inventory changed; remove the direct access or review and update the lint baseline:' >&2
+  echo "$inventory" >&2
+  exit 1
+fi
+
+# Also give contributors a focused diagnostic for newly introduced remote
+# literals and direct mechanisms instead of only reporting the fingerprint.
 base=${TEST_NETWORK_LINT_BASE:-HEAD}
 violations=$(git diff --unified=0 "$base" -- api pkg core tests backend | \
 	  rg '^\+[^+].*(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)"|https?://)' | \

test-resources/manifests/aio.json

@@ -1 +1,12 @@
-{"version":1,"target":"aio"}
+{
+  "version": 1,
+  "target": "aio",
+  "files": [
+    {
+      "url": "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav",
+      "sha256": "37de21902b32aa2fc147ccbfdcc0566cc7061fffb2c0b10874f05147c0b9de0f",
+      "destination": "audio/micro-machines.wav",
+      "environment": "AIO_AUDIO_FIXTURE"
+    }
+  ]
+}

test-resources/manifests/default-darwin.json

@@ -0,0 +1,7 @@
+{
+  "version": 1,
+  "target": "default-darwin",
+  "http": [],
+  "images": [],
+  "files": []
+}

test-resources/manifests/default.json

@@ -1 +1,10 @@
-{"version":1,"target":"default"}
+{
+  "version": 1,
+  "target": "default",
+  "images": [
+    {
+      "reference": "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20",
+      "sha256": "bd98262690143a2a05167b3e924cde07e82ea269e45d5610e00a45f50e76d45a"
+    }
+  ]
+}

test-resources/manifests/distributed-e2e.json

@@ -1 +1,14 @@
-{"version":1,"target":"distributed-e2e"}
+{
+  "version": 1,
+  "target": "distributed-e2e",
+  "images": [
+    {
+      "reference": "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777",
+      "sha256": "fee330cbd34786da2b211fe2e4b7424d7bf974466cd3db0e376b2e4c457a339c"
+    },
+    {
+      "reference": "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0",
+      "sha256": "96e0f53430696eadaf1d7e250914ab78779cadc40df715a168ebe342e039b6f2"
+    }
+  ]
+}

test-resources/manifests/lock.json

@@ -1,10 +1,11 @@
 {
   "version": 1,
   "bundles": {
-    "aio": "embedded",
+    "aio": "sha256:03b05eedb51c853b0f05f2f3f592e2edc210e337388d3ff5a766680c0a166e55",
     "backend": "embedded",
-    "default": "embedded",
-    "distributed-e2e": "embedded",
+    "default": "sha256:185ebd3cfb994b9c1d1d1d9fad0a5d95c9de698b45e010276db388a9d66474bd",
+    "default-darwin": "embedded",
+    "distributed-e2e": "sha256:797dad68952914bf6612a42486086aa45eb17112067bec9955e2f5cf65a030dc",
     "external-probes": "embedded",
     "hardware": "embedded"
   }

tests/e2e-aio/e2e_suite_test.go

@@ -6,14 +6,13 @@ import (
 	"os"
 	"runtime"
 	"testing"
-	"time"
 
+	"github.com/mudler/LocalAI/internal/testfixtures"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/openai/openai-go/v3"
 	"github.com/openai/openai-go/v3/option"
 	"github.com/testcontainers/testcontainers-go"
-	"github.com/testcontainers/testcontainers-go/wait"
 )
 
 var container testcontainers.Container
@@ -39,10 +38,10 @@ var _ = BeforeSuite(func() {
 
 	if apiEndpoint == "" {
 		startDockerImage()
-		apiPort, err := container.MappedPort(context.Background(), defaultApiPort)
+		apiAddress, err := testfixtures.ContainerEndpoint(context.Background(), container, defaultApiPort)
 		Expect(err).To(Not(HaveOccurred()))
 
-		apiEndpoint = "http://localhost:" + apiPort.Port() + "/v1" // So that other tests can reference this value safely.
+		apiEndpoint = "http://" + apiAddress + "/v1" // test-network: fixture
 	} else {
 		GinkgoWriter.Printf("docker apiEndpoint set from env: %q\n", apiEndpoint)
 	}
@@ -122,15 +121,16 @@ func startDockerImage() {
 				Target: "/backends",
 			},
 		},
-		WaitingFor: wait.ForAll(
-			wait.ForListeningPort(defaultApiPort).WithStartupTimeout(10*time.Minute),
-			wait.ForHTTP("/v1/models").WithPort(defaultApiPort).WithStartupTimeout(10*time.Minute),
-		),
 	}
 
 	GinkgoWriter.Printf("Launching Docker Container %s:%s\n", containerImage, containerImageTag)
 
 	ctx := context.Background()
+	imageReference := fmt.Sprintf("%s:%s", containerImage, containerImageTag)
+	Expect(testfixtures.RequireImage(ctx, imageReference, "aio")).To(Succeed())
+	testNetwork, err := testfixtures.DockerNetwork()
+	Expect(err).NotTo(HaveOccurred())
+	req.Networks = []string{testNetwork}
 	c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
 		ContainerRequest: req,
 		Started:          true,

tests/e2e-aio/e2e_test.go

@@ -3,11 +3,13 @@ package e2e_test
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"path/filepath"
 
 	"github.com/mudler/LocalAI/core/schema"
 	. "github.com/onsi/ginkgo/v2"
@@ -257,6 +259,9 @@ var _ = Describe("E2E test", func() {
 
 		Context("vision", func() {
 			It("correctly", func() {
+				image, err := os.ReadFile(filepath.Join("..", "..", "core", "http", "static", "logo.png"))
+				Expect(err).NotTo(HaveOccurred())
+				imageURI := "data:image/png;base64," + base64.StdEncoding.EncodeToString(image)
 				model := "gpt-4o"
 				resp, err := client.Chat.Completions.New(context.TODO(),
 					openai.ChatCompletionNewParams{
@@ -276,7 +281,7 @@ var _ = Describe("E2E test", func() {
 											{
 												OfImageURL: &openai.ChatCompletionContentPartImageParam{
 													ImageURL: openai.ChatCompletionContentPartImageImageURLParam{
-														URL:    "https://picsum.photos/id/22/4434/3729",
+														URL:    imageURI,
 														Detail: "low",
 													},
 												},
@@ -289,7 +294,7 @@ var _ = Describe("E2E test", func() {
 					})
 				Expect(err).ToNot(HaveOccurred())
 				Expect(len(resp.Choices)).To(Equal(1), fmt.Sprint(resp))
-				Expect(resp.Choices[0].Message.Content).To(Or(ContainSubstring("man"), ContainSubstring("road")), fmt.Sprint(resp.Choices[0].Message.Content))
+				Expect(resp.Choices[0].Message.Content).NotTo(BeEmpty())
 			})
 		})
 
@@ -310,11 +315,7 @@ var _ = Describe("E2E test", func() {
 
 		Context("audio to text", func() {
 			It("correctly", func() {
-				downloadURL := "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav"
-				file, err := downloadHttpFile(downloadURL)
-				Expect(err).ToNot(HaveOccurred())
-
-				fileHandle, err := os.Open(file)
+				fileHandle, err := os.Open(preparedAudioFixture())
 				Expect(err).ToNot(HaveOccurred())
 				defer fileHandle.Close()
 
@@ -328,11 +329,7 @@ var _ = Describe("E2E test", func() {
 			})
 
 			It("with VTT format", func() {
-				downloadURL := "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav"
-				file, err := downloadHttpFile(downloadURL)
-				Expect(err).ToNot(HaveOccurred())
-
-				fileHandle, err := os.Open(file)
+				fileHandle, err := os.Open(preparedAudioFixture())
 				Expect(err).ToNot(HaveOccurred())
 				defer fileHandle.Close()
 
@@ -443,25 +440,10 @@ var _ = Describe("E2E test", func() {
 	})
 })
 
-func downloadHttpFile(url string) (string, error) {
-	resp, err := http.Get(url)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-
-	tmpfile, err := os.CreateTemp("", "example")
-	if err != nil {
-		return "", err
-	}
-	defer tmpfile.Close()
-
-	_, err = io.Copy(tmpfile, resp.Body)
-	if err != nil {
-		return "", err
-	}
-
-	return tmpfile.Name(), nil
+func preparedAudioFixture() string {
+	path := os.Getenv("AIO_AUDIO_FIXTURE")
+	Expect(path).NotTo(BeEmpty(), "run `make test-resources TARGET=aio` before the AIO suite")
+	return path
 }
 
 func requestRerank(modelName, query string, documents []string, topN *int, apiEndpoint string) (*http.Response, []byte) {

tests/e2e/distributed/nats_jwt_helpers_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/mudler/LocalAI/core/services/messaging"
+	"github.com/mudler/LocalAI/internal/testfixtures"
 	"github.com/mudler/LocalAI/pkg/natsauth"
 	"github.com/nats-io/jwt/v2"
 	"github.com/nats-io/nkeys"
@@ -17,6 +18,8 @@ import (
 
 	"github.com/testcontainers/testcontainers-go"
 	tcnats "github.com/testcontainers/testcontainers-go/modules/nats"
+	tcnetwork "github.com/testcontainers/testcontainers-go/network"
+	"github.com/testcontainers/testcontainers-go/wait"
 )
 
 // JWTTestInfra holds a NATS server configured with JWT auth and minted worker credentials.
@@ -34,6 +37,9 @@ func SetupJWTInfra() *JWTTestInfra {
 	GinkgoHelper()
 
 	infra := &JWTTestInfra{TestInfra: &TestInfra{Ctx: context.Background()}}
+	Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed())
+	testNetwork, err := testfixtures.DockerNetwork()
+	Expect(err).NotTo(HaveOccurred())
 
 	operatorJWT, accountJWT, accountSeed, err := jwtResolverMaterial()
 	Expect(err).ToNot(HaveOccurred())
@@ -51,15 +57,18 @@ resolver_preload: {
 
 	var natsContainer *tcnats.NATSContainer
 	// Override default testcontainers -js: JetStream fails without a system account in JWT mode.
-	natsContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine",
+	natsContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine,
 		tcnats.WithConfigFile(bytes.NewBufferString(conf)),
 		testcontainers.WithCmd("-c", "/etc/nats.conf"),
+		tcnetwork.WithNetworkName([]string{"nats"}, testNetwork),
+		testcontainers.WithWaitStrategy(wait.ForLog("Server is ready")),
 	)
 	Expect(err).ToNot(HaveOccurred())
 	infra.NATSContainer = natsContainer
 
-	infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx)
+	natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222")
 	Expect(err).ToNot(HaveOccurred())
+	infra.NatsURL = "nats://" + natsEndpoint
 
 	infra.NodeID = "550e8400-e29b-41d4-a716-446655440000"
 	cfg := natsauth.Config{AccountSeed: infra.AccountSeed, WorkerJWTTTL: time.Hour}
@@ -153,4 +162,4 @@ func accountPublicKeyFromSeed(accountSeed string) string {
 func nodeSubjectPrefix(nodeID string) string {
 	tok := strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-").Replace(nodeID)
 	return "nodes." + tok
-}
+}

tests/e2e/distributed/testhelpers_test.go

@@ -2,9 +2,11 @@ package distributed_test
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/mudler/LocalAI/core/services/messaging"
+	"github.com/mudler/LocalAI/internal/testfixtures"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -12,6 +14,7 @@ import (
 	"github.com/testcontainers/testcontainers-go"
 	tcnats "github.com/testcontainers/testcontainers-go/modules/nats"
 	tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres"
+	tcnetwork "github.com/testcontainers/testcontainers-go/network"
 	"github.com/testcontainers/testcontainers-go/wait"
 )
 
@@ -32,9 +35,13 @@ func SetupInfra(dbName string) *TestInfra {
 
 	infra := &TestInfra{Ctx: context.Background()}
 	var err error
+	Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.Postgres16Alpine, "distributed-e2e")).To(Succeed())
+	Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed())
+	testNetwork, err := testfixtures.DockerNetwork()
+	Expect(err).NotTo(HaveOccurred())
 
 	// Start PostgreSQL container
-	infra.PGContainer, err = tcpostgres.Run(infra.Ctx, "postgres:16-alpine",
+	infra.PGContainer, err = tcpostgres.Run(infra.Ctx, testfixtures.Postgres16Alpine,
 		tcpostgres.WithDatabase(dbName),
 		tcpostgres.WithUsername("test"),
 		tcpostgres.WithPassword("test"),
@@ -43,18 +50,23 @@ func SetupInfra(dbName string) *TestInfra {
 				WithOccurrence(2).
 				WithStartupTimeout(30*time.Second),
 		),
+		tcnetwork.WithNetworkName([]string{"postgres"}, testNetwork),
 	)
 	Expect(err).ToNot(HaveOccurred())
 
-	infra.PGURL, err = infra.PGContainer.ConnectionString(infra.Ctx, "sslmode=disable")
+	pgEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.PGContainer, "5432")
 	Expect(err).ToNot(HaveOccurred())
+	infra.PGURL = fmt.Sprintf("postgres://test:test@%s/%s?sslmode=disable", pgEndpoint, dbName)
 
 	// Start NATS container
-	infra.NATSContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine")
+	infra.NATSContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine,
+		tcnetwork.WithNetworkName([]string{"nats"}, testNetwork),
+		testcontainers.WithWaitStrategy(wait.ForLog("Server is ready")))
 	Expect(err).ToNot(HaveOccurred())
 
-	infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx)
+	natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222")
 	Expect(err).ToNot(HaveOccurred())
+	infra.NatsURL = "nats://" + natsEndpoint
 
 	// Connect messaging client
 	infra.NC, err = messaging.New(infra.NatsURL)
@@ -83,12 +95,18 @@ func SetupNATSOnly() *TestInfra {
 
 	infra := &TestInfra{Ctx: context.Background()}
 	var err error
+	Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed())
+	testNetwork, err := testfixtures.DockerNetwork()
+	Expect(err).NotTo(HaveOccurred())
 
-	infra.NATSContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine")
+	infra.NATSContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine,
+		tcnetwork.WithNetworkName([]string{"nats"}, testNetwork),
+		testcontainers.WithWaitStrategy(wait.ForLog("Server is ready")))
 	Expect(err).ToNot(HaveOccurred())
 
-	infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx)
+	natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222")
 	Expect(err).ToNot(HaveOccurred())
+	infra.NatsURL = "nats://" + natsEndpoint
 
 	infra.NC, err = messaging.New(infra.NatsURL)
 	Expect(err).ToNot(HaveOccurred())