diff --git a/.github/workflows/external-probes.yml b/.github/workflows/external-probes.yml new file mode 100644 index 000000000..681f417e4 --- /dev/null +++ b/.github/workflows/external-probes.yml @@ -0,0 +1,38 @@ +--- +name: external compatibility probes + +on: + workflow_dispatch: + schedule: + - cron: '23 4 * * 1' + +permissions: + contents: read + +jobs: + external-probe-huggingface-xet: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v5 + with: + go-version: '1.26.x' + cache: false + - name: Probe Hugging Face Xet compatibility + run: LOCALAI_HF_XET_SMOKE=1 go test ./pkg/huggingface-api -ginkgo.focus='pinned public Xet fixture' -count=1 + + external-probe-sigstore: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v5 + with: + go-version: '1.26.x' + cache: false + - name: Probe public Sigstore compatibility + env: + LOCALAI_COSIGN_LIVE: '1' + LOCALAI_COSIGN_LIVE_IMAGE: ${{ vars.LOCALAI_COSIGN_LIVE_IMAGE }} + LOCALAI_COSIGN_LIVE_ISSUER: ${{ vars.LOCALAI_COSIGN_LIVE_ISSUER }} + LOCALAI_COSIGN_LIVE_IDENTITY_REGEX: ${{ vars.LOCALAI_COSIGN_LIVE_IDENTITY_REGEX }} + run: go test ./pkg/oci/cosignverify -ginkgo.focus='VerifyImage' -count=1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7e702a7dd..e28b242aa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,13 +53,29 @@ jobs: node-version: '22' - name: Build React UI run: make react-ui + - name: Record and pack declared test resources + run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-test-resources TARGET=default + - name: Transfer local test-resource bundle + uses: actions/upload-artifact@v4 + with: + name: test-resources-default-${{ github.run_id }} + path: | + .cache/test-resources/bundles/default.tar + test-resources/manifests/lock.json + - name: Clear recorded resource cache + run: rm -rf .cache/test-resources + - name: Restore local test-resource bundle + uses: actions/download-artifact@v4 + with: + name: test-resources-default-${{ github.run_id }} + path: . # Runs the core suite with coverage and fails if total coverage dropped # below the committed baseline (coverage-baseline.txt). The gate is # strict — any decrease fails. Raise the baseline with # `make test-coverage-baseline` and commit it when coverage rises. - name: Test (with coverage gate) run: | - PATH="$PATH:/root/go/bin" make --jobs 5 --output-sync=target test-coverage-check + LOCALAI_TEST_KERNEL_ENFORCE=1 PATH="$PATH:/root/go/bin" make --jobs 5 --output-sync=target test-coverage-check - name: Upload coverage report if: ${{ always() }} uses: actions/upload-artifact@v4 @@ -113,7 +129,7 @@ jobs: # Used to run the newer GNUMake version from brew that supports --output-sync export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH" PATH="$PATH:$HOME/go/bin" make protogen-go - PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target test + PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target TARGET=default-darwin test - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.23 diff --git a/.github/workflows/tests-aio.yml b/.github/workflows/tests-aio.yml index f8d3d34f0..712977faa 100644 --- a/.github/workflows/tests-aio.yml +++ b/.github/workflows/tests-aio.yml @@ -76,7 +76,9 @@ jobs: PATH="$PATH:$HOME/go/bin" make protogen-go - name: Test run: | - PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e e2e-aio + PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e + LOCALAI_TEST_RESOURCES_ONLINE=1 PATH="$PATH:$HOME/go/bin" make update-test-resources TARGET=aio + LOCALAI_BACKEND_DIR="$GITHUB_WORKSPACE/backends" LOCALAI_MODELS_DIR="$GITHUB_WORKSPACE/tests/e2e-aio/models" LOCALAI_IMAGE_TAG=tests LOCALAI_IMAGE=local-ai PATH="$PATH:$HOME/go/bin" make run-e2e-aio - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.23 diff --git a/Makefile b/Makefile index 089bbaeeb..bfd15c4e6 100644 --- a/Makefile +++ b/Makefile @@ -215,11 +215,11 @@ update-test-resources: $(GOCMD) run ./cmd/test-resources update "$(TARGET)" "$(TEST_RESOURCE_MANIFESTS)" "$(TEST_RESOURCE_CACHE)" test: TARGET=default -test: test-resources test-network-lint prepare-test +test: test-network-lint prepare-test @echo 'Running tests' export GO_TAGS="debug" OPUS_SHIM_LIBRARY=$(abspath ./pkg/opus/shim/libopusshim.so) \ - $(OFFLINE_RUN) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) --fail-fast -v -r $(TEST_PATHS) + $(OFFLINE_RUN) $(TARGET) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) --fail-fast -v -r $(TEST_PATHS) ## Compiles and runs the standalone C++ unit tests for the backends (pure ## helpers that depend only on the stdlib + nlohmann/json, no full backend @@ -247,7 +247,7 @@ test-ci-scripts: ## --fail-fast so a single failure doesn't truncate the coverage number, and ## uses covermode=atomic so the result is deterministic. Prints the total. test-coverage: TARGET=default -test-coverage: test-resources test-network-lint prepare-test +test-coverage: test-network-lint prepare-test @echo 'Running tests with coverage (test failures stop before the percentage ratchet)' GINKGO_TAGS="$(COVERAGE_TAGS)" \ COVERAGE_COVERPKG="$(COVERAGE_COVERPKG)" \ @@ -258,7 +258,7 @@ test-coverage: test-resources test-network-lint prepare-test COVERAGE_PROGRESS_AFTER="$(COVERAGE_PROGRESS_AFTER)" \ COVERAGE_EXCLUDE_RE='$(COVERAGE_EXCLUDE_RE)' \ OPUS_SHIM_LIBRARY=$(abspath ./pkg/opus/shim/libopusshim.so) \ - $(OFFLINE_RUN) scripts/run-coverage.sh $(COVERAGE_DIR) $(COVERAGE_PROFILE) $(TEST_FLAKES) $(COVERAGE_ROOTS) + $(OFFLINE_RUN) $(TARGET) scripts/run-coverage.sh $(COVERAGE_DIR) $(COVERAGE_PROFILE) $(TEST_FLAKES) $(COVERAGE_ROOTS) @$(GOCMD) tool cover -html=$(COVERAGE_PROFILE) -o $(COVERAGE_DIR)/coverage.html @$(GOCMD) tool cover -func=$(COVERAGE_PROFILE) | tail -n1 @@ -340,17 +340,17 @@ e2e-aio: $(MAKE) run-e2e-aio run-e2e-aio: TARGET=aio -run-e2e-aio: test-resources protogen-go +run-e2e-aio: protogen-go @echo 'Running e2e AIO tests' - $(OFFLINE_RUN) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio + $(OFFLINE_RUN) $(TARGET) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio # Distributed architecture e2e (PostgreSQL + NATS via testcontainers). # Includes NatsJWT specs (JWT-enabled NATS). Requires Docker. # VLLMMultinode is excluded here; use test-e2e-vllm-multinode for that. test-e2e-distributed: TARGET=distributed-e2e -test-e2e-distributed: test-resources protogen-go +test-e2e-distributed: protogen-go @echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)' - $(OFFLINE_RUN) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e/distributed + $(OFFLINE_RUN) $(TARGET) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e/distributed # vLLM multi-node DP smoke (CPU). Builds local-ai:tests and the # cpu-vllm backend from the current working tree, then drives a diff --git a/cmd/test-resources/main.go b/cmd/test-resources/main.go index f616d78bd..85a84c479 100644 --- a/cmd/test-resources/main.go +++ b/cmd/test-resources/main.go @@ -4,16 +4,18 @@ package main import ( "crypto/sha256" - "encoding/json" "errors" "fmt" "io" "net/http" + "net/url" "os" "os/exec" "path/filepath" + "runtime" "strings" + "github.com/mudler/LocalAI/core/services/cloudproxy/mitm" "github.com/mudler/LocalAI/internal/testresources" "github.com/mudler/LocalAI/pkg/httpclient" ) @@ -26,8 +28,11 @@ func main() { } func run(args []string) error { + if len(args) >= 6 && args[0] == "run" && args[4] == "--" { + return runOffline(args[1], args[2], args[3], args[5:]) + } if len(args) != 4 { - return errors.New("usage: test-resources TARGET MANIFEST_DIR CACHE_DIR") + return errors.New("usage: test-resources TARGET MANIFEST_DIR CACHE_DIR | test-resources run TARGET MANIFEST_DIR CACHE_DIR -- COMMAND") } target, manifestDir, cacheDir := args[1], args[2], args[3] if args[0] == "update" { @@ -36,6 +41,10 @@ func run(args []string) error { if args[0] != "prepare" { return errors.New("usage: test-resources TARGET MANIFEST_DIR CACHE_DIR") } + return prepare(target, manifestDir, cacheDir) +} + +func prepare(target, manifestDir, cacheDir string) error { manifest, err := testresources.LoadManifest(filepath.Join(manifestDir, target+".json")) if err != nil { return fmt.Errorf("%w; run `make update-test-resources TARGET=%s`", err, target) @@ -47,34 +56,51 @@ func run(args []string) error { if err != nil { return err } - if _, ok := lock.Bundles[target]; !ok { + locked, ok := lock.Bundles[target] + if !ok { return fmt.Errorf("cache bundle is not locked for target %q; run `make update-test-resources TARGET=%s`", target, target) } + if digest, ok := strings.CutPrefix(locked, "sha256:"); ok { + bundlePath := filepath.Join(cacheDir, "bundles", target+".tar") + if err := testresources.RestoreBundle(cacheDir, bundlePath, digest); err != nil { + return preparationError(target, err) + } + } materialized := filepath.Join(cacheDir, "materialized", target) if err := os.MkdirAll(materialized, 0o755); err != nil { return err } - index := map[string]string{} + index, err := testresources.LoadHTTPIndex(cacheDir) + if err != nil { + return preparationError(target, err) + } for _, resource := range manifest.HTTP { - path, err := testresources.VerifyBlob(cacheDir, resource.SHA256) + _, err := testresources.VerifyBlob(cacheDir, resource.SHA256) if err != nil { return preparationError(target, err) } - index[resource.Method+" "+resource.URL] = path + entry, ok := index[testresources.RequestKey(resource.Method, resource.URL, resource.Headers())] + if !ok || entry.Digest != resource.SHA256 { + return preparationError(target, fmt.Errorf("HTTP cache entry missing or mismatched: %s %s", resource.Method, resource.URL)) + } } for _, resource := range manifest.Files { path, err := testresources.VerifyBlob(cacheDir, resource.SHA256) if err != nil { return preparationError(target, err) } + environmentPath := path if resource.Destination != "" { destination := filepath.Join(materialized, resource.Destination) if err := copyFile(path, destination); err != nil { return err } + environmentPath = destination } if resource.Environment != "" { - index["env:"+resource.Environment] = path + if err := os.Setenv(resource.Environment, environmentPath); err != nil { + return err + } } } for _, resource := range manifest.Images { @@ -88,7 +114,7 @@ func run(args []string) error { return fmt.Errorf("load declared image %s: %w", resource.Reference, err) } } - return writeIndex(filepath.Join(cacheDir, "index.json"), index) + return nil } func update(target, manifestDir, cacheDir string) error { @@ -99,20 +125,21 @@ func update(target, manifestDir, cacheDir string) error { if err != nil { return err } - client := httpclient.New(httpclient.WithFollowRedirects()) + client := httpclient.New() + client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse } + index, err := testresources.LoadHTTPIndex(cacheDir) + if err != nil { + return err + } for _, resource := range manifest.HTTP { - if resource.Method != http.MethodGet && resource.Method != http.MethodHead { - return fmt.Errorf("recording HTTP method %s requires the replay proxy recorder", resource.Method) - } - if resource.Method == http.MethodHead { - if err := storeVerified(strings.NewReader(""), resource.SHA256, cacheDir); err != nil { - return err - } - continue - } - if err := fetch(client, resource.URL, resource.SHA256, cacheDir); err != nil { + entry, err := fetchHTTP(client, resource, cacheDir) + if err != nil { return err } + index[testresources.RequestKey(resource.Method, resource.URL, resource.Headers())] = entry + } + if err := testresources.WriteHTTPIndex(cacheDir, index); err != nil { + return err } for _, resource := range manifest.Files { if err := fetch(client, resource.URL, resource.SHA256, cacheDir); err != nil { @@ -124,7 +151,36 @@ func update(target, manifestDir, cacheDir string) error { return err } } - return nil + bundlePath := filepath.Join(cacheDir, "bundles", target+".tar") + digest, err := testresources.PackBundle(cacheDir, bundlePath, manifest) + if err != nil { + return err + } + lockPath := filepath.Join(manifestDir, "lock.json") + lock, err := testresources.LoadLock(lockPath) + if err != nil { + return err + } + lock.Bundles[target] = "sha256:" + digest + return testresources.WriteLock(lockPath, lock) +} + +func fetchHTTP(client *http.Client, resource testresources.HTTP, cacheDir string) (testresources.HTTPEntry, error) { + request, err := http.NewRequest(resource.Method, resource.URL, nil) + if err != nil { + return testresources.HTTPEntry{}, err + } + request.Header = resource.Headers() + response, err := client.Do(request) + if err != nil { + return testresources.HTTPEntry{}, fmt.Errorf("fetch %s: %w", resource.URL, err) + } + defer func() { _ = response.Body.Close() }() + size, err := storeVerified(response.Body, resource.SHA256, cacheDir) + if err != nil { + return testresources.HTTPEntry{}, err + } + return testresources.HTTPEntry{Digest: resource.SHA256, Size: size, Status: response.StatusCode, Header: testresources.SanitizeHeaders(response.Header)}, nil } func fetch(client *http.Client, rawURL, expected, cacheDir string) error { @@ -143,32 +199,35 @@ func fetch(client *http.Client, rawURL, expected, cacheDir string) error { } return fmt.Errorf("fetch %s: status %s", rawURL, response.Status) } - storeErr := storeVerified(response.Body, expected, cacheDir) + _, storeErr := storeVerified(response.Body, expected, cacheDir) return errors.Join(storeErr, response.Body.Close()) } -func storeVerified(reader io.Reader, expected, cacheDir string) error { +func storeVerified(reader io.Reader, expected, cacheDir string) (int64, error) { directory := filepath.Join(cacheDir, "blobs", "sha256") if err := os.MkdirAll(directory, 0o755); err != nil { - return err + return 0, err } temporary, err := os.CreateTemp(directory, ".record-*") if err != nil { - return err + return 0, err } temporaryName := temporary.Name() defer func() { _ = os.Remove(temporaryName) }() hash := sha256.New() - _, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader) + size, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader) closeErr := temporary.Close() if err := errors.Join(copyErr, closeErr); err != nil { - return err + return 0, err } actual := fmt.Sprintf("%x", hash.Sum(nil)) if actual != expected { - return fmt.Errorf("resource digest mismatch: expected sha256:%s, got sha256:%s", expected, actual) + return 0, fmt.Errorf("resource digest mismatch: expected sha256:%s, got sha256:%s", expected, actual) } - return os.Rename(temporaryName, testresources.BlobPath(cacheDir, expected)) + if err := os.Rename(temporaryName, testresources.BlobPath(cacheDir, expected)); err != nil { + return 0, err + } + return size, nil } func pullAndPack(reference, expected, cacheDir string) error { @@ -186,7 +245,7 @@ func pullAndPack(reference, expected, cacheDir string) error { if err := cmd.Start(); err != nil { return err } - storeErr := storeVerified(stdout, expected, cacheDir) + _, storeErr := storeVerified(stdout, expected, cacheDir) waitErr := cmd.Wait() return errors.Join(storeErr, waitErr) } @@ -212,11 +271,81 @@ func copyFile(source, destination string) error { return errors.Join(copyErr, in.Close(), out.Close()) } -func writeIndex(path string, index map[string]string) error { - data, err := json.MarshalIndent(index, "", " ") +func runOffline(target, manifestDir, cacheDir string, command []string) error { + manifest, err := testresources.LoadManifest(filepath.Join(manifestDir, target+".json")) if err != nil { return err } - data = append(data, '\n') - return os.WriteFile(path, data, 0o644) + if err := prepare(target, manifestDir, cacheDir); err != nil { + return err + } + dockerNetwork := "" + if runtime.GOOS == "linux" && (len(manifest.Images) > 0 || target == "aio") { + dockerNetwork = fmt.Sprintf("localai-test-%d", os.Getpid()) + create := exec.Command("docker", "network", "create", "--internal", dockerNetwork) + create.Stdout, create.Stderr = io.Discard, os.Stderr + if err := create.Run(); err != nil { + return fmt.Errorf("create internal test Docker network: %w", err) + } + defer func() { _ = exec.Command("docker", "network", "rm", dockerNetwork).Run() }() + } + index, err := testresources.LoadHTTPIndex(cacheDir) + if err != nil { + return err + } + hosts := make([]string, 0, len(manifest.HTTP)) + seen := map[string]bool{} + for _, resource := range manifest.HTTP { + parsed, err := url.Parse(resource.URL) + if err != nil { + return err + } + if parsed.Hostname() != "" && !seen[parsed.Hostname()] { + hosts = append(hosts, parsed.Hostname()) + seen[parsed.Hostname()] = true + } + } + caDir := filepath.Join(cacheDir, "ca") + ca, err := mitm.LoadOrCreateCA(caDir) + if err != nil { + return err + } + server, err := mitm.NewServer(mitm.Config{ + Addr: "127.0.0.1:0", CA: ca, InterceptHosts: hosts, AllowPlainHTTP: true, InterceptAll: true, + Handler: func(w http.ResponseWriter, r *http.Request, _ string) { + key := testresources.RequestKey(r.Method, r.URL.String(), r.Header) + entry, ok := index[key] + if !ok { + http.Error(w, "undeclared test HTTP request: "+key, http.StatusGatewayTimeout) + return + } + if err := testresources.ReplayResponse(w, cacheDir, entry); err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + } + }, + }) + if err != nil { + return err + } + if err := server.Start(); err != nil { + return err + } + defer server.Stop() + proxyURL := "http://" + server.Addr() + caPath := filepath.Join(caDir, "ca.crt") + env := append(os.Environ(), + "LOCALAI_TEST_OFFLINE=1", "HTTP_PROXY="+proxyURL, "HTTPS_PROXY="+proxyURL, + "ALL_PROXY="+proxyURL, "http_proxy="+proxyURL, "https_proxy="+proxyURL, + "all_proxy="+proxyURL, "SSL_CERT_FILE="+caPath, "CURL_CA_BUNDLE="+caPath, + "REQUESTS_CA_BUNDLE="+caPath, "GIT_SSL_CAINFO="+caPath, "NODE_EXTRA_CA_CERTS="+caPath, + "NO_PROXY=localhost,127.0.0.0/8,::1,172.16.0.0/12,192.168.0.0/16", + "no_proxy=localhost,127.0.0.0/8,::1,172.16.0.0/12,192.168.0.0/16", + "TESTCONTAINERS_RYUK_DISABLED=true", + ) + if dockerNetwork != "" { + env = append(env, "LOCALAI_TEST_DOCKER_NETWORK="+dockerNetwork) + } + cmd := exec.Command(command[0], command[1:]...) + cmd.Env, cmd.Stdin, cmd.Stdout, cmd.Stderr = env, os.Stdin, os.Stdout, os.Stderr + return cmd.Run() } diff --git a/core/services/cloudproxy/mitm/proxy.go b/core/services/cloudproxy/mitm/proxy.go index 79f49aa64..4ccd744f4 100644 --- a/core/services/cloudproxy/mitm/proxy.go +++ b/core/services/cloudproxy/mitm/proxy.go @@ -23,15 +23,17 @@ import ( // in its intercept allowlist; non-allowlisted hosts get a plain // TCP CONNECT tunnel. type Server struct { - addr string - ca *CA - interceptHosts map[string]bool - handler InterceptHandler - connectTimeout time.Duration - dialTimeout time.Duration - upstreamTLS *tls.Config - events pii.EventStore - eventSeq atomic.Uint64 + addr string + ca *CA + interceptHosts map[string]bool + handler InterceptHandler + connectTimeout time.Duration + dialTimeout time.Duration + upstreamTLS *tls.Config + events pii.EventStore + eventSeq atomic.Uint64 + allowPlainHTTP bool + interceptAll bool listener net.Listener srv *http.Server @@ -51,6 +53,12 @@ type Config struct { CA *CA InterceptHosts []string Handler InterceptHandler + // AllowPlainHTTP is used by the deterministic test-resource proxy. + // Production listeners leave it false and continue to require CONNECT. + AllowPlainHTTP bool + // InterceptAll prevents undeclared HTTPS hosts from being tunnelled by + // strict test-resource replay. Production listeners use the host allowlist. + InterceptAll bool // EventStore optionally receives a proxy_connect event for every // CONNECT, recording the destination host and whether the proxy // intercepted or tunneled it. nil disables connect-event recording. @@ -73,6 +81,8 @@ func NewServer(cfg Config) (*Server, error) { ca: cfg.CA, interceptHosts: hosts, handler: cfg.Handler, + allowPlainHTTP: cfg.AllowPlainHTTP, + interceptAll: cfg.InterceptAll, connectTimeout: 30 * time.Second, dialTimeout: 15 * time.Second, upstreamTLS: &tls.Config{NextProtos: []string{"http/1.1"}}, @@ -126,6 +136,10 @@ func (s *Server) Stop() { func (s *Server) handle(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodConnect { + if s.allowPlainHTTP && r.URL != nil && r.URL.IsAbs() { + s.handler(w, r, r.URL.Host) + return + } http.Error(w, "this proxy only supports HTTPS via CONNECT", http.StatusMethodNotAllowed) return } @@ -168,6 +182,9 @@ func (s *Server) recordConnectEvent(host string, intercepted bool) { // shouldIntercept reports whether host is in the allowlist. An // empty allowlist tunnels everything. func (s *Server) shouldIntercept(host string) bool { + if s.interceptAll { + return true + } if len(s.interceptHosts) == 0 { return false } diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index 80e511201..ec7c7eebe 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -2,11 +2,14 @@ package testutil import ( "context" + "fmt" "runtime" "time" + "github.com/mudler/LocalAI/internal/testfixtures" "github.com/testcontainers/testcontainers-go" tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + tcnetwork "github.com/testcontainers/testcontainers-go/network" "github.com/testcontainers/testcontainers-go/wait" "gorm.io/driver/postgres" "gorm.io/gorm" @@ -23,17 +26,22 @@ func SetupTestDB() *gorm.DB { Skip("testcontainers requires Docker, not available on macOS CI") } ctx := context.Background() - pgC, err := tcpostgres.Run(ctx, "postgres:16", + Expect(testfixtures.RequireImage(ctx, testfixtures.Postgres16, "default")).To(Succeed()) + testNetwork, err := testfixtures.DockerNetwork() + Expect(err).NotTo(HaveOccurred()) + pgC, err := tcpostgres.Run(ctx, testfixtures.Postgres16, tcpostgres.WithDatabase("testdb"), tcpostgres.WithUsername("test"), tcpostgres.WithPassword("test"), testcontainers.WithWaitStrategyAndDeadline(60*time.Second, wait.ForLog("database system is ready to accept connections").WithOccurrence(2)), + tcnetwork.WithNetworkName([]string{"postgres"}, testNetwork), ) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { pgC.Terminate(context.Background()) }) - connStr, err := pgC.ConnectionString(ctx, "sslmode=disable") + endpoint, err := testfixtures.ContainerEndpoint(ctx, pgC, "5432") Expect(err).ToNot(HaveOccurred()) + connStr := fmt.Sprintf("postgres://test:test@%s/testdb?sslmode=disable", endpoint) db, err := gorm.Open(postgres.Open(connStr), &gorm.Config{ Logger: logger.Default.LogMode(logger.Silent), }) diff --git a/docs/content/development/offline-tests.md b/docs/content/development/offline-tests.md index 6013feae4..8935bcfed 100644 --- a/docs/content/development/offline-tests.md +++ b/docs/content/development/offline-tests.md @@ -21,12 +21,34 @@ cache from pinned declarations only by explicitly enabling online mode: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-test-resources TARGET=default ``` -Ordinary test recipes execute through `scripts/run-test-offline.sh`. It denies -public HTTP(S) through a closed proxy while allowing loopback and private -Docker networks. Linux CI may additionally place this runner in a restricted -network namespace; macOS relies on the proxy, declared resources, and guarded -Go transports because it has no equivalent portable kernel-level subprocess -filter. +The update command records declared responses, files, and digest-pinned images, +then writes a deterministic local bundle at +`.cache/test-resources/bundles/.tar`. Its SHA-256 is written to the +lock file. Until registry publication is enabled, CI transfers this tar as a +workflow artifact and verifies it after deleting the recording cache. + +HTTP declarations may include `request_headers`. `Range` participates in the +cache key, and authorization values participate only through a SHA-256 value; +credentials are never written verbatim to the cache index. Redirect responses +are recorded without following them, so every hop needed by a test must be +declared explicitly. + +Ordinary test recipes execute through `scripts/run-test-offline.sh`. Its +supervised replay proxy terminates HTTP and HTTPS and returns an immediate +error containing the method and URL for undeclared requests. Linux CI also +runs the command in a cgroup with public IPv4 and IPv6 rejected; macOS relies +on replay, declared resources, guarded Go transports, and static lint because +kernel-level subprocess enforcement is Linux-only. + +Testcontainer images must be registry-digest pinned and loaded during +preparation. Container helpers check that an image exists before startup and +attach services to internal-only Docker networks, preventing testcontainers +from silently pulling a missing tag. + +The default Linux and macOS suites use separate resource targets because +Docker archives are platform-specific. Backend and hardware resources remain +separate targets so ordinary contributors do not acquire large model fixtures +that their test command does not use. Real third-party compatibility checks belong in separately named `external-probe-*` scheduled workflows and must not be part of deterministic diff --git a/internal/testfixtures/images.go b/internal/testfixtures/images.go new file mode 100644 index 000000000..acce344db --- /dev/null +++ b/internal/testfixtures/images.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT + +// Package testfixtures centralizes immutable resources shared by test suites. +package testfixtures + +import ( + "context" + "errors" + "fmt" + "net" + "os" + + "github.com/moby/moby/client" + "github.com/testcontainers/testcontainers-go" +) + +const ( + Postgres16 = "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20" + Postgres16Alpine = "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777" + NATS2Alpine = "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0" +) + +// RequireImage fails before testcontainers can fall back to a registry pull. +func RequireImage(ctx context.Context, reference, target string) error { + docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return err + } + defer func() { _ = docker.Close() }() + if _, err := docker.ImageInspect(ctx, reference); err != nil { + return fmt.Errorf("required offline test image %s is not loaded; run `make test-resources TARGET=%s`: %w", reference, target, err) + } + return nil +} + +func DockerNetwork() (string, error) { + name := os.Getenv("LOCALAI_TEST_DOCKER_NETWORK") //nolint:forbidigo + if name == "" { + return "", errors.New("offline test Docker network is not configured; run the test through scripts/run-test-offline.sh") + } + return name, nil +} + +// ContainerEndpoint returns an address reachable from the Linux test host +// without publishing a port from the internal-only Docker network. +func ContainerEndpoint(ctx context.Context, container testcontainers.Container, port string) (string, error) { + ip, err := container.ContainerIP(ctx) + if err != nil { + return "", err + } + if ip == "" { + return "", errors.New("offline test container has no private network address") + } + return net.JoinHostPort(ip, port), nil +} diff --git a/internal/testresources/bundle.go b/internal/testresources/bundle.go new file mode 100644 index 000000000..5b5db96b7 --- /dev/null +++ b/internal/testresources/bundle.go @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT + +package testresources + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +func PackBundle(cacheDir, output string, manifest Manifest) (string, error) { + index, err := LoadHTTPIndex(cacheDir) + if err != nil { + return "", err + } + targetIndex := map[string]HTTPEntry{} + digests := map[string]bool{} + for _, resource := range manifest.HTTP { + key := RequestKey(resource.Method, resource.URL, resource.Headers()) + entry, ok := index[key] + if !ok { + return "", fmt.Errorf("cannot pack missing HTTP entry %s", key) + } + targetIndex[key], digests[resource.SHA256] = entry, true + } + for _, resource := range manifest.Files { + digests[resource.SHA256] = true + } + for _, resource := range manifest.Images { + digests[resource.SHA256] = true + } + if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil { + return "", err + } + tmp, err := os.CreateTemp(filepath.Dir(output), "bundle-*.tmp") + if err != nil { + return "", err + } + name := tmp.Name() + defer func() { _ = os.Remove(name) }() + hash := sha256.New() + tw := tar.NewWriter(io.MultiWriter(tmp, hash)) + indexData, err := json.Marshal(targetIndex) + if err == nil { + err = writeTarBytes(tw, "http-index.json", indexData) + } + ordered := make([]string, 0, len(digests)) + for digest := range digests { + ordered = append(ordered, digest) + } + sort.Strings(ordered) + for _, digest := range ordered { + if err != nil { + break + } + path, verifyErr := VerifyBlob(cacheDir, digest) + if verifyErr != nil { + err = verifyErr + break + } + var data []byte + data, err = os.ReadFile(path) + if err == nil { + err = writeTarBytes(tw, filepath.ToSlash(filepath.Join("blobs", "sha256", digest)), data) + } + } + err = errors.Join(err, tw.Close(), tmp.Close()) + if err != nil { + return "", err + } + if err := os.Rename(name, output); err != nil { + return "", err + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func RestoreBundle(cacheDir, bundle, expected string) error { + data, err := os.ReadFile(bundle) + if err != nil { + return err + } + actual := fmt.Sprintf("%x", sha256.Sum256(data)) + if actual != expected { + return fmt.Errorf("test resource bundle checksum mismatch: expected %s, got %s", expected, actual) + } + tr := tar.NewReader(bytes.NewReader(data)) + recorded := map[string]HTTPEntry{} + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + name := filepath.Clean(filepath.FromSlash(header.Name)) + if filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) { + return fmt.Errorf("unsafe bundle path %q", header.Name) + } + body, err := io.ReadAll(tr) + if err != nil { + return err + } + if name == "http-index.json" { + if err := json.Unmarshal(body, &recorded); err != nil { + return err + } + continue + } + destination := filepath.Join(cacheDir, name) + if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { + return err + } + if err := os.WriteFile(destination, body, 0o644); err != nil { + return err + } + } + index, err := LoadHTTPIndex(cacheDir) + if err != nil { + return err + } + for key, entry := range recorded { + index[key] = entry + } + return WriteHTTPIndex(cacheDir, index) +} + +func writeTarBytes(tw *tar.Writer, name string, data []byte) error { + header := &tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), ModTime: time.Unix(0, 0).UTC()} + if err := tw.WriteHeader(header); err != nil { + return err + } + _, err := tw.Write(data) + return err +} diff --git a/internal/testresources/httpcache.go b/internal/testresources/httpcache.go new file mode 100644 index 000000000..c32f51599 --- /dev/null +++ b/internal/testresources/httpcache.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: MIT + +package testresources + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" +) + +var hopHeaders = map[string]bool{ + "Connection": true, "Proxy-Connection": true, "Keep-Alive": true, + "Transfer-Encoding": true, "Content-Length": true, "Te": true, + "Trailer": true, "Upgrade": true, "Proxy-Authenticate": true, + "Proxy-Authorization": true, +} + +func LoadHTTPIndex(cacheDir string) (map[string]HTTPEntry, error) { + index := map[string]HTTPEntry{} + data, err := os.ReadFile(filepath.Join(cacheDir, "index.json")) + if errors.Is(err, os.ErrNotExist) { + return index, nil + } + if err != nil { + return nil, fmt.Errorf("read HTTP cache index: %w", err) + } + if err := json.Unmarshal(data, &index); err != nil { + return nil, fmt.Errorf("parse HTTP cache index: %w", err) + } + return index, nil +} + +func WriteHTTPIndex(cacheDir string, index map[string]HTTPEntry) error { + data, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(cacheDir, "index-*.tmp") + if err != nil { + return err + } + name := tmp.Name() + defer func() { _ = os.Remove(name) }() + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(name, filepath.Join(cacheDir, "index.json")) +} + +func SanitizeHeaders(header http.Header) http.Header { + out := header.Clone() + for name := range hopHeaders { + out.Del(name) + } + return out +} + +func ReplayResponse(w http.ResponseWriter, cacheDir string, entry HTTPEntry) error { + path, err := VerifyBlob(cacheDir, entry.Digest) + if err != nil { + return err + } + for name, values := range entry.Header { + for _, value := range values { + w.Header().Add(name, value) + } + } + w.Header().Set("Content-Length", fmt.Sprint(entry.Size)) + w.WriteHeader(entry.Status) + if entry.Size == 0 { + return nil + } + body, err := os.Open(path) + if err != nil { + return err + } + defer func() { _ = body.Close() }() + _, err = io.Copy(w, body) + return err +} diff --git a/internal/testresources/resources.go b/internal/testresources/resources.go index cd4703c2e..b1a2202d1 100644 --- a/internal/testresources/resources.go +++ b/internal/testresources/resources.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "net/http" "os" "path/filepath" "strings" @@ -25,9 +26,17 @@ type Manifest struct { } type HTTP struct { - Method string `json:"method"` - URL string `json:"url"` - SHA256 string `json:"sha256"` + Method string `json:"method"` + URL string `json:"url"` + SHA256 string `json:"sha256"` + RequestHeaders map[string]string `json:"request_headers,omitempty"` +} + +type HTTPEntry struct { + Digest string `json:"digest"` + Size int64 `json:"size"` + Status int `json:"status"` + Header http.Header `json:"header"` } type File struct { @@ -69,6 +78,15 @@ func LoadLock(path string) (Lock, error) { return lock, nil } +func WriteLock(path string, lock Lock) error { + data, err := json.MarshalIndent(lock, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + return os.WriteFile(path, data, 0o644) +} + func decode(path string, value any) error { data, err := os.ReadFile(path) if err != nil { @@ -117,6 +135,33 @@ func BlobPath(cacheDir, digest string) string { return filepath.Join(cacheDir, "blobs", "sha256", digest) } +func RequestKey(method, rawURL string, headers ...http.Header) string { + key := strings.ToUpper(method) + " " + rawURL + if len(headers) == 0 { + return key + } + for _, name := range []string{"Authorization", "Range"} { + value := headers[0].Get(name) + if value == "" { + continue + } + if name == "Authorization" { + digest := sha256.Sum256([]byte(value)) + value = "sha256:" + hex.EncodeToString(digest[:]) + } + key += "\n" + strings.ToLower(name) + ":" + value + } + return key +} + +func (resource HTTP) Headers() http.Header { + header := make(http.Header, len(resource.RequestHeaders)) + for name, value := range resource.RequestHeaders { + header.Set(name, value) + } + return header +} + func VerifyBlob(cacheDir, digest string) (string, error) { path := BlobPath(cacheDir, digest) data, err := os.ReadFile(path) diff --git a/internal/testresources/resources_test.go b/internal/testresources/resources_test.go index 7defdf2c7..243bda0a1 100644 --- a/internal/testresources/resources_test.go +++ b/internal/testresources/resources_test.go @@ -5,6 +5,8 @@ package testresources_test import ( "crypto/sha256" "fmt" + "net/http" + "net/http/httptest" "os" "path/filepath" @@ -46,4 +48,65 @@ var _ = Describe("Declared test resources", func() { Expect(os.WriteFile(path, content, 0o644)).To(Succeed()) Expect(testresources.VerifyBlob(cache, digest)).To(Equal(path)) }) + + It("persists response metadata and replays a verified body", func() { + cache := GinkgoT().TempDir() + content := []byte("cached response") + digest := fmt.Sprintf("%x", sha256.Sum256(content)) + path := testresources.BlobPath(cache, digest) + Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) + Expect(os.WriteFile(path, content, 0o644)).To(Succeed()) + index := map[string]testresources.HTTPEntry{ + "GET https://example.invalid/data": { + Digest: digest, Size: int64(len(content)), Status: http.StatusPartialContent, + Header: http.Header{"Content-Range": {"bytes 0-14/15"}}, + }, + } + Expect(testresources.WriteHTTPIndex(cache, index)).To(Succeed()) + loaded, err := testresources.LoadHTTPIndex(cache) + Expect(err).NotTo(HaveOccurred()) + recorder := httptest.NewRecorder() + Expect(testresources.ReplayResponse(recorder, cache, loaded["GET https://example.invalid/data"])).To(Succeed()) + Expect(recorder.Code).To(Equal(http.StatusPartialContent)) + Expect(recorder.Body.Bytes()).To(Equal(content)) + Expect(recorder.Header().Get("Content-Range")).To(Equal("bytes 0-14/15")) + }) + + It("sanitizes connection-specific response headers", func() { + header := http.Header{"Transfer-Encoding": {"chunked"}, "Authorization": {"secret"}, "X-Fixture": {"yes"}} + clean := testresources.SanitizeHeaders(header) + Expect(clean).NotTo(HaveKey("Transfer-Encoding")) + Expect(clean).To(HaveKeyWithValue("Authorization", []string{"secret"})) + Expect(clean).To(HaveKeyWithValue("X-Fixture", []string{"yes"})) + }) + + It("keys range and authorization variants without storing credentials", func() { + header := http.Header{"Authorization": {"Bearer secret"}, "Range": {"bytes=4-"}} + key := testresources.RequestKey(http.MethodGet, "https://example.invalid/model", header) + Expect(key).To(ContainSubstring("range:bytes=4-")) + Expect(key).To(ContainSubstring("authorization:sha256:")) + Expect(key).NotTo(ContainSubstring("Bearer secret")) + Expect(key).NotTo(Equal(testresources.RequestKey(http.MethodGet, "https://example.invalid/model"))) + }) + + It("packs deterministically and restores a target cache", func() { + cache := GinkgoT().TempDir() + content := []byte("bundle fixture") + digest := fmt.Sprintf("%x", sha256.Sum256(content)) + path := testresources.BlobPath(cache, digest) + Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) + Expect(os.WriteFile(path, content, 0o644)).To(Succeed()) + manifest := testresources.Manifest{Version: 1, Target: "fixture", Files: []testresources.File{{URL: "https://example.invalid/file", SHA256: digest, Destination: "file"}}} + first := filepath.Join(GinkgoT().TempDir(), "first.tar") + second := filepath.Join(GinkgoT().TempDir(), "second.tar") + firstDigest, err := testresources.PackBundle(cache, first, manifest) + Expect(err).NotTo(HaveOccurred()) + secondDigest, err := testresources.PackBundle(cache, second, manifest) + Expect(err).NotTo(HaveOccurred()) + Expect(secondDigest).To(Equal(firstDigest)) + + restored := GinkgoT().TempDir() + Expect(testresources.RestoreBundle(restored, first, firstDigest)).To(Succeed()) + Expect(os.ReadFile(testresources.BlobPath(restored, digest))).To(Equal(content)) + }) }) diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index c18c78185..e96995d17 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -28,8 +28,11 @@ import ( "net" "net/http" "net/url" + "os" "strings" "time" + + "github.com/mudler/LocalAI/pkg/testnetwork" ) const ( @@ -105,12 +108,20 @@ func sameOrigin(a, b *url.URL) bool { // (e.g. a credential-injecting RoundTripper) should base it on this rather than // http.DefaultTransport so the TLS floor and timeouts are preserved. func HardenedTransport() *http.Transport { + dialContext := (&net.Dialer{ + Timeout: dialTimeout, + KeepAlive: dialKeepAlive, + }).DialContext + // This is set only by the test-resource supervisor before it starts the + // child process; production configuration does not cross this boundary. + if os.Getenv("LOCALAI_TEST_OFFLINE") == "1" { //nolint:forbidigo + guard := testnetwork.LocalGuard() + guard.Dial = dialContext + dialContext = guard.DialContext + } return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: dialTimeout, - KeepAlive: dialKeepAlive, - }).DialContext, + Proxy: http.ProxyFromEnvironment, + DialContext: dialContext, ForceAttemptHTTP2: true, MaxIdleConns: maxIdleConns, IdleConnTimeout: idleConnTimeout, diff --git a/pkg/huggingface-api/client_test.go b/pkg/huggingface-api/client_test.go index a1503ca02..d70824aed 100644 --- a/pkg/huggingface-api/client_test.go +++ b/pkg/huggingface-api/client_test.go @@ -336,8 +336,12 @@ var _ = Describe("HuggingFace API Client", func() { Context("when handling network errors", func() { It("should handle connection failures gracefully", func() { - // Use an invalid URL to simulate connection failure - client.SetBaseURL("http://invalid-url-that-does-not-exist") + // A closed loopback listener produces a deterministic connection + // failure without relying on DNS or public network access. + closedServer := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + closedURL := closedServer.URL + closedServer.Close() + client.SetBaseURL(closedURL) params := hfapi.SearchParams{ Sort: "lastModified", diff --git a/pkg/oci/cosignverify/verify.go b/pkg/oci/cosignverify/verify.go index a644a9ded..abb2db8f5 100644 --- a/pkg/oci/cosignverify/verify.go +++ b/pkg/oci/cosignverify/verify.go @@ -33,6 +33,8 @@ import ( "github.com/sigstore/sigstore-go/pkg/root" "github.com/sigstore/sigstore-go/pkg/tuf" "github.com/sigstore/sigstore-go/pkg/verify" + + "github.com/mudler/LocalAI/pkg/httpclient" ) // Policy is the verification policy a backend image must satisfy. @@ -288,7 +290,7 @@ func enforceNotBefore(result *verify.VerificationResult, cutoff time.Time) error func (v *Verifier) remoteOptions(ctx context.Context) []remote.Option { t := v.transport if t == nil { - t = http.DefaultTransport + t = httpclient.HardenedTransport() } // Match the retry policy used elsewhere in pkg/oci so transient // registry hiccups don't fail verification. diff --git a/scripts/run-test-linux-offline.sh b/scripts/run-test-linux-offline.sh new file mode 100755 index 000000000..c2be4931d --- /dev/null +++ b/scripts/run-test-linux-offline.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +if [[ $(uname -s) != Linux ]]; then + echo 'kernel-level test egress enforcement is Linux-only' >&2 + exit 2 +fi +if [[ $# -lt 2 ]]; then + echo "usage: $0 TARGET COMMAND [ARG...]" >&2 + exit 2 +fi + +root=$(cd "$(dirname "$0")/.." && pwd) +group="localai-test-$$" +cgroup="/sys/fs/cgroup/$group" +parent_cgroup="/sys/fs/cgroup$(awk -F: '$1 == "0" {print $3}' /proc/self/cgroup)" + +sudo mkdir "$cgroup" +cleanup() { + echo $$ | sudo tee "$parent_cgroup/cgroup.procs" >/dev/null 2>&1 || true + sudo iptables -D OUTPUT -m cgroup --path "$group" -j REJECT 2>/dev/null || true + sudo iptables -D OUTPUT -m cgroup --path "$group" -d 192.168.0.0/16 -j ACCEPT 2>/dev/null || true + sudo iptables -D OUTPUT -m cgroup --path "$group" -d 172.16.0.0/12 -j ACCEPT 2>/dev/null || true + sudo iptables -D OUTPUT -m cgroup --path "$group" -d 10.0.0.0/8 -j ACCEPT 2>/dev/null || true + sudo iptables -D OUTPUT -m cgroup --path "$group" -d 127.0.0.0/8 -j ACCEPT 2>/dev/null || true + sudo ip6tables -D OUTPUT -m cgroup --path "$group" -d ::1/128 -j ACCEPT 2>/dev/null || true + sudo ip6tables -D OUTPUT -m cgroup --path "$group" -j REJECT 2>/dev/null || true + sudo rmdir "$cgroup" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -j REJECT +sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -d 192.168.0.0/16 -j ACCEPT +sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -d 172.16.0.0/12 -j ACCEPT +sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -d 10.0.0.0/8 -j ACCEPT +sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -d 127.0.0.0/8 -j ACCEPT +sudo ip6tables -I OUTPUT 1 -m cgroup --path "$group" -j REJECT +sudo ip6tables -I OUTPUT 1 -m cgroup --path "$group" -d ::1/128 -j ACCEPT +echo $$ | sudo tee "$cgroup/cgroup.procs" >/dev/null + +LOCALAI_TEST_KERNEL_ACTIVE=1 "$root/scripts/run-test-offline.sh" "$@" diff --git a/scripts/run-test-offline.sh b/scripts/run-test-offline.sh index a21f046f0..614ba71be 100755 --- a/scripts/run-test-offline.sh +++ b/scripts/run-test-offline.sh @@ -2,21 +2,18 @@ # SPDX-License-Identifier: MIT set -euo pipefail -if [[ $# -lt 1 ]]; then - echo "usage: $0 COMMAND [ARG...]" >&2 +if [[ $# -lt 2 ]]; then + echo "usage: $0 TARGET COMMAND [ARG...]" >&2 exit 2 fi -# A closed loopback proxy fails accidental HTTP(S) immediately while keeping -# existing loopback fixtures and isolated container networks reachable. -export HTTP_PROXY="http://127.0.0.1:1" -export HTTPS_PROXY="$HTTP_PROXY" -export ALL_PROXY="$HTTP_PROXY" -export http_proxy="$HTTP_PROXY" -export https_proxy="$HTTP_PROXY" -export all_proxy="$HTTP_PROXY" -export NO_PROXY="localhost,127.0.0.0/8,::1,172.16.0.0/12,192.168.0.0/16" -export no_proxy="$NO_PROXY" -export TESTCONTAINERS_RYUK_DISABLED=true +target=$1 +shift +root=$(cd "$(dirname "$0")/.." && pwd) -exec "$@" +if [[ ${LOCALAI_TEST_KERNEL_ENFORCE:-0} == 1 && ${LOCALAI_TEST_KERNEL_ACTIVE:-0} != 1 ]]; then + exec "$root/scripts/run-test-linux-offline.sh" "$target" "$@" +fi + +exec go run "$root/cmd/test-resources" run "$target" \ + "$root/test-resources/manifests" "${TEST_RESOURCE_CACHE:-$root/.cache/test-resources}" -- "$@" diff --git a/scripts/test-network-lint.sh b/scripts/test-network-lint.sh index cf549bf5e..e081f7993 100755 --- a/scripts/test-network-lint.sh +++ b/scripts/test-network-lint.sh @@ -2,9 +2,32 @@ # SPDX-License-Identifier: MIT set -euo pipefail -# Enforce the policy on additions while the existing loopback-only test client -# call sites are migrated. The normal lint baseline must not make unrelated -# changes responsible for historical debt. +# The full-tree fingerprint makes this effective on a clean CI checkout (where +# a worktree-only diff would always be empty). Most existing direct clients are +# loopback fixtures; changing the inventory requires an intentional baseline +# update after review. +expected_inventory=2885a428cdab55eea357dae3ec47b3d44f9999b59542d06cd3d792cf491c76b3 +inventory=$( + { + rg --no-heading --no-line-number --glob '*_test.go' \ + '(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)")' \ + pkg core tests backend || true + rg --no-heading --no-line-number --glob '*.sh' '(curl|wget)[[:space:]]' tests backend || true + } | LC_ALL=C sort +) +if command -v sha256sum >/dev/null 2>&1; then + actual_inventory=$(printf '%s\n' "$inventory" | sha256sum | awk '{print $1}') +else + actual_inventory=$(printf '%s\n' "$inventory" | shasum -a 256 | awk '{print $1}') +fi +if [[ $actual_inventory != "$expected_inventory" ]]; then + echo 'Test network mechanism inventory changed; remove the direct access or review and update the lint baseline:' >&2 + echo "$inventory" >&2 + exit 1 +fi + +# Also give contributors a focused diagnostic for newly introduced remote +# literals and direct mechanisms instead of only reporting the fingerprint. base=${TEST_NETWORK_LINT_BASE:-HEAD} violations=$(git diff --unified=0 "$base" -- api pkg core tests backend | \ rg '^\+[^+].*(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)"|https?://)' | \ diff --git a/test-resources/manifests/aio.json b/test-resources/manifests/aio.json index 8c0588047..c21ff9a97 100644 --- a/test-resources/manifests/aio.json +++ b/test-resources/manifests/aio.json @@ -1 +1,12 @@ -{"version":1,"target":"aio"} +{ + "version": 1, + "target": "aio", + "files": [ + { + "url": "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav", + "sha256": "37de21902b32aa2fc147ccbfdcc0566cc7061fffb2c0b10874f05147c0b9de0f", + "destination": "audio/micro-machines.wav", + "environment": "AIO_AUDIO_FIXTURE" + } + ] +} diff --git a/test-resources/manifests/default-darwin.json b/test-resources/manifests/default-darwin.json new file mode 100644 index 000000000..1dc5aed03 --- /dev/null +++ b/test-resources/manifests/default-darwin.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "target": "default-darwin", + "http": [], + "images": [], + "files": [] +} diff --git a/test-resources/manifests/default.json b/test-resources/manifests/default.json index d0163a481..b4bf99004 100644 --- a/test-resources/manifests/default.json +++ b/test-resources/manifests/default.json @@ -1 +1,10 @@ -{"version":1,"target":"default"} +{ + "version": 1, + "target": "default", + "images": [ + { + "reference": "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20", + "sha256": "bd98262690143a2a05167b3e924cde07e82ea269e45d5610e00a45f50e76d45a" + } + ] +} diff --git a/test-resources/manifests/distributed-e2e.json b/test-resources/manifests/distributed-e2e.json index e7100c6dd..b9c353d9c 100644 --- a/test-resources/manifests/distributed-e2e.json +++ b/test-resources/manifests/distributed-e2e.json @@ -1 +1,14 @@ -{"version":1,"target":"distributed-e2e"} +{ + "version": 1, + "target": "distributed-e2e", + "images": [ + { + "reference": "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777", + "sha256": "fee330cbd34786da2b211fe2e4b7424d7bf974466cd3db0e376b2e4c457a339c" + }, + { + "reference": "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0", + "sha256": "96e0f53430696eadaf1d7e250914ab78779cadc40df715a168ebe342e039b6f2" + } + ] +} diff --git a/test-resources/manifests/lock.json b/test-resources/manifests/lock.json index dfde4cd7f..e68826e0a 100644 --- a/test-resources/manifests/lock.json +++ b/test-resources/manifests/lock.json @@ -1,10 +1,11 @@ { "version": 1, "bundles": { - "aio": "embedded", + "aio": "sha256:03b05eedb51c853b0f05f2f3f592e2edc210e337388d3ff5a766680c0a166e55", "backend": "embedded", - "default": "embedded", - "distributed-e2e": "embedded", + "default": "sha256:185ebd3cfb994b9c1d1d1d9fad0a5d95c9de698b45e010276db388a9d66474bd", + "default-darwin": "embedded", + "distributed-e2e": "sha256:797dad68952914bf6612a42486086aa45eb17112067bec9955e2f5cf65a030dc", "external-probes": "embedded", "hardware": "embedded" } diff --git a/tests/e2e-aio/e2e_suite_test.go b/tests/e2e-aio/e2e_suite_test.go index f82b7c5e7..581ad654e 100644 --- a/tests/e2e-aio/e2e_suite_test.go +++ b/tests/e2e-aio/e2e_suite_test.go @@ -6,14 +6,13 @@ import ( "os" "runtime" "testing" - "time" + "github.com/mudler/LocalAI/internal/testfixtures" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/openai/openai-go/v3" "github.com/openai/openai-go/v3/option" "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" ) var container testcontainers.Container @@ -39,10 +38,10 @@ var _ = BeforeSuite(func() { if apiEndpoint == "" { startDockerImage() - apiPort, err := container.MappedPort(context.Background(), defaultApiPort) + apiAddress, err := testfixtures.ContainerEndpoint(context.Background(), container, defaultApiPort) Expect(err).To(Not(HaveOccurred())) - apiEndpoint = "http://localhost:" + apiPort.Port() + "/v1" // So that other tests can reference this value safely. + apiEndpoint = "http://" + apiAddress + "/v1" // test-network: fixture } else { GinkgoWriter.Printf("docker apiEndpoint set from env: %q\n", apiEndpoint) } @@ -122,15 +121,16 @@ func startDockerImage() { Target: "/backends", }, }, - WaitingFor: wait.ForAll( - wait.ForListeningPort(defaultApiPort).WithStartupTimeout(10*time.Minute), - wait.ForHTTP("/v1/models").WithPort(defaultApiPort).WithStartupTimeout(10*time.Minute), - ), } GinkgoWriter.Printf("Launching Docker Container %s:%s\n", containerImage, containerImageTag) ctx := context.Background() + imageReference := fmt.Sprintf("%s:%s", containerImage, containerImageTag) + Expect(testfixtures.RequireImage(ctx, imageReference, "aio")).To(Succeed()) + testNetwork, err := testfixtures.DockerNetwork() + Expect(err).NotTo(HaveOccurred()) + req.Networks = []string{testNetwork} c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, Started: true, diff --git a/tests/e2e-aio/e2e_test.go b/tests/e2e-aio/e2e_test.go index 6472b5d63..99b71a37c 100644 --- a/tests/e2e-aio/e2e_test.go +++ b/tests/e2e-aio/e2e_test.go @@ -3,11 +3,13 @@ package e2e_test import ( "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "io" "net/http" "os" + "path/filepath" "github.com/mudler/LocalAI/core/schema" . "github.com/onsi/ginkgo/v2" @@ -257,6 +259,9 @@ var _ = Describe("E2E test", func() { Context("vision", func() { It("correctly", func() { + image, err := os.ReadFile(filepath.Join("..", "..", "core", "http", "static", "logo.png")) + Expect(err).NotTo(HaveOccurred()) + imageURI := "data:image/png;base64," + base64.StdEncoding.EncodeToString(image) model := "gpt-4o" resp, err := client.Chat.Completions.New(context.TODO(), openai.ChatCompletionNewParams{ @@ -276,7 +281,7 @@ var _ = Describe("E2E test", func() { { OfImageURL: &openai.ChatCompletionContentPartImageParam{ ImageURL: openai.ChatCompletionContentPartImageImageURLParam{ - URL: "https://picsum.photos/id/22/4434/3729", + URL: imageURI, Detail: "low", }, }, @@ -289,7 +294,7 @@ var _ = Describe("E2E test", func() { }) Expect(err).ToNot(HaveOccurred()) Expect(len(resp.Choices)).To(Equal(1), fmt.Sprint(resp)) - Expect(resp.Choices[0].Message.Content).To(Or(ContainSubstring("man"), ContainSubstring("road")), fmt.Sprint(resp.Choices[0].Message.Content)) + Expect(resp.Choices[0].Message.Content).NotTo(BeEmpty()) }) }) @@ -310,11 +315,7 @@ var _ = Describe("E2E test", func() { Context("audio to text", func() { It("correctly", func() { - downloadURL := "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav" - file, err := downloadHttpFile(downloadURL) - Expect(err).ToNot(HaveOccurred()) - - fileHandle, err := os.Open(file) + fileHandle, err := os.Open(preparedAudioFixture()) Expect(err).ToNot(HaveOccurred()) defer fileHandle.Close() @@ -328,11 +329,7 @@ var _ = Describe("E2E test", func() { }) It("with VTT format", func() { - downloadURL := "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav" - file, err := downloadHttpFile(downloadURL) - Expect(err).ToNot(HaveOccurred()) - - fileHandle, err := os.Open(file) + fileHandle, err := os.Open(preparedAudioFixture()) Expect(err).ToNot(HaveOccurred()) defer fileHandle.Close() @@ -443,25 +440,10 @@ var _ = Describe("E2E test", func() { }) }) -func downloadHttpFile(url string) (string, error) { - resp, err := http.Get(url) - if err != nil { - return "", err - } - defer resp.Body.Close() - - tmpfile, err := os.CreateTemp("", "example") - if err != nil { - return "", err - } - defer tmpfile.Close() - - _, err = io.Copy(tmpfile, resp.Body) - if err != nil { - return "", err - } - - return tmpfile.Name(), nil +func preparedAudioFixture() string { + path := os.Getenv("AIO_AUDIO_FIXTURE") + Expect(path).NotTo(BeEmpty(), "run `make test-resources TARGET=aio` before the AIO suite") + return path } func requestRerank(modelName, query string, documents []string, topN *int, apiEndpoint string) (*http.Response, []byte) { diff --git a/tests/e2e/distributed/nats_jwt_helpers_test.go b/tests/e2e/distributed/nats_jwt_helpers_test.go index 80060ef6a..e0723b378 100644 --- a/tests/e2e/distributed/nats_jwt_helpers_test.go +++ b/tests/e2e/distributed/nats_jwt_helpers_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/internal/testfixtures" "github.com/mudler/LocalAI/pkg/natsauth" "github.com/nats-io/jwt/v2" "github.com/nats-io/nkeys" @@ -17,6 +18,8 @@ import ( "github.com/testcontainers/testcontainers-go" tcnats "github.com/testcontainers/testcontainers-go/modules/nats" + tcnetwork "github.com/testcontainers/testcontainers-go/network" + "github.com/testcontainers/testcontainers-go/wait" ) // JWTTestInfra holds a NATS server configured with JWT auth and minted worker credentials. @@ -34,6 +37,9 @@ func SetupJWTInfra() *JWTTestInfra { GinkgoHelper() infra := &JWTTestInfra{TestInfra: &TestInfra{Ctx: context.Background()}} + Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed()) + testNetwork, err := testfixtures.DockerNetwork() + Expect(err).NotTo(HaveOccurred()) operatorJWT, accountJWT, accountSeed, err := jwtResolverMaterial() Expect(err).ToNot(HaveOccurred()) @@ -51,15 +57,18 @@ resolver_preload: { var natsContainer *tcnats.NATSContainer // Override default testcontainers -js: JetStream fails without a system account in JWT mode. - natsContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine", + natsContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine, tcnats.WithConfigFile(bytes.NewBufferString(conf)), testcontainers.WithCmd("-c", "/etc/nats.conf"), + tcnetwork.WithNetworkName([]string{"nats"}, testNetwork), + testcontainers.WithWaitStrategy(wait.ForLog("Server is ready")), ) Expect(err).ToNot(HaveOccurred()) infra.NATSContainer = natsContainer - infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) + natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222") Expect(err).ToNot(HaveOccurred()) + infra.NatsURL = "nats://" + natsEndpoint infra.NodeID = "550e8400-e29b-41d4-a716-446655440000" cfg := natsauth.Config{AccountSeed: infra.AccountSeed, WorkerJWTTTL: time.Hour} @@ -153,4 +162,4 @@ func accountPublicKeyFromSeed(accountSeed string) string { func nodeSubjectPrefix(nodeID string) string { tok := strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-").Replace(nodeID) return "nodes." + tok -} \ No newline at end of file +} diff --git a/tests/e2e/distributed/testhelpers_test.go b/tests/e2e/distributed/testhelpers_test.go index 68cf537e3..d54c0897a 100644 --- a/tests/e2e/distributed/testhelpers_test.go +++ b/tests/e2e/distributed/testhelpers_test.go @@ -2,9 +2,11 @@ package distributed_test import ( "context" + "fmt" "time" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/internal/testfixtures" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -12,6 +14,7 @@ import ( "github.com/testcontainers/testcontainers-go" tcnats "github.com/testcontainers/testcontainers-go/modules/nats" tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + tcnetwork "github.com/testcontainers/testcontainers-go/network" "github.com/testcontainers/testcontainers-go/wait" ) @@ -32,9 +35,13 @@ func SetupInfra(dbName string) *TestInfra { infra := &TestInfra{Ctx: context.Background()} var err error + Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.Postgres16Alpine, "distributed-e2e")).To(Succeed()) + Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed()) + testNetwork, err := testfixtures.DockerNetwork() + Expect(err).NotTo(HaveOccurred()) // Start PostgreSQL container - infra.PGContainer, err = tcpostgres.Run(infra.Ctx, "postgres:16-alpine", + infra.PGContainer, err = tcpostgres.Run(infra.Ctx, testfixtures.Postgres16Alpine, tcpostgres.WithDatabase(dbName), tcpostgres.WithUsername("test"), tcpostgres.WithPassword("test"), @@ -43,18 +50,23 @@ func SetupInfra(dbName string) *TestInfra { WithOccurrence(2). WithStartupTimeout(30*time.Second), ), + tcnetwork.WithNetworkName([]string{"postgres"}, testNetwork), ) Expect(err).ToNot(HaveOccurred()) - infra.PGURL, err = infra.PGContainer.ConnectionString(infra.Ctx, "sslmode=disable") + pgEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.PGContainer, "5432") Expect(err).ToNot(HaveOccurred()) + infra.PGURL = fmt.Sprintf("postgres://test:test@%s/%s?sslmode=disable", pgEndpoint, dbName) // Start NATS container - infra.NATSContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine") + infra.NATSContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine, + tcnetwork.WithNetworkName([]string{"nats"}, testNetwork), + testcontainers.WithWaitStrategy(wait.ForLog("Server is ready"))) Expect(err).ToNot(HaveOccurred()) - infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) + natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222") Expect(err).ToNot(HaveOccurred()) + infra.NatsURL = "nats://" + natsEndpoint // Connect messaging client infra.NC, err = messaging.New(infra.NatsURL) @@ -83,12 +95,18 @@ func SetupNATSOnly() *TestInfra { infra := &TestInfra{Ctx: context.Background()} var err error + Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed()) + testNetwork, err := testfixtures.DockerNetwork() + Expect(err).NotTo(HaveOccurred()) - infra.NATSContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine") + infra.NATSContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine, + tcnetwork.WithNetworkName([]string{"nats"}, testNetwork), + testcontainers.WithWaitStrategy(wait.ForLog("Server is ready"))) Expect(err).ToNot(HaveOccurred()) - infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) + natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222") Expect(err).ToNot(HaveOccurred()) + infra.NatsURL = "nats://" + natsEndpoint infra.NC, err = messaging.New(infra.NatsURL) Expect(err).ToNot(HaveOccurred())