From 2383726d6d6e18adc30dc49df4c1b7a0937fde50 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 19 Aug 2026 16:39:39 +0200 Subject: [PATCH] Revert "chore(tests): Avoid network, sleep and more during tests" (#11601) Revert "chore(tests): Avoid network, sleep and more during tests (#11050)" This reverts commit cb3bf7af3fa7a5312e285a657088bacabc6e4d6d. --- .agents/building-and-testing.md | 3 - .agents/ci-caching.md | 29 -- .docker/apt-mirror.sh | 14 +- .docker/bonsai-compile.sh | 8 +- .docker/ik-llama-cpp-compile.sh | 2 - .docker/install-build-proxy-ca.sh | 26 - .docker/llama-cpp-compile.sh | 2 - .docker/turboquant-compile.sh | 2 - .../actions/configure-apt-mirror/action.yml | 13 +- .github/jetson-wheels.json | 25 - .github/scripts/inject-build-proxy-ca.sh | 21 - .github/scripts/start-build-proxy.sh | 48 -- .github/scripts/stop-build-proxy.sh | 55 -- .github/workflows/backend_build.yml | 80 --- .github/workflows/external-probes.yml | 38 -- .github/workflows/image_build.yml | 43 -- .github/workflows/jetson-wheels.yml | 131 ----- .github/workflows/test-resource-refresh.yml | 76 --- .github/workflows/test.yml | 25 +- .github/workflows/tests-aio.yml | 4 +- .github/workflows/tests-e2e.yml | 3 - AGENTS.md | 2 +- Dockerfile | 81 ++- Makefile | 42 +- backend/Dockerfile.audio-cpp | 7 +- backend/Dockerfile.base-grpc-builder | 6 +- backend/Dockerfile.bonsai | 18 +- backend/Dockerfile.ds4 | 9 +- backend/Dockerfile.golang | 37 +- backend/Dockerfile.ik-llama-cpp | 18 +- backend/Dockerfile.llama-cpp | 18 +- backend/Dockerfile.privacy-filter | 16 +- backend/Dockerfile.python | 44 +- backend/Dockerfile.rust | 14 +- backend/Dockerfile.turboquant | 18 +- backend/python/common/libbackend.sh | 73 +-- backend/python/common/pypi_mirror_server.py | 226 --------- .../python/common/pypi_mirror_server_test.py | 100 ---- cmd/build-proxy/main.go | 52 -- cmd/test-resources/main.go | 468 ------------------ cmd/test-resources/main_test.go | 54 -- core/cli/workerregistry/client.go | 13 +- core/cli/workerregistry/credentials.go | 12 +- core/config/model_config_test.go | 14 +- .../config/testdata/hermes-2-pro-mistral.yaml | 7 - core/explorer/database_test.go | 8 +- core/gallery/backends_test.go | 62 +-- .../importers/discovery_options_test.go | 49 -- core/gallery/importers/importers.go | 38 +- .../gallery/importers/importers_suite_test.go | 63 --- core/gallery/importers/llama-cpp.go | 33 +- core/gallery/request_test.go | 6 +- core/http/app_test.go | 26 +- .../endpoints/localai/import_model_test.go | 11 - .../endpoints/localai/localai_suite_test.go | 11 - .../endpoints/ollama/helpers_internal_test.go | 2 +- core/http/openresponses_test.go | 2 +- .../services/agentpool/services_suite_test.go | 11 - core/services/agents/agents_suite_test.go | 11 - .../buildproxy/buildproxy_suite_test.go | 13 - core/services/buildproxy/proxy.go | 280 ----------- core/services/buildproxy/proxy_test.go | 74 --- core/services/buildproxy/server.go | 233 --------- core/services/buildproxy/server_test.go | 46 -- core/services/cloudproxy/mitm/proxy.go | 35 +- core/services/jobs/jobs_suite_test.go | 11 - core/services/nodes/file_stager_http.go | 11 +- core/services/nodes/nodes_suite_test.go | 11 - core/services/nodes/registry.go | 22 +- core/services/testutil/testdb.go | 101 +--- core/services/worker/free_timeout_test.go | 25 +- core/services/worker/lifecycle.go | 2 +- core/services/worker/supervisor.go | 17 +- core/startup/model_preload.go | 18 +- core/startup/model_preload_test.go | 14 +- docs/content/development/offline-tests.md | 77 --- internal/backoff/backoff.go | 23 - internal/backoff/backoff_suite_test.go | 13 - internal/backoff/backoff_test.go | 35 -- internal/testfixtures/images.go | 55 -- internal/testresources/bundle.go | 164 ------ internal/testresources/httpcache.go | 92 ---- internal/testresources/resources.go | 215 -------- .../testresources/resources_suite_test.go | 15 - internal/testresources/resources_test.go | 126 ----- pkg/downloader/cancel_test.go | 6 +- pkg/downloader/retry.go | 9 +- pkg/downloader/stall_test.go | 4 +- pkg/downloader/uri_test.go | 26 +- pkg/httpclient/client.go | 21 +- pkg/huggingface-api/client.go | 57 +-- pkg/huggingface-api/client_test.go | 121 +---- pkg/model/loader.go | 14 +- pkg/model/loader_test.go | 8 +- pkg/modelartifacts/materializer.go | 5 +- pkg/oci/blob.go | 5 - pkg/oci/blob_test.go | 21 +- pkg/oci/cosignverify/verify.go | 4 +- pkg/oci/image_test.go | 32 +- pkg/oci/ollama.go | 20 +- pkg/oci/ollama_test.go | 26 +- pkg/testnetwork/guard.go | 57 --- pkg/testnetwork/guard_suite_test.go | 15 - pkg/testnetwork/guard_test.go | 35 -- scripts/jetson-wheels-sync.py | 178 ------- scripts/run-coverage.sh | 160 +----- scripts/run-test-linux-offline.sh | 70 --- scripts/run-test-offline.sh | 19 - scripts/summarize-ginkgo-waits.sh | 29 -- test-resources/manifests/aio.json | 12 - test-resources/manifests/backend.json | 1 - test-resources/manifests/default-darwin.json | 7 - test-resources/manifests/default.json | 10 - test-resources/manifests/distributed-e2e.json | 14 - test-resources/manifests/external-probes.json | 1 - test-resources/manifests/hardware.json | 1 - test-resources/manifests/lock.json | 12 - tests/e2e-aio/e2e_suite_test.go | 16 +- tests/e2e-aio/e2e_test.go | 44 +- .../e2e/distributed/nats_jwt_helpers_test.go | 15 +- tests/e2e/distributed/testhelpers_test.go | 30 +- 121 files changed, 407 insertions(+), 4756 deletions(-) delete mode 100644 .docker/install-build-proxy-ca.sh delete mode 100644 .github/jetson-wheels.json delete mode 100755 .github/scripts/inject-build-proxy-ca.sh delete mode 100755 .github/scripts/start-build-proxy.sh delete mode 100755 .github/scripts/stop-build-proxy.sh delete mode 100644 .github/workflows/external-probes.yml delete mode 100644 .github/workflows/jetson-wheels.yml delete mode 100644 .github/workflows/test-resource-refresh.yml delete mode 100644 backend/python/common/pypi_mirror_server.py delete mode 100644 backend/python/common/pypi_mirror_server_test.py delete mode 100644 cmd/build-proxy/main.go delete mode 100644 cmd/test-resources/main.go delete mode 100644 cmd/test-resources/main_test.go delete mode 100644 core/config/testdata/hermes-2-pro-mistral.yaml delete mode 100644 core/gallery/importers/discovery_options_test.go delete mode 100644 core/services/buildproxy/buildproxy_suite_test.go delete mode 100644 core/services/buildproxy/proxy.go delete mode 100644 core/services/buildproxy/proxy_test.go delete mode 100644 core/services/buildproxy/server.go delete mode 100644 core/services/buildproxy/server_test.go delete mode 100644 docs/content/development/offline-tests.md delete mode 100644 internal/backoff/backoff.go delete mode 100644 internal/backoff/backoff_suite_test.go delete mode 100644 internal/backoff/backoff_test.go delete mode 100644 internal/testfixtures/images.go delete mode 100644 internal/testresources/bundle.go delete mode 100644 internal/testresources/httpcache.go delete mode 100644 internal/testresources/resources.go delete mode 100644 internal/testresources/resources_suite_test.go delete mode 100644 internal/testresources/resources_test.go delete mode 100644 pkg/testnetwork/guard.go delete mode 100644 pkg/testnetwork/guard_suite_test.go delete mode 100644 pkg/testnetwork/guard_test.go delete mode 100644 scripts/jetson-wheels-sync.py delete mode 100755 scripts/run-test-linux-offline.sh delete mode 100755 scripts/run-test-offline.sh delete mode 100755 scripts/summarize-ginkgo-waits.sh delete mode 100644 test-resources/manifests/aio.json delete mode 100644 test-resources/manifests/backend.json delete mode 100644 test-resources/manifests/default-darwin.json delete mode 100644 test-resources/manifests/default.json delete mode 100644 test-resources/manifests/distributed-e2e.json delete mode 100644 test-resources/manifests/external-probes.json delete mode 100644 test-resources/manifests/hardware.json delete mode 100644 test-resources/manifests/lock.json diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 346620c3a..021d555ec 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -21,9 +21,6 @@ Let's say the user wants to build a particular backend for a given platform. For The core Go suites (`./pkg`, `./core`, plus the in-process integration suite `./tests/e2e`) are covered by a **strict, monotonic coverage ratchet**: - `make test-coverage` — runs the suites with `covermode=atomic` instrumentation and writes a merged profile to `coverage/coverage.out`. Uses the same prerequisites as `make test`. - - Prints per-root wall time and the slowest specs/hooks exceeding `COVERAGE_SLOW_SPEC_THRESHOLD` (default 3 seconds, capped by `COVERAGE_SLOW_SPEC_LIMIT`, default 25 per root); machine-readable root timings are written to `coverage/timings.tsv`. - - Verbose Ginkgo output is written to `coverage/logs/.log`, with the prior run retained as `.log.previous`. The terminal prints one status line per root and a short failure extract. If any suite fails, no merged profile is produced and the percentage ratchet is explicitly not run. A lock under `coverage/` rejects concurrent runs, which would otherwise corrupt their shared profiles and logs. - - Suites run in parallel by default and each recursive root invocation has a five-minute budget. Override auto-detected parallelism with `COVERAGE_PROCS`; tune diagnostics with `COVERAGE_SUITE_TIMEOUT` and `COVERAGE_PROGRESS_AFTER`. A timeout is a performance failure to investigate, not a reason to raise the committed default. - **`--coverpkg` (`COVERAGE_COVERPKG = core/...,pkg/...`):** coverage is attributed to the core+pkg packages, not just the package under test. This is what lets the in-process `tests/e2e` suite (which drives the real HTTP server over loopback via `application.New`) credit the `core/http/endpoints/...` handlers it exercises — folding it in roughly doubled endpoint coverage (e.g. `endpoints/openai` 13.6% → 52%). The denominator is therefore *all* of `core`+`pkg` (minus generated proto, dropped via `COVERAGE_EXCLUDE_RE`), so the number isn't comparable to a plain per-package figure. - **Integration suites (`COVERAGE_E2E_ROOTS = ./tests/e2e`)** run non-recursively (excludes `tests/e2e/distributed`, which needs containers) with `--label-filter=!real-models` (those need a downloaded model) against the mock backend built by `prepare-test`. `tests/integration` is deliberately excluded — it needs `make backends/local-store`, which the coverage CI job doesn't build. - **Flake note:** folding integration tests into a *strict* gate means a hard e2e failure (or a spec that silently stops running) can fail the coverage gate, not just the test. `--flake-attempts` absorbs transient retryable failures; covermode=atomic keeps line coverage deterministic otherwise. diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index ed993dafb..6742049e6 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -1,20 +1,5 @@ # CI Build Caching -## Build network inventory and defensive proxy - -Backend and main-image builds use `cmd/build-proxy` as a strict HTTPS-intercepting -proxy through the host network. Its short-lived CA is injected into the running -BuildKit daemon and mounted over the conventional CA bundle during every -Dockerfile `RUN`. Plain HTTP and opaque CONNECT traffic fail the job. Every -destination is retained for 14 days as JSONL plus an aggregate host/method/byte -summary. Responses are spooled and checked against `Content-Length`; GET/HEAD -requests retry transient status codes or incomplete responses with exponential -backoff capped at 500ms. Request headers, bodies, credentials and query strings -are never recorded. - -The inventory is intended to size a future content-addressed cache and identify -hosts worth adding to curated OCI mirrors, such as the Jetson wheels mirror. - Container builds — both the root LocalAI image (`Dockerfile`) and the per-backend images (`backend/Dockerfile.*`) — share a registry-backed BuildKit cache plus a layered set of prebuilt base images. This file explains how the cache is laid out, what invalidates it, and how to bypass it. ## Workflow surfaces @@ -246,20 +231,6 @@ This applies only to `Dockerfile.python` because: Bump the format to daily (`+%Y-%m-%d`) or hourly (`+%Y-%m-%d-%H`) for faster refreshes. For one-shot rebuilds without changing the schedule, append a marker to the tag-suffix in the matrix or temporarily delete that backend's cache tag in quay. -## The jetson wheels mirror (l4t builds) - -The `requirements-l4t12.txt` / `-l4t13.txt` files pull CUDA aarch64 torch wheels from `pypi.jetson-ai-lab.io` via `--extra-index-url`. That index has a history of multi-hour 502 outages, and a 502 on **any** project page aborts the whole uv resolution — uv consults every configured index for every requirement, so even PyPI-hosted packages die with it. To keep l4t builds green through outages, CI serves those wheels from a mirror it controls: - -- **Storage**: `ghcr.io/mudler/localai/jetson-wheels:{jp6-cu129,jp7-cu130}` — scratch OCI images holding the wheel subset, laid out like the upstream index (`/jp6/cu129/torch/`). -- **Sync**: `.github/workflows/jetson-wheels.yml` (Saturdays 03:00 UTC, ahead of the weekly `DEPS_REFRESH` re-resolve; also `workflow_dispatch` and master pushes touching its inputs) runs `scripts/jetson-wheels-sync.py` against the package list in `.github/jetson-wheels.json`. During an upstream outage the sync keeps the last-known-good wheels and exits green. -- **Consumption**: `backend_build.yml` resolves the matching tag for `build-type: l4t` entries (cuda 12 → `jp6-cu129`, 13 → `jp7-cu130`) and passes it as the `JETSON_WHEELS_IMAGE` build-arg; `Dockerfile.python` bind-mounts it at `/jetson-wheels`; `installRequirements` in `backend/python/common/libbackend.sh` serves that directory on localhost as a PEP 503 index (`backend/python/common/pypi_mirror_server.py`) and rewrites the jetson index host in the requirements files to it. The local index 404s for anything it doesn't carry, which uv follows up on PyPI — only the jetson-built wheels resolve locally. -- **Fallbacks**: if the mirror tag doesn't exist (bootstrap) `backend_build.yml` passes `scratch`, the mount is empty, and the build talks to the upstream index exactly as before. Builds outside CI (local, real Jetsons) never set `JETSON_WHEELS_IMAGE` and are unaffected. -- **Cache interaction**: the bind mount's content is part of the `RUN ... make` layer's BuildKit hash, so a refreshed wheels image invalidates the install layer on the next build — no extra cache-buster needed. - -**Extending the package list**: a package a build needs from the jetson index but missing from `.github/jetson-wheels.json` resolves from PyPI instead — for compiled CUDA packages that silently means a CPU build. When adding an l4t backend with new compiled deps, add them to the list and dispatch `jetson-wheels.yml`. - -**Bootstrap** (one-time): `gh workflow run jetson-wheels.yml --ref master`, then make the `jetson-wheels` ghcr package public so anonymous pulls work (Settings → Packages). - ## ccache for C++ backend builds `Dockerfile.{llama-cpp,ik-llama-cpp,turboquant}` declare a BuildKit cache mount on `/root/.ccache`: diff --git a/.docker/apt-mirror.sh b/.docker/apt-mirror.sh index d00de21d2..8bd41d1f7 100755 --- a/.docker/apt-mirror.sh +++ b/.docker/apt-mirror.sh @@ -7,7 +7,8 @@ # # Inputs (env): # APT_MIRROR Replacement for archive.ubuntu.com and security.ubuntu.com -# (e.g. "https://azure.archive.ubuntu.com"). +# (e.g. "http://azure.archive.ubuntu.com" or +# "https://mirrors.edge.kernel.org"). # Leave empty to keep upstream. The trailing "/ubuntu/..." # path is preserved by the rewrite. # APT_PORTS_MIRROR Replacement for ports.ubuntu.com (arm64/ppc64el/...). @@ -17,8 +18,8 @@ set -e -if [ -f /usr/local/sbin/install-build-proxy-ca ]; then - sh /usr/local/sbin/install-build-proxy-ca +if [ -z "${APT_MIRROR}" ] && [ -z "${APT_PORTS_MIRROR}" ]; then + exit 0 fi # Ubuntu 24.04 (noble) ships DEB822 sources at /etc/apt/sources.list.d/ubuntu.sources; @@ -35,11 +36,4 @@ for f in /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list; do fi done -# Build networking is HTTPS-only. Upgrade any untouched distribution defaults -# as well (notably ports.ubuntu.com when a caller leaves its override empty). -for f in /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list; do - [ -f "$f" ] || continue - sed -i -E 's,http://,https://,g' "$f" -done - echo "apt-mirror: rewrote sources (APT_MIRROR='${APT_MIRROR}', APT_PORTS_MIRROR='${APT_PORTS_MIRROR}')" diff --git a/.docker/bonsai-compile.sh b/.docker/bonsai-compile.sh index 23d1cf378..38439439d 100755 --- a/.docker/bonsai-compile.sh +++ b/.docker/bonsai-compile.sh @@ -4,8 +4,6 @@ set -euxo pipefail -sh /LocalAI/.docker/install-build-proxy-ca.sh - export CCACHE_DIR=/root/.ccache ccache --max-size=5G || true ccache -z || true @@ -25,11 +23,7 @@ if [ -z "${BUILD_TYPE:-}" ]; then # Pure CPU image: one ggml CPU_ALL_VARIANTS build replaces the per-microarch binaries. # arm64: the armv9.2 SME variants need gcc-14 (gcc-13 rejects +sme). if [ "${TARGETARCH}" = "arm64" ]; then - APT_MIRROR="${APT_MIRROR:-https://azure.archive.ubuntu.com}" \ - APT_PORTS_MIRROR="${APT_PORTS_MIRROR:-https://azure.ports.ubuntu.com}" \ - sh /LocalAI/.docker/apt-mirror.sh - apt-get update -qq - apt-get install -y -qq gcc-14 g++-14 + apt-get update -qq && apt-get install -y -qq gcc-14 g++-14 export CC=gcc-14 CXX=g++-14 fi make bonsai-cpu-all diff --git a/.docker/ik-llama-cpp-compile.sh b/.docker/ik-llama-cpp-compile.sh index 5409c78d6..3da869007 100755 --- a/.docker/ik-llama-cpp-compile.sh +++ b/.docker/ik-llama-cpp-compile.sh @@ -4,8 +4,6 @@ set -euxo pipefail -sh /LocalAI/.docker/install-build-proxy-ca.sh - export CCACHE_DIR=/root/.ccache ccache --max-size=5G || true ccache -z || true diff --git a/.docker/install-build-proxy-ca.sh b/.docker/install-build-proxy-ca.sh deleted file mode 100644 index 2475fdb7f..000000000 --- a/.docker/install-build-proxy-ca.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# Install CI's optional HTTPS interception CA without replacing public roots. - -set -e - -proxy_ca=/run/secrets/build_proxy_ca -if [ ! -s "$proxy_ca" ]; then - exit 0 -fi - -mkdir -p /usr/local/share/ca-certificates -cp "$proxy_ca" /usr/local/share/ca-certificates/localai-build-proxy.crt -if command -v update-ca-certificates >/dev/null 2>&1; then - update-ca-certificates -elif [ -f /etc/ssl/certs/ca-certificates.crt ]; then - cat "$proxy_ca" >>/etc/ssl/certs/ca-certificates.crt -else - echo 'build proxy: no system CA bundle found' >&2 - exit 1 -fi - -if [ -d /etc/apt/apt.conf.d ]; then - cat > /etc/apt/apt.conf.d/99localai-build-proxy-ca <&2 - exit 1 -fi - -docker exec "$container" mkdir -p /usr/local/share/ca-certificates /etc/ssl/certs -docker cp "$ca" "$container:/usr/local/share/ca-certificates/localai-build-proxy.crt" -docker exec "$container" sh -eu -c ' - if command -v update-ca-certificates >/dev/null 2>&1; then - update-ca-certificates - else - cat /usr/local/share/ca-certificates/localai-build-proxy.crt >>/etc/ssl/certs/ca-certificates.crt - fi -' -docker restart "$container" >/dev/null diff --git a/.github/scripts/start-build-proxy.sh b/.github/scripts/start-build-proxy.sh deleted file mode 100755 index 7d3a78978..000000000 --- a/.github/scripts/start-build-proxy.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -output="${RUNNER_TEMP}/localai-build-proxy" -mkdir -p "$output" -CGO_ENABLED=0 GOCACHE="${RUNNER_TEMP}/go-build-cache" go build -o "$output/build-proxy" ./cmd/build-proxy -nohup "$output/build-proxy" --listen 127.0.0.1:18080 --output "$output" >"$output/proxy.log" 2>&1 & -echo "$!" >"$output/proxy.pid" -for _ in $(seq 1 50); do - grep -q '^ca=' "$output/proxy.log" && break - sleep 0.1 -done -grep '^proxy=' "$output/proxy.log" -grep '^ca=' "$output/proxy.log" -proxy_ca="$output/ca/ca.crt" -ca_bundle="$output/ca/ca-bundle.crt" -system_ca="" -for candidate in \ - /etc/ssl/certs/ca-certificates.crt \ - /etc/ssl/cert.pem \ - /etc/pki/tls/certs/ca-bundle.crt \ - /etc/openssl/certs/ca-certificates.crt; do - if test -s "$candidate"; then - system_ca="$candidate" - break - fi -done -if test -z "$system_ca"; then - echo 'build proxy: unable to find the runner system CA bundle' >&2 - exit 1 -fi -cat "$system_ca" "$proxy_ca" >"$ca_bundle" -{ - echo "LOCALAI_BUILD_PROXY=http://127.0.0.1:18080" - echo "LOCALAI_BUILD_PROXY_OUTPUT=$output" - echo "LOCALAI_BUILD_PROXY_CA=$proxy_ca" - echo "LOCALAI_BUILD_PROXY_CA_BUNDLE=$ca_bundle" - echo "HTTP_PROXY=http://127.0.0.1:18080" - echo "HTTPS_PROXY=http://127.0.0.1:18080" - echo "http_proxy=http://127.0.0.1:18080" - echo "https_proxy=http://127.0.0.1:18080" - echo "SSL_CERT_FILE=$ca_bundle" - echo "CURL_CA_BUNDLE=$ca_bundle" - echo "REQUESTS_CA_BUNDLE=$ca_bundle" - echo "GIT_SSL_CAINFO=$ca_bundle" - echo "NODE_EXTRA_CA_CERTS=$proxy_ca" - echo "NO_PROXY=localhost,127.0.0.1" - echo "no_proxy=localhost,127.0.0.1" -} >>"$GITHUB_ENV" diff --git a/.github/scripts/stop-build-proxy.sh b/.github/scripts/stop-build-proxy.sh deleted file mode 100755 index 0268ce029..000000000 --- a/.github/scripts/stop-build-proxy.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -if test -z "${LOCALAI_BUILD_PROXY_OUTPUT:-}"; then - echo 'Build proxy did not start; skipping inventory finalization' - exit 0 -fi -output="$LOCALAI_BUILD_PROXY_OUTPUT" -if test -f "$output/proxy.pid"; then - kill -TERM "$(cat "$output/proxy.pid")" 2>/dev/null || true - for _ in $(seq 1 50); do - test -f "$output/summary.json" && break - sleep 0.1 - done -fi - -# Later artifact uploads and action post-hooks must not target a stopped proxy. -{ - echo 'HTTP_PROXY=' - echo 'HTTPS_PROXY=' - echo 'http_proxy=' - echo 'https_proxy=' - echo 'SSL_CERT_FILE=' - echo 'CURL_CA_BUNDLE=' - echo 'REQUESTS_CA_BUNDLE=' - echo 'GIT_SSL_CAINFO=' - echo 'NODE_EXTRA_CA_CERTS=' -} >>"$GITHUB_ENV" -if test -f "$output/summary.json"; then - { - echo '### Build network inventory' - echo - echo 'HTTPS without the generated CA is reported as CONNECT because its HTTP method is encrypted.' - echo - echo '```json' - cat "$output/summary.json" - echo '```' - } >>"$GITHUB_STEP_SUMMARY" -fi - -# A matrix cancellation can interrupt checkout or a BuildKit request at any -# point. Preserve whatever inventory exists, but do not replace the canceled -# conclusion with a misleading proxy-enforcement failure. -if test "${LOCALAI_BUILD_JOB_STATUS:-}" = cancelled; then - echo 'Build was cancelled; skipping network inventory enforcement' - exit 0 -fi - -if ! test -s "$output/events.jsonl"; then - echo 'Build proxy produced no network inventory' >&2 - exit 1 -fi -if grep -qE '"method":"CONNECT"|"error":"plain HTTP is forbidden"' "$output/events.jsonl"; then - echo 'Build traffic bypassed HTTPS interception or attempted plain HTTP' >&2 - exit 1 -fi diff --git a/.github/workflows/backend_build.yml b/.github/workflows/backend_build.yml index 4a9acfb9f..05d50cf82 100644 --- a/.github/workflows/backend_build.yml +++ b/.github/workflows/backend_build.yml @@ -153,27 +153,9 @@ jobs: with: platforms: all - - name: Set up Go for build proxy - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Start build network proxy - run: .github/scripts/start-build-proxy.sh - - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@master - with: - driver-opts: | - network=host - env.http_proxy=${{ env.LOCALAI_BUILD_PROXY }} - env.https_proxy=${{ env.LOCALAI_BUILD_PROXY }} - - - name: Trust build proxy CA in BuildKit - env: - BUILDER_NAME: ${{ steps.buildx.outputs.name }} - run: .github/scripts/inject-build-proxy-ca.sh - name: Login to DockerHub if: github.event_name != 'pull_request' @@ -199,41 +181,6 @@ jobs: id: deps_refresh run: echo "key=$(date -u +%Y-W%V)" >> "$GITHUB_OUTPUT" - - name: Login to ghcr.io (jetson wheels mirror) - if: inputs.build-type == 'l4t' - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} - - # l4t builds pull their CUDA aarch64 torch wheels from - # pypi.jetson-ai-lab.io, which has a history of multi-hour 502 outages - # that fail every l4t job. jetson-wheels.yml mirrors those wheels into - # ghcr weekly; here we hand the mirror image to Dockerfile.python, - # which serves it as a local package index during pip install (see - # installRequirements in backend/python/common/libbackend.sh). Falls - # back to scratch — i.e. building straight against the upstream index — - # when the mirror tag doesn't exist yet, so the mirror can bootstrap - # without a chicken-and-egg failure. - - name: Resolve jetson wheels mirror image - id: jetson_wheels - if: inputs.build-type == 'l4t' - run: | - repo="ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')/jetson-wheels" - case "${{ inputs.cuda-major-version }}" in - 12) tag="jp6-cu129" ;; - 13) tag="jp7-cu130" ;; - *) tag="" ;; - esac - img="" - if [ -n "$tag" ] && docker buildx imagetools inspect "$repo:$tag" >/dev/null 2>&1; then - img="$repo:$tag" - else - echo "jetson wheels image $repo:$tag not found; building against the upstream index" - fi - echo "image=$img" >> "$GITHUB_OUTPUT" - - name: Build and push by digest id: build uses: docker/build-push-action@v7 @@ -241,9 +188,6 @@ jobs: with: builder: ${{ steps.buildx.outputs.name }} build-args: | - HTTP_PROXY=${{ env.LOCALAI_BUILD_PROXY }} - HTTPS_PROXY=${{ env.LOCALAI_BUILD_PROXY }} - NO_PROXY=localhost,127.0.0.1 BUILD_TYPE=${{ inputs.build-type }} SKIP_DRIVERS=${{ inputs.skip-drivers }} CUDA_MAJOR_VERSION=${{ inputs.cuda-major-version }} @@ -257,9 +201,6 @@ jobs: DEPS_REFRESH=${{ steps.deps_refresh.outputs.key }} BUILDER_BASE_IMAGE=${{ inputs.builder-base-image }} BUILDER_TARGET=${{ inputs.builder-base-image != '' && 'builder-prebuilt' || 'builder-fromsource' }} - JETSON_WHEELS_IMAGE=${{ steps.jetson_wheels.outputs.image || 'scratch' }} - secret-files: | - build_proxy_ca=${{ env.LOCALAI_BUILD_PROXY_CA }} context: ${{ inputs.context }} file: ${{ inputs.dockerfile }} cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache${{ inputs.tag-suffix }}-${{ inputs.platform-tag }} @@ -319,9 +260,6 @@ jobs: with: builder: ${{ steps.buildx.outputs.name }} build-args: | - HTTP_PROXY=${{ env.LOCALAI_BUILD_PROXY }} - HTTPS_PROXY=${{ env.LOCALAI_BUILD_PROXY }} - NO_PROXY=localhost,127.0.0.1 BUILD_TYPE=${{ inputs.build-type }} SKIP_DRIVERS=${{ inputs.skip-drivers }} CUDA_MAJOR_VERSION=${{ inputs.cuda-major-version }} @@ -335,9 +273,6 @@ jobs: DEPS_REFRESH=${{ steps.deps_refresh.outputs.key }} BUILDER_BASE_IMAGE=${{ inputs.builder-base-image }} BUILDER_TARGET=${{ inputs.builder-base-image != '' && 'builder-prebuilt' || 'builder-fromsource' }} - JETSON_WHEELS_IMAGE=${{ steps.jetson_wheels.outputs.image || 'scratch' }} - secret-files: | - build_proxy_ca=${{ env.LOCALAI_BUILD_PROXY_CA }} context: ${{ inputs.context }} file: ${{ inputs.dockerfile }} cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache${{ inputs.tag-suffix }}-${{ inputs.platform-tag }} @@ -351,18 +286,3 @@ jobs: - name: job summary run: | echo "Built image: ${{ steps.meta.outputs.labels }}" >> $GITHUB_STEP_SUMMARY - - - name: Stop build network proxy - if: ${{ always() && env.LOCALAI_BUILD_PROXY_OUTPUT != '' }} - env: - LOCALAI_BUILD_JOB_STATUS: ${{ job.status }} - run: .github/scripts/stop-build-proxy.sh - - - name: Upload build network inventory - if: ${{ always() && env.LOCALAI_BUILD_PROXY_OUTPUT != '' }} - uses: actions/upload-artifact@v7 - with: - name: build-network-${{ inputs.backend }}-${{ inputs.tag-suffix }}-${{ inputs.platform-tag || 'single' }} - path: ${{ env.LOCALAI_BUILD_PROXY_OUTPUT }} - if-no-files-found: warn - retention-days: 14 diff --git a/.github/workflows/external-probes.yml b/.github/workflows/external-probes.yml deleted file mode 100644 index 681f417e4..000000000 --- a/.github/workflows/external-probes.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: external compatibility probes - -on: - workflow_dispatch: - schedule: - - cron: '23 4 * * 1' - -permissions: - contents: read - -jobs: - external-probe-huggingface-xet: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 - with: - go-version: '1.26.x' - cache: false - - name: Probe Hugging Face Xet compatibility - run: LOCALAI_HF_XET_SMOKE=1 go test ./pkg/huggingface-api -ginkgo.focus='pinned public Xet fixture' -count=1 - - external-probe-sigstore: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 - with: - go-version: '1.26.x' - cache: false - - name: Probe public Sigstore compatibility - env: - LOCALAI_COSIGN_LIVE: '1' - LOCALAI_COSIGN_LIVE_IMAGE: ${{ vars.LOCALAI_COSIGN_LIVE_IMAGE }} - LOCALAI_COSIGN_LIVE_ISSUER: ${{ vars.LOCALAI_COSIGN_LIVE_ISSUER }} - LOCALAI_COSIGN_LIVE_IDENTITY_REGEX: ${{ vars.LOCALAI_COSIGN_LIVE_IDENTITY_REGEX }} - run: go test ./pkg/oci/cosignverify -ginkgo.focus='VerifyImage' -count=1 diff --git a/.github/workflows/image_build.yml b/.github/workflows/image_build.yml index 609c1f2c5..89bc4124f 100644 --- a/.github/workflows/image_build.yml +++ b/.github/workflows/image_build.yml @@ -129,27 +129,9 @@ jobs: with: platforms: all - - name: Set up Go for build proxy - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Start build network proxy - run: .github/scripts/start-build-proxy.sh - - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@master - with: - driver-opts: | - network=host - env.http_proxy=${{ env.LOCALAI_BUILD_PROXY }} - env.https_proxy=${{ env.LOCALAI_BUILD_PROXY }} - - - name: Trust build proxy CA in BuildKit - env: - BUILDER_NAME: ${{ steps.buildx.outputs.name }} - run: .github/scripts/inject-build-proxy-ca.sh - name: Login to DockerHub if: github.event_name != 'pull_request' @@ -173,9 +155,6 @@ jobs: with: builder: ${{ steps.buildx.outputs.name }} build-args: | - HTTP_PROXY=${{ env.LOCALAI_BUILD_PROXY }} - HTTPS_PROXY=${{ env.LOCALAI_BUILD_PROXY }} - NO_PROXY=localhost,127.0.0.1 BUILD_TYPE=${{ inputs.build-type }} CUDA_MAJOR_VERSION=${{ inputs.cuda-major-version }} CUDA_MINOR_VERSION=${{ inputs.cuda-minor-version }} @@ -186,8 +165,6 @@ jobs: UBUNTU_CODENAME=${{ inputs.ubuntu-codename }} APT_MIRROR=${{ steps.apt_mirror.outputs.effective-mirror }} APT_PORTS_MIRROR=${{ steps.apt_mirror.outputs.effective-ports-mirror }} - secret-files: | - build_proxy_ca=${{ env.LOCALAI_BUILD_PROXY_CA }} context: . file: ./Dockerfile cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache-localai${{ inputs.tag-suffix }}-${{ inputs.platform-tag }} @@ -241,9 +218,6 @@ jobs: with: builder: ${{ steps.buildx.outputs.name }} build-args: | - HTTP_PROXY=${{ env.LOCALAI_BUILD_PROXY }} - HTTPS_PROXY=${{ env.LOCALAI_BUILD_PROXY }} - NO_PROXY=localhost,127.0.0.1 BUILD_TYPE=${{ inputs.build-type }} CUDA_MAJOR_VERSION=${{ inputs.cuda-major-version }} CUDA_MINOR_VERSION=${{ inputs.cuda-minor-version }} @@ -254,8 +228,6 @@ jobs: UBUNTU_CODENAME=${{ inputs.ubuntu-codename }} APT_MIRROR=${{ steps.apt_mirror.outputs.effective-mirror }} APT_PORTS_MIRROR=${{ steps.apt_mirror.outputs.effective-ports-mirror }} - secret-files: | - build_proxy_ca=${{ env.LOCALAI_BUILD_PROXY_CA }} context: . file: ./Dockerfile cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache-localai${{ inputs.tag-suffix }}-${{ inputs.platform-tag }} @@ -267,18 +239,3 @@ jobs: - name: job summary run: | echo "Built image: ${{ steps.meta.outputs.labels }}" >> $GITHUB_STEP_SUMMARY - - - name: Stop build network proxy - if: ${{ always() && env.LOCALAI_BUILD_PROXY_OUTPUT != '' }} - env: - LOCALAI_BUILD_JOB_STATUS: ${{ job.status }} - run: .github/scripts/stop-build-proxy.sh - - - name: Upload build network inventory - if: ${{ always() && env.LOCALAI_BUILD_PROXY_OUTPUT != '' }} - uses: actions/upload-artifact@v7 - with: - name: build-network-localai-${{ inputs.build-type }}-${{ inputs.cuda-major-version }}-${{ inputs.cuda-minor-version }}-${{ inputs.platform-tag || 'single' }} - path: ${{ env.LOCALAI_BUILD_PROXY_OUTPUT }} - if-no-files-found: warn - retention-days: 14 diff --git a/.github/workflows/jetson-wheels.yml b/.github/workflows/jetson-wheels.yml deleted file mode 100644 index a943418a8..000000000 --- a/.github/workflows/jetson-wheels.yml +++ /dev/null @@ -1,131 +0,0 @@ ---- -name: 'sync jetson wheels mirror' - -# Mirrors the CUDA aarch64 wheels our l4t backends need from -# pypi.jetson-ai-lab.io into scratch OCI images on ghcr -# (ghcr.io/mudler/localai/jetson-wheels:, one tag per JetPack index). -# backend_build.yml hands the matching tag to Dockerfile.python, which -# bind-mounts it and serves it as a local package index during pip install -# (see installRequirements in backend/python/common/libbackend.sh), so the -# upstream index's recurring multi-hour 502 outages can no longer fail l4t -# builds. -# -# The package subset lives in .github/jetson-wheels.json. A package that a -# build needs from the jetson index but that is missing from that list will -# resolve from PyPI instead — for compiled CUDA packages that silently means -# a CPU build, so extend the list when adding an l4t backend with new -# compiled deps. -# -# When upstream is unreachable the sync keeps the previously mirrored wheels -# and exits green — the mirror serves last-known-good through outages. It -# only fails when upstream is down and the tag has never been published -# (bootstrap during an outage: nothing to serve yet). -# -# Triggers: -# - schedule (Saturdays 03:00 UTC) — refreshes ahead of base-images.yml -# (Saturdays 05:00 UTC) and the backend.yml weekly cron (Sundays), whose -# DEPS_REFRESH cache-bust re-resolves the python deps. -# - workflow_dispatch — manual one-off sync; also the bootstrap run: -# gh workflow run jetson-wheels.yml --ref master -# - push to master touching the config, the sync script, or this workflow. - -on: - schedule: - - cron: '0 3 * * 6' - workflow_dispatch: - push: - branches: [master] - paths: - - '.github/jetson-wheels.json' - - 'scripts/jetson-wheels-sync.py' - - '.github/workflows/jetson-wheels.yml' - -permissions: - contents: read - packages: write - -concurrency: - group: jetson-wheels-${{ github.repository }} - cancel-in-progress: false - -jobs: - sync: - if: github.repository == 'mudler/LocalAI' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - index: 'jp6/cu129' - tag: 'jp6-cu129' - - index: 'jp7/cu130' - tag: 'jp7-cu130' - steps: - - name: Checkout - uses: actions/checkout@v7 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@master - - - name: Login to ghcr.io - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} - - - name: Compute image name - id: image - run: | - repo="ghcr.io/$(echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')/jetson-wheels" - echo "ref=${repo}:${{ matrix.tag }}" >> "$GITHUB_OUTPUT" - - # Seed the working dir with the current mirror contents so the sync is - # incremental and an upstream outage keeps last-known-good wheels. - - name: Pull current mirror contents - run: | - mkdir -p wheels - # The image is declared linux/arm64 (its only consumers are arm64 - # l4t builds); pulling on this amd64 runner needs the explicit - # platform. The content is just wheel files — never executed here. - if docker pull --platform linux/arm64 "${{ steps.image.outputs.ref }}"; then - # scratch images have no command; docker create still needs one, - # but the container is never started so any path works. - cid="$(docker create "${{ steps.image.outputs.ref }}" /noop)" - docker export "${cid}" | tar -x -C wheels - docker rm "${cid}" - find wheels -name '*.whl' | sed 's/^/ existing: /' - else - echo "no existing mirror image (bootstrap run)" - fi - - - name: Sync from upstream - id: sync - run: | - python3 scripts/jetson-wheels-sync.py \ - --config .github/jetson-wheels.json \ - --index '${{ matrix.index }}' \ - --dest wheels \ - --changed-file /tmp/jetson-wheels-changed - if [ -f /tmp/jetson-wheels-changed ]; then - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - - - name: Push mirror image - if: steps.sync.outputs.changed == 'true' - run: | - cat > Dockerfile.jetson-wheels <<'EOF' - FROM scratch - COPY wheels/ / - EOF - # linux/arm64 because the consumers (l4t builds in - # backend_build.yml) build for arm64 and BuildKit refuses a - # platform-mismatched FROM; COPY-only, so no emulation is needed. - # provenance=false keeps the pushed ref a plain single manifest - # instead of an OCI index wrapping an attestation. - docker buildx build --push \ - --platform linux/arm64 \ - --provenance=false \ - -f Dockerfile.jetson-wheels \ - -t "${{ steps.image.outputs.ref }}" \ - . diff --git a/.github/workflows/test-resource-refresh.yml b/.github/workflows/test-resource-refresh.yml deleted file mode 100644 index 24e19f341..000000000 --- a/.github/workflows/test-resource-refresh.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- -name: refresh offline test resources - -on: - workflow_dispatch: - schedule: - - cron: '17 3 * * 1' - -permissions: - contents: read - issues: write - packages: write - -jobs: - refresh: - strategy: - fail-fast: false - matrix: - resource-set: [default, distributed-e2e, aio] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 - with: - go-version: '1.26.x' - cache: true - - uses: oras-project/setup-oras@v1 - - name: Verify upstream resources and build compressed cache - id: refresh - continue-on-error: true - env: - LOCALAI_TEST_RESOURCES_ONLINE: '1' - run: | - set -o pipefail - make update-offline-test-cache TEST_RESOURCE_SET=${{ matrix.resource-set }} 2>&1 | tee resource-refresh.log - - name: Upload investigation evidence - if: steps.refresh.outcome == 'failure' - uses: actions/upload-artifact@v4 - with: - name: test-resource-investigation-${{ matrix.resource-set }}-${{ github.run_id }} - path: resource-refresh.log - - name: Open or update investigation issue - if: steps.refresh.outcome == 'failure' - env: - GH_TOKEN: ${{ secrets.LOCALAI_BOT_TOKEN || github.token }} - RESOURCE_SET: ${{ matrix.resource-set }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - title="test resource integrity investigation: ${RESOURCE_SET}" - body=$(printf '%s\n\n%s\n' \ - "The scheduled offline-resource refresh failed for \`${RESOURCE_SET}\`." \ - "Do not update the manifest digest blindly. Download the evidence artifact from ${RUN_URL}, compare upstream checksums/signatures and release notes, inspect redirects, and search the [GitHub Advisory Database](https://github.com/advisories) and [OSV](https://osv.dev). Retry from a declared mirror to distinguish source drift from corruption.") - existing=$(gh issue list --state open --search "${title} in:title" --json number --jq '.[0].number // empty') - if [ -n "$existing" ]; then - gh issue comment "$existing" --body "$body" - else - gh issue create --title "$title" --body "$body" - fi - - name: Log in to GHCR - if: steps.refresh.outcome == 'success' - run: echo "${{ github.token }}" | oras login ghcr.io -u "${{ github.actor }}" --password-stdin - - name: Publish compressed cache as an OCI artifact - if: steps.refresh.outcome == 'success' - env: - RESOURCE_SET: ${{ matrix.resource-set }} - run: | - repository=$(printf '%s' "ghcr.io/${GITHUB_REPOSITORY}/localai-test-resources" | tr '[:upper:]' '[:lower:]') - digest=$(jq -r --arg set "$RESOURCE_SET" '.bundles[$set] | sub("sha256:"; "sha256-")' test-resources/manifests/lock.json) - oras push \ - --artifact-type application/vnd.localai.test-resources.v1 \ - "${repository}:${RESOURCE_SET},${RESOURCE_SET}-${digest}" \ - ".cache/test-resources/bundles/${RESOURCE_SET}.tar.zst:application/vnd.localai.test-resources.bundle.v1+zstd" \ - "test-resources/manifests/${RESOURCE_SET}.json:application/vnd.localai.test-resources.manifest.v1+json" - - name: Fail after preserving evidence - if: steps.refresh.outcome == 'failure' - run: exit 1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 71b2469ca..7929f6b65 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,8 +39,6 @@ jobs: # You can test your matrix by printing the current Go version - name: Display Go version run: go version - - name: Download Go modules - run: go mod download - name: Proto Dependencies run: | # Install protoc @@ -60,30 +58,13 @@ jobs: node-version: '22' - name: Build React UI run: make react-ui - - name: Record and pack declared test resources - run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=default - - name: Transfer local test-resource bundle - uses: actions/upload-artifact@v4 - with: - name: test-resources-default-${{ github.run_id }} - include-hidden-files: true - path: | - .cache/test-resources/bundles/default.tar.zst - test-resources/manifests/lock.json - - name: Clear recorded resource cache - run: rm -rf .cache/test-resources - - name: Restore local test-resource bundle - uses: actions/download-artifact@v4 - with: - name: test-resources-default-${{ github.run_id }} - path: . # Runs the core suite with coverage and fails if total coverage dropped # below the committed baseline (coverage-baseline.txt). The gate is # strict — any decrease fails. Raise the baseline with # `make test-coverage-baseline` and commit it when coverage rises. - name: Test (with coverage gate) run: | - LOCALAI_TEST_KERNEL_ENFORCE=1 PATH="$PATH:/root/go/bin" make --jobs 5 --output-sync=target test-coverage-check + PATH="$PATH:/root/go/bin" make --jobs 5 --output-sync=target test-coverage-check # tests/integration is outside the coverage roots because its store specs # need a live backend. test-stores builds and installs local-store before # running the complete suite, so new local-store specs are collected @@ -125,8 +106,6 @@ jobs: # You can test your matrix by printing the current Go version - name: Display Go version run: go version - - name: Download Go modules - run: go mod download - name: Dependencies run: | brew install protobuf grpc make protoc-gen-go protoc-gen-go-grpc libomp llvm opus ffmpeg @@ -145,7 +124,7 @@ jobs: # Used to run the newer GNUMake version from brew that supports --output-sync export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH" PATH="$PATH:$HOME/go/bin" make protogen-go - PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target TEST_RESOURCE_SET=default-darwin test + PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target test - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.23 diff --git a/.github/workflows/tests-aio.yml b/.github/workflows/tests-aio.yml index 4ca29730c..f8d3d34f0 100644 --- a/.github/workflows/tests-aio.yml +++ b/.github/workflows/tests-aio.yml @@ -76,9 +76,7 @@ jobs: PATH="$PATH:$HOME/go/bin" make protogen-go - name: Test run: | - PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e - LOCALAI_TEST_RESOURCES_ONLINE=1 PATH="$PATH:$HOME/go/bin" make update-offline-test-cache TEST_RESOURCE_SET=aio - LOCALAI_BACKEND_DIR="$GITHUB_WORKSPACE/backends" LOCALAI_MODELS_DIR="$GITHUB_WORKSPACE/tests/e2e-aio/models" LOCALAI_IMAGE_TAG=tests LOCALAI_IMAGE=local-ai PATH="$PATH:$HOME/go/bin" make run-e2e-aio + PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e e2e-aio - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.23 diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index df31ff98b..3c1cb711c 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -60,12 +60,9 @@ jobs: node-version: '22' - name: Build React UI run: make react-ui - - name: Record declared distributed test resources - run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=distributed-e2e - name: Test Backend E2E run: | PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e - PATH="$PATH:$HOME/go/bin" make test-e2e-distributed - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.23 diff --git a/AGENTS.md b/AGENTS.md index 74a15f034..71302268a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ LocalAI follows the Linux kernel project's [guidelines for AI coding assistants] |------|-------------| | [.agents/ai-coding-assistants.md](.agents/ai-coding-assistants.md) | Policy for AI-assisted contributions — licensing, DCO, attribution | | [.agents/building-and-testing.md](.agents/building-and-testing.md) | Building the project, running tests, Docker builds for specific platforms | -| [.agents/ci-caching.md](.agents/ci-caching.md) | CI build cache layout (registry-backed BuildKit cache on quay.io/go-skynet/ci-cache, per-arch keys), `DEPS_REFRESH` weekly cache-buster for unpinned Python deps, jetson wheels mirror for l4t builds (ghcr-hosted, survives pypi.jetson-ai-lab.io outages), prebuilt `base-grpc-*` images for llama.cpp variants, per-arch native + manifest-merge pattern, `setup-build-disk` `/mnt` relocation, path filter on master push, manual eviction | +| [.agents/ci-caching.md](.agents/ci-caching.md) | CI build cache layout (registry-backed BuildKit cache on quay.io/go-skynet/ci-cache, per-arch keys), `DEPS_REFRESH` weekly cache-buster for unpinned Python deps, prebuilt `base-grpc-*` images for llama.cpp variants, per-arch native + manifest-merge pattern, `setup-build-disk` `/mnt` relocation, path filter on master push, manual eviction | | [.agents/adding-backends.md](.agents/adding-backends.md) | Adding a new backend (Python, Go, or C++) — full step-by-step checklist, including importer integration (the `/import-model` dropdown is server-driven from `GET /backends/known`) | | [.agents/coding-style.md](.agents/coding-style.md) | Code style, editorconfig, logging, documentation conventions | | [.agents/llama-cpp-backend.md](.agents/llama-cpp-backend.md) | Working on the llama.cpp backend — architecture, updating, tool call parsing | diff --git a/Dockerfile b/Dockerfile index f9358a7d0..4ca9b3279 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,15 +6,7 @@ ARG UBUNTU_CODENAME=noble ARG APT_MIRROR="" ARG APT_PORTS_MIRROR="" -FROM alpine:3.22 AS ca-certificates - FROM ${BASE_IMAGE} AS requirements -COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt - -ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ - CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ - REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ - PIP_CERT=/etc/ssl/certs/ca-certificates.crt ARG APT_MIRROR ARG APT_PORTS_MIRROR @@ -23,7 +15,7 @@ ENV DEBIAN_FRONTEND=noninteractive # hwdata ships /usr/share/hwdata/pci.ids. Without it, the ghw library we use # for hardware detection cannot resolve PCI vendor IDs and fails to enumerate # GPUs at all, so the image reports "No GPU detected" (see issue #10941). -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ +RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ APT_MIRROR="${APT_MIRROR}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR}" sh /usr/local/sbin/apt-mirror && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -45,11 +37,11 @@ ARG TARGETVARIANT ENV BUILD_TYPE=${BUILD_TYPE} ARG UBUNTU_VERSION=2404 -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 mkdir -p /run/localai -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 echo "default" > /run/localai/capability +RUN mkdir -p /run/localai +RUN echo "default" > /run/localai/capability # Vulkan requirements -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 < /run/localai/capability fi EOT # https://github.com/NVIDIA/Isaac-GR00T/issues/343 -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 < /run/localai/capability || echo "not intel" +RUN expr "${BUILD_TYPE}" = intel && echo "intel" > /run/localai/capability || echo "not intel" # Cuda ENV PATH=/usr/local/cuda/bin:${PATH} @@ -214,7 +206,7 @@ ARG CMAKE_FROM_SOURCE=false ARG TARGETARCH ARG TARGETVARIANT -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 apt-get update && \ +RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ ccache \ @@ -228,7 +220,7 @@ RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mod rm -rf /var/lib/apt/lists/* # Install CMake (the version in 22.04 is too old) -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 < /etc/apt/sources.list.d/intel-graphics.list -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ +RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu ${UBUNTU_CODENAME}/lts/2350 unified" > /etc/apt/sources.list.d/intel-graphics.list +RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ APT_MIRROR="${APT_MIRROR}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR}" sh /usr/local/sbin/apt-mirror && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -307,13 +298,13 @@ ENV NVIDIA_REQUIRE_CUDA="cuda>=${CUDA_MAJOR_VERSION}.0" ENV NVIDIA_VISIBLE_DEVICES=all ENV LD_FLAGS=${LD_FLAGS} -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 echo "GO_TAGS: $GO_TAGS" && echo "TARGETARCH: $TARGETARCH" +RUN echo "GO_TAGS: $GO_TAGS" && echo "TARGETARCH: $TARGETARCH" WORKDIR /build # We need protoc installed, and the version in 22.04 is too old. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 </dev/null 2>&1; then \ echo "==> prebuilding engine for ${BACKEND} (cacheable layer)" && \ make engine; \ @@ -421,7 +418,7 @@ COPY . /LocalAI # The engine variants built above survive this COPY (they are build outputs, not # tracked files) and are newer than the pinned clone, so make treats them as up # to date and goes straight to the Go binary. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cd /LocalAI && make protogen-go && make -C /LocalAI/backend/go/${BACKEND} build +RUN cd /LocalAI && make protogen-go && make -C /LocalAI/backend/go/${BACKEND} build FROM scratch ARG BACKEND=rerankers diff --git a/backend/Dockerfile.ik-llama-cpp b/backend/Dockerfile.ik-llama-cpp index 9088d96db..9694441b0 100644 --- a/backend/Dockerfile.ik-llama-cpp +++ b/backend/Dockerfile.ik-llama-cpp @@ -24,10 +24,7 @@ ARG APT_PORTS_MIRROR="" # runs, so the result is bit-equivalent to the prebuilt-base path # (builder-prebuilt below). # ============================================================================ -FROM alpine:3.22 AS ca-certificates - FROM ${BASE_IMAGE} AS builder-fromsource -COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt ARG BUILD_TYPE ARG CUDA_MAJOR_VERSION ARG CUDA_MINOR_VERSION @@ -76,14 +73,13 @@ WORKDIR /build # Install everything via the shared script — the same one that # backend/Dockerfile.base-grpc-builder runs, so the prebuilt CI base and # this from-source path are bit-equivalent. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ +RUN --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ - --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca \ bash /usr/local/sbin/install-base-deps # Mirror builder-prebuilt: copy gRPC from /opt/grpc to /usr/local so # CMake's find_package finds it at the canonical prefix the Makefile expects. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ +RUN cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI @@ -93,13 +89,13 @@ COPY . /LocalAI # different source. # # The compile body is shared with builder-prebuilt via .docker/ik-llama-cpp-compile.sh. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/ik-llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ +RUN --mount=type=bind,source=.docker/ik-llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ --mount=type=cache,target=/root/.ccache,id=ik-llama-cpp-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ bash /usr/local/sbin/compile.sh # Copy libraries using a script to handle architecture differences -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 make -BC /LocalAI/backend/cpp/ik-llama-cpp package +RUN make -BC /LocalAI/backend/cpp/ik-llama-cpp package # ============================================================================ @@ -124,15 +120,15 @@ ARG TARGETVARIANT # The base-grpc-* image installs gRPC to /opt/grpc but doesn't copy it to # /usr/local. Mirror what the from-source path does so the compile step # can find gRPC at the canonical prefix the Makefile expects. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ +RUN cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/ik-llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ +RUN --mount=type=bind,source=.docker/ik-llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ --mount=type=cache,target=/root/.ccache,id=ik-llama-cpp-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ bash /usr/local/sbin/compile.sh -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 make -BC /LocalAI/backend/cpp/ik-llama-cpp package +RUN make -BC /LocalAI/backend/cpp/ik-llama-cpp package # ============================================================================ diff --git a/backend/Dockerfile.llama-cpp b/backend/Dockerfile.llama-cpp index 455356ffa..2f21aaa8e 100644 --- a/backend/Dockerfile.llama-cpp +++ b/backend/Dockerfile.llama-cpp @@ -24,10 +24,7 @@ ARG APT_PORTS_MIRROR="" # runs, so the result is bit-equivalent to the prebuilt-base path # (builder-prebuilt below). # ============================================================================ -FROM alpine:3.22 AS ca-certificates - FROM ${BASE_IMAGE} AS builder-fromsource -COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt ARG BUILD_TYPE ARG CUDA_MAJOR_VERSION ARG CUDA_MINOR_VERSION @@ -75,14 +72,13 @@ WORKDIR /build # Install everything via the shared script — the same one that # backend/Dockerfile.base-grpc-builder runs, so the prebuilt CI base and # this from-source path are bit-equivalent. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ +RUN --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ - --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca \ bash /usr/local/sbin/install-base-deps # Mirror builder-prebuilt: copy gRPC from /opt/grpc to /usr/local so # CMake's find_package finds it at the canonical prefix the Makefile expects. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ +RUN cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI @@ -96,13 +92,13 @@ COPY . /LocalAI # share the same cache mount id. # # The compile body is shared with builder-prebuilt via .docker/llama-cpp-compile.sh. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ +RUN --mount=type=bind,source=.docker/llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ --mount=type=cache,target=/root/.ccache,id=llama-cpp-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ bash /usr/local/sbin/compile.sh # Copy libraries using a script to handle architecture differences -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 make -BC /LocalAI/backend/cpp/llama-cpp package +RUN make -BC /LocalAI/backend/cpp/llama-cpp package # ============================================================================ @@ -134,15 +130,15 @@ ARG TARGETVARIANT # /usr/local. The variant Dockerfile's from-source path does that too; # mirror it here so the compile step can find gRPC at the canonical # prefix the Makefile expects. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ +RUN cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ +RUN --mount=type=bind,source=.docker/llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ --mount=type=cache,target=/root/.ccache,id=llama-cpp-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ bash /usr/local/sbin/compile.sh -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 make -BC /LocalAI/backend/cpp/llama-cpp package +RUN make -BC /LocalAI/backend/cpp/llama-cpp package # ============================================================================ diff --git a/backend/Dockerfile.privacy-filter b/backend/Dockerfile.privacy-filter index 81a20c5ea..97bd48380 100644 --- a/backend/Dockerfile.privacy-filter +++ b/backend/Dockerfile.privacy-filter @@ -28,10 +28,7 @@ ARG APT_PORTS_MIRROR="" # bit-equivalent to the prebuilt base. Used when BUILDER_TARGET=builder-fromsource # (the default; local `make backends/privacy-filter`). # ============================================================================ -FROM alpine:3.22 AS ca-certificates - FROM ${BASE_IMAGE} AS builder-fromsource -COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt ARG BUILD_TYPE ARG CUDA_MAJOR_VERSION ARG CUDA_MINOR_VERSION @@ -66,19 +63,17 @@ WORKDIR /build # apt deps + cmake + protoc + gRPC + conditional CUDA/Vulkan, all from the # shared script (the source of truth that base-grpc-builder also runs). -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ +RUN --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ - --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca \ bash /usr/local/sbin/install-base-deps # install-base-deps installs gRPC under /opt/grpc; copy it to /usr/local so the # backend's find_package(gRPC CONFIG) resolves it at the canonical prefix. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ +RUN cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=cache,target=/root/.ccache,id=privacy-filter-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ - sh /LocalAI/.docker/install-build-proxy-ca.sh && \ +RUN --mount=type=cache,target=/root/.ccache,id=privacy-filter-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ make -C /LocalAI/backend/cpp/privacy-filter BUILD_TYPE=${BUILD_TYPE} NATIVE=false grpc-server package # ============================================================================ @@ -96,12 +91,11 @@ ENV PATH=/usr/local/cuda/bin:${PATH} # Mirror builder-fromsource: the base-grpc image installs gRPC to /opt/grpc but # does not copy it to /usr/local. -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ +RUN cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=cache,target=/root/.ccache,id=privacy-filter-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ - sh /LocalAI/.docker/install-build-proxy-ca.sh && \ +RUN --mount=type=cache,target=/root/.ccache,id=privacy-filter-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ make -C /LocalAI/backend/cpp/privacy-filter BUILD_TYPE=${BUILD_TYPE} NATIVE=false grpc-server package # ============================================================================ diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 0f7d5af66..2522a6f56 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -1,23 +1,8 @@ ARG BASE_IMAGE=ubuntu:24.04 ARG APT_MIRROR="" ARG APT_PORTS_MIRROR="" -# CI mirror of the CUDA aarch64 wheels from pypi.jetson-ai-lab.io, kept warm -# by .github/workflows/jetson-wheels.yml so l4t builds survive the upstream -# index's recurring multi-hour outages. The default (scratch) mounts an empty -# directory, which makes installRequirements fall through to the upstream -# index unchanged — local and Jetson-native builds are unaffected. -ARG JETSON_WHEELS_IMAGE=scratch - -FROM ${JETSON_WHEELS_IMAGE} AS jetson-wheels -FROM alpine:3.22 AS ca-certificates FROM ${BASE_IMAGE} AS builder -COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt - -ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ - CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ - REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ - PIP_CERT=/etc/ssl/certs/ca-certificates.crt ARG BACKEND=rerankers ARG BUILD_TYPE ENV BUILD_TYPE=${BUILD_TYPE} @@ -33,7 +18,7 @@ ARG UBUNTU_VERSION=2404 ARG APT_MIRROR ARG APT_PORTS_MIRROR -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ +RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ APT_MIRROR="${APT_MIRROR}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR}" sh /usr/local/sbin/apt-mirror && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -55,7 +40,7 @@ RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mod apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 </dev/null || true - _JETSON_MIRROR_PID="" - _JETSON_MIRROR_URL="" - fi -} - -function _startJetsonMirror() { - local script_dir port_file port tries - # An empty dir is the JETSON_WHEELS_IMAGE=scratch default in - # Dockerfile.python: no mirror was provided, use upstream as-is. - if [ -z "$(find "${JETSON_WHEELS_DIR}" -name '*.whl' -print -quit 2>/dev/null)" ]; then - echo "jetson wheels dir ${JETSON_WHEELS_DIR} has no wheels, using upstream ${JETSON_PYPI_HOST}" - return 0 - fi - script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - port_file="$(mktemp)" - rm -f "${port_file}" - python3 "${script_dir}/pypi_mirror_server.py" --root "${JETSON_WHEELS_DIR}" --port-file "${port_file}" & - _JETSON_MIRROR_PID=$! - trap _stopJetsonMirror EXIT - tries=0 - until [ -s "${port_file}" ]; do - tries=$((tries + 1)) - if [ ${tries} -gt 50 ] || ! kill -0 "${_JETSON_MIRROR_PID}" 2>/dev/null; then - echo "WARNING: local jetson wheel mirror failed to start, using upstream ${JETSON_PYPI_HOST}" - _stopJetsonMirror - return 0 - fi - sleep 0.2 - done - port="$(cat "${port_file}")" - rm -f "${port_file}" - _JETSON_MIRROR_URL="http://127.0.0.1:${port}" - echo "serving jetson wheels from ${JETSON_WHEELS_DIR} at ${_JETSON_MIRROR_URL}" -} - # installRequirements looks for several requirements files and if they exist runs the install for them in order # # - requirements-install.txt @@ -576,31 +520,18 @@ function installRequirements() { export C_INCLUDE_PATH="${C_INCLUDE_PATH:-}:$(_portable_dir)/include/python${PYTHON_VERSION}" fi - if [ -n "${JETSON_WHEELS_DIR:-}" ] && [ -d "${JETSON_WHEELS_DIR}" ]; then - _startJetsonMirror - fi - - local installFile for reqFile in ${requirementFiles[@]}; do if [ -f "${reqFile}" ]; then - installFile="${reqFile}" - if [ -n "${_JETSON_MIRROR_URL}" ] && grep -q "${JETSON_PYPI_HOST}" "${reqFile}"; then - installFile="$(mktemp)" - sed "s,https://${JETSON_PYPI_HOST},${_JETSON_MIRROR_URL},g" "${reqFile}" > "${installFile}" - echo "rewrote ${JETSON_PYPI_HOST} in ${reqFile} to the local wheel mirror (${installFile})" - fi echo "starting requirements install for ${reqFile}" if [ "x${USE_PIP}" == "xtrue" ]; then - pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${installFile}" + pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${reqFile}" else - uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${installFile}" + uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${reqFile}" fi echo "finished requirements install for ${reqFile}" fi done - _stopJetsonMirror - runProtogen } diff --git a/backend/python/common/pypi_mirror_server.py b/backend/python/common/pypi_mirror_server.py deleted file mode 100644 index 394c3f000..000000000 --- a/backend/python/common/pypi_mirror_server.py +++ /dev/null @@ -1,226 +0,0 @@ -"""Ephemeral PEP 503 "simple" index over a local directory of wheels. - -Serves a directory tree laid out like a package index (e.g. -``/jp6/cu129/torch/torch-2.8.0-cp312-...whl``) as a standards-compliant -"simple" index on localhost, so uv/pip can resolve against it exactly as they -would against the real remote index — same per-project pages, same 404 -fall-through to PyPI for projects the mirror does not carry. - -This exists because pypi.jetson-ai-lab.io (the only source of CUDA-enabled -aarch64 torch wheels for JetPack) has a history of multi-hour 502 outages, -and an --extra-index-url that errors is fatal to the whole resolution: uv -consults every configured index for every requirement, so one 502 on any -project page kills the install even for projects hosted on PyPI. CI mirrors -the handful of jetson-only wheels into an OCI image, bind-mounts it into the -backend build, and libbackend.sh serves it with this script while rewriting -the index host in the requirements files to 127.0.0.1 (see -installRequirements in libbackend.sh). A 404 from this server is a clean -"not here" that resolvers follow up on PyPI; the upstream 502 never was. - -Standard library only — it runs inside every python backend's build -container, before any venv exists. - -Usage: - python3 pypi_mirror_server.py --root /jetson-wheels --port-file /tmp/port - -Run the tests standalone: - python3 -m unittest pypi_mirror_server_test -""" - -import argparse -import hashlib -import html -import os -import re -import sys -import threading -import urllib.parse -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -# Extensions treated as distribution files: a directory containing at least -# one of these is a project page; any other directory is a sub-index listing. -DIST_SUFFIXES = (".whl", ".tar.gz", ".zip") - -_hash_cache = {} -_hash_lock = threading.Lock() - - -def normalize(name): - """PEP 503 project-name normalization.""" - return re.sub(r"[-_.]+", "-", name).lower() - - -def _file_sha256(path): - """sha256 of a file, cached on (path, mtime, size) — wheels are large.""" - st = os.stat(path) - key = (path, st.st_mtime_ns, st.st_size) - with _hash_lock: - cached = _hash_cache.get(key) - if cached: - return cached - digest = hashlib.sha256() - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(1 << 20), b""): - digest.update(chunk) - value = digest.hexdigest() - with _hash_lock: - _hash_cache[key] = value - return value - - -def resolve_path(root, url_path): - """Map a URL path onto the tree under root, or None. - - Path segments match either exactly or via PEP 503 normalization - (resolvers request ``liquid-audio`` even if the directory on disk is - named ``liquid_audio``). Rejects any segment that would escape root. - """ - current = root - for segment in url_path.split("/"): - if segment in ("", "."): - continue - if segment == ".." or "/" in segment or "\\" in segment: - return None - candidate = os.path.join(current, segment) - if not os.path.exists(candidate): - try: - entries = os.listdir(current) - except (NotADirectoryError, FileNotFoundError): - return None - wanted = normalize(segment) - matches = [e for e in entries if normalize(e) == wanted] - if not matches: - return None - candidate = os.path.join(current, matches[0]) - current = candidate - return current - - -class SimpleIndexHandler(BaseHTTPRequestHandler): - root = None - protocol_version = "HTTP/1.1" - - def do_GET(self): - self._respond(head_only=False) - - def do_HEAD(self): - self._respond(head_only=True) - - def _respond(self, head_only): - url_path = urllib.parse.unquote(urllib.parse.urlsplit(self.path).path) - local = resolve_path(self.root, url_path) - if local is None: - self._send_error(404, "not found") - return - if os.path.isfile(local): - self._send_file(local, head_only) - return - # Relative hrefs on index pages resolve against the request URL, so - # directory URLs must end in "/" — redirect like real indexes do. - if not url_path.endswith("/"): - self.send_response(301) - self.send_header("Location", self.path + "/") - self.send_header("Content-Length", "0") - self.end_headers() - return - entries = sorted(os.listdir(local)) - files = [e for e in entries if e.endswith(DIST_SUFFIXES)] - if files: - body = self._project_page(local, files) - else: - dirs = [e for e in entries if os.path.isdir(os.path.join(local, e))] - body = self._listing_page(dirs) - self._send_html(body, head_only) - - def _project_page(self, project_dir, files): - anchors = [] - for name in files: - digest = _file_sha256(os.path.join(project_dir, name)) - anchors.append( - '%s
' - % (urllib.parse.quote(name), digest, html.escape(name)) - ) - return self._page(anchors) - - def _listing_page(self, dirs): - anchors = [ - '%s
' - % (urllib.parse.quote(normalize(d)), html.escape(normalize(d))) - for d in dirs - ] - return self._page(anchors) - - def _page(self, anchors): - return ( - "" - '' - "simple index\n" - + "\n".join(anchors) - + "\n" - ).encode() - - def _send_html(self, body, head_only): - self.send_response(200) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - if not head_only: - self.wfile.write(body) - - def _send_file(self, path, head_only): - size = os.path.getsize(path) - self.send_response(200) - self.send_header("Content-Type", "application/octet-stream") - self.send_header("Content-Length", str(size)) - self.end_headers() - if head_only: - return - with open(path, "rb") as f: - while True: - chunk = f.read(1 << 20) - if not chunk: - break - self.wfile.write(chunk) - - def _send_error(self, code, message): - body = message.encode() - self.send_response(code) - self.send_header("Content-Type", "text/plain") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format, *args): - sys.stderr.write("pypi-mirror: %s\n" % (format % args)) - - -def make_server(root, host="127.0.0.1", port=0): - handler = type("Handler", (SimpleIndexHandler,), {"root": os.path.abspath(root)}) - return ThreadingHTTPServer((host, port), handler) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", required=True, help="directory tree to serve") - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=0, help="0 picks a free port") - parser.add_argument( - "--port-file", - help="write the bound port here once listening (readiness signal)", - ) - args = parser.parse_args() - - server = make_server(args.root, args.host, args.port) - port = server.server_address[1] - if args.port_file: - # Write-then-rename so a reader never sees a partially written port. - tmp = args.port_file + ".tmp" - with open(tmp, "w") as f: - f.write(str(port)) - os.replace(tmp, args.port_file) - sys.stderr.write("pypi-mirror: serving %s on %s:%d\n" % (args.root, args.host, port)) - server.serve_forever() - - -if __name__ == "__main__": - main() diff --git a/backend/python/common/pypi_mirror_server_test.py b/backend/python/common/pypi_mirror_server_test.py deleted file mode 100644 index 5121b159d..000000000 --- a/backend/python/common/pypi_mirror_server_test.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Unit tests for the ephemeral PEP 503 index (pypi_mirror_server.py). - -Run standalone (Python standard library only, no backend venv needed): - python3 -m unittest pypi_mirror_server_test -""" - -import hashlib -import os -import shutil -import tempfile -import threading -import unittest -import urllib.error -import urllib.request - -from pypi_mirror_server import make_server, normalize, resolve_path - -WHEEL_BYTES = b"not a real wheel, but the server must serve it verbatim" - - -class TestHelpers(unittest.TestCase): - def test_normalize(self): - self.assertEqual(normalize("Liquid_Audio.Extra"), "liquid-audio-extra") - self.assertEqual(normalize("torch"), "torch") - - def test_resolve_rejects_traversal(self): - root = tempfile.mkdtemp() - try: - self.assertIsNone(resolve_path(root, "/../etc/passwd")) - self.assertIsNone(resolve_path(root, "/a/../../etc")) - finally: - shutil.rmtree(root) - - def test_resolve_normalized_segment(self): - root = tempfile.mkdtemp() - try: - os.makedirs(os.path.join(root, "jp6", "liquid_audio")) - found = resolve_path(root, "/jp6/liquid-audio/") - self.assertEqual(found, os.path.join(root, "jp6", "liquid_audio")) - finally: - shutil.rmtree(root) - - -class TestServer(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.root = tempfile.mkdtemp() - project = os.path.join(cls.root, "jp6", "cu129", "torch") - os.makedirs(project) - cls.wheel_name = "torch-2.8.0-cp312-cp312-linux_aarch64.whl" - with open(os.path.join(project, cls.wheel_name), "wb") as f: - f.write(WHEEL_BYTES) - cls.server = make_server(cls.root) - cls.base = "http://127.0.0.1:%d" % cls.server.server_address[1] - cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) - cls.thread.start() - - @classmethod - def tearDownClass(cls): - cls.server.shutdown() - cls.server.server_close() - shutil.rmtree(cls.root) - - def _get(self, path): - with urllib.request.urlopen(self.base + path) as resp: - return resp.status, resp.read() - - def test_project_page_lists_wheel_with_hash(self): - status, body = self._get("/jp6/cu129/torch/") - self.assertEqual(status, 200) - digest = hashlib.sha256(WHEEL_BYTES).hexdigest() - self.assertIn( - ('' % (self.wheel_name, digest)).encode(), body - ) - - def test_index_listing_names_projects(self): - status, body = self._get("/jp6/cu129/") - self.assertEqual(status, 200) - self.assertIn(b'torch', body) - - def test_wheel_download_is_verbatim(self): - status, body = self._get("/jp6/cu129/torch/" + self.wheel_name) - self.assertEqual(status, 200) - self.assertEqual(body, WHEEL_BYTES) - - def test_unknown_project_is_404(self): - # 404 (not 5xx) matters: resolvers treat it as "not in this index" - # and fall back to PyPI, which is the whole point of the mirror. - with self.assertRaises(urllib.error.HTTPError) as ctx: - self._get("/jp6/cu129/liquid-audio/") - self.assertEqual(ctx.exception.code, 404) - - def test_directory_without_slash_redirects(self): - status, _ = self._get("/jp6/cu129/torch") - # urllib follows the 301; landing on the page proves the redirect - self.assertEqual(status, 200) - - -if __name__ == "__main__": - unittest.main() diff --git a/cmd/build-proxy/main.go b/cmd/build-proxy/main.go deleted file mode 100644 index f9656fbf3..000000000 --- a/cmd/build-proxy/main.go +++ /dev/null @@ -1,52 +0,0 @@ -package main - -import ( - "context" - "flag" - "fmt" - "net/http" - "os" - "os/signal" - "path/filepath" - "syscall" - "time" - - "github.com/mudler/LocalAI/core/services/buildproxy" -) - -func main() { - listen := flag.String("listen", "127.0.0.1:18080", "proxy listen address") - output := flag.String("output", ".cache/build-proxy", "telemetry directory") - flag.Parse() - if err := run(*listen, *output); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func run(listen, output string) error { - recorder, err := buildproxy.NewRecorder(filepath.Join(output, "events.jsonl")) - if err != nil { - return err - } - defer func() { _ = recorder.Close() }() - proxyHandler := buildproxy.NewHandler(buildproxy.Options{Recorder: recorder}) - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { proxyHandler(w, r, r.URL.Host) }) - server, err := buildproxy.NewServer(listen, filepath.Join(output, "ca"), handler, recorder) - if err != nil { - return err - } - if err := server.Start(); err != nil { - return err - } - fmt.Printf("proxy=http://%s\nca=%s\n", server.Addr(), server.CAPath()) - stop := make(chan os.Signal, 1) - signal.Notify(stop, os.Interrupt, syscall.SIGTERM) - <-stop - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if err := server.Stop(ctx); err != nil { - return err - } - return recorder.WriteSummary(filepath.Join(output, "summary.json")) -} diff --git a/cmd/test-resources/main.go b/cmd/test-resources/main.go deleted file mode 100644 index 7e5eaad3b..000000000 --- a/cmd/test-resources/main.go +++ /dev/null @@ -1,468 +0,0 @@ -// SPDX-License-Identifier: MIT - -package main - -import ( - "archive/tar" - "crypto/sha256" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "time" - - "github.com/mudler/LocalAI/core/services/cloudproxy/mitm" - "github.com/mudler/LocalAI/internal/testresources" - "github.com/mudler/LocalAI/pkg/httpclient" -) - -func main() { - if err := run(os.Args[1:]); err != nil { - fmt.Fprintln(os.Stderr, "test-resources:", err) - os.Exit(1) - } -} - -func run(args []string) error { - if len(args) >= 6 && args[0] == "run" && args[4] == "--" { - return runOffline(args[1], args[2], args[3], args[5:]) - } - if len(args) != 4 { - return errors.New("usage: test-resources RESOURCE_SET MANIFEST_DIR CACHE_DIR | test-resources run RESOURCE_SET MANIFEST_DIR CACHE_DIR -- COMMAND") - } - target, manifestDir, cacheDir := args[1], args[2], args[3] - if args[0] == "update" { - return update(target, manifestDir, cacheDir) - } - if args[0] != "prepare" { - return errors.New("usage: test-resources RESOURCE_SET MANIFEST_DIR CACHE_DIR") - } - return prepare(target, manifestDir, cacheDir) -} - -func prepare(target, manifestDir, cacheDir string) error { - manifest, err := testresources.LoadManifest(filepath.Join(manifestDir, target+".json")) - if err != nil { - return fmt.Errorf("%w; run `make update-offline-test-cache TEST_RESOURCE_SET=%s`", err, target) - } - if manifest.Target != target { - return fmt.Errorf("manifest target %q does not match %q", manifest.Target, target) - } - lock, err := testresources.LoadLock(filepath.Join(manifestDir, "lock.json")) - if err != nil { - return err - } - locked, ok := lock.Bundles[target] - if !ok { - return fmt.Errorf("cache bundle is not locked for resource set %q; run `make update-offline-test-cache TEST_RESOURCE_SET=%s`", target, target) - } - if digest, ok := strings.CutPrefix(locked, "sha256:"); ok { - bundlePath := filepath.Join(cacheDir, "bundles", target+".tar.zst") - if _, err := os.Stat(bundlePath); errors.Is(err, os.ErrNotExist) { - bundlePath = filepath.Join(cacheDir, "bundles", target+".tar") - } - if err := testresources.RestoreBundle(cacheDir, bundlePath, digest); err != nil { - return preparationError(target, err) - } - } - materialized := filepath.Join(cacheDir, "materialized", target) - if err := os.MkdirAll(materialized, 0o755); err != nil { - return err - } - index, err := testresources.LoadHTTPIndex(cacheDir) - if err != nil { - return preparationError(target, err) - } - for _, resource := range manifest.HTTP { - _, err := testresources.VerifyBlob(cacheDir, resource.SHA256) - if err != nil { - return preparationError(target, err) - } - entry, ok := index[testresources.RequestKey(resource.Method, resource.URL, resource.Headers())] - if !ok || entry.Digest != resource.SHA256 { - return preparationError(target, fmt.Errorf("HTTP cache entry missing or mismatched: %s %s", resource.Method, resource.URL)) - } - } - for _, resource := range manifest.Files { - path, err := testresources.VerifyBlob(cacheDir, resource.SHA256) - if err != nil { - return preparationError(target, err) - } - environmentPath := path - if resource.Destination != "" { - destination := filepath.Join(materialized, resource.Destination) - if err := copyFile(path, destination); err != nil { - return err - } - environmentPath = destination - } - if resource.Environment != "" { - if err := os.Setenv(resource.Environment, environmentPath); err != nil { - return err - } - } - } - for _, resource := range manifest.Images { - path, err := testresources.VerifyBlob(cacheDir, resource.SHA256) - if err != nil { - return preparationError(target, err) - } - cmd := exec.Command("docker", "load", "--input", path) - cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("load declared image %s: %w (verify Docker is running and this user can access its socket)", resource.Reference, err) - } - } - return nil -} - -func update(target, manifestDir, cacheDir string) error { - if os.Getenv("LOCALAI_TEST_RESOURCES_ONLINE") != "1" { - return errors.New("update requires explicit online record mode: LOCALAI_TEST_RESOURCES_ONLINE=1") - } - manifest, err := testresources.LoadManifest(filepath.Join(manifestDir, target+".json")) - if err != nil { - return err - } - client := httpclient.New() - client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse } - index, err := testresources.LoadHTTPIndex(cacheDir) - if err != nil { - return err - } - for _, resource := range manifest.HTTP { - entry, err := fetchHTTPWithMirrors(client, resource, cacheDir) - if err != nil { - return err - } - index[testresources.RequestKey(resource.Method, resource.URL, resource.Headers())] = entry - } - if err := testresources.WriteHTTPIndex(cacheDir, index); err != nil { - return err - } - for _, resource := range manifest.Files { - if err := fetchWithMirrors(client, resource.URL, resource.Mirrors, resource.SHA256, cacheDir); err != nil { - return err - } - } - for i := range manifest.Images { - digest, err := pullAndPack(manifest.Images[i].Reference, cacheDir) - if err != nil { - return err - } - manifest.Images[i].SHA256 = digest - } - bundlePath := filepath.Join(cacheDir, "bundles", target+".tar.zst") - digest, err := testresources.PackBundle(cacheDir, bundlePath, manifest) - if err != nil { - return err - } - if err := testresources.WriteManifest(filepath.Join(manifestDir, target+".json"), manifest); err != nil { - return err - } - lockPath := filepath.Join(manifestDir, "lock.json") - lock, err := testresources.LoadLock(lockPath) - if err != nil { - return err - } - lock.Bundles[target] = "sha256:" + digest - return testresources.WriteLock(lockPath, lock) -} - -func fetchHTTPWithMirrors(client *http.Client, resource testresources.HTTP, cacheDir string) (testresources.HTTPEntry, error) { - urls := append([]string{resource.URL}, resource.Mirrors...) - var failures []error - for _, candidate := range urls { - for attempt := 1; attempt <= 2; attempt++ { - started := time.Now() - entry, err := fetchHTTP(client, resource, candidate, cacheDir) - fmt.Fprintf(os.Stderr, "test-resources: download %s attempt %d took %s\n", candidate, attempt, time.Since(started).Round(time.Millisecond)) - if err == nil { - return entry, nil - } - failures = append(failures, fmt.Errorf("%s attempt %d: %w", candidate, attempt, err)) - } - } - return testresources.HTTPEntry{}, resourceChangeError(resource.URL, resource.SHA256, failures) -} - -func fetchHTTP(client *http.Client, resource testresources.HTTP, sourceURL, cacheDir string) (testresources.HTTPEntry, error) { - request, err := http.NewRequest(resource.Method, sourceURL, nil) - if err != nil { - return testresources.HTTPEntry{}, err - } - request.Header = resource.Headers() - response, err := client.Do(request) - if err != nil { - return testresources.HTTPEntry{}, fmt.Errorf("fetch %s: %w", resource.URL, err) - } - defer func() { _ = response.Body.Close() }() - size, err := storeVerified(response.Body, resource.SHA256, cacheDir) - if err != nil { - return testresources.HTTPEntry{}, err - } - return testresources.HTTPEntry{Digest: resource.SHA256, Size: size, Status: response.StatusCode, Header: testresources.SanitizeHeaders(response.Header)}, nil -} - -func fetchWithMirrors(client *http.Client, primary string, mirrors []string, expected, cacheDir string) error { - urls := append([]string{primary}, mirrors...) - var failures []error - for _, candidate := range urls { - for attempt := 1; attempt <= 2; attempt++ { - started := time.Now() - err := fetch(client, candidate, expected, cacheDir) - fmt.Fprintf(os.Stderr, "test-resources: download %s attempt %d took %s\n", candidate, attempt, time.Since(started).Round(time.Millisecond)) - if err == nil { - return nil - } - failures = append(failures, fmt.Errorf("%s attempt %d: %w", candidate, attempt, err)) - } - } - return resourceChangeError(primary, expected, failures) -} - -func resourceChangeError(resourceURL, expected string, failures []error) error { - return fmt.Errorf("resource verification failed for %s (expected sha256:%s) after retrying every declared mirror: %w\nsecurity review required before changing the manifest: compare the upstream release checksum/signature and changelog, inspect redirects, and search https://github.com/advisories and https://osv.dev; a mismatch may be an upstream release, mirror corruption, or a supply-chain incident", resourceURL, expected, errors.Join(failures...)) -} - -func fetch(client *http.Client, rawURL, expected, cacheDir string) error { - request, err := http.NewRequest(http.MethodGet, rawURL, nil) - if err != nil { - return err - } - response, err := client.Do(request) - if err != nil { - return fmt.Errorf("fetch %s: %w", rawURL, err) - } - if response.StatusCode != http.StatusOK { - closeErr := response.Body.Close() - if closeErr != nil { - return errors.Join(fmt.Errorf("fetch %s: status %s", rawURL, response.Status), closeErr) - } - return fmt.Errorf("fetch %s: status %s", rawURL, response.Status) - } - _, storeErr := storeVerified(response.Body, expected, cacheDir) - return errors.Join(storeErr, response.Body.Close()) -} - -func storeVerified(reader io.Reader, expected, cacheDir string) (int64, error) { - directory := filepath.Join(cacheDir, "blobs", "sha256") - if err := os.MkdirAll(directory, 0o755); err != nil { - return 0, err - } - temporary, err := os.CreateTemp(directory, ".record-*") - if err != nil { - return 0, err - } - temporaryName := temporary.Name() - defer func() { _ = os.Remove(temporaryName) }() - hash := sha256.New() - size, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader) - closeErr := temporary.Close() - if err := errors.Join(copyErr, closeErr); err != nil { - return 0, err - } - actual := fmt.Sprintf("%x", hash.Sum(nil)) - if actual != expected { - return 0, fmt.Errorf("resource digest mismatch: expected sha256:%s, got sha256:%s", expected, actual) - } - if err := os.Rename(temporaryName, testresources.BlobPath(cacheDir, expected)); err != nil { - return 0, err - } - return size, nil -} - -func pullAndPack(reference, cacheDir string) (string, error) { - if !strings.Contains(reference, "@sha256:") { - return "", fmt.Errorf("refusing mutable image reference %s", reference) - } - if err := exec.Command("docker", "pull", reference).Run(); err != nil { - return "", fmt.Errorf("pull image %s: %w", reference, err) - } - cmd := exec.Command("docker", "save", reference) - stdout, err := cmd.StdoutPipe() - if err != nil { - return "", err - } - if err := cmd.Start(); err != nil { - return "", err - } - if err := os.MkdirAll(cacheDir, 0o755); err != nil { - return "", err - } - normalized, err := os.CreateTemp(cacheDir, ".docker-save-*.tar") - if err != nil { - return "", err - } - normalizedName := normalized.Name() - defer func() { _ = os.Remove(normalizedName) }() - normalizeErr := normalizeDockerArchive(stdout, normalized) - waitErr := cmd.Wait() - closeErr := normalized.Close() - if err := errors.Join(normalizeErr, waitErr, closeErr); err != nil { - return "", err - } - input, err := os.Open(normalizedName) - if err != nil { - return "", err - } - digest, _, storeErr := storeContentAddressed(input, cacheDir) - return digest, errors.Join(storeErr, input.Close()) -} - -func normalizeDockerArchive(reader io.Reader, writer io.Writer) error { - tr := tar.NewReader(reader) - tw := tar.NewWriter(writer) - for { - header, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return err - } - stable := *header - stable.Uid, stable.Gid = 0, 0 - stable.Uname, stable.Gname = "", "" - stable.ModTime = time.Unix(0, 0).UTC() - stable.AccessTime, stable.ChangeTime = time.Time{}, time.Time{} - stable.PAXRecords, stable.Xattrs = nil, nil - if err := tw.WriteHeader(&stable); err != nil { - return err - } - if _, err := io.Copy(tw, tr); err != nil { - return err - } - } - return tw.Close() -} - -func storeContentAddressed(reader io.Reader, cacheDir string) (string, int64, error) { - directory := filepath.Join(cacheDir, "blobs", "sha256") - if err := os.MkdirAll(directory, 0o755); err != nil { - return "", 0, err - } - temporary, err := os.CreateTemp(directory, ".record-*") - if err != nil { - return "", 0, err - } - name := temporary.Name() - defer func() { _ = os.Remove(name) }() - hash := sha256.New() - size, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader) - closeErr := temporary.Close() - if err := errors.Join(copyErr, closeErr); err != nil { - return "", 0, err - } - digest := fmt.Sprintf("%x", hash.Sum(nil)) - if err := os.Rename(name, testresources.BlobPath(cacheDir, digest)); err != nil { - return "", 0, err - } - return digest, size, nil -} - -func preparationError(target string, err error) error { - return fmt.Errorf("%w; run `make prepare-offline-test-cache TEST_RESOURCE_SET=%s` during the network-enabled preparation phase", err, target) -} - -func copyFile(source, destination string) error { - if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { - return err - } - in, err := os.Open(source) - if err != nil { - return err - } - out, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) - if err != nil { - _ = in.Close() - return err - } - _, copyErr := io.Copy(out, in) - return errors.Join(copyErr, in.Close(), out.Close()) -} - -func runOffline(target, manifestDir, cacheDir string, command []string) error { - manifest, err := testresources.LoadManifest(filepath.Join(manifestDir, target+".json")) - if err != nil { - return err - } - if err := prepare(target, manifestDir, cacheDir); err != nil { - return err - } - dockerNetwork := "" - if runtime.GOOS == "linux" && (len(manifest.Images) > 0 || target == "aio") { - dockerNetwork = fmt.Sprintf("localai-test-%d", os.Getpid()) - create := exec.Command("docker", "network", "create", "--internal", dockerNetwork) - create.Stdout, create.Stderr = io.Discard, os.Stderr - if err := create.Run(); err != nil { - return fmt.Errorf("create internal test Docker network: %w", err) - } - defer func() { _ = exec.Command("docker", "network", "rm", dockerNetwork).Run() }() - } - index, err := testresources.LoadHTTPIndex(cacheDir) - if err != nil { - return err - } - hosts := make([]string, 0, len(manifest.HTTP)) - seen := map[string]bool{} - for _, resource := range manifest.HTTP { - parsed, err := url.Parse(resource.URL) - if err != nil { - return err - } - if parsed.Hostname() != "" && !seen[parsed.Hostname()] { - hosts = append(hosts, parsed.Hostname()) - seen[parsed.Hostname()] = true - } - } - caDir := filepath.Join(cacheDir, "ca") - ca, err := mitm.LoadOrCreateCA(caDir) - if err != nil { - return err - } - server, err := mitm.NewServer(mitm.Config{ - Addr: "127.0.0.1:0", CA: ca, InterceptHosts: hosts, AllowPlainHTTP: true, InterceptAll: true, - Handler: func(w http.ResponseWriter, r *http.Request, _ string) { - key := testresources.RequestKey(r.Method, r.URL.String(), r.Header) - entry, ok := index[key] - if !ok { - http.Error(w, "undeclared test HTTP request: "+key, http.StatusGatewayTimeout) - return - } - if err := testresources.ReplayResponse(w, cacheDir, entry); err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - } - }, - }) - if err != nil { - return err - } - if err := server.Start(); err != nil { - return err - } - defer server.Stop() - proxyURL := "http://" + server.Addr() - caPath := filepath.Join(caDir, "ca.crt") - env := append(os.Environ(), - "LOCALAI_TEST_OFFLINE=1", "HTTP_PROXY="+proxyURL, "HTTPS_PROXY="+proxyURL, - "ALL_PROXY="+proxyURL, "http_proxy="+proxyURL, "https_proxy="+proxyURL, - "all_proxy="+proxyURL, "SSL_CERT_FILE="+caPath, "CURL_CA_BUNDLE="+caPath, - "REQUESTS_CA_BUNDLE="+caPath, "GIT_SSL_CAINFO="+caPath, "NODE_EXTRA_CA_CERTS="+caPath, - "NO_PROXY=localhost,127.0.0.0/8,::1,172.16.0.0/12,192.168.0.0/16", - "no_proxy=localhost,127.0.0.0/8,::1,172.16.0.0/12,192.168.0.0/16", - "TESTCONTAINERS_RYUK_DISABLED=true", - ) - if dockerNetwork != "" { - env = append(env, "LOCALAI_TEST_DOCKER_NETWORK="+dockerNetwork) - } - cmd := exec.Command(command[0], command[1:]...) - cmd.Env, cmd.Stdin, cmd.Stdout, cmd.Stderr = env, os.Stdin, os.Stdout, os.Stderr - return cmd.Run() -} diff --git a/cmd/test-resources/main_test.go b/cmd/test-resources/main_test.go deleted file mode 100644 index 8783bd20e..000000000 --- a/cmd/test-resources/main_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-License-Identifier: MIT - -package main - -import ( - "archive/tar" - "bytes" - "crypto/sha256" - "fmt" - "io" - "testing" - "time" - - "github.com/onsi/gomega" -) - -func TestNormalizeDockerArchiveIgnoresTarMetadata(t *testing.T) { - g := gomega.NewWithT(t) - first := dockerArchive(g, time.Unix(100, 0), 12, "builder") - second := dockerArchive(g, time.Unix(200, 0), 34, "runner") - - var normalizedFirst, normalizedSecond bytes.Buffer - g.Expect(normalizeDockerArchive(bytes.NewReader(first), &normalizedFirst)).To(gomega.Succeed()) - g.Expect(normalizeDockerArchive(bytes.NewReader(second), &normalizedSecond)).To(gomega.Succeed()) - firstDigest := fmt.Sprintf("%x", sha256.Sum256(normalizedFirst.Bytes())) - secondDigest := fmt.Sprintf("%x", sha256.Sum256(normalizedSecond.Bytes())) - g.Expect(secondDigest).To(gomega.Equal(firstDigest)) - - tr := tar.NewReader(bytes.NewReader(normalizedFirst.Bytes())) - header, err := tr.Next() - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(header.Uid).To(gomega.Equal(0)) - g.Expect(header.Gid).To(gomega.Equal(0)) - g.Expect(header.Uname).To(gomega.BeEmpty()) - g.Expect(header.Gname).To(gomega.BeEmpty()) - g.Expect(header.ModTime).To(gomega.Equal(time.Unix(0, 0))) - content, err := io.ReadAll(tr) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(string(content)).To(gomega.Equal("image data")) -} - -func dockerArchive(g *gomega.WithT, modTime time.Time, uid int, user string) []byte { - var archive bytes.Buffer - tw := tar.NewWriter(&archive) - content := []byte("image data") - g.Expect(tw.WriteHeader(&tar.Header{ - Name: "layer.tar", Mode: 0o644, Size: int64(len(content)), - ModTime: modTime, Uid: uid, Gid: uid, Uname: user, Gname: user, - })).To(gomega.Succeed()) - _, err := tw.Write(content) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(tw.Close()).To(gomega.Succeed()) - return archive.Bytes() -} diff --git a/core/cli/workerregistry/client.go b/core/cli/workerregistry/client.go index d4917a29a..cf46455c9 100644 --- a/core/cli/workerregistry/client.go +++ b/core/cli/workerregistry/client.go @@ -16,7 +16,6 @@ import ( "github.com/mudler/xlog" - "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/httpclient" ) @@ -110,10 +109,8 @@ func (c *RegistrationClient) Register(ctx context.Context, body map[string]any) // RegisterWithRetry retries registration with exponential backoff. func (c *RegistrationClient) RegisterWithRetry(ctx context.Context, body map[string]any, maxRetries int) (nodeID, apiToken, natsJWT, natsSeed string, err error) { - const ( - baseBackoff = 2 * time.Second - maxBackoff = 30 * time.Second - ) + backoff := 2 * time.Second + maxBackoff := 30 * time.Second for attempt := 1; attempt <= maxRetries; attempt++ { nodeID, apiToken, natsJWT, natsSeed, err = c.Register(ctx, body) @@ -123,13 +120,13 @@ func (c *RegistrationClient) RegisterWithRetry(ctx context.Context, body map[str if attempt == maxRetries { return "", "", "", "", fmt.Errorf("failed after %d attempts: %w", maxRetries, err) } - delay := backoff.Exponential(baseBackoff, maxBackoff, uint(attempt-1)) - xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", delay, "error", err) + xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", backoff, "error", err) select { case <-ctx.Done(): return "", "", "", "", ctx.Err() - case <-time.After(delay): + case <-time.After(backoff): } + backoff = min(backoff*2, maxBackoff) } return nodeID, apiToken, natsJWT, natsSeed, err } diff --git a/core/cli/workerregistry/credentials.go b/core/cli/workerregistry/credentials.go index e19bbfcee..24dd6f3c8 100644 --- a/core/cli/workerregistry/credentials.go +++ b/core/cli/workerregistry/credentials.go @@ -6,7 +6,6 @@ import ( "sync" "time" - "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/natsauth" "github.com/mudler/xlog" ) @@ -121,23 +120,23 @@ func (m *NATSCredentialManager) HasCredentials() bool { // credentials are minted. Without requireCreds it returns the first successful // response (the historical one-shot behavior, preserved for anonymous NATS). func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, error) { + backoff := m.initialBackoff var lastReason error for attempt := 1; m.maxAttempts <= 0 || attempt <= m.maxAttempts; attempt++ { - delay := backoff.Exponential(m.initialBackoff, m.maxBackoff, uint(attempt-1)) res, err := m.register(ctx) switch { case err != nil: lastReason = err - xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", delay, "error", err) + xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", backoff, "error", err) case !m.requireCreds: m.store(res) return res, nil case res.Status == statusPending: lastReason = fmt.Errorf("node %s still pending admin approval", res.ID) - xlog.Info("Node pending admin approval; waiting", "node", res.ID, "attempt", attempt, "next_retry", delay) + xlog.Info("Node pending admin approval; waiting", "node", res.ID, "attempt", attempt, "next_retry", backoff) case res.NatsJWT == "" || res.NatsUserSeed == "": lastReason = fmt.Errorf("node %s approved but NATS credentials not minted", res.ID) - xlog.Info("Node approved but NATS credentials not yet minted; waiting", "node", res.ID, "attempt", attempt, "next_retry", delay) + xlog.Info("Node approved but NATS credentials not yet minted; waiting", "node", res.ID, "attempt", attempt, "next_retry", backoff) default: m.store(res) return res, nil @@ -145,8 +144,9 @@ func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, select { case <-ctx.Done(): return nil, ctx.Err() - case <-time.After(delay): + case <-time.After(backoff): } + backoff = min(backoff*2, m.maxBackoff) } return nil, fmt.Errorf("giving up acquiring NATS credentials after %d attempts: %w", m.maxAttempts, lastReason) } diff --git a/core/config/model_config_test.go b/core/config/model_config_test.go index 5d008c43e..0abc0c7b6 100644 --- a/core/config/model_config_test.go +++ b/core/config/model_config_test.go @@ -1,6 +1,8 @@ package config import ( + "io" + "net/http" "os" "path/filepath" @@ -315,7 +317,17 @@ parameters: Expect(valid).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) - configs, err = readModelConfigsFromFile(filepath.Join("testdata", "hermes-2-pro-mistral.yaml")) + // download https://raw.githubusercontent.com/mudler/LocalAI/v2.25.0/embedded/models/hermes-2-pro-mistral.yaml + httpClient := http.Client{} + resp, err := httpClient.Get("https://raw.githubusercontent.com/mudler/LocalAI/v2.25.0/embedded/models/hermes-2-pro-mistral.yaml") + Expect(err).To(BeNil()) + defer resp.Body.Close() + tmp, err = os.CreateTemp("", "config.yaml") + Expect(err).To(BeNil()) + defer os.Remove(tmp.Name()) + _, err = io.Copy(tmp, resp.Body) + Expect(err).To(BeNil()) + configs, err = readModelConfigsFromFile(tmp.Name()) config = configs[0] Expect(err).To(BeNil()) Expect(config).ToNot(BeNil()) diff --git a/core/config/testdata/hermes-2-pro-mistral.yaml b/core/config/testdata/hermes-2-pro-mistral.yaml deleted file mode 100644 index 058e14ef3..000000000 --- a/core/config/testdata/hermes-2-pro-mistral.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: hermes-2-pro-mistral -backend: llama-cpp -context_size: 4096 -parameters: - model: hermes-2-pro-mistral.Q4_K_M.gguf -template: - chat: chatml diff --git a/core/explorer/database_test.go b/core/explorer/database_test.go index 3b05955ca..7f2cbd268 100644 --- a/core/explorer/database_test.go +++ b/core/explorer/database_test.go @@ -2,7 +2,6 @@ package explorer_test import ( "os" - "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -18,9 +17,8 @@ var _ = Describe("Database", func() { ) BeforeEach(func() { - // Keep each spec isolated: coverage runs can execute package tests in - // overlapping processes, so a repository-relative filename is racy. - dbPath = filepath.Join(GinkgoT().TempDir(), "test_db.json") + // Create a temporary file path for the database + dbPath = "test_db.json" db, err = explorer.NewDatabase(dbPath) Expect(err).To(BeNil()) }) @@ -80,7 +78,7 @@ var _ = Describe("Database", func() { Context("when loading an empty or non-existent file", func() { It("should start with an empty database", func() { - dbPath = filepath.Join(GinkgoT().TempDir(), "empty_db.json") + dbPath = "empty_db.json" db, err = explorer.NewDatabase(dbPath) Expect(err).To(BeNil()) diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index df1194b5d..1b7e059be 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -1,18 +1,12 @@ package gallery import ( - "archive/tar" - "bytes" "context" "encoding/json" "os" "path/filepath" "runtime" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" @@ -21,21 +15,9 @@ import ( "gopkg.in/yaml.v3" ) -func writeBackendImageFixture(path string) { - var layerTar bytes.Buffer - w := tar.NewWriter(&layerTar) - contents := []byte("#!/bin/sh\necho test backend\n") - Expect(w.WriteHeader(&tar.Header{Name: "run.sh", Mode: 0o755, Size: int64(len(contents))})).To(Succeed()) - _, err := w.Write(contents) - Expect(err).NotTo(HaveOccurred()) - Expect(w.Close()).To(Succeed()) - - layer, err := tarball.LayerFromReader(bytes.NewReader(layerTar.Bytes())) - Expect(err).NotTo(HaveOccurred()) - image, err := mutate.AppendLayers(empty.Image, layer) - Expect(err).NotTo(HaveOccurred()) - Expect(tarball.WriteToFile(path, name.MustParseReference("localai/backend-test:fixture"), image)).To(Succeed()) -} +const ( + testImage = "quay.io/mudler/tests:localai-backend-test" +) var _ = Describe("Runtime capability-based backend selection", func() { var tempDir string @@ -153,8 +135,6 @@ var _ = Describe("Gallery Backends", func() { galleries []config.Gallery ml *model.ModelLoader systemState *system.SystemState - testImage string - fixtureDir string ) BeforeEach(func() { @@ -162,22 +142,11 @@ var _ = Describe("Gallery Backends", func() { tempDir, err = os.MkdirTemp("", "gallery-test-*") Expect(err).NotTo(HaveOccurred()) - fixtureDir, err = os.MkdirTemp("", "backend-fixture-*") - Expect(err).NotTo(HaveOccurred()) - Expect(os.WriteFile(filepath.Join(fixtureDir, "run.sh"), []byte("#!/bin/sh\necho test backend\n"), 0o755)).To(Succeed()) - testImage = fixtureDir - - galleryPath := filepath.Join(tempDir, "backend-gallery.yaml") - galleryData, err := yaml.Marshal(GalleryBackends{ - &GalleryBackend{Metadata: Metadata{Name: "test-backend"}, URI: testImage}, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(os.WriteFile(galleryPath, galleryData, 0o644)).To(Succeed()) - + // Setup test galleries galleries = []config.Gallery{ { Name: "test-gallery", - URL: "file://" + galleryPath, + URL: "https://gist.githubusercontent.com/mudler/71d5376bc2aa168873fa519fa9f4bd56/raw/0557f9c640c159fa8e4eab29e8d98df6a3d6e80f/backend-gallery.yaml", }, } systemState, err = system.GetSystemState(system.WithBackendPath(tempDir)) @@ -186,8 +155,7 @@ var _ = Describe("Gallery Backends", func() { }) AfterEach(func() { - Expect(os.RemoveAll(tempDir)).To(Succeed()) - Expect(os.RemoveAll(fixtureDir)).To(Succeed()) + os.RemoveAll(tempDir) }) Describe("InstallBackendFromGallery", func() { @@ -203,18 +171,6 @@ var _ = Describe("Gallery Backends", func() { Expect(filepath.Join(tempDir, "test-backend", "run.sh")).To(BeARegularFile()) }) - It("should install a local OCI backend image", func() { - imagePath := filepath.Join(tempDir, "backend-image.tar") - writeBackendImageFixture(imagePath) - backend := &GalleryBackend{ - Metadata: Metadata{Name: "oci-test-backend"}, - URI: "ocifile://" + imagePath, - } - - Expect(InstallBackend(context.TODO(), systemState, ml, backend, nil, false)).To(Succeed()) - Expect(filepath.Join(tempDir, "oci-test-backend", "run.sh")).To(BeARegularFile()) - }) - It("removes files from a previous install that are absent in the new artifact", func() { // A reinstall must fully replace the installed backend, not overlay // the new artifact onto the old one: a stale library or package @@ -957,7 +913,7 @@ var _ = Describe("Gallery Backends", func() { Metadata: Metadata{ Name: "test-backend", }, - URI: testImage, + URI: "quay.io/mudler/tests:localai-backend-test", Alias: "test-alias", } @@ -987,7 +943,7 @@ var _ = Describe("Gallery Backends", func() { Metadata: Metadata{ Name: "test-backend", }, - URI: testImage, + URI: "quay.io/mudler/tests:localai-backend-test", Alias: "test-alias", } @@ -1011,7 +967,7 @@ var _ = Describe("Gallery Backends", func() { Metadata: Metadata{ Name: "test-backend", }, - URI: testImage, + URI: "quay.io/mudler/tests:localai-backend-test", Alias: "test-alias", } diff --git a/core/gallery/importers/discovery_options_test.go b/core/gallery/importers/discovery_options_test.go deleted file mode 100644 index 163caa365..000000000 --- a/core/gallery/importers/discovery_options_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: MIT - -package importers_test - -import ( - "context" - "encoding/json" - "errors" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/mudler/LocalAI/core/gallery/importers" - hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" -) - -type fixtureMetadata struct { - details *hfapi.ModelDetails - err error - calls []string -} - -func (f *fixtureMetadata) GetModelDetails(repo string) (*hfapi.ModelDetails, error) { - f.calls = append(f.calls, repo) - return f.details, f.err -} - -var _ = Describe("DiscoverModelConfigWithOptions", func() { - It("uses fixture metadata without creating a live client", func() { - metadata := &fixtureMetadata{details: &hfapi.ModelDetails{ - ModelID: "fixture/whisper", - PipelineTag: "automatic-speech-recognition", - Files: []hfapi.ModelFile{{Path: "ggml-model.bin"}}, - }} - config, err := importers.DiscoverModelConfigWithOptions(context.Background(), "hf://fixture/whisper", json.RawMessage(`{}`), importers.DiscoverOptions{HuggingFace: metadata}) - Expect(err).NotTo(HaveOccurred()) - Expect(config.Name).NotTo(BeEmpty()) - Expect(metadata.calls).To(Equal([]string{"fixture/whisper"})) - }) - - It("does not invoke metadata after cancellation", func() { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - metadata := &fixtureMetadata{err: errors.New("must not be returned")} - _, err := importers.DiscoverModelConfigWithOptions(ctx, "hf://fixture/model", json.RawMessage(`{}`), importers.DiscoverOptions{HuggingFace: metadata}) - Expect(err).To(MatchError(context.Canceled)) - Expect(metadata.calls).To(BeEmpty()) - }) -}) diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index d97c558e9..a86e86530 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -1,7 +1,6 @@ package importers import ( - "context" "encoding/json" "errors" "fmt" @@ -27,8 +26,6 @@ import ( // this sentinel so legacy callers keep working. var ErrAmbiguousImport = errors.New("importer: ambiguous — specify preferences.backend") -var newHuggingFaceMetadata = func() HuggingFaceMetadata { return hfapi.NewClient() } - // AmbiguousImportError is the concrete error DiscoverModelConfig returns when // it can't pick an importer automatically. It carries the importer-modality // key (e.g. "tts", "asr") and the list of candidate backend names so HTTP @@ -254,48 +251,15 @@ func hasYAMLExtension(uri string) bool { } func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.ModelConfig, error) { - return DiscoverModelConfigWithOptions(context.Background(), uri, preferences, DiscoverOptions{}) -} - -// HuggingFaceMetadata provides only the repository metadata needed during -// importer discovery. Tests can supply fixtures without constructing a live -// Hugging Face client. -type HuggingFaceMetadata interface { - GetModelDetails(string) (*hfapi.ModelDetails, error) -} - -// DiscoverOptions contains optional dependencies for model discovery. -type DiscoverOptions struct { - HuggingFace HuggingFaceMetadata -} - -// SetHuggingFaceMetadataFactoryForTest replaces the production metadata -// client factory and returns a restore function. It must only be called by a -// serial test-suite setup before discovery begins. -func SetHuggingFaceMetadataFactoryForTest(factory func() HuggingFaceMetadata) func() { - previous := newHuggingFaceMetadata - newHuggingFaceMetadata = factory - return func() { newHuggingFaceMetadata = previous } -} - -// DiscoverModelConfigWithOptions discovers a model using explicitly supplied -// dependencies. A nil metadata client retains the production behavior. -func DiscoverModelConfigWithOptions(ctx context.Context, uri string, preferences json.RawMessage, opts DiscoverOptions) (gallery.ModelConfig, error) { var err error var modelConfig gallery.ModelConfig - hf := opts.HuggingFace - if hf == nil { - hf = newHuggingFaceMetadata() - } + hf := hfapi.NewClient() hfrepoID := strings.ReplaceAll(uri, "huggingface://", "") hfrepoID = strings.ReplaceAll(hfrepoID, "hf://", "") hfrepoID = strings.ReplaceAll(hfrepoID, "https://huggingface.co/", "") - if err := ctx.Err(); err != nil { - return gallery.ModelConfig{}, err - } hfDetails, err := hf.GetModelDetails(hfrepoID) if err != nil { // maybe not a HF repository diff --git a/core/gallery/importers/importers_suite_test.go b/core/gallery/importers/importers_suite_test.go index a39e2a9c6..a65b8163a 100644 --- a/core/gallery/importers/importers_suite_test.go +++ b/core/gallery/importers/importers_suite_test.go @@ -1,76 +1,13 @@ package importers_test import ( - "context" - "errors" "testing" - gguf "github.com/gpustack/gguf-parser-go" - "github.com/mudler/LocalAI/core/gallery/importers" - hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -type metadataFixtures map[string]*hfapi.ModelDetails - -func (f metadataFixtures) GetModelDetails(repo string) (*hfapi.ModelDetails, error) { - details, ok := f[repo] - if !ok { - return nil, errors.New("metadata fixture not declared: " + repo) - } - return details, nil -} - -func file(repo, path, sha string) hfapi.ModelFile { - return hfapi.ModelFile{Path: path, SHA256: sha, URL: "https://huggingface.co/" + repo + "/resolve/main/" + path} -} - -var fixtures = metadataFixtures{ - "mudler/vibevoice.cpp-models": { - ModelID: "mudler/vibevoice.cpp-models", Author: "mudler", - Files: []hfapi.ModelFile{ - file("mudler/vibevoice.cpp-models", "vibevoice-realtime-Q4_K_M.gguf", "01"), - file("mudler/vibevoice.cpp-models", "vibevoice-asr-Q4_K_M.gguf", "02"), - file("mudler/vibevoice.cpp-models", "tokenizer.gguf", "03"), - file("mudler/vibevoice.cpp-models", "voice-Alice.gguf", "04"), - }, - }, - "UsefulSensors/moonshine-tiny": {ModelID: "UsefulSensors/moonshine-tiny", Author: "UsefulSensors", PipelineTag: "automatic-speech-recognition", Files: []hfapi.ModelFile{file("UsefulSensors/moonshine-tiny", "model.onnx", "05")}}, - "nvidia/parakeet-tdt-0.6b-v3": {ModelID: "nvidia/parakeet-tdt-0.6b-v3", Author: "nvidia", PipelineTag: "automatic-speech-recognition", Files: []hfapi.ModelFile{file("nvidia/parakeet-tdt-0.6b-v3", "parakeet.nemo", "06")}}, - "LiquidAI/LFM2.5-Audio-1.5B": {ModelID: "LiquidAI/LFM2.5-Audio-1.5B", Author: "LiquidAI"}, - "LiquidAI/LFM2-Audio-1.5B": {ModelID: "LiquidAI/LFM2-Audio-1.5B", Author: "LiquidAI"}, - "LiquidAI/LFM2.5-Audio-1.5B-GGUF": {ModelID: "LiquidAI/LFM2.5-Audio-1.5B-GGUF", Author: "LiquidAI", Files: []hfapi.ModelFile{file("LiquidAI/LFM2.5-Audio-1.5B-GGUF", "LFM2.5-Audio-Q4_K_M.gguf", "07")}}, - "hexgrad/Kokoro-82M": {ModelID: "hexgrad/Kokoro-82M", Author: "hexgrad", PipelineTag: "text-to-speech", Files: []hfapi.ModelFile{file("hexgrad/Kokoro-82M", "kokoro-v1_0.pth", "08")}}, - "Qwen/Qwen3-ASR-1.7B": {ModelID: "Qwen/Qwen3-ASR-1.7B", Author: "Qwen", PipelineTag: "automatic-speech-recognition"}, - "HirCoir/piper-voice-es-mx-lucas-melor": {ModelID: "HirCoir/piper-voice-es-mx-lucas-melor", Author: "HirCoir", PipelineTag: "text-to-speech", Files: []hfapi.ModelFile{file("HirCoir/piper-voice-es-mx-lucas-melor", "es_MX-lucas-medium.onnx", "09"), file("HirCoir/piper-voice-es-mx-lucas-melor", "es_MX-lucas-medium.onnx.json", "10")}}, - "h94/IP-Adapter-FaceID": {ModelID: "h94/IP-Adapter-FaceID", Author: "h94", PipelineTag: "text-to-image"}, - "LocalAI-io/whisper-large-v3-it-yodas-only-ggml": {ModelID: "LocalAI-io/whisper-large-v3-it-yodas-only-ggml", Author: "LocalAI-io", PipelineTag: "automatic-speech-recognition", Files: []hfapi.ModelFile{file("LocalAI-io/whisper-large-v3-it-yodas-only-ggml", "ggml-model-q4_0.bin", "11"), file("LocalAI-io/whisper-large-v3-it-yodas-only-ggml", "ggml-model-q5_0.bin", "12"), file("LocalAI-io/whisper-large-v3-it-yodas-only-ggml", "ggml-model-q8_0.bin", "13")}}, - "Systran/faster-whisper-large-v3": {ModelID: "Systran/faster-whisper-large-v3", Author: "Systran", PipelineTag: "automatic-speech-recognition", Files: []hfapi.ModelFile{file("Systran/faster-whisper-large-v3", "model.bin", "14"), file("Systran/faster-whisper-large-v3", "config.json", "15")}}, - "nari-labs/Dia-1.6B": {ModelID: "nari-labs/Dia-1.6B", Author: "nari-labs", PipelineTag: "text-to-speech"}, - "mudler/rfdetr-cpp-nano": {ModelID: "mudler/rfdetr-cpp-nano", Author: "mudler", PipelineTag: "object-detection", Files: []hfapi.ModelFile{file("mudler/rfdetr-cpp-nano", "rfdetr-nano-Q4_K_M.gguf", "16")}}, - "Qdrant/bm25": {ModelID: "Qdrant/bm25", Author: "Qdrant", PipelineTag: "sentence-similarity"}, - "pyannote/voice-activity-detection": {ModelID: "pyannote/voice-activity-detection", Author: "pyannote", PipelineTag: "automatic-speech-recognition"}, - "mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF": {ModelID: "mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF", Author: "mudler", Files: []hfapi.ModelFile{file("mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF", "localai-functioncall-qwen2.5-7b-v0.5-q4_k_m.gguf", "4e7b7fe1d54b881f1ef90799219dc6cc285d29db24f559c8998d1addb35713d4")}}, - "Qwen/Qwen3-VL-2B-Instruct-GGUF": {ModelID: "Qwen/Qwen3-VL-2B-Instruct-GGUF", Author: "Qwen", Files: []hfapi.ModelFile{file("Qwen/Qwen3-VL-2B-Instruct-GGUF", "Qwen3VL-2B-Instruct-Q4_K_M.gguf", "17"), file("Qwen/Qwen3-VL-2B-Instruct-GGUF", "Qwen3VL-2B-Instruct-Q8_0.gguf", "18"), file("Qwen/Qwen3-VL-2B-Instruct-GGUF", "mmproj-Qwen3VL-2B-Instruct-F16.gguf", "20"), file("Qwen/Qwen3-VL-2B-Instruct-GGUF", "mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf", "19")}}, -} - func TestImporters(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Importers test suite") } - -var _ = BeforeSuite(func() { - restoreMetadata := importers.SetHuggingFaceMetadataFactoryForTest(func() importers.HuggingFaceMetadata { return fixtures }) - restoreMTP := importers.SetMTPProbeForTest(func(context.Context, string) (*gguf.GGUFFile, error) { - return nil, errors.New("remote GGUF probing disabled in fixture-backed importer tests") - }) - restoreGenAudio := importers.SetGenAudioProbeForTest(func(context.Context, string) (*gguf.GGUFFile, error) { - return nil, errors.New("remote GGUF probing disabled in fixture-backed importer tests") - }) - DeferCleanup(func() { - restoreGenAudio() - restoreMTP() - restoreMetadata() - }) -}) diff --git a/core/gallery/importers/llama-cpp.go b/core/gallery/importers/llama-cpp.go index 15ff39d52..a1cbb6d1b 100644 --- a/core/gallery/importers/llama-cpp.go +++ b/core/gallery/importers/llama-cpp.go @@ -19,32 +19,10 @@ import ( ) var ( - _ Importer = &LlamaCPPImporter{} - _ AdditionalBackendsProvider = &LlamaCPPImporter{} - parseRemoteGGUF = func(ctx context.Context, url string) (*gguf.GGUFFile, error) { - return gguf.ParseGGUFFileRemote(ctx, url, gguf.SkipLargeMetadata()) - } - parseRemoteGenAudioGGUF = func(ctx context.Context, url string) (*gguf.GGUFFile, error) { - return gguf.ParseGGUFFileRemote(ctx, url) - } + _ Importer = &LlamaCPPImporter{} + _ AdditionalBackendsProvider = &LlamaCPPImporter{} ) -// SetMTPProbeForTest replaces the remote GGUF header reader and returns a -// restore function. It must only be called during serial suite setup. -func SetMTPProbeForTest(probe func(context.Context, string) (*gguf.GGUFFile, error)) func() { - previous := parseRemoteGGUF - parseRemoteGGUF = probe - return func() { parseRemoteGGUF = previous } -} - -// SetGenAudioProbeForTest replaces the remote projector header reader and -// returns a restore function. It must only be called during serial suite setup. -func SetGenAudioProbeForTest(probe func(context.Context, string) (*gguf.GGUFFile, error)) func() { - previous := parseRemoteGenAudioGGUF - parseRemoteGenAudioGGUF = probe - return func() { parseRemoteGenAudioGGUF = previous } -} - type LlamaCPPImporter struct{} func (i *LlamaCPPImporter) Name() string { return "llama-cpp" } @@ -437,7 +415,10 @@ func maybeApplyMTPDefaults(modelConfig *config.ModelConfig, details Details, cfg } }() - f, err := parseRemoteGGUF(ctx, probeURL) + // MTP markers are architecture scalars. Avoid allocating tokenizer and + // other large arrays from an untrusted remote header; panic recovery cannot + // contain a fatal out-of-memory condition. + f, err := gguf.ParseGGUFFileRemote(ctx, probeURL, gguf.SkipLargeMetadata()) if err != nil { xlog.Debug("[mtp-importer] failed to read remote GGUF header for MTP detection", "uri", probeURL, "error", err) return @@ -476,7 +457,7 @@ func maybeApplyTTSUsecase(modelConfig *config.ModelConfig, cfg *gallery.ModelCon } }() - f, err := parseRemoteGenAudioGGUF(ctx, probeURL) + f, err := gguf.ParseGGUFFileRemote(ctx, probeURL) if err != nil { xlog.Debug("[tts-importer] failed to read remote mmproj header for gen-audio detection", "uri", probeURL, "error", err) return diff --git a/core/gallery/request_test.go b/core/gallery/request_test.go index efd83ed70..116756965 100644 --- a/core/gallery/request_test.go +++ b/core/gallery/request_test.go @@ -6,7 +6,6 @@ import ( "net/http/httptest" . "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/pkg/downloader" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -19,8 +18,9 @@ var _ = Describe("Gallery API tests", func() { URL: "github:go-skynet/model-gallery/gpt4all-j.yaml@main", }, } - resolved := downloader.URI(req.URL).ResolveURL() - Expect(resolved).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) + e, err := GetGalleryConfigFromURL[ModelConfig](req.URL, "") + Expect(err).ToNot(HaveOccurred()) + Expect(e.Name).To(Equal("gpt4all-j")) }) }) diff --git a/core/http/app_test.go b/core/http/app_test.go index 90c60532e..36072b96e 100644 --- a/core/http/app_test.go +++ b/core/http/app_test.go @@ -297,7 +297,9 @@ func getRequest(url string, header http.Header) (error, int, []byte) { return nil, resp.StatusCode, body } -var _ = Describe("API test", Serial, func() { +const bertEmbeddingsURL = `https://gist.githubusercontent.com/mudler/0a080b166b87640e8644b09c2aee6e3b/raw/f0e8c26bb72edc16d9fbafbfd6638072126ff225/bert-embeddings-gallery.yaml` + +var _ = Describe("API test", func() { var app *echo.Echo var client *openai.Client @@ -306,7 +308,6 @@ var _ = Describe("API test", Serial, func() { var cancel context.CancelFunc var tmpdir string var modelDir string - var bertEmbeddingsURL string // localAIApp captures the Application so AfterEach can synchronously // stop the spawned gRPC backend processes. application.New cancels // them asynchronously on context cancel, which races with test-binary @@ -331,17 +332,6 @@ var _ = Describe("API test", Serial, func() { modelDir = filepath.Join(tmpdir, "models") err = os.Mkdir(modelDir, 0750) Expect(err).ToNot(HaveOccurred()) - fixtureDir := filepath.Join(modelDir, ".fixtures") - err = os.Mkdir(fixtureDir, 0750) - Expect(err).ToNot(HaveOccurred()) - galleryFixturePath := filepath.Join(fixtureDir, "bert-embeddings-gallery.yaml") - err = os.WriteFile(galleryFixturePath, []byte("name: bert\nconfig_file: |\n name: bert\n backend: embeddings\n usage: You can test this model with curl like this\n parameters:\n model: bert\n"), 0600) - Expect(err).ToNot(HaveOccurred()) - bertEmbeddingsURL = "file://" + galleryFixturePath - // Additional files are cache inputs, not behavior under test here. Seed the - // destination so model application never reaches the public network. - err = os.WriteFile(filepath.Join(modelDir, "foo.yaml"), []byte("fixture: true\n"), 0600) - Expect(err).ToNot(HaveOccurred()) c, cancel = context.WithCancel(context.Background()) @@ -521,7 +511,7 @@ var _ = Describe("API test", Serial, func() { fmt.Println(response) resp = response return response["processed"].(bool) - }, "30s", "50ms").Should(Equal(true)) + }, "360s", "10s").Should(Equal(true)) Expect(resp["message"]).ToNot(ContainSubstring("error")) dat, err := os.ReadFile(filepath.Join(modelDir, "bert2.yaml")) @@ -566,7 +556,7 @@ var _ = Describe("API test", Serial, func() { Eventually(func() bool { response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid) return response["processed"].(bool) - }, "30s", "50ms").Should(Equal(true)) + }, "360s", "10s").Should(Equal(true)) dat, err := os.ReadFile(filepath.Join(modelDir, "bert.yaml")) Expect(err).ToNot(HaveOccurred()) @@ -590,7 +580,7 @@ var _ = Describe("API test", Serial, func() { Eventually(func() bool { response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid) return response["processed"].(bool) - }, "30s", "50ms").Should(Equal(true)) + }, "360s", "10s").Should(Equal(true)) dat, err := os.ReadFile(filepath.Join(modelDir, "bert.yaml")) Expect(err).ToNot(HaveOccurred()) @@ -642,7 +632,7 @@ parameters: response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid) resp = response return response["processed"].(bool) - }, "360s", "50ms").Should(Equal(true)) + }, "360s", "10s").Should(Equal(true)) // Check that the model was imported successfully Expect(resp["message"]).ToNot(ContainSubstring("error")) @@ -713,7 +703,7 @@ parameters: response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid) resp = response return response["processed"].(bool) - }, "360s", "50ms").Should(Equal(true)) + }, "360s", "10s").Should(Equal(true)) // Check that the model was imported successfully Expect(resp["message"]).To(ContainSubstring("error")) diff --git a/core/http/endpoints/localai/import_model_test.go b/core/http/endpoints/localai/import_model_test.go index 2b2bbede2..96c20160e 100644 --- a/core/http/endpoints/localai/import_model_test.go +++ b/core/http/endpoints/localai/import_model_test.go @@ -10,22 +10,14 @@ import ( "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/config" - "github.com/mudler/LocalAI/core/gallery/importers" . "github.com/mudler/LocalAI/core/http/endpoints/localai" "github.com/mudler/LocalAI/core/services/galleryop" - hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -type ambiguityMetadata struct{} - -func (ambiguityMetadata) GetModelDetails(repo string) (*hfapi.ModelDetails, error) { - return &hfapi.ModelDetails{ModelID: repo, Author: "nari-labs", PipelineTag: "text-to-speech"}, nil -} - var _ = Describe("ImportModelURIEndpoint ambiguity handling", func() { var ( @@ -34,9 +26,6 @@ var _ = Describe("ImportModelURIEndpoint ambiguity handling", func() { ) BeforeEach(func() { - restore := importers.SetHuggingFaceMetadataFactoryForTest(func() importers.HuggingFaceMetadata { return ambiguityMetadata{} }) - DeferCleanup(restore) - var err error tempDir, err = os.MkdirTemp("", "import-model-test") Expect(err).ToNot(HaveOccurred()) diff --git a/core/http/endpoints/localai/localai_suite_test.go b/core/http/endpoints/localai/localai_suite_test.go index fd64e6dd7..ea415bf70 100644 --- a/core/http/endpoints/localai/localai_suite_test.go +++ b/core/http/endpoints/localai/localai_suite_test.go @@ -3,7 +3,6 @@ package localai_test import ( "testing" - "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -12,13 +11,3 @@ func TestLocalAIEndpoints(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "LocalAI Endpoints test suite") } - -var _ = SynchronizedBeforeSuite(func() []byte { - return []byte(testutil.StartSharedTestDB()) -}, func(endpoint []byte) { - testutil.SetSharedTestDBEndpoint(string(endpoint)) -}) - -var _ = SynchronizedAfterSuite(func() {}, func() { - testutil.StopSharedTestDB() -}) diff --git a/core/http/endpoints/ollama/helpers_internal_test.go b/core/http/endpoints/ollama/helpers_internal_test.go index 92469eb8f..cb2194f76 100644 --- a/core/http/endpoints/ollama/helpers_internal_test.go +++ b/core/http/endpoints/ollama/helpers_internal_test.go @@ -62,4 +62,4 @@ var _ = Describe("applyOllamaOptions num_ctx clamping (issue #11022)", func() { Expect(cfg.ContextSize).To(BeNil()) }) -}) +}) \ No newline at end of file diff --git a/core/http/openresponses_test.go b/core/http/openresponses_test.go index ab5dd716a..f30674362 100644 --- a/core/http/openresponses_test.go +++ b/core/http/openresponses_test.go @@ -28,7 +28,7 @@ import ( // the registered model is "Qwen3-VL-2B-Instruct-Q4_K_M", not the repo name. const testModel = "Qwen3-VL-2B-Instruct-Q4_K_M" -var _ = Describe("Open Responses API", Serial, func() { +var _ = Describe("Open Responses API", func() { var app *echo.Echo var localApp *application.Application var localModelDir string diff --git a/core/services/agentpool/services_suite_test.go b/core/services/agentpool/services_suite_test.go index ecca27ad4..7a60db0d4 100644 --- a/core/services/agentpool/services_suite_test.go +++ b/core/services/agentpool/services_suite_test.go @@ -3,7 +3,6 @@ package agentpool_test import ( "testing" - "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -12,13 +11,3 @@ func TestServices(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "LocalAI services test") } - -var _ = SynchronizedBeforeSuite(func() []byte { - return []byte(testutil.StartSharedTestDB()) -}, func(endpoint []byte) { - testutil.SetSharedTestDBEndpoint(string(endpoint)) -}) - -var _ = SynchronizedAfterSuite(func() {}, func() { - testutil.StopSharedTestDB() -}) diff --git a/core/services/agents/agents_suite_test.go b/core/services/agents/agents_suite_test.go index 29f76733c..6cc46193b 100644 --- a/core/services/agents/agents_suite_test.go +++ b/core/services/agents/agents_suite_test.go @@ -3,7 +3,6 @@ package agents import ( "testing" - "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -12,13 +11,3 @@ func TestAgents(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Agents test suite") } - -var _ = SynchronizedBeforeSuite(func() []byte { - return []byte(testutil.StartSharedTestDB()) -}, func(endpoint []byte) { - testutil.SetSharedTestDBEndpoint(string(endpoint)) -}) - -var _ = SynchronizedAfterSuite(func() {}, func() { - testutil.StopSharedTestDB() -}) diff --git a/core/services/buildproxy/buildproxy_suite_test.go b/core/services/buildproxy/buildproxy_suite_test.go deleted file mode 100644 index 02fa71876..000000000 --- a/core/services/buildproxy/buildproxy_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package buildproxy_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestBuildProxy(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Build proxy test suite") -} diff --git a/core/services/buildproxy/proxy.go b/core/services/buildproxy/proxy.go deleted file mode 100644 index 39a0cd8ee..000000000 --- a/core/services/buildproxy/proxy.go +++ /dev/null @@ -1,280 +0,0 @@ -// Package buildproxy provides conservative retrying and traffic telemetry for -// CI build downloads. -package buildproxy - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - "time" -) - -type Event struct { - Time time.Time `json:"time"` - Host string `json:"host"` - Method string `json:"method"` - Path string `json:"path,omitempty"` - Status int `json:"status,omitempty"` - Attempts int `json:"attempts"` - BytesSent int64 `json:"bytes_sent,omitempty"` - BytesRead int64 `json:"bytes_read,omitempty"` - Intercepted bool `json:"intercepted,omitempty"` - Error string `json:"error,omitempty"` -} - -type SummaryRow struct { - Host string `json:"host"` - Method string `json:"method"` - Requests int64 `json:"requests"` - Retries int64 `json:"retries"` - BytesSent int64 `json:"bytes_sent"` - BytesRead int64 `json:"bytes_read"` - Errors int64 `json:"errors"` -} - -type Recorder struct { - mu sync.Mutex - file *os.File - rows map[string]*SummaryRow -} - -func NewRecorder(eventsPath string) (*Recorder, error) { - if err := os.MkdirAll(filepath.Dir(eventsPath), 0o755); err != nil { - return nil, err - } - f, err := os.OpenFile(eventsPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) - if err != nil { - return nil, err - } - return &Recorder{file: f, rows: map[string]*SummaryRow{}}, nil -} - -func (r *Recorder) Record(event Event) { - r.mu.Lock() - defer r.mu.Unlock() - if event.Time.IsZero() { - event.Time = time.Now().UTC() - } - _ = json.NewEncoder(r.file).Encode(event) - key := event.Host + "\x00" + event.Method - row := r.rows[key] - if row == nil { - row = &SummaryRow{Host: event.Host, Method: event.Method} - r.rows[key] = row - } - row.Requests++ - if event.Attempts > 1 { - row.Retries += int64(event.Attempts - 1) - } - row.BytesSent += event.BytesSent - row.BytesRead += event.BytesRead - if event.Error != "" || event.Status >= 400 { - row.Errors++ - } -} - -func (r *Recorder) WriteSummary(path string) error { - r.mu.Lock() - defer r.mu.Unlock() - rows := make([]SummaryRow, 0, len(r.rows)) - for _, row := range r.rows { - rows = append(rows, *row) - } - b, err := json.MarshalIndent(rows, "", " ") - if err != nil { - return err - } - return os.WriteFile(path, append(b, '\n'), 0o644) -} - -func (r *Recorder) Close() error { return r.file.Close() } - -type Options struct { - Transport http.RoundTripper - Recorder *Recorder - MaxAttempts int - SpoolDir string - BaseDelay time.Duration - MaxDelay time.Duration -} - -func NewHandler(opts Options) func(http.ResponseWriter, *http.Request, string) { - transport := opts.Transport - if transport == nil { - transport = http.DefaultTransport - } - if opts.MaxAttempts < 1 { - opts.MaxAttempts = 3 - } - if opts.BaseDelay <= 0 { - opts.BaseDelay = 100 * time.Millisecond - } - if opts.MaxDelay <= 0 { - opts.MaxDelay = 500 * time.Millisecond - } - return func(w http.ResponseWriter, request *http.Request, host string) { - event := Event{Host: hostname(host), Method: request.Method, Path: request.URL.EscapedPath(), Intercepted: true} - defer func() { opts.Recorder.Record(event) }() - if request.Body != nil { - defer func() { _ = request.Body.Close() }() - } - event.BytesSent = max(request.ContentLength, 0) - attempts := 1 - if request.Method == http.MethodGet || request.Method == http.MethodHead { - attempts = opts.MaxAttempts - } - for attempt := 1; attempt <= attempts; attempt++ { - event.Attempts = attempt - event.Error = "" - resp, path, size, err := fetch(request.Context(), transport, request, host, opts.SpoolDir) - if err == nil && !retryStatus(resp.StatusCode) { - event.Status, event.BytesRead = resp.StatusCode, size - copyResponse(w, resp, path, request.Method) - return - } - if resp != nil { - event.Status = resp.StatusCode - } - if path != "" { - _ = os.Remove(path) - } - if err != nil { - event.Error = err.Error() - } - if attempt == attempts { - break - } - if err := sleep(request.Context(), delay(opts.BaseDelay, opts.MaxDelay, attempt)); err != nil { - event.Error = err.Error() - break - } - } - http.Error(w, "build proxy: upstream request failed", http.StatusBadGateway) - } -} - -func fetch(ctx context.Context, transport http.RoundTripper, original *http.Request, host, spoolDir string) (*http.Response, string, int64, error) { - u := *original.URL - u.Scheme = original.URL.Scheme - if u.Scheme == "" { - u.Scheme = "https" - } - u.Host = host - req, err := http.NewRequestWithContext(ctx, original.Method, u.String(), original.Body) - if err != nil { - return nil, "", 0, err - } - req.Header = cloneHeaders(original.Header) - resp, err := transport.RoundTrip(req) - if err != nil { - return nil, "", 0, err - } - file, err := os.CreateTemp(spoolDir, "localai-build-proxy-*") - if err != nil { - _ = resp.Body.Close() - return resp, "", 0, err - } - path := file.Name() - size, copyErr := io.Copy(file, resp.Body) - closeErr := errors.Join(resp.Body.Close(), file.Close()) - if copyErr == nil { - copyErr = closeErr - } - if copyErr == nil && original.Method != http.MethodHead && resp.ContentLength >= 0 && size != resp.ContentLength { - copyErr = fmt.Errorf("short response: got %d bytes, expected %d", size, resp.ContentLength) - } - return resp, path, size, copyErr -} - -func copyResponse(w http.ResponseWriter, resp *http.Response, path, method string) { - defer func() { _ = os.Remove(path) }() - for key, values := range resp.Header { - if hopHeader(key) || strings.EqualFold(key, "Content-Length") { - continue - } - for _, value := range values { - w.Header().Add(key, value) - } - } - if method == http.MethodHead && resp.ContentLength >= 0 { - w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10)) - } else if info, err := os.Stat(path); err == nil { - w.Header().Set("Content-Length", strconv.FormatInt(info.Size(), 10)) - } - w.WriteHeader(resp.StatusCode) - file, err := os.Open(path) - if err == nil { - defer func() { _ = file.Close() }() - _, _ = io.Copy(w, file) - } -} - -func retryStatus(status int) bool { - switch status { - case 408, 429, 500, 502, 503, 504: - return true - default: - return false - } -} - -func delay(base, limit time.Duration, attempt int) time.Duration { - d := base - for i := 1; i < attempt && d < limit; i++ { - if d > limit/2 { - return limit - } - d *= 2 - } - if d > limit { - return limit - } - return d -} - -func sleep(ctx context.Context, d time.Duration) error { - timer := time.NewTimer(d) - defer timer.Stop() - select { - case <-timer.C: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -func hostname(host string) string { - if u, err := url.Parse("//" + host); err == nil && u.Hostname() != "" { - return strings.ToLower(u.Hostname()) - } - return strings.ToLower(host) -} - -func cloneHeaders(in http.Header) http.Header { - out := make(http.Header, len(in)) - for key, values := range in { - if hopHeader(key) || strings.EqualFold(key, "Proxy-Authorization") { - continue - } - out[key] = append([]string(nil), values...) - } - return out -} - -func hopHeader(name string) bool { - switch http.CanonicalHeaderKey(name) { - case "Connection", "Proxy-Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "Te", "Trailer", "Transfer-Encoding", "Upgrade": - return true - default: - return false - } -} diff --git a/core/services/buildproxy/proxy_test.go b/core/services/buildproxy/proxy_test.go deleted file mode 100644 index 378b76d55..000000000 --- a/core/services/buildproxy/proxy_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package buildproxy_test - -import ( - "io" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "time" - - "github.com/mudler/LocalAI/core/services/buildproxy" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -type roundTripFunc func(*http.Request) (*http.Response, error) - -func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } - -var _ = Describe("Handler", func() { - It("retries an idempotent transient response and records bytes", func() { - dir := GinkgoT().TempDir() - recorder, err := buildproxy.NewRecorder(dir + "/events.jsonl") - Expect(err).NotTo(HaveOccurred()) - defer func() { _ = recorder.Close() }() - var calls atomic.Int32 - transport := roundTripFunc(func(*http.Request) (*http.Response, error) { - status, body := http.StatusServiceUnavailable, "retry" - if calls.Add(1) == 2 { - status, body = http.StatusOK, "complete" - } - return &http.Response{StatusCode: status, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body)), ContentLength: int64(len(body))}, nil - }) - handler := buildproxy.NewHandler(buildproxy.Options{Transport: transport, Recorder: recorder, SpoolDir: dir, BaseDelay: time.Nanosecond}) - req := httptest.NewRequest(http.MethodGet, "https://example.test/archive", nil) - response := httptest.NewRecorder() - handler(response, req, "example.test") - Expect(response.Code).To(Equal(http.StatusOK)) - Expect(response.Body.String()).To(Equal("complete")) - Expect(calls.Load()).To(Equal(int32(2))) - }) - - It("does not retry a mutating request", func() { - dir := GinkgoT().TempDir() - recorder, err := buildproxy.NewRecorder(dir + "/events.jsonl") - Expect(err).NotTo(HaveOccurred()) - defer func() { _ = recorder.Close() }() - var calls atomic.Int32 - transport := roundTripFunc(func(*http.Request) (*http.Response, error) { - calls.Add(1) - return &http.Response{StatusCode: http.StatusServiceUnavailable, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("no")), ContentLength: 2}, nil - }) - handler := buildproxy.NewHandler(buildproxy.Options{Transport: transport, Recorder: recorder, SpoolDir: dir}) - handler(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "https://example.test/token", strings.NewReader("secret")), "example.test") - Expect(calls.Load()).To(Equal(int32(1))) - }) - - It("preserves HEAD metadata without expecting a response body", func() { - dir := GinkgoT().TempDir() - recorder, err := buildproxy.NewRecorder(dir + "/events.jsonl") - Expect(err).NotTo(HaveOccurred()) - defer func() { _ = recorder.Close() }() - var calls atomic.Int32 - transport := roundTripFunc(func(*http.Request) (*http.Response, error) { - calls.Add(1) - return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: http.NoBody, ContentLength: 1234}, nil - }) - handler := buildproxy.NewHandler(buildproxy.Options{Transport: transport, Recorder: recorder, SpoolDir: dir}) - response := httptest.NewRecorder() - handler(response, httptest.NewRequest(http.MethodHead, "https://example.test/blob", nil), "example.test") - Expect(calls.Load()).To(Equal(int32(1))) - Expect(response.Header().Get("Content-Length")).To(Equal("1234")) - }) -}) diff --git a/core/services/buildproxy/server.go b/core/services/buildproxy/server.go deleted file mode 100644 index 8d7e3e3bb..000000000 --- a/core/services/buildproxy/server.go +++ /dev/null @@ -1,233 +0,0 @@ -package buildproxy - -import ( - "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "errors" - "fmt" - "io" - "math/big" - "net" - "net/http" - "os" - "path/filepath" - "strings" - "sync" - "time" -) - -type certificateAuthority struct { - cert *x509.Certificate - key *ecdsa.PrivateKey - mu sync.Mutex - leaves map[string]*tls.Certificate -} - -type Server struct { - server *http.Server - listener net.Listener - handler http.Handler - recorder *Recorder - ca *certificateAuthority - caPath string - wg sync.WaitGroup -} - -func NewServer(address, caDir string, handler http.Handler, recorder *Recorder) (*Server, error) { - ca, caPath, err := createCA(caDir) - if err != nil { - return nil, err - } - s := &Server{handler: handler, recorder: recorder, ca: ca, caPath: caPath} - s.server = &http.Server{Addr: address, Handler: http.HandlerFunc(s.serveHTTP), ReadHeaderTimeout: 30 * time.Second} - return s, nil -} - -func (s *Server) CAPath() string { return s.caPath } -func (s *Server) Start() error { - ln, err := net.Listen("tcp", s.server.Addr) - if err != nil { - return err - } - s.listener = ln - s.wg.Add(1) - go func() { - defer s.wg.Done() - if err := s.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { - s.recorder.Record(Event{Method: "PROXY", Attempts: 1, Error: err.Error()}) - } - }() - return nil -} -func (s *Server) Addr() string { return s.listener.Addr().String() } -func (s *Server) Stop(ctx context.Context) error { - err := s.server.Shutdown(ctx) - s.wg.Wait() - return err -} - -func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodConnect { - // Some minimal clients (notably BusyBox wget) send an absolute HTTPS - // request to an HTTP forward proxy instead of opening CONNECT. The - // resource hop remains TLS and is handled by the same verified upstream - // transport; only absolute http:// resource URLs are forbidden. - if r.URL != nil && r.URL.IsAbs() && r.URL.Scheme == "https" { - // BusyBox closes its request side after writing the absolute-form - // request. Detach that connection cancellation while the proxy - // completes and verifies the upstream response. - s.handler.ServeHTTP(w, r.Clone(context.WithoutCancel(r.Context()))) - return - } - s.recorder.Record(Event{Host: hostname(r.Host), Method: r.Method, Path: r.URL.EscapedPath(), Attempts: 1, Error: "plain HTTP is forbidden"}) - http.Error(w, "build proxy: plain HTTP is forbidden", http.StatusUpgradeRequired) - return - } - s.intercept(w, r) -} - -func (s *Server) intercept(w http.ResponseWriter, r *http.Request) { - host := hostname(r.Host) - leaf, err := s.ca.leaf(host) - if err != nil { - http.Error(w, err.Error(), 500) - return - } - h, ok := w.(http.Hijacker) - if !ok { - http.Error(w, "hijacking unavailable", 500) - return - } - conn, _, err := h.Hijack() - if err != nil { - return - } - defer func() { _ = conn.Close() }() - if _, err = io.WriteString(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { - return - } - tlsConn := tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{*leaf}, NextProtos: []string{"http/1.1"}}) - if err = tlsConn.SetDeadline(time.Now().Add(30 * time.Second)); err != nil { - return - } - if err = tlsConn.Handshake(); err != nil { - s.recorder.Record(Event{Host: host, Method: "CONNECT", Attempts: 1, Error: err.Error()}) - return - } - _ = tlsConn.SetDeadline(time.Time{}) - ln := &singleListener{conn: tlsConn, done: make(chan struct{})} - inner := &http.Server{Handler: http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - req.URL.Scheme = "https" - req.URL.Host = r.Host - s.handler.ServeHTTP(rw, req) - })} - _ = inner.Serve(ln) -} - -type singleListener struct { - conn net.Conn - once sync.Once - done chan struct{} -} - -func (l *singleListener) Accept() (net.Conn, error) { - var c net.Conn - l.once.Do(func() { c = &signalConn{Conn: l.conn, done: l.done} }) - if c != nil { - return c, nil - } - <-l.done - return nil, net.ErrClosed -} -func (l *singleListener) Close() error { return nil } -func (l *singleListener) Addr() net.Addr { return l.conn.LocalAddr() } - -type signalConn struct { - net.Conn - done chan struct{} - once sync.Once -} - -func (c *signalConn) Close() error { - err := c.Conn.Close() - c.once.Do(func() { close(c.done) }) - return err -} - -func createCA(dir string) (*certificateAuthority, string, error) { - if err := os.MkdirAll(dir, 0700); err != nil { - return nil, "", err - } - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return nil, "", err - } - serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - return nil, "", err - } - now := time.Now() - t := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: "LocalAI CI Build Proxy"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(24 * time.Hour), KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, BasicConstraintsValid: true, IsCA: true} - der, err := x509.CreateCertificate(rand.Reader, t, t, &key.PublicKey, key) - if err != nil { - return nil, "", err - } - cert, err := x509.ParseCertificate(der) - if err != nil { - return nil, "", err - } - path := filepath.Join(dir, "ca.crt") - if err = os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0644); err != nil { - return nil, "", err - } - return &certificateAuthority{cert: cert, key: key, leaves: map[string]*tls.Certificate{}}, path, nil -} -func (c *certificateAuthority) leaf(host string) (*tls.Certificate, error) { - c.mu.Lock() - defer c.mu.Unlock() - if v := c.leaves[host]; v != nil { - return v, nil - } - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return nil, err - } - serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - return nil, err - } - now := time.Now() - t := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: host}, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(24 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}} - if ip := net.ParseIP(host); ip != nil { - t.IPAddresses = []net.IP{ip} - } else { - t.DNSNames = []string{host} - } - der, err := x509.CreateCertificate(rand.Reader, t, c.cert, &key.PublicKey, c.key) - if err != nil { - return nil, err - } - keyDER, err := x509.MarshalECPrivateKey(key) - if err != nil { - return nil, err - } - pair, err := tls.X509KeyPair(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})) - if err != nil { - return nil, err - } - c.leaves[host] = &pair - return &pair, nil -} - -func ParseListenAddress(address string) (string, error) { - if strings.TrimSpace(address) == "" { - return "", fmt.Errorf("listen address is empty") - } - return address, nil -} diff --git a/core/services/buildproxy/server_test.go b/core/services/buildproxy/server_test.go deleted file mode 100644 index 5edec8335..000000000 --- a/core/services/buildproxy/server_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package buildproxy - -import ( - "crypto/x509" - "encoding/pem" - "net/http" - "net/http/httptest" - "os" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Interception certificates", func() { - It("issues host certificates trusted by the generated CA", func() { - ca, path, err := createCA(GinkgoT().TempDir()) - Expect(err).NotTo(HaveOccurred()) - leaf, err := ca.leaf("registry.example.test") - Expect(err).NotTo(HaveOccurred()) - - caPEM, err := os.ReadFile(path) - Expect(err).NotTo(HaveOccurred()) - block, _ := pem.Decode(caPEM) - Expect(block).NotTo(BeNil()) - root, err := x509.ParseCertificate(block.Bytes) - Expect(err).NotTo(HaveOccurred()) - roots := x509.NewCertPool() - roots.AddCert(root) - certificate, err := x509.ParseCertificate(leaf.Certificate[0]) - Expect(err).NotTo(HaveOccurred()) - _, err = certificate.Verify(x509.VerifyOptions{DNSName: "registry.example.test", Roots: roots}) - Expect(err).NotTo(HaveOccurred()) - }) - - It("rejects plain HTTP", func() { - dir := GinkgoT().TempDir() - recorder, err := NewRecorder(dir + "/events.jsonl") - Expect(err).NotTo(HaveOccurred()) - defer func() { _ = recorder.Close() }() - server, err := NewServer("127.0.0.1:0", dir+"/ca", http.NotFoundHandler(), recorder) - Expect(err).NotTo(HaveOccurred()) - response := httptest.NewRecorder() - server.serveHTTP(response, httptest.NewRequest(http.MethodGet, "http://example.test/file", nil)) - Expect(response.Code).To(Equal(http.StatusUpgradeRequired)) - }) -}) diff --git a/core/services/cloudproxy/mitm/proxy.go b/core/services/cloudproxy/mitm/proxy.go index 4ccd744f4..79f49aa64 100644 --- a/core/services/cloudproxy/mitm/proxy.go +++ b/core/services/cloudproxy/mitm/proxy.go @@ -23,17 +23,15 @@ import ( // in its intercept allowlist; non-allowlisted hosts get a plain // TCP CONNECT tunnel. type Server struct { - addr string - ca *CA - interceptHosts map[string]bool - handler InterceptHandler - connectTimeout time.Duration - dialTimeout time.Duration - upstreamTLS *tls.Config - events pii.EventStore - eventSeq atomic.Uint64 - allowPlainHTTP bool - interceptAll bool + addr string + ca *CA + interceptHosts map[string]bool + handler InterceptHandler + connectTimeout time.Duration + dialTimeout time.Duration + upstreamTLS *tls.Config + events pii.EventStore + eventSeq atomic.Uint64 listener net.Listener srv *http.Server @@ -53,12 +51,6 @@ type Config struct { CA *CA InterceptHosts []string Handler InterceptHandler - // AllowPlainHTTP is used by the deterministic test-resource proxy. - // Production listeners leave it false and continue to require CONNECT. - AllowPlainHTTP bool - // InterceptAll prevents undeclared HTTPS hosts from being tunnelled by - // strict test-resource replay. Production listeners use the host allowlist. - InterceptAll bool // EventStore optionally receives a proxy_connect event for every // CONNECT, recording the destination host and whether the proxy // intercepted or tunneled it. nil disables connect-event recording. @@ -81,8 +73,6 @@ func NewServer(cfg Config) (*Server, error) { ca: cfg.CA, interceptHosts: hosts, handler: cfg.Handler, - allowPlainHTTP: cfg.AllowPlainHTTP, - interceptAll: cfg.InterceptAll, connectTimeout: 30 * time.Second, dialTimeout: 15 * time.Second, upstreamTLS: &tls.Config{NextProtos: []string{"http/1.1"}}, @@ -136,10 +126,6 @@ func (s *Server) Stop() { func (s *Server) handle(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodConnect { - if s.allowPlainHTTP && r.URL != nil && r.URL.IsAbs() { - s.handler(w, r, r.URL.Host) - return - } http.Error(w, "this proxy only supports HTTPS via CONNECT", http.StatusMethodNotAllowed) return } @@ -182,9 +168,6 @@ func (s *Server) recordConnectEvent(host string, intercepted bool) { // shouldIntercept reports whether host is in the allowlist. An // empty allowlist tunnels everything. func (s *Server) shouldIntercept(host string) bool { - if s.interceptAll { - return true - } if len(s.interceptHosts) == 0 { return false } diff --git a/core/services/jobs/jobs_suite_test.go b/core/services/jobs/jobs_suite_test.go index c183767e8..957e5ead7 100644 --- a/core/services/jobs/jobs_suite_test.go +++ b/core/services/jobs/jobs_suite_test.go @@ -3,7 +3,6 @@ package jobs import ( "testing" - "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -12,13 +11,3 @@ func TestJobs(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Jobs test suite") } - -var _ = SynchronizedBeforeSuite(func() []byte { - return []byte(testutil.StartSharedTestDB()) -}, func(endpoint []byte) { - testutil.SetSharedTestDBEndpoint(string(endpoint)) -}) - -var _ = SynchronizedAfterSuite(func() {}, func() { - testutil.StopSharedTestDB() -}) diff --git a/core/services/nodes/file_stager_http.go b/core/services/nodes/file_stager_http.go index 62ffc10a7..79047aad6 100644 --- a/core/services/nodes/file_stager_http.go +++ b/core/services/nodes/file_stager_http.go @@ -21,7 +21,6 @@ import ( "github.com/mudler/xlog" "github.com/mudler/LocalAI/core/services/storage" - "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/httpclient" ) @@ -221,7 +220,15 @@ func nextBackoff(attempt int) time.Duration { base = 1 * time.Second ceiling = 30 * time.Second ) - return backoff.Exponential(base, ceiling, uint(attempt-2)) + shift := uint(attempt - 2) + if shift > 30 { + shift = 30 // saturate before time.Duration overflows + } + b := base << shift + if b > ceiling || b < 0 { + b = ceiling + } + return b } // resumeOffset asks the server (via HEAD) how many bytes of the current upload diff --git a/core/services/nodes/nodes_suite_test.go b/core/services/nodes/nodes_suite_test.go index 56cf49a68..a6a24852b 100644 --- a/core/services/nodes/nodes_suite_test.go +++ b/core/services/nodes/nodes_suite_test.go @@ -3,7 +3,6 @@ package nodes import ( "testing" - "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -12,13 +11,3 @@ func TestNodes(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Nodes test suite") } - -var _ = SynchronizedBeforeSuite(func() []byte { - return []byte(testutil.StartSharedTestDB()) -}, func(endpoint []byte) { - testutil.SetSharedTestDBEndpoint(string(endpoint)) -}) - -var _ = SynchronizedAfterSuite(func() {}, func() { - testutil.StopSharedTestDB() -}) diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index cccca7e43..1a59e361f 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -9,7 +9,6 @@ import ( "github.com/google/uuid" "github.com/mudler/LocalAI/core/services/advisorylock" - "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/vrambudget" "github.com/mudler/xlog" @@ -2201,15 +2200,20 @@ func (r *NodeRegistry) RecordPendingBackendOpInFlight(ctx context.Context, id ui // backoffForAttempt is exponential from 30s doubling up to a 15m cap. The // reconciler tick is 30s so anything shorter would just re-fire immediately. func backoffForAttempt(attempts int) time.Duration { - const ( - base = 30 * time.Second - maximum = 15 * time.Minute - ) - exponent := 0 - if attempts > 1 { - exponent = attempts - 1 + const cap = 15 * time.Minute + base := 30 * time.Second + shift := attempts - 1 + if shift < 0 { + shift = 0 } - return backoff.Exponential(base, maximum, uint(exponent)) + if shift > 10 { // 2^10 * 30s already exceeds the cap + shift = 10 + } + d := base << shift + if d > cap { + return cap + } + return d } // CountPendingBackendOpsByBackend returns a map of backend name to the count diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index c5b7597fd..80e511201 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -2,16 +2,11 @@ package testutil import ( "context" - "fmt" "runtime" - "sync" - "sync/atomic" "time" - "github.com/mudler/LocalAI/internal/testfixtures" "github.com/testcontainers/testcontainers-go" tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" - tcnetwork "github.com/testcontainers/testcontainers-go/network" "github.com/testcontainers/testcontainers-go/wait" "gorm.io/driver/postgres" "gorm.io/gorm" @@ -21,110 +16,24 @@ import ( . "github.com/onsi/gomega" ) -var ( - sharedDBMu sync.Mutex - sharedDBContainer testcontainers.Container - sharedDBEndpoint string - sharedDBSequence atomic.Uint64 -) - -// StartSharedTestDB starts one PostgreSQL container and returns its endpoint. -// Pass that endpoint to SetSharedTestDBEndpoint in every parallel test process. -func StartSharedTestDB() string { - if runtime.GOOS == "darwin" { - return "" - } - sharedDBMu.Lock() - defer sharedDBMu.Unlock() - if sharedDBContainer != nil { - return sharedDBEndpoint - } - - container, endpoint := startTestDBContainer() - sharedDBContainer = container - sharedDBEndpoint = endpoint - return endpoint -} - -// SetSharedTestDBEndpoint attaches this test process to the suite database. -func SetSharedTestDBEndpoint(endpoint string) { - sharedDBMu.Lock() - defer sharedDBMu.Unlock() - sharedDBEndpoint = endpoint -} - -// StopSharedTestDB terminates the process-scoped PostgreSQL fixture. -func StopSharedTestDB() { - sharedDBMu.Lock() - defer sharedDBMu.Unlock() - if sharedDBContainer == nil { - return - } - Expect(sharedDBContainer.Terminate(context.Background())).To(Succeed()) - sharedDBContainer = nil - sharedDBEndpoint = "" -} - -// SetupTestDB returns an isolated PostgreSQL database fixture. Suites that call -// StartSharedTestDB get a fresh schema; other suites retain a fresh container. +// SetupTestDB creates a fresh PostgreSQL 16 container and returns a gorm.DB. +// The container is cleaned up via DeferCleanup when the test completes. func SetupTestDB() *gorm.DB { if runtime.GOOS == "darwin" { Skip("testcontainers requires Docker, not available on macOS CI") } - - sharedDBMu.Lock() - endpoint := sharedDBEndpoint - sharedDBMu.Unlock() - if endpoint != "" { - return setupIsolatedSchema(endpoint) - } - - pgC, endpoint := startTestDBContainer() - DeferCleanup(func() { _ = pgC.Terminate(context.Background()) }) - return openTestDB(endpoint, "") -} - -func startTestDBContainer() (testcontainers.Container, string) { ctx := context.Background() - Expect(testfixtures.RequireImage(ctx, testfixtures.Postgres16, "default")).To(Succeed()) - testNetwork, err := testfixtures.DockerNetwork() - Expect(err).NotTo(HaveOccurred()) - pgC, err := tcpostgres.Run(ctx, testfixtures.Postgres16, + pgC, err := tcpostgres.Run(ctx, "postgres:16", tcpostgres.WithDatabase("testdb"), tcpostgres.WithUsername("test"), tcpostgres.WithPassword("test"), testcontainers.WithWaitStrategyAndDeadline(60*time.Second, wait.ForLog("database system is ready to accept connections").WithOccurrence(2)), - tcnetwork.WithNetworkName([]string{"postgres"}, testNetwork), ) Expect(err).ToNot(HaveOccurred()) - endpoint, err := testfixtures.ContainerEndpoint(ctx, pgC, "5432") + DeferCleanup(func() { pgC.Terminate(context.Background()) }) + connStr, err := pgC.ConnectionString(ctx, "sslmode=disable") Expect(err).ToNot(HaveOccurred()) - return pgC, endpoint -} - -func setupIsolatedSchema(endpoint string) *gorm.DB { - schema := fmt.Sprintf("test_%d_%d", GinkgoParallelProcess(), sharedDBSequence.Add(1)) - admin := openTestDB(endpoint, "") - Expect(admin.Exec("CREATE SCHEMA " + schema).Error).ToNot(HaveOccurred()) - db := openTestDB(endpoint, schema) - DeferCleanup(func() { - if sqlDB, err := db.DB(); err == nil { - _ = sqlDB.Close() - } - Expect(admin.Exec("DROP SCHEMA " + schema + " CASCADE").Error).ToNot(HaveOccurred()) - if sqlDB, err := admin.DB(); err == nil { - _ = sqlDB.Close() - } - }) - return db -} - -func openTestDB(endpoint, schema string) *gorm.DB { - connStr := fmt.Sprintf("postgres://test:test@%s/testdb?sslmode=disable", endpoint) - if schema != "" { - connStr += "&search_path=" + schema - } db, err := gorm.Open(postgres.Open(connStr), &gorm.Config{ Logger: logger.Default.LogMode(logger.Silent), }) diff --git a/core/services/worker/free_timeout_test.go b/core/services/worker/free_timeout_test.go index f27d4dcf3..4f1b6346e 100644 --- a/core/services/worker/free_timeout_test.go +++ b/core/services/worker/free_timeout_test.go @@ -6,8 +6,6 @@ import ( "os" "strconv" "syscall" - "testing" - "time" process "github.com/mudler/go-processmanager" gogrpc "google.golang.org/grpc" @@ -18,19 +16,6 @@ import ( . "github.com/onsi/gomega" ) -// TestWorkerFixtureProcess turns the current test binary into a portable -// long-running child for the process-stop assertions below. Using the test -// binary avoids assuming Unix utilities live at paths such as /bin/sleep, -// which is not true in Nix environments. -func TestWorkerFixtureProcess(t *testing.T) { - if os.Getenv("LOCALAI_WORKER_FIXTURE_PROCESS") != "1" { - return - } - for { - time.Sleep(time.Hour) - } -} - // pidAlive probes the OS directly for a process ID. The supervisor's own // liveness helpers all go through go-processmanager's pidfile, which Stop // deletes as part of releasing the handle, so they report "not alive" even if @@ -97,13 +82,10 @@ var _ = Describe("Stopping a backend whose Free never returns", func() { // actually dead afterwards, not merely that Stop() returned. It // outlives every timeout below, so if it is gone at the end it is // because the supervisor signalled it. - executable, err := os.Executable() - Expect(err).ToNot(HaveOccurred()) proc = process.New( process.WithTemporaryStateDir(), - process.WithName(executable), - process.WithArgs("-test.run=^TestWorkerFixtureProcess$"), - process.WithEnvironment(append(os.Environ(), "LOCALAI_WORKER_FIXTURE_PROCESS=1")...), + process.WithName("/bin/sleep"), + process.WithArgs("300"), ) Expect(proc.Run()).To(Succeed()) @@ -112,8 +94,7 @@ var _ = Describe("Stopping a backend whose Free never returns", func() { Expect(pidAlive(procPID)).To(BeTrue(), "the fixture process must be running before the stop") s = &backendSupervisor{ - cfg: &Config{}, - backendFreeTimeout: 20 * time.Millisecond, + cfg: &Config{}, processes: map[string]*backendProcess{ "wedged-model#0": { proc: proc, diff --git a/core/services/worker/lifecycle.go b/core/services/worker/lifecycle.go index 968b8d1ab..c80e00ea0 100644 --- a/core/services/worker/lifecycle.go +++ b/core/services/worker/lifecycle.go @@ -322,7 +322,7 @@ func (s *backendSupervisor) handleModelUnload(data []byte, reply func([]byte)) { // Best-effort bounded gRPC Free(). A model.unload request must not // occupy the NATS reply handler forever when a backend is wedged. client := grpc.NewClientWithToken(targetAddr, false, nil, false, s.cfg.RegistrationToken) - freeCtx, cancel := context.WithTimeout(context.Background(), s.freeTimeout()) + freeCtx, cancel := context.WithTimeout(context.Background(), workerBackendFreeTimeout) if err := client.Free(freeCtx); err != nil { xlog.Warn("Free() failed during model.unload", "error", err, "addr", targetAddr) } diff --git a/core/services/worker/supervisor.go b/core/services/worker/supervisor.go index 49c9b75f7..1e01cf441 100644 --- a/core/services/worker/supervisor.go +++ b/core/services/worker/supervisor.go @@ -133,11 +133,6 @@ type backendSupervisor struct { // the same not-yet-cached backend) are serialized here so the gallery // download path doesn't race itself on the same directory. backendLocks map[string]*sync.Mutex - - // backendFreeTimeout bounds the best-effort Free call before process - // termination. Zero uses workerBackendFreeTimeout; tests use a shorter - // deadline to exercise a wedged backend without waiting five seconds. - backendFreeTimeout time.Duration } // defaultPortQuarantine is how long a released gRPC port waits before it can be @@ -160,13 +155,6 @@ type backendSupervisor struct { // rows; raising this value is not a substitute for it. const defaultPortQuarantine = 15 * time.Second -func (s *backendSupervisor) freeTimeout() time.Duration { - if s.backendFreeTimeout > 0 { - return s.backendFreeTimeout - } - return workerBackendFreeTimeout -} - // quarantinedPort is a released port that must not be re-bound until `until`. type quarantinedPort struct { port int @@ -839,9 +827,8 @@ func (s *backendSupervisor) stopBackendExact(key string, force bool) error { if !force { client := grpc.NewClientWithToken(bp.addr, false, nil, false, s.cfg.RegistrationToken) - freeTimeout := s.freeTimeout() - freeCtx, cancel := context.WithTimeout(context.Background(), freeTimeout) - xlog.Debug("Calling bounded Free() before stopping backend", "backend", key, "timeout", freeTimeout) + freeCtx, cancel := context.WithTimeout(context.Background(), workerBackendFreeTimeout) + xlog.Debug("Calling bounded Free() before stopping backend", "backend", key, "timeout", workerBackendFreeTimeout) if err := client.Free(freeCtx); err != nil { xlog.Warn("Free() failed (best-effort)", "backend", key, "error", err) } diff --git a/core/startup/model_preload.go b/core/startup/model_preload.go index 17738b3ea..4f3bb1683 100644 --- a/core/startup/model_preload.go +++ b/core/startup/model_preload.go @@ -13,18 +13,12 @@ import ( "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/gallery/importers" "github.com/mudler/LocalAI/core/services/galleryop" - "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/xlog" ) -const ( - modelImportPollInterval = 50 * time.Millisecond - modelImportMaxPollInterval = 500 * time.Millisecond -) - // InstallModels will preload models from the given list of URLs and galleries // It will download the model if it is not already present in the model path // It will also try to resolve if the model is an embedded model YAML configuration @@ -81,21 +75,13 @@ func InstallModelsWithOptions(ctx context.Context, galleryService *galleryop.Gal } var status *galleryop.OpStatus - pollInterval := modelImportPollInterval - poll := time.NewTimer(pollInterval) - defer poll.Stop() + // wait for op to finish for { status = galleryService.GetStatus(uuid.String()) if status != nil && status.Processed { break } - select { - case <-ctx.Done(): - return ctx.Err() - case <-poll.C: - pollInterval = backoff.Exponential(pollInterval, modelImportMaxPollInterval, 1) - poll.Reset(pollInterval) - } + time.Sleep(1 * time.Second) } if status.Error != nil { diff --git a/core/startup/model_preload_test.go b/core/startup/model_preload_test.go index ad662a5fa..525f183cf 100644 --- a/core/startup/model_preload_test.go +++ b/core/startup/model_preload_test.go @@ -3,8 +3,6 @@ package startup_test import ( "context" "fmt" - "net/http" - "net/http/httptest" "os" "path/filepath" @@ -41,11 +39,7 @@ var _ = Describe("Preload test", func() { Context("Preloading from strings", func() { It("loads from embedded full-urls", func() { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte("name: phi-2\nbackend: llama-cpp\nparameters:\n model: phi-2.gguf\n")) - })) - defer server.Close() - url := server.URL + "/phi-2.yaml" + url := "https://raw.githubusercontent.com/mudler/LocalAI-examples/main/configurations/phi-2.yaml" fileName := fmt.Sprintf("%s.yaml", "phi-2") galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{ @@ -65,11 +59,7 @@ var _ = Describe("Preload test", func() { Expect(string(content)).To(ContainSubstring("name: phi-2")) }) It("downloads from urls", func() { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte("tiny local GGUF fixture")) - })) - defer server.Close() - url := server.URL + "/tinyllama-1.1b-chat-v0.3.Q2_K.gguf" + url := "huggingface://TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/tinyllama-1.1b-chat-v0.3.Q2_K.gguf" fileName := fmt.Sprintf("%s.gguf", "tinyllama-1.1b-chat-v0.3.Q2_K") galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{ diff --git a/docs/content/development/offline-tests.md b/docs/content/development/offline-tests.md deleted file mode 100644 index 104a04f4d..000000000 --- a/docs/content/development/offline-tests.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: "Offline test resources" ---- - -LocalAI tests separate resource acquisition from test execution. Resources are -declared by resource set in `test-resources/manifests/`; files and packed container -images are content-addressed by SHA-256 under -`.cache/test-resources/blobs/sha256/`. - -Prepare the resources before running a target: - -```sh -make prepare-offline-test-cache TEST_RESOURCE_SET=default -``` - -Preparation verifies every cached blob and fails closed. It never substitutes -a live request for a missing or corrupt entry. Maintainers can populate a -cache from pinned declarations only by explicitly enabling online mode: - -```sh -LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=default -``` - -The update command records declared responses, files, and digest-pinned images, -then writes a deterministic, zstd level-1 bundle at -`.cache/test-resources/bundles/.tar.zst`. Its SHA-256 is written to the -lock file. The test workflow transfers the bundle as a workflow artifact and -verifies it after deleting the recording cache; the scheduled refresh workflow -also publishes verified bundles to GHCR as OCI artifacts. - -HTTP declarations may include `request_headers`. `Range` participates in the -cache key, and authorization values participate only through a SHA-256 value; -credentials are never written verbatim to the cache index. Redirect responses -are recorded without following them, so every hop needed by a test must be -declared explicitly. - -File and HTTP declarations may list HTTPS `mirrors`. Recording tries the -canonical URL twice, then each mirror twice, and reports the duration of every -attempt. Every candidate must produce the same declared SHA-256; mirrors are -alternate transports, not alternate content. - -A digest mismatch is never accepted automatically. The updater prints the -observed failure for every source and directs maintainers to compare upstream -checksums, signatures, release notes, and redirects, then check the GitHub -Advisory Database and OSV before approving a new digest. Repeated mismatches can -mean a legitimate upstream release, a corrupt mirror, or a supply-chain event. - -Ordinary test recipes execute through `scripts/run-test-offline.sh`. Its -supervised replay proxy terminates HTTP and HTTPS and returns an immediate -error containing the method and URL for undeclared requests. Linux CI also -runs the command in a cgroup with public IPv4 and IPv6 rejected; macOS relies -on replay, declared resources, guarded Go transports, and static lint because -kernel-level subprocess enforcement is Linux-only. - -Testcontainer images must be registry-digest pinned and loaded during -preparation. Container helpers check that an image exists before startup and -attach services to internal-only Docker networks, preventing testcontainers -from silently pulling a missing tag. - -The default Linux and macOS suites use separate resource sets because -Docker archives are platform-specific. Backend and hardware resources remain -separate targets so ordinary contributors do not acquire large model fixtures -that their test command does not use. - -Coverage runs print a wall-clock summary for each test root and list every -Ginkgo spec or hook taking at least three seconds, including its source -location. Set `COVERAGE_SLOW_SPEC_THRESHOLD=` to tune the reporting -threshold. This measures the whole spec or hook, so it exposes time spent in -sleeps, polling, channel waits, cleanup, and resource contention without -replacing Go's global clock or changing test semantics. The same timings are -written to `coverage/timings.tsv` for CI artifacts and comparisons. The report -shows the slowest 25 entries per root by default; set -`COVERAGE_SLOW_SPEC_LIMIT=` to change the cap. - -Real third-party compatibility checks belong in separately named -`external-probe-*` scheduled workflows and must not be part of deterministic -test or coverage gates. diff --git a/internal/backoff/backoff.go b/internal/backoff/backoff.go deleted file mode 100644 index 075e6874a..000000000 --- a/internal/backoff/backoff.go +++ /dev/null @@ -1,23 +0,0 @@ -// Package backoff provides bounded retry-delay calculations. -package backoff - -import "time" - -// Exponential returns base*2^exponent capped at maximum. It saturates before -// multiplying so time.Duration cannot overflow. -func Exponential(base, maximum time.Duration, exponent uint) time.Duration { - if base <= 0 || maximum <= 0 { - return 0 - } - if base >= maximum { - return maximum - } - - for ; exponent > 0; exponent-- { - if base > maximum/2 { - return maximum - } - base *= 2 - } - return base -} diff --git a/internal/backoff/backoff_suite_test.go b/internal/backoff/backoff_suite_test.go deleted file mode 100644 index c8137b7c8..000000000 --- a/internal/backoff/backoff_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package backoff_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestBackoff(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Backoff Suite") -} diff --git a/internal/backoff/backoff_test.go b/internal/backoff/backoff_test.go deleted file mode 100644 index 8452a99c4..000000000 --- a/internal/backoff/backoff_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package backoff_test - -import ( - "math" - "time" - - "github.com/mudler/LocalAI/internal/backoff" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Exponential", func() { - It("doubles the base delay up to the maximum", func() { - base := 50 * time.Millisecond - maximum := 500 * time.Millisecond - - Expect(backoff.Exponential(base, maximum, 0)).To(Equal(50 * time.Millisecond)) - Expect(backoff.Exponential(base, maximum, 1)).To(Equal(100 * time.Millisecond)) - Expect(backoff.Exponential(base, maximum, 2)).To(Equal(200 * time.Millisecond)) - Expect(backoff.Exponential(base, maximum, 3)).To(Equal(400 * time.Millisecond)) - Expect(backoff.Exponential(base, maximum, 4)).To(Equal(maximum)) - Expect(backoff.Exponential(base, maximum, math.MaxUint)).To(Equal(maximum)) - }) - - It("saturates without overflowing a duration", func() { - maximum := time.Duration(math.MaxInt64) - Expect(backoff.Exponential(maximum/2+1, maximum, 1)).To(Equal(maximum)) - Expect(backoff.Exponential(2, 5, 1)).To(Equal(time.Duration(4))) - }) - - It("returns zero when backoff is disabled", func() { - Expect(backoff.Exponential(0, time.Second, 1)).To(BeZero()) - Expect(backoff.Exponential(time.Second, 0, 1)).To(BeZero()) - }) -}) diff --git a/internal/testfixtures/images.go b/internal/testfixtures/images.go deleted file mode 100644 index fe16d766d..000000000 --- a/internal/testfixtures/images.go +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-License-Identifier: MIT - -// Package testfixtures centralizes immutable resources shared by test suites. -package testfixtures - -import ( - "context" - "errors" - "fmt" - "net" - "os" - - "github.com/moby/moby/client" - "github.com/testcontainers/testcontainers-go" -) - -const ( - Postgres16 = "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20" - Postgres16Alpine = "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777" - NATS2Alpine = "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0" -) - -// RequireImage fails before testcontainers can fall back to a registry pull. -func RequireImage(ctx context.Context, reference, target string) error { - docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - return err - } - defer func() { _ = docker.Close() }() - if _, err := docker.ImageInspect(ctx, reference); err != nil { - return fmt.Errorf("required offline test image %s is not loaded; run `make prepare-offline-test-cache TEST_RESOURCE_SET=%s`: %w", reference, target, err) - } - return nil -} - -func DockerNetwork() (string, error) { - name := os.Getenv("LOCALAI_TEST_DOCKER_NETWORK") //nolint:forbidigo - if name == "" { - return "", errors.New("offline test Docker network is not configured; run the test through scripts/run-test-offline.sh") - } - return name, nil -} - -// ContainerEndpoint returns an address reachable from the Linux test host -// without publishing a port from the internal-only Docker network. -func ContainerEndpoint(ctx context.Context, container testcontainers.Container, port string) (string, error) { - ip, err := container.ContainerIP(ctx) - if err != nil { - return "", err - } - if ip == "" { - return "", errors.New("offline test container has no private network address") - } - return net.JoinHostPort(ip, port), nil -} diff --git a/internal/testresources/bundle.go b/internal/testresources/bundle.go deleted file mode 100644 index eeffa3f1f..000000000 --- a/internal/testresources/bundle.go +++ /dev/null @@ -1,164 +0,0 @@ -// SPDX-License-Identifier: MIT - -package testresources - -import ( - "archive/tar" - "bytes" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/klauspost/compress/zstd" -) - -func PackBundle(cacheDir, output string, manifest Manifest) (string, error) { - index, err := LoadHTTPIndex(cacheDir) - if err != nil { - return "", err - } - targetIndex := map[string]HTTPEntry{} - digests := map[string]bool{} - for _, resource := range manifest.HTTP { - key := RequestKey(resource.Method, resource.URL, resource.Headers()) - entry, ok := index[key] - if !ok { - return "", fmt.Errorf("cannot pack missing HTTP entry %s", key) - } - targetIndex[key], digests[resource.SHA256] = entry, true - } - for _, resource := range manifest.Files { - digests[resource.SHA256] = true - } - for _, resource := range manifest.Images { - digests[resource.SHA256] = true - } - if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil { - return "", err - } - tmp, err := os.CreateTemp(filepath.Dir(output), "bundle-*.tmp") - if err != nil { - return "", err - } - name := tmp.Name() - defer func() { _ = os.Remove(name) }() - hash := sha256.New() - zstdWriter, err := zstd.NewWriter(io.MultiWriter(tmp, hash), - zstd.WithEncoderLevel(zstd.SpeedFastest), - zstd.WithEncoderConcurrency(1), - zstd.WithEncoderCRC(true), - ) - if err != nil { - _ = tmp.Close() - return "", err - } - tw := tar.NewWriter(zstdWriter) - indexData, err := json.Marshal(targetIndex) - if err == nil { - err = writeTarBytes(tw, "http-index.json", indexData) - } - ordered := make([]string, 0, len(digests)) - for digest := range digests { - ordered = append(ordered, digest) - } - sort.Strings(ordered) - for _, digest := range ordered { - if err != nil { - break - } - path, verifyErr := VerifyBlob(cacheDir, digest) - if verifyErr != nil { - err = verifyErr - break - } - var data []byte - data, err = os.ReadFile(path) - if err == nil { - err = writeTarBytes(tw, filepath.ToSlash(filepath.Join("blobs", "sha256", digest)), data) - } - } - err = errors.Join(err, tw.Close(), zstdWriter.Close(), tmp.Close()) - if err != nil { - return "", err - } - if err := os.Rename(name, output); err != nil { - return "", err - } - return fmt.Sprintf("%x", hash.Sum(nil)), nil -} - -func RestoreBundle(cacheDir, bundle, expected string) error { - data, err := os.ReadFile(bundle) - if err != nil { - return err - } - actual := fmt.Sprintf("%x", sha256.Sum256(data)) - if actual != expected { - return fmt.Errorf("test resource bundle checksum mismatch: expected %s, got %s", expected, actual) - } - var bundleReader io.Reader = bytes.NewReader(data) - if len(data) >= 4 && bytes.Equal(data[:4], []byte{0x28, 0xb5, 0x2f, 0xfd}) { - zstdReader, err := zstd.NewReader(bundleReader, zstd.WithDecoderConcurrency(1)) - if err != nil { - return err - } - defer zstdReader.Close() - bundleReader = zstdReader - } - tr := tar.NewReader(bundleReader) - recorded := map[string]HTTPEntry{} - for { - header, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return err - } - name := filepath.Clean(filepath.FromSlash(header.Name)) - if filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) { - return fmt.Errorf("unsafe bundle path %q", header.Name) - } - body, err := io.ReadAll(tr) - if err != nil { - return err - } - if name == "http-index.json" { - if err := json.Unmarshal(body, &recorded); err != nil { - return err - } - continue - } - destination := filepath.Join(cacheDir, name) - if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { - return err - } - if err := os.WriteFile(destination, body, 0o644); err != nil { - return err - } - } - index, err := LoadHTTPIndex(cacheDir) - if err != nil { - return err - } - for key, entry := range recorded { - index[key] = entry - } - return WriteHTTPIndex(cacheDir, index) -} - -func writeTarBytes(tw *tar.Writer, name string, data []byte) error { - header := &tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), ModTime: time.Unix(0, 0).UTC()} - if err := tw.WriteHeader(header); err != nil { - return err - } - _, err := tw.Write(data) - return err -} diff --git a/internal/testresources/httpcache.go b/internal/testresources/httpcache.go deleted file mode 100644 index c32f51599..000000000 --- a/internal/testresources/httpcache.go +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-License-Identifier: MIT - -package testresources - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" -) - -var hopHeaders = map[string]bool{ - "Connection": true, "Proxy-Connection": true, "Keep-Alive": true, - "Transfer-Encoding": true, "Content-Length": true, "Te": true, - "Trailer": true, "Upgrade": true, "Proxy-Authenticate": true, - "Proxy-Authorization": true, -} - -func LoadHTTPIndex(cacheDir string) (map[string]HTTPEntry, error) { - index := map[string]HTTPEntry{} - data, err := os.ReadFile(filepath.Join(cacheDir, "index.json")) - if errors.Is(err, os.ErrNotExist) { - return index, nil - } - if err != nil { - return nil, fmt.Errorf("read HTTP cache index: %w", err) - } - if err := json.Unmarshal(data, &index); err != nil { - return nil, fmt.Errorf("parse HTTP cache index: %w", err) - } - return index, nil -} - -func WriteHTTPIndex(cacheDir string, index map[string]HTTPEntry) error { - data, err := json.MarshalIndent(index, "", " ") - if err != nil { - return err - } - data = append(data, '\n') - if err := os.MkdirAll(cacheDir, 0o755); err != nil { - return err - } - tmp, err := os.CreateTemp(cacheDir, "index-*.tmp") - if err != nil { - return err - } - name := tmp.Name() - defer func() { _ = os.Remove(name) }() - if _, err := tmp.Write(data); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Close(); err != nil { - return err - } - return os.Rename(name, filepath.Join(cacheDir, "index.json")) -} - -func SanitizeHeaders(header http.Header) http.Header { - out := header.Clone() - for name := range hopHeaders { - out.Del(name) - } - return out -} - -func ReplayResponse(w http.ResponseWriter, cacheDir string, entry HTTPEntry) error { - path, err := VerifyBlob(cacheDir, entry.Digest) - if err != nil { - return err - } - for name, values := range entry.Header { - for _, value := range values { - w.Header().Add(name, value) - } - } - w.Header().Set("Content-Length", fmt.Sprint(entry.Size)) - w.WriteHeader(entry.Status) - if entry.Size == 0 { - return nil - } - body, err := os.Open(path) - if err != nil { - return err - } - defer func() { _ = body.Close() }() - _, err = io.Copy(w, body) - return err -} diff --git a/internal/testresources/resources.go b/internal/testresources/resources.go deleted file mode 100644 index f46ff3ca2..000000000 --- a/internal/testresources/resources.go +++ /dev/null @@ -1,215 +0,0 @@ -// SPDX-License-Identifier: MIT - -package testresources - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" -) - -const ManifestVersion = 1 - -type Manifest struct { - Version int `json:"version"` - Target string `json:"target"` - HTTP []HTTP `json:"http,omitempty"` - Files []File `json:"files,omitempty"` - Images []OCIImage `json:"images,omitempty"` -} - -type HTTP struct { - Method string `json:"method"` - URL string `json:"url"` - Mirrors []string `json:"mirrors,omitempty"` - SHA256 string `json:"sha256"` - RequestHeaders map[string]string `json:"request_headers,omitempty"` -} - -type HTTPEntry struct { - Digest string `json:"digest"` - Size int64 `json:"size"` - Status int `json:"status"` - Header http.Header `json:"header"` -} - -type File struct { - URL string `json:"url"` - Mirrors []string `json:"mirrors,omitempty"` - SHA256 string `json:"sha256"` - Destination string `json:"destination,omitempty"` - Environment string `json:"environment,omitempty"` -} - -type OCIImage struct { - Reference string `json:"reference"` - SHA256 string `json:"sha256"` -} - -type Lock struct { - Version int `json:"version"` - Bundles map[string]string `json:"bundles"` -} - -func LoadManifest(path string) (Manifest, error) { - var manifest Manifest - if err := decode(path, &manifest); err != nil { - return manifest, err - } - if err := manifest.Validate(); err != nil { - return manifest, fmt.Errorf("%s: %w", path, err) - } - return manifest, nil -} - -func LoadLock(path string) (Lock, error) { - var lock Lock - if err := decode(path, &lock); err != nil { - return lock, err - } - if lock.Version != ManifestVersion { - return lock, fmt.Errorf("%s: unsupported version %d", path, lock.Version) - } - return lock, nil -} - -func WriteLock(path string, lock Lock) error { - return writeJSON(path, lock) -} - -func WriteManifest(path string, manifest Manifest) error { - return writeJSON(path, manifest) -} - -func writeJSON(path string, value any) error { - data, err := json.MarshalIndent(value, "", " ") - if err != nil { - return err - } - data = append(data, '\n') - return os.WriteFile(path, data, 0o644) -} - -func decode(path string, value any) error { - data, err := os.ReadFile(path) - if err != nil { - return err - } - decoder := json.NewDecoder(strings.NewReader(string(data))) - decoder.DisallowUnknownFields() - if err := decoder.Decode(value); err != nil { - return err - } - if decoder.Decode(&struct{}{}) != io.EOF { - return errors.New("manifest has trailing JSON data") - } - return nil -} - -func (m Manifest) Validate() error { - if m.Version != ManifestVersion { - return fmt.Errorf("unsupported version %d", m.Version) - } - if strings.TrimSpace(m.Target) == "" { - return errors.New("target is required") - } - for _, resource := range m.HTTP { - if resource.Method == "" || resource.URL == "" || !validDigest(resource.SHA256) { - return fmt.Errorf("HTTP resources require method, URL, and lowercase sha256: %s %s", resource.Method, resource.URL) - } - if err := validateMirrors(resource.Mirrors); err != nil { - return fmt.Errorf("HTTP resource %s: %w", resource.URL, err) - } - } - for _, resource := range m.Files { - if resource.URL == "" || !validDigest(resource.SHA256) || (resource.Destination == "" && resource.Environment == "") { - return fmt.Errorf("file resources require URL, sha256, and destination or environment: %s", resource.URL) - } - if filepath.IsAbs(resource.Destination) || strings.HasPrefix(filepath.Clean(resource.Destination), "..") { - return fmt.Errorf("file destination must stay inside the resource directory: %s", resource.Destination) - } - if err := validateMirrors(resource.Mirrors); err != nil { - return fmt.Errorf("file resource %s: %w", resource.URL, err) - } - } - for _, resource := range m.Images { - if !strings.Contains(resource.Reference, "@sha256:") || !validDigest(resource.SHA256) { - return fmt.Errorf("OCI image must be digest-pinned and have a packed sha256: %s", resource.Reference) - } - } - return nil -} - -func validateMirrors(mirrors []string) error { - seen := map[string]bool{} - for _, mirror := range mirrors { - if !strings.HasPrefix(mirror, "https://") { - return fmt.Errorf("mirror must use HTTPS: %s", mirror) - } - if seen[mirror] { - return fmt.Errorf("duplicate mirror: %s", mirror) - } - seen[mirror] = true - } - return nil -} - -func BlobPath(cacheDir, digest string) string { - return filepath.Join(cacheDir, "blobs", "sha256", digest) -} - -func RequestKey(method, rawURL string, headers ...http.Header) string { - key := strings.ToUpper(method) + " " + rawURL - if len(headers) == 0 { - return key - } - for _, name := range []string{"Authorization", "Range"} { - value := headers[0].Get(name) - if value == "" { - continue - } - if name == "Authorization" { - digest := sha256.Sum256([]byte(value)) - value = "sha256:" + hex.EncodeToString(digest[:]) - } - key += "\n" + strings.ToLower(name) + ":" + value - } - return key -} - -func (resource HTTP) Headers() http.Header { - header := make(http.Header, len(resource.RequestHeaders)) - for name, value := range resource.RequestHeaders { - header.Set(name, value) - } - return header -} - -func VerifyBlob(cacheDir, digest string) (string, error) { - path := BlobPath(cacheDir, digest) - data, err := os.ReadFile(path) - if err != nil { - return "", fmt.Errorf("missing CAS blob %s: %w", digest, err) - } - sum := sha256.Sum256(data) - actual := hex.EncodeToString(sum[:]) - if actual != digest { - return "", fmt.Errorf("corrupt CAS blob %s: got sha256:%s", digest, actual) - } - return path, nil -} - -func validDigest(value string) bool { - if len(value) != sha256.Size*2 || strings.ToLower(value) != value { - return false - } - _, err := hex.DecodeString(value) - return err == nil -} diff --git a/internal/testresources/resources_suite_test.go b/internal/testresources/resources_suite_test.go deleted file mode 100644 index ead665b25..000000000 --- a/internal/testresources/resources_suite_test.go +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: MIT - -package testresources_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestResources(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Test resources suite") -} diff --git a/internal/testresources/resources_test.go b/internal/testresources/resources_test.go deleted file mode 100644 index ae6203b5e..000000000 --- a/internal/testresources/resources_test.go +++ /dev/null @@ -1,126 +0,0 @@ -// SPDX-License-Identifier: MIT - -package testresources_test - -import ( - "crypto/sha256" - "fmt" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/mudler/LocalAI/internal/testresources" -) - -var _ = Describe("Declared test resources", func() { - It("rejects mutable and unpinned resources", func() { - manifest := testresources.Manifest{ - Version: testresources.ManifestVersion, - Target: "backend", - Images: []testresources.OCIImage{{Reference: "postgres:latest", SHA256: fmt.Sprintf("%064d", 0)}}, - } - Expect(manifest.Validate()).To(MatchError(ContainSubstring("digest-pinned"))) - }) - - It("requires HTTPS and unique mirrors", func() { - digest := fmt.Sprintf("%064d", 0) - manifest := testresources.Manifest{Version: 1, Target: "fixture", Files: []testresources.File{{ - URL: "https://primary.invalid/file", Mirrors: []string{"http://mirror.invalid/file"}, - SHA256: digest, Destination: "file", - }}} - Expect(manifest.Validate()).To(MatchError(ContainSubstring("mirror must use HTTPS"))) - manifest.Files[0].Mirrors = []string{"https://mirror.invalid/file", "https://mirror.invalid/file"} - Expect(manifest.Validate()).To(MatchError(ContainSubstring("duplicate mirror"))) - }) - - It("fails before tests when a CAS blob is missing or corrupt", func() { - cache := GinkgoT().TempDir() - digest := fmt.Sprintf("%064d", 0) - _, err := testresources.VerifyBlob(cache, digest) - Expect(err).To(MatchError(ContainSubstring("missing CAS blob"))) - - path := testresources.BlobPath(cache, digest) - Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) - Expect(os.WriteFile(path, []byte("corrupt"), 0o644)).To(Succeed()) - _, err = testresources.VerifyBlob(cache, digest) - Expect(err).To(MatchError(ContainSubstring("corrupt CAS blob"))) - }) - - It("accepts and verifies a content-addressed blob", func() { - cache := GinkgoT().TempDir() - content := []byte("offline fixture") - digest := fmt.Sprintf("%x", sha256.Sum256(content)) - path := testresources.BlobPath(cache, digest) - Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) - Expect(os.WriteFile(path, content, 0o644)).To(Succeed()) - Expect(testresources.VerifyBlob(cache, digest)).To(Equal(path)) - }) - - It("persists response metadata and replays a verified body", func() { - cache := GinkgoT().TempDir() - content := []byte("cached response") - digest := fmt.Sprintf("%x", sha256.Sum256(content)) - path := testresources.BlobPath(cache, digest) - Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) - Expect(os.WriteFile(path, content, 0o644)).To(Succeed()) - index := map[string]testresources.HTTPEntry{ - "GET https://example.invalid/data": { - Digest: digest, Size: int64(len(content)), Status: http.StatusPartialContent, - Header: http.Header{"Content-Range": {"bytes 0-14/15"}}, - }, - } - Expect(testresources.WriteHTTPIndex(cache, index)).To(Succeed()) - loaded, err := testresources.LoadHTTPIndex(cache) - Expect(err).NotTo(HaveOccurred()) - recorder := httptest.NewRecorder() - Expect(testresources.ReplayResponse(recorder, cache, loaded["GET https://example.invalid/data"])).To(Succeed()) - Expect(recorder.Code).To(Equal(http.StatusPartialContent)) - Expect(recorder.Body.Bytes()).To(Equal(content)) - Expect(recorder.Header().Get("Content-Range")).To(Equal("bytes 0-14/15")) - }) - - It("sanitizes connection-specific response headers", func() { - header := http.Header{"Transfer-Encoding": {"chunked"}, "Authorization": {"secret"}, "X-Fixture": {"yes"}} - clean := testresources.SanitizeHeaders(header) - Expect(clean).NotTo(HaveKey("Transfer-Encoding")) - Expect(clean).To(HaveKeyWithValue("Authorization", []string{"secret"})) - Expect(clean).To(HaveKeyWithValue("X-Fixture", []string{"yes"})) - }) - - It("keys range and authorization variants without storing credentials", func() { - header := http.Header{"Authorization": {"Bearer secret"}, "Range": {"bytes=4-"}} - key := testresources.RequestKey(http.MethodGet, "https://example.invalid/model", header) - Expect(key).To(ContainSubstring("range:bytes=4-")) - Expect(key).To(ContainSubstring("authorization:sha256:")) - Expect(key).NotTo(ContainSubstring("Bearer secret")) - Expect(key).NotTo(Equal(testresources.RequestKey(http.MethodGet, "https://example.invalid/model"))) - }) - - It("packs deterministically and restores a target cache", func() { - cache := GinkgoT().TempDir() - content := []byte("bundle fixture") - digest := fmt.Sprintf("%x", sha256.Sum256(content)) - path := testresources.BlobPath(cache, digest) - Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) - Expect(os.WriteFile(path, content, 0o644)).To(Succeed()) - manifest := testresources.Manifest{Version: 1, Target: "fixture", Files: []testresources.File{{URL: "https://example.invalid/file", SHA256: digest, Destination: "file"}}} - first := filepath.Join(GinkgoT().TempDir(), "first.tar.zst") - second := filepath.Join(GinkgoT().TempDir(), "second.tar.zst") - firstDigest, err := testresources.PackBundle(cache, first, manifest) - Expect(err).NotTo(HaveOccurred()) - secondDigest, err := testresources.PackBundle(cache, second, manifest) - Expect(err).NotTo(HaveOccurred()) - Expect(secondDigest).To(Equal(firstDigest)) - compressed, err := os.ReadFile(first) - Expect(err).NotTo(HaveOccurred()) - Expect(compressed[:4]).To(Equal([]byte{0x28, 0xb5, 0x2f, 0xfd})) - - restored := GinkgoT().TempDir() - Expect(testresources.RestoreBundle(restored, first, firstDigest)).To(Succeed()) - Expect(os.ReadFile(testresources.BlobPath(restored, digest))).To(Equal(content)) - }) -}) diff --git a/pkg/downloader/cancel_test.go b/pkg/downloader/cancel_test.go index 57f9bba95..76f8a2df5 100644 --- a/pkg/downloader/cancel_test.go +++ b/pkg/downloader/cancel_test.go @@ -59,7 +59,9 @@ var _ = Describe("Download cancellation", func() { } BeforeEach(func() { - filePath = GinkgoT().TempDir() + "/cancel_model" + dir, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + filePath = dir + "/cancel_model" }) AfterEach(func() { @@ -110,7 +112,7 @@ var _ = Describe("Download cancellation", func() { Expect(err).To(HaveOccurred()) Expect(errors.Is(err, context.Canceled)).To(BeTrue()) - Expect(filePath+".partial").ToNot(BeAnExistingFile(), + Expect(filePath + ".partial").ToNot(BeAnExistingFile(), "a deliberate user cancel must not leave a dangling .partial behind") }) diff --git a/pkg/downloader/retry.go b/pkg/downloader/retry.go index cd956f7d5..5dc6584c7 100644 --- a/pkg/downloader/retry.go +++ b/pkg/downloader/retry.go @@ -4,10 +4,7 @@ import ( "context" "errors" "io" - "math" "time" - - "github.com/mudler/LocalAI/internal/backoff" ) // ErrTransientDownload marks a download failure that a later attempt has a @@ -92,11 +89,7 @@ func (t *readErrorRecorder) Read(p []byte) (int, error) { // waitBeforeRetry sleeps for the backoff interval of the given attempt // (1-based), returning the context error if the caller gives up while waiting. func waitBeforeRetry(ctx context.Context, attempt int) error { - exponent := 0 - if attempt > 1 { - exponent = attempt - 1 - } - delay := backoff.Exponential(DownloadRetryBaseDelay, time.Duration(math.MaxInt64), uint(exponent)) + delay := DownloadRetryBaseDelay << (attempt - 1) timer := time.NewTimer(delay) defer timer.Stop() select { diff --git a/pkg/downloader/stall_test.go b/pkg/downloader/stall_test.go index 7c9f2e5f6..34ae2d348 100644 --- a/pkg/downloader/stall_test.go +++ b/pkg/downloader/stall_test.go @@ -18,7 +18,9 @@ var _ = Describe("Download stall timeout", func() { var savedTimeout time.Duration BeforeEach(func() { - filePath = GinkgoT().TempDir() + "/stall_model" + dir, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + filePath = dir + "/stall_model" savedTimeout = DownloadStallTimeout }) diff --git a/pkg/downloader/uri_test.go b/pkg/downloader/uri_test.go index 20f22454c..9cb667b57 100644 --- a/pkg/downloader/uri_test.go +++ b/pkg/downloader/uri_test.go @@ -22,15 +22,31 @@ var _ = Describe("Gallery API tests", func() { Context("URI", func() { It("parses github with a branch", func() { uri := URI("github:go-skynet/model-gallery/gpt4all-j.yaml") - Expect(uri.ResolveURL()).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) + Expect( + uri.ReadWithCallback("", func(url string, i []byte) error { + Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) + return nil + }), + ).ToNot(HaveOccurred()) }) It("parses github without a branch", func() { uri := URI("github:go-skynet/model-gallery/gpt4all-j.yaml@main") - Expect(uri.ResolveURL()).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) + + Expect( + uri.ReadWithCallback("", func(url string, i []byte) error { + Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) + return nil + }), + ).ToNot(HaveOccurred()) }) It("parses github with urls", func() { uri := URI("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml") - Expect(uri.ResolveURL()).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) + Expect( + uri.ReadWithCallback("", func(url string, i []byte) error { + Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) + return nil + }), + ).ToNot(HaveOccurred()) }) }) @@ -247,7 +263,9 @@ var _ = Describe("Download Test", func() { _, err = _mockDataSha.Write(mockData) Expect(err).ToNot(HaveOccurred()) mockDataSha = fmt.Sprintf("%x", _mockDataSha.Sum(nil)) - filePath = GinkgoT().TempDir() + "/my_supercool_model" + dir, err := os.Getwd() + filePath = dir + "/my_supercool_model" + Expect(err).NotTo(HaveOccurred()) }) Context("URI DownloadFile", func() { diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index e96995d17..c18c78185 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -28,11 +28,8 @@ import ( "net" "net/http" "net/url" - "os" "strings" "time" - - "github.com/mudler/LocalAI/pkg/testnetwork" ) const ( @@ -108,20 +105,12 @@ func sameOrigin(a, b *url.URL) bool { // (e.g. a credential-injecting RoundTripper) should base it on this rather than // http.DefaultTransport so the TLS floor and timeouts are preserved. func HardenedTransport() *http.Transport { - dialContext := (&net.Dialer{ - Timeout: dialTimeout, - KeepAlive: dialKeepAlive, - }).DialContext - // This is set only by the test-resource supervisor before it starts the - // child process; production configuration does not cross this boundary. - if os.Getenv("LOCALAI_TEST_OFFLINE") == "1" { //nolint:forbidigo - guard := testnetwork.LocalGuard() - guard.Dial = dialContext - dialContext = guard.DialContext - } return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: dialContext, + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: dialTimeout, + KeepAlive: dialKeepAlive, + }).DialContext, ForceAttemptHTTP2: true, MaxIdleConns: maxIdleConns, IdleConnTimeout: idleConnTimeout, diff --git a/pkg/huggingface-api/client.go b/pkg/huggingface-api/client.go index 05ccd3465..1d1c7ae3c 100644 --- a/pkg/huggingface-api/client.go +++ b/pkg/huggingface-api/client.go @@ -14,7 +14,6 @@ import ( "strings" "time" - "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/httpclient" ) @@ -97,49 +96,21 @@ type Client struct { maxRetries int retryBackoff time.Duration maxBackoff time.Duration - clock Clock -} - -// Clock is the small portion of wall-clock time used by retry handling. -// Supplying a fake clock lets tests verify backoff behavior without sleeping. -type Clock interface { - Now() time.Time - Sleep(time.Duration) -} - -type realClock struct{} - -func (realClock) Now() time.Time { return time.Now() } -func (realClock) Sleep(d time.Duration) { time.Sleep(d) } - -// ClientOption configures a Hugging Face API client. -type ClientOption func(*Client) - -// WithClock replaces the clock used for retry delays. -func WithClock(clock Clock) ClientOption { - return func(client *Client) { - if clock != nil { - client.clock = clock - } - } + sleepFn func(time.Duration) } var ErrRateLimited = errors.New("huggingface API rate limited") // NewClient creates a new Hugging Face API client -func NewClient(options ...ClientOption) *Client { - client := &Client{ +func NewClient() *Client { + return &Client{ baseURL: "https://huggingface.co/api/models", client: httpclient.New(httpclient.WithFollowRedirects()), maxRetries: 5, retryBackoff: 1 * time.Second, maxBackoff: 30 * time.Second, - clock: realClock{}, + sleepFn: time.Sleep, } - for _, option := range options { - option(client) - } - return client } func (c *Client) newRequest(ctx context.Context, method, rawURL, token string) (*http.Request, error) { @@ -172,7 +143,7 @@ func (c *Client) SearchModels(params SearchParams) ([]Model, error) { resp, err := c.client.Do(req) if err != nil { if attempt < c.maxRetries { - c.clock.Sleep(c.exponentialBackoff(attempt)) + c.sleepFn(c.exponentialBackoff(attempt)) continue } return nil, fmt.Errorf("failed to make request: %w", err) @@ -183,7 +154,7 @@ func (c *Client) SearchModels(params SearchParams) ([]Model, error) { return nil, fmt.Errorf("failed to close response body: %w", err) } if c.isRetryableStatus(resp.StatusCode) && attempt < c.maxRetries { - c.clock.Sleep(c.retryDelay(resp, attempt)) + c.sleepFn(c.retryDelay(resp, attempt)) continue } if resp.StatusCode == http.StatusTooManyRequests { @@ -228,7 +199,7 @@ func (c *Client) retryDelay(resp *http.Response, attempt int) time.Duration { return delay } if at, err := http.ParseTime(retryAfter); err == nil { - delay := at.Sub(c.clock.Now()) + delay := time.Until(at) if delay > 0 { if delay > c.maxBackoff { return c.maxBackoff @@ -242,11 +213,17 @@ func (c *Client) retryDelay(resp *http.Response, attempt int) time.Duration { } func (c *Client) exponentialBackoff(attempt int) time.Duration { - exponent := 0 - if attempt > 1 { - exponent = attempt - 1 + delay := c.retryBackoff + for i := 1; i < attempt; i++ { + delay *= 2 + if delay >= c.maxBackoff { + return c.maxBackoff + } } - return backoff.Exponential(c.retryBackoff, c.maxBackoff, uint(exponent)) + if delay > c.maxBackoff { + return c.maxBackoff + } + return delay } // GetLatest fetches the latest GGUF models diff --git a/pkg/huggingface-api/client_test.go b/pkg/huggingface-api/client_test.go index 591c26aaf..feac4dba3 100644 --- a/pkg/huggingface-api/client_test.go +++ b/pkg/huggingface-api/client_test.go @@ -14,28 +14,14 @@ import ( hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" ) -type fakeClock struct { - now time.Time - sleeps []time.Duration -} - -func (c *fakeClock) Now() time.Time { return c.now } - -func (c *fakeClock) Sleep(d time.Duration) { - c.sleeps = append(c.sleeps, d) - c.now = c.now.Add(d) -} - var _ = Describe("HuggingFace API Client", func() { var ( client *hfapi.Client server *httptest.Server - clock *fakeClock ) BeforeEach(func() { - clock = &fakeClock{now: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)} - client = hfapi.NewClient(hfapi.WithClock(clock)) + client = hfapi.NewClient() }) AfterEach(func() { @@ -225,35 +211,14 @@ var _ = Describe("HuggingFace API Client", func() { Search: "GGUF", } + start := time.Now() models, err := client.SearchModels(params) + elapsed := time.Since(start) Expect(err).ToNot(HaveOccurred()) Expect(models).To(HaveLen(0)) Expect(attempts).To(Equal(2)) - Expect(clock.sleeps).To(Equal([]time.Duration{time.Second})) - }) - - It("should calculate HTTP-date Retry-After using the injected clock", func() { - attempts := 0 - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - attempts++ - if attempts == 1 { - w.Header().Set("Retry-After", clock.now.Add(2*time.Second).Format(http.TimeFormat)) - w.WriteHeader(http.StatusTooManyRequests) - return - } - w.Header().Set("Content-Type", "application/json") - _, err := w.Write([]byte("[]")) - Expect(err).ToNot(HaveOccurred()) - })) - client.SetBaseURL(server.URL) - - models, err := client.SearchModels(hfapi.SearchParams{Search: "GGUF"}) - - Expect(err).ToNot(HaveOccurred()) - Expect(models).To(BeEmpty()) - Expect(attempts).To(Equal(2)) - Expect(clock.sleeps).To(Equal([]time.Duration{2 * time.Second})) + Expect(elapsed).To(BeNumerically(">=", 900*time.Millisecond)) }) It("should fail fast on non-retryable 4xx responses", func() { @@ -302,9 +267,6 @@ var _ = Describe("HuggingFace API Client", func() { Expect(errors.Is(err, hfapi.ErrRateLimited)).To(BeTrue()) Expect(err.Error()).To(ContainSubstring("Status code: 429")) Expect(models).To(BeNil()) - Expect(clock.sleeps).To(Equal([]time.Duration{ - time.Second, time.Second, time.Second, time.Second, - })) }) }) @@ -374,12 +336,8 @@ var _ = Describe("HuggingFace API Client", func() { Context("when handling network errors", func() { It("should handle connection failures gracefully", func() { - // A closed loopback listener produces a deterministic connection - // failure without relying on DNS or public network access. - closedServer := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) - closedURL := closedServer.URL - closedServer.Close() - client.SetBaseURL(closedURL) + // Use an invalid URL to simulate connection failure + client.SetBaseURL("http://invalid-url-that-does-not-exist") params := hfapi.SearchParams{ Sort: "lastModified", @@ -393,26 +351,11 @@ var _ = Describe("HuggingFace API Client", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to make request")) Expect(models).To(BeNil()) - Expect(clock.sleeps).To(Equal([]time.Duration{ - time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, - })) }) }) - Context("when getting file SHA from repository metadata", func() { + Context("when getting file SHA on remote model", func() { It("should get file SHA successfully", func() { - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, err := w.Write([]byte(`[{ - "type":"file", - "path":"localai-functioncall-qwen2.5-7b-v0.5-q4_k_m.gguf", - "size":42, - "oid":"pointer-oid", - "lfs":{"oid":"4e7b7fe1d54b881f1ef90799219dc6cc285d29db24f559c8998d1addb35713d4","size":42,"pointerSize":128} - }]`)) - Expect(err).NotTo(HaveOccurred()) - })) - client.SetBaseURL(server.URL + "/api/models") sha, err := client.GetFileSHA( "mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF", "localai-functioncall-qwen2.5-7b-v0.5-q4_k_m.gguf") Expect(err).ToNot(HaveOccurred()) @@ -943,33 +886,14 @@ var _ = Describe("HuggingFace API Client", func() { }) }) - Context("repository API compatibility fixtures", func() { - It("should recursively list all files including subfolders", func() { - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - var response string - switch { - case strings.HasSuffix(r.URL.Path, "/tree/main"): - response = `[ - {"type":"file","path":"README.md","size":100,"oid":"readme-oid"}, - {"type":"directory","path":"Q4_K_M","size":0,"oid":"directory-oid"} - ]` - case strings.HasSuffix(r.URL.Path, "/tree/main/Q4_K_M"): - response = `[ - {"type":"file","path":"Q4_K_M/model-00001-of-00002.gguf","size":1000,"oid":"model-oid"} - ]` - default: - w.WriteHeader(http.StatusNotFound) - return - } - _, err := w.Write([]byte(response)) - Expect(err).NotTo(HaveOccurred()) - })) - fixtureClient := hfapi.NewClient() - fixtureClient.SetBaseURL(server.URL + "/api/models") + Context("integration test with real HuggingFace API", func() { + It("should recursively list all files including subfolders from real repository", func() { + // This test makes actual API calls to HuggingFace + // Skip if running in CI or if network is not available + realClient := hfapi.NewClient() repoID := "bartowski/Qwen_Qwen3-Next-80B-A3B-Instruct-GGUF" - files, err := fixtureClient.ListFiles(repoID) + files, err := realClient.ListFiles(repoID) Expect(err).ToNot(HaveOccurred()) Expect(files).ToNot(BeEmpty(), "should return at least some files") @@ -1032,19 +956,12 @@ var _ = Describe("HuggingFace API Client", func() { }) It("should populate PipelineTag and LibraryName on ModelDetails", func() { - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - if strings.Contains(r.URL.Path, "/tree/main") { - _, err := w.Write([]byte(`[{"type":"file","path":"config.json","size":100,"oid":"config-oid"}]`)) - Expect(err).NotTo(HaveOccurred()) - return - } - _, err := w.Write([]byte(`{"pipeline_tag":"sentence-similarity","library_name":"sentence-transformers"}`)) - Expect(err).NotTo(HaveOccurred()) - })) - fixtureClient := hfapi.NewClient() - fixtureClient.SetBaseURL(server.URL + "/api/models") - details, err := fixtureClient.GetModelDetails("sentence-transformers/all-MiniLM-L6-v2") + // Sentence-transformers/all-MiniLM-L6-v2 is a public, stable repo: + // pipeline_tag: sentence-similarity, library_name: sentence-transformers. + // This exercises the /api/models/{repo} metadata fetch layered on top + // of ListFiles in GetModelDetails. + realClient := hfapi.NewClient() + details, err := realClient.GetModelDetails("sentence-transformers/all-MiniLM-L6-v2") Expect(err).ToNot(HaveOccurred()) Expect(details).ToNot(BeNil()) Expect(details.PipelineTag).To(Equal("sentence-similarity")) diff --git a/pkg/model/loader.go b/pkg/model/loader.go index e2c8c336c..77b7a0ed6 100644 --- a/pkg/model/loader.go +++ b/pkg/model/loader.go @@ -12,7 +12,6 @@ import ( "sync/atomic" "time" - "github.com/mudler/LocalAI/internal/backoff" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" @@ -202,8 +201,17 @@ func (ml *ModelLoader) recordLoadFailure(modelID string) { ml.loadFailures[modelID] = st } st.consecutive++ - delay := backoff.Exponential(ml.loadFailureBaseCooldown, ml.loadFailureMaxCooldown, uint(st.consecutive-1)) - st.cooldownUntil = time.Now().Add(delay) + // base * 2^(consecutive-1), clamped. Cap the shift to avoid overflowing + // the Duration; anything past the cap collapses to loadFailureMaxCooldown. + shift := st.consecutive - 1 + if shift > 20 { + shift = 20 + } + backoff := ml.loadFailureBaseCooldown * (1 << shift) + if backoff <= 0 || backoff > ml.loadFailureMaxCooldown { + backoff = ml.loadFailureMaxCooldown + } + st.cooldownUntil = time.Now().Add(backoff) } // clearLoadFailure resets the modelID's failure state after a successful load. diff --git a/pkg/model/loader_test.go b/pkg/model/loader_test.go index da5b2f037..1a8829431 100644 --- a/pkg/model/loader_test.go +++ b/pkg/model/loader_test.go @@ -65,7 +65,8 @@ var _ = Describe("ModelLoader", func() { BeforeEach(func() { // Setup the model loader with a test directory - modelPath = GinkgoT().TempDir() + modelPath = "/tmp/test_model_path" + os.Mkdir(modelPath, 0755) systemState, err := system.GetSystemState( system.WithModelPath(modelPath), @@ -74,6 +75,11 @@ var _ = Describe("ModelLoader", func() { modelLoader = model.NewModelLoader(systemState) }) + AfterEach(func() { + // Cleanup test directory + os.RemoveAll(modelPath) + }) + Context("NewModelLoader", func() { It("should create a new ModelLoader with an empty model map", func() { Expect(modelLoader).ToNot(BeNil()) diff --git a/pkg/modelartifacts/materializer.go b/pkg/modelartifacts/materializer.go index 005be3871..a3e2a9e8e 100644 --- a/pkg/modelartifacts/materializer.go +++ b/pkg/modelartifacts/materializer.go @@ -20,7 +20,6 @@ import ( "github.com/gofrs/flock" "github.com/mudler/xlog" - "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/downloader" hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" ) @@ -327,7 +326,9 @@ func (m *Manager) acquireLock(ctx context.Context, locker Locker, lockPath strin return ctx.Err() case <-time.After(interval): } - interval = backoff.Exponential(interval, maxLockRetryInterval, 1) + if interval < maxLockRetryInterval { + interval = min(interval*2, maxLockRetryInterval) + } } } diff --git a/pkg/oci/blob.go b/pkg/oci/blob.go index 63aa44d5f..e034c4162 100644 --- a/pkg/oci/blob.go +++ b/pkg/oci/blob.go @@ -16,10 +16,6 @@ import ( ) func FetchImageBlob(ctx context.Context, r, reference, dst string, statusReader func(ocispec.Descriptor) io.Writer) error { - return fetchImageBlob(ctx, r, reference, dst, statusReader, false) -} - -func fetchImageBlob(ctx context.Context, r, reference, dst string, statusReader func(ocispec.Descriptor) io.Writer, plainHTTP bool) error { // 0. Create a file store for the output fs, err := os.Create(dst) if err != nil { @@ -33,7 +29,6 @@ func fetchImageBlob(ctx context.Context, r, reference, dst string, statusReader return fmt.Errorf("failed to create repository: %v", err) } repo.SkipReferrersGC = true - repo.PlainHTTP = plainHTTP // Identify LocalAI to the registry. This mirrors oras' auth.DefaultClient // (same retry policy) but advertises a LocalAI User-Agent instead of the diff --git a/pkg/oci/blob_test.go b/pkg/oci/blob_test.go index 76b7d49b6..cef29a972 100644 --- a/pkg/oci/blob_test.go +++ b/pkg/oci/blob_test.go @@ -1,14 +1,10 @@ -package oci +package oci_test import ( "context" - "crypto/sha256" - "fmt" - "net/http" - "net/http/httptest" "os" - "strings" + . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -16,22 +12,11 @@ import ( var _ = Describe("OCI", func() { Context("pulling images", func() { It("should fetch blobs correctly", func() { - payload := []byte("local OCI blob fixture") - digest := fmt.Sprintf("sha256:%x", sha256.Sum256(payload)) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") - w.Header().Set("Content-Length", fmt.Sprint(len(payload))) - if r.Method != http.MethodHead { - _, _ = w.Write(payload) - } - })) - defer server.Close() f, err := os.CreateTemp("", "ollama") Expect(err).NotTo(HaveOccurred()) defer os.RemoveAll(f.Name()) - err = fetchImageBlob(context.Background(), strings.TrimPrefix(server.URL, "http://")+"/library/gemma", digest, f.Name(), nil, true) + err = FetchImageBlob(context.TODO(), "registry.ollama.ai/library/gemma", "sha256:c1864a5eb19305c40519da12cc543519e48a0697ecd30e15d5ac228644957d12", f.Name(), nil) Expect(err).NotTo(HaveOccurred()) - Expect(os.ReadFile(f.Name())).To(Equal(payload)) }) }) }) diff --git a/pkg/oci/cosignverify/verify.go b/pkg/oci/cosignverify/verify.go index ac1d4fdc6..579b0d8c6 100644 --- a/pkg/oci/cosignverify/verify.go +++ b/pkg/oci/cosignverify/verify.go @@ -34,8 +34,6 @@ import ( "github.com/sigstore/sigstore-go/pkg/root" "github.com/sigstore/sigstore-go/pkg/tuf" "github.com/sigstore/sigstore-go/pkg/verify" - - "github.com/mudler/LocalAI/pkg/httpclient" ) // Policy is the verification policy a backend image must satisfy. @@ -291,7 +289,7 @@ func enforceNotBefore(result *verify.VerificationResult, cutoff time.Time) error func (v *Verifier) remoteOptions(ctx context.Context) []remote.Option { t := v.transport if t == nil { - t = httpclient.HardenedTransport() + t = http.DefaultTransport } // Match the retry policy used elsewhere in pkg/oci so transient // registry hiccups don't fail verification. diff --git a/pkg/oci/image_test.go b/pkg/oci/image_test.go index 300cfc683..447bc90f6 100644 --- a/pkg/oci/image_test.go +++ b/pkg/oci/image_test.go @@ -1,15 +1,10 @@ package oci_test import ( - "archive/tar" - "bytes" "context" "os" - "path/filepath" + "runtime" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/mudler/LocalAI/pkg/oci" . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path . "github.com/onsi/ginkgo/v2" @@ -20,30 +15,25 @@ var _ = Describe("OCI", func() { Context("when template is loaded successfully", func() { It("should evaluate the template correctly", func() { - var layerTar bytes.Buffer - writer := tar.NewWriter(&layerTar) - content := []byte("offline OCI fixture\n") - Expect(writer.WriteHeader(&tar.Header{Name: "fixture.txt", Mode: 0o644, Size: int64(len(content))})).To(Succeed()) - _, err := writer.Write(content) + if runtime.GOOS == "darwin" { + Skip("Skipping test on darwin") + } + imageName := "alpine" + img, err := GetImage(imageName, "", nil, nil) Expect(err).NotTo(HaveOccurred()) - Expect(writer.Close()).To(Succeed()) - layer, err := tarball.LayerFromReader(bytes.NewReader(layerTar.Bytes())) + size, err := GetOCIImageSize(imageName, "", nil, nil) Expect(err).NotTo(HaveOccurred()) - img, err := mutate.AppendLayers(empty.Image, layer) - Expect(err).NotTo(HaveOccurred()) - size, err := layer.Size() - Expect(err).NotTo(HaveOccurred()) - Expect(size).To(BeNumerically(">", 0)) + + Expect(size).ToNot(Equal(int64(0))) // Create tempdir dir, err := os.MkdirTemp("", "example") Expect(err).NotTo(HaveOccurred()) - DeferCleanup(os.RemoveAll, dir) + defer os.RemoveAll(dir) - err = ExtractOCIImage(context.TODO(), img, "fixture:offline", dir, nil) + err = ExtractOCIImage(context.TODO(), img, imageName, dir, nil) Expect(err).NotTo(HaveOccurred()) - Expect(os.ReadFile(filepath.Join(dir, "fixture.txt"))).To(Equal(content)) }) }) }) diff --git a/pkg/oci/ollama.go b/pkg/oci/ollama.go index 42bf64bfc..f0a874013 100644 --- a/pkg/oci/ollama.go +++ b/pkg/oci/ollama.go @@ -35,10 +35,6 @@ type LayerDetail struct { } func OllamaModelManifest(image string) (*Manifest, error) { - return ollamaModelManifest("https", "registry.ollama.ai", image) -} - -func ollamaModelManifest(scheme, registry, image string) (*Manifest, error) { // parse the repository and tag from `image`. `image` should be for e.g. gemma:2b, or foobar/gemma:2b // if there is a : in the image, then split it @@ -46,7 +42,7 @@ func ollamaModelManifest(scheme, registry, image string) (*Manifest, error) { tag, repository, image := ParseImageParts(image) // get e.g. https://registry.ollama.ai/v2/library/llama3/manifests/latest - req, err := http.NewRequest("GET", scheme+"://"+registry+"/v2/"+repository+"/"+image+"/manifests/"+tag, nil) + req, err := http.NewRequest("GET", "https://registry.ollama.ai/v2/"+repository+"/"+image+"/manifests/"+tag, nil) if err != nil { return nil, err } @@ -69,11 +65,7 @@ func ollamaModelManifest(scheme, registry, image string) (*Manifest, error) { } func OllamaModelBlob(image string) (string, error) { - return ollamaModelBlob("https", "registry.ollama.ai", image) -} - -func ollamaModelBlob(scheme, registry, image string) (string, error) { - manifest, err := ollamaModelManifest(scheme, registry, image) + manifest, err := OllamaModelManifest(image) if err != nil { return "", err } @@ -89,16 +81,12 @@ func ollamaModelBlob(scheme, registry, image string) (string, error) { } func OllamaFetchModel(ctx context.Context, image string, output string, statusWriter func(ocispec.Descriptor) io.Writer) error { - return ollamaFetchModel(ctx, "https", "registry.ollama.ai", image, output, statusWriter) -} - -func ollamaFetchModel(ctx context.Context, scheme, registry, image string, output string, statusWriter func(ocispec.Descriptor) io.Writer) error { _, repository, imageNoTag := ParseImageParts(image) - blobID, err := ollamaModelBlob(scheme, registry, image) + blobID, err := OllamaModelBlob(image) if err != nil { return err } - return fetchImageBlob(ctx, fmt.Sprintf("%s/%s/%s", registry, repository, imageNoTag), blobID, output, statusWriter, scheme == "http") + return FetchImageBlob(ctx, fmt.Sprintf("registry.ollama.ai/%s/%s", repository, imageNoTag), blobID, output, statusWriter) } diff --git a/pkg/oci/ollama_test.go b/pkg/oci/ollama_test.go index bed92a19c..fbda69e6b 100644 --- a/pkg/oci/ollama_test.go +++ b/pkg/oci/ollama_test.go @@ -1,15 +1,10 @@ -package oci +package oci_test import ( "context" - "crypto/sha256" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" "os" - "strings" + . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -17,26 +12,11 @@ import ( var _ = Describe("OCI", func() { Context("ollama", func() { It("pulls model files", func() { - payload := []byte("local Ollama model fixture") - digest := fmt.Sprintf("sha256:%x", sha256.Sum256(payload)) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") - if strings.Contains(r.URL.Path, "/manifests/") { - _ = json.NewEncoder(w).Encode(Manifest{SchemaVersion: 2, Layers: []LayerDetail{{Digest: digest, MediaType: "application/vnd.ollama.image.model", Size: len(payload)}}}) - return - } - w.Header().Set("Content-Length", fmt.Sprint(len(payload))) - if r.Method != http.MethodHead { - _, _ = w.Write(payload) - } - })) - defer server.Close() f, err := os.CreateTemp("", "ollama") Expect(err).NotTo(HaveOccurred()) defer os.RemoveAll(f.Name()) - err = ollamaFetchModel(context.Background(), "http", strings.TrimPrefix(server.URL, "http://"), "gemma:2b", f.Name(), nil) + err = OllamaFetchModel(context.TODO(), "gemma:2b", f.Name(), nil) Expect(err).NotTo(HaveOccurred()) - Expect(os.ReadFile(f.Name())).To(Equal(payload)) }) }) }) diff --git a/pkg/testnetwork/guard.go b/pkg/testnetwork/guard.go deleted file mode 100644 index 17db55c0e..000000000 --- a/pkg/testnetwork/guard.go +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-License-Identifier: MIT - -// Package testnetwork provides an explicit outbound-network guard for tests. -package testnetwork - -import ( - "context" - "fmt" - "net" - "net/netip" - "strings" -) - -type Guard struct { - Dialer net.Dialer - Dial func(context.Context, string, string) (net.Conn, error) - Allowed []netip.Prefix -} - -func LocalGuard() *Guard { - prefixes := []string{"127.0.0.0/8", "::1/128", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"} - guard := &Guard{} - for _, prefix := range prefixes { - guard.Allowed = append(guard.Allowed, netip.MustParsePrefix(prefix)) - } - return guard -} - -func (g *Guard) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - originalAddress := address - host, _, err := net.SplitHostPort(address) - if err != nil { - return nil, fmt.Errorf("test network guard: invalid address %q: %w", address, err) - } - addresses, err := net.DefaultResolver.LookupNetIP(ctx, "ip", strings.Trim(host, "[]")) - if err != nil { - return nil, fmt.Errorf("test network guard: resolve %q: %w", host, err) - } - for _, resolved := range addresses { - if !g.allowed(resolved.Unmap()) { - return nil, fmt.Errorf("test network guard: public dial blocked: %s (%s)", host, resolved) - } - } - if g.Dial != nil { - return g.Dial(ctx, network, originalAddress) - } - return g.Dialer.DialContext(ctx, network, originalAddress) -} - -func (g *Guard) allowed(address netip.Addr) bool { - for _, prefix := range g.Allowed { - if prefix.Contains(address) { - return true - } - } - return false -} diff --git a/pkg/testnetwork/guard_suite_test.go b/pkg/testnetwork/guard_suite_test.go deleted file mode 100644 index 9259f5a64..000000000 --- a/pkg/testnetwork/guard_suite_test.go +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: MIT - -package testnetwork_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestNetworkGuard(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Test network guard suite") -} diff --git a/pkg/testnetwork/guard_test.go b/pkg/testnetwork/guard_test.go deleted file mode 100644 index 5f658eb19..000000000 --- a/pkg/testnetwork/guard_test.go +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: MIT - -package testnetwork_test - -import ( - "context" - "errors" - "net" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/mudler/LocalAI/pkg/testnetwork" -) - -var _ = Describe("Guard", func() { - It("blocks a public IP before dialing", func() { - _, err := testnetwork.LocalGuard().DialContext(context.Background(), "tcp", "203.0.113.1:443") - Expect(err).To(MatchError(ContainSubstring("public dial blocked"))) - }) - - It("allows loopback fixtures", func() { - guard := testnetwork.LocalGuard() - called := false - guard.Dial = func(_ context.Context, network, address string) (net.Conn, error) { - called = true - Expect(network).To(Equal("tcp")) - Expect(address).To(Equal("127.0.0.1:8080")) - return nil, errors.New("fixture dial sentinel") - } - _, err := guard.DialContext(context.Background(), "tcp", "127.0.0.1:8080") - Expect(err).To(MatchError("fixture dial sentinel")) - Expect(called).To(BeTrue()) - }) -}) diff --git a/scripts/jetson-wheels-sync.py b/scripts/jetson-wheels-sync.py deleted file mode 100644 index fd41c623e..000000000 --- a/scripts/jetson-wheels-sync.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python3 -"""Mirror the jetson-only wheel subset from pypi.jetson-ai-lab.io. - -Downloads the wheels for the packages listed in .github/jetson-wheels.json -(one list per JetPack index path) into a local directory laid out exactly -like the upstream index (///). The jetson-wheels -CI workflow publishes that directory as a scratch OCI image on ghcr, and -backend builds serve it as a local package index during pip install — see -pypi_mirror_server.py and installRequirements in -backend/python/common/libbackend.sh for the consuming side and the -motivation (recurring multi-hour upstream outages). - -The sync is additive-with-pruning against a *successfully fetched* project -page: files no longer listed upstream are removed, but a package whose page -cannot be fetched is left exactly as mirrored last time. When the whole -upstream is unreachable the existing mirror is kept as-is (exit 0) so a CI -run during an outage never destroys the last known-good wheels; it only -fails (exit 2) when upstream is down AND there is nothing mirrored yet, -i.e. the bootstrap run has nothing to publish. - -Standard library only. Usage: - python3 scripts/jetson-wheels-sync.py --config .github/jetson-wheels.json \ - --index jp6/cu129 --dest wheels [--changed-file /tmp/changed] -""" - -import argparse -import hashlib -import html.parser -import json -import os -import re -import sys -import urllib.error -import urllib.parse -import urllib.request - -DIST_SUFFIXES = (".whl", ".tar.gz", ".zip") -TIMEOUT = 120 - - -def normalize(name): - """PEP 503 project-name normalization.""" - return re.sub(r"[-_.]+", "-", name).lower() - - -class _LinkParser(html.parser.HTMLParser): - def __init__(self): - super().__init__() - self.hrefs = [] - - def handle_starttag(self, tag, attrs): - if tag == "a": - for key, value in attrs: - if key == "href" and value: - self.hrefs.append(value) - - -def parse_links(page, base_url): - """Extract (filename, absolute_url, sha256|None) for each dist link.""" - parser = _LinkParser() - parser.feed(page) - links = [] - for href in parser.hrefs: - split = urllib.parse.urlsplit(href) - filename = os.path.basename(urllib.parse.unquote(split.path)) - if not filename.endswith(DIST_SUFFIXES): - continue - sha256 = None - if split.fragment.startswith("sha256="): - sha256 = split.fragment[len("sha256="):] - url = urllib.parse.urljoin(base_url, split._replace(fragment="").geturl()) - links.append((filename, url, sha256)) - return links - - -def _fetch(url): - request = urllib.request.Request(url, headers={"User-Agent": "localai-jetson-wheels-sync"}) - return urllib.request.urlopen(request, timeout=TIMEOUT) - - -def _download(url, dest_path, sha256): - digest = hashlib.sha256() - tmp = dest_path + ".tmp" - with _fetch(url) as resp, open(tmp, "wb") as out: - for chunk in iter(lambda: resp.read(1 << 20), b""): - digest.update(chunk) - out.write(chunk) - if sha256 and digest.hexdigest() != sha256: - os.unlink(tmp) - raise RuntimeError(f"sha256 mismatch for {url}: expected {sha256}, got {digest.hexdigest()}") - os.replace(tmp, dest_path) - - -def _has_wheels(dest): - for _, _, files in os.walk(dest): - if any(f.endswith(DIST_SUFFIXES) for f in files): - return True - return False - - -def sync_package(base_url, package, dest_dir): - """Returns (fetched_ok, changed).""" - page_url = urllib.parse.urljoin(base_url, normalize(package) + "/") - try: - with _fetch(page_url) as resp: - page = resp.read().decode("utf-8", "replace") - except urllib.error.HTTPError as err: - if err.code == 404: - # Upstream simply doesn't host this package for this index — - # normal (the config list is a superset across JetPack versions). - print(f" {package}: not hosted upstream (404), skipping") - return True, False - print(f" {package}: upstream error {err.code}, keeping mirrored files") - return False, False - except (urllib.error.URLError, TimeoutError, OSError) as err: - print(f" {package}: upstream unreachable ({err}), keeping mirrored files") - return False, False - - links = parse_links(page, page_url) - listed = {name for name, _, _ in links} - changed = False - os.makedirs(dest_dir, exist_ok=True) - for name, url, sha256 in links: - path = os.path.join(dest_dir, name) - if os.path.exists(path): - continue - print(f" {package}: downloading {name}") - _download(url, path, sha256) - changed = True - # Prune only against a page we actually fetched: upstream removing a - # wheel is tracked, an outage never empties the mirror. - for existing in os.listdir(dest_dir): - if existing.endswith(DIST_SUFFIXES) and existing not in listed: - print(f" {package}: pruning {existing} (no longer listed upstream)") - os.unlink(os.path.join(dest_dir, existing)) - changed = True - return True, changed - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--config", required=True) - parser.add_argument("--index", required=True, help="index path, e.g. jp6/cu129") - parser.add_argument("--dest", required=True) - parser.add_argument("--upstream", help="override the config's upstream (for tests)") - parser.add_argument("--changed-file", help="created iff the mirror content changed") - args = parser.parse_args() - - with open(args.config) as f: - config = json.load(f) - packages = config["indexes"][args.index] - upstream = args.upstream or config["upstream"] - base_url = upstream.rstrip("/") + "/" + args.index.strip("/") + "/" - - print(f"syncing {args.index} from {base_url}: {', '.join(packages)}") - any_fetched = False - any_changed = False - for package in packages: - dest_dir = os.path.join(args.dest, args.index, normalize(package)) - fetched, changed = sync_package(base_url, package, dest_dir) - any_fetched = any_fetched or fetched - any_changed = any_changed or changed - - if not any_fetched: - if _has_wheels(os.path.join(args.dest, args.index)): - print("upstream unreachable; keeping existing mirror unchanged") - return 0 - print("upstream unreachable and nothing mirrored yet — nothing to publish") - return 2 - if any_changed and args.changed_file: - with open(args.changed_file, "w") as f: - f.write("changed\n") - print("sync complete" + (" (changes)" if any_changed else " (no changes)")) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/run-coverage.sh b/scripts/run-coverage.sh index 771fa4242..430ca1f11 100755 --- a/scripts/run-coverage.sh +++ b/scripts/run-coverage.sh @@ -21,13 +21,6 @@ # "!real-models" (those specs need a downloaded model). # COVERAGE_EXCLUDE_RE egrep pattern of profile lines to drop before merging, # e.g. generated protobuf (grpc/proto/.*\.pb\.go). -# COVERAGE_PROCS parallel Ginkgo processes; 0 lets Ginkgo detect CPUs. -# COVERAGE_SUITE_TIMEOUT maximum duration of each recursive root (default 5m). -# COVERAGE_PROGRESS_AFTER emit diagnostics when a spec is slow (default 30s). -# -# Verbose Ginkgo output is retained in OUTPUT_DIR/logs. The previous run's log -# for each root is kept with a .previous suffix, so a noisy failure remains -# available without flooding the commit-hook output. # # Why one ginkgo invocation per root: passing several recursive roots to a # single ginkgo run only merges ONE root's coverprofile into --output-dir @@ -48,43 +41,11 @@ shift 3 unit_roots="$*" # space-free tokens (./pkg ./core) mkdir -p "$out_dir" -lock_dir="$out_dir/.run-coverage.lock" -if ! mkdir "$lock_dir" 2>/dev/null; then - echo "run-coverage: another coverage run is using $out_dir" >&2 - echo "run-coverage: wait for it to finish; if none is running, remove stale lock $lock_dir" >&2 - exit 2 -fi -cleanup() { - for root in $unit_roots ${COVERAGE_E2E_ROOTS:-}; do - # --keep-separate-coverprofiles leaves Go's package test binaries behind. - # They are deterministic runner artifacts, never source inputs. - find "$root" -type f -name '*.test' -delete 2>/dev/null || : - done - rmdir "$lock_dir" 2>/dev/null || : -} -trap cleanup EXIT -trap 'exit 130' HUP INT TERM - -log_dir="$out_dir/logs" -mkdir -p "$log_dir" # Clear per-root profiles from a previous run: the merge collects them by glob, # so a stale profile (e.g. from a root that failed to rebuild this run) must not # leak into the merged result. rm -f "$out_dir"/cover-*.out -rm -f "$out_dir"/*_cover-*.out -rm -f "$merged" fail=0 -timings="$out_dir/timings.tsv" -: > "$timings" -run_started="$(date +%s)" - -procs="${COVERAGE_PROCS:-0}" -suite_timeout="${COVERAGE_SUITE_TIMEOUT:-5m}" -progress_after="${COVERAGE_PROGRESS_AFTER:-30s}" -parallel_flags="-p --keep-going --timeout=$suite_timeout --poll-progress-after=$progress_after --poll-progress-interval=10s" -if [ "$procs" -gt 0 ] 2>/dev/null; then - parallel_flags="$parallel_flags --procs=$procs --compilers=$procs" -fi # Common optional flags go into "$@"; unquoted ${VAR:+...} would word-split a # --tags value that contains a space. The unit roots were captured above, so @@ -98,133 +59,26 @@ profile_name() { printf 'cover-%s.out' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')" } -log_name() { - printf '%s.log' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')" -} - -rotate_log() { - log="$1" - if [ -f "$log" ]; then - mv -f "$log" "$log.previous" - fi -} - -# Ginkgo's recursive-run merger can fail after every suite has passed when a -# large --coverpkg run produces many profiles. Keep its profiles separate and -# merge them here using the same block-summing rule as the cross-root merge. -consolidate_root_profiles() { - base="$1" - set -- "$out_dir"/*_"$base" - if [ ! -e "$1" ]; then - echo "run-coverage: no per-package profiles produced for $base" >&2 - return 1 - fi - tmp="$out_dir/.${base}.tmp" - { - echo "mode: atomic" - awk ' - /^mode:/ { next } - { stmts[$1] = $2; cnt[$1] += $3 } - END { for (k in stmts) print k, stmts[k], cnt[k] } - ' "$@" - } > "$tmp" - mv "$tmp" "$out_dir/$base" - rm -f "$@" -} - -report_failure() { - root="$1" - log="$2" - echo "run-coverage: FAIL — tests under coverage failed for $root" >&2 - echo "run-coverage: full output: $log" >&2 - echo "run-coverage: relevant tail:" >&2 - # Keep the terminal useful even when Ginkgo emits thousands of verbose lines. - # The complete log remains available when this short extract is insufficient. - summary="$(grep -E 'Summarizing|\[FAIL(ED)?\]|FAIL!|--- FAIL:|Test Suite Failed|could not finalize|Status code: 429|HTTP 429|rate limit|timed out|panic:|fork/exec|no such file or directory|Expected.*(but got|success)' "$log" \ - | tail -n 30)" - if [ -n "$summary" ]; then - printf '%s\n' "$summary" >&2 - else - tail -n 30 "$log" >&2 - fi -} - -record_timing() { - root="$1" - started="$2" - elapsed="$(( $(date +%s) - started ))" - printf '%s\t%s\n' "$root" "$elapsed" >> "$timings" - echo "run-coverage: TIMING — $root ${elapsed}s" -} - -print_timing_summary() { - echo "run-coverage: wall-clock summary" - sort -t "$(printf '\t')" -k2,2nr "$timings" | awk -F '\t' '{ printf " %5ss %s\n", $2, $1 }' - echo " $(( $(date +%s) - run_started ))s total" - echo "run-coverage: slowest specs/hooks taking at least ${COVERAGE_SLOW_SPEC_THRESHOLD:-3}s (up to ${COVERAGE_SLOW_SPEC_LIMIT:-25} per root)" - found=0 - while IFS="$(printf '\t')" read -r root elapsed; do - log="$log_dir/$(log_name "$root")" - entries="$(scripts/summarize-ginkgo-waits.sh "${COVERAGE_SLOW_SPEC_THRESHOLD:-3}" "$root" "$log" "${COVERAGE_SLOW_SPEC_LIMIT:-25}")" - if [ -n "$entries" ]; then - printf '%s\n' "$entries" - found=1 - fi - done < "$timings" - [ "$found" -eq 1 ] || echo " none" -} - # Unit/suite roots: recursive. for root in $unit_roots; do base="$(profile_name "$root")" - log="$log_dir/$(log_name "$root")" - rotate_log "$log" - echo "run-coverage: testing $root (full output: $log)" - started="$(date +%s)" - # parallel_flags is intentionally word-split: it contains CLI arguments only. - # shellcheck disable=SC2086 - go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --keep-separate-coverprofiles --flake-attempts "$flakes" -v -r "$@" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ - && consolidate_root_profiles "$base" \ - && echo "run-coverage: PASS — $root" \ - || { fail=1; report_failure "$root" "$log"; } - record_timing "$root" "$started" + go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v -r "$@" \ + --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 done # In-process integration roots: NON-recursive + optional label filter. for root in ${COVERAGE_E2E_ROOTS:-}; do base="$(profile_name "$root")" - log="$log_dir/$(log_name "$root")" - rotate_log "$log" - echo "run-coverage: testing $root (full output: $log)" - started="$(date +%s)" if [ -n "${COVERAGE_E2E_LABELS:-}" ]; then - # shellcheck disable=SC2086 - go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --keep-separate-coverprofiles --flake-attempts "$flakes" -v "$@" \ + go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \ --label-filter="$COVERAGE_E2E_LABELS" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ - && consolidate_root_profiles "$base" \ - && echo "run-coverage: PASS — $root" \ - || { fail=1; report_failure "$root" "$log"; } + --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 else - # shellcheck disable=SC2086 - go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --keep-separate-coverprofiles --flake-attempts "$flakes" -v "$@" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ - && consolidate_root_profiles "$base" \ - && echo "run-coverage: PASS — $root" \ - || { fail=1; report_failure "$root" "$log"; } + go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \ + --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 fi - record_timing "$root" "$started" done -print_timing_summary - -if [ "$fail" -ne 0 ]; then - echo "run-coverage: FAILED — one or more test suites failed; no merged profile was produced." >&2 - echo "run-coverage: the coverage percentage ratchet was not run." >&2 - exit "$fail" -fi - # Collect the per-root profiles by glob (space-safe, no list to track). set -- "$out_dir"/cover-*.out if [ ! -e "$1" ]; then @@ -244,4 +98,4 @@ fi ' "$@" } > "$merged" -echo "run-coverage: all test suites passed; merged profile: $merged" +exit "$fail" diff --git a/scripts/run-test-linux-offline.sh b/scripts/run-test-linux-offline.sh deleted file mode 100755 index b2392aa7e..000000000 --- a/scripts/run-test-linux-offline.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: MIT -set -euo pipefail - -if [[ $(uname -s) != Linux ]]; then - echo 'kernel-level test egress enforcement is Linux-only' >&2 - exit 2 -fi -if [[ $# -lt 2 ]]; then - echo "usage: $0 TARGET COMMAND [ARG...]" >&2 - exit 2 -fi - -root=$(cd "$(dirname "$0")/.." && pwd) -group="localai-test-$$" -cgroup="/sys/fs/cgroup/$group" -parent_cgroup="/sys/fs/cgroup$(awk -F: '$1 == "0" {print $3}' /proc/self/cgroup)" -chain="LAI_TEST_$$" - -sudo mkdir "$cgroup" -cleanup() { - echo $$ | sudo tee "$parent_cgroup/cgroup.procs" >/dev/null 2>&1 || true - sudo iptables -D OUTPUT -m cgroup --path "$group" -j "$chain" 2>/dev/null || true - sudo iptables -F "$chain" 2>/dev/null || true - sudo iptables -X "$chain" 2>/dev/null || true - sudo ip6tables -D OUTPUT -m cgroup --path "$group" -j "$chain" 2>/dev/null || true - sudo ip6tables -F "$chain" 2>/dev/null || true - sudo ip6tables -X "$chain" 2>/dev/null || true - sudo rmdir "$cgroup" 2>/dev/null || true -} -trap cleanup EXIT INT TERM - -sudo iptables -N "$chain" -sudo iptables -A "$chain" -d 127.0.0.0/8 -j ACCEPT -sudo iptables -A "$chain" -d 10.0.0.0/8 -j ACCEPT -sudo iptables -A "$chain" -d 172.16.0.0/12 -j ACCEPT -sudo iptables -A "$chain" -d 192.168.0.0/16 -j ACCEPT -sudo iptables -A "$chain" -j REJECT -sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -j "$chain" - -sudo ip6tables -N "$chain" -sudo ip6tables -A "$chain" -d ::1/128 -j ACCEPT -sudo ip6tables -A "$chain" -j REJECT -sudo ip6tables -I OUTPUT 1 -m cgroup --path "$group" -j "$chain" -echo $$ | sudo tee "$cgroup/cgroup.procs" >/dev/null - -set +e -LOCALAI_TEST_KERNEL_ACTIVE=1 "$root/scripts/run-test-offline.sh" "$@" -command_status=$? -set -e - -# The final rule in each private chain is the external-egress REJECT. Reading -# its counter makes even an optional probe that swallows its network error a -# hard, visible test failure. Report aggregate counters only: packet-by-packet -# kernel logging can itself exhaust CI logs when a client retries aggressively. -ipv4_blocked=$(sudo iptables -L "$chain" -v -x -n | awk '$3 == "REJECT" { print $1; exit }') -ipv6_blocked=$(sudo ip6tables -L "$chain" -v -x -n | awk '$3 == "REJECT" { print $1; exit }') -ipv4_blocked=${ipv4_blocked:-0} -ipv6_blocked=${ipv6_blocked:-0} - -if (( ipv4_blocked > 0 || ipv6_blocked > 0 )); then - { - echo 'offline-test: external network access was attempted and blocked' - echo "offline-test: blocked packets: ipv4=$ipv4_blocked ipv6=$ipv6_blocked" - echo 'offline-test: replace real network clients with fixtures or injected test doubles' - } >&2 - exit 1 -fi - -exit "$command_status" diff --git a/scripts/run-test-offline.sh b/scripts/run-test-offline.sh deleted file mode 100755 index 614ba71be..000000000 --- a/scripts/run-test-offline.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: MIT -set -euo pipefail - -if [[ $# -lt 2 ]]; then - echo "usage: $0 TARGET COMMAND [ARG...]" >&2 - exit 2 -fi - -target=$1 -shift -root=$(cd "$(dirname "$0")/.." && pwd) - -if [[ ${LOCALAI_TEST_KERNEL_ENFORCE:-0} == 1 && ${LOCALAI_TEST_KERNEL_ACTIVE:-0} != 1 ]]; then - exec "$root/scripts/run-test-linux-offline.sh" "$target" "$@" -fi - -exec go run "$root/cmd/test-resources" run "$target" \ - "$root/test-resources/manifests" "${TEST_RESOURCE_CACHE:-$root/.cache/test-resources}" -- "$@" diff --git a/scripts/summarize-ginkgo-waits.sh b/scripts/summarize-ginkgo-waits.sh deleted file mode 100755 index 545e55340..000000000 --- a/scripts/summarize-ginkgo-waits.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env sh -# summarize-ginkgo-waits.sh THRESHOLD_SECONDS ROOT LOG [LIMIT] -set -eu - -threshold="${1:?missing threshold in seconds}" -root="${2:?missing test root}" -log="${3:?missing Ginkgo log}" -limit="${4:-25}" -case "$limit" in - ''|*[!0-9]*) echo "limit must be a non-negative integer" >&2; exit 2 ;; -esac - -# Strip terminal colour sequences, then report slow specs and hooks. This -# catches sleeps, polling, channel waits, teardown and any other idle time -# without unsafe attempts to replace Go's process-wide clock primitives. -sed 's/\[[0-9;]*[[:alpha:]]//g' "$log" | awk -v threshold="$threshold" -v root="$root" ' - /\[[0-9]+([.][0-9]+)? seconds\]/ { - line = $0 - sub(/^.*\[/, "", line) - sub(/ seconds\].*$/, "", line) - seconds = line + 0 - if (seconds < threshold) next - description = "(description unavailable)" - location = "(location unavailable)" - if (getline > 0) description = $0 - if (getline > 0) location = $0 - printf "%010.3f\t%-18s\t%s\t%s\n", seconds, root, description, location - } -' | sort -t "$(printf '\t')" -k1,1nr | sed -n "1,${limit}p" | awk -F '\t' '{ printf " %7.3fs %s %s %s\n", $1 + 0, $2, $3, $4 }' diff --git a/test-resources/manifests/aio.json b/test-resources/manifests/aio.json deleted file mode 100644 index c21ff9a97..000000000 --- a/test-resources/manifests/aio.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "version": 1, - "target": "aio", - "files": [ - { - "url": "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav", - "sha256": "37de21902b32aa2fc147ccbfdcc0566cc7061fffb2c0b10874f05147c0b9de0f", - "destination": "audio/micro-machines.wav", - "environment": "AIO_AUDIO_FIXTURE" - } - ] -} diff --git a/test-resources/manifests/backend.json b/test-resources/manifests/backend.json deleted file mode 100644 index 178b8b9c5..000000000 --- a/test-resources/manifests/backend.json +++ /dev/null @@ -1 +0,0 @@ -{"version":1,"target":"backend"} diff --git a/test-resources/manifests/default-darwin.json b/test-resources/manifests/default-darwin.json deleted file mode 100644 index 1dc5aed03..000000000 --- a/test-resources/manifests/default-darwin.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "version": 1, - "target": "default-darwin", - "http": [], - "images": [], - "files": [] -} diff --git a/test-resources/manifests/default.json b/test-resources/manifests/default.json deleted file mode 100644 index 1acaac816..000000000 --- a/test-resources/manifests/default.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "version": 1, - "target": "default", - "images": [ - { - "reference": "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20", - "sha256": "f4e8a437601f09ad619c6a8df831cd71c24c5c2276652e6ce89c042185759ed9" - } - ] -} diff --git a/test-resources/manifests/distributed-e2e.json b/test-resources/manifests/distributed-e2e.json deleted file mode 100644 index 297bf9d6b..000000000 --- a/test-resources/manifests/distributed-e2e.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "version": 1, - "target": "distributed-e2e", - "images": [ - { - "reference": "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777", - "sha256": "c8d5971aa1c74f0130dcde5ab942e3613d2179cb62cda4e9c08e8ec4c7252220" - }, - { - "reference": "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0", - "sha256": "9e833c05b393c5ac68a06ab348ce45aac9a7cd72a19cce7ed5e41fb3af56c423" - } - ] -} diff --git a/test-resources/manifests/external-probes.json b/test-resources/manifests/external-probes.json deleted file mode 100644 index af7a85e52..000000000 --- a/test-resources/manifests/external-probes.json +++ /dev/null @@ -1 +0,0 @@ -{"version":1,"target":"external-probes"} diff --git a/test-resources/manifests/hardware.json b/test-resources/manifests/hardware.json deleted file mode 100644 index cddf8b0c3..000000000 --- a/test-resources/manifests/hardware.json +++ /dev/null @@ -1 +0,0 @@ -{"version":1,"target":"hardware"} diff --git a/test-resources/manifests/lock.json b/test-resources/manifests/lock.json deleted file mode 100644 index 623021f9e..000000000 --- a/test-resources/manifests/lock.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "version": 1, - "bundles": { - "aio": "sha256:03b05eedb51c853b0f05f2f3f592e2edc210e337388d3ff5a766680c0a166e55", - "backend": "embedded", - "default": "sha256:533e2744151ce8a4df3a9b0ba073222135f17c3e2ee0e5cfeddfb836f41efbca", - "default-darwin": "embedded", - "distributed-e2e": "sha256:524db2b4cedb091c0604eaf6dcd7b0d73fd84e3b262405b6cdb98df099f54965", - "external-probes": "embedded", - "hardware": "embedded" - } -} diff --git a/tests/e2e-aio/e2e_suite_test.go b/tests/e2e-aio/e2e_suite_test.go index 321b9b5ab..f82b7c5e7 100644 --- a/tests/e2e-aio/e2e_suite_test.go +++ b/tests/e2e-aio/e2e_suite_test.go @@ -6,13 +6,14 @@ import ( "os" "runtime" "testing" + "time" - "github.com/mudler/LocalAI/internal/testfixtures" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/openai/openai-go/v3" "github.com/openai/openai-go/v3/option" "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" ) var container testcontainers.Container @@ -38,10 +39,10 @@ var _ = BeforeSuite(func() { if apiEndpoint == "" { startDockerImage() - apiAddress, err := testfixtures.ContainerEndpoint(context.Background(), container, defaultApiPort) + apiPort, err := container.MappedPort(context.Background(), defaultApiPort) Expect(err).To(Not(HaveOccurred())) - apiEndpoint = "http://" + apiAddress + "/v1" + apiEndpoint = "http://localhost:" + apiPort.Port() + "/v1" // So that other tests can reference this value safely. } else { GinkgoWriter.Printf("docker apiEndpoint set from env: %q\n", apiEndpoint) } @@ -121,16 +122,15 @@ func startDockerImage() { Target: "/backends", }, }, + WaitingFor: wait.ForAll( + wait.ForListeningPort(defaultApiPort).WithStartupTimeout(10*time.Minute), + wait.ForHTTP("/v1/models").WithPort(defaultApiPort).WithStartupTimeout(10*time.Minute), + ), } GinkgoWriter.Printf("Launching Docker Container %s:%s\n", containerImage, containerImageTag) ctx := context.Background() - imageReference := fmt.Sprintf("%s:%s", containerImage, containerImageTag) - Expect(testfixtures.RequireImage(ctx, imageReference, "aio")).To(Succeed()) - testNetwork, err := testfixtures.DockerNetwork() - Expect(err).NotTo(HaveOccurred()) - req.Networks = []string{testNetwork} c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, Started: true, diff --git a/tests/e2e-aio/e2e_test.go b/tests/e2e-aio/e2e_test.go index 338288593..6472b5d63 100644 --- a/tests/e2e-aio/e2e_test.go +++ b/tests/e2e-aio/e2e_test.go @@ -3,13 +3,11 @@ package e2e_test import ( "bytes" "context" - "encoding/base64" "encoding/json" "fmt" "io" "net/http" "os" - "path/filepath" "github.com/mudler/LocalAI/core/schema" . "github.com/onsi/ginkgo/v2" @@ -259,9 +257,6 @@ var _ = Describe("E2E test", func() { Context("vision", func() { It("correctly", func() { - image, err := os.ReadFile(filepath.Join("..", "..", "core", "http", "static", "logo.png")) - Expect(err).NotTo(HaveOccurred()) - imageURI := "data:image/png;base64," + base64.StdEncoding.EncodeToString(image) model := "gpt-4o" resp, err := client.Chat.Completions.New(context.TODO(), openai.ChatCompletionNewParams{ @@ -281,7 +276,7 @@ var _ = Describe("E2E test", func() { { OfImageURL: &openai.ChatCompletionContentPartImageParam{ ImageURL: openai.ChatCompletionContentPartImageImageURLParam{ - URL: imageURI, + URL: "https://picsum.photos/id/22/4434/3729", Detail: "low", }, }, @@ -294,7 +289,7 @@ var _ = Describe("E2E test", func() { }) Expect(err).ToNot(HaveOccurred()) Expect(len(resp.Choices)).To(Equal(1), fmt.Sprint(resp)) - Expect(resp.Choices[0].Message.Content).NotTo(BeEmpty()) + Expect(resp.Choices[0].Message.Content).To(Or(ContainSubstring("man"), ContainSubstring("road")), fmt.Sprint(resp.Choices[0].Message.Content)) }) }) @@ -315,7 +310,11 @@ var _ = Describe("E2E test", func() { Context("audio to text", func() { It("correctly", func() { - fileHandle, err := os.Open(preparedAudioFixture()) + downloadURL := "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav" + file, err := downloadHttpFile(downloadURL) + Expect(err).ToNot(HaveOccurred()) + + fileHandle, err := os.Open(file) Expect(err).ToNot(HaveOccurred()) defer fileHandle.Close() @@ -329,7 +328,11 @@ var _ = Describe("E2E test", func() { }) It("with VTT format", func() { - fileHandle, err := os.Open(preparedAudioFixture()) + downloadURL := "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav" + file, err := downloadHttpFile(downloadURL) + Expect(err).ToNot(HaveOccurred()) + + fileHandle, err := os.Open(file) Expect(err).ToNot(HaveOccurred()) defer fileHandle.Close() @@ -440,10 +443,25 @@ var _ = Describe("E2E test", func() { }) }) -func preparedAudioFixture() string { - path := os.Getenv("AIO_AUDIO_FIXTURE") - Expect(path).NotTo(BeEmpty(), "run `make prepare-offline-test-cache TEST_RESOURCE_SET=aio` before the AIO suite") - return path +func downloadHttpFile(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + tmpfile, err := os.CreateTemp("", "example") + if err != nil { + return "", err + } + defer tmpfile.Close() + + _, err = io.Copy(tmpfile, resp.Body) + if err != nil { + return "", err + } + + return tmpfile.Name(), nil } func requestRerank(modelName, query string, documents []string, topN *int, apiEndpoint string) (*http.Response, []byte) { diff --git a/tests/e2e/distributed/nats_jwt_helpers_test.go b/tests/e2e/distributed/nats_jwt_helpers_test.go index e0723b378..80060ef6a 100644 --- a/tests/e2e/distributed/nats_jwt_helpers_test.go +++ b/tests/e2e/distributed/nats_jwt_helpers_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/internal/testfixtures" "github.com/mudler/LocalAI/pkg/natsauth" "github.com/nats-io/jwt/v2" "github.com/nats-io/nkeys" @@ -18,8 +17,6 @@ import ( "github.com/testcontainers/testcontainers-go" tcnats "github.com/testcontainers/testcontainers-go/modules/nats" - tcnetwork "github.com/testcontainers/testcontainers-go/network" - "github.com/testcontainers/testcontainers-go/wait" ) // JWTTestInfra holds a NATS server configured with JWT auth and minted worker credentials. @@ -37,9 +34,6 @@ func SetupJWTInfra() *JWTTestInfra { GinkgoHelper() infra := &JWTTestInfra{TestInfra: &TestInfra{Ctx: context.Background()}} - Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed()) - testNetwork, err := testfixtures.DockerNetwork() - Expect(err).NotTo(HaveOccurred()) operatorJWT, accountJWT, accountSeed, err := jwtResolverMaterial() Expect(err).ToNot(HaveOccurred()) @@ -57,18 +51,15 @@ resolver_preload: { var natsContainer *tcnats.NATSContainer // Override default testcontainers -js: JetStream fails without a system account in JWT mode. - natsContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine, + natsContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine", tcnats.WithConfigFile(bytes.NewBufferString(conf)), testcontainers.WithCmd("-c", "/etc/nats.conf"), - tcnetwork.WithNetworkName([]string{"nats"}, testNetwork), - testcontainers.WithWaitStrategy(wait.ForLog("Server is ready")), ) Expect(err).ToNot(HaveOccurred()) infra.NATSContainer = natsContainer - natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222") + infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) Expect(err).ToNot(HaveOccurred()) - infra.NatsURL = "nats://" + natsEndpoint infra.NodeID = "550e8400-e29b-41d4-a716-446655440000" cfg := natsauth.Config{AccountSeed: infra.AccountSeed, WorkerJWTTTL: time.Hour} @@ -162,4 +153,4 @@ func accountPublicKeyFromSeed(accountSeed string) string { func nodeSubjectPrefix(nodeID string) string { tok := strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-").Replace(nodeID) return "nodes." + tok -} +} \ No newline at end of file diff --git a/tests/e2e/distributed/testhelpers_test.go b/tests/e2e/distributed/testhelpers_test.go index d54c0897a..68cf537e3 100644 --- a/tests/e2e/distributed/testhelpers_test.go +++ b/tests/e2e/distributed/testhelpers_test.go @@ -2,11 +2,9 @@ package distributed_test import ( "context" - "fmt" "time" "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/internal/testfixtures" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -14,7 +12,6 @@ import ( "github.com/testcontainers/testcontainers-go" tcnats "github.com/testcontainers/testcontainers-go/modules/nats" tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" - tcnetwork "github.com/testcontainers/testcontainers-go/network" "github.com/testcontainers/testcontainers-go/wait" ) @@ -35,13 +32,9 @@ func SetupInfra(dbName string) *TestInfra { infra := &TestInfra{Ctx: context.Background()} var err error - Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.Postgres16Alpine, "distributed-e2e")).To(Succeed()) - Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed()) - testNetwork, err := testfixtures.DockerNetwork() - Expect(err).NotTo(HaveOccurred()) // Start PostgreSQL container - infra.PGContainer, err = tcpostgres.Run(infra.Ctx, testfixtures.Postgres16Alpine, + infra.PGContainer, err = tcpostgres.Run(infra.Ctx, "postgres:16-alpine", tcpostgres.WithDatabase(dbName), tcpostgres.WithUsername("test"), tcpostgres.WithPassword("test"), @@ -50,23 +43,18 @@ func SetupInfra(dbName string) *TestInfra { WithOccurrence(2). WithStartupTimeout(30*time.Second), ), - tcnetwork.WithNetworkName([]string{"postgres"}, testNetwork), ) Expect(err).ToNot(HaveOccurred()) - pgEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.PGContainer, "5432") + infra.PGURL, err = infra.PGContainer.ConnectionString(infra.Ctx, "sslmode=disable") Expect(err).ToNot(HaveOccurred()) - infra.PGURL = fmt.Sprintf("postgres://test:test@%s/%s?sslmode=disable", pgEndpoint, dbName) // Start NATS container - infra.NATSContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine, - tcnetwork.WithNetworkName([]string{"nats"}, testNetwork), - testcontainers.WithWaitStrategy(wait.ForLog("Server is ready"))) + infra.NATSContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine") Expect(err).ToNot(HaveOccurred()) - natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222") + infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) Expect(err).ToNot(HaveOccurred()) - infra.NatsURL = "nats://" + natsEndpoint // Connect messaging client infra.NC, err = messaging.New(infra.NatsURL) @@ -95,18 +83,12 @@ func SetupNATSOnly() *TestInfra { infra := &TestInfra{Ctx: context.Background()} var err error - Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed()) - testNetwork, err := testfixtures.DockerNetwork() - Expect(err).NotTo(HaveOccurred()) - infra.NATSContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine, - tcnetwork.WithNetworkName([]string{"nats"}, testNetwork), - testcontainers.WithWaitStrategy(wait.ForLog("Server is ready"))) + infra.NATSContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine") Expect(err).ToNot(HaveOccurred()) - natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222") + infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) Expect(err).ToNot(HaveOccurred()) - infra.NatsURL = "nats://" + natsEndpoint infra.NC, err = messaging.New(infra.NatsURL) Expect(err).ToNot(HaveOccurred())