From cb3bf7af3fa7a5312e285a657088bacabc6e4d6d Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Wed, 19 Aug 2026 09:59:31 +0100 Subject: [PATCH] chore(tests): Avoid network, sleep and more during tests (#11050) * test: make coverage failures observable Keep per-root logs, reject concurrent coverage runs, and avoid relying on /bin/sleep in the worker timeout test. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * test: parallelize coverage without remote fixtures Assisted-by: Codex:gpt-5 [apply_patch] [exec_command] Signed-off-by: Richard Palethorpe * test: add offline resource infrastructure Introduce versioned resource manifests, a checksum-verified CAS preparer, offline test wrappers, and a guarded network transport. Replace live Hugging Face, GitHub, and OCI cases with deterministic fixtures and inject fixture metadata into importer discovery. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * test: enforce offline resource replay Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * test: harden offline resource refresh Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * test: expose slow coverage waits Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * test: eliminate avoidable wall-clock waits Inject a clock into Hugging Face retry handling, reuse a process-scoped PostgreSQL container with per-spec schemas in the nodes suite, and poll local import jobs promptly. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * test: remove repeated fixture startup waits Share PostgreSQL fixtures across parallel endpoint and agent suite workers, and make the worker Free deadline injectable so the wedged-backend test does not spend five seconds on wall-clock time. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * test: fix offline resource CI portability Normalize Docker archive metadata before content addressing, derive archive checksums during explicit refreshes, make network lint portable to macOS, and prepare distributed images before running their offline suite. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * ci: cache Go modules before offline tests Warm the complete module graph before the Linux and macOS test jobs enter offline replay mode, so tool dependencies such as Ginkgo are not fetched through the guarded proxy. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * test: drop the static network lint in favour of real isolation The offline test suite already prevents tests from reaching the network twice over: run-test-linux-offline.sh puts the test process in a cgroup and REJECTs egress outside the private ranges, and HardenedTransport installs testnetwork.LocalGuard to refuse dials that resolve to a public address. Both fail the test with a precise error at the moment of the dial. test-network-lint.sh added neither. Its diff stage defaulted to a HEAD base, so on a clean checkout it compared the tree against itself and inspected nothing; the branch's own commits were never examined. It only produced output when an earlier job step dirtied the tree, and then it matched a bare https?:// against whatever changed. make react-ui runs npm install rather than npm ci, so CI rewrote core/http/react-ui/package-lock.json and the lint reported an npm registry URL as forbidden test network access: + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.25.tgz", Its fingerprint stage was self-defeating in a quieter way: hashing the whole tree's network-mechanism inventory meant every rebase onto a master that touched any _test.go needed a manual baseline bump, so the check mostly caught its own staleness. Remove the script, its make target and the two prerequisite edges, along with the test-network: fixture markers that existed only to suppress it. The isolation itself is untouched. Assisted-by: Claude:claude-opus-5 [go vet] Signed-off-by: Richard Palethorpe * ci: keep hidden files in the offline test bundle artifact Cherry-picked from 15a37b0ac on the remote branch. The offline bundle lives under .cache/, which actions/upload-artifact skips by default, so the Linux job packed an artifact missing the very file the next step restores. The other half of 15a37b0ac moved test-network-lint out of the `test` and `test-coverage` prerequisite lists into a recipe line, so parallel make could not fingerprint the tree while generated fixtures were still changing. That is dropped: the preceding commit removes the lint entirely, and the race it worked around is one more reason a whole-tree fingerprint was the wrong mechanism. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * refactor: share bounded exponential backoff Use overflow-safe saturating arithmetic for retry delays across model import polling, downloads, registration, node operations, and model loading. Keep model import status checks responsive initially while capping their interval at 500ms. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * ci: mirror Jetson Python wheels Keep the CUDA aarch64 wheel subset in GHCR and serve it as a local PEP 503 index during L4T backend builds, preserving last-known-good packages through upstream outages. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * docs(agents): index the Jetson wheels mirror Mention the GHCR-hosted L4T wheel mirror in the CI caching guide summary so maintainers can find its outage and cache documentation. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * ci: add defensive build network proxy Record build destinations and byte counts, retry observable idempotent HTTP downloads, and isolate explorer database tests that race under coverage. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * fix(kokoros): implement updated backend trait Return unimplemented for image upscaling, matching the backend's other unsupported modalities after the protobuf API update. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * fix(ci): clear recovered proxy errors Do not mark a request failed when a later safe retry succeeds. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * ci: require HTTPS build interception Inject a short-lived proxy CA into BuildKit and Dockerfile RUN steps, reject plain HTTP and opaque tunnels, and retain method/status/byte telemetry for verified HTTPS traffic. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * fix(ci): preserve system trust in unproxied builds Mount the generated interception CA at a dedicated secret path and add it to the trust bundle only in proxy-aware dependency stages. This prevents optional secret mounts from masking the system CA bundle in ordinary backend test builds. Install the requested Go toolchain before starting the proxy and satisfy cleanup error checks found by CI lint. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * fix(ci): persist build proxy trust Install the generated proxy CA through the system-managed local certificate directory so ca-certificates upgrades retain it. Avoid turning canceled matrix jobs into proxy cleanup failures. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * fix(ci): trust proxy in nested build scripts Install the build proxy CA before nested source fetches, route the DS4 package setup through the HTTPS mirror helper, and avoid repeated OCI setup in gallery behavior tests. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * fix(ci): use HTTPS apt sources for Bonsai Rewrite ARM64 package sources before installing GCC and check gallery fixture cleanup errors so the optimized tests satisfy errcheck. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * fix(privacy-filter): trust build proxy CA Install the mounted build proxy certificate before privacy-filter's make target fetches its HTTPS sources, for both source and prebuilt builder paths.\n\nAssisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * test: fail on hidden offline egress Count cgroup-scoped firewall rejects and fail the offline test harness with bounded aggregate diagnostics. Inject the gen-audio GGUF probe so fixture-backed importer tests do not attempt real network access. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * fix(ci): preserve system CA trust Build a combined runner certificate bundle instead of replacing public roots with the generated proxy CA. Centralize additive container installation in the shared proxy CA helper. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe --------- Signed-off-by: Richard Palethorpe Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- .agents/building-and-testing.md | 3 + .agents/ci-caching.md | 29 ++ .docker/apt-mirror.sh | 14 +- .docker/bonsai-compile.sh | 8 +- .docker/ik-llama-cpp-compile.sh | 2 + .docker/install-build-proxy-ca.sh | 26 + .docker/llama-cpp-compile.sh | 2 + .docker/turboquant-compile.sh | 2 + .../actions/configure-apt-mirror/action.yml | 13 +- .github/jetson-wheels.json | 25 + .github/scripts/inject-build-proxy-ca.sh | 21 + .github/scripts/start-build-proxy.sh | 48 ++ .github/scripts/stop-build-proxy.sh | 55 ++ .github/workflows/backend_build.yml | 80 +++ .github/workflows/external-probes.yml | 38 ++ .github/workflows/image_build.yml | 43 ++ .github/workflows/jetson-wheels.yml | 131 +++++ .github/workflows/test-resource-refresh.yml | 76 +++ .github/workflows/test.yml | 25 +- .github/workflows/tests-aio.yml | 4 +- .github/workflows/tests-e2e.yml | 3 + AGENTS.md | 2 +- Dockerfile | 81 +-- Makefile | 42 +- backend/Dockerfile.audio-cpp | 7 +- backend/Dockerfile.base-grpc-builder | 6 +- backend/Dockerfile.bonsai | 18 +- backend/Dockerfile.ds4 | 9 +- backend/Dockerfile.golang | 37 +- backend/Dockerfile.ik-llama-cpp | 18 +- backend/Dockerfile.llama-cpp | 18 +- backend/Dockerfile.privacy-filter | 16 +- backend/Dockerfile.python | 44 +- backend/Dockerfile.rust | 14 +- backend/Dockerfile.turboquant | 18 +- backend/python/common/libbackend.sh | 73 ++- backend/python/common/pypi_mirror_server.py | 226 +++++++++ .../python/common/pypi_mirror_server_test.py | 100 ++++ cmd/build-proxy/main.go | 52 ++ cmd/test-resources/main.go | 468 ++++++++++++++++++ cmd/test-resources/main_test.go | 54 ++ core/cli/workerregistry/client.go | 13 +- core/cli/workerregistry/credentials.go | 12 +- core/config/model_config_test.go | 14 +- .../config/testdata/hermes-2-pro-mistral.yaml | 7 + core/explorer/database_test.go | 8 +- core/gallery/backends_test.go | 62 ++- .../importers/discovery_options_test.go | 49 ++ core/gallery/importers/importers.go | 38 +- .../gallery/importers/importers_suite_test.go | 63 +++ core/gallery/importers/llama-cpp.go | 33 +- core/gallery/request_test.go | 6 +- core/http/app_test.go | 26 +- .../endpoints/localai/import_model_test.go | 11 + .../endpoints/localai/localai_suite_test.go | 11 + .../endpoints/ollama/helpers_internal_test.go | 2 +- core/http/openresponses_test.go | 2 +- .../services/agentpool/services_suite_test.go | 11 + core/services/agents/agents_suite_test.go | 11 + .../buildproxy/buildproxy_suite_test.go | 13 + core/services/buildproxy/proxy.go | 280 +++++++++++ core/services/buildproxy/proxy_test.go | 74 +++ core/services/buildproxy/server.go | 233 +++++++++ core/services/buildproxy/server_test.go | 46 ++ core/services/cloudproxy/mitm/proxy.go | 35 +- core/services/jobs/jobs_suite_test.go | 11 + core/services/nodes/file_stager_http.go | 11 +- core/services/nodes/nodes_suite_test.go | 11 + core/services/nodes/registry.go | 22 +- core/services/testutil/testdb.go | 101 +++- core/services/worker/free_timeout_test.go | 25 +- core/services/worker/lifecycle.go | 2 +- core/services/worker/supervisor.go | 17 +- core/startup/model_preload.go | 18 +- core/startup/model_preload_test.go | 14 +- docs/content/development/offline-tests.md | 77 +++ internal/backoff/backoff.go | 23 + internal/backoff/backoff_suite_test.go | 13 + internal/backoff/backoff_test.go | 35 ++ internal/testfixtures/images.go | 55 ++ internal/testresources/bundle.go | 164 ++++++ internal/testresources/httpcache.go | 92 ++++ internal/testresources/resources.go | 215 ++++++++ .../testresources/resources_suite_test.go | 15 + internal/testresources/resources_test.go | 126 +++++ pkg/downloader/cancel_test.go | 6 +- pkg/downloader/retry.go | 9 +- pkg/downloader/stall_test.go | 4 +- pkg/downloader/uri_test.go | 26 +- pkg/httpclient/client.go | 21 +- pkg/huggingface-api/client.go | 57 ++- pkg/huggingface-api/client_test.go | 121 ++++- pkg/model/loader.go | 14 +- pkg/model/loader_test.go | 8 +- pkg/modelartifacts/materializer.go | 5 +- pkg/oci/blob.go | 5 + pkg/oci/blob_test.go | 21 +- pkg/oci/cosignverify/verify.go | 4 +- pkg/oci/image_test.go | 32 +- pkg/oci/ollama.go | 20 +- pkg/oci/ollama_test.go | 26 +- pkg/testnetwork/guard.go | 57 +++ pkg/testnetwork/guard_suite_test.go | 15 + pkg/testnetwork/guard_test.go | 35 ++ scripts/jetson-wheels-sync.py | 178 +++++++ scripts/run-coverage.sh | 160 +++++- scripts/run-test-linux-offline.sh | 70 +++ scripts/run-test-offline.sh | 19 + scripts/summarize-ginkgo-waits.sh | 29 ++ test-resources/manifests/aio.json | 12 + test-resources/manifests/backend.json | 1 + test-resources/manifests/default-darwin.json | 7 + test-resources/manifests/default.json | 10 + test-resources/manifests/distributed-e2e.json | 14 + test-resources/manifests/external-probes.json | 1 + test-resources/manifests/hardware.json | 1 + test-resources/manifests/lock.json | 12 + tests/e2e-aio/e2e_suite_test.go | 16 +- tests/e2e-aio/e2e_test.go | 44 +- .../e2e/distributed/nats_jwt_helpers_test.go | 15 +- tests/e2e/distributed/testhelpers_test.go | 30 +- 121 files changed, 4756 insertions(+), 407 deletions(-) create mode 100644 .docker/install-build-proxy-ca.sh create mode 100644 .github/jetson-wheels.json create mode 100755 .github/scripts/inject-build-proxy-ca.sh create mode 100755 .github/scripts/start-build-proxy.sh create mode 100755 .github/scripts/stop-build-proxy.sh create mode 100644 .github/workflows/external-probes.yml create mode 100644 .github/workflows/jetson-wheels.yml create mode 100644 .github/workflows/test-resource-refresh.yml create mode 100644 backend/python/common/pypi_mirror_server.py create mode 100644 backend/python/common/pypi_mirror_server_test.py create mode 100644 cmd/build-proxy/main.go create mode 100644 cmd/test-resources/main.go create mode 100644 cmd/test-resources/main_test.go create mode 100644 core/config/testdata/hermes-2-pro-mistral.yaml create mode 100644 core/gallery/importers/discovery_options_test.go create mode 100644 core/services/buildproxy/buildproxy_suite_test.go create mode 100644 core/services/buildproxy/proxy.go create mode 100644 core/services/buildproxy/proxy_test.go create mode 100644 core/services/buildproxy/server.go create mode 100644 core/services/buildproxy/server_test.go create mode 100644 docs/content/development/offline-tests.md create mode 100644 internal/backoff/backoff.go create mode 100644 internal/backoff/backoff_suite_test.go create mode 100644 internal/backoff/backoff_test.go create mode 100644 internal/testfixtures/images.go create mode 100644 internal/testresources/bundle.go create mode 100644 internal/testresources/httpcache.go create mode 100644 internal/testresources/resources.go create mode 100644 internal/testresources/resources_suite_test.go create mode 100644 internal/testresources/resources_test.go create mode 100644 pkg/testnetwork/guard.go create mode 100644 pkg/testnetwork/guard_suite_test.go create mode 100644 pkg/testnetwork/guard_test.go create mode 100644 scripts/jetson-wheels-sync.py create mode 100755 scripts/run-test-linux-offline.sh create mode 100755 scripts/run-test-offline.sh create mode 100755 scripts/summarize-ginkgo-waits.sh create mode 100644 test-resources/manifests/aio.json create mode 100644 test-resources/manifests/backend.json create mode 100644 test-resources/manifests/default-darwin.json create mode 100644 test-resources/manifests/default.json create mode 100644 test-resources/manifests/distributed-e2e.json create mode 100644 test-resources/manifests/external-probes.json create mode 100644 test-resources/manifests/hardware.json create mode 100644 test-resources/manifests/lock.json diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 021d555ec..346620c3a 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -21,6 +21,9 @@ Let's say the user wants to build a particular backend for a given platform. For The core Go suites (`./pkg`, `./core`, plus the in-process integration suite `./tests/e2e`) are covered by a **strict, monotonic coverage ratchet**: - `make test-coverage` — runs the suites with `covermode=atomic` instrumentation and writes a merged profile to `coverage/coverage.out`. Uses the same prerequisites as `make test`. + - Prints per-root wall time and the slowest specs/hooks exceeding `COVERAGE_SLOW_SPEC_THRESHOLD` (default 3 seconds, capped by `COVERAGE_SLOW_SPEC_LIMIT`, default 25 per root); machine-readable root timings are written to `coverage/timings.tsv`. + - Verbose Ginkgo output is written to `coverage/logs/.log`, with the prior run retained as `.log.previous`. The terminal prints one status line per root and a short failure extract. If any suite fails, no merged profile is produced and the percentage ratchet is explicitly not run. A lock under `coverage/` rejects concurrent runs, which would otherwise corrupt their shared profiles and logs. + - Suites run in parallel by default and each recursive root invocation has a five-minute budget. Override auto-detected parallelism with `COVERAGE_PROCS`; tune diagnostics with `COVERAGE_SUITE_TIMEOUT` and `COVERAGE_PROGRESS_AFTER`. A timeout is a performance failure to investigate, not a reason to raise the committed default. - **`--coverpkg` (`COVERAGE_COVERPKG = core/...,pkg/...`):** coverage is attributed to the core+pkg packages, not just the package under test. This is what lets the in-process `tests/e2e` suite (which drives the real HTTP server over loopback via `application.New`) credit the `core/http/endpoints/...` handlers it exercises — folding it in roughly doubled endpoint coverage (e.g. `endpoints/openai` 13.6% → 52%). The denominator is therefore *all* of `core`+`pkg` (minus generated proto, dropped via `COVERAGE_EXCLUDE_RE`), so the number isn't comparable to a plain per-package figure. - **Integration suites (`COVERAGE_E2E_ROOTS = ./tests/e2e`)** run non-recursively (excludes `tests/e2e/distributed`, which needs containers) with `--label-filter=!real-models` (those need a downloaded model) against the mock backend built by `prepare-test`. `tests/integration` is deliberately excluded — it needs `make backends/local-store`, which the coverage CI job doesn't build. - **Flake note:** folding integration tests into a *strict* gate means a hard e2e failure (or a spec that silently stops running) can fail the coverage gate, not just the test. `--flake-attempts` absorbs transient retryable failures; covermode=atomic keeps line coverage deterministic otherwise. diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 6742049e6..ed993dafb 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -1,5 +1,20 @@ # CI Build Caching +## Build network inventory and defensive proxy + +Backend and main-image builds use `cmd/build-proxy` as a strict HTTPS-intercepting +proxy through the host network. Its short-lived CA is injected into the running +BuildKit daemon and mounted over the conventional CA bundle during every +Dockerfile `RUN`. Plain HTTP and opaque CONNECT traffic fail the job. Every +destination is retained for 14 days as JSONL plus an aggregate host/method/byte +summary. Responses are spooled and checked against `Content-Length`; GET/HEAD +requests retry transient status codes or incomplete responses with exponential +backoff capped at 500ms. Request headers, bodies, credentials and query strings +are never recorded. + +The inventory is intended to size a future content-addressed cache and identify +hosts worth adding to curated OCI mirrors, such as the Jetson wheels mirror. + Container builds — both the root LocalAI image (`Dockerfile`) and the per-backend images (`backend/Dockerfile.*`) — share a registry-backed BuildKit cache plus a layered set of prebuilt base images. This file explains how the cache is laid out, what invalidates it, and how to bypass it. ## Workflow surfaces @@ -231,6 +246,20 @@ This applies only to `Dockerfile.python` because: Bump the format to daily (`+%Y-%m-%d`) or hourly (`+%Y-%m-%d-%H`) for faster refreshes. For one-shot rebuilds without changing the schedule, append a marker to the tag-suffix in the matrix or temporarily delete that backend's cache tag in quay. +## The jetson wheels mirror (l4t builds) + +The `requirements-l4t12.txt` / `-l4t13.txt` files pull CUDA aarch64 torch wheels from `pypi.jetson-ai-lab.io` via `--extra-index-url`. That index has a history of multi-hour 502 outages, and a 502 on **any** project page aborts the whole uv resolution — uv consults every configured index for every requirement, so even PyPI-hosted packages die with it. To keep l4t builds green through outages, CI serves those wheels from a mirror it controls: + +- **Storage**: `ghcr.io/mudler/localai/jetson-wheels:{jp6-cu129,jp7-cu130}` — scratch OCI images holding the wheel subset, laid out like the upstream index (`/jp6/cu129/torch/`). +- **Sync**: `.github/workflows/jetson-wheels.yml` (Saturdays 03:00 UTC, ahead of the weekly `DEPS_REFRESH` re-resolve; also `workflow_dispatch` and master pushes touching its inputs) runs `scripts/jetson-wheels-sync.py` against the package list in `.github/jetson-wheels.json`. During an upstream outage the sync keeps the last-known-good wheels and exits green. +- **Consumption**: `backend_build.yml` resolves the matching tag for `build-type: l4t` entries (cuda 12 → `jp6-cu129`, 13 → `jp7-cu130`) and passes it as the `JETSON_WHEELS_IMAGE` build-arg; `Dockerfile.python` bind-mounts it at `/jetson-wheels`; `installRequirements` in `backend/python/common/libbackend.sh` serves that directory on localhost as a PEP 503 index (`backend/python/common/pypi_mirror_server.py`) and rewrites the jetson index host in the requirements files to it. The local index 404s for anything it doesn't carry, which uv follows up on PyPI — only the jetson-built wheels resolve locally. +- **Fallbacks**: if the mirror tag doesn't exist (bootstrap) `backend_build.yml` passes `scratch`, the mount is empty, and the build talks to the upstream index exactly as before. Builds outside CI (local, real Jetsons) never set `JETSON_WHEELS_IMAGE` and are unaffected. +- **Cache interaction**: the bind mount's content is part of the `RUN ... make` layer's BuildKit hash, so a refreshed wheels image invalidates the install layer on the next build — no extra cache-buster needed. + +**Extending the package list**: a package a build needs from the jetson index but missing from `.github/jetson-wheels.json` resolves from PyPI instead — for compiled CUDA packages that silently means a CPU build. When adding an l4t backend with new compiled deps, add them to the list and dispatch `jetson-wheels.yml`. + +**Bootstrap** (one-time): `gh workflow run jetson-wheels.yml --ref master`, then make the `jetson-wheels` ghcr package public so anonymous pulls work (Settings → Packages). + ## ccache for C++ backend builds `Dockerfile.{llama-cpp,ik-llama-cpp,turboquant}` declare a BuildKit cache mount on `/root/.ccache`: diff --git a/.docker/apt-mirror.sh b/.docker/apt-mirror.sh index 8bd41d1f7..d00de21d2 100755 --- a/.docker/apt-mirror.sh +++ b/.docker/apt-mirror.sh @@ -7,8 +7,7 @@ # # Inputs (env): # APT_MIRROR Replacement for archive.ubuntu.com and security.ubuntu.com -# (e.g. "http://azure.archive.ubuntu.com" or -# "https://mirrors.edge.kernel.org"). +# (e.g. "https://azure.archive.ubuntu.com"). # Leave empty to keep upstream. The trailing "/ubuntu/..." # path is preserved by the rewrite. # APT_PORTS_MIRROR Replacement for ports.ubuntu.com (arm64/ppc64el/...). @@ -18,8 +17,8 @@ set -e -if [ -z "${APT_MIRROR}" ] && [ -z "${APT_PORTS_MIRROR}" ]; then - exit 0 +if [ -f /usr/local/sbin/install-build-proxy-ca ]; then + sh /usr/local/sbin/install-build-proxy-ca fi # Ubuntu 24.04 (noble) ships DEB822 sources at /etc/apt/sources.list.d/ubuntu.sources; @@ -36,4 +35,11 @@ for f in /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list; do fi done +# Build networking is HTTPS-only. Upgrade any untouched distribution defaults +# as well (notably ports.ubuntu.com when a caller leaves its override empty). +for f in /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list; do + [ -f "$f" ] || continue + sed -i -E 's,http://,https://,g' "$f" +done + echo "apt-mirror: rewrote sources (APT_MIRROR='${APT_MIRROR}', APT_PORTS_MIRROR='${APT_PORTS_MIRROR}')" diff --git a/.docker/bonsai-compile.sh b/.docker/bonsai-compile.sh index 38439439d..23d1cf378 100755 --- a/.docker/bonsai-compile.sh +++ b/.docker/bonsai-compile.sh @@ -4,6 +4,8 @@ set -euxo pipefail +sh /LocalAI/.docker/install-build-proxy-ca.sh + export CCACHE_DIR=/root/.ccache ccache --max-size=5G || true ccache -z || true @@ -23,7 +25,11 @@ if [ -z "${BUILD_TYPE:-}" ]; then # Pure CPU image: one ggml CPU_ALL_VARIANTS build replaces the per-microarch binaries. # arm64: the armv9.2 SME variants need gcc-14 (gcc-13 rejects +sme). if [ "${TARGETARCH}" = "arm64" ]; then - apt-get update -qq && apt-get install -y -qq gcc-14 g++-14 + APT_MIRROR="${APT_MIRROR:-https://azure.archive.ubuntu.com}" \ + APT_PORTS_MIRROR="${APT_PORTS_MIRROR:-https://azure.ports.ubuntu.com}" \ + sh /LocalAI/.docker/apt-mirror.sh + apt-get update -qq + apt-get install -y -qq gcc-14 g++-14 export CC=gcc-14 CXX=g++-14 fi make bonsai-cpu-all diff --git a/.docker/ik-llama-cpp-compile.sh b/.docker/ik-llama-cpp-compile.sh index 3da869007..5409c78d6 100755 --- a/.docker/ik-llama-cpp-compile.sh +++ b/.docker/ik-llama-cpp-compile.sh @@ -4,6 +4,8 @@ set -euxo pipefail +sh /LocalAI/.docker/install-build-proxy-ca.sh + export CCACHE_DIR=/root/.ccache ccache --max-size=5G || true ccache -z || true diff --git a/.docker/install-build-proxy-ca.sh b/.docker/install-build-proxy-ca.sh new file mode 100644 index 000000000..2475fdb7f --- /dev/null +++ b/.docker/install-build-proxy-ca.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Install CI's optional HTTPS interception CA without replacing public roots. + +set -e + +proxy_ca=/run/secrets/build_proxy_ca +if [ ! -s "$proxy_ca" ]; then + exit 0 +fi + +mkdir -p /usr/local/share/ca-certificates +cp "$proxy_ca" /usr/local/share/ca-certificates/localai-build-proxy.crt +if command -v update-ca-certificates >/dev/null 2>&1; then + update-ca-certificates +elif [ -f /etc/ssl/certs/ca-certificates.crt ]; then + cat "$proxy_ca" >>/etc/ssl/certs/ca-certificates.crt +else + echo 'build proxy: no system CA bundle found' >&2 + exit 1 +fi + +if [ -d /etc/apt/apt.conf.d ]; then + cat > /etc/apt/apt.conf.d/99localai-build-proxy-ca <&2 + exit 1 +fi + +docker exec "$container" mkdir -p /usr/local/share/ca-certificates /etc/ssl/certs +docker cp "$ca" "$container:/usr/local/share/ca-certificates/localai-build-proxy.crt" +docker exec "$container" sh -eu -c ' + if command -v update-ca-certificates >/dev/null 2>&1; then + update-ca-certificates + else + cat /usr/local/share/ca-certificates/localai-build-proxy.crt >>/etc/ssl/certs/ca-certificates.crt + fi +' +docker restart "$container" >/dev/null diff --git a/.github/scripts/start-build-proxy.sh b/.github/scripts/start-build-proxy.sh new file mode 100755 index 000000000..7d3a78978 --- /dev/null +++ b/.github/scripts/start-build-proxy.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail +output="${RUNNER_TEMP}/localai-build-proxy" +mkdir -p "$output" +CGO_ENABLED=0 GOCACHE="${RUNNER_TEMP}/go-build-cache" go build -o "$output/build-proxy" ./cmd/build-proxy +nohup "$output/build-proxy" --listen 127.0.0.1:18080 --output "$output" >"$output/proxy.log" 2>&1 & +echo "$!" >"$output/proxy.pid" +for _ in $(seq 1 50); do + grep -q '^ca=' "$output/proxy.log" && break + sleep 0.1 +done +grep '^proxy=' "$output/proxy.log" +grep '^ca=' "$output/proxy.log" +proxy_ca="$output/ca/ca.crt" +ca_bundle="$output/ca/ca-bundle.crt" +system_ca="" +for candidate in \ + /etc/ssl/certs/ca-certificates.crt \ + /etc/ssl/cert.pem \ + /etc/pki/tls/certs/ca-bundle.crt \ + /etc/openssl/certs/ca-certificates.crt; do + if test -s "$candidate"; then + system_ca="$candidate" + break + fi +done +if test -z "$system_ca"; then + echo 'build proxy: unable to find the runner system CA bundle' >&2 + exit 1 +fi +cat "$system_ca" "$proxy_ca" >"$ca_bundle" +{ + echo "LOCALAI_BUILD_PROXY=http://127.0.0.1:18080" + echo "LOCALAI_BUILD_PROXY_OUTPUT=$output" + echo "LOCALAI_BUILD_PROXY_CA=$proxy_ca" + echo "LOCALAI_BUILD_PROXY_CA_BUNDLE=$ca_bundle" + echo "HTTP_PROXY=http://127.0.0.1:18080" + echo "HTTPS_PROXY=http://127.0.0.1:18080" + echo "http_proxy=http://127.0.0.1:18080" + echo "https_proxy=http://127.0.0.1:18080" + echo "SSL_CERT_FILE=$ca_bundle" + echo "CURL_CA_BUNDLE=$ca_bundle" + echo "REQUESTS_CA_BUNDLE=$ca_bundle" + echo "GIT_SSL_CAINFO=$ca_bundle" + echo "NODE_EXTRA_CA_CERTS=$proxy_ca" + echo "NO_PROXY=localhost,127.0.0.1" + echo "no_proxy=localhost,127.0.0.1" +} >>"$GITHUB_ENV" diff --git a/.github/scripts/stop-build-proxy.sh b/.github/scripts/stop-build-proxy.sh new file mode 100755 index 000000000..0268ce029 --- /dev/null +++ b/.github/scripts/stop-build-proxy.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail +if test -z "${LOCALAI_BUILD_PROXY_OUTPUT:-}"; then + echo 'Build proxy did not start; skipping inventory finalization' + exit 0 +fi +output="$LOCALAI_BUILD_PROXY_OUTPUT" +if test -f "$output/proxy.pid"; then + kill -TERM "$(cat "$output/proxy.pid")" 2>/dev/null || true + for _ in $(seq 1 50); do + test -f "$output/summary.json" && break + sleep 0.1 + done +fi + +# Later artifact uploads and action post-hooks must not target a stopped proxy. +{ + echo 'HTTP_PROXY=' + echo 'HTTPS_PROXY=' + echo 'http_proxy=' + echo 'https_proxy=' + echo 'SSL_CERT_FILE=' + echo 'CURL_CA_BUNDLE=' + echo 'REQUESTS_CA_BUNDLE=' + echo 'GIT_SSL_CAINFO=' + echo 'NODE_EXTRA_CA_CERTS=' +} >>"$GITHUB_ENV" +if test -f "$output/summary.json"; then + { + echo '### Build network inventory' + echo + echo 'HTTPS without the generated CA is reported as CONNECT because its HTTP method is encrypted.' + echo + echo '```json' + cat "$output/summary.json" + echo '```' + } >>"$GITHUB_STEP_SUMMARY" +fi + +# A matrix cancellation can interrupt checkout or a BuildKit request at any +# point. Preserve whatever inventory exists, but do not replace the canceled +# conclusion with a misleading proxy-enforcement failure. +if test "${LOCALAI_BUILD_JOB_STATUS:-}" = cancelled; then + echo 'Build was cancelled; skipping network inventory enforcement' + exit 0 +fi + +if ! test -s "$output/events.jsonl"; then + echo 'Build proxy produced no network inventory' >&2 + exit 1 +fi +if grep -qE '"method":"CONNECT"|"error":"plain HTTP is forbidden"' "$output/events.jsonl"; then + echo 'Build traffic bypassed HTTPS interception or attempted plain HTTP' >&2 + exit 1 +fi diff --git a/.github/workflows/backend_build.yml b/.github/workflows/backend_build.yml index 05d50cf82..4a9acfb9f 100644 --- a/.github/workflows/backend_build.yml +++ b/.github/workflows/backend_build.yml @@ -153,9 +153,27 @@ jobs: with: platforms: all + - name: Set up Go for build proxy + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Start build network proxy + run: .github/scripts/start-build-proxy.sh + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@master + with: + driver-opts: | + network=host + env.http_proxy=${{ env.LOCALAI_BUILD_PROXY }} + env.https_proxy=${{ env.LOCALAI_BUILD_PROXY }} + + - name: Trust build proxy CA in BuildKit + env: + BUILDER_NAME: ${{ steps.buildx.outputs.name }} + run: .github/scripts/inject-build-proxy-ca.sh - name: Login to DockerHub if: github.event_name != 'pull_request' @@ -181,6 +199,41 @@ jobs: id: deps_refresh run: echo "key=$(date -u +%Y-W%V)" >> "$GITHUB_OUTPUT" + - name: Login to ghcr.io (jetson wheels mirror) + if: inputs.build-type == 'l4t' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + # l4t builds pull their CUDA aarch64 torch wheels from + # pypi.jetson-ai-lab.io, which has a history of multi-hour 502 outages + # that fail every l4t job. jetson-wheels.yml mirrors those wheels into + # ghcr weekly; here we hand the mirror image to Dockerfile.python, + # which serves it as a local package index during pip install (see + # installRequirements in backend/python/common/libbackend.sh). Falls + # back to scratch — i.e. building straight against the upstream index — + # when the mirror tag doesn't exist yet, so the mirror can bootstrap + # without a chicken-and-egg failure. + - name: Resolve jetson wheels mirror image + id: jetson_wheels + if: inputs.build-type == 'l4t' + run: | + repo="ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')/jetson-wheels" + case "${{ inputs.cuda-major-version }}" in + 12) tag="jp6-cu129" ;; + 13) tag="jp7-cu130" ;; + *) tag="" ;; + esac + img="" + if [ -n "$tag" ] && docker buildx imagetools inspect "$repo:$tag" >/dev/null 2>&1; then + img="$repo:$tag" + else + echo "jetson wheels image $repo:$tag not found; building against the upstream index" + fi + echo "image=$img" >> "$GITHUB_OUTPUT" + - name: Build and push by digest id: build uses: docker/build-push-action@v7 @@ -188,6 +241,9 @@ jobs: with: builder: ${{ steps.buildx.outputs.name }} build-args: | + HTTP_PROXY=${{ env.LOCALAI_BUILD_PROXY }} + HTTPS_PROXY=${{ env.LOCALAI_BUILD_PROXY }} + NO_PROXY=localhost,127.0.0.1 BUILD_TYPE=${{ inputs.build-type }} SKIP_DRIVERS=${{ inputs.skip-drivers }} CUDA_MAJOR_VERSION=${{ inputs.cuda-major-version }} @@ -201,6 +257,9 @@ jobs: DEPS_REFRESH=${{ steps.deps_refresh.outputs.key }} BUILDER_BASE_IMAGE=${{ inputs.builder-base-image }} BUILDER_TARGET=${{ inputs.builder-base-image != '' && 'builder-prebuilt' || 'builder-fromsource' }} + JETSON_WHEELS_IMAGE=${{ steps.jetson_wheels.outputs.image || 'scratch' }} + secret-files: | + build_proxy_ca=${{ env.LOCALAI_BUILD_PROXY_CA }} context: ${{ inputs.context }} file: ${{ inputs.dockerfile }} cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache${{ inputs.tag-suffix }}-${{ inputs.platform-tag }} @@ -260,6 +319,9 @@ jobs: with: builder: ${{ steps.buildx.outputs.name }} build-args: | + HTTP_PROXY=${{ env.LOCALAI_BUILD_PROXY }} + HTTPS_PROXY=${{ env.LOCALAI_BUILD_PROXY }} + NO_PROXY=localhost,127.0.0.1 BUILD_TYPE=${{ inputs.build-type }} SKIP_DRIVERS=${{ inputs.skip-drivers }} CUDA_MAJOR_VERSION=${{ inputs.cuda-major-version }} @@ -273,6 +335,9 @@ jobs: DEPS_REFRESH=${{ steps.deps_refresh.outputs.key }} BUILDER_BASE_IMAGE=${{ inputs.builder-base-image }} BUILDER_TARGET=${{ inputs.builder-base-image != '' && 'builder-prebuilt' || 'builder-fromsource' }} + JETSON_WHEELS_IMAGE=${{ steps.jetson_wheels.outputs.image || 'scratch' }} + secret-files: | + build_proxy_ca=${{ env.LOCALAI_BUILD_PROXY_CA }} context: ${{ inputs.context }} file: ${{ inputs.dockerfile }} cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache${{ inputs.tag-suffix }}-${{ inputs.platform-tag }} @@ -286,3 +351,18 @@ jobs: - name: job summary run: | echo "Built image: ${{ steps.meta.outputs.labels }}" >> $GITHUB_STEP_SUMMARY + + - name: Stop build network proxy + if: ${{ always() && env.LOCALAI_BUILD_PROXY_OUTPUT != '' }} + env: + LOCALAI_BUILD_JOB_STATUS: ${{ job.status }} + run: .github/scripts/stop-build-proxy.sh + + - name: Upload build network inventory + if: ${{ always() && env.LOCALAI_BUILD_PROXY_OUTPUT != '' }} + uses: actions/upload-artifact@v7 + with: + name: build-network-${{ inputs.backend }}-${{ inputs.tag-suffix }}-${{ inputs.platform-tag || 'single' }} + path: ${{ env.LOCALAI_BUILD_PROXY_OUTPUT }} + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/external-probes.yml b/.github/workflows/external-probes.yml new file mode 100644 index 000000000..681f417e4 --- /dev/null +++ b/.github/workflows/external-probes.yml @@ -0,0 +1,38 @@ +--- +name: external compatibility probes + +on: + workflow_dispatch: + schedule: + - cron: '23 4 * * 1' + +permissions: + contents: read + +jobs: + external-probe-huggingface-xet: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v5 + with: + go-version: '1.26.x' + cache: false + - name: Probe Hugging Face Xet compatibility + run: LOCALAI_HF_XET_SMOKE=1 go test ./pkg/huggingface-api -ginkgo.focus='pinned public Xet fixture' -count=1 + + external-probe-sigstore: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v5 + with: + go-version: '1.26.x' + cache: false + - name: Probe public Sigstore compatibility + env: + LOCALAI_COSIGN_LIVE: '1' + LOCALAI_COSIGN_LIVE_IMAGE: ${{ vars.LOCALAI_COSIGN_LIVE_IMAGE }} + LOCALAI_COSIGN_LIVE_ISSUER: ${{ vars.LOCALAI_COSIGN_LIVE_ISSUER }} + LOCALAI_COSIGN_LIVE_IDENTITY_REGEX: ${{ vars.LOCALAI_COSIGN_LIVE_IDENTITY_REGEX }} + run: go test ./pkg/oci/cosignverify -ginkgo.focus='VerifyImage' -count=1 diff --git a/.github/workflows/image_build.yml b/.github/workflows/image_build.yml index 89bc4124f..609c1f2c5 100644 --- a/.github/workflows/image_build.yml +++ b/.github/workflows/image_build.yml @@ -129,9 +129,27 @@ jobs: with: platforms: all + - name: Set up Go for build proxy + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Start build network proxy + run: .github/scripts/start-build-proxy.sh + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@master + with: + driver-opts: | + network=host + env.http_proxy=${{ env.LOCALAI_BUILD_PROXY }} + env.https_proxy=${{ env.LOCALAI_BUILD_PROXY }} + + - name: Trust build proxy CA in BuildKit + env: + BUILDER_NAME: ${{ steps.buildx.outputs.name }} + run: .github/scripts/inject-build-proxy-ca.sh - name: Login to DockerHub if: github.event_name != 'pull_request' @@ -155,6 +173,9 @@ jobs: with: builder: ${{ steps.buildx.outputs.name }} build-args: | + HTTP_PROXY=${{ env.LOCALAI_BUILD_PROXY }} + HTTPS_PROXY=${{ env.LOCALAI_BUILD_PROXY }} + NO_PROXY=localhost,127.0.0.1 BUILD_TYPE=${{ inputs.build-type }} CUDA_MAJOR_VERSION=${{ inputs.cuda-major-version }} CUDA_MINOR_VERSION=${{ inputs.cuda-minor-version }} @@ -165,6 +186,8 @@ jobs: UBUNTU_CODENAME=${{ inputs.ubuntu-codename }} APT_MIRROR=${{ steps.apt_mirror.outputs.effective-mirror }} APT_PORTS_MIRROR=${{ steps.apt_mirror.outputs.effective-ports-mirror }} + secret-files: | + build_proxy_ca=${{ env.LOCALAI_BUILD_PROXY_CA }} context: . file: ./Dockerfile cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache-localai${{ inputs.tag-suffix }}-${{ inputs.platform-tag }} @@ -218,6 +241,9 @@ jobs: with: builder: ${{ steps.buildx.outputs.name }} build-args: | + HTTP_PROXY=${{ env.LOCALAI_BUILD_PROXY }} + HTTPS_PROXY=${{ env.LOCALAI_BUILD_PROXY }} + NO_PROXY=localhost,127.0.0.1 BUILD_TYPE=${{ inputs.build-type }} CUDA_MAJOR_VERSION=${{ inputs.cuda-major-version }} CUDA_MINOR_VERSION=${{ inputs.cuda-minor-version }} @@ -228,6 +254,8 @@ jobs: UBUNTU_CODENAME=${{ inputs.ubuntu-codename }} APT_MIRROR=${{ steps.apt_mirror.outputs.effective-mirror }} APT_PORTS_MIRROR=${{ steps.apt_mirror.outputs.effective-ports-mirror }} + secret-files: | + build_proxy_ca=${{ env.LOCALAI_BUILD_PROXY_CA }} context: . file: ./Dockerfile cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache-localai${{ inputs.tag-suffix }}-${{ inputs.platform-tag }} @@ -239,3 +267,18 @@ jobs: - name: job summary run: | echo "Built image: ${{ steps.meta.outputs.labels }}" >> $GITHUB_STEP_SUMMARY + + - name: Stop build network proxy + if: ${{ always() && env.LOCALAI_BUILD_PROXY_OUTPUT != '' }} + env: + LOCALAI_BUILD_JOB_STATUS: ${{ job.status }} + run: .github/scripts/stop-build-proxy.sh + + - name: Upload build network inventory + if: ${{ always() && env.LOCALAI_BUILD_PROXY_OUTPUT != '' }} + uses: actions/upload-artifact@v7 + with: + name: build-network-localai-${{ inputs.build-type }}-${{ inputs.cuda-major-version }}-${{ inputs.cuda-minor-version }}-${{ inputs.platform-tag || 'single' }} + path: ${{ env.LOCALAI_BUILD_PROXY_OUTPUT }} + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/jetson-wheels.yml b/.github/workflows/jetson-wheels.yml new file mode 100644 index 000000000..a943418a8 --- /dev/null +++ b/.github/workflows/jetson-wheels.yml @@ -0,0 +1,131 @@ +--- +name: 'sync jetson wheels mirror' + +# Mirrors the CUDA aarch64 wheels our l4t backends need from +# pypi.jetson-ai-lab.io into scratch OCI images on ghcr +# (ghcr.io/mudler/localai/jetson-wheels:, one tag per JetPack index). +# backend_build.yml hands the matching tag to Dockerfile.python, which +# bind-mounts it and serves it as a local package index during pip install +# (see installRequirements in backend/python/common/libbackend.sh), so the +# upstream index's recurring multi-hour 502 outages can no longer fail l4t +# builds. +# +# The package subset lives in .github/jetson-wheels.json. A package that a +# build needs from the jetson index but that is missing from that list will +# resolve from PyPI instead — for compiled CUDA packages that silently means +# a CPU build, so extend the list when adding an l4t backend with new +# compiled deps. +# +# When upstream is unreachable the sync keeps the previously mirrored wheels +# and exits green — the mirror serves last-known-good through outages. It +# only fails when upstream is down and the tag has never been published +# (bootstrap during an outage: nothing to serve yet). +# +# Triggers: +# - schedule (Saturdays 03:00 UTC) — refreshes ahead of base-images.yml +# (Saturdays 05:00 UTC) and the backend.yml weekly cron (Sundays), whose +# DEPS_REFRESH cache-bust re-resolves the python deps. +# - workflow_dispatch — manual one-off sync; also the bootstrap run: +# gh workflow run jetson-wheels.yml --ref master +# - push to master touching the config, the sync script, or this workflow. + +on: + schedule: + - cron: '0 3 * * 6' + workflow_dispatch: + push: + branches: [master] + paths: + - '.github/jetson-wheels.json' + - 'scripts/jetson-wheels-sync.py' + - '.github/workflows/jetson-wheels.yml' + +permissions: + contents: read + packages: write + +concurrency: + group: jetson-wheels-${{ github.repository }} + cancel-in-progress: false + +jobs: + sync: + if: github.repository == 'mudler/LocalAI' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - index: 'jp6/cu129' + tag: 'jp6-cu129' + - index: 'jp7/cu130' + tag: 'jp7-cu130' + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@master + + - name: Login to ghcr.io + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Compute image name + id: image + run: | + repo="ghcr.io/$(echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')/jetson-wheels" + echo "ref=${repo}:${{ matrix.tag }}" >> "$GITHUB_OUTPUT" + + # Seed the working dir with the current mirror contents so the sync is + # incremental and an upstream outage keeps last-known-good wheels. + - name: Pull current mirror contents + run: | + mkdir -p wheels + # The image is declared linux/arm64 (its only consumers are arm64 + # l4t builds); pulling on this amd64 runner needs the explicit + # platform. The content is just wheel files — never executed here. + if docker pull --platform linux/arm64 "${{ steps.image.outputs.ref }}"; then + # scratch images have no command; docker create still needs one, + # but the container is never started so any path works. + cid="$(docker create "${{ steps.image.outputs.ref }}" /noop)" + docker export "${cid}" | tar -x -C wheels + docker rm "${cid}" + find wheels -name '*.whl' | sed 's/^/ existing: /' + else + echo "no existing mirror image (bootstrap run)" + fi + + - name: Sync from upstream + id: sync + run: | + python3 scripts/jetson-wheels-sync.py \ + --config .github/jetson-wheels.json \ + --index '${{ matrix.index }}' \ + --dest wheels \ + --changed-file /tmp/jetson-wheels-changed + if [ -f /tmp/jetson-wheels-changed ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Push mirror image + if: steps.sync.outputs.changed == 'true' + run: | + cat > Dockerfile.jetson-wheels <<'EOF' + FROM scratch + COPY wheels/ / + EOF + # linux/arm64 because the consumers (l4t builds in + # backend_build.yml) build for arm64 and BuildKit refuses a + # platform-mismatched FROM; COPY-only, so no emulation is needed. + # provenance=false keeps the pushed ref a plain single manifest + # instead of an OCI index wrapping an attestation. + docker buildx build --push \ + --platform linux/arm64 \ + --provenance=false \ + -f Dockerfile.jetson-wheels \ + -t "${{ steps.image.outputs.ref }}" \ + . diff --git a/.github/workflows/test-resource-refresh.yml b/.github/workflows/test-resource-refresh.yml new file mode 100644 index 000000000..24e19f341 --- /dev/null +++ b/.github/workflows/test-resource-refresh.yml @@ -0,0 +1,76 @@ +--- +name: refresh offline test resources + +on: + workflow_dispatch: + schedule: + - cron: '17 3 * * 1' + +permissions: + contents: read + issues: write + packages: write + +jobs: + refresh: + strategy: + fail-fast: false + matrix: + resource-set: [default, distributed-e2e, aio] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v5 + with: + go-version: '1.26.x' + cache: true + - uses: oras-project/setup-oras@v1 + - name: Verify upstream resources and build compressed cache + id: refresh + continue-on-error: true + env: + LOCALAI_TEST_RESOURCES_ONLINE: '1' + run: | + set -o pipefail + make update-offline-test-cache TEST_RESOURCE_SET=${{ matrix.resource-set }} 2>&1 | tee resource-refresh.log + - name: Upload investigation evidence + if: steps.refresh.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: test-resource-investigation-${{ matrix.resource-set }}-${{ github.run_id }} + path: resource-refresh.log + - name: Open or update investigation issue + if: steps.refresh.outcome == 'failure' + env: + GH_TOKEN: ${{ secrets.LOCALAI_BOT_TOKEN || github.token }} + RESOURCE_SET: ${{ matrix.resource-set }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + title="test resource integrity investigation: ${RESOURCE_SET}" + body=$(printf '%s\n\n%s\n' \ + "The scheduled offline-resource refresh failed for \`${RESOURCE_SET}\`." \ + "Do not update the manifest digest blindly. Download the evidence artifact from ${RUN_URL}, compare upstream checksums/signatures and release notes, inspect redirects, and search the [GitHub Advisory Database](https://github.com/advisories) and [OSV](https://osv.dev). Retry from a declared mirror to distinguish source drift from corruption.") + existing=$(gh issue list --state open --search "${title} in:title" --json number --jq '.[0].number // empty') + if [ -n "$existing" ]; then + gh issue comment "$existing" --body "$body" + else + gh issue create --title "$title" --body "$body" + fi + - name: Log in to GHCR + if: steps.refresh.outcome == 'success' + run: echo "${{ github.token }}" | oras login ghcr.io -u "${{ github.actor }}" --password-stdin + - name: Publish compressed cache as an OCI artifact + if: steps.refresh.outcome == 'success' + env: + RESOURCE_SET: ${{ matrix.resource-set }} + run: | + repository=$(printf '%s' "ghcr.io/${GITHUB_REPOSITORY}/localai-test-resources" | tr '[:upper:]' '[:lower:]') + digest=$(jq -r --arg set "$RESOURCE_SET" '.bundles[$set] | sub("sha256:"; "sha256-")' test-resources/manifests/lock.json) + oras push \ + --artifact-type application/vnd.localai.test-resources.v1 \ + "${repository}:${RESOURCE_SET},${RESOURCE_SET}-${digest}" \ + ".cache/test-resources/bundles/${RESOURCE_SET}.tar.zst:application/vnd.localai.test-resources.bundle.v1+zstd" \ + "test-resources/manifests/${RESOURCE_SET}.json:application/vnd.localai.test-resources.manifest.v1+json" + - name: Fail after preserving evidence + if: steps.refresh.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7929f6b65..71b2469ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,8 @@ jobs: # You can test your matrix by printing the current Go version - name: Display Go version run: go version + - name: Download Go modules + run: go mod download - name: Proto Dependencies run: | # Install protoc @@ -58,13 +60,30 @@ jobs: node-version: '22' - name: Build React UI run: make react-ui + - name: Record and pack declared test resources + run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=default + - name: Transfer local test-resource bundle + uses: actions/upload-artifact@v4 + with: + name: test-resources-default-${{ github.run_id }} + include-hidden-files: true + path: | + .cache/test-resources/bundles/default.tar.zst + test-resources/manifests/lock.json + - name: Clear recorded resource cache + run: rm -rf .cache/test-resources + - name: Restore local test-resource bundle + uses: actions/download-artifact@v4 + with: + name: test-resources-default-${{ github.run_id }} + path: . # Runs the core suite with coverage and fails if total coverage dropped # below the committed baseline (coverage-baseline.txt). The gate is # strict — any decrease fails. Raise the baseline with # `make test-coverage-baseline` and commit it when coverage rises. - name: Test (with coverage gate) run: | - PATH="$PATH:/root/go/bin" make --jobs 5 --output-sync=target test-coverage-check + LOCALAI_TEST_KERNEL_ENFORCE=1 PATH="$PATH:/root/go/bin" make --jobs 5 --output-sync=target test-coverage-check # tests/integration is outside the coverage roots because its store specs # need a live backend. test-stores builds and installs local-store before # running the complete suite, so new local-store specs are collected @@ -106,6 +125,8 @@ jobs: # You can test your matrix by printing the current Go version - name: Display Go version run: go version + - name: Download Go modules + run: go mod download - name: Dependencies run: | brew install protobuf grpc make protoc-gen-go protoc-gen-go-grpc libomp llvm opus ffmpeg @@ -124,7 +145,7 @@ jobs: # Used to run the newer GNUMake version from brew that supports --output-sync export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH" PATH="$PATH:$HOME/go/bin" make protogen-go - PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target test + PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target TEST_RESOURCE_SET=default-darwin test - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.23 diff --git a/.github/workflows/tests-aio.yml b/.github/workflows/tests-aio.yml index f8d3d34f0..4ca29730c 100644 --- a/.github/workflows/tests-aio.yml +++ b/.github/workflows/tests-aio.yml @@ -76,7 +76,9 @@ jobs: PATH="$PATH:$HOME/go/bin" make protogen-go - name: Test run: | - PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e e2e-aio + PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e + LOCALAI_TEST_RESOURCES_ONLINE=1 PATH="$PATH:$HOME/go/bin" make update-offline-test-cache TEST_RESOURCE_SET=aio + LOCALAI_BACKEND_DIR="$GITHUB_WORKSPACE/backends" LOCALAI_MODELS_DIR="$GITHUB_WORKSPACE/tests/e2e-aio/models" LOCALAI_IMAGE_TAG=tests LOCALAI_IMAGE=local-ai PATH="$PATH:$HOME/go/bin" make run-e2e-aio - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.23 diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index 3c1cb711c..df31ff98b 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -60,9 +60,12 @@ jobs: node-version: '22' - name: Build React UI run: make react-ui + - name: Record declared distributed test resources + run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=distributed-e2e - name: Test Backend E2E run: | PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e + PATH="$PATH:$HOME/go/bin" make test-e2e-distributed - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.23 diff --git a/AGENTS.md b/AGENTS.md index 71302268a..74a15f034 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ LocalAI follows the Linux kernel project's [guidelines for AI coding assistants] |------|-------------| | [.agents/ai-coding-assistants.md](.agents/ai-coding-assistants.md) | Policy for AI-assisted contributions — licensing, DCO, attribution | | [.agents/building-and-testing.md](.agents/building-and-testing.md) | Building the project, running tests, Docker builds for specific platforms | -| [.agents/ci-caching.md](.agents/ci-caching.md) | CI build cache layout (registry-backed BuildKit cache on quay.io/go-skynet/ci-cache, per-arch keys), `DEPS_REFRESH` weekly cache-buster for unpinned Python deps, prebuilt `base-grpc-*` images for llama.cpp variants, per-arch native + manifest-merge pattern, `setup-build-disk` `/mnt` relocation, path filter on master push, manual eviction | +| [.agents/ci-caching.md](.agents/ci-caching.md) | CI build cache layout (registry-backed BuildKit cache on quay.io/go-skynet/ci-cache, per-arch keys), `DEPS_REFRESH` weekly cache-buster for unpinned Python deps, jetson wheels mirror for l4t builds (ghcr-hosted, survives pypi.jetson-ai-lab.io outages), prebuilt `base-grpc-*` images for llama.cpp variants, per-arch native + manifest-merge pattern, `setup-build-disk` `/mnt` relocation, path filter on master push, manual eviction | | [.agents/adding-backends.md](.agents/adding-backends.md) | Adding a new backend (Python, Go, or C++) — full step-by-step checklist, including importer integration (the `/import-model` dropdown is server-driven from `GET /backends/known`) | | [.agents/coding-style.md](.agents/coding-style.md) | Code style, editorconfig, logging, documentation conventions | | [.agents/llama-cpp-backend.md](.agents/llama-cpp-backend.md) | Working on the llama.cpp backend — architecture, updating, tool call parsing | diff --git a/Dockerfile b/Dockerfile index 4ca9b3279..f9358a7d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,15 @@ ARG UBUNTU_CODENAME=noble ARG APT_MIRROR="" ARG APT_PORTS_MIRROR="" +FROM alpine:3.22 AS ca-certificates + FROM ${BASE_IMAGE} AS requirements +COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ + CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + PIP_CERT=/etc/ssl/certs/ca-certificates.crt ARG APT_MIRROR ARG APT_PORTS_MIRROR @@ -15,7 +23,7 @@ ENV DEBIAN_FRONTEND=noninteractive # hwdata ships /usr/share/hwdata/pci.ids. Without it, the ghw library we use # for hardware detection cannot resolve PCI vendor IDs and fails to enumerate # GPUs at all, so the image reports "No GPU detected" (see issue #10941). -RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ APT_MIRROR="${APT_MIRROR}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR}" sh /usr/local/sbin/apt-mirror && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -37,11 +45,11 @@ ARG TARGETVARIANT ENV BUILD_TYPE=${BUILD_TYPE} ARG UBUNTU_VERSION=2404 -RUN mkdir -p /run/localai -RUN echo "default" > /run/localai/capability +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 mkdir -p /run/localai +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 echo "default" > /run/localai/capability # Vulkan requirements -RUN < /run/localai/capability fi EOT # https://github.com/NVIDIA/Isaac-GR00T/issues/343 -RUN < /run/localai/capability || echo "not intel" +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 expr "${BUILD_TYPE}" = intel && echo "intel" > /run/localai/capability || echo "not intel" # Cuda ENV PATH=/usr/local/cuda/bin:${PATH} @@ -206,7 +214,7 @@ ARG CMAKE_FROM_SOURCE=false ARG TARGETARCH ARG TARGETVARIANT -RUN apt-get update && \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ ccache \ @@ -220,7 +228,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* # Install CMake (the version in 22.04 is too old) -RUN < /etc/apt/sources.list.d/intel-graphics.list -RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu ${UBUNTU_CODENAME}/lts/2350 unified" > /etc/apt/sources.list.d/intel-graphics.list +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ APT_MIRROR="${APT_MIRROR}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR}" sh /usr/local/sbin/apt-mirror && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -298,13 +307,13 @@ ENV NVIDIA_REQUIRE_CUDA="cuda>=${CUDA_MAJOR_VERSION}.0" ENV NVIDIA_VISIBLE_DEVICES=all ENV LD_FLAGS=${LD_FLAGS} -RUN echo "GO_TAGS: $GO_TAGS" && echo "TARGETARCH: $TARGETARCH" +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 echo "GO_TAGS: $GO_TAGS" && echo "TARGETARCH: $TARGETARCH" WORKDIR /build # We need protoc installed, and the version in 22.04 is too old. -RUN </dev/null 2>&1; then \ echo "==> prebuilding engine for ${BACKEND} (cacheable layer)" && \ make engine; \ @@ -418,7 +421,7 @@ COPY . /LocalAI # The engine variants built above survive this COPY (they are build outputs, not # tracked files) and are newer than the pinned clone, so make treats them as up # to date and goes straight to the Go binary. -RUN cd /LocalAI && make protogen-go && make -C /LocalAI/backend/go/${BACKEND} build +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cd /LocalAI && make protogen-go && make -C /LocalAI/backend/go/${BACKEND} build FROM scratch ARG BACKEND=rerankers diff --git a/backend/Dockerfile.ik-llama-cpp b/backend/Dockerfile.ik-llama-cpp index 9694441b0..9088d96db 100644 --- a/backend/Dockerfile.ik-llama-cpp +++ b/backend/Dockerfile.ik-llama-cpp @@ -24,7 +24,10 @@ ARG APT_PORTS_MIRROR="" # runs, so the result is bit-equivalent to the prebuilt-base path # (builder-prebuilt below). # ============================================================================ +FROM alpine:3.22 AS ca-certificates + FROM ${BASE_IMAGE} AS builder-fromsource +COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt ARG BUILD_TYPE ARG CUDA_MAJOR_VERSION ARG CUDA_MINOR_VERSION @@ -73,13 +76,14 @@ WORKDIR /build # Install everything via the shared script — the same one that # backend/Dockerfile.base-grpc-builder runs, so the prebuilt CI base and # this from-source path are bit-equivalent. -RUN --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ + --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca \ bash /usr/local/sbin/install-base-deps # Mirror builder-prebuilt: copy gRPC from /opt/grpc to /usr/local so # CMake's find_package finds it at the canonical prefix the Makefile expects. -RUN cp -a /opt/grpc/. /usr/local/ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI @@ -89,13 +93,13 @@ COPY . /LocalAI # different source. # # The compile body is shared with builder-prebuilt via .docker/ik-llama-cpp-compile.sh. -RUN --mount=type=bind,source=.docker/ik-llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/ik-llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ --mount=type=cache,target=/root/.ccache,id=ik-llama-cpp-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ bash /usr/local/sbin/compile.sh # Copy libraries using a script to handle architecture differences -RUN make -BC /LocalAI/backend/cpp/ik-llama-cpp package +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 make -BC /LocalAI/backend/cpp/ik-llama-cpp package # ============================================================================ @@ -120,15 +124,15 @@ ARG TARGETVARIANT # The base-grpc-* image installs gRPC to /opt/grpc but doesn't copy it to # /usr/local. Mirror what the from-source path does so the compile step # can find gRPC at the canonical prefix the Makefile expects. -RUN cp -a /opt/grpc/. /usr/local/ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI -RUN --mount=type=bind,source=.docker/ik-llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/ik-llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ --mount=type=cache,target=/root/.ccache,id=ik-llama-cpp-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ bash /usr/local/sbin/compile.sh -RUN make -BC /LocalAI/backend/cpp/ik-llama-cpp package +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 make -BC /LocalAI/backend/cpp/ik-llama-cpp package # ============================================================================ diff --git a/backend/Dockerfile.llama-cpp b/backend/Dockerfile.llama-cpp index 2f21aaa8e..455356ffa 100644 --- a/backend/Dockerfile.llama-cpp +++ b/backend/Dockerfile.llama-cpp @@ -24,7 +24,10 @@ ARG APT_PORTS_MIRROR="" # runs, so the result is bit-equivalent to the prebuilt-base path # (builder-prebuilt below). # ============================================================================ +FROM alpine:3.22 AS ca-certificates + FROM ${BASE_IMAGE} AS builder-fromsource +COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt ARG BUILD_TYPE ARG CUDA_MAJOR_VERSION ARG CUDA_MINOR_VERSION @@ -72,13 +75,14 @@ WORKDIR /build # Install everything via the shared script — the same one that # backend/Dockerfile.base-grpc-builder runs, so the prebuilt CI base and # this from-source path are bit-equivalent. -RUN --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ + --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca \ bash /usr/local/sbin/install-base-deps # Mirror builder-prebuilt: copy gRPC from /opt/grpc to /usr/local so # CMake's find_package finds it at the canonical prefix the Makefile expects. -RUN cp -a /opt/grpc/. /usr/local/ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI @@ -92,13 +96,13 @@ COPY . /LocalAI # share the same cache mount id. # # The compile body is shared with builder-prebuilt via .docker/llama-cpp-compile.sh. -RUN --mount=type=bind,source=.docker/llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ --mount=type=cache,target=/root/.ccache,id=llama-cpp-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ bash /usr/local/sbin/compile.sh # Copy libraries using a script to handle architecture differences -RUN make -BC /LocalAI/backend/cpp/llama-cpp package +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 make -BC /LocalAI/backend/cpp/llama-cpp package # ============================================================================ @@ -130,15 +134,15 @@ ARG TARGETVARIANT # /usr/local. The variant Dockerfile's from-source path does that too; # mirror it here so the compile step can find gRPC at the canonical # prefix the Makefile expects. -RUN cp -a /opt/grpc/. /usr/local/ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI -RUN --mount=type=bind,source=.docker/llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/llama-cpp-compile.sh,target=/usr/local/sbin/compile.sh \ --mount=type=cache,target=/root/.ccache,id=llama-cpp-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ bash /usr/local/sbin/compile.sh -RUN make -BC /LocalAI/backend/cpp/llama-cpp package +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 make -BC /LocalAI/backend/cpp/llama-cpp package # ============================================================================ diff --git a/backend/Dockerfile.privacy-filter b/backend/Dockerfile.privacy-filter index 97bd48380..81a20c5ea 100644 --- a/backend/Dockerfile.privacy-filter +++ b/backend/Dockerfile.privacy-filter @@ -28,7 +28,10 @@ ARG APT_PORTS_MIRROR="" # bit-equivalent to the prebuilt base. Used when BUILDER_TARGET=builder-fromsource # (the default; local `make backends/privacy-filter`). # ============================================================================ +FROM alpine:3.22 AS ca-certificates + FROM ${BASE_IMAGE} AS builder-fromsource +COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt ARG BUILD_TYPE ARG CUDA_MAJOR_VERSION ARG CUDA_MINOR_VERSION @@ -63,17 +66,19 @@ WORKDIR /build # apt deps + cmake + protoc + gRPC + conditional CUDA/Vulkan, all from the # shared script (the source of truth that base-grpc-builder also runs). -RUN --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \ --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ + --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca \ bash /usr/local/sbin/install-base-deps # install-base-deps installs gRPC under /opt/grpc; copy it to /usr/local so the # backend's find_package(gRPC CONFIG) resolves it at the canonical prefix. -RUN cp -a /opt/grpc/. /usr/local/ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI -RUN --mount=type=cache,target=/root/.ccache,id=privacy-filter-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=cache,target=/root/.ccache,id=privacy-filter-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ + sh /LocalAI/.docker/install-build-proxy-ca.sh && \ make -C /LocalAI/backend/cpp/privacy-filter BUILD_TYPE=${BUILD_TYPE} NATIVE=false grpc-server package # ============================================================================ @@ -91,11 +96,12 @@ ENV PATH=/usr/local/cuda/bin:${PATH} # Mirror builder-fromsource: the base-grpc image installs gRPC to /opt/grpc but # does not copy it to /usr/local. -RUN cp -a /opt/grpc/. /usr/local/ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 cp -a /opt/grpc/. /usr/local/ COPY . /LocalAI -RUN --mount=type=cache,target=/root/.ccache,id=privacy-filter-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=cache,target=/root/.ccache,id=privacy-filter-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ + sh /LocalAI/.docker/install-build-proxy-ca.sh && \ make -C /LocalAI/backend/cpp/privacy-filter BUILD_TYPE=${BUILD_TYPE} NATIVE=false grpc-server package # ============================================================================ diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 2522a6f56..0f7d5af66 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -1,8 +1,23 @@ ARG BASE_IMAGE=ubuntu:24.04 ARG APT_MIRROR="" ARG APT_PORTS_MIRROR="" +# CI mirror of the CUDA aarch64 wheels from pypi.jetson-ai-lab.io, kept warm +# by .github/workflows/jetson-wheels.yml so l4t builds survive the upstream +# index's recurring multi-hour outages. The default (scratch) mounts an empty +# directory, which makes installRequirements fall through to the upstream +# index unchanged — local and Jetson-native builds are unaffected. +ARG JETSON_WHEELS_IMAGE=scratch + +FROM ${JETSON_WHEELS_IMAGE} AS jetson-wheels +FROM alpine:3.22 AS ca-certificates FROM ${BASE_IMAGE} AS builder +COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ + CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + PIP_CERT=/etc/ssl/certs/ca-certificates.crt ARG BACKEND=rerankers ARG BUILD_TYPE ENV BUILD_TYPE=${BUILD_TYPE} @@ -18,7 +33,7 @@ ARG UBUNTU_VERSION=2404 ARG APT_MIRROR ARG APT_PORTS_MIRROR -RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ APT_MIRROR="${APT_MIRROR}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR}" sh /usr/local/sbin/apt-mirror && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -40,7 +55,7 @@ RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mi apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN </dev/null || true + _JETSON_MIRROR_PID="" + _JETSON_MIRROR_URL="" + fi +} + +function _startJetsonMirror() { + local script_dir port_file port tries + # An empty dir is the JETSON_WHEELS_IMAGE=scratch default in + # Dockerfile.python: no mirror was provided, use upstream as-is. + if [ -z "$(find "${JETSON_WHEELS_DIR}" -name '*.whl' -print -quit 2>/dev/null)" ]; then + echo "jetson wheels dir ${JETSON_WHEELS_DIR} has no wheels, using upstream ${JETSON_PYPI_HOST}" + return 0 + fi + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + port_file="$(mktemp)" + rm -f "${port_file}" + python3 "${script_dir}/pypi_mirror_server.py" --root "${JETSON_WHEELS_DIR}" --port-file "${port_file}" & + _JETSON_MIRROR_PID=$! + trap _stopJetsonMirror EXIT + tries=0 + until [ -s "${port_file}" ]; do + tries=$((tries + 1)) + if [ ${tries} -gt 50 ] || ! kill -0 "${_JETSON_MIRROR_PID}" 2>/dev/null; then + echo "WARNING: local jetson wheel mirror failed to start, using upstream ${JETSON_PYPI_HOST}" + _stopJetsonMirror + return 0 + fi + sleep 0.2 + done + port="$(cat "${port_file}")" + rm -f "${port_file}" + _JETSON_MIRROR_URL="http://127.0.0.1:${port}" + echo "serving jetson wheels from ${JETSON_WHEELS_DIR} at ${_JETSON_MIRROR_URL}" +} + # installRequirements looks for several requirements files and if they exist runs the install for them in order # # - requirements-install.txt @@ -520,18 +576,31 @@ function installRequirements() { export C_INCLUDE_PATH="${C_INCLUDE_PATH:-}:$(_portable_dir)/include/python${PYTHON_VERSION}" fi + if [ -n "${JETSON_WHEELS_DIR:-}" ] && [ -d "${JETSON_WHEELS_DIR}" ]; then + _startJetsonMirror + fi + + local installFile for reqFile in ${requirementFiles[@]}; do if [ -f "${reqFile}" ]; then + installFile="${reqFile}" + if [ -n "${_JETSON_MIRROR_URL}" ] && grep -q "${JETSON_PYPI_HOST}" "${reqFile}"; then + installFile="$(mktemp)" + sed "s,https://${JETSON_PYPI_HOST},${_JETSON_MIRROR_URL},g" "${reqFile}" > "${installFile}" + echo "rewrote ${JETSON_PYPI_HOST} in ${reqFile} to the local wheel mirror (${installFile})" + fi echo "starting requirements install for ${reqFile}" if [ "x${USE_PIP}" == "xtrue" ]; then - pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${reqFile}" + pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${installFile}" else - uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${reqFile}" + uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${installFile}" fi echo "finished requirements install for ${reqFile}" fi done + _stopJetsonMirror + runProtogen } diff --git a/backend/python/common/pypi_mirror_server.py b/backend/python/common/pypi_mirror_server.py new file mode 100644 index 000000000..394c3f000 --- /dev/null +++ b/backend/python/common/pypi_mirror_server.py @@ -0,0 +1,226 @@ +"""Ephemeral PEP 503 "simple" index over a local directory of wheels. + +Serves a directory tree laid out like a package index (e.g. +``/jp6/cu129/torch/torch-2.8.0-cp312-...whl``) as a standards-compliant +"simple" index on localhost, so uv/pip can resolve against it exactly as they +would against the real remote index — same per-project pages, same 404 +fall-through to PyPI for projects the mirror does not carry. + +This exists because pypi.jetson-ai-lab.io (the only source of CUDA-enabled +aarch64 torch wheels for JetPack) has a history of multi-hour 502 outages, +and an --extra-index-url that errors is fatal to the whole resolution: uv +consults every configured index for every requirement, so one 502 on any +project page kills the install even for projects hosted on PyPI. CI mirrors +the handful of jetson-only wheels into an OCI image, bind-mounts it into the +backend build, and libbackend.sh serves it with this script while rewriting +the index host in the requirements files to 127.0.0.1 (see +installRequirements in libbackend.sh). A 404 from this server is a clean +"not here" that resolvers follow up on PyPI; the upstream 502 never was. + +Standard library only — it runs inside every python backend's build +container, before any venv exists. + +Usage: + python3 pypi_mirror_server.py --root /jetson-wheels --port-file /tmp/port + +Run the tests standalone: + python3 -m unittest pypi_mirror_server_test +""" + +import argparse +import hashlib +import html +import os +import re +import sys +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +# Extensions treated as distribution files: a directory containing at least +# one of these is a project page; any other directory is a sub-index listing. +DIST_SUFFIXES = (".whl", ".tar.gz", ".zip") + +_hash_cache = {} +_hash_lock = threading.Lock() + + +def normalize(name): + """PEP 503 project-name normalization.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def _file_sha256(path): + """sha256 of a file, cached on (path, mtime, size) — wheels are large.""" + st = os.stat(path) + key = (path, st.st_mtime_ns, st.st_size) + with _hash_lock: + cached = _hash_cache.get(key) + if cached: + return cached + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + value = digest.hexdigest() + with _hash_lock: + _hash_cache[key] = value + return value + + +def resolve_path(root, url_path): + """Map a URL path onto the tree under root, or None. + + Path segments match either exactly or via PEP 503 normalization + (resolvers request ``liquid-audio`` even if the directory on disk is + named ``liquid_audio``). Rejects any segment that would escape root. + """ + current = root + for segment in url_path.split("/"): + if segment in ("", "."): + continue + if segment == ".." or "/" in segment or "\\" in segment: + return None + candidate = os.path.join(current, segment) + if not os.path.exists(candidate): + try: + entries = os.listdir(current) + except (NotADirectoryError, FileNotFoundError): + return None + wanted = normalize(segment) + matches = [e for e in entries if normalize(e) == wanted] + if not matches: + return None + candidate = os.path.join(current, matches[0]) + current = candidate + return current + + +class SimpleIndexHandler(BaseHTTPRequestHandler): + root = None + protocol_version = "HTTP/1.1" + + def do_GET(self): + self._respond(head_only=False) + + def do_HEAD(self): + self._respond(head_only=True) + + def _respond(self, head_only): + url_path = urllib.parse.unquote(urllib.parse.urlsplit(self.path).path) + local = resolve_path(self.root, url_path) + if local is None: + self._send_error(404, "not found") + return + if os.path.isfile(local): + self._send_file(local, head_only) + return + # Relative hrefs on index pages resolve against the request URL, so + # directory URLs must end in "/" — redirect like real indexes do. + if not url_path.endswith("/"): + self.send_response(301) + self.send_header("Location", self.path + "/") + self.send_header("Content-Length", "0") + self.end_headers() + return + entries = sorted(os.listdir(local)) + files = [e for e in entries if e.endswith(DIST_SUFFIXES)] + if files: + body = self._project_page(local, files) + else: + dirs = [e for e in entries if os.path.isdir(os.path.join(local, e))] + body = self._listing_page(dirs) + self._send_html(body, head_only) + + def _project_page(self, project_dir, files): + anchors = [] + for name in files: + digest = _file_sha256(os.path.join(project_dir, name)) + anchors.append( + '%s
' + % (urllib.parse.quote(name), digest, html.escape(name)) + ) + return self._page(anchors) + + def _listing_page(self, dirs): + anchors = [ + '%s
' + % (urllib.parse.quote(normalize(d)), html.escape(normalize(d))) + for d in dirs + ] + return self._page(anchors) + + def _page(self, anchors): + return ( + "" + '' + "simple index\n" + + "\n".join(anchors) + + "\n" + ).encode() + + def _send_html(self, body, head_only): + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if not head_only: + self.wfile.write(body) + + def _send_file(self, path, head_only): + size = os.path.getsize(path) + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(size)) + self.end_headers() + if head_only: + return + with open(path, "rb") as f: + while True: + chunk = f.read(1 << 20) + if not chunk: + break + self.wfile.write(chunk) + + def _send_error(self, code, message): + body = message.encode() + self.send_response(code) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + sys.stderr.write("pypi-mirror: %s\n" % (format % args)) + + +def make_server(root, host="127.0.0.1", port=0): + handler = type("Handler", (SimpleIndexHandler,), {"root": os.path.abspath(root)}) + return ThreadingHTTPServer((host, port), handler) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", required=True, help="directory tree to serve") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=0, help="0 picks a free port") + parser.add_argument( + "--port-file", + help="write the bound port here once listening (readiness signal)", + ) + args = parser.parse_args() + + server = make_server(args.root, args.host, args.port) + port = server.server_address[1] + if args.port_file: + # Write-then-rename so a reader never sees a partially written port. + tmp = args.port_file + ".tmp" + with open(tmp, "w") as f: + f.write(str(port)) + os.replace(tmp, args.port_file) + sys.stderr.write("pypi-mirror: serving %s on %s:%d\n" % (args.root, args.host, port)) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/backend/python/common/pypi_mirror_server_test.py b/backend/python/common/pypi_mirror_server_test.py new file mode 100644 index 000000000..5121b159d --- /dev/null +++ b/backend/python/common/pypi_mirror_server_test.py @@ -0,0 +1,100 @@ +"""Unit tests for the ephemeral PEP 503 index (pypi_mirror_server.py). + +Run standalone (Python standard library only, no backend venv needed): + python3 -m unittest pypi_mirror_server_test +""" + +import hashlib +import os +import shutil +import tempfile +import threading +import unittest +import urllib.error +import urllib.request + +from pypi_mirror_server import make_server, normalize, resolve_path + +WHEEL_BYTES = b"not a real wheel, but the server must serve it verbatim" + + +class TestHelpers(unittest.TestCase): + def test_normalize(self): + self.assertEqual(normalize("Liquid_Audio.Extra"), "liquid-audio-extra") + self.assertEqual(normalize("torch"), "torch") + + def test_resolve_rejects_traversal(self): + root = tempfile.mkdtemp() + try: + self.assertIsNone(resolve_path(root, "/../etc/passwd")) + self.assertIsNone(resolve_path(root, "/a/../../etc")) + finally: + shutil.rmtree(root) + + def test_resolve_normalized_segment(self): + root = tempfile.mkdtemp() + try: + os.makedirs(os.path.join(root, "jp6", "liquid_audio")) + found = resolve_path(root, "/jp6/liquid-audio/") + self.assertEqual(found, os.path.join(root, "jp6", "liquid_audio")) + finally: + shutil.rmtree(root) + + +class TestServer(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.root = tempfile.mkdtemp() + project = os.path.join(cls.root, "jp6", "cu129", "torch") + os.makedirs(project) + cls.wheel_name = "torch-2.8.0-cp312-cp312-linux_aarch64.whl" + with open(os.path.join(project, cls.wheel_name), "wb") as f: + f.write(WHEEL_BYTES) + cls.server = make_server(cls.root) + cls.base = "http://127.0.0.1:%d" % cls.server.server_address[1] + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + shutil.rmtree(cls.root) + + def _get(self, path): + with urllib.request.urlopen(self.base + path) as resp: + return resp.status, resp.read() + + def test_project_page_lists_wheel_with_hash(self): + status, body = self._get("/jp6/cu129/torch/") + self.assertEqual(status, 200) + digest = hashlib.sha256(WHEEL_BYTES).hexdigest() + self.assertIn( + ('' % (self.wheel_name, digest)).encode(), body + ) + + def test_index_listing_names_projects(self): + status, body = self._get("/jp6/cu129/") + self.assertEqual(status, 200) + self.assertIn(b'torch', body) + + def test_wheel_download_is_verbatim(self): + status, body = self._get("/jp6/cu129/torch/" + self.wheel_name) + self.assertEqual(status, 200) + self.assertEqual(body, WHEEL_BYTES) + + def test_unknown_project_is_404(self): + # 404 (not 5xx) matters: resolvers treat it as "not in this index" + # and fall back to PyPI, which is the whole point of the mirror. + with self.assertRaises(urllib.error.HTTPError) as ctx: + self._get("/jp6/cu129/liquid-audio/") + self.assertEqual(ctx.exception.code, 404) + + def test_directory_without_slash_redirects(self): + status, _ = self._get("/jp6/cu129/torch") + # urllib follows the 301; landing on the page proves the redirect + self.assertEqual(status, 200) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmd/build-proxy/main.go b/cmd/build-proxy/main.go new file mode 100644 index 000000000..f9656fbf3 --- /dev/null +++ b/cmd/build-proxy/main.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/mudler/LocalAI/core/services/buildproxy" +) + +func main() { + listen := flag.String("listen", "127.0.0.1:18080", "proxy listen address") + output := flag.String("output", ".cache/build-proxy", "telemetry directory") + flag.Parse() + if err := run(*listen, *output); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(listen, output string) error { + recorder, err := buildproxy.NewRecorder(filepath.Join(output, "events.jsonl")) + if err != nil { + return err + } + defer func() { _ = recorder.Close() }() + proxyHandler := buildproxy.NewHandler(buildproxy.Options{Recorder: recorder}) + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { proxyHandler(w, r, r.URL.Host) }) + server, err := buildproxy.NewServer(listen, filepath.Join(output, "ca"), handler, recorder) + if err != nil { + return err + } + if err := server.Start(); err != nil { + return err + } + fmt.Printf("proxy=http://%s\nca=%s\n", server.Addr(), server.CAPath()) + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + <-stop + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := server.Stop(ctx); err != nil { + return err + } + return recorder.WriteSummary(filepath.Join(output, "summary.json")) +} diff --git a/cmd/test-resources/main.go b/cmd/test-resources/main.go new file mode 100644 index 000000000..7e5eaad3b --- /dev/null +++ b/cmd/test-resources/main.go @@ -0,0 +1,468 @@ +// SPDX-License-Identifier: MIT + +package main + +import ( + "archive/tar" + "crypto/sha256" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/mudler/LocalAI/core/services/cloudproxy/mitm" + "github.com/mudler/LocalAI/internal/testresources" + "github.com/mudler/LocalAI/pkg/httpclient" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, "test-resources:", err) + os.Exit(1) + } +} + +func run(args []string) error { + if len(args) >= 6 && args[0] == "run" && args[4] == "--" { + return runOffline(args[1], args[2], args[3], args[5:]) + } + if len(args) != 4 { + return errors.New("usage: test-resources RESOURCE_SET MANIFEST_DIR CACHE_DIR | test-resources run RESOURCE_SET MANIFEST_DIR CACHE_DIR -- COMMAND") + } + target, manifestDir, cacheDir := args[1], args[2], args[3] + if args[0] == "update" { + return update(target, manifestDir, cacheDir) + } + if args[0] != "prepare" { + return errors.New("usage: test-resources RESOURCE_SET MANIFEST_DIR CACHE_DIR") + } + return prepare(target, manifestDir, cacheDir) +} + +func prepare(target, manifestDir, cacheDir string) error { + manifest, err := testresources.LoadManifest(filepath.Join(manifestDir, target+".json")) + if err != nil { + return fmt.Errorf("%w; run `make update-offline-test-cache TEST_RESOURCE_SET=%s`", err, target) + } + if manifest.Target != target { + return fmt.Errorf("manifest target %q does not match %q", manifest.Target, target) + } + lock, err := testresources.LoadLock(filepath.Join(manifestDir, "lock.json")) + if err != nil { + return err + } + locked, ok := lock.Bundles[target] + if !ok { + return fmt.Errorf("cache bundle is not locked for resource set %q; run `make update-offline-test-cache TEST_RESOURCE_SET=%s`", target, target) + } + if digest, ok := strings.CutPrefix(locked, "sha256:"); ok { + bundlePath := filepath.Join(cacheDir, "bundles", target+".tar.zst") + if _, err := os.Stat(bundlePath); errors.Is(err, os.ErrNotExist) { + bundlePath = filepath.Join(cacheDir, "bundles", target+".tar") + } + if err := testresources.RestoreBundle(cacheDir, bundlePath, digest); err != nil { + return preparationError(target, err) + } + } + materialized := filepath.Join(cacheDir, "materialized", target) + if err := os.MkdirAll(materialized, 0o755); err != nil { + return err + } + index, err := testresources.LoadHTTPIndex(cacheDir) + if err != nil { + return preparationError(target, err) + } + for _, resource := range manifest.HTTP { + _, err := testresources.VerifyBlob(cacheDir, resource.SHA256) + if err != nil { + return preparationError(target, err) + } + entry, ok := index[testresources.RequestKey(resource.Method, resource.URL, resource.Headers())] + if !ok || entry.Digest != resource.SHA256 { + return preparationError(target, fmt.Errorf("HTTP cache entry missing or mismatched: %s %s", resource.Method, resource.URL)) + } + } + for _, resource := range manifest.Files { + path, err := testresources.VerifyBlob(cacheDir, resource.SHA256) + if err != nil { + return preparationError(target, err) + } + environmentPath := path + if resource.Destination != "" { + destination := filepath.Join(materialized, resource.Destination) + if err := copyFile(path, destination); err != nil { + return err + } + environmentPath = destination + } + if resource.Environment != "" { + if err := os.Setenv(resource.Environment, environmentPath); err != nil { + return err + } + } + } + for _, resource := range manifest.Images { + path, err := testresources.VerifyBlob(cacheDir, resource.SHA256) + if err != nil { + return preparationError(target, err) + } + cmd := exec.Command("docker", "load", "--input", path) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("load declared image %s: %w (verify Docker is running and this user can access its socket)", resource.Reference, err) + } + } + return nil +} + +func update(target, manifestDir, cacheDir string) error { + if os.Getenv("LOCALAI_TEST_RESOURCES_ONLINE") != "1" { + return errors.New("update requires explicit online record mode: LOCALAI_TEST_RESOURCES_ONLINE=1") + } + manifest, err := testresources.LoadManifest(filepath.Join(manifestDir, target+".json")) + if err != nil { + return err + } + client := httpclient.New() + client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse } + index, err := testresources.LoadHTTPIndex(cacheDir) + if err != nil { + return err + } + for _, resource := range manifest.HTTP { + entry, err := fetchHTTPWithMirrors(client, resource, cacheDir) + if err != nil { + return err + } + index[testresources.RequestKey(resource.Method, resource.URL, resource.Headers())] = entry + } + if err := testresources.WriteHTTPIndex(cacheDir, index); err != nil { + return err + } + for _, resource := range manifest.Files { + if err := fetchWithMirrors(client, resource.URL, resource.Mirrors, resource.SHA256, cacheDir); err != nil { + return err + } + } + for i := range manifest.Images { + digest, err := pullAndPack(manifest.Images[i].Reference, cacheDir) + if err != nil { + return err + } + manifest.Images[i].SHA256 = digest + } + bundlePath := filepath.Join(cacheDir, "bundles", target+".tar.zst") + digest, err := testresources.PackBundle(cacheDir, bundlePath, manifest) + if err != nil { + return err + } + if err := testresources.WriteManifest(filepath.Join(manifestDir, target+".json"), manifest); err != nil { + return err + } + lockPath := filepath.Join(manifestDir, "lock.json") + lock, err := testresources.LoadLock(lockPath) + if err != nil { + return err + } + lock.Bundles[target] = "sha256:" + digest + return testresources.WriteLock(lockPath, lock) +} + +func fetchHTTPWithMirrors(client *http.Client, resource testresources.HTTP, cacheDir string) (testresources.HTTPEntry, error) { + urls := append([]string{resource.URL}, resource.Mirrors...) + var failures []error + for _, candidate := range urls { + for attempt := 1; attempt <= 2; attempt++ { + started := time.Now() + entry, err := fetchHTTP(client, resource, candidate, cacheDir) + fmt.Fprintf(os.Stderr, "test-resources: download %s attempt %d took %s\n", candidate, attempt, time.Since(started).Round(time.Millisecond)) + if err == nil { + return entry, nil + } + failures = append(failures, fmt.Errorf("%s attempt %d: %w", candidate, attempt, err)) + } + } + return testresources.HTTPEntry{}, resourceChangeError(resource.URL, resource.SHA256, failures) +} + +func fetchHTTP(client *http.Client, resource testresources.HTTP, sourceURL, cacheDir string) (testresources.HTTPEntry, error) { + request, err := http.NewRequest(resource.Method, sourceURL, nil) + if err != nil { + return testresources.HTTPEntry{}, err + } + request.Header = resource.Headers() + response, err := client.Do(request) + if err != nil { + return testresources.HTTPEntry{}, fmt.Errorf("fetch %s: %w", resource.URL, err) + } + defer func() { _ = response.Body.Close() }() + size, err := storeVerified(response.Body, resource.SHA256, cacheDir) + if err != nil { + return testresources.HTTPEntry{}, err + } + return testresources.HTTPEntry{Digest: resource.SHA256, Size: size, Status: response.StatusCode, Header: testresources.SanitizeHeaders(response.Header)}, nil +} + +func fetchWithMirrors(client *http.Client, primary string, mirrors []string, expected, cacheDir string) error { + urls := append([]string{primary}, mirrors...) + var failures []error + for _, candidate := range urls { + for attempt := 1; attempt <= 2; attempt++ { + started := time.Now() + err := fetch(client, candidate, expected, cacheDir) + fmt.Fprintf(os.Stderr, "test-resources: download %s attempt %d took %s\n", candidate, attempt, time.Since(started).Round(time.Millisecond)) + if err == nil { + return nil + } + failures = append(failures, fmt.Errorf("%s attempt %d: %w", candidate, attempt, err)) + } + } + return resourceChangeError(primary, expected, failures) +} + +func resourceChangeError(resourceURL, expected string, failures []error) error { + return fmt.Errorf("resource verification failed for %s (expected sha256:%s) after retrying every declared mirror: %w\nsecurity review required before changing the manifest: compare the upstream release checksum/signature and changelog, inspect redirects, and search https://github.com/advisories and https://osv.dev; a mismatch may be an upstream release, mirror corruption, or a supply-chain incident", resourceURL, expected, errors.Join(failures...)) +} + +func fetch(client *http.Client, rawURL, expected, cacheDir string) error { + request, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + return err + } + response, err := client.Do(request) + if err != nil { + return fmt.Errorf("fetch %s: %w", rawURL, err) + } + if response.StatusCode != http.StatusOK { + closeErr := response.Body.Close() + if closeErr != nil { + return errors.Join(fmt.Errorf("fetch %s: status %s", rawURL, response.Status), closeErr) + } + return fmt.Errorf("fetch %s: status %s", rawURL, response.Status) + } + _, storeErr := storeVerified(response.Body, expected, cacheDir) + return errors.Join(storeErr, response.Body.Close()) +} + +func storeVerified(reader io.Reader, expected, cacheDir string) (int64, error) { + directory := filepath.Join(cacheDir, "blobs", "sha256") + if err := os.MkdirAll(directory, 0o755); err != nil { + return 0, err + } + temporary, err := os.CreateTemp(directory, ".record-*") + if err != nil { + return 0, err + } + temporaryName := temporary.Name() + defer func() { _ = os.Remove(temporaryName) }() + hash := sha256.New() + size, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader) + closeErr := temporary.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return 0, err + } + actual := fmt.Sprintf("%x", hash.Sum(nil)) + if actual != expected { + return 0, fmt.Errorf("resource digest mismatch: expected sha256:%s, got sha256:%s", expected, actual) + } + if err := os.Rename(temporaryName, testresources.BlobPath(cacheDir, expected)); err != nil { + return 0, err + } + return size, nil +} + +func pullAndPack(reference, cacheDir string) (string, error) { + if !strings.Contains(reference, "@sha256:") { + return "", fmt.Errorf("refusing mutable image reference %s", reference) + } + if err := exec.Command("docker", "pull", reference).Run(); err != nil { + return "", fmt.Errorf("pull image %s: %w", reference, err) + } + cmd := exec.Command("docker", "save", reference) + stdout, err := cmd.StdoutPipe() + if err != nil { + return "", err + } + if err := cmd.Start(); err != nil { + return "", err + } + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return "", err + } + normalized, err := os.CreateTemp(cacheDir, ".docker-save-*.tar") + if err != nil { + return "", err + } + normalizedName := normalized.Name() + defer func() { _ = os.Remove(normalizedName) }() + normalizeErr := normalizeDockerArchive(stdout, normalized) + waitErr := cmd.Wait() + closeErr := normalized.Close() + if err := errors.Join(normalizeErr, waitErr, closeErr); err != nil { + return "", err + } + input, err := os.Open(normalizedName) + if err != nil { + return "", err + } + digest, _, storeErr := storeContentAddressed(input, cacheDir) + return digest, errors.Join(storeErr, input.Close()) +} + +func normalizeDockerArchive(reader io.Reader, writer io.Writer) error { + tr := tar.NewReader(reader) + tw := tar.NewWriter(writer) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + stable := *header + stable.Uid, stable.Gid = 0, 0 + stable.Uname, stable.Gname = "", "" + stable.ModTime = time.Unix(0, 0).UTC() + stable.AccessTime, stable.ChangeTime = time.Time{}, time.Time{} + stable.PAXRecords, stable.Xattrs = nil, nil + if err := tw.WriteHeader(&stable); err != nil { + return err + } + if _, err := io.Copy(tw, tr); err != nil { + return err + } + } + return tw.Close() +} + +func storeContentAddressed(reader io.Reader, cacheDir string) (string, int64, error) { + directory := filepath.Join(cacheDir, "blobs", "sha256") + if err := os.MkdirAll(directory, 0o755); err != nil { + return "", 0, err + } + temporary, err := os.CreateTemp(directory, ".record-*") + if err != nil { + return "", 0, err + } + name := temporary.Name() + defer func() { _ = os.Remove(name) }() + hash := sha256.New() + size, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader) + closeErr := temporary.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return "", 0, err + } + digest := fmt.Sprintf("%x", hash.Sum(nil)) + if err := os.Rename(name, testresources.BlobPath(cacheDir, digest)); err != nil { + return "", 0, err + } + return digest, size, nil +} + +func preparationError(target string, err error) error { + return fmt.Errorf("%w; run `make prepare-offline-test-cache TEST_RESOURCE_SET=%s` during the network-enabled preparation phase", err, target) +} + +func copyFile(source, destination string) error { + if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { + return err + } + in, err := os.Open(source) + if err != nil { + return err + } + out, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + _ = in.Close() + return err + } + _, copyErr := io.Copy(out, in) + return errors.Join(copyErr, in.Close(), out.Close()) +} + +func runOffline(target, manifestDir, cacheDir string, command []string) error { + manifest, err := testresources.LoadManifest(filepath.Join(manifestDir, target+".json")) + if err != nil { + return err + } + if err := prepare(target, manifestDir, cacheDir); err != nil { + return err + } + dockerNetwork := "" + if runtime.GOOS == "linux" && (len(manifest.Images) > 0 || target == "aio") { + dockerNetwork = fmt.Sprintf("localai-test-%d", os.Getpid()) + create := exec.Command("docker", "network", "create", "--internal", dockerNetwork) + create.Stdout, create.Stderr = io.Discard, os.Stderr + if err := create.Run(); err != nil { + return fmt.Errorf("create internal test Docker network: %w", err) + } + defer func() { _ = exec.Command("docker", "network", "rm", dockerNetwork).Run() }() + } + index, err := testresources.LoadHTTPIndex(cacheDir) + if err != nil { + return err + } + hosts := make([]string, 0, len(manifest.HTTP)) + seen := map[string]bool{} + for _, resource := range manifest.HTTP { + parsed, err := url.Parse(resource.URL) + if err != nil { + return err + } + if parsed.Hostname() != "" && !seen[parsed.Hostname()] { + hosts = append(hosts, parsed.Hostname()) + seen[parsed.Hostname()] = true + } + } + caDir := filepath.Join(cacheDir, "ca") + ca, err := mitm.LoadOrCreateCA(caDir) + if err != nil { + return err + } + server, err := mitm.NewServer(mitm.Config{ + Addr: "127.0.0.1:0", CA: ca, InterceptHosts: hosts, AllowPlainHTTP: true, InterceptAll: true, + Handler: func(w http.ResponseWriter, r *http.Request, _ string) { + key := testresources.RequestKey(r.Method, r.URL.String(), r.Header) + entry, ok := index[key] + if !ok { + http.Error(w, "undeclared test HTTP request: "+key, http.StatusGatewayTimeout) + return + } + if err := testresources.ReplayResponse(w, cacheDir, entry); err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + } + }, + }) + if err != nil { + return err + } + if err := server.Start(); err != nil { + return err + } + defer server.Stop() + proxyURL := "http://" + server.Addr() + caPath := filepath.Join(caDir, "ca.crt") + env := append(os.Environ(), + "LOCALAI_TEST_OFFLINE=1", "HTTP_PROXY="+proxyURL, "HTTPS_PROXY="+proxyURL, + "ALL_PROXY="+proxyURL, "http_proxy="+proxyURL, "https_proxy="+proxyURL, + "all_proxy="+proxyURL, "SSL_CERT_FILE="+caPath, "CURL_CA_BUNDLE="+caPath, + "REQUESTS_CA_BUNDLE="+caPath, "GIT_SSL_CAINFO="+caPath, "NODE_EXTRA_CA_CERTS="+caPath, + "NO_PROXY=localhost,127.0.0.0/8,::1,172.16.0.0/12,192.168.0.0/16", + "no_proxy=localhost,127.0.0.0/8,::1,172.16.0.0/12,192.168.0.0/16", + "TESTCONTAINERS_RYUK_DISABLED=true", + ) + if dockerNetwork != "" { + env = append(env, "LOCALAI_TEST_DOCKER_NETWORK="+dockerNetwork) + } + cmd := exec.Command(command[0], command[1:]...) + cmd.Env, cmd.Stdin, cmd.Stdout, cmd.Stderr = env, os.Stdin, os.Stdout, os.Stderr + return cmd.Run() +} diff --git a/cmd/test-resources/main_test.go b/cmd/test-resources/main_test.go new file mode 100644 index 000000000..8783bd20e --- /dev/null +++ b/cmd/test-resources/main_test.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT + +package main + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "fmt" + "io" + "testing" + "time" + + "github.com/onsi/gomega" +) + +func TestNormalizeDockerArchiveIgnoresTarMetadata(t *testing.T) { + g := gomega.NewWithT(t) + first := dockerArchive(g, time.Unix(100, 0), 12, "builder") + second := dockerArchive(g, time.Unix(200, 0), 34, "runner") + + var normalizedFirst, normalizedSecond bytes.Buffer + g.Expect(normalizeDockerArchive(bytes.NewReader(first), &normalizedFirst)).To(gomega.Succeed()) + g.Expect(normalizeDockerArchive(bytes.NewReader(second), &normalizedSecond)).To(gomega.Succeed()) + firstDigest := fmt.Sprintf("%x", sha256.Sum256(normalizedFirst.Bytes())) + secondDigest := fmt.Sprintf("%x", sha256.Sum256(normalizedSecond.Bytes())) + g.Expect(secondDigest).To(gomega.Equal(firstDigest)) + + tr := tar.NewReader(bytes.NewReader(normalizedFirst.Bytes())) + header, err := tr.Next() + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(header.Uid).To(gomega.Equal(0)) + g.Expect(header.Gid).To(gomega.Equal(0)) + g.Expect(header.Uname).To(gomega.BeEmpty()) + g.Expect(header.Gname).To(gomega.BeEmpty()) + g.Expect(header.ModTime).To(gomega.Equal(time.Unix(0, 0))) + content, err := io.ReadAll(tr) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(string(content)).To(gomega.Equal("image data")) +} + +func dockerArchive(g *gomega.WithT, modTime time.Time, uid int, user string) []byte { + var archive bytes.Buffer + tw := tar.NewWriter(&archive) + content := []byte("image data") + g.Expect(tw.WriteHeader(&tar.Header{ + Name: "layer.tar", Mode: 0o644, Size: int64(len(content)), + ModTime: modTime, Uid: uid, Gid: uid, Uname: user, Gname: user, + })).To(gomega.Succeed()) + _, err := tw.Write(content) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(tw.Close()).To(gomega.Succeed()) + return archive.Bytes() +} diff --git a/core/cli/workerregistry/client.go b/core/cli/workerregistry/client.go index cf46455c9..d4917a29a 100644 --- a/core/cli/workerregistry/client.go +++ b/core/cli/workerregistry/client.go @@ -16,6 +16,7 @@ import ( "github.com/mudler/xlog" + "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/httpclient" ) @@ -109,8 +110,10 @@ func (c *RegistrationClient) Register(ctx context.Context, body map[string]any) // RegisterWithRetry retries registration with exponential backoff. func (c *RegistrationClient) RegisterWithRetry(ctx context.Context, body map[string]any, maxRetries int) (nodeID, apiToken, natsJWT, natsSeed string, err error) { - backoff := 2 * time.Second - maxBackoff := 30 * time.Second + const ( + baseBackoff = 2 * time.Second + maxBackoff = 30 * time.Second + ) for attempt := 1; attempt <= maxRetries; attempt++ { nodeID, apiToken, natsJWT, natsSeed, err = c.Register(ctx, body) @@ -120,13 +123,13 @@ func (c *RegistrationClient) RegisterWithRetry(ctx context.Context, body map[str if attempt == maxRetries { return "", "", "", "", fmt.Errorf("failed after %d attempts: %w", maxRetries, err) } - xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", backoff, "error", err) + delay := backoff.Exponential(baseBackoff, maxBackoff, uint(attempt-1)) + xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", delay, "error", err) select { case <-ctx.Done(): return "", "", "", "", ctx.Err() - case <-time.After(backoff): + case <-time.After(delay): } - backoff = min(backoff*2, maxBackoff) } return nodeID, apiToken, natsJWT, natsSeed, err } diff --git a/core/cli/workerregistry/credentials.go b/core/cli/workerregistry/credentials.go index 24dd6f3c8..e19bbfcee 100644 --- a/core/cli/workerregistry/credentials.go +++ b/core/cli/workerregistry/credentials.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/natsauth" "github.com/mudler/xlog" ) @@ -120,23 +121,23 @@ func (m *NATSCredentialManager) HasCredentials() bool { // credentials are minted. Without requireCreds it returns the first successful // response (the historical one-shot behavior, preserved for anonymous NATS). func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, error) { - backoff := m.initialBackoff var lastReason error for attempt := 1; m.maxAttempts <= 0 || attempt <= m.maxAttempts; attempt++ { + delay := backoff.Exponential(m.initialBackoff, m.maxBackoff, uint(attempt-1)) res, err := m.register(ctx) switch { case err != nil: lastReason = err - xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", backoff, "error", err) + xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", delay, "error", err) case !m.requireCreds: m.store(res) return res, nil case res.Status == statusPending: lastReason = fmt.Errorf("node %s still pending admin approval", res.ID) - xlog.Info("Node pending admin approval; waiting", "node", res.ID, "attempt", attempt, "next_retry", backoff) + xlog.Info("Node pending admin approval; waiting", "node", res.ID, "attempt", attempt, "next_retry", delay) case res.NatsJWT == "" || res.NatsUserSeed == "": lastReason = fmt.Errorf("node %s approved but NATS credentials not minted", res.ID) - xlog.Info("Node approved but NATS credentials not yet minted; waiting", "node", res.ID, "attempt", attempt, "next_retry", backoff) + xlog.Info("Node approved but NATS credentials not yet minted; waiting", "node", res.ID, "attempt", attempt, "next_retry", delay) default: m.store(res) return res, nil @@ -144,9 +145,8 @@ func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, select { case <-ctx.Done(): return nil, ctx.Err() - case <-time.After(backoff): + case <-time.After(delay): } - backoff = min(backoff*2, m.maxBackoff) } return nil, fmt.Errorf("giving up acquiring NATS credentials after %d attempts: %w", m.maxAttempts, lastReason) } diff --git a/core/config/model_config_test.go b/core/config/model_config_test.go index 0abc0c7b6..5d008c43e 100644 --- a/core/config/model_config_test.go +++ b/core/config/model_config_test.go @@ -1,8 +1,6 @@ package config import ( - "io" - "net/http" "os" "path/filepath" @@ -317,17 +315,7 @@ parameters: Expect(valid).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) - // download https://raw.githubusercontent.com/mudler/LocalAI/v2.25.0/embedded/models/hermes-2-pro-mistral.yaml - httpClient := http.Client{} - resp, err := httpClient.Get("https://raw.githubusercontent.com/mudler/LocalAI/v2.25.0/embedded/models/hermes-2-pro-mistral.yaml") - Expect(err).To(BeNil()) - defer resp.Body.Close() - tmp, err = os.CreateTemp("", "config.yaml") - Expect(err).To(BeNil()) - defer os.Remove(tmp.Name()) - _, err = io.Copy(tmp, resp.Body) - Expect(err).To(BeNil()) - configs, err = readModelConfigsFromFile(tmp.Name()) + configs, err = readModelConfigsFromFile(filepath.Join("testdata", "hermes-2-pro-mistral.yaml")) config = configs[0] Expect(err).To(BeNil()) Expect(config).ToNot(BeNil()) diff --git a/core/config/testdata/hermes-2-pro-mistral.yaml b/core/config/testdata/hermes-2-pro-mistral.yaml new file mode 100644 index 000000000..058e14ef3 --- /dev/null +++ b/core/config/testdata/hermes-2-pro-mistral.yaml @@ -0,0 +1,7 @@ +name: hermes-2-pro-mistral +backend: llama-cpp +context_size: 4096 +parameters: + model: hermes-2-pro-mistral.Q4_K_M.gguf +template: + chat: chatml diff --git a/core/explorer/database_test.go b/core/explorer/database_test.go index 7f2cbd268..3b05955ca 100644 --- a/core/explorer/database_test.go +++ b/core/explorer/database_test.go @@ -2,6 +2,7 @@ package explorer_test import ( "os" + "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -17,8 +18,9 @@ var _ = Describe("Database", func() { ) BeforeEach(func() { - // Create a temporary file path for the database - dbPath = "test_db.json" + // Keep each spec isolated: coverage runs can execute package tests in + // overlapping processes, so a repository-relative filename is racy. + dbPath = filepath.Join(GinkgoT().TempDir(), "test_db.json") db, err = explorer.NewDatabase(dbPath) Expect(err).To(BeNil()) }) @@ -78,7 +80,7 @@ var _ = Describe("Database", func() { Context("when loading an empty or non-existent file", func() { It("should start with an empty database", func() { - dbPath = "empty_db.json" + dbPath = filepath.Join(GinkgoT().TempDir(), "empty_db.json") db, err = explorer.NewDatabase(dbPath) Expect(err).To(BeNil()) diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index 1b7e059be..df1194b5d 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -1,12 +1,18 @@ package gallery import ( + "archive/tar" + "bytes" "context" "encoding/json" "os" "path/filepath" "runtime" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" @@ -15,9 +21,21 @@ import ( "gopkg.in/yaml.v3" ) -const ( - testImage = "quay.io/mudler/tests:localai-backend-test" -) +func writeBackendImageFixture(path string) { + var layerTar bytes.Buffer + w := tar.NewWriter(&layerTar) + contents := []byte("#!/bin/sh\necho test backend\n") + Expect(w.WriteHeader(&tar.Header{Name: "run.sh", Mode: 0o755, Size: int64(len(contents))})).To(Succeed()) + _, err := w.Write(contents) + Expect(err).NotTo(HaveOccurred()) + Expect(w.Close()).To(Succeed()) + + layer, err := tarball.LayerFromReader(bytes.NewReader(layerTar.Bytes())) + Expect(err).NotTo(HaveOccurred()) + image, err := mutate.AppendLayers(empty.Image, layer) + Expect(err).NotTo(HaveOccurred()) + Expect(tarball.WriteToFile(path, name.MustParseReference("localai/backend-test:fixture"), image)).To(Succeed()) +} var _ = Describe("Runtime capability-based backend selection", func() { var tempDir string @@ -135,6 +153,8 @@ var _ = Describe("Gallery Backends", func() { galleries []config.Gallery ml *model.ModelLoader systemState *system.SystemState + testImage string + fixtureDir string ) BeforeEach(func() { @@ -142,11 +162,22 @@ var _ = Describe("Gallery Backends", func() { tempDir, err = os.MkdirTemp("", "gallery-test-*") Expect(err).NotTo(HaveOccurred()) - // Setup test galleries + fixtureDir, err = os.MkdirTemp("", "backend-fixture-*") + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(fixtureDir, "run.sh"), []byte("#!/bin/sh\necho test backend\n"), 0o755)).To(Succeed()) + testImage = fixtureDir + + galleryPath := filepath.Join(tempDir, "backend-gallery.yaml") + galleryData, err := yaml.Marshal(GalleryBackends{ + &GalleryBackend{Metadata: Metadata{Name: "test-backend"}, URI: testImage}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(galleryPath, galleryData, 0o644)).To(Succeed()) + galleries = []config.Gallery{ { Name: "test-gallery", - URL: "https://gist.githubusercontent.com/mudler/71d5376bc2aa168873fa519fa9f4bd56/raw/0557f9c640c159fa8e4eab29e8d98df6a3d6e80f/backend-gallery.yaml", + URL: "file://" + galleryPath, }, } systemState, err = system.GetSystemState(system.WithBackendPath(tempDir)) @@ -155,7 +186,8 @@ var _ = Describe("Gallery Backends", func() { }) AfterEach(func() { - os.RemoveAll(tempDir) + Expect(os.RemoveAll(tempDir)).To(Succeed()) + Expect(os.RemoveAll(fixtureDir)).To(Succeed()) }) Describe("InstallBackendFromGallery", func() { @@ -171,6 +203,18 @@ var _ = Describe("Gallery Backends", func() { Expect(filepath.Join(tempDir, "test-backend", "run.sh")).To(BeARegularFile()) }) + It("should install a local OCI backend image", func() { + imagePath := filepath.Join(tempDir, "backend-image.tar") + writeBackendImageFixture(imagePath) + backend := &GalleryBackend{ + Metadata: Metadata{Name: "oci-test-backend"}, + URI: "ocifile://" + imagePath, + } + + Expect(InstallBackend(context.TODO(), systemState, ml, backend, nil, false)).To(Succeed()) + Expect(filepath.Join(tempDir, "oci-test-backend", "run.sh")).To(BeARegularFile()) + }) + It("removes files from a previous install that are absent in the new artifact", func() { // A reinstall must fully replace the installed backend, not overlay // the new artifact onto the old one: a stale library or package @@ -913,7 +957,7 @@ var _ = Describe("Gallery Backends", func() { Metadata: Metadata{ Name: "test-backend", }, - URI: "quay.io/mudler/tests:localai-backend-test", + URI: testImage, Alias: "test-alias", } @@ -943,7 +987,7 @@ var _ = Describe("Gallery Backends", func() { Metadata: Metadata{ Name: "test-backend", }, - URI: "quay.io/mudler/tests:localai-backend-test", + URI: testImage, Alias: "test-alias", } @@ -967,7 +1011,7 @@ var _ = Describe("Gallery Backends", func() { Metadata: Metadata{ Name: "test-backend", }, - URI: "quay.io/mudler/tests:localai-backend-test", + URI: testImage, Alias: "test-alias", } diff --git a/core/gallery/importers/discovery_options_test.go b/core/gallery/importers/discovery_options_test.go new file mode 100644 index 000000000..163caa365 --- /dev/null +++ b/core/gallery/importers/discovery_options_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT + +package importers_test + +import ( + "context" + "encoding/json" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/gallery/importers" + hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" +) + +type fixtureMetadata struct { + details *hfapi.ModelDetails + err error + calls []string +} + +func (f *fixtureMetadata) GetModelDetails(repo string) (*hfapi.ModelDetails, error) { + f.calls = append(f.calls, repo) + return f.details, f.err +} + +var _ = Describe("DiscoverModelConfigWithOptions", func() { + It("uses fixture metadata without creating a live client", func() { + metadata := &fixtureMetadata{details: &hfapi.ModelDetails{ + ModelID: "fixture/whisper", + PipelineTag: "automatic-speech-recognition", + Files: []hfapi.ModelFile{{Path: "ggml-model.bin"}}, + }} + config, err := importers.DiscoverModelConfigWithOptions(context.Background(), "hf://fixture/whisper", json.RawMessage(`{}`), importers.DiscoverOptions{HuggingFace: metadata}) + Expect(err).NotTo(HaveOccurred()) + Expect(config.Name).NotTo(BeEmpty()) + Expect(metadata.calls).To(Equal([]string{"fixture/whisper"})) + }) + + It("does not invoke metadata after cancellation", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + metadata := &fixtureMetadata{err: errors.New("must not be returned")} + _, err := importers.DiscoverModelConfigWithOptions(ctx, "hf://fixture/model", json.RawMessage(`{}`), importers.DiscoverOptions{HuggingFace: metadata}) + Expect(err).To(MatchError(context.Canceled)) + Expect(metadata.calls).To(BeEmpty()) + }) +}) diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index a86e86530..d97c558e9 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -1,6 +1,7 @@ package importers import ( + "context" "encoding/json" "errors" "fmt" @@ -26,6 +27,8 @@ import ( // this sentinel so legacy callers keep working. var ErrAmbiguousImport = errors.New("importer: ambiguous — specify preferences.backend") +var newHuggingFaceMetadata = func() HuggingFaceMetadata { return hfapi.NewClient() } + // AmbiguousImportError is the concrete error DiscoverModelConfig returns when // it can't pick an importer automatically. It carries the importer-modality // key (e.g. "tts", "asr") and the list of candidate backend names so HTTP @@ -251,15 +254,48 @@ func hasYAMLExtension(uri string) bool { } func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.ModelConfig, error) { + return DiscoverModelConfigWithOptions(context.Background(), uri, preferences, DiscoverOptions{}) +} + +// HuggingFaceMetadata provides only the repository metadata needed during +// importer discovery. Tests can supply fixtures without constructing a live +// Hugging Face client. +type HuggingFaceMetadata interface { + GetModelDetails(string) (*hfapi.ModelDetails, error) +} + +// DiscoverOptions contains optional dependencies for model discovery. +type DiscoverOptions struct { + HuggingFace HuggingFaceMetadata +} + +// SetHuggingFaceMetadataFactoryForTest replaces the production metadata +// client factory and returns a restore function. It must only be called by a +// serial test-suite setup before discovery begins. +func SetHuggingFaceMetadataFactoryForTest(factory func() HuggingFaceMetadata) func() { + previous := newHuggingFaceMetadata + newHuggingFaceMetadata = factory + return func() { newHuggingFaceMetadata = previous } +} + +// DiscoverModelConfigWithOptions discovers a model using explicitly supplied +// dependencies. A nil metadata client retains the production behavior. +func DiscoverModelConfigWithOptions(ctx context.Context, uri string, preferences json.RawMessage, opts DiscoverOptions) (gallery.ModelConfig, error) { var err error var modelConfig gallery.ModelConfig - hf := hfapi.NewClient() + hf := opts.HuggingFace + if hf == nil { + hf = newHuggingFaceMetadata() + } hfrepoID := strings.ReplaceAll(uri, "huggingface://", "") hfrepoID = strings.ReplaceAll(hfrepoID, "hf://", "") hfrepoID = strings.ReplaceAll(hfrepoID, "https://huggingface.co/", "") + if err := ctx.Err(); err != nil { + return gallery.ModelConfig{}, err + } hfDetails, err := hf.GetModelDetails(hfrepoID) if err != nil { // maybe not a HF repository diff --git a/core/gallery/importers/importers_suite_test.go b/core/gallery/importers/importers_suite_test.go index a65b8163a..a39e2a9c6 100644 --- a/core/gallery/importers/importers_suite_test.go +++ b/core/gallery/importers/importers_suite_test.go @@ -1,13 +1,76 @@ package importers_test import ( + "context" + "errors" "testing" + gguf "github.com/gpustack/gguf-parser-go" + "github.com/mudler/LocalAI/core/gallery/importers" + hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) +type metadataFixtures map[string]*hfapi.ModelDetails + +func (f metadataFixtures) GetModelDetails(repo string) (*hfapi.ModelDetails, error) { + details, ok := f[repo] + if !ok { + return nil, errors.New("metadata fixture not declared: " + repo) + } + return details, nil +} + +func file(repo, path, sha string) hfapi.ModelFile { + return hfapi.ModelFile{Path: path, SHA256: sha, URL: "https://huggingface.co/" + repo + "/resolve/main/" + path} +} + +var fixtures = metadataFixtures{ + "mudler/vibevoice.cpp-models": { + ModelID: "mudler/vibevoice.cpp-models", Author: "mudler", + Files: []hfapi.ModelFile{ + file("mudler/vibevoice.cpp-models", "vibevoice-realtime-Q4_K_M.gguf", "01"), + file("mudler/vibevoice.cpp-models", "vibevoice-asr-Q4_K_M.gguf", "02"), + file("mudler/vibevoice.cpp-models", "tokenizer.gguf", "03"), + file("mudler/vibevoice.cpp-models", "voice-Alice.gguf", "04"), + }, + }, + "UsefulSensors/moonshine-tiny": {ModelID: "UsefulSensors/moonshine-tiny", Author: "UsefulSensors", PipelineTag: "automatic-speech-recognition", Files: []hfapi.ModelFile{file("UsefulSensors/moonshine-tiny", "model.onnx", "05")}}, + "nvidia/parakeet-tdt-0.6b-v3": {ModelID: "nvidia/parakeet-tdt-0.6b-v3", Author: "nvidia", PipelineTag: "automatic-speech-recognition", Files: []hfapi.ModelFile{file("nvidia/parakeet-tdt-0.6b-v3", "parakeet.nemo", "06")}}, + "LiquidAI/LFM2.5-Audio-1.5B": {ModelID: "LiquidAI/LFM2.5-Audio-1.5B", Author: "LiquidAI"}, + "LiquidAI/LFM2-Audio-1.5B": {ModelID: "LiquidAI/LFM2-Audio-1.5B", Author: "LiquidAI"}, + "LiquidAI/LFM2.5-Audio-1.5B-GGUF": {ModelID: "LiquidAI/LFM2.5-Audio-1.5B-GGUF", Author: "LiquidAI", Files: []hfapi.ModelFile{file("LiquidAI/LFM2.5-Audio-1.5B-GGUF", "LFM2.5-Audio-Q4_K_M.gguf", "07")}}, + "hexgrad/Kokoro-82M": {ModelID: "hexgrad/Kokoro-82M", Author: "hexgrad", PipelineTag: "text-to-speech", Files: []hfapi.ModelFile{file("hexgrad/Kokoro-82M", "kokoro-v1_0.pth", "08")}}, + "Qwen/Qwen3-ASR-1.7B": {ModelID: "Qwen/Qwen3-ASR-1.7B", Author: "Qwen", PipelineTag: "automatic-speech-recognition"}, + "HirCoir/piper-voice-es-mx-lucas-melor": {ModelID: "HirCoir/piper-voice-es-mx-lucas-melor", Author: "HirCoir", PipelineTag: "text-to-speech", Files: []hfapi.ModelFile{file("HirCoir/piper-voice-es-mx-lucas-melor", "es_MX-lucas-medium.onnx", "09"), file("HirCoir/piper-voice-es-mx-lucas-melor", "es_MX-lucas-medium.onnx.json", "10")}}, + "h94/IP-Adapter-FaceID": {ModelID: "h94/IP-Adapter-FaceID", Author: "h94", PipelineTag: "text-to-image"}, + "LocalAI-io/whisper-large-v3-it-yodas-only-ggml": {ModelID: "LocalAI-io/whisper-large-v3-it-yodas-only-ggml", Author: "LocalAI-io", PipelineTag: "automatic-speech-recognition", Files: []hfapi.ModelFile{file("LocalAI-io/whisper-large-v3-it-yodas-only-ggml", "ggml-model-q4_0.bin", "11"), file("LocalAI-io/whisper-large-v3-it-yodas-only-ggml", "ggml-model-q5_0.bin", "12"), file("LocalAI-io/whisper-large-v3-it-yodas-only-ggml", "ggml-model-q8_0.bin", "13")}}, + "Systran/faster-whisper-large-v3": {ModelID: "Systran/faster-whisper-large-v3", Author: "Systran", PipelineTag: "automatic-speech-recognition", Files: []hfapi.ModelFile{file("Systran/faster-whisper-large-v3", "model.bin", "14"), file("Systran/faster-whisper-large-v3", "config.json", "15")}}, + "nari-labs/Dia-1.6B": {ModelID: "nari-labs/Dia-1.6B", Author: "nari-labs", PipelineTag: "text-to-speech"}, + "mudler/rfdetr-cpp-nano": {ModelID: "mudler/rfdetr-cpp-nano", Author: "mudler", PipelineTag: "object-detection", Files: []hfapi.ModelFile{file("mudler/rfdetr-cpp-nano", "rfdetr-nano-Q4_K_M.gguf", "16")}}, + "Qdrant/bm25": {ModelID: "Qdrant/bm25", Author: "Qdrant", PipelineTag: "sentence-similarity"}, + "pyannote/voice-activity-detection": {ModelID: "pyannote/voice-activity-detection", Author: "pyannote", PipelineTag: "automatic-speech-recognition"}, + "mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF": {ModelID: "mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF", Author: "mudler", Files: []hfapi.ModelFile{file("mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF", "localai-functioncall-qwen2.5-7b-v0.5-q4_k_m.gguf", "4e7b7fe1d54b881f1ef90799219dc6cc285d29db24f559c8998d1addb35713d4")}}, + "Qwen/Qwen3-VL-2B-Instruct-GGUF": {ModelID: "Qwen/Qwen3-VL-2B-Instruct-GGUF", Author: "Qwen", Files: []hfapi.ModelFile{file("Qwen/Qwen3-VL-2B-Instruct-GGUF", "Qwen3VL-2B-Instruct-Q4_K_M.gguf", "17"), file("Qwen/Qwen3-VL-2B-Instruct-GGUF", "Qwen3VL-2B-Instruct-Q8_0.gguf", "18"), file("Qwen/Qwen3-VL-2B-Instruct-GGUF", "mmproj-Qwen3VL-2B-Instruct-F16.gguf", "20"), file("Qwen/Qwen3-VL-2B-Instruct-GGUF", "mmproj-Qwen3VL-2B-Instruct-Q8_0.gguf", "19")}}, +} + func TestImporters(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Importers test suite") } + +var _ = BeforeSuite(func() { + restoreMetadata := importers.SetHuggingFaceMetadataFactoryForTest(func() importers.HuggingFaceMetadata { return fixtures }) + restoreMTP := importers.SetMTPProbeForTest(func(context.Context, string) (*gguf.GGUFFile, error) { + return nil, errors.New("remote GGUF probing disabled in fixture-backed importer tests") + }) + restoreGenAudio := importers.SetGenAudioProbeForTest(func(context.Context, string) (*gguf.GGUFFile, error) { + return nil, errors.New("remote GGUF probing disabled in fixture-backed importer tests") + }) + DeferCleanup(func() { + restoreGenAudio() + restoreMTP() + restoreMetadata() + }) +}) diff --git a/core/gallery/importers/llama-cpp.go b/core/gallery/importers/llama-cpp.go index a1cbb6d1b..15ff39d52 100644 --- a/core/gallery/importers/llama-cpp.go +++ b/core/gallery/importers/llama-cpp.go @@ -19,10 +19,32 @@ import ( ) var ( - _ Importer = &LlamaCPPImporter{} - _ AdditionalBackendsProvider = &LlamaCPPImporter{} + _ Importer = &LlamaCPPImporter{} + _ AdditionalBackendsProvider = &LlamaCPPImporter{} + parseRemoteGGUF = func(ctx context.Context, url string) (*gguf.GGUFFile, error) { + return gguf.ParseGGUFFileRemote(ctx, url, gguf.SkipLargeMetadata()) + } + parseRemoteGenAudioGGUF = func(ctx context.Context, url string) (*gguf.GGUFFile, error) { + return gguf.ParseGGUFFileRemote(ctx, url) + } ) +// SetMTPProbeForTest replaces the remote GGUF header reader and returns a +// restore function. It must only be called during serial suite setup. +func SetMTPProbeForTest(probe func(context.Context, string) (*gguf.GGUFFile, error)) func() { + previous := parseRemoteGGUF + parseRemoteGGUF = probe + return func() { parseRemoteGGUF = previous } +} + +// SetGenAudioProbeForTest replaces the remote projector header reader and +// returns a restore function. It must only be called during serial suite setup. +func SetGenAudioProbeForTest(probe func(context.Context, string) (*gguf.GGUFFile, error)) func() { + previous := parseRemoteGenAudioGGUF + parseRemoteGenAudioGGUF = probe + return func() { parseRemoteGenAudioGGUF = previous } +} + type LlamaCPPImporter struct{} func (i *LlamaCPPImporter) Name() string { return "llama-cpp" } @@ -415,10 +437,7 @@ func maybeApplyMTPDefaults(modelConfig *config.ModelConfig, details Details, cfg } }() - // MTP markers are architecture scalars. Avoid allocating tokenizer and - // other large arrays from an untrusted remote header; panic recovery cannot - // contain a fatal out-of-memory condition. - f, err := gguf.ParseGGUFFileRemote(ctx, probeURL, gguf.SkipLargeMetadata()) + f, err := parseRemoteGGUF(ctx, probeURL) if err != nil { xlog.Debug("[mtp-importer] failed to read remote GGUF header for MTP detection", "uri", probeURL, "error", err) return @@ -457,7 +476,7 @@ func maybeApplyTTSUsecase(modelConfig *config.ModelConfig, cfg *gallery.ModelCon } }() - f, err := gguf.ParseGGUFFileRemote(ctx, probeURL) + f, err := parseRemoteGenAudioGGUF(ctx, probeURL) if err != nil { xlog.Debug("[tts-importer] failed to read remote mmproj header for gen-audio detection", "uri", probeURL, "error", err) return diff --git a/core/gallery/request_test.go b/core/gallery/request_test.go index 116756965..efd83ed70 100644 --- a/core/gallery/request_test.go +++ b/core/gallery/request_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" . "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/downloader" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -18,9 +19,8 @@ var _ = Describe("Gallery API tests", func() { URL: "github:go-skynet/model-gallery/gpt4all-j.yaml@main", }, } - e, err := GetGalleryConfigFromURL[ModelConfig](req.URL, "") - Expect(err).ToNot(HaveOccurred()) - Expect(e.Name).To(Equal("gpt4all-j")) + resolved := downloader.URI(req.URL).ResolveURL() + Expect(resolved).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) }) }) diff --git a/core/http/app_test.go b/core/http/app_test.go index 36072b96e..90c60532e 100644 --- a/core/http/app_test.go +++ b/core/http/app_test.go @@ -297,9 +297,7 @@ func getRequest(url string, header http.Header) (error, int, []byte) { return nil, resp.StatusCode, body } -const bertEmbeddingsURL = `https://gist.githubusercontent.com/mudler/0a080b166b87640e8644b09c2aee6e3b/raw/f0e8c26bb72edc16d9fbafbfd6638072126ff225/bert-embeddings-gallery.yaml` - -var _ = Describe("API test", func() { +var _ = Describe("API test", Serial, func() { var app *echo.Echo var client *openai.Client @@ -308,6 +306,7 @@ var _ = Describe("API test", func() { var cancel context.CancelFunc var tmpdir string var modelDir string + var bertEmbeddingsURL string // localAIApp captures the Application so AfterEach can synchronously // stop the spawned gRPC backend processes. application.New cancels // them asynchronously on context cancel, which races with test-binary @@ -332,6 +331,17 @@ var _ = Describe("API test", func() { modelDir = filepath.Join(tmpdir, "models") err = os.Mkdir(modelDir, 0750) Expect(err).ToNot(HaveOccurred()) + fixtureDir := filepath.Join(modelDir, ".fixtures") + err = os.Mkdir(fixtureDir, 0750) + Expect(err).ToNot(HaveOccurred()) + galleryFixturePath := filepath.Join(fixtureDir, "bert-embeddings-gallery.yaml") + err = os.WriteFile(galleryFixturePath, []byte("name: bert\nconfig_file: |\n name: bert\n backend: embeddings\n usage: You can test this model with curl like this\n parameters:\n model: bert\n"), 0600) + Expect(err).ToNot(HaveOccurred()) + bertEmbeddingsURL = "file://" + galleryFixturePath + // Additional files are cache inputs, not behavior under test here. Seed the + // destination so model application never reaches the public network. + err = os.WriteFile(filepath.Join(modelDir, "foo.yaml"), []byte("fixture: true\n"), 0600) + Expect(err).ToNot(HaveOccurred()) c, cancel = context.WithCancel(context.Background()) @@ -511,7 +521,7 @@ var _ = Describe("API test", func() { fmt.Println(response) resp = response return response["processed"].(bool) - }, "360s", "10s").Should(Equal(true)) + }, "30s", "50ms").Should(Equal(true)) Expect(resp["message"]).ToNot(ContainSubstring("error")) dat, err := os.ReadFile(filepath.Join(modelDir, "bert2.yaml")) @@ -556,7 +566,7 @@ var _ = Describe("API test", func() { Eventually(func() bool { response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid) return response["processed"].(bool) - }, "360s", "10s").Should(Equal(true)) + }, "30s", "50ms").Should(Equal(true)) dat, err := os.ReadFile(filepath.Join(modelDir, "bert.yaml")) Expect(err).ToNot(HaveOccurred()) @@ -580,7 +590,7 @@ var _ = Describe("API test", func() { Eventually(func() bool { response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid) return response["processed"].(bool) - }, "360s", "10s").Should(Equal(true)) + }, "30s", "50ms").Should(Equal(true)) dat, err := os.ReadFile(filepath.Join(modelDir, "bert.yaml")) Expect(err).ToNot(HaveOccurred()) @@ -632,7 +642,7 @@ parameters: response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid) resp = response return response["processed"].(bool) - }, "360s", "10s").Should(Equal(true)) + }, "360s", "50ms").Should(Equal(true)) // Check that the model was imported successfully Expect(resp["message"]).ToNot(ContainSubstring("error")) @@ -703,7 +713,7 @@ parameters: response := getModelStatus("http://" + testHTTPAddr + "/models/jobs/" + uuid) resp = response return response["processed"].(bool) - }, "360s", "10s").Should(Equal(true)) + }, "360s", "50ms").Should(Equal(true)) // Check that the model was imported successfully Expect(resp["message"]).To(ContainSubstring("error")) diff --git a/core/http/endpoints/localai/import_model_test.go b/core/http/endpoints/localai/import_model_test.go index 96c20160e..2b2bbede2 100644 --- a/core/http/endpoints/localai/import_model_test.go +++ b/core/http/endpoints/localai/import_model_test.go @@ -10,14 +10,22 @@ import ( "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery/importers" . "github.com/mudler/LocalAI/core/http/endpoints/localai" "github.com/mudler/LocalAI/core/services/galleryop" + hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) +type ambiguityMetadata struct{} + +func (ambiguityMetadata) GetModelDetails(repo string) (*hfapi.ModelDetails, error) { + return &hfapi.ModelDetails{ModelID: repo, Author: "nari-labs", PipelineTag: "text-to-speech"}, nil +} + var _ = Describe("ImportModelURIEndpoint ambiguity handling", func() { var ( @@ -26,6 +34,9 @@ var _ = Describe("ImportModelURIEndpoint ambiguity handling", func() { ) BeforeEach(func() { + restore := importers.SetHuggingFaceMetadataFactoryForTest(func() importers.HuggingFaceMetadata { return ambiguityMetadata{} }) + DeferCleanup(restore) + var err error tempDir, err = os.MkdirTemp("", "import-model-test") Expect(err).ToNot(HaveOccurred()) diff --git a/core/http/endpoints/localai/localai_suite_test.go b/core/http/endpoints/localai/localai_suite_test.go index ea415bf70..fd64e6dd7 100644 --- a/core/http/endpoints/localai/localai_suite_test.go +++ b/core/http/endpoints/localai/localai_suite_test.go @@ -3,6 +3,7 @@ package localai_test import ( "testing" + "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -11,3 +12,13 @@ func TestLocalAIEndpoints(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "LocalAI Endpoints test suite") } + +var _ = SynchronizedBeforeSuite(func() []byte { + return []byte(testutil.StartSharedTestDB()) +}, func(endpoint []byte) { + testutil.SetSharedTestDBEndpoint(string(endpoint)) +}) + +var _ = SynchronizedAfterSuite(func() {}, func() { + testutil.StopSharedTestDB() +}) diff --git a/core/http/endpoints/ollama/helpers_internal_test.go b/core/http/endpoints/ollama/helpers_internal_test.go index cb2194f76..92469eb8f 100644 --- a/core/http/endpoints/ollama/helpers_internal_test.go +++ b/core/http/endpoints/ollama/helpers_internal_test.go @@ -62,4 +62,4 @@ var _ = Describe("applyOllamaOptions num_ctx clamping (issue #11022)", func() { Expect(cfg.ContextSize).To(BeNil()) }) -}) \ No newline at end of file +}) diff --git a/core/http/openresponses_test.go b/core/http/openresponses_test.go index f30674362..ab5dd716a 100644 --- a/core/http/openresponses_test.go +++ b/core/http/openresponses_test.go @@ -28,7 +28,7 @@ import ( // the registered model is "Qwen3-VL-2B-Instruct-Q4_K_M", not the repo name. const testModel = "Qwen3-VL-2B-Instruct-Q4_K_M" -var _ = Describe("Open Responses API", func() { +var _ = Describe("Open Responses API", Serial, func() { var app *echo.Echo var localApp *application.Application var localModelDir string diff --git a/core/services/agentpool/services_suite_test.go b/core/services/agentpool/services_suite_test.go index 7a60db0d4..ecca27ad4 100644 --- a/core/services/agentpool/services_suite_test.go +++ b/core/services/agentpool/services_suite_test.go @@ -3,6 +3,7 @@ package agentpool_test import ( "testing" + "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -11,3 +12,13 @@ func TestServices(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "LocalAI services test") } + +var _ = SynchronizedBeforeSuite(func() []byte { + return []byte(testutil.StartSharedTestDB()) +}, func(endpoint []byte) { + testutil.SetSharedTestDBEndpoint(string(endpoint)) +}) + +var _ = SynchronizedAfterSuite(func() {}, func() { + testutil.StopSharedTestDB() +}) diff --git a/core/services/agents/agents_suite_test.go b/core/services/agents/agents_suite_test.go index 6cc46193b..29f76733c 100644 --- a/core/services/agents/agents_suite_test.go +++ b/core/services/agents/agents_suite_test.go @@ -3,6 +3,7 @@ package agents import ( "testing" + "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -11,3 +12,13 @@ func TestAgents(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Agents test suite") } + +var _ = SynchronizedBeforeSuite(func() []byte { + return []byte(testutil.StartSharedTestDB()) +}, func(endpoint []byte) { + testutil.SetSharedTestDBEndpoint(string(endpoint)) +}) + +var _ = SynchronizedAfterSuite(func() {}, func() { + testutil.StopSharedTestDB() +}) diff --git a/core/services/buildproxy/buildproxy_suite_test.go b/core/services/buildproxy/buildproxy_suite_test.go new file mode 100644 index 000000000..02fa71876 --- /dev/null +++ b/core/services/buildproxy/buildproxy_suite_test.go @@ -0,0 +1,13 @@ +package buildproxy_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBuildProxy(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Build proxy test suite") +} diff --git a/core/services/buildproxy/proxy.go b/core/services/buildproxy/proxy.go new file mode 100644 index 000000000..39a0cd8ee --- /dev/null +++ b/core/services/buildproxy/proxy.go @@ -0,0 +1,280 @@ +// Package buildproxy provides conservative retrying and traffic telemetry for +// CI build downloads. +package buildproxy + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +type Event struct { + Time time.Time `json:"time"` + Host string `json:"host"` + Method string `json:"method"` + Path string `json:"path,omitempty"` + Status int `json:"status,omitempty"` + Attempts int `json:"attempts"` + BytesSent int64 `json:"bytes_sent,omitempty"` + BytesRead int64 `json:"bytes_read,omitempty"` + Intercepted bool `json:"intercepted,omitempty"` + Error string `json:"error,omitempty"` +} + +type SummaryRow struct { + Host string `json:"host"` + Method string `json:"method"` + Requests int64 `json:"requests"` + Retries int64 `json:"retries"` + BytesSent int64 `json:"bytes_sent"` + BytesRead int64 `json:"bytes_read"` + Errors int64 `json:"errors"` +} + +type Recorder struct { + mu sync.Mutex + file *os.File + rows map[string]*SummaryRow +} + +func NewRecorder(eventsPath string) (*Recorder, error) { + if err := os.MkdirAll(filepath.Dir(eventsPath), 0o755); err != nil { + return nil, err + } + f, err := os.OpenFile(eventsPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return nil, err + } + return &Recorder{file: f, rows: map[string]*SummaryRow{}}, nil +} + +func (r *Recorder) Record(event Event) { + r.mu.Lock() + defer r.mu.Unlock() + if event.Time.IsZero() { + event.Time = time.Now().UTC() + } + _ = json.NewEncoder(r.file).Encode(event) + key := event.Host + "\x00" + event.Method + row := r.rows[key] + if row == nil { + row = &SummaryRow{Host: event.Host, Method: event.Method} + r.rows[key] = row + } + row.Requests++ + if event.Attempts > 1 { + row.Retries += int64(event.Attempts - 1) + } + row.BytesSent += event.BytesSent + row.BytesRead += event.BytesRead + if event.Error != "" || event.Status >= 400 { + row.Errors++ + } +} + +func (r *Recorder) WriteSummary(path string) error { + r.mu.Lock() + defer r.mu.Unlock() + rows := make([]SummaryRow, 0, len(r.rows)) + for _, row := range r.rows { + rows = append(rows, *row) + } + b, err := json.MarshalIndent(rows, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(b, '\n'), 0o644) +} + +func (r *Recorder) Close() error { return r.file.Close() } + +type Options struct { + Transport http.RoundTripper + Recorder *Recorder + MaxAttempts int + SpoolDir string + BaseDelay time.Duration + MaxDelay time.Duration +} + +func NewHandler(opts Options) func(http.ResponseWriter, *http.Request, string) { + transport := opts.Transport + if transport == nil { + transport = http.DefaultTransport + } + if opts.MaxAttempts < 1 { + opts.MaxAttempts = 3 + } + if opts.BaseDelay <= 0 { + opts.BaseDelay = 100 * time.Millisecond + } + if opts.MaxDelay <= 0 { + opts.MaxDelay = 500 * time.Millisecond + } + return func(w http.ResponseWriter, request *http.Request, host string) { + event := Event{Host: hostname(host), Method: request.Method, Path: request.URL.EscapedPath(), Intercepted: true} + defer func() { opts.Recorder.Record(event) }() + if request.Body != nil { + defer func() { _ = request.Body.Close() }() + } + event.BytesSent = max(request.ContentLength, 0) + attempts := 1 + if request.Method == http.MethodGet || request.Method == http.MethodHead { + attempts = opts.MaxAttempts + } + for attempt := 1; attempt <= attempts; attempt++ { + event.Attempts = attempt + event.Error = "" + resp, path, size, err := fetch(request.Context(), transport, request, host, opts.SpoolDir) + if err == nil && !retryStatus(resp.StatusCode) { + event.Status, event.BytesRead = resp.StatusCode, size + copyResponse(w, resp, path, request.Method) + return + } + if resp != nil { + event.Status = resp.StatusCode + } + if path != "" { + _ = os.Remove(path) + } + if err != nil { + event.Error = err.Error() + } + if attempt == attempts { + break + } + if err := sleep(request.Context(), delay(opts.BaseDelay, opts.MaxDelay, attempt)); err != nil { + event.Error = err.Error() + break + } + } + http.Error(w, "build proxy: upstream request failed", http.StatusBadGateway) + } +} + +func fetch(ctx context.Context, transport http.RoundTripper, original *http.Request, host, spoolDir string) (*http.Response, string, int64, error) { + u := *original.URL + u.Scheme = original.URL.Scheme + if u.Scheme == "" { + u.Scheme = "https" + } + u.Host = host + req, err := http.NewRequestWithContext(ctx, original.Method, u.String(), original.Body) + if err != nil { + return nil, "", 0, err + } + req.Header = cloneHeaders(original.Header) + resp, err := transport.RoundTrip(req) + if err != nil { + return nil, "", 0, err + } + file, err := os.CreateTemp(spoolDir, "localai-build-proxy-*") + if err != nil { + _ = resp.Body.Close() + return resp, "", 0, err + } + path := file.Name() + size, copyErr := io.Copy(file, resp.Body) + closeErr := errors.Join(resp.Body.Close(), file.Close()) + if copyErr == nil { + copyErr = closeErr + } + if copyErr == nil && original.Method != http.MethodHead && resp.ContentLength >= 0 && size != resp.ContentLength { + copyErr = fmt.Errorf("short response: got %d bytes, expected %d", size, resp.ContentLength) + } + return resp, path, size, copyErr +} + +func copyResponse(w http.ResponseWriter, resp *http.Response, path, method string) { + defer func() { _ = os.Remove(path) }() + for key, values := range resp.Header { + if hopHeader(key) || strings.EqualFold(key, "Content-Length") { + continue + } + for _, value := range values { + w.Header().Add(key, value) + } + } + if method == http.MethodHead && resp.ContentLength >= 0 { + w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10)) + } else if info, err := os.Stat(path); err == nil { + w.Header().Set("Content-Length", strconv.FormatInt(info.Size(), 10)) + } + w.WriteHeader(resp.StatusCode) + file, err := os.Open(path) + if err == nil { + defer func() { _ = file.Close() }() + _, _ = io.Copy(w, file) + } +} + +func retryStatus(status int) bool { + switch status { + case 408, 429, 500, 502, 503, 504: + return true + default: + return false + } +} + +func delay(base, limit time.Duration, attempt int) time.Duration { + d := base + for i := 1; i < attempt && d < limit; i++ { + if d > limit/2 { + return limit + } + d *= 2 + } + if d > limit { + return limit + } + return d +} + +func sleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func hostname(host string) string { + if u, err := url.Parse("//" + host); err == nil && u.Hostname() != "" { + return strings.ToLower(u.Hostname()) + } + return strings.ToLower(host) +} + +func cloneHeaders(in http.Header) http.Header { + out := make(http.Header, len(in)) + for key, values := range in { + if hopHeader(key) || strings.EqualFold(key, "Proxy-Authorization") { + continue + } + out[key] = append([]string(nil), values...) + } + return out +} + +func hopHeader(name string) bool { + switch http.CanonicalHeaderKey(name) { + case "Connection", "Proxy-Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "Te", "Trailer", "Transfer-Encoding", "Upgrade": + return true + default: + return false + } +} diff --git a/core/services/buildproxy/proxy_test.go b/core/services/buildproxy/proxy_test.go new file mode 100644 index 000000000..378b76d55 --- /dev/null +++ b/core/services/buildproxy/proxy_test.go @@ -0,0 +1,74 @@ +package buildproxy_test + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "time" + + "github.com/mudler/LocalAI/core/services/buildproxy" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +var _ = Describe("Handler", func() { + It("retries an idempotent transient response and records bytes", func() { + dir := GinkgoT().TempDir() + recorder, err := buildproxy.NewRecorder(dir + "/events.jsonl") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = recorder.Close() }() + var calls atomic.Int32 + transport := roundTripFunc(func(*http.Request) (*http.Response, error) { + status, body := http.StatusServiceUnavailable, "retry" + if calls.Add(1) == 2 { + status, body = http.StatusOK, "complete" + } + return &http.Response{StatusCode: status, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body)), ContentLength: int64(len(body))}, nil + }) + handler := buildproxy.NewHandler(buildproxy.Options{Transport: transport, Recorder: recorder, SpoolDir: dir, BaseDelay: time.Nanosecond}) + req := httptest.NewRequest(http.MethodGet, "https://example.test/archive", nil) + response := httptest.NewRecorder() + handler(response, req, "example.test") + Expect(response.Code).To(Equal(http.StatusOK)) + Expect(response.Body.String()).To(Equal("complete")) + Expect(calls.Load()).To(Equal(int32(2))) + }) + + It("does not retry a mutating request", func() { + dir := GinkgoT().TempDir() + recorder, err := buildproxy.NewRecorder(dir + "/events.jsonl") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = recorder.Close() }() + var calls atomic.Int32 + transport := roundTripFunc(func(*http.Request) (*http.Response, error) { + calls.Add(1) + return &http.Response{StatusCode: http.StatusServiceUnavailable, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("no")), ContentLength: 2}, nil + }) + handler := buildproxy.NewHandler(buildproxy.Options{Transport: transport, Recorder: recorder, SpoolDir: dir}) + handler(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "https://example.test/token", strings.NewReader("secret")), "example.test") + Expect(calls.Load()).To(Equal(int32(1))) + }) + + It("preserves HEAD metadata without expecting a response body", func() { + dir := GinkgoT().TempDir() + recorder, err := buildproxy.NewRecorder(dir + "/events.jsonl") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = recorder.Close() }() + var calls atomic.Int32 + transport := roundTripFunc(func(*http.Request) (*http.Response, error) { + calls.Add(1) + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: http.NoBody, ContentLength: 1234}, nil + }) + handler := buildproxy.NewHandler(buildproxy.Options{Transport: transport, Recorder: recorder, SpoolDir: dir}) + response := httptest.NewRecorder() + handler(response, httptest.NewRequest(http.MethodHead, "https://example.test/blob", nil), "example.test") + Expect(calls.Load()).To(Equal(int32(1))) + Expect(response.Header().Get("Content-Length")).To(Equal("1234")) + }) +}) diff --git a/core/services/buildproxy/server.go b/core/services/buildproxy/server.go new file mode 100644 index 000000000..8d7e3e3bb --- /dev/null +++ b/core/services/buildproxy/server.go @@ -0,0 +1,233 @@ +package buildproxy + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +type certificateAuthority struct { + cert *x509.Certificate + key *ecdsa.PrivateKey + mu sync.Mutex + leaves map[string]*tls.Certificate +} + +type Server struct { + server *http.Server + listener net.Listener + handler http.Handler + recorder *Recorder + ca *certificateAuthority + caPath string + wg sync.WaitGroup +} + +func NewServer(address, caDir string, handler http.Handler, recorder *Recorder) (*Server, error) { + ca, caPath, err := createCA(caDir) + if err != nil { + return nil, err + } + s := &Server{handler: handler, recorder: recorder, ca: ca, caPath: caPath} + s.server = &http.Server{Addr: address, Handler: http.HandlerFunc(s.serveHTTP), ReadHeaderTimeout: 30 * time.Second} + return s, nil +} + +func (s *Server) CAPath() string { return s.caPath } +func (s *Server) Start() error { + ln, err := net.Listen("tcp", s.server.Addr) + if err != nil { + return err + } + s.listener = ln + s.wg.Add(1) + go func() { + defer s.wg.Done() + if err := s.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { + s.recorder.Record(Event{Method: "PROXY", Attempts: 1, Error: err.Error()}) + } + }() + return nil +} +func (s *Server) Addr() string { return s.listener.Addr().String() } +func (s *Server) Stop(ctx context.Context) error { + err := s.server.Shutdown(ctx) + s.wg.Wait() + return err +} + +func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodConnect { + // Some minimal clients (notably BusyBox wget) send an absolute HTTPS + // request to an HTTP forward proxy instead of opening CONNECT. The + // resource hop remains TLS and is handled by the same verified upstream + // transport; only absolute http:// resource URLs are forbidden. + if r.URL != nil && r.URL.IsAbs() && r.URL.Scheme == "https" { + // BusyBox closes its request side after writing the absolute-form + // request. Detach that connection cancellation while the proxy + // completes and verifies the upstream response. + s.handler.ServeHTTP(w, r.Clone(context.WithoutCancel(r.Context()))) + return + } + s.recorder.Record(Event{Host: hostname(r.Host), Method: r.Method, Path: r.URL.EscapedPath(), Attempts: 1, Error: "plain HTTP is forbidden"}) + http.Error(w, "build proxy: plain HTTP is forbidden", http.StatusUpgradeRequired) + return + } + s.intercept(w, r) +} + +func (s *Server) intercept(w http.ResponseWriter, r *http.Request) { + host := hostname(r.Host) + leaf, err := s.ca.leaf(host) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + h, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "hijacking unavailable", 500) + return + } + conn, _, err := h.Hijack() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + if _, err = io.WriteString(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + return + } + tlsConn := tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{*leaf}, NextProtos: []string{"http/1.1"}}) + if err = tlsConn.SetDeadline(time.Now().Add(30 * time.Second)); err != nil { + return + } + if err = tlsConn.Handshake(); err != nil { + s.recorder.Record(Event{Host: host, Method: "CONNECT", Attempts: 1, Error: err.Error()}) + return + } + _ = tlsConn.SetDeadline(time.Time{}) + ln := &singleListener{conn: tlsConn, done: make(chan struct{})} + inner := &http.Server{Handler: http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + req.URL.Scheme = "https" + req.URL.Host = r.Host + s.handler.ServeHTTP(rw, req) + })} + _ = inner.Serve(ln) +} + +type singleListener struct { + conn net.Conn + once sync.Once + done chan struct{} +} + +func (l *singleListener) Accept() (net.Conn, error) { + var c net.Conn + l.once.Do(func() { c = &signalConn{Conn: l.conn, done: l.done} }) + if c != nil { + return c, nil + } + <-l.done + return nil, net.ErrClosed +} +func (l *singleListener) Close() error { return nil } +func (l *singleListener) Addr() net.Addr { return l.conn.LocalAddr() } + +type signalConn struct { + net.Conn + done chan struct{} + once sync.Once +} + +func (c *signalConn) Close() error { + err := c.Conn.Close() + c.once.Do(func() { close(c.done) }) + return err +} + +func createCA(dir string) (*certificateAuthority, string, error) { + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, "", err + } + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, "", err + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, "", err + } + now := time.Now() + t := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: "LocalAI CI Build Proxy"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(24 * time.Hour), KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, BasicConstraintsValid: true, IsCA: true} + der, err := x509.CreateCertificate(rand.Reader, t, t, &key.PublicKey, key) + if err != nil { + return nil, "", err + } + cert, err := x509.ParseCertificate(der) + if err != nil { + return nil, "", err + } + path := filepath.Join(dir, "ca.crt") + if err = os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0644); err != nil { + return nil, "", err + } + return &certificateAuthority{cert: cert, key: key, leaves: map[string]*tls.Certificate{}}, path, nil +} +func (c *certificateAuthority) leaf(host string) (*tls.Certificate, error) { + c.mu.Lock() + defer c.mu.Unlock() + if v := c.leaves[host]; v != nil { + return v, nil + } + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, err + } + now := time.Now() + t := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: host}, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(24 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}} + if ip := net.ParseIP(host); ip != nil { + t.IPAddresses = []net.IP{ip} + } else { + t.DNSNames = []string{host} + } + der, err := x509.CreateCertificate(rand.Reader, t, c.cert, &key.PublicKey, c.key) + if err != nil { + return nil, err + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return nil, err + } + pair, err := tls.X509KeyPair(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})) + if err != nil { + return nil, err + } + c.leaves[host] = &pair + return &pair, nil +} + +func ParseListenAddress(address string) (string, error) { + if strings.TrimSpace(address) == "" { + return "", fmt.Errorf("listen address is empty") + } + return address, nil +} diff --git a/core/services/buildproxy/server_test.go b/core/services/buildproxy/server_test.go new file mode 100644 index 000000000..5edec8335 --- /dev/null +++ b/core/services/buildproxy/server_test.go @@ -0,0 +1,46 @@ +package buildproxy + +import ( + "crypto/x509" + "encoding/pem" + "net/http" + "net/http/httptest" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Interception certificates", func() { + It("issues host certificates trusted by the generated CA", func() { + ca, path, err := createCA(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + leaf, err := ca.leaf("registry.example.test") + Expect(err).NotTo(HaveOccurred()) + + caPEM, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + block, _ := pem.Decode(caPEM) + Expect(block).NotTo(BeNil()) + root, err := x509.ParseCertificate(block.Bytes) + Expect(err).NotTo(HaveOccurred()) + roots := x509.NewCertPool() + roots.AddCert(root) + certificate, err := x509.ParseCertificate(leaf.Certificate[0]) + Expect(err).NotTo(HaveOccurred()) + _, err = certificate.Verify(x509.VerifyOptions{DNSName: "registry.example.test", Roots: roots}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects plain HTTP", func() { + dir := GinkgoT().TempDir() + recorder, err := NewRecorder(dir + "/events.jsonl") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = recorder.Close() }() + server, err := NewServer("127.0.0.1:0", dir+"/ca", http.NotFoundHandler(), recorder) + Expect(err).NotTo(HaveOccurred()) + response := httptest.NewRecorder() + server.serveHTTP(response, httptest.NewRequest(http.MethodGet, "http://example.test/file", nil)) + Expect(response.Code).To(Equal(http.StatusUpgradeRequired)) + }) +}) diff --git a/core/services/cloudproxy/mitm/proxy.go b/core/services/cloudproxy/mitm/proxy.go index 79f49aa64..4ccd744f4 100644 --- a/core/services/cloudproxy/mitm/proxy.go +++ b/core/services/cloudproxy/mitm/proxy.go @@ -23,15 +23,17 @@ import ( // in its intercept allowlist; non-allowlisted hosts get a plain // TCP CONNECT tunnel. type Server struct { - addr string - ca *CA - interceptHosts map[string]bool - handler InterceptHandler - connectTimeout time.Duration - dialTimeout time.Duration - upstreamTLS *tls.Config - events pii.EventStore - eventSeq atomic.Uint64 + addr string + ca *CA + interceptHosts map[string]bool + handler InterceptHandler + connectTimeout time.Duration + dialTimeout time.Duration + upstreamTLS *tls.Config + events pii.EventStore + eventSeq atomic.Uint64 + allowPlainHTTP bool + interceptAll bool listener net.Listener srv *http.Server @@ -51,6 +53,12 @@ type Config struct { CA *CA InterceptHosts []string Handler InterceptHandler + // AllowPlainHTTP is used by the deterministic test-resource proxy. + // Production listeners leave it false and continue to require CONNECT. + AllowPlainHTTP bool + // InterceptAll prevents undeclared HTTPS hosts from being tunnelled by + // strict test-resource replay. Production listeners use the host allowlist. + InterceptAll bool // EventStore optionally receives a proxy_connect event for every // CONNECT, recording the destination host and whether the proxy // intercepted or tunneled it. nil disables connect-event recording. @@ -73,6 +81,8 @@ func NewServer(cfg Config) (*Server, error) { ca: cfg.CA, interceptHosts: hosts, handler: cfg.Handler, + allowPlainHTTP: cfg.AllowPlainHTTP, + interceptAll: cfg.InterceptAll, connectTimeout: 30 * time.Second, dialTimeout: 15 * time.Second, upstreamTLS: &tls.Config{NextProtos: []string{"http/1.1"}}, @@ -126,6 +136,10 @@ func (s *Server) Stop() { func (s *Server) handle(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodConnect { + if s.allowPlainHTTP && r.URL != nil && r.URL.IsAbs() { + s.handler(w, r, r.URL.Host) + return + } http.Error(w, "this proxy only supports HTTPS via CONNECT", http.StatusMethodNotAllowed) return } @@ -168,6 +182,9 @@ func (s *Server) recordConnectEvent(host string, intercepted bool) { // shouldIntercept reports whether host is in the allowlist. An // empty allowlist tunnels everything. func (s *Server) shouldIntercept(host string) bool { + if s.interceptAll { + return true + } if len(s.interceptHosts) == 0 { return false } diff --git a/core/services/jobs/jobs_suite_test.go b/core/services/jobs/jobs_suite_test.go index 957e5ead7..c183767e8 100644 --- a/core/services/jobs/jobs_suite_test.go +++ b/core/services/jobs/jobs_suite_test.go @@ -3,6 +3,7 @@ package jobs import ( "testing" + "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -11,3 +12,13 @@ func TestJobs(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Jobs test suite") } + +var _ = SynchronizedBeforeSuite(func() []byte { + return []byte(testutil.StartSharedTestDB()) +}, func(endpoint []byte) { + testutil.SetSharedTestDBEndpoint(string(endpoint)) +}) + +var _ = SynchronizedAfterSuite(func() {}, func() { + testutil.StopSharedTestDB() +}) diff --git a/core/services/nodes/file_stager_http.go b/core/services/nodes/file_stager_http.go index 79047aad6..62ffc10a7 100644 --- a/core/services/nodes/file_stager_http.go +++ b/core/services/nodes/file_stager_http.go @@ -21,6 +21,7 @@ import ( "github.com/mudler/xlog" "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/httpclient" ) @@ -220,15 +221,7 @@ func nextBackoff(attempt int) time.Duration { base = 1 * time.Second ceiling = 30 * time.Second ) - shift := uint(attempt - 2) - if shift > 30 { - shift = 30 // saturate before time.Duration overflows - } - b := base << shift - if b > ceiling || b < 0 { - b = ceiling - } - return b + return backoff.Exponential(base, ceiling, uint(attempt-2)) } // resumeOffset asks the server (via HEAD) how many bytes of the current upload diff --git a/core/services/nodes/nodes_suite_test.go b/core/services/nodes/nodes_suite_test.go index a6a24852b..56cf49a68 100644 --- a/core/services/nodes/nodes_suite_test.go +++ b/core/services/nodes/nodes_suite_test.go @@ -3,6 +3,7 @@ package nodes import ( "testing" + "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -11,3 +12,13 @@ func TestNodes(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Nodes test suite") } + +var _ = SynchronizedBeforeSuite(func() []byte { + return []byte(testutil.StartSharedTestDB()) +}, func(endpoint []byte) { + testutil.SetSharedTestDBEndpoint(string(endpoint)) +}) + +var _ = SynchronizedAfterSuite(func() {}, func() { + testutil.StopSharedTestDB() +}) diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index 1a59e361f..cccca7e43 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -9,6 +9,7 @@ import ( "github.com/google/uuid" "github.com/mudler/LocalAI/core/services/advisorylock" + "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/vrambudget" "github.com/mudler/xlog" @@ -2200,20 +2201,15 @@ func (r *NodeRegistry) RecordPendingBackendOpInFlight(ctx context.Context, id ui // backoffForAttempt is exponential from 30s doubling up to a 15m cap. The // reconciler tick is 30s so anything shorter would just re-fire immediately. func backoffForAttempt(attempts int) time.Duration { - const cap = 15 * time.Minute - base := 30 * time.Second - shift := attempts - 1 - if shift < 0 { - shift = 0 + const ( + base = 30 * time.Second + maximum = 15 * time.Minute + ) + exponent := 0 + if attempts > 1 { + exponent = attempts - 1 } - if shift > 10 { // 2^10 * 30s already exceeds the cap - shift = 10 - } - d := base << shift - if d > cap { - return cap - } - return d + return backoff.Exponential(base, maximum, uint(exponent)) } // CountPendingBackendOpsByBackend returns a map of backend name to the count diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index 80e511201..c5b7597fd 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -2,11 +2,16 @@ package testutil import ( "context" + "fmt" "runtime" + "sync" + "sync/atomic" "time" + "github.com/mudler/LocalAI/internal/testfixtures" "github.com/testcontainers/testcontainers-go" tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + tcnetwork "github.com/testcontainers/testcontainers-go/network" "github.com/testcontainers/testcontainers-go/wait" "gorm.io/driver/postgres" "gorm.io/gorm" @@ -16,24 +21,110 @@ import ( . "github.com/onsi/gomega" ) -// SetupTestDB creates a fresh PostgreSQL 16 container and returns a gorm.DB. -// The container is cleaned up via DeferCleanup when the test completes. +var ( + sharedDBMu sync.Mutex + sharedDBContainer testcontainers.Container + sharedDBEndpoint string + sharedDBSequence atomic.Uint64 +) + +// StartSharedTestDB starts one PostgreSQL container and returns its endpoint. +// Pass that endpoint to SetSharedTestDBEndpoint in every parallel test process. +func StartSharedTestDB() string { + if runtime.GOOS == "darwin" { + return "" + } + sharedDBMu.Lock() + defer sharedDBMu.Unlock() + if sharedDBContainer != nil { + return sharedDBEndpoint + } + + container, endpoint := startTestDBContainer() + sharedDBContainer = container + sharedDBEndpoint = endpoint + return endpoint +} + +// SetSharedTestDBEndpoint attaches this test process to the suite database. +func SetSharedTestDBEndpoint(endpoint string) { + sharedDBMu.Lock() + defer sharedDBMu.Unlock() + sharedDBEndpoint = endpoint +} + +// StopSharedTestDB terminates the process-scoped PostgreSQL fixture. +func StopSharedTestDB() { + sharedDBMu.Lock() + defer sharedDBMu.Unlock() + if sharedDBContainer == nil { + return + } + Expect(sharedDBContainer.Terminate(context.Background())).To(Succeed()) + sharedDBContainer = nil + sharedDBEndpoint = "" +} + +// SetupTestDB returns an isolated PostgreSQL database fixture. Suites that call +// StartSharedTestDB get a fresh schema; other suites retain a fresh container. func SetupTestDB() *gorm.DB { if runtime.GOOS == "darwin" { Skip("testcontainers requires Docker, not available on macOS CI") } + + sharedDBMu.Lock() + endpoint := sharedDBEndpoint + sharedDBMu.Unlock() + if endpoint != "" { + return setupIsolatedSchema(endpoint) + } + + pgC, endpoint := startTestDBContainer() + DeferCleanup(func() { _ = pgC.Terminate(context.Background()) }) + return openTestDB(endpoint, "") +} + +func startTestDBContainer() (testcontainers.Container, string) { ctx := context.Background() - pgC, err := tcpostgres.Run(ctx, "postgres:16", + Expect(testfixtures.RequireImage(ctx, testfixtures.Postgres16, "default")).To(Succeed()) + testNetwork, err := testfixtures.DockerNetwork() + Expect(err).NotTo(HaveOccurred()) + pgC, err := tcpostgres.Run(ctx, testfixtures.Postgres16, tcpostgres.WithDatabase("testdb"), tcpostgres.WithUsername("test"), tcpostgres.WithPassword("test"), testcontainers.WithWaitStrategyAndDeadline(60*time.Second, wait.ForLog("database system is ready to accept connections").WithOccurrence(2)), + tcnetwork.WithNetworkName([]string{"postgres"}, testNetwork), ) Expect(err).ToNot(HaveOccurred()) - DeferCleanup(func() { pgC.Terminate(context.Background()) }) - connStr, err := pgC.ConnectionString(ctx, "sslmode=disable") + endpoint, err := testfixtures.ContainerEndpoint(ctx, pgC, "5432") Expect(err).ToNot(HaveOccurred()) + return pgC, endpoint +} + +func setupIsolatedSchema(endpoint string) *gorm.DB { + schema := fmt.Sprintf("test_%d_%d", GinkgoParallelProcess(), sharedDBSequence.Add(1)) + admin := openTestDB(endpoint, "") + Expect(admin.Exec("CREATE SCHEMA " + schema).Error).ToNot(HaveOccurred()) + db := openTestDB(endpoint, schema) + DeferCleanup(func() { + if sqlDB, err := db.DB(); err == nil { + _ = sqlDB.Close() + } + Expect(admin.Exec("DROP SCHEMA " + schema + " CASCADE").Error).ToNot(HaveOccurred()) + if sqlDB, err := admin.DB(); err == nil { + _ = sqlDB.Close() + } + }) + return db +} + +func openTestDB(endpoint, schema string) *gorm.DB { + connStr := fmt.Sprintf("postgres://test:test@%s/testdb?sslmode=disable", endpoint) + if schema != "" { + connStr += "&search_path=" + schema + } db, err := gorm.Open(postgres.Open(connStr), &gorm.Config{ Logger: logger.Default.LogMode(logger.Silent), }) diff --git a/core/services/worker/free_timeout_test.go b/core/services/worker/free_timeout_test.go index 4f1b6346e..f27d4dcf3 100644 --- a/core/services/worker/free_timeout_test.go +++ b/core/services/worker/free_timeout_test.go @@ -6,6 +6,8 @@ import ( "os" "strconv" "syscall" + "testing" + "time" process "github.com/mudler/go-processmanager" gogrpc "google.golang.org/grpc" @@ -16,6 +18,19 @@ import ( . "github.com/onsi/gomega" ) +// TestWorkerFixtureProcess turns the current test binary into a portable +// long-running child for the process-stop assertions below. Using the test +// binary avoids assuming Unix utilities live at paths such as /bin/sleep, +// which is not true in Nix environments. +func TestWorkerFixtureProcess(t *testing.T) { + if os.Getenv("LOCALAI_WORKER_FIXTURE_PROCESS") != "1" { + return + } + for { + time.Sleep(time.Hour) + } +} + // pidAlive probes the OS directly for a process ID. The supervisor's own // liveness helpers all go through go-processmanager's pidfile, which Stop // deletes as part of releasing the handle, so they report "not alive" even if @@ -82,10 +97,13 @@ var _ = Describe("Stopping a backend whose Free never returns", func() { // actually dead afterwards, not merely that Stop() returned. It // outlives every timeout below, so if it is gone at the end it is // because the supervisor signalled it. + executable, err := os.Executable() + Expect(err).ToNot(HaveOccurred()) proc = process.New( process.WithTemporaryStateDir(), - process.WithName("/bin/sleep"), - process.WithArgs("300"), + process.WithName(executable), + process.WithArgs("-test.run=^TestWorkerFixtureProcess$"), + process.WithEnvironment(append(os.Environ(), "LOCALAI_WORKER_FIXTURE_PROCESS=1")...), ) Expect(proc.Run()).To(Succeed()) @@ -94,7 +112,8 @@ var _ = Describe("Stopping a backend whose Free never returns", func() { Expect(pidAlive(procPID)).To(BeTrue(), "the fixture process must be running before the stop") s = &backendSupervisor{ - cfg: &Config{}, + cfg: &Config{}, + backendFreeTimeout: 20 * time.Millisecond, processes: map[string]*backendProcess{ "wedged-model#0": { proc: proc, diff --git a/core/services/worker/lifecycle.go b/core/services/worker/lifecycle.go index c80e00ea0..968b8d1ab 100644 --- a/core/services/worker/lifecycle.go +++ b/core/services/worker/lifecycle.go @@ -322,7 +322,7 @@ func (s *backendSupervisor) handleModelUnload(data []byte, reply func([]byte)) { // Best-effort bounded gRPC Free(). A model.unload request must not // occupy the NATS reply handler forever when a backend is wedged. client := grpc.NewClientWithToken(targetAddr, false, nil, false, s.cfg.RegistrationToken) - freeCtx, cancel := context.WithTimeout(context.Background(), workerBackendFreeTimeout) + freeCtx, cancel := context.WithTimeout(context.Background(), s.freeTimeout()) if err := client.Free(freeCtx); err != nil { xlog.Warn("Free() failed during model.unload", "error", err, "addr", targetAddr) } diff --git a/core/services/worker/supervisor.go b/core/services/worker/supervisor.go index 1e01cf441..49c9b75f7 100644 --- a/core/services/worker/supervisor.go +++ b/core/services/worker/supervisor.go @@ -133,6 +133,11 @@ type backendSupervisor struct { // the same not-yet-cached backend) are serialized here so the gallery // download path doesn't race itself on the same directory. backendLocks map[string]*sync.Mutex + + // backendFreeTimeout bounds the best-effort Free call before process + // termination. Zero uses workerBackendFreeTimeout; tests use a shorter + // deadline to exercise a wedged backend without waiting five seconds. + backendFreeTimeout time.Duration } // defaultPortQuarantine is how long a released gRPC port waits before it can be @@ -155,6 +160,13 @@ type backendSupervisor struct { // rows; raising this value is not a substitute for it. const defaultPortQuarantine = 15 * time.Second +func (s *backendSupervisor) freeTimeout() time.Duration { + if s.backendFreeTimeout > 0 { + return s.backendFreeTimeout + } + return workerBackendFreeTimeout +} + // quarantinedPort is a released port that must not be re-bound until `until`. type quarantinedPort struct { port int @@ -827,8 +839,9 @@ func (s *backendSupervisor) stopBackendExact(key string, force bool) error { if !force { client := grpc.NewClientWithToken(bp.addr, false, nil, false, s.cfg.RegistrationToken) - freeCtx, cancel := context.WithTimeout(context.Background(), workerBackendFreeTimeout) - xlog.Debug("Calling bounded Free() before stopping backend", "backend", key, "timeout", workerBackendFreeTimeout) + freeTimeout := s.freeTimeout() + freeCtx, cancel := context.WithTimeout(context.Background(), freeTimeout) + xlog.Debug("Calling bounded Free() before stopping backend", "backend", key, "timeout", freeTimeout) if err := client.Free(freeCtx); err != nil { xlog.Warn("Free() failed (best-effort)", "backend", key, "error", err) } diff --git a/core/startup/model_preload.go b/core/startup/model_preload.go index 4f3bb1683..17738b3ea 100644 --- a/core/startup/model_preload.go +++ b/core/startup/model_preload.go @@ -13,12 +13,18 @@ import ( "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/gallery/importers" "github.com/mudler/LocalAI/core/services/galleryop" + "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/xlog" ) +const ( + modelImportPollInterval = 50 * time.Millisecond + modelImportMaxPollInterval = 500 * time.Millisecond +) + // InstallModels will preload models from the given list of URLs and galleries // It will download the model if it is not already present in the model path // It will also try to resolve if the model is an embedded model YAML configuration @@ -75,13 +81,21 @@ func InstallModelsWithOptions(ctx context.Context, galleryService *galleryop.Gal } var status *galleryop.OpStatus - // wait for op to finish + pollInterval := modelImportPollInterval + poll := time.NewTimer(pollInterval) + defer poll.Stop() for { status = galleryService.GetStatus(uuid.String()) if status != nil && status.Processed { break } - time.Sleep(1 * time.Second) + select { + case <-ctx.Done(): + return ctx.Err() + case <-poll.C: + pollInterval = backoff.Exponential(pollInterval, modelImportMaxPollInterval, 1) + poll.Reset(pollInterval) + } } if status.Error != nil { diff --git a/core/startup/model_preload_test.go b/core/startup/model_preload_test.go index 525f183cf..ad662a5fa 100644 --- a/core/startup/model_preload_test.go +++ b/core/startup/model_preload_test.go @@ -3,6 +3,8 @@ package startup_test import ( "context" "fmt" + "net/http" + "net/http/httptest" "os" "path/filepath" @@ -39,7 +41,11 @@ var _ = Describe("Preload test", func() { Context("Preloading from strings", func() { It("loads from embedded full-urls", func() { - url := "https://raw.githubusercontent.com/mudler/LocalAI-examples/main/configurations/phi-2.yaml" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("name: phi-2\nbackend: llama-cpp\nparameters:\n model: phi-2.gguf\n")) + })) + defer server.Close() + url := server.URL + "/phi-2.yaml" fileName := fmt.Sprintf("%s.yaml", "phi-2") galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{ @@ -59,7 +65,11 @@ var _ = Describe("Preload test", func() { Expect(string(content)).To(ContainSubstring("name: phi-2")) }) It("downloads from urls", func() { - url := "huggingface://TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/tinyllama-1.1b-chat-v0.3.Q2_K.gguf" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("tiny local GGUF fixture")) + })) + defer server.Close() + url := server.URL + "/tinyllama-1.1b-chat-v0.3.Q2_K.gguf" fileName := fmt.Sprintf("%s.gguf", "tinyllama-1.1b-chat-v0.3.Q2_K") galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{ diff --git a/docs/content/development/offline-tests.md b/docs/content/development/offline-tests.md new file mode 100644 index 000000000..104a04f4d --- /dev/null +++ b/docs/content/development/offline-tests.md @@ -0,0 +1,77 @@ +--- +title: "Offline test resources" +--- + +LocalAI tests separate resource acquisition from test execution. Resources are +declared by resource set in `test-resources/manifests/`; files and packed container +images are content-addressed by SHA-256 under +`.cache/test-resources/blobs/sha256/`. + +Prepare the resources before running a target: + +```sh +make prepare-offline-test-cache TEST_RESOURCE_SET=default +``` + +Preparation verifies every cached blob and fails closed. It never substitutes +a live request for a missing or corrupt entry. Maintainers can populate a +cache from pinned declarations only by explicitly enabling online mode: + +```sh +LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=default +``` + +The update command records declared responses, files, and digest-pinned images, +then writes a deterministic, zstd level-1 bundle at +`.cache/test-resources/bundles/.tar.zst`. Its SHA-256 is written to the +lock file. The test workflow transfers the bundle as a workflow artifact and +verifies it after deleting the recording cache; the scheduled refresh workflow +also publishes verified bundles to GHCR as OCI artifacts. + +HTTP declarations may include `request_headers`. `Range` participates in the +cache key, and authorization values participate only through a SHA-256 value; +credentials are never written verbatim to the cache index. Redirect responses +are recorded without following them, so every hop needed by a test must be +declared explicitly. + +File and HTTP declarations may list HTTPS `mirrors`. Recording tries the +canonical URL twice, then each mirror twice, and reports the duration of every +attempt. Every candidate must produce the same declared SHA-256; mirrors are +alternate transports, not alternate content. + +A digest mismatch is never accepted automatically. The updater prints the +observed failure for every source and directs maintainers to compare upstream +checksums, signatures, release notes, and redirects, then check the GitHub +Advisory Database and OSV before approving a new digest. Repeated mismatches can +mean a legitimate upstream release, a corrupt mirror, or a supply-chain event. + +Ordinary test recipes execute through `scripts/run-test-offline.sh`. Its +supervised replay proxy terminates HTTP and HTTPS and returns an immediate +error containing the method and URL for undeclared requests. Linux CI also +runs the command in a cgroup with public IPv4 and IPv6 rejected; macOS relies +on replay, declared resources, guarded Go transports, and static lint because +kernel-level subprocess enforcement is Linux-only. + +Testcontainer images must be registry-digest pinned and loaded during +preparation. Container helpers check that an image exists before startup and +attach services to internal-only Docker networks, preventing testcontainers +from silently pulling a missing tag. + +The default Linux and macOS suites use separate resource sets because +Docker archives are platform-specific. Backend and hardware resources remain +separate targets so ordinary contributors do not acquire large model fixtures +that their test command does not use. + +Coverage runs print a wall-clock summary for each test root and list every +Ginkgo spec or hook taking at least three seconds, including its source +location. Set `COVERAGE_SLOW_SPEC_THRESHOLD=` to tune the reporting +threshold. This measures the whole spec or hook, so it exposes time spent in +sleeps, polling, channel waits, cleanup, and resource contention without +replacing Go's global clock or changing test semantics. The same timings are +written to `coverage/timings.tsv` for CI artifacts and comparisons. The report +shows the slowest 25 entries per root by default; set +`COVERAGE_SLOW_SPEC_LIMIT=` to change the cap. + +Real third-party compatibility checks belong in separately named +`external-probe-*` scheduled workflows and must not be part of deterministic +test or coverage gates. diff --git a/internal/backoff/backoff.go b/internal/backoff/backoff.go new file mode 100644 index 000000000..075e6874a --- /dev/null +++ b/internal/backoff/backoff.go @@ -0,0 +1,23 @@ +// Package backoff provides bounded retry-delay calculations. +package backoff + +import "time" + +// Exponential returns base*2^exponent capped at maximum. It saturates before +// multiplying so time.Duration cannot overflow. +func Exponential(base, maximum time.Duration, exponent uint) time.Duration { + if base <= 0 || maximum <= 0 { + return 0 + } + if base >= maximum { + return maximum + } + + for ; exponent > 0; exponent-- { + if base > maximum/2 { + return maximum + } + base *= 2 + } + return base +} diff --git a/internal/backoff/backoff_suite_test.go b/internal/backoff/backoff_suite_test.go new file mode 100644 index 000000000..c8137b7c8 --- /dev/null +++ b/internal/backoff/backoff_suite_test.go @@ -0,0 +1,13 @@ +package backoff_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBackoff(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Backoff Suite") +} diff --git a/internal/backoff/backoff_test.go b/internal/backoff/backoff_test.go new file mode 100644 index 000000000..8452a99c4 --- /dev/null +++ b/internal/backoff/backoff_test.go @@ -0,0 +1,35 @@ +package backoff_test + +import ( + "math" + "time" + + "github.com/mudler/LocalAI/internal/backoff" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Exponential", func() { + It("doubles the base delay up to the maximum", func() { + base := 50 * time.Millisecond + maximum := 500 * time.Millisecond + + Expect(backoff.Exponential(base, maximum, 0)).To(Equal(50 * time.Millisecond)) + Expect(backoff.Exponential(base, maximum, 1)).To(Equal(100 * time.Millisecond)) + Expect(backoff.Exponential(base, maximum, 2)).To(Equal(200 * time.Millisecond)) + Expect(backoff.Exponential(base, maximum, 3)).To(Equal(400 * time.Millisecond)) + Expect(backoff.Exponential(base, maximum, 4)).To(Equal(maximum)) + Expect(backoff.Exponential(base, maximum, math.MaxUint)).To(Equal(maximum)) + }) + + It("saturates without overflowing a duration", func() { + maximum := time.Duration(math.MaxInt64) + Expect(backoff.Exponential(maximum/2+1, maximum, 1)).To(Equal(maximum)) + Expect(backoff.Exponential(2, 5, 1)).To(Equal(time.Duration(4))) + }) + + It("returns zero when backoff is disabled", func() { + Expect(backoff.Exponential(0, time.Second, 1)).To(BeZero()) + Expect(backoff.Exponential(time.Second, 0, 1)).To(BeZero()) + }) +}) diff --git a/internal/testfixtures/images.go b/internal/testfixtures/images.go new file mode 100644 index 000000000..fe16d766d --- /dev/null +++ b/internal/testfixtures/images.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT + +// Package testfixtures centralizes immutable resources shared by test suites. +package testfixtures + +import ( + "context" + "errors" + "fmt" + "net" + "os" + + "github.com/moby/moby/client" + "github.com/testcontainers/testcontainers-go" +) + +const ( + Postgres16 = "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20" + Postgres16Alpine = "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777" + NATS2Alpine = "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0" +) + +// RequireImage fails before testcontainers can fall back to a registry pull. +func RequireImage(ctx context.Context, reference, target string) error { + docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return err + } + defer func() { _ = docker.Close() }() + if _, err := docker.ImageInspect(ctx, reference); err != nil { + return fmt.Errorf("required offline test image %s is not loaded; run `make prepare-offline-test-cache TEST_RESOURCE_SET=%s`: %w", reference, target, err) + } + return nil +} + +func DockerNetwork() (string, error) { + name := os.Getenv("LOCALAI_TEST_DOCKER_NETWORK") //nolint:forbidigo + if name == "" { + return "", errors.New("offline test Docker network is not configured; run the test through scripts/run-test-offline.sh") + } + return name, nil +} + +// ContainerEndpoint returns an address reachable from the Linux test host +// without publishing a port from the internal-only Docker network. +func ContainerEndpoint(ctx context.Context, container testcontainers.Container, port string) (string, error) { + ip, err := container.ContainerIP(ctx) + if err != nil { + return "", err + } + if ip == "" { + return "", errors.New("offline test container has no private network address") + } + return net.JoinHostPort(ip, port), nil +} diff --git a/internal/testresources/bundle.go b/internal/testresources/bundle.go new file mode 100644 index 000000000..eeffa3f1f --- /dev/null +++ b/internal/testresources/bundle.go @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: MIT + +package testresources + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/klauspost/compress/zstd" +) + +func PackBundle(cacheDir, output string, manifest Manifest) (string, error) { + index, err := LoadHTTPIndex(cacheDir) + if err != nil { + return "", err + } + targetIndex := map[string]HTTPEntry{} + digests := map[string]bool{} + for _, resource := range manifest.HTTP { + key := RequestKey(resource.Method, resource.URL, resource.Headers()) + entry, ok := index[key] + if !ok { + return "", fmt.Errorf("cannot pack missing HTTP entry %s", key) + } + targetIndex[key], digests[resource.SHA256] = entry, true + } + for _, resource := range manifest.Files { + digests[resource.SHA256] = true + } + for _, resource := range manifest.Images { + digests[resource.SHA256] = true + } + if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil { + return "", err + } + tmp, err := os.CreateTemp(filepath.Dir(output), "bundle-*.tmp") + if err != nil { + return "", err + } + name := tmp.Name() + defer func() { _ = os.Remove(name) }() + hash := sha256.New() + zstdWriter, err := zstd.NewWriter(io.MultiWriter(tmp, hash), + zstd.WithEncoderLevel(zstd.SpeedFastest), + zstd.WithEncoderConcurrency(1), + zstd.WithEncoderCRC(true), + ) + if err != nil { + _ = tmp.Close() + return "", err + } + tw := tar.NewWriter(zstdWriter) + indexData, err := json.Marshal(targetIndex) + if err == nil { + err = writeTarBytes(tw, "http-index.json", indexData) + } + ordered := make([]string, 0, len(digests)) + for digest := range digests { + ordered = append(ordered, digest) + } + sort.Strings(ordered) + for _, digest := range ordered { + if err != nil { + break + } + path, verifyErr := VerifyBlob(cacheDir, digest) + if verifyErr != nil { + err = verifyErr + break + } + var data []byte + data, err = os.ReadFile(path) + if err == nil { + err = writeTarBytes(tw, filepath.ToSlash(filepath.Join("blobs", "sha256", digest)), data) + } + } + err = errors.Join(err, tw.Close(), zstdWriter.Close(), tmp.Close()) + if err != nil { + return "", err + } + if err := os.Rename(name, output); err != nil { + return "", err + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func RestoreBundle(cacheDir, bundle, expected string) error { + data, err := os.ReadFile(bundle) + if err != nil { + return err + } + actual := fmt.Sprintf("%x", sha256.Sum256(data)) + if actual != expected { + return fmt.Errorf("test resource bundle checksum mismatch: expected %s, got %s", expected, actual) + } + var bundleReader io.Reader = bytes.NewReader(data) + if len(data) >= 4 && bytes.Equal(data[:4], []byte{0x28, 0xb5, 0x2f, 0xfd}) { + zstdReader, err := zstd.NewReader(bundleReader, zstd.WithDecoderConcurrency(1)) + if err != nil { + return err + } + defer zstdReader.Close() + bundleReader = zstdReader + } + tr := tar.NewReader(bundleReader) + recorded := map[string]HTTPEntry{} + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + name := filepath.Clean(filepath.FromSlash(header.Name)) + if filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) { + return fmt.Errorf("unsafe bundle path %q", header.Name) + } + body, err := io.ReadAll(tr) + if err != nil { + return err + } + if name == "http-index.json" { + if err := json.Unmarshal(body, &recorded); err != nil { + return err + } + continue + } + destination := filepath.Join(cacheDir, name) + if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { + return err + } + if err := os.WriteFile(destination, body, 0o644); err != nil { + return err + } + } + index, err := LoadHTTPIndex(cacheDir) + if err != nil { + return err + } + for key, entry := range recorded { + index[key] = entry + } + return WriteHTTPIndex(cacheDir, index) +} + +func writeTarBytes(tw *tar.Writer, name string, data []byte) error { + header := &tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), ModTime: time.Unix(0, 0).UTC()} + if err := tw.WriteHeader(header); err != nil { + return err + } + _, err := tw.Write(data) + return err +} diff --git a/internal/testresources/httpcache.go b/internal/testresources/httpcache.go new file mode 100644 index 000000000..c32f51599 --- /dev/null +++ b/internal/testresources/httpcache.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: MIT + +package testresources + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" +) + +var hopHeaders = map[string]bool{ + "Connection": true, "Proxy-Connection": true, "Keep-Alive": true, + "Transfer-Encoding": true, "Content-Length": true, "Te": true, + "Trailer": true, "Upgrade": true, "Proxy-Authenticate": true, + "Proxy-Authorization": true, +} + +func LoadHTTPIndex(cacheDir string) (map[string]HTTPEntry, error) { + index := map[string]HTTPEntry{} + data, err := os.ReadFile(filepath.Join(cacheDir, "index.json")) + if errors.Is(err, os.ErrNotExist) { + return index, nil + } + if err != nil { + return nil, fmt.Errorf("read HTTP cache index: %w", err) + } + if err := json.Unmarshal(data, &index); err != nil { + return nil, fmt.Errorf("parse HTTP cache index: %w", err) + } + return index, nil +} + +func WriteHTTPIndex(cacheDir string, index map[string]HTTPEntry) error { + data, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(cacheDir, "index-*.tmp") + if err != nil { + return err + } + name := tmp.Name() + defer func() { _ = os.Remove(name) }() + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(name, filepath.Join(cacheDir, "index.json")) +} + +func SanitizeHeaders(header http.Header) http.Header { + out := header.Clone() + for name := range hopHeaders { + out.Del(name) + } + return out +} + +func ReplayResponse(w http.ResponseWriter, cacheDir string, entry HTTPEntry) error { + path, err := VerifyBlob(cacheDir, entry.Digest) + if err != nil { + return err + } + for name, values := range entry.Header { + for _, value := range values { + w.Header().Add(name, value) + } + } + w.Header().Set("Content-Length", fmt.Sprint(entry.Size)) + w.WriteHeader(entry.Status) + if entry.Size == 0 { + return nil + } + body, err := os.Open(path) + if err != nil { + return err + } + defer func() { _ = body.Close() }() + _, err = io.Copy(w, body) + return err +} diff --git a/internal/testresources/resources.go b/internal/testresources/resources.go new file mode 100644 index 000000000..f46ff3ca2 --- /dev/null +++ b/internal/testresources/resources.go @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: MIT + +package testresources + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +const ManifestVersion = 1 + +type Manifest struct { + Version int `json:"version"` + Target string `json:"target"` + HTTP []HTTP `json:"http,omitempty"` + Files []File `json:"files,omitempty"` + Images []OCIImage `json:"images,omitempty"` +} + +type HTTP struct { + Method string `json:"method"` + URL string `json:"url"` + Mirrors []string `json:"mirrors,omitempty"` + SHA256 string `json:"sha256"` + RequestHeaders map[string]string `json:"request_headers,omitempty"` +} + +type HTTPEntry struct { + Digest string `json:"digest"` + Size int64 `json:"size"` + Status int `json:"status"` + Header http.Header `json:"header"` +} + +type File struct { + URL string `json:"url"` + Mirrors []string `json:"mirrors,omitempty"` + SHA256 string `json:"sha256"` + Destination string `json:"destination,omitempty"` + Environment string `json:"environment,omitempty"` +} + +type OCIImage struct { + Reference string `json:"reference"` + SHA256 string `json:"sha256"` +} + +type Lock struct { + Version int `json:"version"` + Bundles map[string]string `json:"bundles"` +} + +func LoadManifest(path string) (Manifest, error) { + var manifest Manifest + if err := decode(path, &manifest); err != nil { + return manifest, err + } + if err := manifest.Validate(); err != nil { + return manifest, fmt.Errorf("%s: %w", path, err) + } + return manifest, nil +} + +func LoadLock(path string) (Lock, error) { + var lock Lock + if err := decode(path, &lock); err != nil { + return lock, err + } + if lock.Version != ManifestVersion { + return lock, fmt.Errorf("%s: unsupported version %d", path, lock.Version) + } + return lock, nil +} + +func WriteLock(path string, lock Lock) error { + return writeJSON(path, lock) +} + +func WriteManifest(path string, manifest Manifest) error { + return writeJSON(path, manifest) +} + +func writeJSON(path string, value any) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + return os.WriteFile(path, data, 0o644) +} + +func decode(path string, value any) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(value); err != nil { + return err + } + if decoder.Decode(&struct{}{}) != io.EOF { + return errors.New("manifest has trailing JSON data") + } + return nil +} + +func (m Manifest) Validate() error { + if m.Version != ManifestVersion { + return fmt.Errorf("unsupported version %d", m.Version) + } + if strings.TrimSpace(m.Target) == "" { + return errors.New("target is required") + } + for _, resource := range m.HTTP { + if resource.Method == "" || resource.URL == "" || !validDigest(resource.SHA256) { + return fmt.Errorf("HTTP resources require method, URL, and lowercase sha256: %s %s", resource.Method, resource.URL) + } + if err := validateMirrors(resource.Mirrors); err != nil { + return fmt.Errorf("HTTP resource %s: %w", resource.URL, err) + } + } + for _, resource := range m.Files { + if resource.URL == "" || !validDigest(resource.SHA256) || (resource.Destination == "" && resource.Environment == "") { + return fmt.Errorf("file resources require URL, sha256, and destination or environment: %s", resource.URL) + } + if filepath.IsAbs(resource.Destination) || strings.HasPrefix(filepath.Clean(resource.Destination), "..") { + return fmt.Errorf("file destination must stay inside the resource directory: %s", resource.Destination) + } + if err := validateMirrors(resource.Mirrors); err != nil { + return fmt.Errorf("file resource %s: %w", resource.URL, err) + } + } + for _, resource := range m.Images { + if !strings.Contains(resource.Reference, "@sha256:") || !validDigest(resource.SHA256) { + return fmt.Errorf("OCI image must be digest-pinned and have a packed sha256: %s", resource.Reference) + } + } + return nil +} + +func validateMirrors(mirrors []string) error { + seen := map[string]bool{} + for _, mirror := range mirrors { + if !strings.HasPrefix(mirror, "https://") { + return fmt.Errorf("mirror must use HTTPS: %s", mirror) + } + if seen[mirror] { + return fmt.Errorf("duplicate mirror: %s", mirror) + } + seen[mirror] = true + } + return nil +} + +func BlobPath(cacheDir, digest string) string { + return filepath.Join(cacheDir, "blobs", "sha256", digest) +} + +func RequestKey(method, rawURL string, headers ...http.Header) string { + key := strings.ToUpper(method) + " " + rawURL + if len(headers) == 0 { + return key + } + for _, name := range []string{"Authorization", "Range"} { + value := headers[0].Get(name) + if value == "" { + continue + } + if name == "Authorization" { + digest := sha256.Sum256([]byte(value)) + value = "sha256:" + hex.EncodeToString(digest[:]) + } + key += "\n" + strings.ToLower(name) + ":" + value + } + return key +} + +func (resource HTTP) Headers() http.Header { + header := make(http.Header, len(resource.RequestHeaders)) + for name, value := range resource.RequestHeaders { + header.Set(name, value) + } + return header +} + +func VerifyBlob(cacheDir, digest string) (string, error) { + path := BlobPath(cacheDir, digest) + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("missing CAS blob %s: %w", digest, err) + } + sum := sha256.Sum256(data) + actual := hex.EncodeToString(sum[:]) + if actual != digest { + return "", fmt.Errorf("corrupt CAS blob %s: got sha256:%s", digest, actual) + } + return path, nil +} + +func validDigest(value string) bool { + if len(value) != sha256.Size*2 || strings.ToLower(value) != value { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} diff --git a/internal/testresources/resources_suite_test.go b/internal/testresources/resources_suite_test.go new file mode 100644 index 000000000..ead665b25 --- /dev/null +++ b/internal/testresources/resources_suite_test.go @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT + +package testresources_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestResources(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Test resources suite") +} diff --git a/internal/testresources/resources_test.go b/internal/testresources/resources_test.go new file mode 100644 index 000000000..ae6203b5e --- /dev/null +++ b/internal/testresources/resources_test.go @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: MIT + +package testresources_test + +import ( + "crypto/sha256" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/internal/testresources" +) + +var _ = Describe("Declared test resources", func() { + It("rejects mutable and unpinned resources", func() { + manifest := testresources.Manifest{ + Version: testresources.ManifestVersion, + Target: "backend", + Images: []testresources.OCIImage{{Reference: "postgres:latest", SHA256: fmt.Sprintf("%064d", 0)}}, + } + Expect(manifest.Validate()).To(MatchError(ContainSubstring("digest-pinned"))) + }) + + It("requires HTTPS and unique mirrors", func() { + digest := fmt.Sprintf("%064d", 0) + manifest := testresources.Manifest{Version: 1, Target: "fixture", Files: []testresources.File{{ + URL: "https://primary.invalid/file", Mirrors: []string{"http://mirror.invalid/file"}, + SHA256: digest, Destination: "file", + }}} + Expect(manifest.Validate()).To(MatchError(ContainSubstring("mirror must use HTTPS"))) + manifest.Files[0].Mirrors = []string{"https://mirror.invalid/file", "https://mirror.invalid/file"} + Expect(manifest.Validate()).To(MatchError(ContainSubstring("duplicate mirror"))) + }) + + It("fails before tests when a CAS blob is missing or corrupt", func() { + cache := GinkgoT().TempDir() + digest := fmt.Sprintf("%064d", 0) + _, err := testresources.VerifyBlob(cache, digest) + Expect(err).To(MatchError(ContainSubstring("missing CAS blob"))) + + path := testresources.BlobPath(cache, digest) + Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) + Expect(os.WriteFile(path, []byte("corrupt"), 0o644)).To(Succeed()) + _, err = testresources.VerifyBlob(cache, digest) + Expect(err).To(MatchError(ContainSubstring("corrupt CAS blob"))) + }) + + It("accepts and verifies a content-addressed blob", func() { + cache := GinkgoT().TempDir() + content := []byte("offline fixture") + digest := fmt.Sprintf("%x", sha256.Sum256(content)) + path := testresources.BlobPath(cache, digest) + Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) + Expect(os.WriteFile(path, content, 0o644)).To(Succeed()) + Expect(testresources.VerifyBlob(cache, digest)).To(Equal(path)) + }) + + It("persists response metadata and replays a verified body", func() { + cache := GinkgoT().TempDir() + content := []byte("cached response") + digest := fmt.Sprintf("%x", sha256.Sum256(content)) + path := testresources.BlobPath(cache, digest) + Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) + Expect(os.WriteFile(path, content, 0o644)).To(Succeed()) + index := map[string]testresources.HTTPEntry{ + "GET https://example.invalid/data": { + Digest: digest, Size: int64(len(content)), Status: http.StatusPartialContent, + Header: http.Header{"Content-Range": {"bytes 0-14/15"}}, + }, + } + Expect(testresources.WriteHTTPIndex(cache, index)).To(Succeed()) + loaded, err := testresources.LoadHTTPIndex(cache) + Expect(err).NotTo(HaveOccurred()) + recorder := httptest.NewRecorder() + Expect(testresources.ReplayResponse(recorder, cache, loaded["GET https://example.invalid/data"])).To(Succeed()) + Expect(recorder.Code).To(Equal(http.StatusPartialContent)) + Expect(recorder.Body.Bytes()).To(Equal(content)) + Expect(recorder.Header().Get("Content-Range")).To(Equal("bytes 0-14/15")) + }) + + It("sanitizes connection-specific response headers", func() { + header := http.Header{"Transfer-Encoding": {"chunked"}, "Authorization": {"secret"}, "X-Fixture": {"yes"}} + clean := testresources.SanitizeHeaders(header) + Expect(clean).NotTo(HaveKey("Transfer-Encoding")) + Expect(clean).To(HaveKeyWithValue("Authorization", []string{"secret"})) + Expect(clean).To(HaveKeyWithValue("X-Fixture", []string{"yes"})) + }) + + It("keys range and authorization variants without storing credentials", func() { + header := http.Header{"Authorization": {"Bearer secret"}, "Range": {"bytes=4-"}} + key := testresources.RequestKey(http.MethodGet, "https://example.invalid/model", header) + Expect(key).To(ContainSubstring("range:bytes=4-")) + Expect(key).To(ContainSubstring("authorization:sha256:")) + Expect(key).NotTo(ContainSubstring("Bearer secret")) + Expect(key).NotTo(Equal(testresources.RequestKey(http.MethodGet, "https://example.invalid/model"))) + }) + + It("packs deterministically and restores a target cache", func() { + cache := GinkgoT().TempDir() + content := []byte("bundle fixture") + digest := fmt.Sprintf("%x", sha256.Sum256(content)) + path := testresources.BlobPath(cache, digest) + Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) + Expect(os.WriteFile(path, content, 0o644)).To(Succeed()) + manifest := testresources.Manifest{Version: 1, Target: "fixture", Files: []testresources.File{{URL: "https://example.invalid/file", SHA256: digest, Destination: "file"}}} + first := filepath.Join(GinkgoT().TempDir(), "first.tar.zst") + second := filepath.Join(GinkgoT().TempDir(), "second.tar.zst") + firstDigest, err := testresources.PackBundle(cache, first, manifest) + Expect(err).NotTo(HaveOccurred()) + secondDigest, err := testresources.PackBundle(cache, second, manifest) + Expect(err).NotTo(HaveOccurred()) + Expect(secondDigest).To(Equal(firstDigest)) + compressed, err := os.ReadFile(first) + Expect(err).NotTo(HaveOccurred()) + Expect(compressed[:4]).To(Equal([]byte{0x28, 0xb5, 0x2f, 0xfd})) + + restored := GinkgoT().TempDir() + Expect(testresources.RestoreBundle(restored, first, firstDigest)).To(Succeed()) + Expect(os.ReadFile(testresources.BlobPath(restored, digest))).To(Equal(content)) + }) +}) diff --git a/pkg/downloader/cancel_test.go b/pkg/downloader/cancel_test.go index 76f8a2df5..57f9bba95 100644 --- a/pkg/downloader/cancel_test.go +++ b/pkg/downloader/cancel_test.go @@ -59,9 +59,7 @@ var _ = Describe("Download cancellation", func() { } BeforeEach(func() { - dir, err := os.Getwd() - Expect(err).ToNot(HaveOccurred()) - filePath = dir + "/cancel_model" + filePath = GinkgoT().TempDir() + "/cancel_model" }) AfterEach(func() { @@ -112,7 +110,7 @@ var _ = Describe("Download cancellation", func() { Expect(err).To(HaveOccurred()) Expect(errors.Is(err, context.Canceled)).To(BeTrue()) - Expect(filePath + ".partial").ToNot(BeAnExistingFile(), + Expect(filePath+".partial").ToNot(BeAnExistingFile(), "a deliberate user cancel must not leave a dangling .partial behind") }) diff --git a/pkg/downloader/retry.go b/pkg/downloader/retry.go index 5dc6584c7..cd956f7d5 100644 --- a/pkg/downloader/retry.go +++ b/pkg/downloader/retry.go @@ -4,7 +4,10 @@ import ( "context" "errors" "io" + "math" "time" + + "github.com/mudler/LocalAI/internal/backoff" ) // ErrTransientDownload marks a download failure that a later attempt has a @@ -89,7 +92,11 @@ func (t *readErrorRecorder) Read(p []byte) (int, error) { // waitBeforeRetry sleeps for the backoff interval of the given attempt // (1-based), returning the context error if the caller gives up while waiting. func waitBeforeRetry(ctx context.Context, attempt int) error { - delay := DownloadRetryBaseDelay << (attempt - 1) + exponent := 0 + if attempt > 1 { + exponent = attempt - 1 + } + delay := backoff.Exponential(DownloadRetryBaseDelay, time.Duration(math.MaxInt64), uint(exponent)) timer := time.NewTimer(delay) defer timer.Stop() select { diff --git a/pkg/downloader/stall_test.go b/pkg/downloader/stall_test.go index 34ae2d348..7c9f2e5f6 100644 --- a/pkg/downloader/stall_test.go +++ b/pkg/downloader/stall_test.go @@ -18,9 +18,7 @@ var _ = Describe("Download stall timeout", func() { var savedTimeout time.Duration BeforeEach(func() { - dir, err := os.Getwd() - Expect(err).ToNot(HaveOccurred()) - filePath = dir + "/stall_model" + filePath = GinkgoT().TempDir() + "/stall_model" savedTimeout = DownloadStallTimeout }) diff --git a/pkg/downloader/uri_test.go b/pkg/downloader/uri_test.go index 9cb667b57..20f22454c 100644 --- a/pkg/downloader/uri_test.go +++ b/pkg/downloader/uri_test.go @@ -22,31 +22,15 @@ var _ = Describe("Gallery API tests", func() { Context("URI", func() { It("parses github with a branch", func() { uri := URI("github:go-skynet/model-gallery/gpt4all-j.yaml") - Expect( - uri.ReadWithCallback("", func(url string, i []byte) error { - Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) - return nil - }), - ).ToNot(HaveOccurred()) + Expect(uri.ResolveURL()).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) }) It("parses github without a branch", func() { uri := URI("github:go-skynet/model-gallery/gpt4all-j.yaml@main") - - Expect( - uri.ReadWithCallback("", func(url string, i []byte) error { - Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) - return nil - }), - ).ToNot(HaveOccurred()) + Expect(uri.ResolveURL()).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) }) It("parses github with urls", func() { uri := URI("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml") - Expect( - uri.ReadWithCallback("", func(url string, i []byte) error { - Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) - return nil - }), - ).ToNot(HaveOccurred()) + Expect(uri.ResolveURL()).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")) }) }) @@ -263,9 +247,7 @@ var _ = Describe("Download Test", func() { _, err = _mockDataSha.Write(mockData) Expect(err).ToNot(HaveOccurred()) mockDataSha = fmt.Sprintf("%x", _mockDataSha.Sum(nil)) - dir, err := os.Getwd() - filePath = dir + "/my_supercool_model" - Expect(err).NotTo(HaveOccurred()) + filePath = GinkgoT().TempDir() + "/my_supercool_model" }) Context("URI DownloadFile", func() { diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index c18c78185..e96995d17 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -28,8 +28,11 @@ import ( "net" "net/http" "net/url" + "os" "strings" "time" + + "github.com/mudler/LocalAI/pkg/testnetwork" ) const ( @@ -105,12 +108,20 @@ func sameOrigin(a, b *url.URL) bool { // (e.g. a credential-injecting RoundTripper) should base it on this rather than // http.DefaultTransport so the TLS floor and timeouts are preserved. func HardenedTransport() *http.Transport { + dialContext := (&net.Dialer{ + Timeout: dialTimeout, + KeepAlive: dialKeepAlive, + }).DialContext + // This is set only by the test-resource supervisor before it starts the + // child process; production configuration does not cross this boundary. + if os.Getenv("LOCALAI_TEST_OFFLINE") == "1" { //nolint:forbidigo + guard := testnetwork.LocalGuard() + guard.Dial = dialContext + dialContext = guard.DialContext + } return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: dialTimeout, - KeepAlive: dialKeepAlive, - }).DialContext, + Proxy: http.ProxyFromEnvironment, + DialContext: dialContext, ForceAttemptHTTP2: true, MaxIdleConns: maxIdleConns, IdleConnTimeout: idleConnTimeout, diff --git a/pkg/huggingface-api/client.go b/pkg/huggingface-api/client.go index 1d1c7ae3c..05ccd3465 100644 --- a/pkg/huggingface-api/client.go +++ b/pkg/huggingface-api/client.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/httpclient" ) @@ -96,21 +97,49 @@ type Client struct { maxRetries int retryBackoff time.Duration maxBackoff time.Duration - sleepFn func(time.Duration) + clock Clock +} + +// Clock is the small portion of wall-clock time used by retry handling. +// Supplying a fake clock lets tests verify backoff behavior without sleeping. +type Clock interface { + Now() time.Time + Sleep(time.Duration) +} + +type realClock struct{} + +func (realClock) Now() time.Time { return time.Now() } +func (realClock) Sleep(d time.Duration) { time.Sleep(d) } + +// ClientOption configures a Hugging Face API client. +type ClientOption func(*Client) + +// WithClock replaces the clock used for retry delays. +func WithClock(clock Clock) ClientOption { + return func(client *Client) { + if clock != nil { + client.clock = clock + } + } } var ErrRateLimited = errors.New("huggingface API rate limited") // NewClient creates a new Hugging Face API client -func NewClient() *Client { - return &Client{ +func NewClient(options ...ClientOption) *Client { + client := &Client{ baseURL: "https://huggingface.co/api/models", client: httpclient.New(httpclient.WithFollowRedirects()), maxRetries: 5, retryBackoff: 1 * time.Second, maxBackoff: 30 * time.Second, - sleepFn: time.Sleep, + clock: realClock{}, } + for _, option := range options { + option(client) + } + return client } func (c *Client) newRequest(ctx context.Context, method, rawURL, token string) (*http.Request, error) { @@ -143,7 +172,7 @@ func (c *Client) SearchModels(params SearchParams) ([]Model, error) { resp, err := c.client.Do(req) if err != nil { if attempt < c.maxRetries { - c.sleepFn(c.exponentialBackoff(attempt)) + c.clock.Sleep(c.exponentialBackoff(attempt)) continue } return nil, fmt.Errorf("failed to make request: %w", err) @@ -154,7 +183,7 @@ func (c *Client) SearchModels(params SearchParams) ([]Model, error) { return nil, fmt.Errorf("failed to close response body: %w", err) } if c.isRetryableStatus(resp.StatusCode) && attempt < c.maxRetries { - c.sleepFn(c.retryDelay(resp, attempt)) + c.clock.Sleep(c.retryDelay(resp, attempt)) continue } if resp.StatusCode == http.StatusTooManyRequests { @@ -199,7 +228,7 @@ func (c *Client) retryDelay(resp *http.Response, attempt int) time.Duration { return delay } if at, err := http.ParseTime(retryAfter); err == nil { - delay := time.Until(at) + delay := at.Sub(c.clock.Now()) if delay > 0 { if delay > c.maxBackoff { return c.maxBackoff @@ -213,17 +242,11 @@ func (c *Client) retryDelay(resp *http.Response, attempt int) time.Duration { } func (c *Client) exponentialBackoff(attempt int) time.Duration { - delay := c.retryBackoff - for i := 1; i < attempt; i++ { - delay *= 2 - if delay >= c.maxBackoff { - return c.maxBackoff - } + exponent := 0 + if attempt > 1 { + exponent = attempt - 1 } - if delay > c.maxBackoff { - return c.maxBackoff - } - return delay + return backoff.Exponential(c.retryBackoff, c.maxBackoff, uint(exponent)) } // GetLatest fetches the latest GGUF models diff --git a/pkg/huggingface-api/client_test.go b/pkg/huggingface-api/client_test.go index feac4dba3..591c26aaf 100644 --- a/pkg/huggingface-api/client_test.go +++ b/pkg/huggingface-api/client_test.go @@ -14,14 +14,28 @@ import ( hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" ) +type fakeClock struct { + now time.Time + sleeps []time.Duration +} + +func (c *fakeClock) Now() time.Time { return c.now } + +func (c *fakeClock) Sleep(d time.Duration) { + c.sleeps = append(c.sleeps, d) + c.now = c.now.Add(d) +} + var _ = Describe("HuggingFace API Client", func() { var ( client *hfapi.Client server *httptest.Server + clock *fakeClock ) BeforeEach(func() { - client = hfapi.NewClient() + clock = &fakeClock{now: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)} + client = hfapi.NewClient(hfapi.WithClock(clock)) }) AfterEach(func() { @@ -211,14 +225,35 @@ var _ = Describe("HuggingFace API Client", func() { Search: "GGUF", } - start := time.Now() models, err := client.SearchModels(params) - elapsed := time.Since(start) Expect(err).ToNot(HaveOccurred()) Expect(models).To(HaveLen(0)) Expect(attempts).To(Equal(2)) - Expect(elapsed).To(BeNumerically(">=", 900*time.Millisecond)) + Expect(clock.sleeps).To(Equal([]time.Duration{time.Second})) + }) + + It("should calculate HTTP-date Retry-After using the injected clock", func() { + attempts := 0 + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + if attempts == 1 { + w.Header().Set("Retry-After", clock.now.Add(2*time.Second).Format(http.TimeFormat)) + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte("[]")) + Expect(err).ToNot(HaveOccurred()) + })) + client.SetBaseURL(server.URL) + + models, err := client.SearchModels(hfapi.SearchParams{Search: "GGUF"}) + + Expect(err).ToNot(HaveOccurred()) + Expect(models).To(BeEmpty()) + Expect(attempts).To(Equal(2)) + Expect(clock.sleeps).To(Equal([]time.Duration{2 * time.Second})) }) It("should fail fast on non-retryable 4xx responses", func() { @@ -267,6 +302,9 @@ var _ = Describe("HuggingFace API Client", func() { Expect(errors.Is(err, hfapi.ErrRateLimited)).To(BeTrue()) Expect(err.Error()).To(ContainSubstring("Status code: 429")) Expect(models).To(BeNil()) + Expect(clock.sleeps).To(Equal([]time.Duration{ + time.Second, time.Second, time.Second, time.Second, + })) }) }) @@ -336,8 +374,12 @@ var _ = Describe("HuggingFace API Client", func() { Context("when handling network errors", func() { It("should handle connection failures gracefully", func() { - // Use an invalid URL to simulate connection failure - client.SetBaseURL("http://invalid-url-that-does-not-exist") + // A closed loopback listener produces a deterministic connection + // failure without relying on DNS or public network access. + closedServer := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + closedURL := closedServer.URL + closedServer.Close() + client.SetBaseURL(closedURL) params := hfapi.SearchParams{ Sort: "lastModified", @@ -351,11 +393,26 @@ var _ = Describe("HuggingFace API Client", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to make request")) Expect(models).To(BeNil()) + Expect(clock.sleeps).To(Equal([]time.Duration{ + time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, + })) }) }) - Context("when getting file SHA on remote model", func() { + Context("when getting file SHA from repository metadata", func() { It("should get file SHA successfully", func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(`[{ + "type":"file", + "path":"localai-functioncall-qwen2.5-7b-v0.5-q4_k_m.gguf", + "size":42, + "oid":"pointer-oid", + "lfs":{"oid":"4e7b7fe1d54b881f1ef90799219dc6cc285d29db24f559c8998d1addb35713d4","size":42,"pointerSize":128} + }]`)) + Expect(err).NotTo(HaveOccurred()) + })) + client.SetBaseURL(server.URL + "/api/models") sha, err := client.GetFileSHA( "mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF", "localai-functioncall-qwen2.5-7b-v0.5-q4_k_m.gguf") Expect(err).ToNot(HaveOccurred()) @@ -886,14 +943,33 @@ var _ = Describe("HuggingFace API Client", func() { }) }) - Context("integration test with real HuggingFace API", func() { - It("should recursively list all files including subfolders from real repository", func() { - // This test makes actual API calls to HuggingFace - // Skip if running in CI or if network is not available - realClient := hfapi.NewClient() + Context("repository API compatibility fixtures", func() { + It("should recursively list all files including subfolders", func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var response string + switch { + case strings.HasSuffix(r.URL.Path, "/tree/main"): + response = `[ + {"type":"file","path":"README.md","size":100,"oid":"readme-oid"}, + {"type":"directory","path":"Q4_K_M","size":0,"oid":"directory-oid"} + ]` + case strings.HasSuffix(r.URL.Path, "/tree/main/Q4_K_M"): + response = `[ + {"type":"file","path":"Q4_K_M/model-00001-of-00002.gguf","size":1000,"oid":"model-oid"} + ]` + default: + w.WriteHeader(http.StatusNotFound) + return + } + _, err := w.Write([]byte(response)) + Expect(err).NotTo(HaveOccurred()) + })) + fixtureClient := hfapi.NewClient() + fixtureClient.SetBaseURL(server.URL + "/api/models") repoID := "bartowski/Qwen_Qwen3-Next-80B-A3B-Instruct-GGUF" - files, err := realClient.ListFiles(repoID) + files, err := fixtureClient.ListFiles(repoID) Expect(err).ToNot(HaveOccurred()) Expect(files).ToNot(BeEmpty(), "should return at least some files") @@ -956,12 +1032,19 @@ var _ = Describe("HuggingFace API Client", func() { }) It("should populate PipelineTag and LibraryName on ModelDetails", func() { - // Sentence-transformers/all-MiniLM-L6-v2 is a public, stable repo: - // pipeline_tag: sentence-similarity, library_name: sentence-transformers. - // This exercises the /api/models/{repo} metadata fetch layered on top - // of ListFiles in GetModelDetails. - realClient := hfapi.NewClient() - details, err := realClient.GetModelDetails("sentence-transformers/all-MiniLM-L6-v2") + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.Contains(r.URL.Path, "/tree/main") { + _, err := w.Write([]byte(`[{"type":"file","path":"config.json","size":100,"oid":"config-oid"}]`)) + Expect(err).NotTo(HaveOccurred()) + return + } + _, err := w.Write([]byte(`{"pipeline_tag":"sentence-similarity","library_name":"sentence-transformers"}`)) + Expect(err).NotTo(HaveOccurred()) + })) + fixtureClient := hfapi.NewClient() + fixtureClient.SetBaseURL(server.URL + "/api/models") + details, err := fixtureClient.GetModelDetails("sentence-transformers/all-MiniLM-L6-v2") Expect(err).ToNot(HaveOccurred()) Expect(details).ToNot(BeNil()) Expect(details.PipelineTag).To(Equal("sentence-similarity")) diff --git a/pkg/model/loader.go b/pkg/model/loader.go index 77b7a0ed6..e2c8c336c 100644 --- a/pkg/model/loader.go +++ b/pkg/model/loader.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "github.com/mudler/LocalAI/internal/backoff" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" @@ -201,17 +202,8 @@ func (ml *ModelLoader) recordLoadFailure(modelID string) { ml.loadFailures[modelID] = st } st.consecutive++ - // base * 2^(consecutive-1), clamped. Cap the shift to avoid overflowing - // the Duration; anything past the cap collapses to loadFailureMaxCooldown. - shift := st.consecutive - 1 - if shift > 20 { - shift = 20 - } - backoff := ml.loadFailureBaseCooldown * (1 << shift) - if backoff <= 0 || backoff > ml.loadFailureMaxCooldown { - backoff = ml.loadFailureMaxCooldown - } - st.cooldownUntil = time.Now().Add(backoff) + delay := backoff.Exponential(ml.loadFailureBaseCooldown, ml.loadFailureMaxCooldown, uint(st.consecutive-1)) + st.cooldownUntil = time.Now().Add(delay) } // clearLoadFailure resets the modelID's failure state after a successful load. diff --git a/pkg/model/loader_test.go b/pkg/model/loader_test.go index 1a8829431..da5b2f037 100644 --- a/pkg/model/loader_test.go +++ b/pkg/model/loader_test.go @@ -65,8 +65,7 @@ var _ = Describe("ModelLoader", func() { BeforeEach(func() { // Setup the model loader with a test directory - modelPath = "/tmp/test_model_path" - os.Mkdir(modelPath, 0755) + modelPath = GinkgoT().TempDir() systemState, err := system.GetSystemState( system.WithModelPath(modelPath), @@ -75,11 +74,6 @@ var _ = Describe("ModelLoader", func() { modelLoader = model.NewModelLoader(systemState) }) - AfterEach(func() { - // Cleanup test directory - os.RemoveAll(modelPath) - }) - Context("NewModelLoader", func() { It("should create a new ModelLoader with an empty model map", func() { Expect(modelLoader).ToNot(BeNil()) diff --git a/pkg/modelartifacts/materializer.go b/pkg/modelartifacts/materializer.go index a3e2a9e8e..005be3871 100644 --- a/pkg/modelartifacts/materializer.go +++ b/pkg/modelartifacts/materializer.go @@ -20,6 +20,7 @@ import ( "github.com/gofrs/flock" "github.com/mudler/xlog" + "github.com/mudler/LocalAI/internal/backoff" "github.com/mudler/LocalAI/pkg/downloader" hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" ) @@ -326,9 +327,7 @@ func (m *Manager) acquireLock(ctx context.Context, locker Locker, lockPath strin return ctx.Err() case <-time.After(interval): } - if interval < maxLockRetryInterval { - interval = min(interval*2, maxLockRetryInterval) - } + interval = backoff.Exponential(interval, maxLockRetryInterval, 1) } } diff --git a/pkg/oci/blob.go b/pkg/oci/blob.go index e034c4162..63aa44d5f 100644 --- a/pkg/oci/blob.go +++ b/pkg/oci/blob.go @@ -16,6 +16,10 @@ import ( ) func FetchImageBlob(ctx context.Context, r, reference, dst string, statusReader func(ocispec.Descriptor) io.Writer) error { + return fetchImageBlob(ctx, r, reference, dst, statusReader, false) +} + +func fetchImageBlob(ctx context.Context, r, reference, dst string, statusReader func(ocispec.Descriptor) io.Writer, plainHTTP bool) error { // 0. Create a file store for the output fs, err := os.Create(dst) if err != nil { @@ -29,6 +33,7 @@ func FetchImageBlob(ctx context.Context, r, reference, dst string, statusReader return fmt.Errorf("failed to create repository: %v", err) } repo.SkipReferrersGC = true + repo.PlainHTTP = plainHTTP // Identify LocalAI to the registry. This mirrors oras' auth.DefaultClient // (same retry policy) but advertises a LocalAI User-Agent instead of the diff --git a/pkg/oci/blob_test.go b/pkg/oci/blob_test.go index cef29a972..76b7d49b6 100644 --- a/pkg/oci/blob_test.go +++ b/pkg/oci/blob_test.go @@ -1,10 +1,14 @@ -package oci_test +package oci import ( "context" + "crypto/sha256" + "fmt" + "net/http" + "net/http/httptest" "os" + "strings" - . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -12,11 +16,22 @@ import ( var _ = Describe("OCI", func() { Context("pulling images", func() { It("should fetch blobs correctly", func() { + payload := []byte("local OCI blob fixture") + digest := fmt.Sprintf("sha256:%x", sha256.Sum256(payload)) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") + w.Header().Set("Content-Length", fmt.Sprint(len(payload))) + if r.Method != http.MethodHead { + _, _ = w.Write(payload) + } + })) + defer server.Close() f, err := os.CreateTemp("", "ollama") Expect(err).NotTo(HaveOccurred()) defer os.RemoveAll(f.Name()) - err = FetchImageBlob(context.TODO(), "registry.ollama.ai/library/gemma", "sha256:c1864a5eb19305c40519da12cc543519e48a0697ecd30e15d5ac228644957d12", f.Name(), nil) + err = fetchImageBlob(context.Background(), strings.TrimPrefix(server.URL, "http://")+"/library/gemma", digest, f.Name(), nil, true) Expect(err).NotTo(HaveOccurred()) + Expect(os.ReadFile(f.Name())).To(Equal(payload)) }) }) }) diff --git a/pkg/oci/cosignverify/verify.go b/pkg/oci/cosignverify/verify.go index 579b0d8c6..ac1d4fdc6 100644 --- a/pkg/oci/cosignverify/verify.go +++ b/pkg/oci/cosignverify/verify.go @@ -34,6 +34,8 @@ import ( "github.com/sigstore/sigstore-go/pkg/root" "github.com/sigstore/sigstore-go/pkg/tuf" "github.com/sigstore/sigstore-go/pkg/verify" + + "github.com/mudler/LocalAI/pkg/httpclient" ) // Policy is the verification policy a backend image must satisfy. @@ -289,7 +291,7 @@ func enforceNotBefore(result *verify.VerificationResult, cutoff time.Time) error func (v *Verifier) remoteOptions(ctx context.Context) []remote.Option { t := v.transport if t == nil { - t = http.DefaultTransport + t = httpclient.HardenedTransport() } // Match the retry policy used elsewhere in pkg/oci so transient // registry hiccups don't fail verification. diff --git a/pkg/oci/image_test.go b/pkg/oci/image_test.go index 447bc90f6..300cfc683 100644 --- a/pkg/oci/image_test.go +++ b/pkg/oci/image_test.go @@ -1,10 +1,15 @@ package oci_test import ( + "archive/tar" + "bytes" "context" "os" - "runtime" + "path/filepath" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/mudler/LocalAI/pkg/oci" . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path . "github.com/onsi/ginkgo/v2" @@ -15,25 +20,30 @@ var _ = Describe("OCI", func() { Context("when template is loaded successfully", func() { It("should evaluate the template correctly", func() { - if runtime.GOOS == "darwin" { - Skip("Skipping test on darwin") - } - imageName := "alpine" - img, err := GetImage(imageName, "", nil, nil) + var layerTar bytes.Buffer + writer := tar.NewWriter(&layerTar) + content := []byte("offline OCI fixture\n") + Expect(writer.WriteHeader(&tar.Header{Name: "fixture.txt", Mode: 0o644, Size: int64(len(content))})).To(Succeed()) + _, err := writer.Write(content) Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) - size, err := GetOCIImageSize(imageName, "", nil, nil) + layer, err := tarball.LayerFromReader(bytes.NewReader(layerTar.Bytes())) Expect(err).NotTo(HaveOccurred()) - - Expect(size).ToNot(Equal(int64(0))) + img, err := mutate.AppendLayers(empty.Image, layer) + Expect(err).NotTo(HaveOccurred()) + size, err := layer.Size() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(BeNumerically(">", 0)) // Create tempdir dir, err := os.MkdirTemp("", "example") Expect(err).NotTo(HaveOccurred()) - defer os.RemoveAll(dir) + DeferCleanup(os.RemoveAll, dir) - err = ExtractOCIImage(context.TODO(), img, imageName, dir, nil) + err = ExtractOCIImage(context.TODO(), img, "fixture:offline", dir, nil) Expect(err).NotTo(HaveOccurred()) + Expect(os.ReadFile(filepath.Join(dir, "fixture.txt"))).To(Equal(content)) }) }) }) diff --git a/pkg/oci/ollama.go b/pkg/oci/ollama.go index f0a874013..42bf64bfc 100644 --- a/pkg/oci/ollama.go +++ b/pkg/oci/ollama.go @@ -35,6 +35,10 @@ type LayerDetail struct { } func OllamaModelManifest(image string) (*Manifest, error) { + return ollamaModelManifest("https", "registry.ollama.ai", image) +} + +func ollamaModelManifest(scheme, registry, image string) (*Manifest, error) { // parse the repository and tag from `image`. `image` should be for e.g. gemma:2b, or foobar/gemma:2b // if there is a : in the image, then split it @@ -42,7 +46,7 @@ func OllamaModelManifest(image string) (*Manifest, error) { tag, repository, image := ParseImageParts(image) // get e.g. https://registry.ollama.ai/v2/library/llama3/manifests/latest - req, err := http.NewRequest("GET", "https://registry.ollama.ai/v2/"+repository+"/"+image+"/manifests/"+tag, nil) + req, err := http.NewRequest("GET", scheme+"://"+registry+"/v2/"+repository+"/"+image+"/manifests/"+tag, nil) if err != nil { return nil, err } @@ -65,7 +69,11 @@ func OllamaModelManifest(image string) (*Manifest, error) { } func OllamaModelBlob(image string) (string, error) { - manifest, err := OllamaModelManifest(image) + return ollamaModelBlob("https", "registry.ollama.ai", image) +} + +func ollamaModelBlob(scheme, registry, image string) (string, error) { + manifest, err := ollamaModelManifest(scheme, registry, image) if err != nil { return "", err } @@ -81,12 +89,16 @@ func OllamaModelBlob(image string) (string, error) { } func OllamaFetchModel(ctx context.Context, image string, output string, statusWriter func(ocispec.Descriptor) io.Writer) error { + return ollamaFetchModel(ctx, "https", "registry.ollama.ai", image, output, statusWriter) +} + +func ollamaFetchModel(ctx context.Context, scheme, registry, image string, output string, statusWriter func(ocispec.Descriptor) io.Writer) error { _, repository, imageNoTag := ParseImageParts(image) - blobID, err := OllamaModelBlob(image) + blobID, err := ollamaModelBlob(scheme, registry, image) if err != nil { return err } - return FetchImageBlob(ctx, fmt.Sprintf("registry.ollama.ai/%s/%s", repository, imageNoTag), blobID, output, statusWriter) + return fetchImageBlob(ctx, fmt.Sprintf("%s/%s/%s", registry, repository, imageNoTag), blobID, output, statusWriter, scheme == "http") } diff --git a/pkg/oci/ollama_test.go b/pkg/oci/ollama_test.go index fbda69e6b..bed92a19c 100644 --- a/pkg/oci/ollama_test.go +++ b/pkg/oci/ollama_test.go @@ -1,10 +1,15 @@ -package oci_test +package oci import ( "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" "os" + "strings" - . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -12,11 +17,26 @@ import ( var _ = Describe("OCI", func() { Context("ollama", func() { It("pulls model files", func() { + payload := []byte("local Ollama model fixture") + digest := fmt.Sprintf("sha256:%x", sha256.Sum256(payload)) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") + if strings.Contains(r.URL.Path, "/manifests/") { + _ = json.NewEncoder(w).Encode(Manifest{SchemaVersion: 2, Layers: []LayerDetail{{Digest: digest, MediaType: "application/vnd.ollama.image.model", Size: len(payload)}}}) + return + } + w.Header().Set("Content-Length", fmt.Sprint(len(payload))) + if r.Method != http.MethodHead { + _, _ = w.Write(payload) + } + })) + defer server.Close() f, err := os.CreateTemp("", "ollama") Expect(err).NotTo(HaveOccurred()) defer os.RemoveAll(f.Name()) - err = OllamaFetchModel(context.TODO(), "gemma:2b", f.Name(), nil) + err = ollamaFetchModel(context.Background(), "http", strings.TrimPrefix(server.URL, "http://"), "gemma:2b", f.Name(), nil) Expect(err).NotTo(HaveOccurred()) + Expect(os.ReadFile(f.Name())).To(Equal(payload)) }) }) }) diff --git a/pkg/testnetwork/guard.go b/pkg/testnetwork/guard.go new file mode 100644 index 000000000..17db55c0e --- /dev/null +++ b/pkg/testnetwork/guard.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: MIT + +// Package testnetwork provides an explicit outbound-network guard for tests. +package testnetwork + +import ( + "context" + "fmt" + "net" + "net/netip" + "strings" +) + +type Guard struct { + Dialer net.Dialer + Dial func(context.Context, string, string) (net.Conn, error) + Allowed []netip.Prefix +} + +func LocalGuard() *Guard { + prefixes := []string{"127.0.0.0/8", "::1/128", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"} + guard := &Guard{} + for _, prefix := range prefixes { + guard.Allowed = append(guard.Allowed, netip.MustParsePrefix(prefix)) + } + return guard +} + +func (g *Guard) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + originalAddress := address + host, _, err := net.SplitHostPort(address) + if err != nil { + return nil, fmt.Errorf("test network guard: invalid address %q: %w", address, err) + } + addresses, err := net.DefaultResolver.LookupNetIP(ctx, "ip", strings.Trim(host, "[]")) + if err != nil { + return nil, fmt.Errorf("test network guard: resolve %q: %w", host, err) + } + for _, resolved := range addresses { + if !g.allowed(resolved.Unmap()) { + return nil, fmt.Errorf("test network guard: public dial blocked: %s (%s)", host, resolved) + } + } + if g.Dial != nil { + return g.Dial(ctx, network, originalAddress) + } + return g.Dialer.DialContext(ctx, network, originalAddress) +} + +func (g *Guard) allowed(address netip.Addr) bool { + for _, prefix := range g.Allowed { + if prefix.Contains(address) { + return true + } + } + return false +} diff --git a/pkg/testnetwork/guard_suite_test.go b/pkg/testnetwork/guard_suite_test.go new file mode 100644 index 000000000..9259f5a64 --- /dev/null +++ b/pkg/testnetwork/guard_suite_test.go @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT + +package testnetwork_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestNetworkGuard(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Test network guard suite") +} diff --git a/pkg/testnetwork/guard_test.go b/pkg/testnetwork/guard_test.go new file mode 100644 index 000000000..5f658eb19 --- /dev/null +++ b/pkg/testnetwork/guard_test.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MIT + +package testnetwork_test + +import ( + "context" + "errors" + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/pkg/testnetwork" +) + +var _ = Describe("Guard", func() { + It("blocks a public IP before dialing", func() { + _, err := testnetwork.LocalGuard().DialContext(context.Background(), "tcp", "203.0.113.1:443") + Expect(err).To(MatchError(ContainSubstring("public dial blocked"))) + }) + + It("allows loopback fixtures", func() { + guard := testnetwork.LocalGuard() + called := false + guard.Dial = func(_ context.Context, network, address string) (net.Conn, error) { + called = true + Expect(network).To(Equal("tcp")) + Expect(address).To(Equal("127.0.0.1:8080")) + return nil, errors.New("fixture dial sentinel") + } + _, err := guard.DialContext(context.Background(), "tcp", "127.0.0.1:8080") + Expect(err).To(MatchError("fixture dial sentinel")) + Expect(called).To(BeTrue()) + }) +}) diff --git a/scripts/jetson-wheels-sync.py b/scripts/jetson-wheels-sync.py new file mode 100644 index 000000000..fd41c623e --- /dev/null +++ b/scripts/jetson-wheels-sync.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Mirror the jetson-only wheel subset from pypi.jetson-ai-lab.io. + +Downloads the wheels for the packages listed in .github/jetson-wheels.json +(one list per JetPack index path) into a local directory laid out exactly +like the upstream index (///). The jetson-wheels +CI workflow publishes that directory as a scratch OCI image on ghcr, and +backend builds serve it as a local package index during pip install — see +pypi_mirror_server.py and installRequirements in +backend/python/common/libbackend.sh for the consuming side and the +motivation (recurring multi-hour upstream outages). + +The sync is additive-with-pruning against a *successfully fetched* project +page: files no longer listed upstream are removed, but a package whose page +cannot be fetched is left exactly as mirrored last time. When the whole +upstream is unreachable the existing mirror is kept as-is (exit 0) so a CI +run during an outage never destroys the last known-good wheels; it only +fails (exit 2) when upstream is down AND there is nothing mirrored yet, +i.e. the bootstrap run has nothing to publish. + +Standard library only. Usage: + python3 scripts/jetson-wheels-sync.py --config .github/jetson-wheels.json \ + --index jp6/cu129 --dest wheels [--changed-file /tmp/changed] +""" + +import argparse +import hashlib +import html.parser +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request + +DIST_SUFFIXES = (".whl", ".tar.gz", ".zip") +TIMEOUT = 120 + + +def normalize(name): + """PEP 503 project-name normalization.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +class _LinkParser(html.parser.HTMLParser): + def __init__(self): + super().__init__() + self.hrefs = [] + + def handle_starttag(self, tag, attrs): + if tag == "a": + for key, value in attrs: + if key == "href" and value: + self.hrefs.append(value) + + +def parse_links(page, base_url): + """Extract (filename, absolute_url, sha256|None) for each dist link.""" + parser = _LinkParser() + parser.feed(page) + links = [] + for href in parser.hrefs: + split = urllib.parse.urlsplit(href) + filename = os.path.basename(urllib.parse.unquote(split.path)) + if not filename.endswith(DIST_SUFFIXES): + continue + sha256 = None + if split.fragment.startswith("sha256="): + sha256 = split.fragment[len("sha256="):] + url = urllib.parse.urljoin(base_url, split._replace(fragment="").geturl()) + links.append((filename, url, sha256)) + return links + + +def _fetch(url): + request = urllib.request.Request(url, headers={"User-Agent": "localai-jetson-wheels-sync"}) + return urllib.request.urlopen(request, timeout=TIMEOUT) + + +def _download(url, dest_path, sha256): + digest = hashlib.sha256() + tmp = dest_path + ".tmp" + with _fetch(url) as resp, open(tmp, "wb") as out: + for chunk in iter(lambda: resp.read(1 << 20), b""): + digest.update(chunk) + out.write(chunk) + if sha256 and digest.hexdigest() != sha256: + os.unlink(tmp) + raise RuntimeError(f"sha256 mismatch for {url}: expected {sha256}, got {digest.hexdigest()}") + os.replace(tmp, dest_path) + + +def _has_wheels(dest): + for _, _, files in os.walk(dest): + if any(f.endswith(DIST_SUFFIXES) for f in files): + return True + return False + + +def sync_package(base_url, package, dest_dir): + """Returns (fetched_ok, changed).""" + page_url = urllib.parse.urljoin(base_url, normalize(package) + "/") + try: + with _fetch(page_url) as resp: + page = resp.read().decode("utf-8", "replace") + except urllib.error.HTTPError as err: + if err.code == 404: + # Upstream simply doesn't host this package for this index — + # normal (the config list is a superset across JetPack versions). + print(f" {package}: not hosted upstream (404), skipping") + return True, False + print(f" {package}: upstream error {err.code}, keeping mirrored files") + return False, False + except (urllib.error.URLError, TimeoutError, OSError) as err: + print(f" {package}: upstream unreachable ({err}), keeping mirrored files") + return False, False + + links = parse_links(page, page_url) + listed = {name for name, _, _ in links} + changed = False + os.makedirs(dest_dir, exist_ok=True) + for name, url, sha256 in links: + path = os.path.join(dest_dir, name) + if os.path.exists(path): + continue + print(f" {package}: downloading {name}") + _download(url, path, sha256) + changed = True + # Prune only against a page we actually fetched: upstream removing a + # wheel is tracked, an outage never empties the mirror. + for existing in os.listdir(dest_dir): + if existing.endswith(DIST_SUFFIXES) and existing not in listed: + print(f" {package}: pruning {existing} (no longer listed upstream)") + os.unlink(os.path.join(dest_dir, existing)) + changed = True + return True, changed + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--index", required=True, help="index path, e.g. jp6/cu129") + parser.add_argument("--dest", required=True) + parser.add_argument("--upstream", help="override the config's upstream (for tests)") + parser.add_argument("--changed-file", help="created iff the mirror content changed") + args = parser.parse_args() + + with open(args.config) as f: + config = json.load(f) + packages = config["indexes"][args.index] + upstream = args.upstream or config["upstream"] + base_url = upstream.rstrip("/") + "/" + args.index.strip("/") + "/" + + print(f"syncing {args.index} from {base_url}: {', '.join(packages)}") + any_fetched = False + any_changed = False + for package in packages: + dest_dir = os.path.join(args.dest, args.index, normalize(package)) + fetched, changed = sync_package(base_url, package, dest_dir) + any_fetched = any_fetched or fetched + any_changed = any_changed or changed + + if not any_fetched: + if _has_wheels(os.path.join(args.dest, args.index)): + print("upstream unreachable; keeping existing mirror unchanged") + return 0 + print("upstream unreachable and nothing mirrored yet — nothing to publish") + return 2 + if any_changed and args.changed_file: + with open(args.changed_file, "w") as f: + f.write("changed\n") + print("sync complete" + (" (changes)" if any_changed else " (no changes)")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run-coverage.sh b/scripts/run-coverage.sh index 430ca1f11..771fa4242 100755 --- a/scripts/run-coverage.sh +++ b/scripts/run-coverage.sh @@ -21,6 +21,13 @@ # "!real-models" (those specs need a downloaded model). # COVERAGE_EXCLUDE_RE egrep pattern of profile lines to drop before merging, # e.g. generated protobuf (grpc/proto/.*\.pb\.go). +# COVERAGE_PROCS parallel Ginkgo processes; 0 lets Ginkgo detect CPUs. +# COVERAGE_SUITE_TIMEOUT maximum duration of each recursive root (default 5m). +# COVERAGE_PROGRESS_AFTER emit diagnostics when a spec is slow (default 30s). +# +# Verbose Ginkgo output is retained in OUTPUT_DIR/logs. The previous run's log +# for each root is kept with a .previous suffix, so a noisy failure remains +# available without flooding the commit-hook output. # # Why one ginkgo invocation per root: passing several recursive roots to a # single ginkgo run only merges ONE root's coverprofile into --output-dir @@ -41,11 +48,43 @@ shift 3 unit_roots="$*" # space-free tokens (./pkg ./core) mkdir -p "$out_dir" +lock_dir="$out_dir/.run-coverage.lock" +if ! mkdir "$lock_dir" 2>/dev/null; then + echo "run-coverage: another coverage run is using $out_dir" >&2 + echo "run-coverage: wait for it to finish; if none is running, remove stale lock $lock_dir" >&2 + exit 2 +fi +cleanup() { + for root in $unit_roots ${COVERAGE_E2E_ROOTS:-}; do + # --keep-separate-coverprofiles leaves Go's package test binaries behind. + # They are deterministic runner artifacts, never source inputs. + find "$root" -type f -name '*.test' -delete 2>/dev/null || : + done + rmdir "$lock_dir" 2>/dev/null || : +} +trap cleanup EXIT +trap 'exit 130' HUP INT TERM + +log_dir="$out_dir/logs" +mkdir -p "$log_dir" # Clear per-root profiles from a previous run: the merge collects them by glob, # so a stale profile (e.g. from a root that failed to rebuild this run) must not # leak into the merged result. rm -f "$out_dir"/cover-*.out +rm -f "$out_dir"/*_cover-*.out +rm -f "$merged" fail=0 +timings="$out_dir/timings.tsv" +: > "$timings" +run_started="$(date +%s)" + +procs="${COVERAGE_PROCS:-0}" +suite_timeout="${COVERAGE_SUITE_TIMEOUT:-5m}" +progress_after="${COVERAGE_PROGRESS_AFTER:-30s}" +parallel_flags="-p --keep-going --timeout=$suite_timeout --poll-progress-after=$progress_after --poll-progress-interval=10s" +if [ "$procs" -gt 0 ] 2>/dev/null; then + parallel_flags="$parallel_flags --procs=$procs --compilers=$procs" +fi # Common optional flags go into "$@"; unquoted ${VAR:+...} would word-split a # --tags value that contains a space. The unit roots were captured above, so @@ -59,26 +98,133 @@ profile_name() { printf 'cover-%s.out' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')" } +log_name() { + printf '%s.log' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')" +} + +rotate_log() { + log="$1" + if [ -f "$log" ]; then + mv -f "$log" "$log.previous" + fi +} + +# Ginkgo's recursive-run merger can fail after every suite has passed when a +# large --coverpkg run produces many profiles. Keep its profiles separate and +# merge them here using the same block-summing rule as the cross-root merge. +consolidate_root_profiles() { + base="$1" + set -- "$out_dir"/*_"$base" + if [ ! -e "$1" ]; then + echo "run-coverage: no per-package profiles produced for $base" >&2 + return 1 + fi + tmp="$out_dir/.${base}.tmp" + { + echo "mode: atomic" + awk ' + /^mode:/ { next } + { stmts[$1] = $2; cnt[$1] += $3 } + END { for (k in stmts) print k, stmts[k], cnt[k] } + ' "$@" + } > "$tmp" + mv "$tmp" "$out_dir/$base" + rm -f "$@" +} + +report_failure() { + root="$1" + log="$2" + echo "run-coverage: FAIL — tests under coverage failed for $root" >&2 + echo "run-coverage: full output: $log" >&2 + echo "run-coverage: relevant tail:" >&2 + # Keep the terminal useful even when Ginkgo emits thousands of verbose lines. + # The complete log remains available when this short extract is insufficient. + summary="$(grep -E 'Summarizing|\[FAIL(ED)?\]|FAIL!|--- FAIL:|Test Suite Failed|could not finalize|Status code: 429|HTTP 429|rate limit|timed out|panic:|fork/exec|no such file or directory|Expected.*(but got|success)' "$log" \ + | tail -n 30)" + if [ -n "$summary" ]; then + printf '%s\n' "$summary" >&2 + else + tail -n 30 "$log" >&2 + fi +} + +record_timing() { + root="$1" + started="$2" + elapsed="$(( $(date +%s) - started ))" + printf '%s\t%s\n' "$root" "$elapsed" >> "$timings" + echo "run-coverage: TIMING — $root ${elapsed}s" +} + +print_timing_summary() { + echo "run-coverage: wall-clock summary" + sort -t "$(printf '\t')" -k2,2nr "$timings" | awk -F '\t' '{ printf " %5ss %s\n", $2, $1 }' + echo " $(( $(date +%s) - run_started ))s total" + echo "run-coverage: slowest specs/hooks taking at least ${COVERAGE_SLOW_SPEC_THRESHOLD:-3}s (up to ${COVERAGE_SLOW_SPEC_LIMIT:-25} per root)" + found=0 + while IFS="$(printf '\t')" read -r root elapsed; do + log="$log_dir/$(log_name "$root")" + entries="$(scripts/summarize-ginkgo-waits.sh "${COVERAGE_SLOW_SPEC_THRESHOLD:-3}" "$root" "$log" "${COVERAGE_SLOW_SPEC_LIMIT:-25}")" + if [ -n "$entries" ]; then + printf '%s\n' "$entries" + found=1 + fi + done < "$timings" + [ "$found" -eq 1 ] || echo " none" +} + # Unit/suite roots: recursive. for root in $unit_roots; do base="$(profile_name "$root")" - go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v -r "$@" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 + log="$log_dir/$(log_name "$root")" + rotate_log "$log" + echo "run-coverage: testing $root (full output: $log)" + started="$(date +%s)" + # parallel_flags is intentionally word-split: it contains CLI arguments only. + # shellcheck disable=SC2086 + go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --keep-separate-coverprofiles --flake-attempts "$flakes" -v -r "$@" \ + --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ + && consolidate_root_profiles "$base" \ + && echo "run-coverage: PASS — $root" \ + || { fail=1; report_failure "$root" "$log"; } + record_timing "$root" "$started" done # In-process integration roots: NON-recursive + optional label filter. for root in ${COVERAGE_E2E_ROOTS:-}; do base="$(profile_name "$root")" + log="$log_dir/$(log_name "$root")" + rotate_log "$log" + echo "run-coverage: testing $root (full output: $log)" + started="$(date +%s)" if [ -n "${COVERAGE_E2E_LABELS:-}" ]; then - go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \ + # shellcheck disable=SC2086 + go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --keep-separate-coverprofiles --flake-attempts "$flakes" -v "$@" \ --label-filter="$COVERAGE_E2E_LABELS" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 + --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ + && consolidate_root_profiles "$base" \ + && echo "run-coverage: PASS — $root" \ + || { fail=1; report_failure "$root" "$log"; } else - go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 + # shellcheck disable=SC2086 + go run github.com/onsi/ginkgo/v2/ginkgo $parallel_flags --keep-separate-coverprofiles --flake-attempts "$flakes" -v "$@" \ + --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" >"$log" 2>&1 \ + && consolidate_root_profiles "$base" \ + && echo "run-coverage: PASS — $root" \ + || { fail=1; report_failure "$root" "$log"; } fi + record_timing "$root" "$started" done +print_timing_summary + +if [ "$fail" -ne 0 ]; then + echo "run-coverage: FAILED — one or more test suites failed; no merged profile was produced." >&2 + echo "run-coverage: the coverage percentage ratchet was not run." >&2 + exit "$fail" +fi + # Collect the per-root profiles by glob (space-safe, no list to track). set -- "$out_dir"/cover-*.out if [ ! -e "$1" ]; then @@ -98,4 +244,4 @@ fi ' "$@" } > "$merged" -exit "$fail" +echo "run-coverage: all test suites passed; merged profile: $merged" diff --git a/scripts/run-test-linux-offline.sh b/scripts/run-test-linux-offline.sh new file mode 100755 index 000000000..b2392aa7e --- /dev/null +++ b/scripts/run-test-linux-offline.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +if [[ $(uname -s) != Linux ]]; then + echo 'kernel-level test egress enforcement is Linux-only' >&2 + exit 2 +fi +if [[ $# -lt 2 ]]; then + echo "usage: $0 TARGET COMMAND [ARG...]" >&2 + exit 2 +fi + +root=$(cd "$(dirname "$0")/.." && pwd) +group="localai-test-$$" +cgroup="/sys/fs/cgroup/$group" +parent_cgroup="/sys/fs/cgroup$(awk -F: '$1 == "0" {print $3}' /proc/self/cgroup)" +chain="LAI_TEST_$$" + +sudo mkdir "$cgroup" +cleanup() { + echo $$ | sudo tee "$parent_cgroup/cgroup.procs" >/dev/null 2>&1 || true + sudo iptables -D OUTPUT -m cgroup --path "$group" -j "$chain" 2>/dev/null || true + sudo iptables -F "$chain" 2>/dev/null || true + sudo iptables -X "$chain" 2>/dev/null || true + sudo ip6tables -D OUTPUT -m cgroup --path "$group" -j "$chain" 2>/dev/null || true + sudo ip6tables -F "$chain" 2>/dev/null || true + sudo ip6tables -X "$chain" 2>/dev/null || true + sudo rmdir "$cgroup" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +sudo iptables -N "$chain" +sudo iptables -A "$chain" -d 127.0.0.0/8 -j ACCEPT +sudo iptables -A "$chain" -d 10.0.0.0/8 -j ACCEPT +sudo iptables -A "$chain" -d 172.16.0.0/12 -j ACCEPT +sudo iptables -A "$chain" -d 192.168.0.0/16 -j ACCEPT +sudo iptables -A "$chain" -j REJECT +sudo iptables -I OUTPUT 1 -m cgroup --path "$group" -j "$chain" + +sudo ip6tables -N "$chain" +sudo ip6tables -A "$chain" -d ::1/128 -j ACCEPT +sudo ip6tables -A "$chain" -j REJECT +sudo ip6tables -I OUTPUT 1 -m cgroup --path "$group" -j "$chain" +echo $$ | sudo tee "$cgroup/cgroup.procs" >/dev/null + +set +e +LOCALAI_TEST_KERNEL_ACTIVE=1 "$root/scripts/run-test-offline.sh" "$@" +command_status=$? +set -e + +# The final rule in each private chain is the external-egress REJECT. Reading +# its counter makes even an optional probe that swallows its network error a +# hard, visible test failure. Report aggregate counters only: packet-by-packet +# kernel logging can itself exhaust CI logs when a client retries aggressively. +ipv4_blocked=$(sudo iptables -L "$chain" -v -x -n | awk '$3 == "REJECT" { print $1; exit }') +ipv6_blocked=$(sudo ip6tables -L "$chain" -v -x -n | awk '$3 == "REJECT" { print $1; exit }') +ipv4_blocked=${ipv4_blocked:-0} +ipv6_blocked=${ipv6_blocked:-0} + +if (( ipv4_blocked > 0 || ipv6_blocked > 0 )); then + { + echo 'offline-test: external network access was attempted and blocked' + echo "offline-test: blocked packets: ipv4=$ipv4_blocked ipv6=$ipv6_blocked" + echo 'offline-test: replace real network clients with fixtures or injected test doubles' + } >&2 + exit 1 +fi + +exit "$command_status" diff --git a/scripts/run-test-offline.sh b/scripts/run-test-offline.sh new file mode 100755 index 000000000..614ba71be --- /dev/null +++ b/scripts/run-test-offline.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +if [[ $# -lt 2 ]]; then + echo "usage: $0 TARGET COMMAND [ARG...]" >&2 + exit 2 +fi + +target=$1 +shift +root=$(cd "$(dirname "$0")/.." && pwd) + +if [[ ${LOCALAI_TEST_KERNEL_ENFORCE:-0} == 1 && ${LOCALAI_TEST_KERNEL_ACTIVE:-0} != 1 ]]; then + exec "$root/scripts/run-test-linux-offline.sh" "$target" "$@" +fi + +exec go run "$root/cmd/test-resources" run "$target" \ + "$root/test-resources/manifests" "${TEST_RESOURCE_CACHE:-$root/.cache/test-resources}" -- "$@" diff --git a/scripts/summarize-ginkgo-waits.sh b/scripts/summarize-ginkgo-waits.sh new file mode 100755 index 000000000..545e55340 --- /dev/null +++ b/scripts/summarize-ginkgo-waits.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env sh +# summarize-ginkgo-waits.sh THRESHOLD_SECONDS ROOT LOG [LIMIT] +set -eu + +threshold="${1:?missing threshold in seconds}" +root="${2:?missing test root}" +log="${3:?missing Ginkgo log}" +limit="${4:-25}" +case "$limit" in + ''|*[!0-9]*) echo "limit must be a non-negative integer" >&2; exit 2 ;; +esac + +# Strip terminal colour sequences, then report slow specs and hooks. This +# catches sleeps, polling, channel waits, teardown and any other idle time +# without unsafe attempts to replace Go's process-wide clock primitives. +sed 's/\[[0-9;]*[[:alpha:]]//g' "$log" | awk -v threshold="$threshold" -v root="$root" ' + /\[[0-9]+([.][0-9]+)? seconds\]/ { + line = $0 + sub(/^.*\[/, "", line) + sub(/ seconds\].*$/, "", line) + seconds = line + 0 + if (seconds < threshold) next + description = "(description unavailable)" + location = "(location unavailable)" + if (getline > 0) description = $0 + if (getline > 0) location = $0 + printf "%010.3f\t%-18s\t%s\t%s\n", seconds, root, description, location + } +' | sort -t "$(printf '\t')" -k1,1nr | sed -n "1,${limit}p" | awk -F '\t' '{ printf " %7.3fs %s %s %s\n", $1 + 0, $2, $3, $4 }' diff --git a/test-resources/manifests/aio.json b/test-resources/manifests/aio.json new file mode 100644 index 000000000..c21ff9a97 --- /dev/null +++ b/test-resources/manifests/aio.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "target": "aio", + "files": [ + { + "url": "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav", + "sha256": "37de21902b32aa2fc147ccbfdcc0566cc7061fffb2c0b10874f05147c0b9de0f", + "destination": "audio/micro-machines.wav", + "environment": "AIO_AUDIO_FIXTURE" + } + ] +} diff --git a/test-resources/manifests/backend.json b/test-resources/manifests/backend.json new file mode 100644 index 000000000..178b8b9c5 --- /dev/null +++ b/test-resources/manifests/backend.json @@ -0,0 +1 @@ +{"version":1,"target":"backend"} diff --git a/test-resources/manifests/default-darwin.json b/test-resources/manifests/default-darwin.json new file mode 100644 index 000000000..1dc5aed03 --- /dev/null +++ b/test-resources/manifests/default-darwin.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "target": "default-darwin", + "http": [], + "images": [], + "files": [] +} diff --git a/test-resources/manifests/default.json b/test-resources/manifests/default.json new file mode 100644 index 000000000..1acaac816 --- /dev/null +++ b/test-resources/manifests/default.json @@ -0,0 +1,10 @@ +{ + "version": 1, + "target": "default", + "images": [ + { + "reference": "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20", + "sha256": "f4e8a437601f09ad619c6a8df831cd71c24c5c2276652e6ce89c042185759ed9" + } + ] +} diff --git a/test-resources/manifests/distributed-e2e.json b/test-resources/manifests/distributed-e2e.json new file mode 100644 index 000000000..297bf9d6b --- /dev/null +++ b/test-resources/manifests/distributed-e2e.json @@ -0,0 +1,14 @@ +{ + "version": 1, + "target": "distributed-e2e", + "images": [ + { + "reference": "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777", + "sha256": "c8d5971aa1c74f0130dcde5ab942e3613d2179cb62cda4e9c08e8ec4c7252220" + }, + { + "reference": "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0", + "sha256": "9e833c05b393c5ac68a06ab348ce45aac9a7cd72a19cce7ed5e41fb3af56c423" + } + ] +} diff --git a/test-resources/manifests/external-probes.json b/test-resources/manifests/external-probes.json new file mode 100644 index 000000000..af7a85e52 --- /dev/null +++ b/test-resources/manifests/external-probes.json @@ -0,0 +1 @@ +{"version":1,"target":"external-probes"} diff --git a/test-resources/manifests/hardware.json b/test-resources/manifests/hardware.json new file mode 100644 index 000000000..cddf8b0c3 --- /dev/null +++ b/test-resources/manifests/hardware.json @@ -0,0 +1 @@ +{"version":1,"target":"hardware"} diff --git a/test-resources/manifests/lock.json b/test-resources/manifests/lock.json new file mode 100644 index 000000000..623021f9e --- /dev/null +++ b/test-resources/manifests/lock.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "bundles": { + "aio": "sha256:03b05eedb51c853b0f05f2f3f592e2edc210e337388d3ff5a766680c0a166e55", + "backend": "embedded", + "default": "sha256:533e2744151ce8a4df3a9b0ba073222135f17c3e2ee0e5cfeddfb836f41efbca", + "default-darwin": "embedded", + "distributed-e2e": "sha256:524db2b4cedb091c0604eaf6dcd7b0d73fd84e3b262405b6cdb98df099f54965", + "external-probes": "embedded", + "hardware": "embedded" + } +} diff --git a/tests/e2e-aio/e2e_suite_test.go b/tests/e2e-aio/e2e_suite_test.go index f82b7c5e7..321b9b5ab 100644 --- a/tests/e2e-aio/e2e_suite_test.go +++ b/tests/e2e-aio/e2e_suite_test.go @@ -6,14 +6,13 @@ import ( "os" "runtime" "testing" - "time" + "github.com/mudler/LocalAI/internal/testfixtures" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/openai/openai-go/v3" "github.com/openai/openai-go/v3/option" "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" ) var container testcontainers.Container @@ -39,10 +38,10 @@ var _ = BeforeSuite(func() { if apiEndpoint == "" { startDockerImage() - apiPort, err := container.MappedPort(context.Background(), defaultApiPort) + apiAddress, err := testfixtures.ContainerEndpoint(context.Background(), container, defaultApiPort) Expect(err).To(Not(HaveOccurred())) - apiEndpoint = "http://localhost:" + apiPort.Port() + "/v1" // So that other tests can reference this value safely. + apiEndpoint = "http://" + apiAddress + "/v1" } else { GinkgoWriter.Printf("docker apiEndpoint set from env: %q\n", apiEndpoint) } @@ -122,15 +121,16 @@ func startDockerImage() { Target: "/backends", }, }, - WaitingFor: wait.ForAll( - wait.ForListeningPort(defaultApiPort).WithStartupTimeout(10*time.Minute), - wait.ForHTTP("/v1/models").WithPort(defaultApiPort).WithStartupTimeout(10*time.Minute), - ), } GinkgoWriter.Printf("Launching Docker Container %s:%s\n", containerImage, containerImageTag) ctx := context.Background() + imageReference := fmt.Sprintf("%s:%s", containerImage, containerImageTag) + Expect(testfixtures.RequireImage(ctx, imageReference, "aio")).To(Succeed()) + testNetwork, err := testfixtures.DockerNetwork() + Expect(err).NotTo(HaveOccurred()) + req.Networks = []string{testNetwork} c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, Started: true, diff --git a/tests/e2e-aio/e2e_test.go b/tests/e2e-aio/e2e_test.go index 6472b5d63..338288593 100644 --- a/tests/e2e-aio/e2e_test.go +++ b/tests/e2e-aio/e2e_test.go @@ -3,11 +3,13 @@ package e2e_test import ( "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "io" "net/http" "os" + "path/filepath" "github.com/mudler/LocalAI/core/schema" . "github.com/onsi/ginkgo/v2" @@ -257,6 +259,9 @@ var _ = Describe("E2E test", func() { Context("vision", func() { It("correctly", func() { + image, err := os.ReadFile(filepath.Join("..", "..", "core", "http", "static", "logo.png")) + Expect(err).NotTo(HaveOccurred()) + imageURI := "data:image/png;base64," + base64.StdEncoding.EncodeToString(image) model := "gpt-4o" resp, err := client.Chat.Completions.New(context.TODO(), openai.ChatCompletionNewParams{ @@ -276,7 +281,7 @@ var _ = Describe("E2E test", func() { { OfImageURL: &openai.ChatCompletionContentPartImageParam{ ImageURL: openai.ChatCompletionContentPartImageImageURLParam{ - URL: "https://picsum.photos/id/22/4434/3729", + URL: imageURI, Detail: "low", }, }, @@ -289,7 +294,7 @@ var _ = Describe("E2E test", func() { }) Expect(err).ToNot(HaveOccurred()) Expect(len(resp.Choices)).To(Equal(1), fmt.Sprint(resp)) - Expect(resp.Choices[0].Message.Content).To(Or(ContainSubstring("man"), ContainSubstring("road")), fmt.Sprint(resp.Choices[0].Message.Content)) + Expect(resp.Choices[0].Message.Content).NotTo(BeEmpty()) }) }) @@ -310,11 +315,7 @@ var _ = Describe("E2E test", func() { Context("audio to text", func() { It("correctly", func() { - downloadURL := "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav" - file, err := downloadHttpFile(downloadURL) - Expect(err).ToNot(HaveOccurred()) - - fileHandle, err := os.Open(file) + fileHandle, err := os.Open(preparedAudioFixture()) Expect(err).ToNot(HaveOccurred()) defer fileHandle.Close() @@ -328,11 +329,7 @@ var _ = Describe("E2E test", func() { }) It("with VTT format", func() { - downloadURL := "https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav" - file, err := downloadHttpFile(downloadURL) - Expect(err).ToNot(HaveOccurred()) - - fileHandle, err := os.Open(file) + fileHandle, err := os.Open(preparedAudioFixture()) Expect(err).ToNot(HaveOccurred()) defer fileHandle.Close() @@ -443,25 +440,10 @@ var _ = Describe("E2E test", func() { }) }) -func downloadHttpFile(url string) (string, error) { - resp, err := http.Get(url) - if err != nil { - return "", err - } - defer resp.Body.Close() - - tmpfile, err := os.CreateTemp("", "example") - if err != nil { - return "", err - } - defer tmpfile.Close() - - _, err = io.Copy(tmpfile, resp.Body) - if err != nil { - return "", err - } - - return tmpfile.Name(), nil +func preparedAudioFixture() string { + path := os.Getenv("AIO_AUDIO_FIXTURE") + Expect(path).NotTo(BeEmpty(), "run `make prepare-offline-test-cache TEST_RESOURCE_SET=aio` before the AIO suite") + return path } func requestRerank(modelName, query string, documents []string, topN *int, apiEndpoint string) (*http.Response, []byte) { diff --git a/tests/e2e/distributed/nats_jwt_helpers_test.go b/tests/e2e/distributed/nats_jwt_helpers_test.go index 80060ef6a..e0723b378 100644 --- a/tests/e2e/distributed/nats_jwt_helpers_test.go +++ b/tests/e2e/distributed/nats_jwt_helpers_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/internal/testfixtures" "github.com/mudler/LocalAI/pkg/natsauth" "github.com/nats-io/jwt/v2" "github.com/nats-io/nkeys" @@ -17,6 +18,8 @@ import ( "github.com/testcontainers/testcontainers-go" tcnats "github.com/testcontainers/testcontainers-go/modules/nats" + tcnetwork "github.com/testcontainers/testcontainers-go/network" + "github.com/testcontainers/testcontainers-go/wait" ) // JWTTestInfra holds a NATS server configured with JWT auth and minted worker credentials. @@ -34,6 +37,9 @@ func SetupJWTInfra() *JWTTestInfra { GinkgoHelper() infra := &JWTTestInfra{TestInfra: &TestInfra{Ctx: context.Background()}} + Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed()) + testNetwork, err := testfixtures.DockerNetwork() + Expect(err).NotTo(HaveOccurred()) operatorJWT, accountJWT, accountSeed, err := jwtResolverMaterial() Expect(err).ToNot(HaveOccurred()) @@ -51,15 +57,18 @@ resolver_preload: { var natsContainer *tcnats.NATSContainer // Override default testcontainers -js: JetStream fails without a system account in JWT mode. - natsContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine", + natsContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine, tcnats.WithConfigFile(bytes.NewBufferString(conf)), testcontainers.WithCmd("-c", "/etc/nats.conf"), + tcnetwork.WithNetworkName([]string{"nats"}, testNetwork), + testcontainers.WithWaitStrategy(wait.ForLog("Server is ready")), ) Expect(err).ToNot(HaveOccurred()) infra.NATSContainer = natsContainer - infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) + natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222") Expect(err).ToNot(HaveOccurred()) + infra.NatsURL = "nats://" + natsEndpoint infra.NodeID = "550e8400-e29b-41d4-a716-446655440000" cfg := natsauth.Config{AccountSeed: infra.AccountSeed, WorkerJWTTTL: time.Hour} @@ -153,4 +162,4 @@ func accountPublicKeyFromSeed(accountSeed string) string { func nodeSubjectPrefix(nodeID string) string { tok := strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-").Replace(nodeID) return "nodes." + tok -} \ No newline at end of file +} diff --git a/tests/e2e/distributed/testhelpers_test.go b/tests/e2e/distributed/testhelpers_test.go index 68cf537e3..d54c0897a 100644 --- a/tests/e2e/distributed/testhelpers_test.go +++ b/tests/e2e/distributed/testhelpers_test.go @@ -2,9 +2,11 @@ package distributed_test import ( "context" + "fmt" "time" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/internal/testfixtures" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -12,6 +14,7 @@ import ( "github.com/testcontainers/testcontainers-go" tcnats "github.com/testcontainers/testcontainers-go/modules/nats" tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + tcnetwork "github.com/testcontainers/testcontainers-go/network" "github.com/testcontainers/testcontainers-go/wait" ) @@ -32,9 +35,13 @@ func SetupInfra(dbName string) *TestInfra { infra := &TestInfra{Ctx: context.Background()} var err error + Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.Postgres16Alpine, "distributed-e2e")).To(Succeed()) + Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed()) + testNetwork, err := testfixtures.DockerNetwork() + Expect(err).NotTo(HaveOccurred()) // Start PostgreSQL container - infra.PGContainer, err = tcpostgres.Run(infra.Ctx, "postgres:16-alpine", + infra.PGContainer, err = tcpostgres.Run(infra.Ctx, testfixtures.Postgres16Alpine, tcpostgres.WithDatabase(dbName), tcpostgres.WithUsername("test"), tcpostgres.WithPassword("test"), @@ -43,18 +50,23 @@ func SetupInfra(dbName string) *TestInfra { WithOccurrence(2). WithStartupTimeout(30*time.Second), ), + tcnetwork.WithNetworkName([]string{"postgres"}, testNetwork), ) Expect(err).ToNot(HaveOccurred()) - infra.PGURL, err = infra.PGContainer.ConnectionString(infra.Ctx, "sslmode=disable") + pgEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.PGContainer, "5432") Expect(err).ToNot(HaveOccurred()) + infra.PGURL = fmt.Sprintf("postgres://test:test@%s/%s?sslmode=disable", pgEndpoint, dbName) // Start NATS container - infra.NATSContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine") + infra.NATSContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine, + tcnetwork.WithNetworkName([]string{"nats"}, testNetwork), + testcontainers.WithWaitStrategy(wait.ForLog("Server is ready"))) Expect(err).ToNot(HaveOccurred()) - infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) + natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222") Expect(err).ToNot(HaveOccurred()) + infra.NatsURL = "nats://" + natsEndpoint // Connect messaging client infra.NC, err = messaging.New(infra.NatsURL) @@ -83,12 +95,18 @@ func SetupNATSOnly() *TestInfra { infra := &TestInfra{Ctx: context.Background()} var err error + Expect(testfixtures.RequireImage(infra.Ctx, testfixtures.NATS2Alpine, "distributed-e2e")).To(Succeed()) + testNetwork, err := testfixtures.DockerNetwork() + Expect(err).NotTo(HaveOccurred()) - infra.NATSContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine") + infra.NATSContainer, err = tcnats.Run(infra.Ctx, testfixtures.NATS2Alpine, + tcnetwork.WithNetworkName([]string{"nats"}, testNetwork), + testcontainers.WithWaitStrategy(wait.ForLog("Server is ready"))) Expect(err).ToNot(HaveOccurred()) - infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) + natsEndpoint, err := testfixtures.ContainerEndpoint(infra.Ctx, infra.NATSContainer, "4222") Expect(err).ToNot(HaveOccurred()) + infra.NatsURL = "nats://" + natsEndpoint infra.NC, err = messaging.New(infra.NatsURL) Expect(err).ToNot(HaveOccurred())