From 0c75d13d98c49eaf730280d568480f0b03ba3dba Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Thu, 23 Jul 2026 09:40:17 +0100 Subject: [PATCH] test: fix offline resource CI portability Normalize Docker archive metadata before content addressing, derive archive checksums during explicit refreshes, make network lint portable to macOS, and prepare distributed images before running their offline suite. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe --- .github/workflows/tests-e2e.yml | 3 + Makefile | 2 +- cmd/test-resources/main.go | 93 +++++++++++++++++-- cmd/test-resources/main_test.go | 54 +++++++++++ internal/testresources/resources.go | 10 +- scripts/test-network-lint.sh | 30 ++++-- test-resources/manifests/default.json | 2 +- test-resources/manifests/distributed-e2e.json | 4 +- test-resources/manifests/lock.json | 4 +- 9 files changed, 179 insertions(+), 23 deletions(-) create mode 100644 cmd/test-resources/main_test.go diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index eef0a1a1c..0923df113 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -52,9 +52,12 @@ jobs: node-version: '22' - name: Build React UI run: make react-ui + - name: Record declared distributed test resources + run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=distributed-e2e - name: Test Backend E2E run: | PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e + PATH="$PATH:$HOME/go/bin" make test-e2e-distributed - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.23 diff --git a/Makefile b/Makefile index 99074f4df..793188ed8 100644 --- a/Makefile +++ b/Makefile @@ -396,7 +396,7 @@ test-e2e: build-mock-backend build-cloud-proxy-backend prepare-e2e run-e2e-image @echo 'Running e2e tests' BUILD_TYPE=$(BUILD_TYPE) \ LOCALAI_API=http://$(E2E_BRIDGE_IP):5390 \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='!Distributed' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e $(MAKE) clean-mock-backend $(MAKE) clean-cloud-proxy-backend $(MAKE) teardown-e2e diff --git a/cmd/test-resources/main.go b/cmd/test-resources/main.go index 133d3cc27..7e5eaad3b 100644 --- a/cmd/test-resources/main.go +++ b/cmd/test-resources/main.go @@ -3,6 +3,7 @@ package main import ( + "archive/tar" "crypto/sha256" "errors" "fmt" @@ -150,16 +151,21 @@ func update(target, manifestDir, cacheDir string) error { return err } } - for _, resource := range manifest.Images { - if err := pullAndPack(resource.Reference, resource.SHA256, cacheDir); err != nil { + for i := range manifest.Images { + digest, err := pullAndPack(manifest.Images[i].Reference, cacheDir) + if err != nil { return err } + manifest.Images[i].SHA256 = digest } bundlePath := filepath.Join(cacheDir, "bundles", target+".tar.zst") digest, err := testresources.PackBundle(cacheDir, bundlePath, manifest) if err != nil { return err } + if err := testresources.WriteManifest(filepath.Join(manifestDir, target+".json"), manifest); err != nil { + return err + } lockPath := filepath.Join(manifestDir, "lock.json") lock, err := testresources.LoadLock(lockPath) if err != nil { @@ -272,24 +278,93 @@ func storeVerified(reader io.Reader, expected, cacheDir string) (int64, error) { return size, nil } -func pullAndPack(reference, expected, cacheDir string) error { +func pullAndPack(reference, cacheDir string) (string, error) { if !strings.Contains(reference, "@sha256:") { - return fmt.Errorf("refusing mutable image reference %s", reference) + return "", fmt.Errorf("refusing mutable image reference %s", reference) } if err := exec.Command("docker", "pull", reference).Run(); err != nil { - return fmt.Errorf("pull image %s: %w", reference, err) + return "", fmt.Errorf("pull image %s: %w", reference, err) } cmd := exec.Command("docker", "save", reference) stdout, err := cmd.StdoutPipe() if err != nil { - return err + return "", err } if err := cmd.Start(); err != nil { - return err + return "", err } - _, storeErr := storeVerified(stdout, expected, cacheDir) + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return "", err + } + normalized, err := os.CreateTemp(cacheDir, ".docker-save-*.tar") + if err != nil { + return "", err + } + normalizedName := normalized.Name() + defer func() { _ = os.Remove(normalizedName) }() + normalizeErr := normalizeDockerArchive(stdout, normalized) waitErr := cmd.Wait() - return errors.Join(storeErr, waitErr) + closeErr := normalized.Close() + if err := errors.Join(normalizeErr, waitErr, closeErr); err != nil { + return "", err + } + input, err := os.Open(normalizedName) + if err != nil { + return "", err + } + digest, _, storeErr := storeContentAddressed(input, cacheDir) + return digest, errors.Join(storeErr, input.Close()) +} + +func normalizeDockerArchive(reader io.Reader, writer io.Writer) error { + tr := tar.NewReader(reader) + tw := tar.NewWriter(writer) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + stable := *header + stable.Uid, stable.Gid = 0, 0 + stable.Uname, stable.Gname = "", "" + stable.ModTime = time.Unix(0, 0).UTC() + stable.AccessTime, stable.ChangeTime = time.Time{}, time.Time{} + stable.PAXRecords, stable.Xattrs = nil, nil + if err := tw.WriteHeader(&stable); err != nil { + return err + } + if _, err := io.Copy(tw, tr); err != nil { + return err + } + } + return tw.Close() +} + +func storeContentAddressed(reader io.Reader, cacheDir string) (string, int64, error) { + directory := filepath.Join(cacheDir, "blobs", "sha256") + if err := os.MkdirAll(directory, 0o755); err != nil { + return "", 0, err + } + temporary, err := os.CreateTemp(directory, ".record-*") + if err != nil { + return "", 0, err + } + name := temporary.Name() + defer func() { _ = os.Remove(name) }() + hash := sha256.New() + size, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader) + closeErr := temporary.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return "", 0, err + } + digest := fmt.Sprintf("%x", hash.Sum(nil)) + if err := os.Rename(name, testresources.BlobPath(cacheDir, digest)); err != nil { + return "", 0, err + } + return digest, size, nil } func preparationError(target string, err error) error { diff --git a/cmd/test-resources/main_test.go b/cmd/test-resources/main_test.go new file mode 100644 index 000000000..8783bd20e --- /dev/null +++ b/cmd/test-resources/main_test.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT + +package main + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "fmt" + "io" + "testing" + "time" + + "github.com/onsi/gomega" +) + +func TestNormalizeDockerArchiveIgnoresTarMetadata(t *testing.T) { + g := gomega.NewWithT(t) + first := dockerArchive(g, time.Unix(100, 0), 12, "builder") + second := dockerArchive(g, time.Unix(200, 0), 34, "runner") + + var normalizedFirst, normalizedSecond bytes.Buffer + g.Expect(normalizeDockerArchive(bytes.NewReader(first), &normalizedFirst)).To(gomega.Succeed()) + g.Expect(normalizeDockerArchive(bytes.NewReader(second), &normalizedSecond)).To(gomega.Succeed()) + firstDigest := fmt.Sprintf("%x", sha256.Sum256(normalizedFirst.Bytes())) + secondDigest := fmt.Sprintf("%x", sha256.Sum256(normalizedSecond.Bytes())) + g.Expect(secondDigest).To(gomega.Equal(firstDigest)) + + tr := tar.NewReader(bytes.NewReader(normalizedFirst.Bytes())) + header, err := tr.Next() + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(header.Uid).To(gomega.Equal(0)) + g.Expect(header.Gid).To(gomega.Equal(0)) + g.Expect(header.Uname).To(gomega.BeEmpty()) + g.Expect(header.Gname).To(gomega.BeEmpty()) + g.Expect(header.ModTime).To(gomega.Equal(time.Unix(0, 0))) + content, err := io.ReadAll(tr) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(string(content)).To(gomega.Equal("image data")) +} + +func dockerArchive(g *gomega.WithT, modTime time.Time, uid int, user string) []byte { + var archive bytes.Buffer + tw := tar.NewWriter(&archive) + content := []byte("image data") + g.Expect(tw.WriteHeader(&tar.Header{ + Name: "layer.tar", Mode: 0o644, Size: int64(len(content)), + ModTime: modTime, Uid: uid, Gid: uid, Uname: user, Gname: user, + })).To(gomega.Succeed()) + _, err := tw.Write(content) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(tw.Close()).To(gomega.Succeed()) + return archive.Bytes() +} diff --git a/internal/testresources/resources.go b/internal/testresources/resources.go index a5c5afd8a..f46ff3ca2 100644 --- a/internal/testresources/resources.go +++ b/internal/testresources/resources.go @@ -81,7 +81,15 @@ func LoadLock(path string) (Lock, error) { } func WriteLock(path string, lock Lock) error { - data, err := json.MarshalIndent(lock, "", " ") + return writeJSON(path, lock) +} + +func WriteManifest(path string, manifest Manifest) error { + return writeJSON(path, manifest) +} + +func writeJSON(path string, value any) error { + data, err := json.MarshalIndent(value, "", " ") if err != nil { return err } diff --git a/scripts/test-network-lint.sh b/scripts/test-network-lint.sh index e081f7993..4ec828d9f 100755 --- a/scripts/test-network-lint.sh +++ b/scripts/test-network-lint.sh @@ -6,13 +6,29 @@ set -euo pipefail # a worktree-only diff would always be empty). Most existing direct clients are # loopback fixtures; changing the inventory requires an intentional baseline # update after review. -expected_inventory=2885a428cdab55eea357dae3ec47b3d44f9999b59542d06cd3d792cf491c76b3 +expected_inventory=6b8a611b9d01ea1b58f446ccdd92efdbb01c34663ee5946194ccc0f00e8e877d +search_test_files() { + local pattern=$1 + shift + while IFS= read -r -d '' file; do + [[ $file == *_test.go ]] && grep -hE "$pattern" "$file" || true + done < <(git ls-files -co --exclude-standard -z -- "$@") +} + +search_shell_files() { + local pattern=$1 + shift + while IFS= read -r -d '' file; do + [[ $file == *.sh ]] && grep -hE "$pattern" "$file" || true + done < <(git ls-files -co --exclude-standard -z -- "$@") +} + inventory=$( { - rg --no-heading --no-line-number --glob '*_test.go' \ + search_test_files \ '(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)")' \ - pkg core tests backend || true - rg --no-heading --no-line-number --glob '*.sh' '(curl|wget)[[:space:]]' tests backend || true + pkg core tests backend + search_shell_files '(curl|wget)[[:space:]]' tests backend } | LC_ALL=C sort ) if command -v sha256sum >/dev/null 2>&1; then @@ -21,7 +37,7 @@ else actual_inventory=$(printf '%s\n' "$inventory" | shasum -a 256 | awk '{print $1}') fi if [[ $actual_inventory != "$expected_inventory" ]]; then - echo 'Test network mechanism inventory changed; remove the direct access or review and update the lint baseline:' >&2 + echo "Test network mechanism inventory changed (expected $expected_inventory, got $actual_inventory); remove the direct access or review and update the lint baseline:" >&2 echo "$inventory" >&2 exit 1 fi @@ -30,8 +46,8 @@ fi # literals and direct mechanisms instead of only reporting the fingerprint. base=${TEST_NETWORK_LINT_BASE:-HEAD} violations=$(git diff --unified=0 "$base" -- api pkg core tests backend | \ - rg '^\+[^+].*(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)"|https?://)' | \ - rg -v 'test-network: fixture' || true) + grep -E '^\+[^+].*(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)"|https?://)' | \ + grep -Ev 'test-network: fixture' || true) if [[ -n "$violations" ]]; then echo 'Direct test network access is forbidden; use a fixture or guarded transport:' >&2 echo "$violations" >&2 diff --git a/test-resources/manifests/default.json b/test-resources/manifests/default.json index b4bf99004..1acaac816 100644 --- a/test-resources/manifests/default.json +++ b/test-resources/manifests/default.json @@ -4,7 +4,7 @@ "images": [ { "reference": "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20", - "sha256": "bd98262690143a2a05167b3e924cde07e82ea269e45d5610e00a45f50e76d45a" + "sha256": "f4e8a437601f09ad619c6a8df831cd71c24c5c2276652e6ce89c042185759ed9" } ] } diff --git a/test-resources/manifests/distributed-e2e.json b/test-resources/manifests/distributed-e2e.json index b9c353d9c..297bf9d6b 100644 --- a/test-resources/manifests/distributed-e2e.json +++ b/test-resources/manifests/distributed-e2e.json @@ -4,11 +4,11 @@ "images": [ { "reference": "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777", - "sha256": "fee330cbd34786da2b211fe2e4b7424d7bf974466cd3db0e376b2e4c457a339c" + "sha256": "c8d5971aa1c74f0130dcde5ab942e3613d2179cb62cda4e9c08e8ec4c7252220" }, { "reference": "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0", - "sha256": "96e0f53430696eadaf1d7e250914ab78779cadc40df715a168ebe342e039b6f2" + "sha256": "9e833c05b393c5ac68a06ab348ce45aac9a7cd72a19cce7ed5e41fb3af56c423" } ] } diff --git a/test-resources/manifests/lock.json b/test-resources/manifests/lock.json index e68826e0a..623021f9e 100644 --- a/test-resources/manifests/lock.json +++ b/test-resources/manifests/lock.json @@ -3,9 +3,9 @@ "bundles": { "aio": "sha256:03b05eedb51c853b0f05f2f3f592e2edc210e337388d3ff5a766680c0a166e55", "backend": "embedded", - "default": "sha256:185ebd3cfb994b9c1d1d1d9fad0a5d95c9de698b45e010276db388a9d66474bd", + "default": "sha256:533e2744151ce8a4df3a9b0ba073222135f17c3e2ee0e5cfeddfb836f41efbca", "default-darwin": "embedded", - "distributed-e2e": "sha256:797dad68952914bf6612a42486086aa45eb17112067bec9955e2f5cf65a030dc", + "distributed-e2e": "sha256:524db2b4cedb091c0604eaf6dcd7b0d73fd84e3b262405b6cdb98df099f54965", "external-probes": "embedded", "hardware": "embedded" }