diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index eef0a1a1c..0923df113 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -52,9 +52,12 @@ jobs: node-version: '22' - name: Build React UI run: make react-ui + - name: Record declared distributed test resources + run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=distributed-e2e - name: Test Backend E2E run: | PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e + PATH="$PATH:$HOME/go/bin" make test-e2e-distributed - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.23 diff --git a/Makefile b/Makefile index 99074f4df..793188ed8 100644 --- a/Makefile +++ b/Makefile @@ -396,7 +396,7 @@ test-e2e: build-mock-backend build-cloud-proxy-backend prepare-e2e run-e2e-image @echo 'Running e2e tests' BUILD_TYPE=$(BUILD_TYPE) \ LOCALAI_API=http://$(E2E_BRIDGE_IP):5390 \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='!Distributed' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e $(MAKE) clean-mock-backend $(MAKE) clean-cloud-proxy-backend $(MAKE) teardown-e2e diff --git a/cmd/test-resources/main.go b/cmd/test-resources/main.go index 133d3cc27..7e5eaad3b 100644 --- a/cmd/test-resources/main.go +++ b/cmd/test-resources/main.go @@ -3,6 +3,7 @@ package main import ( + "archive/tar" "crypto/sha256" "errors" "fmt" @@ -150,16 +151,21 @@ func update(target, manifestDir, cacheDir string) error { return err } } - for _, resource := range manifest.Images { - if err := pullAndPack(resource.Reference, resource.SHA256, cacheDir); err != nil { + for i := range manifest.Images { + digest, err := pullAndPack(manifest.Images[i].Reference, cacheDir) + if err != nil { return err } + manifest.Images[i].SHA256 = digest } bundlePath := filepath.Join(cacheDir, "bundles", target+".tar.zst") digest, err := testresources.PackBundle(cacheDir, bundlePath, manifest) if err != nil { return err } + if err := testresources.WriteManifest(filepath.Join(manifestDir, target+".json"), manifest); err != nil { + return err + } lockPath := filepath.Join(manifestDir, "lock.json") lock, err := testresources.LoadLock(lockPath) if err != nil { @@ -272,24 +278,93 @@ func storeVerified(reader io.Reader, expected, cacheDir string) (int64, error) { return size, nil } -func pullAndPack(reference, expected, cacheDir string) error { +func pullAndPack(reference, cacheDir string) (string, error) { if !strings.Contains(reference, "@sha256:") { - return fmt.Errorf("refusing mutable image reference %s", reference) + return "", fmt.Errorf("refusing mutable image reference %s", reference) } if err := exec.Command("docker", "pull", reference).Run(); err != nil { - return fmt.Errorf("pull image %s: %w", reference, err) + return "", fmt.Errorf("pull image %s: %w", reference, err) } cmd := exec.Command("docker", "save", reference) stdout, err := cmd.StdoutPipe() if err != nil { - return err + return "", err } if err := cmd.Start(); err != nil { - return err + return "", err } - _, storeErr := storeVerified(stdout, expected, cacheDir) + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return "", err + } + normalized, err := os.CreateTemp(cacheDir, ".docker-save-*.tar") + if err != nil { + return "", err + } + normalizedName := normalized.Name() + defer func() { _ = os.Remove(normalizedName) }() + normalizeErr := normalizeDockerArchive(stdout, normalized) waitErr := cmd.Wait() - return errors.Join(storeErr, waitErr) + closeErr := normalized.Close() + if err := errors.Join(normalizeErr, waitErr, closeErr); err != nil { + return "", err + } + input, err := os.Open(normalizedName) + if err != nil { + return "", err + } + digest, _, storeErr := storeContentAddressed(input, cacheDir) + return digest, errors.Join(storeErr, input.Close()) +} + +func normalizeDockerArchive(reader io.Reader, writer io.Writer) error { + tr := tar.NewReader(reader) + tw := tar.NewWriter(writer) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + stable := *header + stable.Uid, stable.Gid = 0, 0 + stable.Uname, stable.Gname = "", "" + stable.ModTime = time.Unix(0, 0).UTC() + stable.AccessTime, stable.ChangeTime = time.Time{}, time.Time{} + stable.PAXRecords, stable.Xattrs = nil, nil + if err := tw.WriteHeader(&stable); err != nil { + return err + } + if _, err := io.Copy(tw, tr); err != nil { + return err + } + } + return tw.Close() +} + +func storeContentAddressed(reader io.Reader, cacheDir string) (string, int64, error) { + directory := filepath.Join(cacheDir, "blobs", "sha256") + if err := os.MkdirAll(directory, 0o755); err != nil { + return "", 0, err + } + temporary, err := os.CreateTemp(directory, ".record-*") + if err != nil { + return "", 0, err + } + name := temporary.Name() + defer func() { _ = os.Remove(name) }() + hash := sha256.New() + size, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader) + closeErr := temporary.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return "", 0, err + } + digest := fmt.Sprintf("%x", hash.Sum(nil)) + if err := os.Rename(name, testresources.BlobPath(cacheDir, digest)); err != nil { + return "", 0, err + } + return digest, size, nil } func preparationError(target string, err error) error { diff --git a/cmd/test-resources/main_test.go b/cmd/test-resources/main_test.go new file mode 100644 index 000000000..8783bd20e --- /dev/null +++ b/cmd/test-resources/main_test.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT + +package main + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "fmt" + "io" + "testing" + "time" + + "github.com/onsi/gomega" +) + +func TestNormalizeDockerArchiveIgnoresTarMetadata(t *testing.T) { + g := gomega.NewWithT(t) + first := dockerArchive(g, time.Unix(100, 0), 12, "builder") + second := dockerArchive(g, time.Unix(200, 0), 34, "runner") + + var normalizedFirst, normalizedSecond bytes.Buffer + g.Expect(normalizeDockerArchive(bytes.NewReader(first), &normalizedFirst)).To(gomega.Succeed()) + g.Expect(normalizeDockerArchive(bytes.NewReader(second), &normalizedSecond)).To(gomega.Succeed()) + firstDigest := fmt.Sprintf("%x", sha256.Sum256(normalizedFirst.Bytes())) + secondDigest := fmt.Sprintf("%x", sha256.Sum256(normalizedSecond.Bytes())) + g.Expect(secondDigest).To(gomega.Equal(firstDigest)) + + tr := tar.NewReader(bytes.NewReader(normalizedFirst.Bytes())) + header, err := tr.Next() + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(header.Uid).To(gomega.Equal(0)) + g.Expect(header.Gid).To(gomega.Equal(0)) + g.Expect(header.Uname).To(gomega.BeEmpty()) + g.Expect(header.Gname).To(gomega.BeEmpty()) + g.Expect(header.ModTime).To(gomega.Equal(time.Unix(0, 0))) + content, err := io.ReadAll(tr) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(string(content)).To(gomega.Equal("image data")) +} + +func dockerArchive(g *gomega.WithT, modTime time.Time, uid int, user string) []byte { + var archive bytes.Buffer + tw := tar.NewWriter(&archive) + content := []byte("image data") + g.Expect(tw.WriteHeader(&tar.Header{ + Name: "layer.tar", Mode: 0o644, Size: int64(len(content)), + ModTime: modTime, Uid: uid, Gid: uid, Uname: user, Gname: user, + })).To(gomega.Succeed()) + _, err := tw.Write(content) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(tw.Close()).To(gomega.Succeed()) + return archive.Bytes() +} diff --git a/internal/testresources/resources.go b/internal/testresources/resources.go index a5c5afd8a..f46ff3ca2 100644 --- a/internal/testresources/resources.go +++ b/internal/testresources/resources.go @@ -81,7 +81,15 @@ func LoadLock(path string) (Lock, error) { } func WriteLock(path string, lock Lock) error { - data, err := json.MarshalIndent(lock, "", " ") + return writeJSON(path, lock) +} + +func WriteManifest(path string, manifest Manifest) error { + return writeJSON(path, manifest) +} + +func writeJSON(path string, value any) error { + data, err := json.MarshalIndent(value, "", " ") if err != nil { return err } diff --git a/scripts/test-network-lint.sh b/scripts/test-network-lint.sh index e081f7993..4ec828d9f 100755 --- a/scripts/test-network-lint.sh +++ b/scripts/test-network-lint.sh @@ -6,13 +6,29 @@ set -euo pipefail # a worktree-only diff would always be empty). Most existing direct clients are # loopback fixtures; changing the inventory requires an intentional baseline # update after review. -expected_inventory=2885a428cdab55eea357dae3ec47b3d44f9999b59542d06cd3d792cf491c76b3 +expected_inventory=6b8a611b9d01ea1b58f446ccdd92efdbb01c34663ee5946194ccc0f00e8e877d +search_test_files() { + local pattern=$1 + shift + while IFS= read -r -d '' file; do + [[ $file == *_test.go ]] && grep -hE "$pattern" "$file" || true + done < <(git ls-files -co --exclude-standard -z -- "$@") +} + +search_shell_files() { + local pattern=$1 + shift + while IFS= read -r -d '' file; do + [[ $file == *.sh ]] && grep -hE "$pattern" "$file" || true + done < <(git ls-files -co --exclude-standard -z -- "$@") +} + inventory=$( { - rg --no-heading --no-line-number --glob '*_test.go' \ + search_test_files \ '(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)")' \ - pkg core tests backend || true - rg --no-heading --no-line-number --glob '*.sh' '(curl|wget)[[:space:]]' tests backend || true + pkg core tests backend + search_shell_files '(curl|wget)[[:space:]]' tests backend } | LC_ALL=C sort ) if command -v sha256sum >/dev/null 2>&1; then @@ -21,7 +37,7 @@ else actual_inventory=$(printf '%s\n' "$inventory" | shasum -a 256 | awk '{print $1}') fi if [[ $actual_inventory != "$expected_inventory" ]]; then - echo 'Test network mechanism inventory changed; remove the direct access or review and update the lint baseline:' >&2 + echo "Test network mechanism inventory changed (expected $expected_inventory, got $actual_inventory); remove the direct access or review and update the lint baseline:" >&2 echo "$inventory" >&2 exit 1 fi @@ -30,8 +46,8 @@ fi # literals and direct mechanisms instead of only reporting the fingerprint. base=${TEST_NETWORK_LINT_BASE:-HEAD} violations=$(git diff --unified=0 "$base" -- api pkg core tests backend | \ - rg '^\+[^+].*(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)"|https?://)' | \ - rg -v 'test-network: fixture' || true) + grep -E '^\+[^+].*(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)"|https?://)' | \ + grep -Ev 'test-network: fixture' || true) if [[ -n "$violations" ]]; then echo 'Direct test network access is forbidden; use a fixture or guarded transport:' >&2 echo "$violations" >&2 diff --git a/test-resources/manifests/default.json b/test-resources/manifests/default.json index b4bf99004..1acaac816 100644 --- a/test-resources/manifests/default.json +++ b/test-resources/manifests/default.json @@ -4,7 +4,7 @@ "images": [ { "reference": "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20", - "sha256": "bd98262690143a2a05167b3e924cde07e82ea269e45d5610e00a45f50e76d45a" + "sha256": "f4e8a437601f09ad619c6a8df831cd71c24c5c2276652e6ce89c042185759ed9" } ] } diff --git a/test-resources/manifests/distributed-e2e.json b/test-resources/manifests/distributed-e2e.json index b9c353d9c..297bf9d6b 100644 --- a/test-resources/manifests/distributed-e2e.json +++ b/test-resources/manifests/distributed-e2e.json @@ -4,11 +4,11 @@ "images": [ { "reference": "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777", - "sha256": "fee330cbd34786da2b211fe2e4b7424d7bf974466cd3db0e376b2e4c457a339c" + "sha256": "c8d5971aa1c74f0130dcde5ab942e3613d2179cb62cda4e9c08e8ec4c7252220" }, { "reference": "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0", - "sha256": "96e0f53430696eadaf1d7e250914ab78779cadc40df715a168ebe342e039b6f2" + "sha256": "9e833c05b393c5ac68a06ab348ce45aac9a7cd72a19cce7ed5e41fb3af56c423" } ] } diff --git a/test-resources/manifests/lock.json b/test-resources/manifests/lock.json index e68826e0a..623021f9e 100644 --- a/test-resources/manifests/lock.json +++ b/test-resources/manifests/lock.json @@ -3,9 +3,9 @@ "bundles": { "aio": "sha256:03b05eedb51c853b0f05f2f3f592e2edc210e337388d3ff5a766680c0a166e55", "backend": "embedded", - "default": "sha256:185ebd3cfb994b9c1d1d1d9fad0a5d95c9de698b45e010276db388a9d66474bd", + "default": "sha256:533e2744151ce8a4df3a9b0ba073222135f17c3e2ee0e5cfeddfb836f41efbca", "default-darwin": "embedded", - "distributed-e2e": "sha256:797dad68952914bf6612a42486086aa45eb17112067bec9955e2f5cf65a030dc", + "distributed-e2e": "sha256:524db2b4cedb091c0604eaf6dcd7b0d73fd84e3b262405b6cdb98df099f54965", "external-probes": "embedded", "hardware": "embedded" }