test: fix offline resource CI portability

Normalize Docker archive metadata before content addressing, derive archive checksums during explicit refreshes, make network lint portable to macOS, and prepare distributed images before running their offline suite.

Assisted-by: Codex:gpt-5
Signed-off-by: Richard Palethorpe <io@richiejp.com>

.github/workflows/tests-e2e.yml

@@ -52,9 +52,12 @@ jobs:
           node-version: '22'
       - name: Build React UI
         run: make react-ui
+      - name: Record declared distributed test resources
+        run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=distributed-e2e
       - name: Test Backend E2E
         run: |
           PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e
+          PATH="$PATH:$HOME/go/bin" make test-e2e-distributed
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3.23

Makefile

@@ -396,7 +396,7 @@ test-e2e: build-mock-backend build-cloud-proxy-backend prepare-e2e run-e2e-image
 	@echo 'Running e2e tests'
 	BUILD_TYPE=$(BUILD_TYPE) \
 	LOCALAI_API=http://$(E2E_BRIDGE_IP):5390 \
-	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e
+	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='!Distributed' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e
 	$(MAKE) clean-mock-backend
 	$(MAKE) clean-cloud-proxy-backend
 	$(MAKE) teardown-e2e

cmd/test-resources/main.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"archive/tar"
 	"crypto/sha256"
 	"errors"
 	"fmt"
@@ -150,16 +151,21 @@ func update(target, manifestDir, cacheDir string) error {
 			return err
 		}
 	}
-	for _, resource := range manifest.Images {
-		if err := pullAndPack(resource.Reference, resource.SHA256, cacheDir); err != nil {
+	for i := range manifest.Images {
+		digest, err := pullAndPack(manifest.Images[i].Reference, cacheDir)
+		if err != nil {
 			return err
 		}
+		manifest.Images[i].SHA256 = digest
 	}
 	bundlePath := filepath.Join(cacheDir, "bundles", target+".tar.zst")
 	digest, err := testresources.PackBundle(cacheDir, bundlePath, manifest)
 	if err != nil {
 		return err
 	}
+	if err := testresources.WriteManifest(filepath.Join(manifestDir, target+".json"), manifest); err != nil {
+		return err
+	}
 	lockPath := filepath.Join(manifestDir, "lock.json")
 	lock, err := testresources.LoadLock(lockPath)
 	if err != nil {
@@ -272,24 +278,93 @@ func storeVerified(reader io.Reader, expected, cacheDir string) (int64, error) {
 	return size, nil
 }
 
-func pullAndPack(reference, expected, cacheDir string) error {
+func pullAndPack(reference, cacheDir string) (string, error) {
 	if !strings.Contains(reference, "@sha256:") {
-		return fmt.Errorf("refusing mutable image reference %s", reference)
+		return "", fmt.Errorf("refusing mutable image reference %s", reference)
 	}
 	if err := exec.Command("docker", "pull", reference).Run(); err != nil {
-		return fmt.Errorf("pull image %s: %w", reference, err)
+		return "", fmt.Errorf("pull image %s: %w", reference, err)
 	}
 	cmd := exec.Command("docker", "save", reference)
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
-		return err
+		return "", err
 	}
 	if err := cmd.Start(); err != nil {
-		return err
+		return "", err
 	}
-	_, storeErr := storeVerified(stdout, expected, cacheDir)
+	if err := os.MkdirAll(cacheDir, 0o755); err != nil {
+		return "", err
+	}
+	normalized, err := os.CreateTemp(cacheDir, ".docker-save-*.tar")
+	if err != nil {
+		return "", err
+	}
+	normalizedName := normalized.Name()
+	defer func() { _ = os.Remove(normalizedName) }()
+	normalizeErr := normalizeDockerArchive(stdout, normalized)
 	waitErr := cmd.Wait()
-	return errors.Join(storeErr, waitErr)
+	closeErr := normalized.Close()
+	if err := errors.Join(normalizeErr, waitErr, closeErr); err != nil {
+		return "", err
+	}
+	input, err := os.Open(normalizedName)
+	if err != nil {
+		return "", err
+	}
+	digest, _, storeErr := storeContentAddressed(input, cacheDir)
+	return digest, errors.Join(storeErr, input.Close())
+}
+
+func normalizeDockerArchive(reader io.Reader, writer io.Writer) error {
+	tr := tar.NewReader(reader)
+	tw := tar.NewWriter(writer)
+	for {
+		header, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		stable := *header
+		stable.Uid, stable.Gid = 0, 0
+		stable.Uname, stable.Gname = "", ""
+		stable.ModTime = time.Unix(0, 0).UTC()
+		stable.AccessTime, stable.ChangeTime = time.Time{}, time.Time{}
+		stable.PAXRecords, stable.Xattrs = nil, nil
+		if err := tw.WriteHeader(&stable); err != nil {
+			return err
+		}
+		if _, err := io.Copy(tw, tr); err != nil {
+			return err
+		}
+	}
+	return tw.Close()
+}
+
+func storeContentAddressed(reader io.Reader, cacheDir string) (string, int64, error) {
+	directory := filepath.Join(cacheDir, "blobs", "sha256")
+	if err := os.MkdirAll(directory, 0o755); err != nil {
+		return "", 0, err
+	}
+	temporary, err := os.CreateTemp(directory, ".record-*")
+	if err != nil {
+		return "", 0, err
+	}
+	name := temporary.Name()
+	defer func() { _ = os.Remove(name) }()
+	hash := sha256.New()
+	size, copyErr := io.Copy(io.MultiWriter(temporary, hash), reader)
+	closeErr := temporary.Close()
+	if err := errors.Join(copyErr, closeErr); err != nil {
+		return "", 0, err
+	}
+	digest := fmt.Sprintf("%x", hash.Sum(nil))
+	if err := os.Rename(name, testresources.BlobPath(cacheDir, digest)); err != nil {
+		return "", 0, err
+	}
+	return digest, size, nil
 }
 
 func preparationError(target string, err error) error {

cmd/test-resources/main_test.go

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/onsi/gomega"
+)
+
+func TestNormalizeDockerArchiveIgnoresTarMetadata(t *testing.T) {
+	g := gomega.NewWithT(t)
+	first := dockerArchive(g, time.Unix(100, 0), 12, "builder")
+	second := dockerArchive(g, time.Unix(200, 0), 34, "runner")
+
+	var normalizedFirst, normalizedSecond bytes.Buffer
+	g.Expect(normalizeDockerArchive(bytes.NewReader(first), &normalizedFirst)).To(gomega.Succeed())
+	g.Expect(normalizeDockerArchive(bytes.NewReader(second), &normalizedSecond)).To(gomega.Succeed())
+	firstDigest := fmt.Sprintf("%x", sha256.Sum256(normalizedFirst.Bytes()))
+	secondDigest := fmt.Sprintf("%x", sha256.Sum256(normalizedSecond.Bytes()))
+	g.Expect(secondDigest).To(gomega.Equal(firstDigest))
+
+	tr := tar.NewReader(bytes.NewReader(normalizedFirst.Bytes()))
+	header, err := tr.Next()
+	g.Expect(err).NotTo(gomega.HaveOccurred())
+	g.Expect(header.Uid).To(gomega.Equal(0))
+	g.Expect(header.Gid).To(gomega.Equal(0))
+	g.Expect(header.Uname).To(gomega.BeEmpty())
+	g.Expect(header.Gname).To(gomega.BeEmpty())
+	g.Expect(header.ModTime).To(gomega.Equal(time.Unix(0, 0)))
+	content, err := io.ReadAll(tr)
+	g.Expect(err).NotTo(gomega.HaveOccurred())
+	g.Expect(string(content)).To(gomega.Equal("image data"))
+}
+
+func dockerArchive(g *gomega.WithT, modTime time.Time, uid int, user string) []byte {
+	var archive bytes.Buffer
+	tw := tar.NewWriter(&archive)
+	content := []byte("image data")
+	g.Expect(tw.WriteHeader(&tar.Header{
+		Name: "layer.tar", Mode: 0o644, Size: int64(len(content)),
+		ModTime: modTime, Uid: uid, Gid: uid, Uname: user, Gname: user,
+	})).To(gomega.Succeed())
+	_, err := tw.Write(content)
+	g.Expect(err).NotTo(gomega.HaveOccurred())
+	g.Expect(tw.Close()).To(gomega.Succeed())
+	return archive.Bytes()
+}

internal/testresources/resources.go

@@ -81,7 +81,15 @@ func LoadLock(path string) (Lock, error) {
 }
 
 func WriteLock(path string, lock Lock) error {
-	data, err := json.MarshalIndent(lock, "", "  ")
+	return writeJSON(path, lock)
+}
+
+func WriteManifest(path string, manifest Manifest) error {
+	return writeJSON(path, manifest)
+}
+
+func writeJSON(path string, value any) error {
+	data, err := json.MarshalIndent(value, "", "  ")
 	if err != nil {
 		return err
 	}

scripts/test-network-lint.sh

@@ -6,13 +6,29 @@ set -euo pipefail
 # a worktree-only diff would always be empty). Most existing direct clients are
 # loopback fixtures; changing the inventory requires an intentional baseline
 # update after review.
-expected_inventory=2885a428cdab55eea357dae3ec47b3d44f9999b59542d06cd3d792cf491c76b3
+expected_inventory=6b8a611b9d01ea1b58f446ccdd92efdbb01c34663ee5946194ccc0f00e8e877d
+search_test_files() {
+  local pattern=$1
+  shift
+  while IFS= read -r -d '' file; do
+    [[ $file == *_test.go ]] && grep -hE "$pattern" "$file" || true
+  done < <(git ls-files -co --exclude-standard -z -- "$@")
+}
+
+search_shell_files() {
+  local pattern=$1
+  shift
+  while IFS= read -r -d '' file; do
+    [[ $file == *.sh ]] && grep -hE "$pattern" "$file" || true
+  done < <(git ls-files -co --exclude-standard -z -- "$@")
+}
+
 inventory=$(
   {
-    rg --no-heading --no-line-number --glob '*_test.go' \
+    search_test_files \
       '(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)")' \
-      pkg core tests backend || true
-    rg --no-heading --no-line-number --glob '*.sh' '(curl|wget)[[:space:]]' tests backend || true
+      pkg core tests backend
+    search_shell_files '(curl|wget)[[:space:]]' tests backend
   } | LC_ALL=C sort
 )
 if command -v sha256sum >/dev/null 2>&1; then
@@ -21,7 +37,7 @@ else
   actual_inventory=$(printf '%s\n' "$inventory" | shasum -a 256 | awk '{print $1}')
 fi
 if [[ $actual_inventory != "$expected_inventory" ]]; then
-  echo 'Test network mechanism inventory changed; remove the direct access or review and update the lint baseline:' >&2
+  echo "Test network mechanism inventory changed (expected $expected_inventory, got $actual_inventory); remove the direct access or review and update the lint baseline:" >&2
   echo "$inventory" >&2
   exit 1
 fi
@@ -30,8 +46,8 @@ fi
 # literals and direct mechanisms instead of only reporting the fingerprint.
 base=${TEST_NETWORK_LINT_BASE:-HEAD}
 violations=$(git diff --unified=0 "$base" -- api pkg core tests backend | \
-	  rg '^\+[^+].*(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)"|https?://)' | \
-	  rg -v 'test-network: fixture' || true)
+  grep -E '^\+[^+].*(http\.(Get|Post|Head)\(|http\.Default(Client|Transport)|net\.Dial\(|exec\.Command\([^,]+,[[:space:]]*"(curl|wget)"|https?://)' | \
+  grep -Ev 'test-network: fixture' || true)
 if [[ -n "$violations" ]]; then
   echo 'Direct test network access is forbidden; use a fixture or guarded transport:' >&2
   echo "$violations" >&2

test-resources/manifests/default.json

@@ -4,7 +4,7 @@
   "images": [
     {
       "reference": "docker.io/library/postgres@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20",
-      "sha256": "bd98262690143a2a05167b3e924cde07e82ea269e45d5610e00a45f50e76d45a"
+      "sha256": "f4e8a437601f09ad619c6a8df831cd71c24c5c2276652e6ce89c042185759ed9"
     }
   ]
 }

test-resources/manifests/distributed-e2e.json

@@ -4,11 +4,11 @@
   "images": [
     {
       "reference": "docker.io/library/postgres@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777",
-      "sha256": "fee330cbd34786da2b211fe2e4b7424d7bf974466cd3db0e376b2e4c457a339c"
+      "sha256": "c8d5971aa1c74f0130dcde5ab942e3613d2179cb62cda4e9c08e8ec4c7252220"
     },
     {
       "reference": "docker.io/library/nats@sha256:c11af972c99ae542de8925e6a7d9c533aa1eb039660420d2074beed6089b3bf0",
-      "sha256": "96e0f53430696eadaf1d7e250914ab78779cadc40df715a168ebe342e039b6f2"
+      "sha256": "9e833c05b393c5ac68a06ab348ce45aac9a7cd72a19cce7ed5e41fb3af56c423"
     }
   ]
 }

test-resources/manifests/lock.json

@@ -3,9 +3,9 @@
   "bundles": {
     "aio": "sha256:03b05eedb51c853b0f05f2f3f592e2edc210e337388d3ff5a766680c0a166e55",
     "backend": "embedded",
-    "default": "sha256:185ebd3cfb994b9c1d1d1d9fad0a5d95c9de698b45e010276db388a9d66474bd",
+    "default": "sha256:533e2744151ce8a4df3a9b0ba073222135f17c3e2ee0e5cfeddfb836f41efbca",
     "default-darwin": "embedded",
-    "distributed-e2e": "sha256:797dad68952914bf6612a42486086aa45eb17112067bec9955e2f5cf65a030dc",
+    "distributed-e2e": "sha256:524db2b4cedb091c0604eaf6dcd7b0d73fd84e3b262405b6cdb98df099f54965",
     "external-probes": "embedded",
     "hardware": "embedded"
   }