fix(oci): install backends on filesystems without symlinks (#11166)

* fix(backends): fall back to copying links when the filesystem rejects symlinks (#10890)

Backend installation extracts the OCI image tar via containerd's
archive.Apply, which calls os.Symlink directly. On filesystems that do
not support symlinks (notably CIFS/SMB mounts, commonly used to back the
/backends volume) the syscall fails with "operation not supported" and
the whole install aborts, leaving an empty backend directory. The CUDA
llama.cpp image trips this on the libcublas.so -> libcublas.so.12.x
symlink.

When archive.Apply fails with a link-unsupported error, reset the
staging directory and re-extract with a pure-Go walker that still
attempts real symlinks/hardlinks first and degrades to copying the link
target's contents in place when the filesystem rejects them.
mutate.Extract already flattened the layers, so the tar carries no
whiteouts to interpret. Link copies are deferred to a second pass so
forward references resolve.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude:opus-4.8 [Claude Code]

* fix(oci): check deferred Close in copyFilePreservingMode (errcheck)

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude:opus-4.8 [Claude Code]

* fix(oci): reject path-traversal tar entries in the link-copy fallback

safeJoin sanitized "../.." entries by clamping them under root instead of
rejecting them, so a malicious entry was silently redirected rather than
refused. Join without the leading-slash trick and reject any entry whose
cleaned path resolves outside root; absolute link targets are still mapped
under root (image-root relative) rather than escaping.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude:opus-4.8 [Claude Code]

* fix(oci): silence gosec on the validated link-copy file ops

Use hdr.FileInfo().Mode() instead of converting the int64 tar mode to
os.FileMode (removes two G115 overflow findings), and annotate the tar
extraction file operations with justified #nosec comments: every path is
validated by safeJoin against the extraction root before use (G304/G305).

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude:opus-4.8 [Claude Code]

* fix(oci): build extraction image from downloaded layers

Avoid appending downloaded layers to the original remote-backed image, which duplicates the layer stack and reopens the source during extraction. Building from an empty image preserves the flattened whiteout semantics while keeping extraction local.

Assisted-by: Codex:gpt-5

* fix(oci): materialize chained links in dependency order

Retry deferred link copies until their targets exist so soname chains work on filesystems without symlink support. Document that copied links can increase backend storage usage on CIFS and SMB mounts.

Assisted-by: Codex:gpt-5

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>
This commit is contained in:
localai-org-maint-bot
2026-07-30 16:23:42 +02:00
committed by GitHub
parent ef724a3c9d
commit 68a0460681
3 changed files with 483 additions and 3 deletions

View File

@@ -202,6 +202,12 @@ must be exactly `/models`, `/backends`, `/configuration`, and `/data`. In
UnRAID and other container-template UIs, create one path mapping for each row
in the table above.
Backend OCI images contain symbolic links. When `/backends` is stored on a
filesystem that cannot create links, such as some CIFS/SMB mounts, LocalAI
materializes each link as a regular file so installation can complete. This can
use more disk space than a local filesystem. Prefer a Docker or Podman named
volume for `/backends` when possible.
To use bind mounts:
```bash

View File

@@ -0,0 +1,231 @@
package oci
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"errors"
"io"
"os"
"path/filepath"
"syscall"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/tarball"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
type compressedOnlyLayer struct {
v1.Layer
digest v1.Hash
}
func (l compressedOnlyLayer) Uncompressed() (io.ReadCloser, error) {
return nil, errors.New("downloaded layer reopened from source")
}
func (l compressedOnlyLayer) Digest() (v1.Hash, error) { return l.digest, nil }
func buildLayer(entries ...tar.Header) v1.Layer {
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for _, header := range entries {
content := []byte(header.PAXRecords["content"])
header.PAXRecords = nil
header.Size = int64(len(content))
Expect(tw.WriteHeader(&header)).To(Succeed())
if len(content) != 0 {
_, err := tw.Write(content)
Expect(err).NotTo(HaveOccurred())
}
}
Expect(tw.Close()).To(Succeed())
Expect(zw.Close()).To(Succeed())
layer, err := tarball.LayerFromReader(bytes.NewReader(buf.Bytes()))
Expect(err).NotTo(HaveOccurred())
digest, _, err := v1.SHA256(bytes.NewReader([]byte{byte(len(entries))}))
Expect(err).NotTo(HaveOccurred())
return compressedOnlyLayer{Layer: layer, digest: digest}
}
// buildTar assembles an in-memory tar carrying a directory, a regular file and
// a relative symlink pointing at that file, mirroring the layout of a backend
// image (e.g. libcublas.so -> libcublas.so.12).
func buildTar() []byte {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
Expect(tw.WriteHeader(&tar.Header{
Name: "lib/",
Typeflag: tar.TypeDir,
Mode: 0755,
})).To(Succeed())
content := []byte("real library bytes")
Expect(tw.WriteHeader(&tar.Header{
Name: "lib/libcublas.so.12",
Typeflag: tar.TypeReg,
Mode: 0644,
Size: int64(len(content)),
})).To(Succeed())
_, err := tw.Write(content)
Expect(err).NotTo(HaveOccurred())
Expect(tw.WriteHeader(&tar.Header{
Name: "lib/libcublas.so",
Typeflag: tar.TypeSymlink,
Linkname: "libcublas.so.12",
Mode: 0777,
})).To(Succeed())
Expect(tw.Close()).To(Succeed())
return buf.Bytes()
}
func buildChainedLinkTar() []byte {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
content := []byte("real library bytes")
Expect(tw.WriteHeader(&tar.Header{
Name: "lib/libcublas.so",
Typeflag: tar.TypeSymlink,
Linkname: "libcublas.so.12",
Mode: 0777,
})).To(Succeed())
Expect(tw.WriteHeader(&tar.Header{
Name: "lib/libcublas.so.12",
Typeflag: tar.TypeSymlink,
Linkname: "libcublas.so.12.8.5.5",
Mode: 0777,
})).To(Succeed())
Expect(tw.WriteHeader(&tar.Header{
Name: "lib/libcublas.so.12.8.5.5",
Typeflag: tar.TypeReg,
Mode: 0644,
Size: int64(len(content)),
})).To(Succeed())
_, err := tw.Write(content)
Expect(err).NotTo(HaveOccurred())
Expect(tw.Close()).To(Succeed())
return buf.Bytes()
}
var _ = Describe("Tar extraction fallback for link-less filesystems", func() {
It("downloads a layered image once and preserves whiteouts before copying links", func() {
base := buildLayer(
tar.Header{Name: "lib/removed.so", Mode: 0644, PAXRecords: map[string]string{"content": "removed"}},
tar.Header{Name: "lib/libcublas.so.12", Mode: 0644, PAXRecords: map[string]string{"content": "old library"}},
)
top := buildLayer(
tar.Header{Name: "lib/.wh.removed.so", Mode: 0644},
tar.Header{Name: "lib/libcublas.so.12", Mode: 0644, PAXRecords: map[string]string{"content": "new library"}},
tar.Header{Name: "lib/libcublas.so", Typeflag: tar.TypeSymlink, Linkname: "libcublas.so.12", Mode: 0777},
)
image, err := mutate.AppendLayers(empty.Image, base, top)
Expect(err).NotTo(HaveOccurred())
tmp := GinkgoT().TempDir()
tarPath := filepath.Join(tmp, "rootfs.tar")
Expect(DownloadOCIImageTar(context.Background(), image, "test/image", tarPath, nil)).To(Succeed())
originalSymlink := symlink
symlink = func(string, string) error { return syscall.ENOTSUP }
DeferCleanup(func() { symlink = originalSymlink })
destination := filepath.Join(tmp, "destination")
Expect(os.Mkdir(destination, 0755)).To(Succeed())
Expect(ExtractOCIImageFromTar(context.Background(), tarPath, "test/image", destination, nil)).To(Succeed())
Expect(filepath.Join(destination, "lib", "removed.so")).NotTo(BeAnExistingFile())
Expect(os.ReadFile(filepath.Join(destination, "lib", "libcublas.so.12"))).To(Equal([]byte("new library")))
Expect(os.ReadFile(filepath.Join(destination, "lib", "libcublas.so"))).To(Equal([]byte("new library")))
})
Describe("isLinkUnsupportedError", func() {
It("recognises filesystem link-unsupported errors", func() {
Expect(isLinkUnsupportedError(syscall.ENOTSUP)).To(BeTrue())
Expect(isLinkUnsupportedError(syscall.EOPNOTSUPP)).To(BeTrue())
Expect(isLinkUnsupportedError(syscall.EPERM)).To(BeTrue())
Expect(isLinkUnsupportedError(&os.LinkError{
Op: "symlink",
Old: "libcublas.so.12",
New: "/backends/lib/libcublas.so",
Err: syscall.ENOTSUP,
})).To(BeTrue())
})
It("does not misclassify unrelated errors", func() {
Expect(isLinkUnsupportedError(os.ErrNotExist)).To(BeFalse())
Expect(isLinkUnsupportedError(syscall.ENOSPC)).To(BeFalse())
})
})
Describe("safeJoin", func() {
It("keeps entries inside the root", func() {
root := "/tmp/extract-root"
p, err := safeJoin(root, "lib/libcublas.so")
Expect(err).NotTo(HaveOccurred())
Expect(p).To(Equal(filepath.Join(root, "lib/libcublas.so")))
})
It("rejects path traversal entries", func() {
_, err := safeJoin("/tmp/extract-root", "../../etc/passwd")
Expect(err).To(HaveOccurred())
})
})
Describe("extractTarCopyingLinks", func() {
It("preserves symlinks when the filesystem supports them", func() {
dir := GinkgoT().TempDir()
Expect(extractTarCopyingLinks(bytes.NewReader(buildTar()), dir)).To(Succeed())
linkPath := filepath.Join(dir, "lib", "libcublas.so")
fi, err := os.Lstat(linkPath)
Expect(err).NotTo(HaveOccurred())
Expect(fi.Mode() & os.ModeSymlink).NotTo(BeZero())
data, err := os.ReadFile(linkPath)
Expect(err).NotTo(HaveOccurred())
Expect(string(data)).To(Equal("real library bytes"))
})
It("copies the target when symlink creation is unsupported", func() {
// Simulate a CIFS/SMB mount: symlink() reports ENOTSUP.
origSymlink := symlink
symlink = func(string, string) error { return syscall.ENOTSUP }
DeferCleanup(func() { symlink = origSymlink })
dir := GinkgoT().TempDir()
Expect(extractTarCopyingLinks(bytes.NewReader(buildTar()), dir)).To(Succeed())
linkPath := filepath.Join(dir, "lib", "libcublas.so")
fi, err := os.Lstat(linkPath)
Expect(err).NotTo(HaveOccurred())
// The entry must now be a real, regular file (a copy), not a symlink.
Expect(fi.Mode() & os.ModeSymlink).To(BeZero())
Expect(fi.Mode().IsRegular()).To(BeTrue())
data, err := os.ReadFile(linkPath)
Expect(err).NotTo(HaveOccurred())
Expect(string(data)).To(Equal("real library bytes"))
})
It("materialises chained symlinks regardless of archive order", func() {
origSymlink := symlink
symlink = func(string, string) error { return syscall.ENOTSUP }
DeferCleanup(func() { symlink = origSymlink })
dir := GinkgoT().TempDir()
Expect(extractTarCopyingLinks(bytes.NewReader(buildChainedLinkTar()), dir)).To(Succeed())
Expect(os.ReadFile(filepath.Join(dir, "lib", "libcublas.so"))).To(Equal([]byte("real library bytes")))
Expect(os.ReadFile(filepath.Join(dir, "lib", "libcublas.so.12"))).To(Equal([]byte("real library bytes")))
})
})
})

View File

@@ -1,12 +1,14 @@
package oci
import (
"archive/tar"
"context"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
@@ -19,6 +21,7 @@ import (
"github.com/google/go-containerregistry/pkg/logs"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
@@ -396,8 +399,9 @@ func DownloadOCIImageTar(ctx context.Context, img v1.Image, imageRef string, tar
downloadedSize += layerSize
}
// Create a local image from the downloaded layers
localImg, err := mutate.AppendLayers(img, downloadedLayers...)
// Build the local image only from the downloaded layers. Appending them to
// img duplicates the layer stack and makes extraction reopen the source.
localImg, err := mutate.AppendLayers(empty.Image, downloadedLayers...)
if err != nil {
return fmt.Errorf("failed to create local image: %v", err)
}
@@ -447,8 +451,247 @@ func ExtractOCIImageFromTar(ctx context.Context, tarFilePath, imageRef, targetDe
_, err = archive.Apply(ctx,
targetDestination, reader,
archive.WithNoSameOwner())
if err == nil {
return nil
}
return err
// Some filesystems (notably CIFS/SMB mounts, which users commonly bind as the
// /backends volume) reject symlink/hardlink creation with "operation not
// supported"/"operation not permitted". containerd's archive.Apply hard-fails
// there, so no backend can be installed. Fall back to a pure-Go extractor that
// degrades unsupported links into plain file copies. mutate.Extract already
// flattened the layers, so this tar carries no whiteouts to interpret.
if !isLinkUnsupportedError(err) {
return err
}
logs.Warn.Printf("symlink/hardlink creation is not supported on filesystem at %q (%v), retrying extraction with links copied in place", targetDestination, err)
// archive.Apply may have written some entries before failing; start from a
// clean destination so the manual pass is deterministic. The caller stages
// into an ephemeral, per-install temp directory, so wiping its contents is safe.
if err := cleanDirContents(targetDestination); err != nil {
return fmt.Errorf("failed to reset destination before fallback extraction: %w", err)
}
// Re-read the tar from the beginning for the second pass.
if _, err := tarFile.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to rewind tar for fallback extraction: %w", err)
}
return extractTarCopyingLinks(tarFile, targetDestination)
}
// symlink and hardlink are indirected so tests can simulate a filesystem that
// rejects link creation (e.g. CIFS/SMB).
var (
symlink = os.Symlink
hardlink = os.Link
)
// isLinkUnsupportedError reports whether err indicates the destination
// filesystem cannot create symlinks or hardlinks (e.g. CIFS/SMB, some FUSE
// mounts). Such filesystems surface ENOTSUP/EOPNOTSUPP, or EPERM in some
// configurations; the error text is also matched because containerd wraps the
// syscall error into a formatted string.
func isLinkUnsupportedError(err error) bool {
if errors.Is(err, syscall.ENOTSUP) || errors.Is(err, syscall.EOPNOTSUPP) || errors.Is(err, syscall.EPERM) {
return true
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "operation not supported") || strings.Contains(msg, "operation not permitted")
}
// cleanDirContents removes the entries inside dir without removing dir itself,
// preserving the directory (and its permissions) the caller created.
func cleanDirContents(dir string) error {
entries, err := os.ReadDir(dir)
if err != nil {
return err
}
for _, entry := range entries {
if err := os.RemoveAll(filepath.Join(dir, entry.Name())); err != nil {
return err
}
}
return nil
}
// extractTarCopyingLinks extracts a flattened image tar into targetDestination,
// copying the target contents of any symlink/hardlink that the filesystem cannot
// represent. Regular symlinks are still attempted first, so link semantics are
// preserved wherever the filesystem allows it. Link copies are deferred to a
// second pass so that forward references (a link appearing before its target in
// the tar) resolve correctly.
func extractTarCopyingLinks(r io.Reader, targetDestination string) error {
root, err := filepath.Abs(targetDestination)
if err != nil {
return err
}
type pendingLink struct {
path string // absolute destination path of the link
targetPath string // absolute path of the file to copy from
}
var pending []pendingLink
tr := tar.NewReader(r)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return fmt.Errorf("failed to read tar entry: %w", err)
}
cleaned, err := safeJoin(root, hdr.Name)
if err != nil {
return err
}
// Skip aufs/overlay whiteout markers defensively; a flattened tar
// should not contain any, but ignoring them is always correct here.
if strings.HasPrefix(filepath.Base(hdr.Name), ".wh.") {
continue
}
switch hdr.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(cleaned, hdr.FileInfo().Mode().Perm()|0700); err != nil {
return fmt.Errorf("failed to create directory %s: %w", cleaned, err)
}
case tar.TypeReg:
if err := os.MkdirAll(filepath.Dir(cleaned), 0700); err != nil {
return fmt.Errorf("failed to create parent directory for %s: %w", cleaned, err)
}
if err := writeRegularFile(cleaned, tr, hdr.FileInfo().Mode().Perm()); err != nil {
return err
}
case tar.TypeSymlink:
if err := os.MkdirAll(filepath.Dir(cleaned), 0700); err != nil {
return fmt.Errorf("failed to create parent directory for %s: %w", cleaned, err)
}
// Remove any pre-existing entry so os.Symlink does not fail with EEXIST.
_ = os.Remove(cleaned)
if err := symlink(hdr.Linkname, cleaned); err == nil {
break
} else if !isLinkUnsupportedError(err) {
return fmt.Errorf("failed to create symlink %s -> %s: %w", cleaned, hdr.Linkname, err)
}
// Resolve the link target: absolute targets are image-root relative,
// relative ones are resolved against the link's own directory.
var src string
if filepath.IsAbs(hdr.Linkname) {
src, err = safeJoin(root, hdr.Linkname)
} else {
// #nosec G305 -- safeJoin rejects any result that resolves outside the extraction root
src, err = safeJoin(root, filepath.Join(filepath.Dir(hdr.Name), hdr.Linkname))
}
if err != nil {
return err
}
pending = append(pending, pendingLink{path: cleaned, targetPath: src})
case tar.TypeLink:
if err := os.MkdirAll(filepath.Dir(cleaned), 0700); err != nil {
return fmt.Errorf("failed to create parent directory for %s: %w", cleaned, err)
}
// Hardlink targets are always relative to the image root.
src, err := safeJoin(root, hdr.Linkname)
if err != nil {
return err
}
_ = os.Remove(cleaned)
if err := hardlink(src, cleaned); err == nil {
break
} else if !isLinkUnsupportedError(err) {
return fmt.Errorf("failed to create hardlink %s -> %s: %w", cleaned, src, err)
}
pending = append(pending, pendingLink{path: cleaned, targetPath: src})
default:
// Ignore device nodes, fifos, etc: backend artifacts do not use them.
logs.Debug.Printf("skipping unsupported tar entry type during fallback extraction: name=%q type=%d", hdr.Name, hdr.Typeflag)
}
}
// Materialise links in dependency order. Soname links are often chained
// (libfoo.so -> libfoo.so.1 -> libfoo.so.1.2), and archives do not guarantee
// that the nearest-to-file link appears first.
for len(pending) > 0 {
var unresolved []pendingLink
for _, link := range pending {
if err := copyFilePreservingMode(link.targetPath, link.path); err != nil {
if errors.Is(err, os.ErrNotExist) {
unresolved = append(unresolved, link)
continue
}
return fmt.Errorf("failed to copy link target %s -> %s: %w", link.targetPath, link.path, err)
}
}
if len(unresolved) == len(pending) {
link := unresolved[0]
return fmt.Errorf("failed to resolve copied link target %s -> %s", link.targetPath, link.path)
}
pending = unresolved
}
return nil
}
// safeJoin joins name onto root and guarantees the result stays within root,
// rejecting path-traversal entries in a malicious tar. An absolute name (e.g. an
// absolute symlink target) is treated as image-root relative, so it is mapped
// under root rather than escaping it.
func safeJoin(root, name string) (string, error) {
cleaned := filepath.Join(root, name)
rel, err := filepath.Rel(root, cleaned)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) || filepath.IsAbs(rel) {
return "", fmt.Errorf("tar entry escapes extraction root: %s", name)
}
return cleaned, nil
}
func writeRegularFile(path string, r io.Reader, mode os.FileMode) error {
// Remove any pre-existing symlink so we do not write through it.
if fi, err := os.Lstat(path); err == nil && fi.Mode()&os.ModeSymlink != 0 {
_ = os.Remove(path)
}
// #nosec G304 -- path is validated by safeJoin to stay within the extraction root
f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode|0600)
if err != nil {
return fmt.Errorf("failed to create file %s: %w", path, err)
}
if _, err := io.Copy(f, r); err != nil {
_ = f.Close()
return fmt.Errorf("failed to write file %s: %w", path, err)
}
if err := f.Close(); err != nil {
return fmt.Errorf("failed to close file %s: %w", path, err)
}
return nil
}
func copyFilePreservingMode(src, dst string) error {
// #nosec G304 -- src is a safeJoin-validated link target within the extraction root
in, err := os.Open(src)
if err != nil {
return err
}
defer func() { _ = in.Close() }()
info, err := in.Stat()
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil {
return err
}
_ = os.Remove(dst)
// #nosec G304 -- dst is a safeJoin-validated path within the extraction root
out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode().Perm()|0600)
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
_ = out.Close()
return err
}
return out.Close()
}
// GetOCIImageUncompressedSize returns the total uncompressed size of an image