diff --git a/core/services/nodes/file_stager_http.go b/core/services/nodes/file_stager_http.go index 29a8d1e9a..17bd8e528 100644 --- a/core/services/nodes/file_stager_http.go +++ b/core/services/nodes/file_stager_http.go @@ -753,7 +753,7 @@ func hashLocalCached(ctx context.Context, path string) (string, error) { return "", err } if sidecarStat, err := os.Stat(sidecar); err == nil && !sidecarStat.ModTime().Before(fileStat.ModTime()) { - if data, err := os.ReadFile(sidecar); err == nil { + if data, err := readHashSidecar(sidecar); err == nil { cached := strings.TrimSpace(string(data)) if len(cached) == 64 { return cached, nil @@ -765,7 +765,7 @@ func hashLocalCached(ctx context.Context, path string) (string, error) { if err != nil { return "", err } - if err := os.WriteFile(sidecar, []byte(hashHex), 0640); err != nil { + if err := writeHashSidecar(sidecar, hashHex); err != nil { xlog.Warn("Failed to write hash sidecar", "path", sidecar, "error", err) } return hashHex, nil diff --git a/core/services/nodes/file_transfer_server.go b/core/services/nodes/file_transfer_server.go index 1d6dd061a..9683e61af 100644 --- a/core/services/nodes/file_transfer_server.go +++ b/core/services/nodes/file_transfer_server.go @@ -566,7 +566,7 @@ func handleHead(w http.ResponseWriter, r *http.Request, stagingDir, modelsDir, d // X-Target-SHA256 and skip emitting X-Content-SHA256 (which would otherwise // be the hash of just the bytes received so far — misleading for clients // trying to decide whether the file is "the right one"). - if target, err := os.ReadFile(filePath + targetSidecarSuffix); err == nil { + if target, err := readHashSidecar(filePath + targetSidecarSuffix); err == nil { t := strings.TrimSpace(string(target)) if len(t) == 64 { w.Header().Set(HeaderTargetSHA256, t) @@ -869,7 +869,7 @@ func handleFullUpload(w http.ResponseWriter, r *http.Request, dstPath, key, expe return } - if err := os.WriteFile(dstPath+hashSidecarSuffix, []byte(hashHex), 0640); err != nil { + if err := writeHashSidecar(dstPath+hashSidecarSuffix, hashHex); err != nil { xlog.Warn("Failed to write hash sidecar", "path", dstPath+hashSidecarSuffix, "error", err) } @@ -910,11 +910,11 @@ func handleRangeUpload(w http.ResponseWriter, r *http.Request, dstPath, key stri if expectedFinalHash != "" { // Compare the client's declared target hash against either an // in-progress target sidecar OR the completed-file sidecar. - if t, err := os.ReadFile(targetSidecar); err == nil { + if t, err := readHashSidecar(targetSidecar); err == nil { if strings.EqualFold(strings.TrimSpace(string(t)), expectedFinalHash) { sameFile = true } - } else if h, err := os.ReadFile(dstPath + hashSidecarSuffix); err == nil { + } else if h, err := readHashSidecar(dstPath + hashSidecarSuffix); err == nil { if strings.EqualFold(strings.TrimSpace(string(h)), expectedFinalHash) { sameFile = true } @@ -941,7 +941,7 @@ func handleRangeUpload(w http.ResponseWriter, r *http.Request, dstPath, key stri // Cross-attempt consistency: if there's an in-progress target sidecar with // a different hash than what's now being claimed, force a restart. if expectedFinalHash != "" && cr.start > 0 { - prev, _ := os.ReadFile(targetSidecar) + prev, _ := readHashSidecar(targetSidecar) prevHash := strings.TrimSpace(string(prev)) if prevHash != "" && !strings.EqualFold(prevHash, expectedFinalHash) { _ = os.Remove(dstPath) @@ -985,7 +985,7 @@ func handleRangeUpload(w http.ResponseWriter, r *http.Request, dstPath, key stri // Persist the declared expected hash so subsequent chunks can be // cross-checked. if expectedFinalHash != "" { - if err := os.WriteFile(targetSidecar, []byte(expectedFinalHash), 0640); err != nil { + if err := writeHashSidecar(targetSidecar, expectedFinalHash); err != nil { xlog.Warn("Failed to write target hash sidecar", "path", targetSidecar, "error", err) } } @@ -1045,7 +1045,7 @@ func finalizeRangeUpload(w http.ResponseWriter, dstPath, key string, size int64, return } - if err := os.WriteFile(dstPath+hashSidecarSuffix, []byte(finalHash), 0640); err != nil { + if err := writeHashSidecar(dstPath+hashSidecarSuffix, finalHash); err != nil { xlog.Warn("Failed to write hash sidecar", "path", dstPath+hashSidecarSuffix, "error", err) } // Clear the in-progress sidecar — upload is committed. @@ -1071,7 +1071,7 @@ func computeAndCacheHash(filePath string) (string, error) { } if sidecarStat, err := os.Stat(sidecar); err == nil && !sidecarStat.ModTime().Before(fileStat.ModTime()) { - if data, err := os.ReadFile(sidecar); err == nil { + if data, err := readHashSidecar(sidecar); err == nil { h := strings.TrimSpace(string(data)) if len(h) == 64 { // valid hex-encoded SHA-256 return h, nil @@ -1084,7 +1084,7 @@ func computeAndCacheHash(filePath string) (string, error) { return "", err } - if err := os.WriteFile(sidecar, []byte(hashHex), 0640); err != nil { + if err := writeHashSidecar(sidecar, hashHex); err != nil { xlog.Warn("Failed to write hash sidecar", "path", sidecar, "error", err) } return hashHex, nil diff --git a/core/services/nodes/hash_sidecar.go b/core/services/nodes/hash_sidecar.go new file mode 100644 index 000000000..d4885d033 --- /dev/null +++ b/core/services/nodes/hash_sidecar.go @@ -0,0 +1,40 @@ +package nodes + +import ( + "crypto/rand" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/pkg/safefile" +) + +// A validated model path does not validate the adjacent metadata: an attacker +// can place a separate symlink at the sidecar name. +func readHashSidecar(path string) ([]byte, error) { + data, _, err := safefile.ReadRegularAt(filepath.Dir(path), filepath.Base(path)) + return data, err +} + +func writeHashSidecar(path, hash string) error { + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return err + } + defer func() { _ = root.Close() }() + // Replace metadata atomically instead of following an existing sidecar link. + name := ".localai-hash-" + rand.Text() + file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + defer func() { _ = root.Remove(name) }() + _, writeErr := file.WriteString(hash) + closeErr := file.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } + return root.Rename(name, filepath.Base(path)) +} diff --git a/core/services/nodes/hash_sidecar_test.go b/core/services/nodes/hash_sidecar_test.go new file mode 100644 index 000000000..da353ba04 --- /dev/null +++ b/core/services/nodes/hash_sidecar_test.go @@ -0,0 +1,90 @@ +package nodes + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestHashSidecarCannotEscapeFileDirectory(t *testing.T) { + for _, mode := range []string{"server", "stager"} { + t.Run(mode, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "model.bin") + outside := filepath.Join(t.TempDir(), "outside") + content := []byte("model contents") + forged := strings.Repeat("a", 64) + if err := os.WriteFile(path, content, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(outside, []byte(forged), 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, path+hashSidecarSuffix); err != nil { + t.Fatal(err) + } + var got string + var err error + if mode == "server" { + got, err = computeAndCacheHash(path) + } else { + got, err = hashLocalCached(context.Background(), path) + } + if err != nil { + t.Fatal(err) + } + if got == forged { + t.Fatal("trusted hash from sidecar outside the model directory") + } + after, err := os.ReadFile(outside) + if err != nil { + t.Fatal(err) + } + if string(after) != forged { + t.Fatal("overwrote file outside the model directory") + } + }) + } +} + +func TestHashSidecarWriteDoesNotFollowSymlinks(t *testing.T) { + for _, external := range []bool{false, true} { + dir := t.TempDir() + targetDir := dir + if external { + targetDir = t.TempDir() + } + target := filepath.Join(targetDir, "original") + path := filepath.Join(dir, "model.bin.sha256") + if err := os.WriteFile(target, []byte("do not overwrite"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, path); err != nil { + t.Fatal(err) + } + if _, err := readHashSidecar(path); err == nil { + t.Fatal("read symlinked sidecar") + } + hash := strings.Repeat("b", 64) + if err := writeHashSidecar(path, hash); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(target) + if err != nil || string(data) != "do not overwrite" { + t.Fatalf("target changed: %q, %v", data, err) + } + data, err = readHashSidecar(path) + if err != nil || string(data) != hash { + t.Fatalf("incorrect sidecar: %q, %v", data, err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0600 { + t.Fatalf("sidecar permissions: %v", info.Mode()) + } + } +} diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 12c82fc15..9df1e335e 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -730,6 +730,8 @@ A worker started without `LOCALAI_STORAGE_URL` does not serve the four staging v When S3 is not configured, model files are transferred directly from the frontend to workers via **HTTP** - no shared filesystem needed. Each worker runs a small HTTP file transfer server alongside the gRPC backend process. This is the default and works out of the box. +HTTP transfers cache SHA-256 hashes in adjacent metadata files. LocalAI ignores symbolic links when reading this metadata and replaces it atomically when writing. New metadata files are readable and writable only by the LocalAI process owner (mode `0600`). + For high-throughput or very large model files, S3 can be more efficient since it avoids streaming through the frontend. ### Shared models directory