Compare commits

..

1 Commits

Author SHA1 Message Date
Ettore Di Giacinto
347cdcf545 fix(watchdog): persist a UI-saved Check Interval across restarts (#10601)
The watchdog Check Interval saved via /api/settings reverted to 500ms on
every restart, while the idle/busy timeouts persisted correctly.

Root cause: NewApplicationConfig baseline-defaulted WatchDogInterval to
500ms, whereas the idle/busy timeouts default to 0. The startup loader
(loadRuntimeSettingsFromFile) applies a persisted runtime_settings.json
value only when the field is still at its zero default - its heuristic
for "this wasn't set by an env var". Because the interval was always
500ms at that point, the loader never read the persisted value back, so
the saved interval was silently discarded on each boot.

Fix: drop the non-zero baseline default so the interval behaves like the
sibling timeouts (0 = unset). The effective 500ms default is now supplied
at the watchdog layer: WithWatchdogInterval ignores a non-positive value
so DefaultWatchDogOptions' 500ms is preserved (and a 0 interval can never
turn the watchdog loop into a busy spin). Also mirror the interval in the
live config file watcher alongside idle/busy, and report the real 500ms
default (not the stale "2s") from ToRuntimeSettings.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude:claude-opus-4-8 [Claude Code]
2026-06-30 08:04:12 +00:00
14 changed files with 68 additions and 121 deletions

View File

@@ -1482,13 +1482,8 @@ build-launcher-darwin:
mv cmd/launcher/LocalAI.app dist/LocalAI.app
bash contrib/macos/sign-and-notarize.sh sign dist/LocalAI.app
# Notarize + staple the .app itself, then wrap it into a drag-to-Applications
# DMG via hdiutil and sign the DMG. The app is stapled BEFORE packaging so the
# bundle carries its own ticket and verifies offline (a dmg-only staple leaves
# the app relying on an online Gatekeeper check, which fails offline / once the
# app is copied out of the dmg). No-op without notary secrets.
# Wrap the (signed) app into a drag-to-Applications DMG via hdiutil, then sign the DMG.
dmg-launcher-darwin: build-launcher-darwin
bash contrib/macos/sign-and-notarize.sh notarize-app dist/LocalAI.app
rm -rf dist/dmg dist/LocalAI.dmg
mkdir -p dist/dmg
cp -R dist/LocalAI.app dist/dmg/LocalAI.app
@@ -1500,7 +1495,7 @@ dmg-launcher-darwin: build-launcher-darwin
notarize-launcher-darwin: dmg-launcher-darwin
bash contrib/macos/sign-and-notarize.sh notarize dist/LocalAI.dmg
# Single entrypoint for CI: build -> sign app -> notarize+staple app -> dmg -> sign dmg -> notarize+staple dmg.
# Single entrypoint for CI: build -> sign app -> dmg -> sign dmg -> notarize -> staple.
release-launcher-darwin: notarize-launcher-darwin
@echo "dist/LocalAI.dmg is ready"

View File

@@ -71,42 +71,13 @@ cmd_notarize() {
echo "[notarize] notarized and stapled $dmg"
}
# Notarize and staple the .app bundle itself. Stapling the dmg alone is not
# enough: an app with no embedded ticket has no local proof of notarization, so
# Gatekeeper falls back to an online check — and the app then fails to launch on
# a machine that is offline / behind a firewall, or once it has been copied out
# of the dmg. Stapling the bundle makes it verify offline. notarytool needs an
# archive for a bundle, so we zip it first.
cmd_notarize_app() {
local app="$1"
if [ -z "${MACOS_NOTARY_KEY:-}" ]; then
echo "[notarize] MACOS_NOTARY_KEY unset: skipping notarization of $app"
return 0
fi
local keyfile zip
keyfile="$(mktemp).p8"
zip="$(mktemp).zip"
echo "$MACOS_NOTARY_KEY" | base64 --decode > "$keyfile"
ditto -c -k --keepParent "$app" "$zip"
xcrun notarytool submit "$zip" \
--key "$keyfile" \
--key-id "${MACOS_NOTARY_KEY_ID:?}" \
--issuer "${MACOS_NOTARY_ISSUER_ID:?}" \
--wait
rm -f "$keyfile" "$zip"
xcrun stapler staple "$app"
xcrun stapler validate "$app"
echo "[notarize] notarized and stapled $app"
}
main() {
local sub="${1:-}"; shift || true
case "$sub" in
import-cert) cmd_import_cert ;;
sign) cmd_sign "$@" ;;
notarize) cmd_notarize "$@" ;;
notarize-app) cmd_notarize_app "$@" ;;
*) echo "usage: $0 {import-cert|sign <path>|notarize <dmg>|notarize-app <app>}" >&2; exit 2 ;;
import-cert) cmd_import_cert ;;
sign) cmd_sign "$@" ;;
notarize) cmd_notarize "$@" ;;
*) echo "usage: $0 {import-cert|sign <path>|notarize <dmg>}" >&2; exit 2 ;;
esac
}

View File

@@ -197,6 +197,7 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
envWatchdogBusy := appConfig.WatchDogBusy == startupAppConfig.WatchDogBusy
envWatchdogIdleTimeout := appConfig.WatchDogIdleTimeout == startupAppConfig.WatchDogIdleTimeout
envWatchdogBusyTimeout := appConfig.WatchDogBusyTimeout == startupAppConfig.WatchDogBusyTimeout
envWatchdogInterval := appConfig.WatchDogInterval == startupAppConfig.WatchDogInterval
envSingleBackend := appConfig.SingleBackend == startupAppConfig.SingleBackend
envMaxActiveBackends := appConfig.MaxActiveBackends == startupAppConfig.MaxActiveBackends
envMemoryReclaimerEnabled := appConfig.MemoryReclaimerEnabled == startupAppConfig.MemoryReclaimerEnabled
@@ -257,6 +258,14 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
xlog.Warn("invalid watchdog busy timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogBusyTimeout)
}
}
if settings.WatchdogInterval != nil && !envWatchdogInterval {
dur, err := time.ParseDuration(*settings.WatchdogInterval)
if err == nil {
appConfig.WatchDogInterval = dur
} else {
xlog.Warn("invalid watchdog interval in runtime_settings.json", "error", err, "interval", *settings.WatchdogInterval)
}
}
// Handle MaxActiveBackends (new) and SingleBackend (deprecated)
if settings.MaxActiveBackends != nil && !envMaxActiveBackends {
appConfig.MaxActiveBackends = *settings.MaxActiveBackends

View File

@@ -87,6 +87,31 @@ var _ = Describe("loadRuntimeSettingsFromFile", func() {
})
})
// Watchdog check interval (issue #10601). Unlike the idle/busy timeouts
// (which default to 0), NewApplicationConfig baseline-defaults the
// interval to 500ms. The loader's "apply file value only if still at the
// zero default" env-detection therefore never fired for the interval, so
// a UI-saved Check Interval silently reverted to 500ms on every restart
// while the idle/busy timeouts persisted. These specs construct the
// config the same way boot does (NewApplicationConfig) so they observe
// the real default the loader sees.
Describe("watchdog interval", func() {
It("loads a UI-saved watchdog_interval on the next startup", func() {
cfg := config.NewApplicationConfig()
cfg.DynamicConfigsDir = seedSettings(`{"watchdog_interval": "2s"}`)
loadRuntimeSettingsFromFile(cfg)
Expect(cfg.WatchDogInterval).To(Equal(2 * time.Second))
})
It("does not override an explicit env/CLI interval", func() {
cfg := config.NewApplicationConfig()
cfg.DynamicConfigsDir = seedSettings(`{"watchdog_interval": "2s"}`)
cfg.WatchDogInterval = 1 * time.Second // simulate SetWatchDogInterval from env
loadRuntimeSettingsFromFile(cfg)
Expect(cfg.WatchDogInterval).To(Equal(1*time.Second), "env/CLI interval must win over the persisted file value")
})
})
// MITM listener address. The file is the only source — no env var
// exists — so a regression here means an admin who configured the
// listener via /api/settings loses it after a reboot, even though

View File

@@ -6,6 +6,7 @@ import (
"regexp"
"time"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/mudler/xlog"
@@ -241,12 +242,19 @@ func NewApplicationConfig(o ...AppOption) *ApplicationConfig {
Context: context.Background(),
UploadLimitMB: 15,
Debug: true,
AgentJobRetentionDays: 30, // Default: 30 days
LRUEvictionMaxRetries: 30, // Default: 30 retries
LRUEvictionRetryInterval: 1 * time.Second, // Default: 1 second
WatchDogInterval: 500 * time.Millisecond, // Default: 500ms
TracingMaxItems: 1024,
TracingMaxBodyBytes: 64 * 1024, // 64 KiB - caps each request/response body in the trace buffer
AgentJobRetentionDays: 30, // Default: 30 days
LRUEvictionMaxRetries: 30, // Default: 30 retries
LRUEvictionRetryInterval: 1 * time.Second, // Default: 1 second
// WatchDogInterval is intentionally left at the zero value here.
// The startup loader applies a persisted runtime_settings.json value
// only when the interval is still 0 (its "not set by env var"
// heuristic, matching the idle/busy timeouts); a non-zero baseline
// default would defeat that and silently revert a UI-saved Check
// Interval to the default on every restart (#10601). The effective
// 500ms default is supplied at the watchdog layer (DefaultWatchdogInterval)
// when the value is still 0.
TracingMaxItems: 1024,
TracingMaxBodyBytes: 64 * 1024, // 64 KiB - caps each request/response body in the trace buffer
AgentPool: AgentPoolConfig{
Enabled: true,
Timeout: "5m",
@@ -1097,7 +1105,7 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings {
if o.WatchDogInterval > 0 {
watchdogInterval = o.WatchDogInterval.String()
} else {
watchdogInterval = "2s" // default
watchdogInterval = model.DefaultWatchdogInterval.String() // default: 500ms
}
var lruEvictionRetryInterval string
if o.LRUEvictionRetryInterval > 0 {

View File

@@ -101,7 +101,7 @@ func (i *DiffuserImporter) Import(details Details) (gallery.ModelConfig, error)
Backend: backend,
PredictionOptions: schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{
Model: LocalModelPath(details.URI),
Model: details.URI,
},
},
Diffusers: config.Diffusers{

View File

@@ -4,24 +4,9 @@ import (
"path/filepath"
"strings"
"github.com/mudler/LocalAI/pkg/downloader"
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
)
// LocalModelPath normalizes a model URI for backends that treat the model
// field as a HuggingFace repo id or local filesystem path (mlx, mlx-vlm,
// vllm, transformers, diffusers). A "file://" import URI is reduced to the
// bare path it points at: mlx-lm and vLLM otherwise mis-read the "file://"
// scheme as a repo id and fail with "Repo id must be in the form
// 'repo_name' or 'namespace/repo_name'" (issue #7461). HuggingFace and HTTP
// URIs are returned unchanged so the existing remote-load path is untouched.
func LocalModelPath(uri string) string {
if path, ok := strings.CutPrefix(uri, downloader.LocalPrefix); ok {
return path
}
return uri
}
// HasFile returns true when any file in files has exactly the given basename.
// Directory components in file.Path are ignored — a nested
// "sub/dir/config.json" is considered a match for name = "config.json".

View File

@@ -86,21 +86,4 @@ var _ = Describe("importer helpers", func() {
Expect(importers.HasGGMLFile(files, "ggml-")).To(BeFalse())
})
})
Describe("LocalModelPath", func() {
It("strips the file:// scheme from an absolute local path", func() {
Expect(importers.LocalModelPath("file:///Users/u/.lmstudio/models/mlx-community/Qwen3-4bit")).
To(Equal("/Users/u/.lmstudio/models/mlx-community/Qwen3-4bit"))
})
It("strips the file:// scheme from a relative local path", func() {
Expect(importers.LocalModelPath("file://my-models/nvidia/Qwen3-30B-A3B-FP4")).
To(Equal("my-models/nvidia/Qwen3-30B-A3B-FP4"))
})
It("leaves HuggingFace and HTTP URIs unchanged", func() {
Expect(importers.LocalModelPath("https://huggingface.co/mlx-community/test-model")).
To(Equal("https://huggingface.co/mlx-community/test-model"))
Expect(importers.LocalModelPath("mlx-community/test-model")).
To(Equal("mlx-community/test-model"))
})
})
})

View File

@@ -87,7 +87,7 @@ func (i *MLXImporter) Import(details Details) (gallery.ModelConfig, error) {
Backend: backend,
PredictionOptions: schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{
Model: LocalModelPath(details.URI),
Model: details.URI,
},
},
TemplateConfig: config.TemplateConfig{

View File

@@ -198,24 +198,5 @@ var _ = Describe("MLXImporter", func() {
Expect(err).ToNot(HaveOccurred())
Expect(modelConfig.Name).To(Equal("model"))
})
It("should emit a bare filesystem path for a file:// local import", func() {
// Regression for #7461: a model imported from a local directory
// (e.g. LM Studio's store) must not carry the file:// scheme into
// the model field — mlx-lm rejects it as an invalid repo id.
preferences := json.RawMessage(`{"backend": "mlx"}`)
details := importers.Details{
URI: "file:///Users/u/.lmstudio/models/mlx-community/Qwen3-Coder-30B-A3B-Instruct-4bit",
Preferences: preferences,
}
modelConfig, err := importer.Import(details)
Expect(err).ToNot(HaveOccurred())
Expect(modelConfig.Name).To(Equal("Qwen3-Coder-30B-A3B-Instruct-4bit"))
Expect(modelConfig.ConfigFile).To(ContainSubstring(
"model: /Users/u/.lmstudio/models/mlx-community/Qwen3-Coder-30B-A3B-Instruct-4bit"))
Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("model: file://"))
})
})
})

View File

@@ -91,7 +91,7 @@ func (i *TransformersImporter) Import(details Details) (gallery.ModelConfig, err
Backend: backend,
PredictionOptions: schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{
Model: LocalModelPath(details.URI),
Model: details.URI,
},
},
TemplateConfig: config.TemplateConfig{

View File

@@ -81,7 +81,7 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) {
Backend: backend,
PredictionOptions: schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{
Model: LocalModelPath(details.URI),
Model: details.URI,
},
},
TemplateConfig: config.TemplateConfig{

View File

@@ -177,22 +177,5 @@ var _ = Describe("VLLMImporter", func() {
Expect(modelConfig.ConfigFile).To(ContainSubstring("known_usecases:"))
Expect(modelConfig.ConfigFile).To(ContainSubstring("- chat"))
})
It("should emit a bare filesystem path for a file:// local import", func() {
// Regression for #7461: vLLM rejects a file:// model field as an
// invalid repo id, so a locally-imported model must carry the bare
// path instead.
preferences := json.RawMessage(`{"backend": "vllm"}`)
details := Details{
URI: "file://my-models/nvidia/Qwen3-30B-A3B-FP4",
Preferences: preferences,
}
modelConfig, err := importer.Import(details)
Expect(err).ToNot(HaveOccurred())
Expect(modelConfig.ConfigFile).To(ContainSubstring("model: my-models/nvidia/Qwen3-30B-A3B-FP4"))
Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("model: file://"))
})
})
})

View File

@@ -60,10 +60,17 @@ func WithIdleTimeout(timeout time.Duration) WatchDogOption {
}
}
// WithWatchdogCheck sets the watchdog check duration
// WithWatchdogInterval sets the watchdog check interval. A non-positive
// interval is ignored so the DefaultWatchdogInterval set by
// DefaultWatchDogOptions is preserved: callers pass the raw
// ApplicationConfig value, which is 0 when neither an env var nor a
// persisted setting configured it (#10601), and a 0 interval would otherwise
// turn the watchdog loop into a busy spin.
func WithWatchdogInterval(interval time.Duration) WatchDogOption {
return func(o *WatchDogOptions) {
o.watchdogInterval = interval
if interval > 0 {
o.watchdogInterval = interval
}
}
}