fix(hipblas): correct amdgpu.ids source package name in comment

Verified against the real rocm/dev-ubuntu-24.04:7.2.1 image with
hipblas-dev/hipblaslt-dev/rocblas-dev installed: /usr/share/libdrm/amdgpu.ids
is owned by libdrm-common, not libdrm-amdgpu1 as the comment said.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Dockerfile

@@ -171,6 +171,17 @@ RUN if [ "${BUILD_TYPE}" = "hipblas" ]; then \
     ln -s /opt/rocm-**/lib/llvm/lib/libomp.so /usr/lib/libomp.so \
     ; fi
 
+# ROCm's bundled libdrm_amdgpu is built with a hardcoded fallback lookup path
+# for the ASIC ID table (/opt/amdgpu/share/libdrm/amdgpu.ids), which only exists
+# if AMD's full amdgpu graphics/DKMS stack is installed. This compute-only image
+# doesn't have it, so hipblas/rocBLAS log "No such file or directory" on every
+# model load and can fail to identify the GPU. Point it at the equivalent file
+# Ubuntu's libdrm-common package already ships.
+RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ -f /usr/share/libdrm/amdgpu.ids ] && [ ! -e /opt/amdgpu/share/libdrm/amdgpu.ids ]; then \
+    mkdir -p /opt/amdgpu/share/libdrm && \
+    ln -s /usr/share/libdrm/amdgpu.ids /opt/amdgpu/share/libdrm/amdgpu.ids \
+    ; fi
+
 RUN expr "${BUILD_TYPE}" = intel && echo "intel" > /run/localai/capability || echo "not intel"
 
 # Cuda

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=f74a6fb87b315b2c3154166e075360e15021a61d
+IK_LLAMA_VERSION?=29431b31c89e79c10f8736e8f2742485ba1713d6
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=6f4f53f2b7da54fcdbbecaaa734337c337ad6176
+LLAMA_VERSION?=0eca4d490e591d4e93058d07540cf47278a72577
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/go/crispasr/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # CrispASR version (release tag)
 CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR
-CRISPASR_VERSION?=3b93758f9725d400eca82976f895e4cec3f31260
+CRISPASR_VERSION?=8fd9db8fec8cb5e929d23d3267ed5817794feb1a
 SO_TARGET?=libgocrispasr.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=3b6c9ca97cfcda8e68e719e6670d06379fcbe943
+STABLEDIFFUSION_GGML_VERSION?=484baa41e5e006c52dcd4addc38c830b9489745f
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/stablediffusion-ggml/cpp/gosd.cpp

@@ -798,6 +798,7 @@ void sd_img_gen_params_set_seed(sd_img_gen_params_t *params, int64_t seed) {
 int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, char *src_image, float strength, char *mask_image, char* ref_images[], int ref_images_count) {
 
     sd_image_t* results;
+    int num_results_out = 0;
 
     std::vector<int> skip_layers = {7, 8, 9};
 
@@ -994,10 +995,14 @@ int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, cha
             sd_ctx_params_to_str(&ctx_params),
             sd_img_gen_params_to_str(p));
 
-    results = generate_image(sd_c, p);
+    bool gen_ok = generate_image(sd_c, p, &results, &num_results_out);
 
     std::free(p);
 
+    if (!gen_ok || num_results_out == 0) {
+        results = NULL;
+    }
+
     if (results == NULL) {
         fprintf (stderr, "NO results\n");
         if (input_image_buffer) free(input_image_buffer);

backend/go/whisper/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=0ae02cdb2c7317b50991367c165736ce42ed96ac
+WHISPER_CPP_VERSION?=0874de3e8e8e48361dba85c7fe6d176f008bf158
 SO_TARGET?=libgowhisper.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/python/vllm/install.sh

@@ -104,7 +104,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
     # can rewrite it. Darwin therefore follows vllm-metal and can lag the Linux
     # vllm pin (requirements-cublas13-after.txt, bumped independently against
     # vllm/vllm) until vllm-metal supports a newer vLLM.
-    VLLM_METAL_VERSION="v0.3.0.dev20260628073537"
+    VLLM_METAL_VERSION="v0.3.0.dev20260630095652"
 
     # The coupled vLLM source version is whatever this vllm-metal release builds
     # against -- it declares it in its own installer as `vllm_v=`. Derive it from

backend/python/vllm/requirements-cublas13-after.txt

@@ -3,8 +3,8 @@
 # on a cu130 host. Pull the cu130-flavoured wheel from vLLM's per-tag index
 # instead — the cublas13 case in install.sh adds --index-strategy=unsafe-best-match
 # so uv consults this index alongside PyPI.
---extra-index-url https://wheels.vllm.ai/0.23.0/cu130
+--extra-index-url https://wheels.vllm.ai/0.24.0/cu130
 # VERSION COUPLING: darwin/Apple-Silicon builds use vllm-metal (see install.sh),
 # which pins this exact vLLM version. Bumping vllm here means coordinating with a
 # vllm-metal release that supports the new version, or macOS/Metal builds break.
-vllm==0.23.0
+vllm==0.24.0

core/application/distributed.go

@@ -356,12 +356,6 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade
 		PrefixConfig:     prefixCfg,
 		Pressure:         pressure,
 		SharedModels:     cfg.Distributed.SharedModels,
-		// Cap how long a cold load may hold the per-model advisory lock: the
-		// configured backend.install deadline plus a margin for file staging and
-		// the remote LoadModel. Derived from the install timeout so raising it
-		// (for slow links pulling multi-GB images) widens the ceiling too,
-		// instead of letting the static default cut a legitimately slow load.
-		ModelLoadCeiling: cfg.Distributed.BackendInstallTimeoutOrDefault() + 10*time.Minute,
 	})
 
 	// Wire staging-progress broadcasting so file-staging shows up on every

core/services/advisorylock/advisorylock.go

@@ -6,39 +6,10 @@ import (
 	"hash/fnv"
 	"strings"
 	"sync"
-	"time"
 
 	"gorm.io/gorm"
 )
 
-// advisoryLockWaitBackstop bounds, server-side, how long we will wait to
-// acquire a blocking advisory lock when the caller's context carries no
-// deadline (e.g. a startup schema migration using context.Background()). It
-// only exists so such a caller cannot hang forever behind a holder whose
-// session never releases the lock; it is far longer than any legitimate
-// guarded section. A var (not const) so tests can shrink it.
-var advisoryLockWaitBackstop = 30 * time.Minute
-
-// advisoryLockTimeoutMargin is added to a context's remaining budget when
-// deriving the server-side lock_timeout, so the Go context's own (cleaner)
-// cancellation fires first and the server bound is only ever a backstop.
-const advisoryLockTimeoutMargin = 30 * time.Second
-
-// advisoryLockWaitBudget returns the server-side lock_timeout to use for a
-// blocking acquire: the caller context's remaining time plus a margin (so the
-// Go context still governs), or the backstop when the context has no deadline.
-// Never returns zero - "wait forever" must not be possible.
-func advisoryLockWaitBudget(ctx context.Context) time.Duration {
-	if dl, ok := ctx.Deadline(); ok {
-		budget := time.Until(dl) + advisoryLockTimeoutMargin
-		if budget < time.Second {
-			budget = time.Second
-		}
-		return budget
-	}
-	return advisoryLockWaitBackstop
-}
-
 // localLocks holds one buffered channel (capacity 1) per lock key, used as an
 // in-process mutex for non-PostgreSQL dialects (SQLite). A SQLite auth DB is
 // effectively single-process, so serializing guarded sections within this
@@ -159,27 +130,6 @@ func WithLockCtx(ctx context.Context, db *gorm.DB, key int64, fn func() error) e
 	}
 	defer conn.Close()
 
-	// Override any deployment-wide lock_timeout on this dedicated connection.
-	// Operators commonly set a short global lock_timeout (on the role or
-	// database) to bound ordinary row-lock waits. Applied to the blocking
-	// pg_advisory_lock below, it aborts the wait with SQLSTATE 55P03 and turns
-	// LocalAI's intentional cross-replica "wait your turn, then re-check"
-	// coordination into a hard error for the caller (e.g. a chat request that
-	// just wanted to reuse a model another replica is loading).
-	//
-	// We do NOT disable it outright (lock_timeout = 0 would wait forever, which
-	// is unsafe for the schema-migration callers that pass context.Background()).
-	// Instead we set a bound derived from the caller's context: its remaining
-	// budget plus a margin so the Go context's cancellation wins with a clean
-	// error, or a finite backstop when the context has no deadline.
-	waitBudget := advisoryLockWaitBudget(ctx)
-	if _, err := conn.ExecContext(ctx,
-		fmt.Sprintf("SET lock_timeout = %d", waitBudget.Milliseconds())); err != nil {
-		return fmt.Errorf("advisorylock: setting lock_timeout: %w", err)
-	}
-	// Restore the session default before this pooled connection is reused.
-	defer func() { _, _ = conn.ExecContext(context.Background(), "RESET lock_timeout") }()
-
 	if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", key); err != nil {
 		return fmt.Errorf("advisorylock: acquiring lock %d: %w", key, err)
 	}

core/services/advisorylock/advisorylock_test.go

@@ -158,87 +158,6 @@ var _ = Describe("AdvisoryLock", func() {
 			Expect(err).To(HaveOccurred())
 		})
 
-		It("waits out a short server-side lock_timeout instead of failing with 55P03", func() {
-			const lockKey int64 = 703
-
-			// Reproduce the production deployment that triggered this: a short
-			// global lock_timeout set on the database. Without the fix, a waiter
-			// blocked on pg_advisory_lock() is aborted by the server after this
-			// window and surfaces SQLSTATE 55P03 ("canceling statement due to
-			// lock timeout") to the caller instead of waiting for its turn.
-			Expect(db.Exec("ALTER DATABASE testdb SET lock_timeout = '300ms'").Error).ToNot(HaveOccurred())
-			sqlDB, err := db.DB()
-			Expect(err).ToNot(HaveOccurred())
-			// Drop pooled connections so subsequent ones reconnect and inherit
-			// the new database-level lock_timeout default.
-			sqlDB.SetMaxIdleConns(0)
-
-			holding := make(chan struct{})
-			released := make(chan struct{})
-			go func() {
-				defer GinkgoRecover()
-				herr := WithLockCtx(context.Background(), db, lockKey, func() error {
-					close(holding)
-					// Hold well past the 300ms server lock_timeout.
-					time.Sleep(1 * time.Second)
-					return nil
-				})
-				Expect(herr).ToNot(HaveOccurred())
-				close(released)
-			}()
-
-			<-holding // ensure the holder owns the lock before we contend
-
-			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-			defer cancel()
-			executed := false
-			start := time.Now()
-			werr := WithLockCtx(ctx, db, lockKey, func() error {
-				executed = true
-				return nil
-			})
-			Expect(werr).ToNot(HaveOccurred(),
-				"waiter should wait out the in-progress hold, not fail with lock_timeout (55P03)")
-			Expect(executed).To(BeTrue())
-			Expect(time.Since(start)).To(BeNumerically(">=", 400*time.Millisecond),
-				"waiter should have actually waited for the holder to release")
-			<-released
-		})
-
-		It("bounds a deadline-less waiter with the backstop instead of waiting forever", func() {
-			const lockKey int64 = 704
-
-			// A caller with no context deadline (e.g. startup schema migration
-			// passing context.Background()) must not hang forever if the holder
-			// never releases. Shrink the backstop so the test is fast.
-			origBackstop := advisoryLockWaitBackstop
-			advisoryLockWaitBackstop = 500 * time.Millisecond
-			DeferCleanup(func() { advisoryLockWaitBackstop = origBackstop })
-
-			holding := make(chan struct{})
-			release := make(chan struct{})
-			go func() {
-				defer GinkgoRecover()
-				_ = WithLockCtx(context.Background(), db, lockKey, func() error {
-					close(holding)
-					<-release // hold until the test releases us
-					return nil
-				})
-			}()
-			defer close(release)
-
-			<-holding
-
-			start := time.Now()
-			err := WithLockCtx(context.Background(), db, lockKey, func() error {
-				Fail("waiter should not have acquired the still-held lock")
-				return nil
-			})
-			Expect(err).To(HaveOccurred(), "deadline-less waiter should give up at the backstop, not hang")
-			Expect(time.Since(start)).To(BeNumerically("<", 5*time.Second),
-				"backstop must cap the wait well under the test timeout")
-		})
-
 		It("serializes concurrent WithLockCtx on same key", func() {
 			const lockKey int64 = 702
 

core/services/nodes/router.go

@@ -68,13 +68,6 @@ type SmartRouterOptions struct {
 	// the absolute model paths untouched so the worker loads them directly from
 	// the shared volume (#10556). See config.DistributedConfig.SharedModels.
 	SharedModels bool
-	// ModelLoadCeiling is the hard upper bound on how long a single cold-load
-	// attempt (node selection -> backend install -> file staging -> LoadModel)
-	// may run while holding the per-model advisory lock. It backstops every
-	// sub-step's own timeout so a wedged worker can never pin the lock - and
-	// every other replica's request for that model - indefinitely. Zero selects
-	// defaultModelLoadCeiling.
-	ModelLoadCeiling time.Duration
 }
 
 // SmartRouter routes inference requests to the best available backend node.
@@ -108,18 +101,8 @@ type SmartRouter struct {
 	// sharedModels skips file staging when all nodes mount the same models
 	// directory at the same path (see SmartRouterOptions.SharedModels).
 	sharedModels bool
-	// modelLoadCeiling bounds how long a cold load may hold the per-model
-	// advisory lock (see SmartRouterOptions.ModelLoadCeiling).
-	modelLoadCeiling time.Duration
 }
 
-// defaultModelLoadCeiling is the fallback hold ceiling for a cold model load.
-// It must comfortably exceed the slowest legitimate load - a multi-GB backend
-// install (DefaultBackendInstallTimeout, 15m) plus staging and the remote
-// LoadModel (5m) - so it never cuts a real load short; it only ever fires when
-// a step is genuinely wedged (e.g. a worker that died mid-install).
-const defaultModelLoadCeiling = 25 * time.Minute
-
 // probeCacheTTL is how long a successful gRPC HealthCheck on a backend is
 // trusted before the next request re-probes. Matches healthCheckTTL in
 // pkg/model/model.go so the single-process and distributed paths share a
@@ -134,10 +117,6 @@ func NewSmartRouter(registry ModelRouter, opts SmartRouterOptions) *SmartRouter
 	if factory == nil {
 		factory = &tokenClientFactory{token: opts.AuthToken}
 	}
-	ceiling := opts.ModelLoadCeiling
-	if ceiling <= 0 {
-		ceiling = defaultModelLoadCeiling
-	}
 	return &SmartRouter{
 		registry:         registry,
 		unloader:         opts.Unloader,
@@ -152,7 +131,6 @@ func NewSmartRouter(registry ModelRouter, opts SmartRouterOptions) *SmartRouter
 		prefixConfig:     opts.PrefixConfig,
 		pressure:         opts.Pressure,
 		sharedModels:     opts.SharedModels,
-		modelLoadCeiling: ceiling,
 	}
 }
 
@@ -405,19 +383,11 @@ func (r *SmartRouter) Route(ctx context.Context, modelID, modelName, backendType
 	// the request context. If staging were bound to it, the multi-GB upload
 	// aborts with "context canceled" mid-transfer and large models can never
 	// finish staging (the model-load outage). WithoutCancel keeps the request's
-	// values (prefix chain, etc.) but drops its cancellation/deadline.
-	//
-	// Detaching from the caller is necessary, but it must not be unbounded: the
-	// load runs while holding the per-model advisory lock, and a worker that
-	// dies mid-install (its backend.install never replies) would otherwise pin
-	// that lock (and every other replica's request for the same model) until
-	// the NATS install deadline alone expires. Re-impose a single hard ceiling
-	// over the whole sequence so the lock is always released in bounded time,
-	// even if a sub-step wedges. Each long step still has its own (tighter)
-	// bound; this only backstops them. The per-model advisory lock below
-	// de-dupes concurrent loaders across replicas.
-	loadCtx, cancelLoad := context.WithTimeout(context.WithoutCancel(ctx), r.modelLoadCeiling)
-	defer cancelLoad()
+	// values (prefix chain, etc.) but drops its cancellation/deadline. Each
+	// long step still has its own bound (the file stager's resume budget,
+	// LoadModel's 5m timeout), and the per-model advisory lock below de-dupes
+	// concurrent loaders across replicas.
+	loadCtx := context.WithoutCancel(ctx)
 	loadModel := func(ctx context.Context) (*RouteResult, error) {
 		// Re-check after acquiring lock — another request may have loaded it
 		node, nm, err := r.registry.FindAndLockNodeWithModel(ctx, trackingKey, candidateNodeIDs, pref)
@@ -946,14 +916,7 @@ func (r *SmartRouter) installBackendOnNode(ctx context.Context, node *BackendNod
 	}
 
 	key := fmt.Sprintf("%s|%s|%s|%d", node.ID, backendType, modelID, replicaIndex)
-	// DoChan rather than Do so this wait honors ctx cancellation. InstallBackend
-	// blocks for its full NATS deadline (15m by default) when a worker accepts
-	// the request but never replies (e.g. it died mid-install). Without ctx
-	// awareness the caller (holding the per-model advisory lock) would sit there
-	// the whole time; here a cancelled ctx (typically the model-load ceiling)
-	// frees the caller promptly. The shared install keeps running in the
-	// background and still coalesces other callers via singleflight.
-	resCh := r.installFlight.DoChan(key, func() (any, error) {
+	v, err, _ := r.installFlight.Do(key, func() (any, error) {
 		reply, err := r.unloader.InstallBackend(node.ID, backendType, modelID, r.galleriesJSON, "", "", "", replicaIndex, "", nil)
 		if err != nil {
 			return "", err
@@ -968,15 +931,10 @@ func (r *SmartRouter) installBackendOnNode(ctx context.Context, node *BackendNod
 		}
 		return addr, nil
 	})
-	select {
-	case <-ctx.Done():
-		return "", ctx.Err()
-	case res := <-resCh:
-		if res.Err != nil {
-			return "", res.Err
-		}
-		return res.Val.(string), nil
+	if err != nil {
+		return "", err
 	}
+	return v.(string), nil
 }
 
 func (r *SmartRouter) buildClientForAddr(node *BackendNode, addr string, parallel bool) grpc.Backend {

core/services/nodes/router_test.go

@@ -493,44 +493,6 @@ var _ = Describe("SmartRouter", func() {
 				Expect(result.Node.ID).To(Equal("n3"))
 			})
 		})
-
-		Context("worker wedges mid-install (dead node holding the lock)", func() {
-			It("aborts the load at the ModelLoadCeiling instead of blocking forever", func() {
-				// Simulate the production incident: the chosen worker accepts the
-				// backend.install but never replies (it died), so InstallBackend
-				// would otherwise block for its full NATS deadline (15m by
-				// default) while pinning the per-model advisory lock. Route must
-				// give up at the ceiling so the lock is released promptly.
-				reg.findAndLockErr = errors.New("not found")
-				reg.findIdleNode = &BackendNode{ID: "n4", Name: "dead-node", Address: "10.0.0.4:50051"}
-
-				block := make(chan struct{})
-				defer close(block) // let the background install goroutine drain at test end
-				unloader.installHook = func() { <-block }
-
-				router := NewSmartRouter(reg, SmartRouterOptions{
-					Unloader:         unloader,
-					ClientFactory:    factory,
-					ModelLoadCeiling: 200 * time.Millisecond,
-				})
-
-				done := make(chan error, 1)
-				start := time.Now()
-				go func() {
-					defer GinkgoRecover()
-					_, err := router.Route(context.Background(), "wedged-model",
-						"models/wedged.gguf", "llama-cpp",
-						&pb.ModelOptions{Model: "models/wedged.gguf"}, false)
-					done <- err
-				}()
-
-				var routeErr error
-				Eventually(done, 5*time.Second).Should(Receive(&routeErr),
-					"Route must not block on a wedged install past the ceiling")
-				Expect(routeErr).To(HaveOccurred())
-				Expect(time.Since(start)).To(BeNumerically("<", 5*time.Second))
-			})
-		})
 	})
 
 	Describe("scheduleNewModel (mock-based, via Route)", func() {

docs/data/version.json

@@ -1,3 +1,3 @@
 {
-  "version": "v4.5.5"
+  "version": "v4.5.6"
 }