diff --git a/core/http/react-ui/e2e/models-gallery.spec.js b/core/http/react-ui/e2e/models-gallery.spec.js index fdf107bfd..5d3934710 100644 --- a/core/http/react-ui/e2e/models-gallery.spec.js +++ b/core/http/react-ui/e2e/models-gallery.spec.js @@ -8,38 +8,10 @@ const MOCK_MODELS_RESPONSE = { backend: "llama-cpp", installed: false, tags: ["chat"], - // memory_bytes is omitempty server-side, so the mlx variant deliberately - // carries no key at all: the UI must render that as unknown, never 0 B. - variants: [ - { - model: "llama-model", - backend: "llama-cpp", - memory_bytes: 4 * 1024 * 1024 * 1024, - fits: true, - is_base: true, - }, - { - model: "llama-model-q8", - backend: "llama-cpp", - memory_bytes: 8 * 1024 * 1024 * 1024, - fits: true, - is_base: false, - }, - { - model: "llama-model-mlx", - backend: "mlx", - fits: true, - is_base: false, - }, - { - model: "llama-model-f16", - backend: "llama-cpp", - memory_bytes: 40 * 1024 * 1024 * 1024, - fits: false, - is_base: false, - }, - ], - auto_variant: "llama-model-q8", + // The listing carries only the declaration flag. Describing variants + // costs the server a network probe each, so the description lives + // behind /api/models/variants/:id and is fetched on demand. + has_variants: true, }, { name: "whisper-model", @@ -468,13 +440,57 @@ test.describe("Models Gallery - Empty State", () => { }); }); +// The variant description the companion endpoint returns for llama-model. +// memory_bytes is omitempty server-side, so the mlx variant deliberately +// carries no key at all: the UI must render that as unknown, never 0 B. +const MOCK_VARIANTS_RESPONSE = { + variants: [ + { + model: "llama-model", + backend: "llama-cpp", + memory_bytes: 4 * 1024 * 1024 * 1024, + fits: true, + is_base: true, + }, + { + model: "llama-model-q8", + backend: "llama-cpp", + memory_bytes: 8 * 1024 * 1024 * 1024, + fits: true, + is_base: false, + }, + { + model: "llama-model-mlx", + backend: "mlx", + fits: true, + is_base: false, + }, + { + model: "llama-model-f16", + backend: "llama-cpp", + memory_bytes: 40 * 1024 * 1024 * 1024, + fits: false, + is_base: false, + }, + ], + auto_selected: "llama-model-q8", +}; + test.describe("Models Gallery - Variant picker", () => { // installUrls records every install request so a test can assert both the // presence and the absence of the ?variant= parameter. let installUrls; + // variantUrls records every companion-endpoint request. It is what proves + // the description is fetched lazily and cached, rather than being paid for + // by every row on page load. + let variantUrls; + // Held requests let a test observe the in-flight state rather than racing it. + let releaseVariants; test.beforeEach(async ({ page }) => { installUrls = []; + variantUrls = []; + releaseVariants = null; await page.route("**/api/models*", (route) => { route.fulfill({ contentType: "application/json", @@ -489,6 +505,15 @@ test.describe("Models Gallery - Variant picker", () => { body: JSON.stringify({ jobID: "variant-install" }), }); }); + await page.route("**/api/models/variants/**", async (route) => { + variantUrls.push(route.request().url()); + if (releaseVariants) await releaseVariants; + return route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(MOCK_VARIANTS_RESPONSE), + }); + }); await page.goto("/app/models"); await expect(page.locator("th", { hasText: "Backend" })).toBeVisible({ timeout: 10_000, @@ -498,6 +523,15 @@ test.describe("Models Gallery - Variant picker", () => { const variantRow = (page) => page.locator("tr", { hasText: "llama-model" }).first(); const plainRow = (page) => page.locator("tr", { hasText: "stablediffusion-model" }).first(); + const openMenu = (page) => + variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + + test("the listing alone fetches no variant descriptions", async ({ page }) => { + // The whole point of the companion endpoint: a page load costs zero + // probes no matter how many entries declare variants. + await expect(page.locator("tbody tr").first()).toBeVisible(); + expect(variantUrls).toHaveLength(0); + }); test("an entry that declares variants shows the split-button chevron", async ({ page, @@ -517,14 +551,57 @@ test.describe("Models Gallery - Variant picker", () => { ).toHaveCount(1); }); + test("an entry without variants fetches nothing even when expanded", async ({ + page, + }) => { + await plainRow(page).click(); + await expect(page.locator('td[colspan="8"]')).toBeVisible(); + expect(variantUrls).toHaveLength(0); + }); + test("plain Install sends no variant parameter", async ({ page }) => { await plainRow(page).locator("button.btn-primary").click(); await expect.poll(() => installUrls.length).toBe(1); expect(installUrls[0]).not.toContain("variant="); }); + test("opening the menu fetches the description once and caches it", async ({ + page, + }) => { + await openMenu(page); + await expect(page.locator(".action-menu")).toBeVisible(); + await expect.poll(() => variantUrls.length).toBe(1); + expect(variantUrls[0]).toContain("/api/models/variants/llama-model"); + + // Close and reopen: the cached answer must be reused. + await page.keyboard.press("Escape"); + await openMenu(page); + await expect( + page.locator(".action-menu__item", { hasText: "llama-model-q8" }), + ).toBeVisible(); + expect(variantUrls).toHaveLength(1); + }); + + test("the menu shows a loading state while the description is in flight", async ({ + page, + }) => { + let unblock; + releaseVariants = new Promise((resolve) => { + unblock = resolve; + }); + await openMenu(page); + await expect(page.locator(".action-menu")).toContainText("Loading variants"); + unblock(); + await expect( + page.locator(".action-menu__item", { hasText: "llama-model-q8" }), + ).toBeVisible(); + await expect(page.locator(".action-menu")).not.toContainText( + "Loading variants", + ); + }); + test("the auto-selected variant is marked in the menu", async ({ page }) => { - await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + await openMenu(page); const menu = page.locator(".action-menu"); await expect(menu).toBeVisible(); const autoItem = menu.locator(".action-menu__item", { @@ -543,7 +620,7 @@ test.describe("Models Gallery - Variant picker", () => { test("a variant with no memory_bytes renders as unknown, not 0", async ({ page, }) => { - await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + await openMenu(page); const mlxItem = page.locator(".action-menu__item", { hasText: "llama-model-mlx", }); @@ -552,7 +629,7 @@ test.describe("Models Gallery - Variant picker", () => { }); test("a variant that does not fit is still selectable", async ({ page }) => { - await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + await openMenu(page); const f16 = page.locator(".action-menu__item", { hasText: "llama-model-f16", }); @@ -563,7 +640,7 @@ test.describe("Models Gallery - Variant picker", () => { test("choosing a specific variant sends ?variant= on the install", async ({ page, }) => { - await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + await openMenu(page); await page .locator(".action-menu__item", { hasText: "llama-model-mlx" }) .click(); @@ -583,5 +660,7 @@ test.describe("Models Gallery - Variant picker", () => { await expect(detail).toContainText("Base build"); await expect(detail).toContainText("Does not fit"); await expect(detail).toContainText("mlx"); + // Expanding is the second trigger point, so it pays for exactly one fetch. + expect(variantUrls).toHaveLength(1); }); }); diff --git a/core/http/react-ui/public/locales/de/models.json b/core/http/react-ui/public/locales/de/models.json index cf99a6567..9142a1b8b 100644 --- a/core/http/react-ui/public/locales/de/models.json +++ b/core/http/react-ui/public/locales/de/models.json @@ -93,6 +93,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/public/locales/en/models.json b/core/http/react-ui/public/locales/en/models.json index ac2e8b71e..9c750774f 100644 --- a/core/http/react-ui/public/locales/en/models.json +++ b/core/http/react-ui/public/locales/en/models.json @@ -118,6 +118,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/public/locales/es/models.json b/core/http/react-ui/public/locales/es/models.json index 578f5b71c..69ca19dae 100644 --- a/core/http/react-ui/public/locales/es/models.json +++ b/core/http/react-ui/public/locales/es/models.json @@ -93,6 +93,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/public/locales/id/models.json b/core/http/react-ui/public/locales/id/models.json index f66622f58..3acff3afb 100644 --- a/core/http/react-ui/public/locales/id/models.json +++ b/core/http/react-ui/public/locales/id/models.json @@ -106,6 +106,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/public/locales/it/models.json b/core/http/react-ui/public/locales/it/models.json index 6f963e514..15eca936f 100644 --- a/core/http/react-ui/public/locales/it/models.json +++ b/core/http/react-ui/public/locales/it/models.json @@ -93,6 +93,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/public/locales/zh-CN/models.json b/core/http/react-ui/public/locales/zh-CN/models.json index d25e608af..88cd49d7d 100644 --- a/core/http/react-ui/public/locales/zh-CN/models.json +++ b/core/http/react-ui/public/locales/zh-CN/models.json @@ -93,6 +93,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/src/pages/Models.jsx b/core/http/react-ui/src/pages/Models.jsx index ee5585d43..75375632b 100644 --- a/core/http/react-ui/src/pages/Models.jsx +++ b/core/http/react-ui/src/pages/Models.jsx @@ -74,6 +74,11 @@ export default function Models() { // Popover is re-anchored per row rather than one instance per row. const [variantMenuFor, setVariantMenuFor] = useState(null) const variantMenuAnchorRef = useRef(null) + // Variant descriptions, keyed by model name. The listing only tells us + // whether an entry declares any; describing them costs the server a network + // probe per variant, so we ask for one entry at a time and keep the answer + // for the rest of the page session. + const [variantData, setVariantData] = useState({}) const [fitsFilter, setFitsFilter] = useState(() => { try { return localStorage.getItem(FITS_FILTER_STORAGE_KEY) === '1' @@ -193,6 +198,21 @@ export default function Models() { } } + // Fetches an entry's variant description once. Called from the two points + // where a user actually asks to see variants: opening the split-button menu + // and expanding the detail row. An entry that declares none never gets here, + // so it issues no request at all. + const loadVariants = useCallback((id) => { + if (!id) return + setVariantData(prev => { + if (prev[id]) return prev + modelsApi.variants(id) + .then(data => setVariantData(p => ({ ...p, [id]: { loading: false, ...data } }))) + .catch(() => setVariantData(p => ({ ...p, [id]: { loading: false, variants: [] } }))) + return { ...prev, [id]: { loading: true, variants: [] } } + }) + }, []) + const handleInstall = async (modelId, variant) => { try { setInstalling(prev => new Map(prev).set(modelId, Date.now())) @@ -429,12 +449,16 @@ export default function Models() { const progress = getOperationProgress(name) const fit = fitsGpu(vramBytes) const isExpanded = expandedRow === idx - const hasVariants = model.variants?.length > 0 + const hasVariants = !!model.has_variants return ( { setExpandedRow(isExpanded ? null : idx); setExpandedFiles(false) }} + onClick={() => { + if (!isExpanded && hasVariants) loadVariants(name) + setExpandedRow(isExpanded ? null : idx) + setExpandedFiles(false) + }} style={{ cursor: 'pointer' }} > {/* Chevron */} @@ -578,7 +602,10 @@ export default function Models() { - ) - })} - - + onChoose={handleInstall} + t={t} + /> ) } +// VariantMenu is the split-button dropdown. It is one instance re-anchored to +// whichever row is active, so Escape, outside-click and focus return come from +// Popover rather than being reimplemented per row. +function VariantMenu({ anchor, model, variantData, onClose, onChoose, t }) { + const name = model ? (model.name || model.id) : null + const data = name ? variantData[name] : null + return ( + +
+ {data?.loading && ( + // The description is a round trip, so the menu says so rather than + // opening empty and looking broken. +
+ + {t('variants.loading')} +
+ )} + {(data?.variants || []).map(v => { + const isAuto = v.model === data?.auto_selected + return ( + + ) + })} +
+ + ) +} + // variantSizeLabel renders a variant footprint. memory_bytes is omitempty on // the wire, so an absent key means the probe could not determine a size; it // must never render as "0 B", which would read as "needs nothing". @@ -707,7 +756,7 @@ function DetailRow({ label, children }) { ) } -function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setExpandedFiles, t }) { +function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setExpandedFiles, variantData, t }) { const files = model.additionalFiles || model.files || [] return (
@@ -747,10 +796,17 @@ function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setE ) : null} - {model.variants?.length > 0 && ( + {variantData?.loading && ( + + + {t('variants.loading')} + + + )} + {variantData?.variants?.length > 0 && (
- {model.variants.map(v => ( + {variantData.variants.map(v => (
{v.model} {v.backend || t('variants.unknownBackend')} @@ -759,7 +815,7 @@ function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setE {v.fits ? t('variants.fits') : t('variants.doesNotFit')} {v.is_base && {t('variants.base')}} - {v.model === model.auto_variant && ( + {v.model === variantData.auto_selected && ( {t('variants.autoSelected')} diff --git a/core/http/react-ui/src/utils/api.js b/core/http/react-ui/src/utils/api.js index 3895dae3c..e2b76ba32 100644 --- a/core/http/react-ui/src/utils/api.js +++ b/core/http/react-ui/src/utils/api.js @@ -98,6 +98,9 @@ export const modelsApi = { buildUrl(API_CONFIG.endpoints.modelEstimate(id), contexts?.length ? { contexts: contexts.join(',') } : {}) ), + // Companion to estimate: the listing reports only has_variants, so the + // description is fetched per entry, on demand. + variants: (id) => fetchJSON(API_CONFIG.endpoints.modelVariants(id)), getConfig: (id) => postJSON(API_CONFIG.endpoints.modelConfig(id), {}), getConfigJson: (name) => fetchJSON(API_CONFIG.endpoints.modelConfigJson(name)), getJob: (uid) => fetchJSON(API_CONFIG.endpoints.modelJob(uid)), diff --git a/core/http/react-ui/src/utils/config.js b/core/http/react-ui/src/utils/config.js index d1bd4d335..3ddf19eb7 100644 --- a/core/http/react-ui/src/utils/config.js +++ b/core/http/react-ui/src/utils/config.js @@ -10,6 +10,7 @@ export const API_CONFIG = { installModel: (id) => `/api/models/install/${id}`, deleteModel: (id) => `/api/models/delete/${id}`, modelEstimate: (id) => `/api/models/estimate/${id}`, + modelVariants: (id) => `/api/models/variants/${id}`, modelConfig: (id) => `/api/models/config/${id}`, modelConfigJson: (name) => `/api/models/config-json/${name}`, configMetadata: '/api/models/config-metadata', diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index b30dd2561..ec5f4258e 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -413,11 +413,6 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model }) } - // Kept before the filters and pagination below narrow `models`: a - // variant references another gallery entry by name, and that entry may - // well have been filtered off this page. - allModels := models - // Get all available tags allTags := map[string]struct{}{} tags := []string{} @@ -590,24 +585,18 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model "voice_cloning": m.VoiceCloningCapability(appConfig.SystemState.Model.ModelsPath), } - // Variants are described only for the entries that declare them. - // DescribeVariants is what probes model sizes, so calling it for - // every entry would put a network round trip behind each of the - // ~1200 entries in the gallery. The HasVariants guard keeps an - // ordinary entry exactly as cheap as it was before variants - // existed; only a declaring entry pays, and only on the page it - // appears on. + // Only the cheap declaration flag, never the description itself. + // Describing a variant probes every referenced entry's weight files + // over the network, so doing it here would cost one page load + // (entries x variants) serial round trips; a client that wants the + // description asks /api/models/variants/:id for one entry at a + // time, exactly as it already does for VRAM estimates. + // + // The flag is omitted rather than sent as false so an entry that + // declares nothing stays byte-for-byte what it was before variants + // existed, and so a client never asks about it. if m.HasVariants() { - env := gallery.HostResolveEnv(c.Request().Context(), appConfig.SystemState) - view, describeErr := gallery.DescribeVariants(allModels, m, env) - if describeErr != nil { - // A malformed variant list must not blank the whole gallery - // page; that entry just renders without a picker. - xlog.Warn("could not describe model variants", "model", m.Name, "error", describeErr) - } else if view != nil { - obj["variants"] = view.Variants - obj["auto_variant"] = view.AutoSelected - } + obj["has_variants"] = true } modelsJSON = append(modelsJSON, obj) @@ -825,6 +814,53 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model return c.JSON(200, result) }, adminMiddleware) + // Returns the selectable builds of a single gallery model and which one + // auto-selection would install on this host. Companion to estimate/:id and + // for the same reason: describing variants probes each referenced entry's + // weight files over the network, so the gallery listing must stay free of + // it and the frontend fills pickers in per-entry, on demand. + // + // An entry that declares no variants is not an error; it answers with an + // empty description, so a client that asks anyway gets a well-formed reply + // rather than a 404 it has to special-case. + app.GET("/api/models/variants/:id", func(c echo.Context) error { + modelID, err := url.QueryUnescape(c.Param("id")) + if err != nil { + return c.JSON(http.StatusBadRequest, map[string]any{"error": "invalid model ID"}) + } + + models, err := gallery.AvailableGalleryModelsCached(appConfig.Galleries, appConfig.SystemState) + if err != nil { + return c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error()}) + } + + model := gallery.FindGalleryElement(models, modelID) + if model == nil { + return c.JSON(http.StatusNotFound, map[string]any{"error": "model not found"}) + } + + // An allocated slice rather than the zero value, so "nothing + // selectable" serializes as [] and a client can iterate the reply + // without first distinguishing it from null. + empty := gallery.EntryVariants{Variants: []gallery.VariantView{}} + + // The full, unpaginated list: a variant references another gallery + // entry by name and that entry need not be anywhere near this one. + env := gallery.HostResolveEnv(c.Request().Context(), appConfig.SystemState) + view, err := gallery.DescribeVariants(models, model, env) + if err != nil { + // A malformed variant list must not break the picker; the entry + // just reports nothing selectable and installs as-is. + xlog.Debug("could not describe model variants", "model", modelID, "error", err) + return c.JSON(200, empty) + } + if view == nil { + return c.JSON(200, empty) + } + + return c.JSON(200, view) + }, adminMiddleware) + app.POST("/api/models/install/:id", func(c echo.Context) error { galleryID := c.Param("id") // URL decode the gallery ID (e.g., "localai%40model" -> "localai@model") diff --git a/core/http/routes/ui_api_variants_test.go b/core/http/routes/ui_api_variants_test.go new file mode 100644 index 000000000..508a45223 --- /dev/null +++ b/core/http/routes/ui_api_variants_test.go @@ -0,0 +1,211 @@ +package routes_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "sync/atomic" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/routes" + "github.com/mudler/LocalAI/core/services/galleryop" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// These specs pin the cost contract of the gallery listing. +// +// Variant description probes every referenced entry's weight files over the +// network. Doing that inline in the listing makes one page load cost +// (entries x variants) serial round trips, so the listing must not do it at +// all: it reports only the cheap `has_variants` flag, and a client that wants +// the description asks for one entry at a time. +// +// The probe counter below is what makes that a behavioral assertion rather +// than a structural one. It counts real HTTP hits on the weight files, so it +// goes red if DescribeVariants becomes reachable from the listing path again, +// through any caller. +var _ = Describe("Model gallery variants API", func() { + var ( + app *echo.Echo + modelsDir string + weightServer *httptest.Server + indexServer *httptest.Server + probes *atomic.Int64 + appConfig *config.ApplicationConfig + ) + + BeforeEach(func() { + var err error + modelsDir, err = os.MkdirTemp("", "ui-api-variants-test-*") + Expect(err).NotTo(HaveOccurred()) + + probes = &atomic.Int64{} + // Stands in for the weight files a variant probe would range-fetch. + // Every hit is a probe; a listing that describes variants inline + // cannot avoid them. + weightServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + probes.Add(1) + w.Header().Set("Content-Length", "1048576") + w.WriteHeader(http.StatusOK) + })) + + index := fmt.Sprintf(` +- name: base-entry + description: An entry that declares variants + backend: llama-cpp + files: + - filename: base.gguf + uri: %s/base.gguf + sha256: "" + variants: + - model: big-entry +- name: big-entry + description: The alternative build + backend: llama-cpp + files: + - filename: big.gguf + uri: %s/big.gguf + sha256: "" +- name: plain-entry + description: An entry that declares nothing + backend: whisper +`, weightServer.URL, weightServer.URL) + + indexServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(index)) + })) + + systemState, err := system.GetSystemState(system.WithModelPath(modelsDir)) + Expect(err).NotTo(HaveOccurred()) + + appConfig = &config.ApplicationConfig{ + Galleries: []config.Gallery{{Name: "test", URL: indexServer.URL + "/index.yaml"}}, + SystemState: systemState, + } + + galleryService := galleryop.NewGalleryService(appConfig, nil) + app = echo.New() + routes.RegisterUIAPIRoutes(app, config.NewModelConfigLoader(modelsDir), model.NewModelLoader(systemState), appConfig, + galleryService, galleryop.NewOpCache(galleryService), &application.Application{}, + func(next echo.HandlerFunc) echo.HandlerFunc { return next }) + }) + + AfterEach(func() { + weightServer.Close() + indexServer.Close() + Expect(os.RemoveAll(modelsDir)).To(Succeed()) + }) + + get := func(path string) (int, map[string]any) { + req := httptest.NewRequest(http.MethodGet, path, nil) + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + var body map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &body)).To(Succeed()) + return rec.Code, body + } + + listing := func() []map[string]any { + code, body := get("/api/models?items=9999") + Expect(code).To(Equal(http.StatusOK)) + raw, ok := body["models"].([]any) + Expect(ok).To(BeTrue(), "listing must return a models array") + out := make([]map[string]any, 0, len(raw)) + for _, m := range raw { + out = append(out, m.(map[string]any)) + } + return out + } + + find := func(models []map[string]any, name string) map[string]any { + for _, m := range models { + if m["name"] == name { + return m + } + } + Fail("no entry named " + name + " in the listing") + return nil + } + + Context("the listing", func() { + It("issues no variant probes at all", func() { + models := listing() + Expect(models).NotTo(BeEmpty()) + + // The whole point. An entry declaring variants must cost the + // listing exactly what an entry declaring none costs it. + Expect(probes.Load()).To(BeZero(), + "the gallery listing probed variant weight files; variant description must not be reachable from the listing path") + }) + + It("omits the described variant payload", func() { + entry := find(listing(), "base-entry") + Expect(entry).NotTo(HaveKey("variants")) + Expect(entry).NotTo(HaveKey("auto_variant")) + }) + + It("reports has_variants so a client knows whether to ask", func() { + models := listing() + Expect(find(models, "base-entry")["has_variants"]).To(BeTrue()) + // An entry declaring nothing must look exactly as it did before + // variants existed, so a client never asks about it. + Expect(find(models, "plain-entry")).NotTo(HaveKey("has_variants")) + }) + }) + + Context("the companion endpoint", func() { + It("returns the description the listing used to carry", func() { + code, body := get("/api/models/variants/test@base-entry") + Expect(code).To(Equal(http.StatusOK)) + + Expect(body).To(HaveKey("auto_selected")) + variants, ok := body["variants"].([]any) + Expect(ok).To(BeTrue()) + Expect(variants).To(HaveLen(2), "the declared variant plus the base") + + byModel := map[string]map[string]any{} + for _, v := range variants { + vm := v.(map[string]any) + byModel[vm["model"].(string)] = vm + } + Expect(byModel).To(HaveKey("big-entry")) + Expect(byModel).To(HaveKey("base-entry")) + Expect(byModel["base-entry"]["is_base"]).To(BeTrue()) + Expect(byModel["big-entry"]["is_base"]).To(BeFalse()) + Expect(byModel["big-entry"]).To(HaveKey("backend")) + Expect(byModel["big-entry"]).To(HaveKey("fits")) + }) + + It("omits memory_bytes entirely when a size cannot be determined", func() { + // The weight server answers without a usable size, so the probe + // comes back unknown. An absent key is the contract: a zero would + // read as 'needs nothing'. + _, body := get("/api/models/variants/test@base-entry") + for _, v := range body["variants"].([]any) { + vm := v.(map[string]any) + if mb, present := vm["memory_bytes"]; present { + Expect(mb).NotTo(BeZero(), "memory_bytes must be omitted rather than serialized as zero") + } + } + }) + + It("returns an empty description for an entry declaring no variants", func() { + code, body := get("/api/models/variants/test@plain-entry") + Expect(code).To(Equal(http.StatusOK)) + Expect(body["variants"]).To(BeEmpty()) + Expect(body["auto_selected"]).To(BeEmpty()) + }) + + It("404s an unknown entry", func() { + code, _ := get("/api/models/variants/test@nope") + Expect(code).To(Equal(http.StatusNotFound)) + }) + }) +}) diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index b02ae336d..e8b2f26ac 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -178,18 +178,26 @@ larger ones, or both. Sizes are measured from the model's weights rather than downloaded, and cached. -The gallery listing reports what an entry offers. Entries with variants carry -two extra fields, `variants` and `auto_variant`: +The gallery listing only flags which entries offer variants, with a +`has_variants` field. It deliberately does not describe them: measuring a +variant is a network round trip per referenced build, so describing every +entry inline would make one listing request cost as many round trips as the +whole page has variants. ```bash -curl http://localhost:8080/api/models | jq '.models[] | select(.variants) | - {name, auto_variant, variants}' +curl http://localhost:8080/api/models | jq '.models[] | select(.has_variants) | .name' +``` + +Ask for the description one entry at a time, as the web UI does when you open +a model's variant menu: + +```bash +curl http://localhost:8080/api/models/variants/localai@nanbeige4.1-3b-q4 ``` ```json { - "name": "nanbeige4.1-3b-q4", - "auto_variant": "nanbeige4.1-3b-q8", + "auto_selected": "nanbeige4.1-3b-q8", "variants": [ { "model": "nanbeige4.1-3b-q8", "backend": "llama-cpp", "memory_bytes": 4187593113, "fits": true, "is_base": false }, { "model": "nanbeige4.1-3b-q4", "backend": "llama-cpp", "fits": true, "is_base": true } @@ -197,12 +205,15 @@ curl http://localhost:8080/api/models | jq '.models[] | select(.variants) | } ``` -`auto_variant` is what installing without a choice would pick right now. `fits` +`auto_selected` is what installing without a choice would pick right now. `fits` is whether auto-selection would consider that variant on this machine, and `is_base` marks the entry's own build. `memory_bytes` is omitted entirely, as on the second entry above, when the size could not be measured; read a missing `memory_bytes` as unknown rather than as a free build. +An entry that declares no variants carries no `has_variants` field and answers +this endpoint with an empty list, so a client never has to ask about it. + To install a specific one, pass its name as `variant`: ```bash