mirror of
https://github.com/mudler/LocalAI.git
synced 2026-06-12 18:58:49 -04:00
085fc53bbc
* fix(router): score classifier production-readiness Conversation trimming runs through the classifier model's chat template and trims by exact token count, sized to the model's n_batch which is now scaled to context so long probes can't crash the backend. Missing chat_message templates are a hard error at router build time. Router- facing factories (Embedder/Scorer/Reranker/TokenCounter) re-resolve ModelConfig per call so a model installed post-startup doesn't bind a stub Backend="" config and silently fall into the loader's auto- iterate path. New 'vector_store' backend trace recorded inside localVectorStore on every Search/Insert — including the backend-load-failure path that previously vanished into an xlog.Warn — with outcome tagging (hit/miss/empty_store/backend_load_error/find_error/insert_error/ok). Companion cleanup drops misleading similarity:0 and input_tokens_count:0 from non-hit and text-mode traces. Gallery local-store-development aliases to 'local-store' so the master image satisfies pkg/model.LocalStoreBackend lookups from the embedding cache. Misc: llama-cpp TokenizeString reads the correct 'prompt' JSON key (the original bug); ModelTokenize nil-guard; non-fatal mitm proxy startup; PII 'route_local' renamed to 'allow' with docs/UI in sync; model-editor footer no longer eats the edit area on small screens; several config-editor template/dropdown/section fixes. Tests: e2e router specs (casual/code-hint + long-conversation trim), vector_store trace specs, lazy-factory specs, gallery dev-alias resolution, Playwright trace badge + scroll regression. Assisted-by: Claude:claude-opus-4-7 [Claude Code] Signed-off-by: Richard Palethorpe <io@richiejp.com> * feat(backend): auto-size batch to context for embedding and rerank models Embedding and rerank models pool over the whole input in a single physical batch (n_ubatch). With batch left at the 512 default, the backend rejects longer inputs with "input is too large to process", silently capping a large-context embedder (e.g. 8k/32k) at 512 tokens. Size n_batch to the context for these single-pass usecases, mirroring the existing FLAG_SCORE behaviour; an explicit batch: still wins. Extracts EffectiveContextSize/EffectiveBatchSize from grpcModelOpts so the effective decode window has one home for other callers to reuse. Adds an e2e-aio regression test that embeds a >512-token input. The AIO embedding model is switched to nomic-embed-text-v1.5 (2048 context) because the previous granite model was capped at 512 tokens and could not exercise the larger batch. Assisted-by: claude-code:claude-opus-4-8 [Claude Code] Signed-off-by: Richard Palethorpe <io@richiejp.com> * fix(gallery): raise arch-router scoring output cap via parallel:64 Scoring decodes the whole prompt+candidate in a single llama_decode and reads one logit row per candidate token. The vendored llama.cpp server caps causal output rows at n_parallel, so the default of 1 aborts with GGML_ASSERT(n_outputs_max <= cparams.n_outputs_max) on multi-token route labels. Set options: [parallel:64] on both arch-router quant entries to lift the cap; kv_unified (the grpc-server default) keeps the full context per sequence, so this does not split the KV cache. Assisted-by: claude-code:claude-opus-4-8 [Claude Code] Signed-off-by: Richard Palethorpe <io@richiejp.com> --------- Signed-off-by: Richard Palethorpe <io@richiejp.com>
92 lines
3.4 KiB
Go
92 lines
3.4 KiB
Go
package meta
|
|
|
|
// Dynamic autocomplete provider constants (runtime lookup required).
|
|
const (
|
|
ProviderBackends = "backends"
|
|
ProviderModels = "models"
|
|
ProviderModelsChat = "models:chat"
|
|
ProviderModelsTTS = "models:tts"
|
|
ProviderModelsTranscript = "models:transcript"
|
|
ProviderModelsVAD = "models:vad"
|
|
ProviderModelsScore = "models:score"
|
|
)
|
|
|
|
// Static option lists embedded directly in field metadata.
|
|
|
|
var QuantizationOptions = []FieldOption{
|
|
{Value: "q4_0", Label: "Q4_0"},
|
|
{Value: "q4_1", Label: "Q4_1"},
|
|
{Value: "q5_0", Label: "Q5_0"},
|
|
{Value: "q5_1", Label: "Q5_1"},
|
|
{Value: "q8_0", Label: "Q8_0"},
|
|
{Value: "q2_K", Label: "Q2_K"},
|
|
{Value: "q3_K_S", Label: "Q3_K_S"},
|
|
{Value: "q3_K_M", Label: "Q3_K_M"},
|
|
{Value: "q3_K_L", Label: "Q3_K_L"},
|
|
{Value: "q4_K_S", Label: "Q4_K_S"},
|
|
{Value: "q4_K_M", Label: "Q4_K_M"},
|
|
{Value: "q5_K_S", Label: "Q5_K_S"},
|
|
{Value: "q5_K_M", Label: "Q5_K_M"},
|
|
{Value: "q6_K", Label: "Q6_K"},
|
|
}
|
|
|
|
var CacheTypeOptions = []FieldOption{
|
|
{Value: "f16", Label: "F16"},
|
|
{Value: "f32", Label: "F32"},
|
|
{Value: "q8_0", Label: "Q8_0"},
|
|
{Value: "q4_0", Label: "Q4_0"},
|
|
{Value: "q4_1", Label: "Q4_1"},
|
|
{Value: "q5_0", Label: "Q5_0"},
|
|
{Value: "q5_1", Label: "Q5_1"},
|
|
}
|
|
|
|
var DiffusersPipelineOptions = []FieldOption{
|
|
{Value: "StableDiffusionPipeline", Label: "StableDiffusionPipeline"},
|
|
{Value: "StableDiffusionImg2ImgPipeline", Label: "StableDiffusionImg2ImgPipeline"},
|
|
{Value: "StableDiffusionXLPipeline", Label: "StableDiffusionXLPipeline"},
|
|
{Value: "StableDiffusionXLImg2ImgPipeline", Label: "StableDiffusionXLImg2ImgPipeline"},
|
|
{Value: "StableDiffusionDepth2ImgPipeline", Label: "StableDiffusionDepth2ImgPipeline"},
|
|
{Value: "DiffusionPipeline", Label: "DiffusionPipeline"},
|
|
{Value: "StableVideoDiffusionPipeline", Label: "StableVideoDiffusionPipeline"},
|
|
}
|
|
|
|
// UsecaseOptions must stay in sync with GetAllModelConfigUsecases in
|
|
// core/config/model_config.go — a value missing here is silently
|
|
// inaccessible from the model editor, which is how `score` (the router
|
|
// classifier usecase) hid for an entire release.
|
|
var UsecaseOptions = []FieldOption{
|
|
{Value: "chat", Label: "Chat"},
|
|
{Value: "completion", Label: "Completion"},
|
|
{Value: "edit", Label: "Edit"},
|
|
{Value: "embeddings", Label: "Embeddings"},
|
|
{Value: "rerank", Label: "Rerank"},
|
|
{Value: "score", Label: "Score (Router Classifier)"},
|
|
{Value: "image", Label: "Image"},
|
|
{Value: "vision", Label: "Vision"},
|
|
{Value: "detection", Label: "Detection"},
|
|
{Value: "face_recognition", Label: "Face Recognition"},
|
|
{Value: "transcript", Label: "Transcript"},
|
|
{Value: "diarization", Label: "Diarization"},
|
|
{Value: "speaker_recognition", Label: "Speaker Recognition"},
|
|
{Value: "tts", Label: "TTS"},
|
|
{Value: "sound_generation", Label: "Sound Generation"},
|
|
{Value: "audio_transform", Label: "Audio Transform"},
|
|
{Value: "realtime_audio", Label: "Realtime Audio"},
|
|
{Value: "tokenize", Label: "Tokenize"},
|
|
{Value: "vad", Label: "VAD"},
|
|
{Value: "video", Label: "Video"},
|
|
}
|
|
|
|
var DiffusersSchedulerOptions = []FieldOption{
|
|
{Value: "ddim", Label: "DDIM"},
|
|
{Value: "ddpm", Label: "DDPM"},
|
|
{Value: "pndm", Label: "PNDM"},
|
|
{Value: "lms", Label: "LMS"},
|
|
{Value: "euler", Label: "Euler"},
|
|
{Value: "euler_a", Label: "Euler A"},
|
|
{Value: "dpm_multistep", Label: "DPM Multistep"},
|
|
{Value: "dpm_singlestep", Label: "DPM Singlestep"},
|
|
{Value: "heun", Label: "Heun"},
|
|
{Value: "unipc", Label: "UniPC"},
|
|
}
|