mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-31 02:18:50 -04:00
backend(audio-cpp): load models and cache sessions per task
Loads one ILoadedVoiceModel and creates an IVoiceTaskSession lazily per (task, mode), so the same model serves both the unary and streaming RPCs. LoadModel derives the family from GGUF metadata or an explicit option and fails with INVALID_ARGUMENT otherwise, so a failed load is a gRPC error the backend probe can see. audiocpp_backend::Task mirrors engine::runtime::VoiceTaskKind positionally, and drift there is silent: every unit still compiles and every test still passes while the backend runs a different task. Two mechanisms pin it. The static_asserts in loaded_model.cpp catch an insertion or a reorder, and -Werror=switch on that one file turns an appended upstream enumerator into a build failure rather than a warning in a 600 file log. Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
committed by
localai-org-maint-bot
localai-org-maint-bot
parent
7c3ce73017
commit
487a2a3dc8
@@ -88,11 +88,26 @@ add_executable(${TARGET}
|
||||
grpc-server.cpp
|
||||
model_options.cpp
|
||||
capability_routing.cpp
|
||||
family_gate.cpp
|
||||
loaded_model.cpp
|
||||
audio_units.cpp
|
||||
transcript_assembly.cpp
|
||||
inference_lane.cpp
|
||||
)
|
||||
|
||||
# loaded_model.cpp mirrors engine::runtime::VoiceTaskKind onto its own Task enum.
|
||||
# Its static_asserts catch an insertion or a reorder, but an enumerator APPENDED
|
||||
# after the last one shifts no value, so no assertion can see it. What does see
|
||||
# it is from_engine_task's switch over the engine enum, which carries no
|
||||
# `default:` label. -Wswitch is only a warning by default, and a warning in a
|
||||
# 600-file build log is a warning nobody reads, so promote it here: this is the
|
||||
# difference between a build failure and a backend that silently runs the wrong
|
||||
# task.
|
||||
if(NOT MSVC)
|
||||
set_source_files_properties(loaded_model.cpp PROPERTIES
|
||||
COMPILE_OPTIONS "-Werror=switch")
|
||||
endif()
|
||||
|
||||
target_include_directories(${TARGET} PRIVATE
|
||||
"${AUDIO_CPP_DIR}/include"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
|
||||
@@ -5,12 +5,17 @@
|
||||
// src/ or tests/ is used: those are application internals, they are where
|
||||
// upstream expects churn, and upstream is Apache-2.0 while LocalAI is MIT.
|
||||
//
|
||||
// This commit is the bind/listen/Health/Status skeleton. Model loading and the
|
||||
// audio RPCs land in later commits.
|
||||
// This commit adds LoadModel/Free/Status over one audio.cpp model. The audio
|
||||
// RPCs themselves land in later commits.
|
||||
|
||||
#include "backend.pb.h"
|
||||
#include "backend.grpc.pb.h"
|
||||
|
||||
#include "capability_routing.h"
|
||||
#include "inference_lane.h"
|
||||
#include "loaded_model.h"
|
||||
#include "model_options.h"
|
||||
|
||||
#include <grpcpp/grpcpp.h>
|
||||
#include <grpcpp/server.h>
|
||||
#include <grpcpp/server_builder.h>
|
||||
@@ -20,9 +25,13 @@
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstdlib>
|
||||
#include <exception>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
using grpc::Server;
|
||||
using grpc::ServerBuilder;
|
||||
@@ -35,6 +44,30 @@ namespace {
|
||||
|
||||
std::atomic<Server *> g_server{nullptr};
|
||||
|
||||
// One model per process, matching every other LocalAI backend. The mutex guards
|
||||
// the pointer itself, not the model: swapping or dropping it races with the
|
||||
// handlers that read it, whereas the model's own concurrency is the inference
|
||||
// lane's job.
|
||||
std::mutex g_model_mu;
|
||||
std::unique_ptr<audiocpp_backend::LoadedModel> g_model;
|
||||
|
||||
// Maps a thrown exception onto the gRPC status the client should see.
|
||||
GStatus to_status(const std::exception &err) {
|
||||
if (dynamic_cast<const audiocpp_backend::ConfigError *>(&err) != nullptr) {
|
||||
return GStatus(grpc::StatusCode::INVALID_ARGUMENT, err.what());
|
||||
}
|
||||
if (dynamic_cast<const audiocpp_backend::CapabilityError *>(&err) != nullptr) {
|
||||
return GStatus(grpc::StatusCode::UNIMPLEMENTED, err.what());
|
||||
}
|
||||
// The lane was busy and this caller ran out of patience. UNAVAILABLE, not
|
||||
// RESOURCE_EXHAUSTED: the request is fine and retrying later is the right
|
||||
// response, which is what UNAVAILABLE tells a client.
|
||||
if (dynamic_cast<const audiocpp_backend::LaneUnavailable *>(&err) != nullptr) {
|
||||
return GStatus(grpc::StatusCode::UNAVAILABLE, err.what());
|
||||
}
|
||||
return GStatus(grpc::StatusCode::INTERNAL, err.what());
|
||||
}
|
||||
|
||||
class AudioCppBackend final : public backend::Backend::Service {
|
||||
public:
|
||||
GStatus Health(ServerContext *, const backend::HealthMessage *,
|
||||
@@ -45,7 +78,75 @@ public:
|
||||
|
||||
GStatus Status(ServerContext *, const backend::HealthMessage *,
|
||||
backend::StatusResponse *response) override {
|
||||
response->set_state(backend::StatusResponse::UNINITIALIZED);
|
||||
std::lock_guard<std::mutex> lock(g_model_mu);
|
||||
response->set_state(g_model ? backend::StatusResponse::READY
|
||||
: backend::StatusResponse::UNINITIALIZED);
|
||||
return GStatus::OK;
|
||||
}
|
||||
|
||||
GStatus LoadModel(ServerContext *, const backend::ModelOptions *request,
|
||||
backend::Result *result) override {
|
||||
try {
|
||||
std::vector<std::string> entries(request->options().begin(),
|
||||
request->options().end());
|
||||
auto parsed = audiocpp_backend::parse_model_options(entries);
|
||||
if (!parsed.error.empty()) {
|
||||
throw audiocpp_backend::ConfigError(parsed.error);
|
||||
}
|
||||
// ModelOptions.Threads is the model YAML's threads setting; the
|
||||
// explicit threads: option wins when both are set.
|
||||
if (parsed.options.threads == 0 && request->threads() > 0) {
|
||||
parsed.options.threads = static_cast<int>(request->threads());
|
||||
}
|
||||
// MainGPU carries the device index for GPU backends.
|
||||
if (parsed.options.device == 0 && !request->maingpu().empty()) {
|
||||
parsed.options.device = std::atoi(request->maingpu().c_str());
|
||||
}
|
||||
|
||||
const std::string path = audiocpp_backend::resolve_model_path(
|
||||
request->modelpath(), request->modelfile(), request->model());
|
||||
|
||||
auto loaded = std::make_unique<audiocpp_backend::LoadedModel>(
|
||||
path, parsed.options);
|
||||
|
||||
// Read everything the reply and the log line need while this scope
|
||||
// still owns the model. After the move, `loaded` is null and the
|
||||
// global is only safe to touch under the mutex.
|
||||
const std::string family = loaded->family();
|
||||
const std::string variant = loaded->variant();
|
||||
const std::string capabilities =
|
||||
audiocpp_backend::describe_capabilities(loaded->capabilities());
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(g_model_mu);
|
||||
g_model = std::move(loaded);
|
||||
}
|
||||
|
||||
// Logged once at load time so an operator can see what the model
|
||||
// can do without making a request. ModelMetadata is deliberately
|
||||
// not implemented: its response message is chat-template oriented
|
||||
// and has no field for family, variant, languages or capabilities,
|
||||
// so there is nothing truthful to put in it.
|
||||
std::cerr << "audio-cpp: loaded family '" << family << "' variant '"
|
||||
<< variant << "' capabilities: " << capabilities << "\n";
|
||||
|
||||
result->set_success(true);
|
||||
result->set_message("loaded audio.cpp family " + family);
|
||||
return GStatus::OK;
|
||||
} catch (const std::exception &err) {
|
||||
result->set_success(false);
|
||||
result->set_message(err.what());
|
||||
// A failed load must also be a gRPC error, or the model loader's
|
||||
// backend probe treats this backend as having accepted the model.
|
||||
return to_status(err);
|
||||
}
|
||||
}
|
||||
|
||||
GStatus Free(ServerContext *, const backend::HealthMessage *,
|
||||
backend::Result *result) override {
|
||||
std::lock_guard<std::mutex> lock(g_model_mu);
|
||||
// Destroying LoadedModel releases its sessions first, then the model.
|
||||
g_model.reset();
|
||||
result->set_success(true);
|
||||
return GStatus::OK;
|
||||
}
|
||||
};
|
||||
|
||||
404
backend/cpp/audio-cpp/loaded_model.cpp
Normal file
404
backend/cpp/audio-cpp/loaded_model.cpp
Normal file
@@ -0,0 +1,404 @@
|
||||
#include "loaded_model.h"
|
||||
|
||||
#include "family_gate.h"
|
||||
|
||||
#include "engine/framework/assets/tensor_source.h"
|
||||
|
||||
#include <filesystem>
|
||||
#include <utility>
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#include <mach-o/dyld.h>
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
#elif defined(__linux__)
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace audiocpp_backend {
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Enum coupling
|
||||
//
|
||||
// audiocpp_backend::Task mirrors engine::runtime::VoiceTaskKind positionally so
|
||||
// capability_routing can stay stdlib-only and testable without an audio.cpp
|
||||
// checkout. Nothing about that mirroring is enforced by the type system, and a
|
||||
// drift is silent in the worst possible way: every unit still compiles, every
|
||||
// test still passes, and the backend runs a different task than the one the
|
||||
// caller asked for.
|
||||
//
|
||||
// Two mechanisms pin it, and both are needed because they catch different edits:
|
||||
//
|
||||
// 1. The assertions below pin every enumerator's value on both sides. An
|
||||
// insertion or a reorder anywhere before the last member shifts the values
|
||||
// after it and fails the build here.
|
||||
// 2. An enumerator APPENDED after the last one shifts nothing, so no value
|
||||
// assertion can see it. What sees it is the switch in from_engine_task,
|
||||
// which covers the engine enum with no `default:` label. CMakeLists.txt
|
||||
// compiles this file with -Werror=switch so that omission is an error
|
||||
// rather than a warning nobody reads.
|
||||
//
|
||||
// Neither mechanism catches a pure RENAME of an upstream enumerator, but that
|
||||
// does not need catching: the switch stops naming an enumerator that exists and
|
||||
// the build fails on its own.
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int kEngine(engine::runtime::VoiceTaskKind kind) {
|
||||
return static_cast<int>(kind);
|
||||
}
|
||||
constexpr int kMirror(Task task) { return static_cast<int>(task); }
|
||||
|
||||
} // namespace
|
||||
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::Vad) == 0, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::Asr) == 1, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::Diarization) == 2, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::SourceSeparation) == 3, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::AudioGeneration) == 4, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::Tts) == 5, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::VoiceCloning) == 6, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::VoiceConversion) == 7, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::SpeechToSpeech) == 8, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::Alignment) == 9, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::VoiceDesign) == 10, "VoiceTaskKind drifted");
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::SpeakerRecognition) == 11, "VoiceTaskKind drifted");
|
||||
// The last member. Pinning it pins the member count too, as long as the
|
||||
// enumerators stay contiguous and unassigned, which upstream's declaration is.
|
||||
static_assert(kEngine(engine::runtime::VoiceTaskKind::Svc) == 12,
|
||||
"engine::runtime::VoiceTaskKind gained, lost or reordered a member. "
|
||||
"audiocpp_backend::Task mirrors it positionally: update capability_routing.h, "
|
||||
"to_engine_task and from_engine_task together, then move this pin.");
|
||||
|
||||
static_assert(kMirror(Task::Vad) == 0, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::Asr) == 1, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::Diarization) == 2, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::SourceSeparation) == 3, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::AudioGeneration) == 4, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::Tts) == 5, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::VoiceCloning) == 6, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::VoiceConversion) == 7, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::SpeechToSpeech) == 8, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::Alignment) == 9, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::VoiceDesign) == 10, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::SpeakerRecognition) == 11, "Task drifted from VoiceTaskKind");
|
||||
static_assert(kMirror(Task::Svc) == 12, "Task drifted from VoiceTaskKind");
|
||||
|
||||
static_assert(static_cast<int>(engine::runtime::RunMode::Offline) == 0, "RunMode drifted");
|
||||
static_assert(static_cast<int>(engine::runtime::RunMode::Streaming) == 1,
|
||||
"engine::runtime::RunMode gained, lost or reordered a member. "
|
||||
"audiocpp_backend::Mode mirrors it positionally.");
|
||||
static_assert(static_cast<int>(Mode::Offline) == 0, "Mode drifted from RunMode");
|
||||
static_assert(static_cast<int>(Mode::Streaming) == 1, "Mode drifted from RunMode");
|
||||
|
||||
namespace {
|
||||
|
||||
engine::core::BackendType parse_backend_type(const std::string &value) {
|
||||
if (value == "cuda") {
|
||||
return engine::core::BackendType::Cuda;
|
||||
}
|
||||
if (value == "vulkan") {
|
||||
return engine::core::BackendType::Vulkan;
|
||||
}
|
||||
if (value == "metal") {
|
||||
return engine::core::BackendType::Metal;
|
||||
}
|
||||
if (value == "best") {
|
||||
return engine::core::BackendType::BestAvailable;
|
||||
}
|
||||
if (value == "cpu" || value.empty()) {
|
||||
return engine::core::BackendType::Cpu;
|
||||
}
|
||||
throw ConfigError("audio-cpp: unknown backend option '" + value +
|
||||
"'. Known backends: cpu, cuda, vulkan, metal, best");
|
||||
}
|
||||
|
||||
std::filesystem::path executable_directory() {
|
||||
#if defined(__APPLE__)
|
||||
std::uint32_t size = 0;
|
||||
_NSGetExecutablePath(nullptr, &size);
|
||||
std::vector<char> buffer(size + 1, '\0');
|
||||
if (_NSGetExecutablePath(buffer.data(), &size) != 0) {
|
||||
return std::filesystem::current_path();
|
||||
}
|
||||
return std::filesystem::path(buffer.data()).parent_path();
|
||||
#elif defined(__linux__)
|
||||
std::error_code ec;
|
||||
const auto self = std::filesystem::read_symlink("/proc/self/exe", ec);
|
||||
if (ec) {
|
||||
return std::filesystem::current_path();
|
||||
}
|
||||
return self.parent_path();
|
||||
#else
|
||||
return std::filesystem::current_path();
|
||||
#endif
|
||||
}
|
||||
|
||||
// Runs the load gate. Throws ConfigError rather than returning a decision,
|
||||
// because the only caller is a delegating constructor whose member initializer
|
||||
// list has nowhere to put a failure.
|
||||
std::string require_family(const std::string &resolved_path,
|
||||
const ModelOptions &options) {
|
||||
const std::filesystem::path path(resolved_path);
|
||||
std::error_code ec;
|
||||
if (!std::filesystem::exists(path, ec)) {
|
||||
throw ConfigError("audio-cpp: model path does not exist: " + resolved_path);
|
||||
}
|
||||
|
||||
const bool is_gguf = path_looks_like_gguf(resolved_path);
|
||||
// Only a GGUF is asked for embedded metadata. A directory has no single
|
||||
// file to read it from, and probing one would make the gate's refusal
|
||||
// depend on which file happened to be inside.
|
||||
const std::string embedded =
|
||||
is_gguf ? read_gguf_family(resolved_path) : std::string();
|
||||
|
||||
const FamilyDecision decision =
|
||||
decide_family(is_gguf, embedded, options.family);
|
||||
if (!decision.ok) {
|
||||
throw ConfigError(decision.error);
|
||||
}
|
||||
return decision.family;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
engine::runtime::VoiceTaskKind to_engine_task(Task task) {
|
||||
using K = engine::runtime::VoiceTaskKind;
|
||||
// An explicit switch, never a cast: a cast would keep compiling through
|
||||
// exactly the drift the assertions above exist to catch.
|
||||
switch (task) {
|
||||
case Task::Vad: return K::Vad;
|
||||
case Task::Asr: return K::Asr;
|
||||
case Task::Diarization: return K::Diarization;
|
||||
case Task::SourceSeparation: return K::SourceSeparation;
|
||||
case Task::AudioGeneration: return K::AudioGeneration;
|
||||
case Task::Tts: return K::Tts;
|
||||
case Task::VoiceCloning: return K::VoiceCloning;
|
||||
case Task::VoiceConversion: return K::VoiceConversion;
|
||||
case Task::SpeechToSpeech: return K::SpeechToSpeech;
|
||||
case Task::Alignment: return K::Alignment;
|
||||
case Task::VoiceDesign: return K::VoiceDesign;
|
||||
case Task::SpeakerRecognition: return K::SpeakerRecognition;
|
||||
case Task::Svc: return K::Svc;
|
||||
}
|
||||
// Unreachable for any valid enumerator. No `default:` label, so -Wswitch
|
||||
// still reports a member this switch stops covering.
|
||||
return K::Vad;
|
||||
}
|
||||
|
||||
Task from_engine_task(engine::runtime::VoiceTaskKind kind) {
|
||||
using K = engine::runtime::VoiceTaskKind;
|
||||
switch (kind) {
|
||||
case K::Vad: return Task::Vad;
|
||||
case K::Asr: return Task::Asr;
|
||||
case K::Diarization: return Task::Diarization;
|
||||
case K::SourceSeparation: return Task::SourceSeparation;
|
||||
case K::AudioGeneration: return Task::AudioGeneration;
|
||||
case K::Tts: return Task::Tts;
|
||||
case K::VoiceCloning: return Task::VoiceCloning;
|
||||
case K::VoiceConversion: return Task::VoiceConversion;
|
||||
case K::SpeechToSpeech: return Task::SpeechToSpeech;
|
||||
case K::Alignment: return Task::Alignment;
|
||||
case K::VoiceDesign: return Task::VoiceDesign;
|
||||
case K::SpeakerRecognition: return Task::SpeakerRecognition;
|
||||
case K::Svc: return Task::Svc;
|
||||
}
|
||||
return Task::Vad;
|
||||
}
|
||||
|
||||
engine::runtime::RunMode to_engine_mode(Mode mode) {
|
||||
using M = engine::runtime::RunMode;
|
||||
switch (mode) {
|
||||
case Mode::Offline: return M::Offline;
|
||||
case Mode::Streaming: return M::Streaming;
|
||||
}
|
||||
return M::Offline;
|
||||
}
|
||||
|
||||
Mode from_engine_mode(engine::runtime::RunMode mode) {
|
||||
using M = engine::runtime::RunMode;
|
||||
switch (mode) {
|
||||
case M::Offline: return Mode::Offline;
|
||||
case M::Streaming: return Mode::Streaming;
|
||||
}
|
||||
return Mode::Offline;
|
||||
}
|
||||
|
||||
Capabilities to_capabilities(const std::string &family,
|
||||
const engine::runtime::CapabilitySet &set) {
|
||||
Capabilities caps;
|
||||
caps.family = family;
|
||||
caps.tasks.reserve(set.supported_tasks.size());
|
||||
for (const auto &supported : set.supported_tasks) {
|
||||
TaskCapability capability;
|
||||
capability.task = from_engine_task(supported.task);
|
||||
capability.modes.reserve(supported.modes.size());
|
||||
for (const auto mode : supported.modes) {
|
||||
capability.modes.push_back(from_engine_mode(mode));
|
||||
}
|
||||
caps.tasks.push_back(std::move(capability));
|
||||
}
|
||||
return caps;
|
||||
}
|
||||
|
||||
std::string read_gguf_family(const std::string &path) {
|
||||
try {
|
||||
const auto spec =
|
||||
engine::assets::read_gguf_embedded_model_spec(std::filesystem::path(path));
|
||||
if (spec.has_value()) {
|
||||
return spec->family;
|
||||
}
|
||||
} catch (...) {
|
||||
// A file that is not a readable GGUF simply has no family. The load
|
||||
// gate turns that into a clear refusal; a throw here would surface as
|
||||
// an opaque internal error during backend probing.
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
std::string resolve_model_path(const std::string &model_path_dir,
|
||||
const std::string &model_file,
|
||||
const std::string &model_name) {
|
||||
const std::string candidate = !model_file.empty() ? model_file : model_name;
|
||||
|
||||
const std::string bundled_prefix = "bundled:";
|
||||
if (candidate.rfind(bundled_prefix, 0) == 0) {
|
||||
const std::string name = candidate.substr(bundled_prefix.size());
|
||||
return (executable_directory() / "assets" / name).string();
|
||||
}
|
||||
|
||||
std::filesystem::path path(candidate);
|
||||
if (path.is_absolute() || model_path_dir.empty()) {
|
||||
return path.string();
|
||||
}
|
||||
return (std::filesystem::path(model_path_dir) / path).string();
|
||||
}
|
||||
|
||||
LoadedModel::LoadedModel(const std::string &resolved_path,
|
||||
const ModelOptions &options)
|
||||
: LoadedModel(resolved_path, options, require_family(resolved_path, options)) {}
|
||||
|
||||
LoadedModel::LoadedModel(const std::string &resolved_path,
|
||||
const ModelOptions &options, std::string family)
|
||||
: lane_(family), registry_(engine::runtime::make_default_registry()) {
|
||||
if (!registry_.supports_family(family)) {
|
||||
throw ConfigError("audio-cpp: unknown audio.cpp family '" + family + "'");
|
||||
}
|
||||
|
||||
engine::runtime::ModelLoadRequest request;
|
||||
request.model_path = std::filesystem::path(resolved_path);
|
||||
request.family_hint = family;
|
||||
if (!options.model_spec_override.empty()) {
|
||||
request.model_spec_override =
|
||||
std::filesystem::path(options.model_spec_override);
|
||||
}
|
||||
for (const auto &entry : options.load_options) {
|
||||
request.options[entry.first] = entry.second;
|
||||
}
|
||||
|
||||
try {
|
||||
model_ = registry_.load(request);
|
||||
} catch (const std::exception &err) {
|
||||
throw ConfigError("audio-cpp: failed to load family '" + family +
|
||||
"' from " + resolved_path + ": " + err.what());
|
||||
}
|
||||
if (model_ == nullptr) {
|
||||
throw ConfigError("audio-cpp: the registry returned no model for " +
|
||||
resolved_path);
|
||||
}
|
||||
|
||||
const auto &metadata = model_->metadata();
|
||||
const auto &engine_caps = model_->capabilities();
|
||||
variant_ = metadata.variant;
|
||||
description_ = metadata.description;
|
||||
languages_ = engine_caps.languages;
|
||||
supports_timestamps_ = engine_caps.supports_timestamps;
|
||||
capabilities_ = to_capabilities(family, engine_caps);
|
||||
|
||||
session_options_.backend.type = parse_backend_type(options.backend);
|
||||
session_options_.backend.device = options.device;
|
||||
if (options.threads > 0) {
|
||||
session_options_.backend.threads = options.threads;
|
||||
}
|
||||
for (const auto &entry : options.session_options) {
|
||||
session_options_.options[entry.first] = entry.second;
|
||||
}
|
||||
|
||||
wait_budget_ceiling_ms_ = options.busy_timeout_ms;
|
||||
}
|
||||
|
||||
LoadedModel::Session LoadedModel::session_for(Rpc rpc, const RequestShape &shape) {
|
||||
const Route route = resolve_route(rpc, shape, capabilities_);
|
||||
if (!route.ok) {
|
||||
throw CapabilityError(route.error);
|
||||
}
|
||||
|
||||
const SessionKey key{static_cast<int>(route.task), static_cast<int>(route.mode)};
|
||||
auto found = sessions_.find(key);
|
||||
if (found == sessions_.end()) {
|
||||
engine::runtime::TaskSpec spec;
|
||||
spec.task = to_engine_task(route.task);
|
||||
spec.mode = to_engine_mode(route.mode);
|
||||
std::unique_ptr<engine::runtime::IVoiceTaskSession> created;
|
||||
try {
|
||||
created = model_->create_task_session(spec, session_options_);
|
||||
} catch (const std::exception &err) {
|
||||
throw CapabilityError(
|
||||
std::string("audio-cpp: family '") + capabilities_.family +
|
||||
"' advertises " + task_name(route.task) + "/" +
|
||||
mode_name(route.mode) + " but refused to create the session: " +
|
||||
err.what());
|
||||
}
|
||||
if (created == nullptr) {
|
||||
throw CapabilityError(std::string("audio-cpp: family '") +
|
||||
capabilities_.family +
|
||||
"' returned no session for " +
|
||||
task_name(route.task) + "/" +
|
||||
mode_name(route.mode));
|
||||
}
|
||||
found = sessions_.emplace(key, std::move(created)).first;
|
||||
}
|
||||
|
||||
Session session;
|
||||
session.task = route.task;
|
||||
session.mode = route.mode;
|
||||
engine::runtime::IVoiceTaskSession *raw = found->second.get();
|
||||
if (route.mode == Mode::Streaming) {
|
||||
session.streaming =
|
||||
dynamic_cast<engine::runtime::IStreamingVoiceTaskSession *>(raw);
|
||||
if (session.streaming == nullptr) {
|
||||
throw CapabilityError(std::string("audio-cpp: family '") +
|
||||
capabilities_.family +
|
||||
"' advertises " + task_name(route.task) +
|
||||
"/streaming but its session is not streaming");
|
||||
}
|
||||
} else {
|
||||
session.offline =
|
||||
dynamic_cast<engine::runtime::IOfflineVoiceTaskSession *>(raw);
|
||||
if (session.offline == nullptr) {
|
||||
throw CapabilityError(std::string("audio-cpp: family '") +
|
||||
capabilities_.family +
|
||||
"' advertises " + task_name(route.task) +
|
||||
"/offline but its session is not offline");
|
||||
}
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
LaneEntry LoadedModel::acquire(int requested_timeout_ms) {
|
||||
// Constructed straight into the return value. C++17 requires that, which is
|
||||
// what lets an immovable type be returned at all; a named local here would
|
||||
// not compile.
|
||||
return LaneEntry(lane_,
|
||||
resolve_wait_budget_ms(wait_budget_ceiling_ms_,
|
||||
requested_timeout_ms));
|
||||
}
|
||||
|
||||
std::unique_ptr<LaneEntry> LoadedModel::acquire_owned(int requested_timeout_ms) {
|
||||
return std::make_unique<LaneEntry>(
|
||||
lane_,
|
||||
resolve_wait_budget_ms(wait_budget_ceiling_ms_, requested_timeout_ms));
|
||||
}
|
||||
|
||||
} // namespace audiocpp_backend
|
||||
146
backend/cpp/audio-cpp/loaded_model.h
Normal file
146
backend/cpp/audio-cpp/loaded_model.h
Normal file
@@ -0,0 +1,146 @@
|
||||
#pragma once
|
||||
|
||||
// Owns one audio.cpp model for the life of the process, plus a lazily created
|
||||
// session per (task, mode) so the same model answers both TTS and TTSStream.
|
||||
// This is the only unit that converts between the stdlib-only mirror types and
|
||||
// engine::runtime types.
|
||||
|
||||
#include "capability_routing.h"
|
||||
#include "inference_lane.h"
|
||||
#include "model_options.h"
|
||||
|
||||
#include "engine/framework/runtime/model.h"
|
||||
#include "engine/framework/runtime/registry.h"
|
||||
#include "engine/framework/runtime/session.h"
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace audiocpp_backend {
|
||||
|
||||
// User-fixable configuration problem. grpc-server maps this to INVALID_ARGUMENT.
|
||||
class ConfigError : public std::runtime_error {
|
||||
public:
|
||||
using std::runtime_error::runtime_error;
|
||||
};
|
||||
|
||||
// The family cannot serve the requested RPC. Maps to UNIMPLEMENTED.
|
||||
class CapabilityError : public std::runtime_error {
|
||||
public:
|
||||
using std::runtime_error::runtime_error;
|
||||
};
|
||||
|
||||
engine::runtime::VoiceTaskKind to_engine_task(Task task);
|
||||
engine::runtime::RunMode to_engine_mode(Mode mode);
|
||||
Task from_engine_task(engine::runtime::VoiceTaskKind kind);
|
||||
Mode from_engine_mode(engine::runtime::RunMode mode);
|
||||
Capabilities to_capabilities(const std::string &family,
|
||||
const engine::runtime::CapabilitySet &set);
|
||||
|
||||
// Reads audiocpp.model_spec.family from a GGUF. Returns an empty string when
|
||||
// the file is not a GGUF, carries no audio.cpp spec, or cannot be read. Never
|
||||
// throws: an unreadable file is the load gate's problem, not a crash.
|
||||
std::string read_gguf_family(const std::string &path);
|
||||
|
||||
// Builds the absolute model path from LocalAI's (ModelPath, ModelFile, Model)
|
||||
// triple. A model file of the form "bundled:<name>" resolves to
|
||||
// <executable dir>/assets/<name>, which is where package.sh puts upstream's
|
||||
// bundled silero_vad and marblenet_vad assets.
|
||||
std::string resolve_model_path(const std::string &model_path_dir,
|
||||
const std::string &model_file,
|
||||
const std::string &model_name);
|
||||
|
||||
class LoadedModel {
|
||||
public:
|
||||
struct Session {
|
||||
Task task = Task::Tts;
|
||||
Mode mode = Mode::Offline;
|
||||
// Exactly one of these is non-null, matching the resolved mode.
|
||||
engine::runtime::IOfflineVoiceTaskSession *offline = nullptr;
|
||||
engine::runtime::IStreamingVoiceTaskSession *streaming = nullptr;
|
||||
};
|
||||
|
||||
// Throws ConfigError when the path does not exist, the family cannot be
|
||||
// determined, or the registry rejects the family.
|
||||
LoadedModel(const std::string &resolved_path, const ModelOptions &options);
|
||||
|
||||
LoadedModel(const LoadedModel &) = delete;
|
||||
LoadedModel &operator=(const LoadedModel &) = delete;
|
||||
|
||||
const std::string &family() const noexcept { return capabilities_.family; }
|
||||
const std::string &variant() const noexcept { return variant_; }
|
||||
const std::string &description() const noexcept { return description_; }
|
||||
const std::vector<std::string> &languages() const noexcept { return languages_; }
|
||||
const Capabilities &capabilities() const noexcept { return capabilities_; }
|
||||
bool supports_timestamps() const noexcept { return supports_timestamps_; }
|
||||
const engine::runtime::SessionOptions &session_options() const noexcept {
|
||||
return session_options_;
|
||||
}
|
||||
|
||||
// Routes the RPC and returns the cached session, creating it on first use.
|
||||
// Throws CapabilityError when this family cannot serve the RPC.
|
||||
//
|
||||
// Call it only while holding this model's lane. The session cache is not
|
||||
// itself synchronised, and it does not need to be: the lane admits one
|
||||
// caller at a time, which is the same constraint the sessions themselves
|
||||
// impose.
|
||||
Session session_for(Rpc rpc, const RequestShape &shape);
|
||||
|
||||
// Takes the inference lane, or throws LaneUnavailable. Serializes runs
|
||||
// against this model. `requested_timeout_ms` is a per-request wait hint
|
||||
// where a value <= 0 means "use the model's configured ceiling"; a hint may
|
||||
// only tighten that ceiling, never loosen it.
|
||||
//
|
||||
// LaneEntry is deliberately immovable, so bind the result to a named local
|
||||
// in the scope the inference happens in:
|
||||
//
|
||||
// LaneEntry entry = model.acquire(request_hint_ms);
|
||||
//
|
||||
// which C++17 initializes in place. A handler that has to keep the lane
|
||||
// beyond one scope, for instance in a member that outlives the call that
|
||||
// took it, wants acquire_owned instead.
|
||||
LaneEntry acquire(int requested_timeout_ms);
|
||||
|
||||
// Same lane, heap-allocated so it can be stored or handed on. Prefer
|
||||
// acquire: this one adds a null state that the scoped form does not have.
|
||||
std::unique_ptr<LaneEntry> acquire_owned(int requested_timeout_ms);
|
||||
|
||||
private:
|
||||
// Keyed by the enum values so the map needs no custom comparator.
|
||||
using SessionKey = std::pair<int, int>;
|
||||
|
||||
// The public constructor runs the load gate, then delegates here. The
|
||||
// detour exists because lane_ has to be built from the family in the member
|
||||
// initializer list, and the family is only known after the gate has run.
|
||||
LoadedModel(const std::string &resolved_path, const ModelOptions &options,
|
||||
std::string family);
|
||||
|
||||
// MEMBER ORDER IS LOAD-BEARING BELOW THIS LINE. Members are destroyed in
|
||||
// reverse declaration order.
|
||||
//
|
||||
// lane_ is first so it is destroyed last: nothing that runs during teardown
|
||||
// can then find a lane that has already gone.
|
||||
InferenceLane lane_;
|
||||
// registry_ before model_: the registry owns the loader that produced the
|
||||
// model, and the model may hold loader-owned state.
|
||||
engine::runtime::ModelRegistry registry_;
|
||||
// model_ before sessions_, so sessions_ is destroyed FIRST and the model
|
||||
// second. A session is created from the model and must not outlive it. Do
|
||||
// not reorder these two.
|
||||
std::unique_ptr<engine::runtime::ILoadedVoiceModel> model_;
|
||||
std::map<SessionKey, std::unique_ptr<engine::runtime::IVoiceTaskSession>> sessions_;
|
||||
|
||||
engine::runtime::SessionOptions session_options_;
|
||||
Capabilities capabilities_;
|
||||
std::string variant_;
|
||||
std::string description_;
|
||||
std::vector<std::string> languages_;
|
||||
bool supports_timestamps_ = false;
|
||||
int wait_budget_ceiling_ms_ = 0;
|
||||
};
|
||||
|
||||
} // namespace audiocpp_backend
|
||||
Reference in New Issue
Block a user