diff --git a/backend/cpp/audio-cpp/CMakeLists.txt b/backend/cpp/audio-cpp/CMakeLists.txt index d927cf073..3cf4d3a31 100644 --- a/backend/cpp/audio-cpp/CMakeLists.txt +++ b/backend/cpp/audio-cpp/CMakeLists.txt @@ -88,11 +88,26 @@ add_executable(${TARGET} grpc-server.cpp model_options.cpp capability_routing.cpp + family_gate.cpp + loaded_model.cpp audio_units.cpp transcript_assembly.cpp inference_lane.cpp ) +# loaded_model.cpp mirrors engine::runtime::VoiceTaskKind onto its own Task enum. +# Its static_asserts catch an insertion or a reorder, but an enumerator APPENDED +# after the last one shifts no value, so no assertion can see it. What does see +# it is from_engine_task's switch over the engine enum, which carries no +# `default:` label. -Wswitch is only a warning by default, and a warning in a +# 600-file build log is a warning nobody reads, so promote it here: this is the +# difference between a build failure and a backend that silently runs the wrong +# task. +if(NOT MSVC) + set_source_files_properties(loaded_model.cpp PROPERTIES + COMPILE_OPTIONS "-Werror=switch") +endif() + target_include_directories(${TARGET} PRIVATE "${AUDIO_CPP_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/backend/cpp/audio-cpp/grpc-server.cpp b/backend/cpp/audio-cpp/grpc-server.cpp index bc07e2cb2..7e84cce89 100644 --- a/backend/cpp/audio-cpp/grpc-server.cpp +++ b/backend/cpp/audio-cpp/grpc-server.cpp @@ -5,12 +5,17 @@ // src/ or tests/ is used: those are application internals, they are where // upstream expects churn, and upstream is Apache-2.0 while LocalAI is MIT. // -// This commit is the bind/listen/Health/Status skeleton. Model loading and the -// audio RPCs land in later commits. +// This commit adds LoadModel/Free/Status over one audio.cpp model. The audio +// RPCs themselves land in later commits. #include "backend.pb.h" #include "backend.grpc.pb.h" +#include "capability_routing.h" +#include "inference_lane.h" +#include "loaded_model.h" +#include "model_options.h" + #include #include #include @@ -20,9 +25,13 @@ #include #include #include +#include #include #include +#include #include +#include +#include using grpc::Server; using grpc::ServerBuilder; @@ -35,6 +44,30 @@ namespace { std::atomic g_server{nullptr}; +// One model per process, matching every other LocalAI backend. The mutex guards +// the pointer itself, not the model: swapping or dropping it races with the +// handlers that read it, whereas the model's own concurrency is the inference +// lane's job. +std::mutex g_model_mu; +std::unique_ptr g_model; + +// Maps a thrown exception onto the gRPC status the client should see. +GStatus to_status(const std::exception &err) { + if (dynamic_cast(&err) != nullptr) { + return GStatus(grpc::StatusCode::INVALID_ARGUMENT, err.what()); + } + if (dynamic_cast(&err) != nullptr) { + return GStatus(grpc::StatusCode::UNIMPLEMENTED, err.what()); + } + // The lane was busy and this caller ran out of patience. UNAVAILABLE, not + // RESOURCE_EXHAUSTED: the request is fine and retrying later is the right + // response, which is what UNAVAILABLE tells a client. + if (dynamic_cast(&err) != nullptr) { + return GStatus(grpc::StatusCode::UNAVAILABLE, err.what()); + } + return GStatus(grpc::StatusCode::INTERNAL, err.what()); +} + class AudioCppBackend final : public backend::Backend::Service { public: GStatus Health(ServerContext *, const backend::HealthMessage *, @@ -45,7 +78,75 @@ public: GStatus Status(ServerContext *, const backend::HealthMessage *, backend::StatusResponse *response) override { - response->set_state(backend::StatusResponse::UNINITIALIZED); + std::lock_guard lock(g_model_mu); + response->set_state(g_model ? backend::StatusResponse::READY + : backend::StatusResponse::UNINITIALIZED); + return GStatus::OK; + } + + GStatus LoadModel(ServerContext *, const backend::ModelOptions *request, + backend::Result *result) override { + try { + std::vector entries(request->options().begin(), + request->options().end()); + auto parsed = audiocpp_backend::parse_model_options(entries); + if (!parsed.error.empty()) { + throw audiocpp_backend::ConfigError(parsed.error); + } + // ModelOptions.Threads is the model YAML's threads setting; the + // explicit threads: option wins when both are set. + if (parsed.options.threads == 0 && request->threads() > 0) { + parsed.options.threads = static_cast(request->threads()); + } + // MainGPU carries the device index for GPU backends. + if (parsed.options.device == 0 && !request->maingpu().empty()) { + parsed.options.device = std::atoi(request->maingpu().c_str()); + } + + const std::string path = audiocpp_backend::resolve_model_path( + request->modelpath(), request->modelfile(), request->model()); + + auto loaded = std::make_unique( + path, parsed.options); + + // Read everything the reply and the log line need while this scope + // still owns the model. After the move, `loaded` is null and the + // global is only safe to touch under the mutex. + const std::string family = loaded->family(); + const std::string variant = loaded->variant(); + const std::string capabilities = + audiocpp_backend::describe_capabilities(loaded->capabilities()); + { + std::lock_guard lock(g_model_mu); + g_model = std::move(loaded); + } + + // Logged once at load time so an operator can see what the model + // can do without making a request. ModelMetadata is deliberately + // not implemented: its response message is chat-template oriented + // and has no field for family, variant, languages or capabilities, + // so there is nothing truthful to put in it. + std::cerr << "audio-cpp: loaded family '" << family << "' variant '" + << variant << "' capabilities: " << capabilities << "\n"; + + result->set_success(true); + result->set_message("loaded audio.cpp family " + family); + return GStatus::OK; + } catch (const std::exception &err) { + result->set_success(false); + result->set_message(err.what()); + // A failed load must also be a gRPC error, or the model loader's + // backend probe treats this backend as having accepted the model. + return to_status(err); + } + } + + GStatus Free(ServerContext *, const backend::HealthMessage *, + backend::Result *result) override { + std::lock_guard lock(g_model_mu); + // Destroying LoadedModel releases its sessions first, then the model. + g_model.reset(); + result->set_success(true); return GStatus::OK; } }; diff --git a/backend/cpp/audio-cpp/loaded_model.cpp b/backend/cpp/audio-cpp/loaded_model.cpp new file mode 100644 index 000000000..cceec7f66 --- /dev/null +++ b/backend/cpp/audio-cpp/loaded_model.cpp @@ -0,0 +1,404 @@ +#include "loaded_model.h" + +#include "family_gate.h" + +#include "engine/framework/assets/tensor_source.h" + +#include +#include + +#if defined(__APPLE__) +#include +#include +#include +#elif defined(__linux__) +#include +#endif + +namespace audiocpp_backend { + +// -------------------------------------------------------------------------- +// Enum coupling +// +// audiocpp_backend::Task mirrors engine::runtime::VoiceTaskKind positionally so +// capability_routing can stay stdlib-only and testable without an audio.cpp +// checkout. Nothing about that mirroring is enforced by the type system, and a +// drift is silent in the worst possible way: every unit still compiles, every +// test still passes, and the backend runs a different task than the one the +// caller asked for. +// +// Two mechanisms pin it, and both are needed because they catch different edits: +// +// 1. The assertions below pin every enumerator's value on both sides. An +// insertion or a reorder anywhere before the last member shifts the values +// after it and fails the build here. +// 2. An enumerator APPENDED after the last one shifts nothing, so no value +// assertion can see it. What sees it is the switch in from_engine_task, +// which covers the engine enum with no `default:` label. CMakeLists.txt +// compiles this file with -Werror=switch so that omission is an error +// rather than a warning nobody reads. +// +// Neither mechanism catches a pure RENAME of an upstream enumerator, but that +// does not need catching: the switch stops naming an enumerator that exists and +// the build fails on its own. +// -------------------------------------------------------------------------- + +namespace { + +constexpr int kEngine(engine::runtime::VoiceTaskKind kind) { + return static_cast(kind); +} +constexpr int kMirror(Task task) { return static_cast(task); } + +} // namespace + +static_assert(kEngine(engine::runtime::VoiceTaskKind::Vad) == 0, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::Asr) == 1, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::Diarization) == 2, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::SourceSeparation) == 3, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::AudioGeneration) == 4, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::Tts) == 5, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::VoiceCloning) == 6, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::VoiceConversion) == 7, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::SpeechToSpeech) == 8, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::Alignment) == 9, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::VoiceDesign) == 10, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::SpeakerRecognition) == 11, "VoiceTaskKind drifted"); +// The last member. Pinning it pins the member count too, as long as the +// enumerators stay contiguous and unassigned, which upstream's declaration is. +static_assert(kEngine(engine::runtime::VoiceTaskKind::Svc) == 12, + "engine::runtime::VoiceTaskKind gained, lost or reordered a member. " + "audiocpp_backend::Task mirrors it positionally: update capability_routing.h, " + "to_engine_task and from_engine_task together, then move this pin."); + +static_assert(kMirror(Task::Vad) == 0, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::Asr) == 1, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::Diarization) == 2, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::SourceSeparation) == 3, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::AudioGeneration) == 4, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::Tts) == 5, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::VoiceCloning) == 6, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::VoiceConversion) == 7, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::SpeechToSpeech) == 8, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::Alignment) == 9, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::VoiceDesign) == 10, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::SpeakerRecognition) == 11, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::Svc) == 12, "Task drifted from VoiceTaskKind"); + +static_assert(static_cast(engine::runtime::RunMode::Offline) == 0, "RunMode drifted"); +static_assert(static_cast(engine::runtime::RunMode::Streaming) == 1, + "engine::runtime::RunMode gained, lost or reordered a member. " + "audiocpp_backend::Mode mirrors it positionally."); +static_assert(static_cast(Mode::Offline) == 0, "Mode drifted from RunMode"); +static_assert(static_cast(Mode::Streaming) == 1, "Mode drifted from RunMode"); + +namespace { + +engine::core::BackendType parse_backend_type(const std::string &value) { + if (value == "cuda") { + return engine::core::BackendType::Cuda; + } + if (value == "vulkan") { + return engine::core::BackendType::Vulkan; + } + if (value == "metal") { + return engine::core::BackendType::Metal; + } + if (value == "best") { + return engine::core::BackendType::BestAvailable; + } + if (value == "cpu" || value.empty()) { + return engine::core::BackendType::Cpu; + } + throw ConfigError("audio-cpp: unknown backend option '" + value + + "'. Known backends: cpu, cuda, vulkan, metal, best"); +} + +std::filesystem::path executable_directory() { +#if defined(__APPLE__) + std::uint32_t size = 0; + _NSGetExecutablePath(nullptr, &size); + std::vector buffer(size + 1, '\0'); + if (_NSGetExecutablePath(buffer.data(), &size) != 0) { + return std::filesystem::current_path(); + } + return std::filesystem::path(buffer.data()).parent_path(); +#elif defined(__linux__) + std::error_code ec; + const auto self = std::filesystem::read_symlink("/proc/self/exe", ec); + if (ec) { + return std::filesystem::current_path(); + } + return self.parent_path(); +#else + return std::filesystem::current_path(); +#endif +} + +// Runs the load gate. Throws ConfigError rather than returning a decision, +// because the only caller is a delegating constructor whose member initializer +// list has nowhere to put a failure. +std::string require_family(const std::string &resolved_path, + const ModelOptions &options) { + const std::filesystem::path path(resolved_path); + std::error_code ec; + if (!std::filesystem::exists(path, ec)) { + throw ConfigError("audio-cpp: model path does not exist: " + resolved_path); + } + + const bool is_gguf = path_looks_like_gguf(resolved_path); + // Only a GGUF is asked for embedded metadata. A directory has no single + // file to read it from, and probing one would make the gate's refusal + // depend on which file happened to be inside. + const std::string embedded = + is_gguf ? read_gguf_family(resolved_path) : std::string(); + + const FamilyDecision decision = + decide_family(is_gguf, embedded, options.family); + if (!decision.ok) { + throw ConfigError(decision.error); + } + return decision.family; +} + +} // namespace + +engine::runtime::VoiceTaskKind to_engine_task(Task task) { + using K = engine::runtime::VoiceTaskKind; + // An explicit switch, never a cast: a cast would keep compiling through + // exactly the drift the assertions above exist to catch. + switch (task) { + case Task::Vad: return K::Vad; + case Task::Asr: return K::Asr; + case Task::Diarization: return K::Diarization; + case Task::SourceSeparation: return K::SourceSeparation; + case Task::AudioGeneration: return K::AudioGeneration; + case Task::Tts: return K::Tts; + case Task::VoiceCloning: return K::VoiceCloning; + case Task::VoiceConversion: return K::VoiceConversion; + case Task::SpeechToSpeech: return K::SpeechToSpeech; + case Task::Alignment: return K::Alignment; + case Task::VoiceDesign: return K::VoiceDesign; + case Task::SpeakerRecognition: return K::SpeakerRecognition; + case Task::Svc: return K::Svc; + } + // Unreachable for any valid enumerator. No `default:` label, so -Wswitch + // still reports a member this switch stops covering. + return K::Vad; +} + +Task from_engine_task(engine::runtime::VoiceTaskKind kind) { + using K = engine::runtime::VoiceTaskKind; + switch (kind) { + case K::Vad: return Task::Vad; + case K::Asr: return Task::Asr; + case K::Diarization: return Task::Diarization; + case K::SourceSeparation: return Task::SourceSeparation; + case K::AudioGeneration: return Task::AudioGeneration; + case K::Tts: return Task::Tts; + case K::VoiceCloning: return Task::VoiceCloning; + case K::VoiceConversion: return Task::VoiceConversion; + case K::SpeechToSpeech: return Task::SpeechToSpeech; + case K::Alignment: return Task::Alignment; + case K::VoiceDesign: return Task::VoiceDesign; + case K::SpeakerRecognition: return Task::SpeakerRecognition; + case K::Svc: return Task::Svc; + } + return Task::Vad; +} + +engine::runtime::RunMode to_engine_mode(Mode mode) { + using M = engine::runtime::RunMode; + switch (mode) { + case Mode::Offline: return M::Offline; + case Mode::Streaming: return M::Streaming; + } + return M::Offline; +} + +Mode from_engine_mode(engine::runtime::RunMode mode) { + using M = engine::runtime::RunMode; + switch (mode) { + case M::Offline: return Mode::Offline; + case M::Streaming: return Mode::Streaming; + } + return Mode::Offline; +} + +Capabilities to_capabilities(const std::string &family, + const engine::runtime::CapabilitySet &set) { + Capabilities caps; + caps.family = family; + caps.tasks.reserve(set.supported_tasks.size()); + for (const auto &supported : set.supported_tasks) { + TaskCapability capability; + capability.task = from_engine_task(supported.task); + capability.modes.reserve(supported.modes.size()); + for (const auto mode : supported.modes) { + capability.modes.push_back(from_engine_mode(mode)); + } + caps.tasks.push_back(std::move(capability)); + } + return caps; +} + +std::string read_gguf_family(const std::string &path) { + try { + const auto spec = + engine::assets::read_gguf_embedded_model_spec(std::filesystem::path(path)); + if (spec.has_value()) { + return spec->family; + } + } catch (...) { + // A file that is not a readable GGUF simply has no family. The load + // gate turns that into a clear refusal; a throw here would surface as + // an opaque internal error during backend probing. + } + return {}; +} + +std::string resolve_model_path(const std::string &model_path_dir, + const std::string &model_file, + const std::string &model_name) { + const std::string candidate = !model_file.empty() ? model_file : model_name; + + const std::string bundled_prefix = "bundled:"; + if (candidate.rfind(bundled_prefix, 0) == 0) { + const std::string name = candidate.substr(bundled_prefix.size()); + return (executable_directory() / "assets" / name).string(); + } + + std::filesystem::path path(candidate); + if (path.is_absolute() || model_path_dir.empty()) { + return path.string(); + } + return (std::filesystem::path(model_path_dir) / path).string(); +} + +LoadedModel::LoadedModel(const std::string &resolved_path, + const ModelOptions &options) + : LoadedModel(resolved_path, options, require_family(resolved_path, options)) {} + +LoadedModel::LoadedModel(const std::string &resolved_path, + const ModelOptions &options, std::string family) + : lane_(family), registry_(engine::runtime::make_default_registry()) { + if (!registry_.supports_family(family)) { + throw ConfigError("audio-cpp: unknown audio.cpp family '" + family + "'"); + } + + engine::runtime::ModelLoadRequest request; + request.model_path = std::filesystem::path(resolved_path); + request.family_hint = family; + if (!options.model_spec_override.empty()) { + request.model_spec_override = + std::filesystem::path(options.model_spec_override); + } + for (const auto &entry : options.load_options) { + request.options[entry.first] = entry.second; + } + + try { + model_ = registry_.load(request); + } catch (const std::exception &err) { + throw ConfigError("audio-cpp: failed to load family '" + family + + "' from " + resolved_path + ": " + err.what()); + } + if (model_ == nullptr) { + throw ConfigError("audio-cpp: the registry returned no model for " + + resolved_path); + } + + const auto &metadata = model_->metadata(); + const auto &engine_caps = model_->capabilities(); + variant_ = metadata.variant; + description_ = metadata.description; + languages_ = engine_caps.languages; + supports_timestamps_ = engine_caps.supports_timestamps; + capabilities_ = to_capabilities(family, engine_caps); + + session_options_.backend.type = parse_backend_type(options.backend); + session_options_.backend.device = options.device; + if (options.threads > 0) { + session_options_.backend.threads = options.threads; + } + for (const auto &entry : options.session_options) { + session_options_.options[entry.first] = entry.second; + } + + wait_budget_ceiling_ms_ = options.busy_timeout_ms; +} + +LoadedModel::Session LoadedModel::session_for(Rpc rpc, const RequestShape &shape) { + const Route route = resolve_route(rpc, shape, capabilities_); + if (!route.ok) { + throw CapabilityError(route.error); + } + + const SessionKey key{static_cast(route.task), static_cast(route.mode)}; + auto found = sessions_.find(key); + if (found == sessions_.end()) { + engine::runtime::TaskSpec spec; + spec.task = to_engine_task(route.task); + spec.mode = to_engine_mode(route.mode); + std::unique_ptr created; + try { + created = model_->create_task_session(spec, session_options_); + } catch (const std::exception &err) { + throw CapabilityError( + std::string("audio-cpp: family '") + capabilities_.family + + "' advertises " + task_name(route.task) + "/" + + mode_name(route.mode) + " but refused to create the session: " + + err.what()); + } + if (created == nullptr) { + throw CapabilityError(std::string("audio-cpp: family '") + + capabilities_.family + + "' returned no session for " + + task_name(route.task) + "/" + + mode_name(route.mode)); + } + found = sessions_.emplace(key, std::move(created)).first; + } + + Session session; + session.task = route.task; + session.mode = route.mode; + engine::runtime::IVoiceTaskSession *raw = found->second.get(); + if (route.mode == Mode::Streaming) { + session.streaming = + dynamic_cast(raw); + if (session.streaming == nullptr) { + throw CapabilityError(std::string("audio-cpp: family '") + + capabilities_.family + + "' advertises " + task_name(route.task) + + "/streaming but its session is not streaming"); + } + } else { + session.offline = + dynamic_cast(raw); + if (session.offline == nullptr) { + throw CapabilityError(std::string("audio-cpp: family '") + + capabilities_.family + + "' advertises " + task_name(route.task) + + "/offline but its session is not offline"); + } + } + return session; +} + +LaneEntry LoadedModel::acquire(int requested_timeout_ms) { + // Constructed straight into the return value. C++17 requires that, which is + // what lets an immovable type be returned at all; a named local here would + // not compile. + return LaneEntry(lane_, + resolve_wait_budget_ms(wait_budget_ceiling_ms_, + requested_timeout_ms)); +} + +std::unique_ptr LoadedModel::acquire_owned(int requested_timeout_ms) { + return std::make_unique( + lane_, + resolve_wait_budget_ms(wait_budget_ceiling_ms_, requested_timeout_ms)); +} + +} // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/loaded_model.h b/backend/cpp/audio-cpp/loaded_model.h new file mode 100644 index 000000000..763155e2a --- /dev/null +++ b/backend/cpp/audio-cpp/loaded_model.h @@ -0,0 +1,146 @@ +#pragma once + +// Owns one audio.cpp model for the life of the process, plus a lazily created +// session per (task, mode) so the same model answers both TTS and TTSStream. +// This is the only unit that converts between the stdlib-only mirror types and +// engine::runtime types. + +#include "capability_routing.h" +#include "inference_lane.h" +#include "model_options.h" + +#include "engine/framework/runtime/model.h" +#include "engine/framework/runtime/registry.h" +#include "engine/framework/runtime/session.h" + +#include +#include +#include +#include +#include +#include + +namespace audiocpp_backend { + +// User-fixable configuration problem. grpc-server maps this to INVALID_ARGUMENT. +class ConfigError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +// The family cannot serve the requested RPC. Maps to UNIMPLEMENTED. +class CapabilityError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +engine::runtime::VoiceTaskKind to_engine_task(Task task); +engine::runtime::RunMode to_engine_mode(Mode mode); +Task from_engine_task(engine::runtime::VoiceTaskKind kind); +Mode from_engine_mode(engine::runtime::RunMode mode); +Capabilities to_capabilities(const std::string &family, + const engine::runtime::CapabilitySet &set); + +// Reads audiocpp.model_spec.family from a GGUF. Returns an empty string when +// the file is not a GGUF, carries no audio.cpp spec, or cannot be read. Never +// throws: an unreadable file is the load gate's problem, not a crash. +std::string read_gguf_family(const std::string &path); + +// Builds the absolute model path from LocalAI's (ModelPath, ModelFile, Model) +// triple. A model file of the form "bundled:" resolves to +// /assets/, which is where package.sh puts upstream's +// bundled silero_vad and marblenet_vad assets. +std::string resolve_model_path(const std::string &model_path_dir, + const std::string &model_file, + const std::string &model_name); + +class LoadedModel { +public: + struct Session { + Task task = Task::Tts; + Mode mode = Mode::Offline; + // Exactly one of these is non-null, matching the resolved mode. + engine::runtime::IOfflineVoiceTaskSession *offline = nullptr; + engine::runtime::IStreamingVoiceTaskSession *streaming = nullptr; + }; + + // Throws ConfigError when the path does not exist, the family cannot be + // determined, or the registry rejects the family. + LoadedModel(const std::string &resolved_path, const ModelOptions &options); + + LoadedModel(const LoadedModel &) = delete; + LoadedModel &operator=(const LoadedModel &) = delete; + + const std::string &family() const noexcept { return capabilities_.family; } + const std::string &variant() const noexcept { return variant_; } + const std::string &description() const noexcept { return description_; } + const std::vector &languages() const noexcept { return languages_; } + const Capabilities &capabilities() const noexcept { return capabilities_; } + bool supports_timestamps() const noexcept { return supports_timestamps_; } + const engine::runtime::SessionOptions &session_options() const noexcept { + return session_options_; + } + + // Routes the RPC and returns the cached session, creating it on first use. + // Throws CapabilityError when this family cannot serve the RPC. + // + // Call it only while holding this model's lane. The session cache is not + // itself synchronised, and it does not need to be: the lane admits one + // caller at a time, which is the same constraint the sessions themselves + // impose. + Session session_for(Rpc rpc, const RequestShape &shape); + + // Takes the inference lane, or throws LaneUnavailable. Serializes runs + // against this model. `requested_timeout_ms` is a per-request wait hint + // where a value <= 0 means "use the model's configured ceiling"; a hint may + // only tighten that ceiling, never loosen it. + // + // LaneEntry is deliberately immovable, so bind the result to a named local + // in the scope the inference happens in: + // + // LaneEntry entry = model.acquire(request_hint_ms); + // + // which C++17 initializes in place. A handler that has to keep the lane + // beyond one scope, for instance in a member that outlives the call that + // took it, wants acquire_owned instead. + LaneEntry acquire(int requested_timeout_ms); + + // Same lane, heap-allocated so it can be stored or handed on. Prefer + // acquire: this one adds a null state that the scoped form does not have. + std::unique_ptr acquire_owned(int requested_timeout_ms); + +private: + // Keyed by the enum values so the map needs no custom comparator. + using SessionKey = std::pair; + + // The public constructor runs the load gate, then delegates here. The + // detour exists because lane_ has to be built from the family in the member + // initializer list, and the family is only known after the gate has run. + LoadedModel(const std::string &resolved_path, const ModelOptions &options, + std::string family); + + // MEMBER ORDER IS LOAD-BEARING BELOW THIS LINE. Members are destroyed in + // reverse declaration order. + // + // lane_ is first so it is destroyed last: nothing that runs during teardown + // can then find a lane that has already gone. + InferenceLane lane_; + // registry_ before model_: the registry owns the loader that produced the + // model, and the model may hold loader-owned state. + engine::runtime::ModelRegistry registry_; + // model_ before sessions_, so sessions_ is destroyed FIRST and the model + // second. A session is created from the model and must not outlive it. Do + // not reorder these two. + std::unique_ptr model_; + std::map> sessions_; + + engine::runtime::SessionOptions session_options_; + Capabilities capabilities_; + std::string variant_; + std::string description_; + std::vector languages_; + bool supports_timestamps_ = false; + int wait_budget_ceiling_ms_ = 0; +}; + +} // namespace audiocpp_backend