#pragma once // Owns one audio.cpp model for the life of the process, plus a lazily created // session per (task, mode) so the same model answers both TTS and TTSStream. // This is the only unit that converts between the stdlib-only mirror types and // engine::runtime types. #include "capability_routing.h" #include "inference_lane.h" #include "model_options.h" #include "engine/framework/runtime/model.h" #include "engine/framework/runtime/registry.h" #include "engine/framework/runtime/session.h" #include #include #include #include #include #include #include namespace audiocpp_backend { // User-fixable configuration problem. grpc-server maps this to INVALID_ARGUMENT. class ConfigError : public std::runtime_error { public: using std::runtime_error::runtime_error; }; // The family cannot serve the requested RPC. Maps to UNIMPLEMENTED. class CapabilityError : public std::runtime_error { public: using std::runtime_error::runtime_error; }; engine::runtime::VoiceTaskKind to_engine_task(Task task); engine::runtime::RunMode to_engine_mode(Mode mode); Task from_engine_task(engine::runtime::VoiceTaskKind kind); Mode from_engine_mode(engine::runtime::RunMode mode); Capabilities to_capabilities(const std::string &family, const engine::runtime::CapabilitySet &set); // Reads audiocpp.model_spec.family from a GGUF. Returns an empty string when // the file is not a GGUF, carries no audio.cpp spec, or cannot be read. Never // throws: an unreadable file is the load gate's problem, not a crash. std::string read_gguf_family(const std::string &path); // Builds the absolute model path from LocalAI's (ModelPath, ModelFile, Model) // triple. Either the Model or the ModelFile field may carry the form // "bundled:", which resolves to /assets/, where // package.sh puts upstream's bundled silero_vad and marblenet_vad assets. BOTH // are checked because LocalAI fills ModelFile by joining ModelPath onto the // configured model string, so a model YAML using the form has it intact only in // Model. std::string resolve_model_path(const std::string &model_path_dir, const std::string &model_file, const std::string &model_name); class LoadedModel { public: struct Session { Task task = Task::Tts; Mode mode = Mode::Offline; // Exactly one of these is non-null, matching the resolved mode. engine::runtime::IOfflineVoiceTaskSession *offline = nullptr; engine::runtime::IStreamingVoiceTaskSession *streaming = nullptr; }; // Throws ConfigError when the path does not exist, the family cannot be // determined, or the registry rejects the family. // // `model_identity` is ModelOptions.Model verbatim: the UNTRANSLATED // controller-side name. It is a constructor argument rather than a setter // so identity and model are inseparable. llama-cpp keeps its equivalent in // a separate global from the model, which leaves a window where a handler // can read one without the other; here a handler that holds the model // through snapshot() necessarily holds the identity it was loaded with. LoadedModel(const std::string &resolved_path, const ModelOptions &options, std::string model_identity); LoadedModel(const LoadedModel &) = delete; LoadedModel &operator=(const LoadedModel &) = delete; const std::string &family() const noexcept { return capabilities_.family; } // Empty when the controller predates ModelOptions.ModelIdentity, which the // identity check reads as "skip". See check_model_identity in grpc-server. const std::string &identity() const noexcept { return identity_; } const std::string &variant() const noexcept { return variant_; } const std::string &description() const noexcept { return description_; } const std::vector &languages() const noexcept { return languages_; } const Capabilities &capabilities() const noexcept { return capabilities_; } bool supports_timestamps() const noexcept { return supports_timestamps_; } const engine::runtime::SessionOptions &session_options() const noexcept { return session_options_; } // The model's `task:` option, empty when unset. Every handler must copy it // into RequestShape::pinned_task before calling session_for: routing is // otherwise derived from the RPC alone, and this is the option's only route // from the load to the request that honours it. const std::string &pinned_task() const noexcept { return pinned_task_; } // The `live_idle_timeout_ms` option: how long AudioTranscriptionLive waits // for the next audio frame before cancelling the stream to give this // model's lane back. 0 means no limit. See the option in model_options.h // for why it exists and how the default was chosen. int live_idle_timeout_ms() const noexcept { return live_idle_timeout_ms_; } // Throws the same CapabilityError session_for would throw when this family // cannot serve the RPC, and RETURNS THE RESOLVED ROUTE otherwise. // // The route is returned rather than computed and dropped because a handler // often has to know which task it is about to run BEFORE running it. // AudioTransform refuses params[stem] on any route but source separation, // and reading that off the route costs microseconds where reading it off // the result costs a whole inference first. A caller with no such need // ignores the value, which is what the three transcription-shaped handlers // do. // // It exists so a refusal does not have to buy a place in the queue first. // resolve_route is a pure function of capabilities_, which is fixed at // construction and never written again, so unlike the session cache it // needs no lane and no lock: a model that cannot transcribe can say so // while another request is halfway through a thirty second run. Without // this the refusal waits for that run to finish only to be told no. // // It does NOT replace the routing inside session_for, and must not be made // to: session_for still needs the route to key the session cache. The two // calls agree because both read the same immutable capabilities. What this // one adds is only the ordering, so call it before acquire(). // // Const and lane-free on purpose. If a future edit makes routing depend on // mutable state, this must grow the lane parameter its siblings carry. Route check_can_serve(Rpc rpc, const RequestShape &shape) const; // Routes the RPC and returns the cached session, creating it on first use. // Throws CapabilityError when this family cannot serve the RPC, and a plain // runtime_error when it can but the session could not be built, which is an // environment fault rather than a capability answer. // // The `lane` parameter is a PROOF OF HOLDING and is otherwise unused: it // exists so the rule below is a compile error rather than prose. The // session cache is an unsynchronised std::map and the sessions themselves // are not reentrant, so this must only be called with the lane held; the // lane admits one caller at a time, which is exactly the constraint the // sessions impose. Pass the LaneEntry from acquire(). // // NON-CONST reference on purpose, and do not "tidy" it to const. A const // reference binds to a temporary, which makes this compile: // // auto session = model->session_for(rpc, shape, model->acquire(0)); // auto result = run_offline(session, task, model->acquire(0)); // // and each temporary dies at the end of its own full-expression, so the // lane is released between the two calls. That is precisely the split this // parameter exists to prevent, and it is the form a future caller is most // likely to reach for because it reads as tidy. Requiring an lvalue forces // a named entry whose scope spans both calls. // // What it proves is bounded, so do not over-trust it: it proves A lane was // taken, not THIS model's lane. A caller determined to defeat it can // construct an entry on an unrelated InferenceLane and pass that. It // therefore catches the two mistakes that actually happen, forgetting the // lane entirely and taking it after routing, and does not catch lane // identity. // // STATE CONTRACT, and it is the CALLER'S to honour. Sessions are cached per // (task, mode), so a streaming session is normally the same warm object the // previous stream used, carrying that stream's state. session_for hands it // back as it is. // // Every streaming caller must therefore begin a stream through // begin_stream() below, which is prepare() then start_stream() in that // order and is the ONLY implementation of that sequence. start_stream's // base implementation is a call to reset(), which is what clears the // previous stream, and reset() is only legal after prepare(): silero_vad // throws "session prepare() must be called before Silero VAD reset()" // otherwise. That ordering constraint is also why session_for cannot do // this for you. Skipping it does not raise an error, it silently continues // the previous stream. // // Offline sessions need no such care: their interface has no reset and // run() takes a whole request. Session session_for(Rpc rpc, const RequestShape &shape, LaneEntry &lane); // Takes the inference lane, or throws LaneUnavailable. Serializes runs // against this model. `requested_timeout_ms` is a per-request wait hint // where a value <= 0 means "use the model's configured ceiling"; a hint may // only tighten that ceiling, never loosen it. // // LaneEntry is deliberately immovable, so bind the result to a named local // in the scope the inference happens in: // // LaneEntry entry = model.acquire(request_hint_ms); // // which C++17 initializes in place. A handler that has to keep the lane // beyond one scope, for instance in a member that outlives the call that // took it, wants acquire_owned instead. LaneEntry acquire(int requested_timeout_ms); // Same lane, heap-allocated so it can be stored or handed on. Prefer // acquire: this one adds a null state that the scoped form does not have. std::unique_ptr acquire_owned(int requested_timeout_ms); private: // Keyed by the enum values so the map needs no custom comparator. using SessionKey = std::pair; // The public constructor runs the load gate, then delegates here. The // detour exists because lane_ has to be built from the family in the member // initializer list, and the family is only known after the gate has run. // Four parameters rather than three so it cannot be confused with the // public constructor, whose third argument is also a std::string. LoadedModel(const std::string &resolved_path, const ModelOptions &options, std::string family, std::string model_identity); // MEMBER ORDER IS LOAD-BEARING BELOW THIS LINE. Members are destroyed in // reverse declaration order. // // lane_ is first so it is destroyed last: nothing that runs during teardown // can then find a lane that has already gone. InferenceLane lane_; // registry_ before model_: the registry owns the loader that produced the // model, and the model may hold loader-owned state. engine::runtime::ModelRegistry registry_; // model_ before sessions_, so sessions_ is destroyed FIRST and the model // second. A session is created from the model and must not outlive it. Do // not reorder these two. std::unique_ptr model_; std::map> sessions_; engine::runtime::SessionOptions session_options_; Capabilities capabilities_; std::string variant_; std::string description_; std::vector languages_; std::string pinned_task_; std::string identity_; bool supports_timestamps_ = false; int wait_budget_ceiling_ms_ = 0; int live_idle_timeout_ms_ = 0; }; // Prepares and runs an offline session. prepare() is called for every run // rather than once per session, because SessionPreparationRequest is derived // from the request itself (audio contract, text, voice condition) and not from // the model: a second request with a different sample rate or length would // otherwise run against the first request's contract. // // `lane` is a PROOF OF HOLDING, unused at runtime, for the same reason // session_for takes one: the session is not reentrant and prepare() mutates it, // so running without the lane is a data race. Making it a parameter turns that // into a compile error instead of a comment. Non-const for the same reason as // session_for's: a const reference would bind to `model.acquire(0)` written // inline, and that temporary dies at the end of this call, releasing the lane // before the caller's next one. // // Throws CapabilityError when the session is not an offline one. That should be // unreachable through session_for, which already refuses a non-offline session // for an offline route, and is checked anyway because the alternative is a null // dereference. engine::runtime::TaskResult run_offline(const LoadedModel::Session &session, const engine::runtime::TaskRequest &request, LaneEntry &lane); // THE ONE IMPLEMENTATION of the streaming state obligation described in // session_for's STATE CONTRACT: prepare(), then start_stream(), in that order. // // It is a function rather than a comment because the obligation is invisible // when it is broken. Streaming sessions are CACHED per (task, mode), so the // object a second stream gets is the warm one the first stream left behind, // still holding its audio, its tokens and its started flag. What clears it is // start_stream, whose base implementation IS a reset() and whose seven family // overrides (nemotron_asr, vibevoice_asr, higgs_audio_stt, voxtral_realtime, // supertonic, omnivoice, voxcpm2) every one call reset() as their first // statement, verified in the pinned checkout. Nothing in the type system pins // that. A future override that dropped the reset would break every call site // at once with no compile error and no exception, only a second transcript // that begins with the first one's audio, so the fewer call sites there are to // break, the better: this is the only one. // // prepare() must come first and cannot be folded into session_for, because // reset() is illegal before prepare() (silero_vad throws "session prepare() // must be called before Silero VAD reset()"), and because the preparation // request is derived from the REQUEST, not the model: build_preparation_request // reads the audio contract, the text and the voice condition off it, so a // second stream with a different sample rate or length would otherwise run // against the first stream's contract. // // `lane` is a PROOF OF HOLDING, unused at runtime, exactly as in run_offline. // // Throws CapabilityError when the session is not a streaming one. void begin_stream(const LoadedModel::Session &session, const engine::runtime::TaskRequest &request, LaneEntry &lane); // Drives a streaming session that takes NO incremental input, which is the TTS // shape (StreamingInputKind::None, StreamingOutputKind::PullEvents): begin the // stream, pull events until the session says there are no more, then finish. // // NO STREAM EVENT SINK IS INSTALLED HERE, and that is deliberate rather than an // omission. voxcpm2's start_stream runs the whole synthesis and pushes every // chunk to the sink, then its next_stream_event replays those same chunks out // of the stored result, so a sink on this path would put every chunk of audio // on the wire twice. supertonic and omnivoice ignore set_stream_event_sink // outright. The pull loop is therefore the single delivery channel. // // The returned TaskResult is the session's own merged whole for all three // families, NOT a tail the pull loop missed. A caller that already emitted the // pulled events must not also emit its audio; see the TTSStream handler. engine::runtime::TaskResult run_streaming_pull( const LoadedModel::Session &session, const engine::runtime::TaskRequest &request, const std::function &on_event, LaneEntry &lane); // Drives a streaming session that CONSUMES audio chunks, which is the ASR shape // (StreamingInputKind::AudioChunks): begin the stream, feed the buffer in // policy-sized chunks, then finalize. // // A STREAM EVENT SINK IS INSTALLED HERE, and it is not optional: nemotron_asr // reports its partial text ONLY through the sink, and only from inside // finalize(), because its decode does not start until the audio is complete. // Without the sink that family streams a transcript with no partials at all. // The sink is cleared again before returning, including on the exception path: // the session is cached and outlives this call, so a sink left holding a // reference to the caller's frame is a use after free waiting for the next // stream. // // Both delivery channels are consumed, the sink and the value process_audio_chunk // returns, because the families do not agree on which they use, and // voxtral_realtime uses BOTH for the same event. The duplicate that produces is // absorbed by TranscriptDeltaTracker in stream_delta.h rather than here. engine::runtime::TaskResult run_streaming_audio( const LoadedModel::Session &session, const engine::runtime::TaskRequest &request, const engine::runtime::AudioBuffer &audio, const std::function &on_event, LaneEntry &lane); // Drives the same ASR shape as run_streaming_audio when the audio DOES NOT // EXIST YET, which is the live-microphone case: instead of slicing a buffer it // pulls frames from the caller until the input side closes. // // `next_frames` fills `out` with interleaved float PCM and returns true, or // returns false when there is no more input. It is expected to BLOCK, since the // only real implementation is a gRPC stream Read, and it may throw: a request // the handler has to refuse mid-stream unwinds through here, and the sink is // cleared on that path like every other. // // The audio contract comes from `request.audio_input`, which for a live stream // is an EMPTY buffer carrying only the sample rate and channel count. It is not // optional: nemotron_asr's streaming prepare() throws "Nemotron ASR streaming // prepare() requires an audio contract" without one, and there is no buffer to // derive it from here. // // Frames are BUFFERED to the family's own preferred window rather than fed in // whatever sizes the wire delivered them in, because that window is a family's // statement about what it can decode (nemotron_asr asks for one second, higgs // for four), and a 512-sample gRPC frame is a property of the client's audio // callback rather than of the model. The tail shorter than a window is fed at // the end. // // A stream that carried NO AUDIO returns an empty TaskResult and never calls // finish_stream. Finalizing an empty stream is not universally legal: // nemotron_asr throws "Nemotron ASR finalize requires streamed audio", so a // client that opens a session and closes it without speaking would receive an // INTERNAL naming an engine internal instead of an empty transcript, which is // the truthful answer to "transcribe nothing". engine::runtime::TaskResult run_streaming_live( const LoadedModel::Session &session, const engine::runtime::TaskRequest &request, const std::function &)> &next_frames, const std::function &on_event, LaneEntry &lane); } // namespace audiocpp_backend