Files
LocalAI/backend/cpp/audio-cpp/CMakeLists.txt
Ettore Di Giacinto 87387d91de backend(audio-cpp): serialize inference through an InferenceLane
One audio.cpp model is loaded per backend process and its sessions are not
reentrant, so concurrent gRPC handlers have to take turns. Serialization alone
is not enough: a wedged GPU call cannot be cancelled from userspace, so an
unbounded queue behind one stuck run would swallow every gRPC worker thread
until the process is useless.

InferenceLane gives handlers a lane with room for one runner. LaneEntry occupies
it for a scope and gives it back on every exit, including an exception, and is
the only way to take the lane at all: occupy/vacate are private with LaneEntry
as the sole friend, so a caller cannot acquire without holding something that
releases. LaneEntry is immovable on purpose, because a moved-from entry would
have to stop releasing while the lane still recorded it as occupied.

A caller either waits indefinitely or brings a millisecond budget. A bounded
caller that cannot get in fails instead of waiting on, and a bounded caller
whose budget is already shorter than the age of the run in the lane fails
immediately, which is what stops a queue forming behind a wedged run. The two
failures carry different text: one names the wait it exhausted, the other states
the measured age of the run without claiming to know why it is long, since a
short budget meeting a legitimately long run lands there too.

The run's age is stamped only after acquisition. A waiter that published itself
as holder would restart the measurement and hide a genuinely stuck holder from
every caller behind it.

Budget negotiation and the overrun decision are pure functions taking their
inputs explicitly, so both are covered without threads or sleeping. The
per-model ceiling arrives as an int of milliseconds; a request may tighten it
and may never loosen it.

Replaces the previous run_guard unit, which was a derivative of an
Apache-2.0 file upstream and could not stay in an MIT tree. Written from a
behaviour contract with no reference to the removed code.

Tests: 65 checks, standard library only, single translation unit, clean under
-Wall -Wextra. Mutation tested at 23/23 killed; two of those mutants exposed
missing coverage and the tests were extended until they died. ThreadSanitizer
clean.

Assisted-by: Claude:claude-opus-5 [Claude Code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
2026-07-26 12:25:32 +00:00

125 lines
4.8 KiB
CMake

cmake_minimum_required(VERSION 3.20)
project(audio-cpp-grpc-server LANGUAGES C CXX)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(TARGET grpc-server)
set(AUDIO_CPP_DIR "${CMAKE_CURRENT_SOURCE_DIR}/audio.cpp"
CACHE PATH "Path to the pinned audio.cpp checkout")
option(AUDIO_CPP_GRPC_BUILD_TESTS "Build engine-linked ctest binaries" OFF)
if(NOT EXISTS "${AUDIO_CPP_DIR}/CMakeLists.txt")
message(FATAL_ERROR
"AUDIO_CPP_DIR does not contain an audio.cpp checkout: ${AUDIO_CPP_DIR}. "
"Run 'make audio.cpp' first.")
endif()
if(APPLE)
# Homebrew installs protobuf/grpc under a non-default prefix.
if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64")
set(HOMEBREW_DEFAULT_PREFIX "/opt/homebrew")
else()
set(HOMEBREW_DEFAULT_PREFIX "/usr/local")
endif()
link_directories("${HOMEBREW_DEFAULT_PREFIX}/lib")
include_directories("${HOMEBREW_DEFAULT_PREFIX}/include")
endif()
find_package(Threads REQUIRED)
find_package(Protobuf CONFIG QUIET)
if(NOT Protobuf_FOUND)
find_package(Protobuf REQUIRED)
endif()
find_package(gRPC CONFIG QUIET)
if(NOT gRPC_FOUND)
# Reached only on distros whose grpc++ packaging ships no CMake config.
# Ubuntu's libgrpc-dev does ship one, so this is dead code on LocalAI's own
# build distro. Kept for the distros that do not.
find_library(GRPCPP_LIB grpc++ REQUIRED)
find_library(GRPCPP_REFLECTION_LIB grpc++_reflection REQUIRED)
add_library(gRPC::grpc++ INTERFACE IMPORTED)
set_target_properties(gRPC::grpc++ PROPERTIES
INTERFACE_LINK_LIBRARIES "${GRPCPP_LIB}")
add_library(gRPC::grpc++_reflection INTERFACE IMPORTED)
set_target_properties(gRPC::grpc++_reflection PROPERTIES
INTERFACE_LINK_LIBRARIES "${GRPCPP_REFLECTION_LIB}")
endif()
find_program(_PROTOC NAMES protoc REQUIRED)
find_program(_GRPC_CPP_PLUGIN NAMES grpc_cpp_plugin REQUIRED)
get_filename_component(HW_PROTO "${CMAKE_CURRENT_SOURCE_DIR}/../../backend.proto" ABSOLUTE)
get_filename_component(HW_PROTO_PATH "${HW_PROTO}" PATH)
set(HW_PROTO_SRCS "${CMAKE_CURRENT_BINARY_DIR}/backend.pb.cc")
set(HW_PROTO_HDRS "${CMAKE_CURRENT_BINARY_DIR}/backend.pb.h")
set(HW_GRPC_SRCS "${CMAKE_CURRENT_BINARY_DIR}/backend.grpc.pb.cc")
set(HW_GRPC_HDRS "${CMAKE_CURRENT_BINARY_DIR}/backend.grpc.pb.h")
add_custom_command(
OUTPUT "${HW_PROTO_SRCS}" "${HW_PROTO_HDRS}" "${HW_GRPC_SRCS}" "${HW_GRPC_HDRS}"
COMMAND ${_PROTOC}
ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
--cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
-I "${HW_PROTO_PATH}"
--plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN}"
"${HW_PROTO}"
DEPENDS "${HW_PROTO}")
add_library(hw_grpc_proto STATIC
${HW_GRPC_SRCS} ${HW_GRPC_HDRS}
${HW_PROTO_SRCS} ${HW_PROTO_HDRS})
target_include_directories(hw_grpc_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
# Required on macOS: without these the Homebrew protobuf/grpc include dirs never
# reach this target and google/protobuf/runtime_version.h is not found.
target_link_libraries(hw_grpc_proto PUBLIC protobuf::libprotobuf gRPC::grpc++)
# Upstream's global add_compile_options(-Wall -Wextra -Wpedantic -pedantic-errors)
# is a directory property of the subdirectory and does not reach our targets.
#
# EXCLUDE_FROM_ALL is load-bearing, do not drop it: upstream's default target set
# includes its CLI, server, converter and test binaries, none of which we ship.
# Without it every build would compile all of them. The targets we do name in
# target_link_libraries below are still built on demand, so nothing is lost.
add_subdirectory("${AUDIO_CPP_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/audio-cpp" EXCLUDE_FROM_ALL)
add_executable(${TARGET}
grpc-server.cpp
model_options.cpp
capability_routing.cpp
audio_units.cpp
transcript_assembly.cpp
inference_lane.cpp
)
target_include_directories(${TARGET} PRIVATE
"${AUDIO_CPP_DIR}/include"
"${CMAKE_CURRENT_SOURCE_DIR}")
target_link_libraries(${TARGET} PRIVATE
hw_grpc_proto
engine_runtime
ggml
gRPC::grpc++
gRPC::grpc++_reflection
protobuf::libprotobuf
Threads::Threads)
# ENGINE_ENABLE_CPU_ALL_VARIANTS builds ggml backends as shared objects that sit
# next to the binary in the package, so the binary must search its own directory.
# BUILD_WITH_INSTALL_RPATH keeps the build-tree binary at exactly "$ORIGIN".
# Upstream sets CMAKE_BUILD_WITH_INSTALL_RPATH in its own directory scope, which
# does not reach ours, so without this CMake also appends its build-tree library
# directory. That absolute build-host path would survive into the copied binary
# and let a package.sh that forgot to bundle libggml*.so still pass on the build
# machine while failing everywhere else.
set_target_properties(${TARGET} PROPERTIES
BUILD_RPATH "$ORIGIN"
INSTALL_RPATH "$ORIGIN"
BUILD_WITH_INSTALL_RPATH TRUE)
if(AUDIO_CPP_GRPC_BUILD_TESTS)
enable_testing()
endif()