mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 09:08:54 -04:00
mudler-agentandEttore Di Giacinto
849c177834
Bump LLAMA_VERSION to b76b13379d3b (upstream 0.7.0 plus later commits). This is the same one-line change as the bot PR #12725. Add the moe_cache_mib model option. It sets upstream's moe_cache_size, a GPU cache for MoE experts kept in host memory, from a size in MiB. It is off by default. Invalid, negative or non-numeric values are ignored with a log line. Forks that predate the field ignore the option, using the same prepare.sh probe as the other fork-skew switches. Document the option, and cover the MiB to bytes conversion with a standalone unit test. Assisted-by: Claude:claude-sonnet-5-5 g++ Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
36 lines
1.1 KiB
C++
36 lines
1.1 KiB
C++
// SPDX-License-Identifier: MIT
|
|
|
|
#pragma once
|
|
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <exception>
|
|
#include <optional>
|
|
#include <string>
|
|
|
|
namespace llama_grpc {
|
|
|
|
// parse_moe_cache_bytes converts the value of the moe_cache_mib model option
|
|
// (a whole number of MiB) to the byte count that common_params::moe_cache_size
|
|
// expects. It returns std::nullopt for anything that is not a non-negative
|
|
// integer that fits in size_t once converted, so the caller can log and keep
|
|
// the default (0, which means the cache is off).
|
|
inline std::optional<size_t> parse_moe_cache_bytes(const std::string & value) {
|
|
constexpr uint64_t MIB = 1024ull * 1024ull;
|
|
try {
|
|
size_t used = 0;
|
|
const long long mib = std::stoll(value, &used);
|
|
if (used != value.size() || mib < 0) {
|
|
return std::nullopt;
|
|
}
|
|
if (static_cast<unsigned long long>(mib) > SIZE_MAX / MIB) {
|
|
return std::nullopt;
|
|
}
|
|
return static_cast<size_t>(mib) * static_cast<size_t>(MIB);
|
|
} catch (const std::exception &) {
|
|
return std::nullopt;
|
|
}
|
|
}
|
|
|
|
} // namespace llama_grpc
|