mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 09:08:54 -04:00
mudler-agentandEttore Di Giacinto
849c177834
Bump LLAMA_VERSION to b76b13379d3b (upstream 0.7.0 plus later commits). This is the same one-line change as the bot PR #12725. Add the moe_cache_mib model option. It sets upstream's moe_cache_size, a GPU cache for MoE experts kept in host memory, from a size in MiB. It is off by default. Invalid, negative or non-numeric values are ignored with a log line. Forks that predate the field ignore the option, using the same prepare.sh probe as the other fork-skew switches. Document the option, and cover the MiB to bytes conversion with a standalone unit test. Assisted-by: Claude:claude-sonnet-5-5 g++ Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
37 lines
1.0 KiB
C++
37 lines
1.0 KiB
C++
// SPDX-License-Identifier: MIT
|
|
|
|
#include "moe_cache_option.h"
|
|
|
|
#include <cstdio>
|
|
|
|
static int failures = 0;
|
|
|
|
static void expect(const std::string & in, std::optional<size_t> want) {
|
|
const auto got = llama_grpc::parse_moe_cache_bytes(in);
|
|
if (got != want) {
|
|
std::fprintf(stderr, "parse_moe_cache_bytes(\"%s\") returned a wrong result\n", in.c_str());
|
|
failures++;
|
|
}
|
|
}
|
|
|
|
int main() {
|
|
// MiB to bytes.
|
|
expect("0", 0);
|
|
expect("1", 1024ull * 1024ull);
|
|
expect("2048", 2048ull * 1024ull * 1024ull);
|
|
// 8 GiB must not overflow a 32-bit intermediate.
|
|
expect("8192", 8192ull * 1024ull * 1024ull);
|
|
|
|
// Invalid values are rejected so the caller keeps the default.
|
|
expect("-1", std::nullopt);
|
|
expect("", std::nullopt);
|
|
expect("abc", std::nullopt);
|
|
expect("12abc", std::nullopt);
|
|
expect("1.5", std::nullopt);
|
|
expect("99999999999999999999999", std::nullopt);
|
|
// Too large to express in bytes.
|
|
expect("9223372036854775807", std::nullopt);
|
|
|
|
return failures == 0 ? 0 : 1;
|
|
}
|