owen/longhaul.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Owen Qwen
2026-07-30 21:20:05 -05:00
parent d5458aa44e
commit 22f42628dd
25 changed files with 1038 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-longhaul.cpp
+20 -11
View File
@@ -191,12 +191,13 @@ struct cpu_decode_result {
bool expert_buffer_is_host = false;
};
static cpu_decode_result decode_cpu_model(
static cpu_decode_result decode_model(
const std::string & path,
llama_load_mode load_mode,
uint64_t cache_bytes) {
uint64_t cache_bytes,
int32_t n_gpu_layers) {
llama_model_params model_params = llama_model_default_params();
model_params.n_gpu_layers = 0;
model_params.n_gpu_layers = n_gpu_layers;
model_params.load_mode = load_mode;
model_params.longhaul_cache_bytes = cache_bytes;
model_params.use_extra_bufts = true;
@@ -256,19 +257,24 @@ static cpu_decode_result decode_cpu_model(
return result;
}
static void test_cpu_model(
static void test_model(
testing & t,
const std::string & path,
uint64_t cache_bytes) {
uint64_t cache_bytes,
int32_t n_gpu_layers) {
const cpu_decode_result regular =
decode_cpu_model(path, LLAMA_LOAD_MODE_NONE, 0);
decode_model(path, LLAMA_LOAD_MODE_NONE, 0, n_gpu_layers);
const cpu_decode_result longhaul =
decode_cpu_model(path, LLAMA_LOAD_MODE_LONGHAUL, cache_bytes);
decode_model(path, LLAMA_LOAD_MODE_LONGHAUL, cache_bytes, n_gpu_layers);
t.assert_equal(1u, longhaul.capacity);
t.assert_true("longhaul loaded at least one expert", longhaul.misses > 0);
t.assert_true("longhaul read expert bytes", longhaul.bytes_read > 0);
t.assert_true("streamed experts use a directly writable host buffer", longhaul.expert_buffer_is_host);
if (n_gpu_layers == 0) {
t.assert_true(
"streamed CPU experts use a directly writable host buffer",
longhaul.expert_buffer_is_host);
}
t.assert_equal(regular.logits.size(), longhaul.logits.size());
double squared_error = 0.0;
@@ -296,11 +302,14 @@ int main(int argc, char ** argv) {
llama_log_set([](ggml_log_level, const char *, void *) {}, nullptr);
}
if (argc == 4 && std::string(argv[1]) == "--cpu-model") {
if (argc == 4 &&
(std::string(argv[1]) == "--cpu-model" ||
std::string(argv[1]) == "--metal-model")) {
const std::string path = argv[2];
const uint64_t cache_bytes = std::stoull(argv[3]);
t.test("cpu_model", [&](testing & current) {
test_cpu_model(current, path, cache_bytes);
const bool metal = std::string(argv[1]) == "--metal-model";
t.test(metal ? "metal_model" : "cpu_model", [&](testing & current) {
test_model(current, path, cache_bytes, metal ? 99 : 0);
});
const int result = t.summary();
llama_backend_free();