perf: accelerate longhaul prompt loading

2026-07-29 12:44:50 -05:00
parent c2baa66534
commit fdd9fcf85e
7 changed files with 525 additions and 55 deletions
@@ -56,6 +56,7 @@ test parameters:
  -ub, --ubatch-size <n>                    (default: 512)
  -ctk, --cache-type-k <t>                  (default: f16)
  -ctv, --cache-type-v <t>                  (default: f16)
+  --longhaul-cache <GiB>                    expert cache size for longhaul mode
  -t, --threads <n>                         (default: system dependent)
  -C, --cpu-mask <hex,hex>                  (default: 0x0)
  --cpu-strict <0|1>                        (default: 0)