diff --git a/python/freetoken/engine/cache_budget.py b/python/freetoken/engine/cache_budget.py index ab7c0a9f1..8f85cf44d 100644 --- a/python/freetoken/engine/cache_budget.py +++ b/python/freetoken/engine/cache_budget.py @@ -118,7 +118,9 @@ def resolve_moe_cache_auto( """ budget_bytes = net_cache_budget_bytes(memory_ratio, baseline_free, weights_bytes, fixed_cache_size) max_slots = 992 if quant_format == "nvfp4_marlin" else total_experts - kv_reserve_pages = div_ceil(kv_reserve_tokens, page_size) + # Every pool keeps page 0 as an unreachable dummy/sentinel. The CLI floor is expressed in + # usable tokens, so reserve that internal page in addition to the user-visible capacity. + kv_reserve_pages = div_ceil(kv_reserve_tokens, page_size) + 1 return plan_cache_budget( budget_bytes=budget_bytes, per_expert_bytes=per_expert_bytes, diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py index 5b4db587d..ab0e0cd63 100644 --- a/python/freetoken/server/args.py +++ b/python/freetoken/server/args.py @@ -537,7 +537,10 @@ def _infer_reasoning_parser(model_path: str) -> str | None: "--kv-reserve-tokens", type=int, default=ServerArgs.kv_reserve_tokens, - help="KV-cache token floor reserved before --moe-cache-auto fills experts.", + help=( + "Usable KV-cache token floor reserved before --moe-cache-auto fills experts " + "(the internal dummy page is additional)." + ), ) parser.add_argument( diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.py index 9ac2a4f4c..40e1055b2 100644 --- a/tests/engine/test_cache_budget.py +++ b/tests/engine/test_cache_budget.py @@ -112,6 +112,26 @@ def test_resolve_auto_applies_ratio_once_and_marlin_cap(): assert size == 8 and pages == 40 and overlap is True +def test_resolve_auto_reserves_usable_tokens_beyond_the_dummy_page(): + size, pages, overlap = resolve_moe_cache_auto( + baseline_free=940, + weights_bytes=0, + memory_ratio=1.0, + cache_per_page=10, + fixed_cache_size=0, + per_expert_bytes=100, + num_experts=2, + total_experts=50, + prefill_overlap=False, + kv_reserve_tokens=256, + page_size=64, + quant_format="bf16", + ) + assert overlap is False + assert (pages - 1) * 64 >= 256 + assert size == 8 and pages == 14 + + def test_resolve_auto_marlin_caps_slots(): size, _, _ = resolve_moe_cache_auto( baseline_free=10_000_000, weights_bytes=0, memory_ratio=1.0,