diff --git a/tests/test_kv_capacity.py b/tests/test_kv_capacity.py new file mode 100644 index 00000000..9f3cb605 --- /dev/null +++ b/tests/test_kv_capacity.py @@ -0,0 +1,66 @@ +"""Final KV capacity arithmetic for startup sizing (CPU-only).""" + +import pytest + +from vllm_rlt.core.memory import budget_blocks, kv_capacity + +MIB = 1024**2 + + +def test_kv_capacity_subtracts_every_overhead_from_the_tighter_free_bound(): + overheads = {"peak": 3 * MIB, "graph": 5 * MIB, "metadata": 1 * MIB} + # Free memory is the binding bound here: min(64, 100 - 20) = 64 MiB. + blocks, budget = kv_capacity( + free_bytes=64 * MIB, + cap_bytes=100 * MIB, + reserved_bytes=20 * MIB, + per_block=MIB, + overheads=overheads, + useful_blocks=10**6, + ) + assert budget == 64 * MIB - 9 * MIB + assert blocks == 55 + + +def test_kv_capacity_uses_utilization_cap_when_it_is_tighter(): + blocks, budget = kv_capacity( + free_bytes=512 * MIB, + cap_bytes=100 * MIB, + reserved_bytes=20 * MIB, + per_block=2 * MIB, + overheads={"graph": 10 * MIB}, + useful_blocks=10**6, + ) + assert budget == 70 * MIB + assert blocks == 35 + + +def test_kv_capacity_is_capped_by_useful_blocks_without_changing_budget(): + blocks, budget = kv_capacity( + free_bytes=100 * MIB, + cap_bytes=100 * MIB, + reserved_bytes=0, + per_block=MIB, + overheads={}, + useful_blocks=7, + ) + assert blocks == 7 + assert budget == 100 * MIB + + +def test_kv_capacity_raises_when_no_block_fits(): + with pytest.raises(ValueError, match="no memory remains for KV blocks"): + kv_capacity( + free_bytes=4 * MIB, + cap_bytes=100 * MIB, + reserved_bytes=0, + per_block=MIB, + overheads={"graph": 4 * MIB}, + useful_blocks=10, + ) + + +def test_budget_blocks_floors_partial_blocks(): + assert budget_blocks(5 * MIB + 1, MIB) == 5 + with pytest.raises(ValueError): + budget_blocks(MIB - 1, MIB) diff --git a/vllm_rlt/core/memory.py b/vllm_rlt/core/memory.py index 1f7ed1e0..51df6a85 100644 --- a/vllm_rlt/core/memory.py +++ b/vllm_rlt/core/memory.py @@ -19,6 +19,17 @@ def budget_blocks(available_bytes, bytes_per_block): return blocks +def kv_capacity(*, free_bytes, cap_bytes, reserved_bytes, per_block, overheads, useful_blocks): + """Blocks that fit after measured free memory, the cap and every overhead. + + ``overheads`` are non-KV bytes that must remain available. Returns the block + count and the KV budget in bytes before block rounding. + """ + budget = min(free_bytes, cap_bytes - reserved_bytes) - sum(overheads.values()) + blocks = min(budget_blocks(budget, per_block), useful_blocks) + return blocks, int(budget) + + def graph_memory_reserve(model, scheduler, execution, speculative=None): """Headroom per pool plus retained speculative coda logits across all entries. @@ -154,17 +165,15 @@ def plan_cache( metadata = 3 * ( (1 << (wide_capacity - 1).bit_length()) * 8 + (1 << (narrow_capacity - 1).bit_length()) * 4 ) - budget = min(free_before, int(total * cache.gpu_memory_utilization) - reserved) - concurrent_peak = peak * (2 if execution.async_scheduling and execution.multi_stream else 1) - graph_reserve = graph_memory_reserve(model, scheduler, execution, speculative) - budget -= ( - concurrent_peak - + context_scratch - + buffers - + metadata - + cache.memory_reserve_bytes - + graph_reserve - ) + concurrent_factor = 2 if execution.async_scheduling and execution.multi_stream else 1 + overheads = { + "concurrent_peak_bytes": peak * concurrent_factor, + "context_scratch_bytes": context_scratch, + "static_buffer_bytes": buffers, + "metadata_bytes": metadata, + "memory_reserve_bytes": cache.memory_reserve_bytes, + "cuda_graph_reserve_bytes": graph_memory_reserve(model, scheduler, execution, speculative), + } # More blocks than all concurrent full-context requests can use only waste # memory, particularly for tiny-model GPU tests with very small pages. useful_blocks = ( @@ -172,15 +181,24 @@ def plan_cache( * (config.total_ut_steps if cache.layout == "last_exited" else 1) * scheduler.max_num_seqs ) - blocks = min(budget_blocks(budget, per_block), useful_blocks) + blocks, budget = kv_capacity( + free_bytes=free_before, + cap_bytes=int(total * cache.gpu_memory_utilization), + reserved_bytes=reserved, + per_block=per_block, + overheads=overheads, + useful_blocks=useful_blocks, + ) return blocks, dict( source="cuda_profile", bytes_per_block=per_block, profile_peak_bytes=peak, - concurrent_peak_bytes=concurrent_peak, + concurrent_peak_bytes=overheads["concurrent_peak_bytes"], max_useful_blocks=useful_blocks, - cuda_graph_reserve_bytes=graph_reserve, + cuda_graph_reserve_bytes=overheads["cuda_graph_reserve_bytes"], static_buffer_bytes=buffers, context_scratch_bytes=context_scratch, - kv_budget_bytes=int(budget), + metadata_bytes=metadata, + kv_budget_bytes=budget, + kv_committed_bytes=blocks * per_block, )