Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions tests/test_kv_capacity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
"""Final KV capacity arithmetic for startup sizing (CPU-only)."""

import pytest

from vllm_rlt.core.memory import budget_blocks, kv_capacity

MIB = 1024**2


def test_kv_capacity_subtracts_every_overhead_from_the_tighter_free_bound():
overheads = {"peak": 3 * MIB, "graph": 5 * MIB, "metadata": 1 * MIB}
# Free memory is the binding bound here: min(64, 100 - 20) = 64 MiB.
blocks, budget = kv_capacity(
free_bytes=64 * MIB,
cap_bytes=100 * MIB,
reserved_bytes=20 * MIB,
per_block=MIB,
overheads=overheads,
useful_blocks=10**6,
)
assert budget == 64 * MIB - 9 * MIB
assert blocks == 55


def test_kv_capacity_uses_utilization_cap_when_it_is_tighter():
blocks, budget = kv_capacity(
free_bytes=512 * MIB,
cap_bytes=100 * MIB,
reserved_bytes=20 * MIB,
per_block=2 * MIB,
overheads={"graph": 10 * MIB},
useful_blocks=10**6,
)
assert budget == 70 * MIB
assert blocks == 35


def test_kv_capacity_is_capped_by_useful_blocks_without_changing_budget():
blocks, budget = kv_capacity(
free_bytes=100 * MIB,
cap_bytes=100 * MIB,
reserved_bytes=0,
per_block=MIB,
overheads={},
useful_blocks=7,
)
assert blocks == 7
assert budget == 100 * MIB


def test_kv_capacity_raises_when_no_block_fits():
with pytest.raises(ValueError, match="no memory remains for KV blocks"):
kv_capacity(
free_bytes=4 * MIB,
cap_bytes=100 * MIB,
reserved_bytes=0,
per_block=MIB,
overheads={"graph": 4 * MIB},
useful_blocks=10,
)


def test_budget_blocks_floors_partial_blocks():
assert budget_blocks(5 * MIB + 1, MIB) == 5
with pytest.raises(ValueError):
budget_blocks(MIB - 1, MIB)
48 changes: 33 additions & 15 deletions vllm_rlt/core/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,17 @@ def budget_blocks(available_bytes, bytes_per_block):
return blocks


def kv_capacity(*, free_bytes, cap_bytes, reserved_bytes, per_block, overheads, useful_blocks):
"""Blocks that fit after measured free memory, the cap and every overhead.

``overheads`` are non-KV bytes that must remain available. Returns the block
count and the KV budget in bytes before block rounding.
"""
budget = min(free_bytes, cap_bytes - reserved_bytes) - sum(overheads.values())
blocks = min(budget_blocks(budget, per_block), useful_blocks)
return blocks, int(budget)


def graph_memory_reserve(model, scheduler, execution, speculative=None):
"""Headroom per pool plus retained speculative coda logits across all entries.

Expand Down Expand Up @@ -154,33 +165,40 @@ def plan_cache(
metadata = 3 * (
(1 << (wide_capacity - 1).bit_length()) * 8 + (1 << (narrow_capacity - 1).bit_length()) * 4
)
budget = min(free_before, int(total * cache.gpu_memory_utilization) - reserved)
concurrent_peak = peak * (2 if execution.async_scheduling and execution.multi_stream else 1)
graph_reserve = graph_memory_reserve(model, scheduler, execution, speculative)
budget -= (
concurrent_peak
+ context_scratch
+ buffers
+ metadata
+ cache.memory_reserve_bytes
+ graph_reserve
)
concurrent_factor = 2 if execution.async_scheduling and execution.multi_stream else 1
overheads = {
"concurrent_peak_bytes": peak * concurrent_factor,
"context_scratch_bytes": context_scratch,
"static_buffer_bytes": buffers,
"metadata_bytes": metadata,
"memory_reserve_bytes": cache.memory_reserve_bytes,
"cuda_graph_reserve_bytes": graph_memory_reserve(model, scheduler, execution, speculative),
}
# More blocks than all concurrent full-context requests can use only waste
# memory, particularly for tiny-model GPU tests with very small pages.
useful_blocks = (
math.ceil(config.max_position_embeddings / cache.block_size)
* (config.total_ut_steps if cache.layout == "last_exited" else 1)
* scheduler.max_num_seqs
)
blocks = min(budget_blocks(budget, per_block), useful_blocks)
blocks, budget = kv_capacity(
free_bytes=free_before,
cap_bytes=int(total * cache.gpu_memory_utilization),
reserved_bytes=reserved,
per_block=per_block,
overheads=overheads,
useful_blocks=useful_blocks,
)
return blocks, dict(
source="cuda_profile",
bytes_per_block=per_block,
profile_peak_bytes=peak,
concurrent_peak_bytes=concurrent_peak,
concurrent_peak_bytes=overheads["concurrent_peak_bytes"],
max_useful_blocks=useful_blocks,
cuda_graph_reserve_bytes=graph_reserve,
cuda_graph_reserve_bytes=overheads["cuda_graph_reserve_bytes"],
static_buffer_bytes=buffers,
context_scratch_bytes=context_scratch,
kv_budget_bytes=int(budget),
metadata_bytes=metadata,
kv_budget_bytes=budget,
kv_committed_bytes=blocks * per_block,
)
Loading