From a608e32782a34047589c959206082b211c566a07 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 12 Nov 2026 12:30:51 +0100 Subject: [PATCH 1/5] feat(dory): synchronous block and token-pipeline self-speculation Dory's input and output stages hold attention KV, so the speculative runner now passes boundary metadata to PRELUDE and CODA. A draft runs the target's first loops over per-loop KV and its prelude depends only on the input tokens, so both are reused by verification. A draft CODA stores draft-depth KV at its own position, never below the committed frontier, and the target CODA of that position stores over it before any target row attends it; a rejected suffix is cut by truncate_suffix. Sliding-window attention needs one query row per token, so verification keeps per-row attention. Speculation with CUDA graphs, asynchronous scheduling, preemption or last_loop KV stays rejected at construction. Signed-off-by: 0z5a --- tests/test_dory_speculative.py | 401 ++++++++++++++++++++++++++++++++ vllm_rlt/engine/stage_engine.py | 10 +- vllm_rlt/models/dory.py | 3 + vllm_rlt/worker/speculative.py | 127 +++++++--- 4 files changed, 512 insertions(+), 29 deletions(-) create mode 100644 tests/test_dory_speculative.py diff --git a/tests/test_dory_speculative.py b/tests/test_dory_speculative.py new file mode 100644 index 00000000..b8881777 --- /dev/null +++ b/tests/test_dory_speculative.py @@ -0,0 +1,401 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Synchronous Dory self-speculation: block and token-pipeline rounds.""" + +from dataclasses import replace + +import pytest +import torch + +from tests.reference.dory import dense_dory_reference +from tests.test_dory import make_cache, tiny_dory_config +from vllm_rlt import ( + CacheConfig, + ExecutionConfig, + SamplingParams, + SchedulerConfig, + SpeculativeConfig, +) +from vllm_rlt.engine.llm_engine import LLMEngine +from vllm_rlt.models import DoryForCausalLM +from vllm_rlt.worker.sampler import Sampler + +DEPTH_PAIRS = [(1, 2), (1, 4), (2, 4)] + + +def dory(loops, seed=42, **overrides): + torch.manual_seed(seed) + return DoryForCausalLM(tiny_dory_config(n_recurrent_loops=loops, **overrides)).eval() + + +def engine(model, k, draft, mode="block", **kwargs): + loops = model.config.total_ut_steps + kwargs.setdefault("cache_config", CacheConfig(num_blocks=1024, block_size=2)) + kwargs.setdefault("scheduler_config", SchedulerConfig(max_num_seqs=4)) + return LLMEngine( + model, + speculative_config=SpeculativeConfig(k, draft_loops=draft, target_loops=loops, mode=mode), + **kwargs, + ) + + +def params(model, max_tokens, **kwargs): + loops = model.config.total_ut_steps + kwargs.setdefault("ignore_eos", True) + return SamplingParams(max_tokens=max_tokens, min_loops=loops, max_loops=loops, **kwargs) + + +def drain(e): + final = {} + for _ in range(4000): + if not e.has_unfinished_requests(): + return final + for output in e.step(): + if output.finished: + final[output.request_id] = output + pytest.fail("speculative scheduler did not drain") + + +def greedy_reference(model, prompt, count): + tokens = list(prompt) + for _ in range(count): + logits = dense_dory_reference(model, torch.tensor(tokens))["logits"] + tokens.append(int(logits[-1].argmax())) + return tokens + + +def serial_cache(model, tokens): + """KV of the reference schedule: one token per step at the full depth.""" + cache = make_cache(model) + tokens = torch.tensor(tokens) + assert cache.allocate("request", len(tokens)) + for position in range(len(tokens)): + boundary = cache._prepare_batch(["request"], [0], [position]) + hidden = model.prelude_prepared(tokens[position : position + 1], boundary, cache) + for depth in range(model.config.total_ut_steps): + core = cache._prepare_batch(["request"], [depth], [position]) + hidden, _ = model.recurrent_prepared(hidden, core, cache) + model.coda_prepared(hidden, boundary, cache) + return cache + + +def assert_committed_kv_is_the_target_history(e, model, request_id, tokens, frontier): + """Every plane of the committed prefix holds what serial execution stores.""" + if not frontier: + return + expected = serial_cache(model, tokens[:frontier]) + cache = e.cache_manager + boundary = (*model.prelude_kv_layers, *model.coda_kv_layers) + for layer in range(model.config.num_kv_layers): + depths = (0,) if layer in boundary else range(model.config.total_ut_steps) + for depth in depths: + actual = cache.read(layer, request_id, depth, frontier) + wanted = expected.read(layer, "request", depth, frontier) + for a, b in zip(actual, wanted): + torch.testing.assert_close(a, b, atol=3e-6, rtol=3e-5) + + +def force_drafts(monkeypatch, e, model, sequence, wrong): + """Make every draft row propose the reference token, or miss where ``wrong`` says. + + ``wrong(position, index)`` receives the input position of the draft row and, + in block mode, its index in the round. Target rows keep the model's logits. + """ + runner = e.speculative_runner + coda, submit_round = runner._coda, runner._submit_round + round_state = {} + + def submit(batch, *args): + round_state.update( + calls=0, + drafts=max(span.token_count for span in batch.requests) - 1, + ) + return submit_round(batch, *args) + + def controlled(hidden, boundary=None): + logits = coda(hidden, boundary) + positions = [position for _, _, position in boundary.rows] + if runner.config.mode == "block": + index = round_state["calls"] + round_state["calls"] += 1 + if index >= round_state["drafts"]: + return logits + drafts = [(row, index) for row in range(len(positions))] + else: + depth = { + token.position: token.depth + for state in runner.pipeline_states.values() + for token in state.tokens + } + drafts = [ + (row, None) + for row, position in enumerate(positions) + if depth[position] == runner.config.draft_loops + ] + for row, index in drafts: + position = positions[row] + if position + 1 >= len(sequence): + continue + token = sequence[position + 1] + if wrong(position, index): + token = (token + 1) % model.config.vocab_size + logits[row] = -100.0 + logits[row, token] = 100.0 + return logits + + monkeypatch.setattr(runner, "_submit_round", submit) + monkeypatch.setattr(runner, "_coda", controlled) + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +@pytest.mark.parametrize("k", [1, 2, 4]) +def test_greedy_matches_the_dense_reference_for_ragged_requests(mode, draft, target, k): + model = dory(target) + e = engine(model, k, draft, mode) + prompts = {"a": [4, 7, 3], "b": [9], "c": [5, 6, 8, 2, 11]} + limits = {"a": 9, "b": 1, "c": 6} + for rid, prompt in prompts.items(): + e.add_request(rid, prompt, params(model, limits[rid])) + final = drain(e) + for rid, prompt in prompts.items(): + expected = greedy_reference(model, prompt, limits[rid])[len(prompt) :] + assert final[rid].token_ids == expected + assert final[rid].finish_reason == "length" + assert final[rid].exit_depths == [target] * limits[rid] + assert e.cache_manager.num_used_blocks == 0 + assert not e.speculative_runner.pipeline_states + + +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +@pytest.mark.parametrize("rejected", [0, 1, 2, 3]) +def test_block_commits_through_each_rejection_position(draft, target, rejected, monkeypatch): + """rejected=3 accepts all three candidates and commits the bonus token.""" + model, k, prompt, count = dory(target), 3, [4, 7, 3], 13 + sequence = greedy_reference(model, prompt, count) + e = engine(model, k, draft) + force_drafts(monkeypatch, e, model, sequence, lambda position, index: index == rejected) + e.add_request("r", prompt, params(model, count)) + request = e.scheduler.requests["r"] + stats = e.speculative_runner.stats + rounds = 0 + while e.has_unfinished_requests(): + before = (len(request.generated_token_ids), stats.rounds, stats.accepted_tokens) + outputs = e.step() + generated = list(request.generated_token_ids) + assert generated == sequence[len(prompt) : len(prompt) + len(generated)] + if stats.rounds == before[1]: + continue + rounds += 1 + # First token comes from prefill, so `count - 1 - before` remain. + remaining = count - before[0] + drafts = min(k + 1, remaining) - 1 + accepted = min(rejected, drafts) + assert stats.accepted_tokens - before[2] == accepted + assert len(generated) - before[0] == accepted + 1 + if outputs and outputs[0].finished: + break + frontier = len(prompt) + len(generated) - 1 + allocation = e.cache_manager._get_allocation("r") + # A rejected suffix is invisible in every plane, draft CODA KV included. + for plane in allocation.written: + assert all(w.prefix <= frontier and not w.pending for w in plane) + assert_committed_kv_is_the_target_history(e, model, "r", sequence, frontier) + assert rounds >= 3 + assert e.cache_manager.num_used_blocks == 0 + + +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +@pytest.mark.parametrize("pattern", ["all", "none", "every_third"]) +def test_pipeline_commits_target_history_under_forced_acceptance( + draft, target, pattern, monkeypatch +): + model, prompt, count = dory(target), [4, 7, 3], 12 + sequence = greedy_reference(model, prompt, count) + wrong = { + "all": lambda position, index: False, + "none": lambda position, index: True, + "every_third": lambda position, index: position % 3 == 0, + }[pattern] + e = engine(model, 3, draft, "pipeline") + force_drafts(monkeypatch, e, model, sequence, wrong) + e.add_request("r", prompt, params(model, count)) + request = e.scheduler.requests["r"] + while e.has_unfinished_requests(): + outputs = e.step() + generated = list(request.generated_token_ids) + assert generated == sequence[len(prompt) : len(prompt) + len(generated)] + if outputs and outputs[0].finished: + break + # Positions below the last committed input have run their target CODA. + frontier = len(prompt) + len(generated) - 1 + assert_committed_kv_is_the_target_history(e, model, "r", sequence, frontier) + stats = e.speculative_runner.stats + verified = count - 1 + assert stats.committed_tokens == verified + if pattern == "all": + assert stats.accepted_tokens == verified + elif pattern == "none": + assert stats.accepted_tokens == 0 + else: + assert 0 < stats.accepted_tokens < verified + assert not e.speculative_runner.pipeline_states + assert e.cache_manager.num_used_blocks == 0 + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_eos_inside_an_accepted_prefix_stops_and_releases(mode, monkeypatch): + model, prompt = dory(2), [4, 7, 3] + sequence = greedy_reference(model, prompt, 8) + stop = sequence[len(prompt) + 2] + assert stop not in sequence[len(prompt) : len(prompt) + 2] + model.config = replace(model.config, eos_token_id=stop) + e = engine(model, 4, 1, mode) + force_drafts(monkeypatch, e, model, sequence, lambda position, index: False) + e.add_request("r", prompt, params(model, 8, ignore_eos=False)) + result = drain(e)["r"] + assert result.finish_reason == "stop" + assert result.token_ids == sequence[len(prompt) : len(prompt) + 3] + assert e.cache_manager.num_used_blocks == 0 + assert not e.speculative_runner.pipeline_states + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +@pytest.mark.parametrize("count", [1, 2, 5]) +def test_output_and_context_limits_bound_the_last_round(mode, count): + model = dory(2) + limit = model.config.max_position_embeddings + # The last output token is never an input, so it needs no position. + prompt = [(3 * i + 2) % model.config.vocab_size for i in range(limit - count + 1)] + e = engine(model, 4, 1, mode) + e.add_request("r", prompt, params(model, count)) + result = drain(e)["r"] + assert result.token_ids == greedy_reference(model, prompt, count)[len(prompt) :] + assert e.cache_manager.num_used_blocks == 0 + with pytest.raises(ValueError, match="context length"): + e.add_request("r", prompt, params(model, count + 1)) + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_cancellation_releases_boundary_state_and_allows_id_reuse(mode): + model = dory(4) + e = engine(model, 3, 2, mode) + e.add_request("a", [4, 7, 3], params(model, 12)) + e.add_request("b", [9, 5], params(model, 6)) + final = {} + for _ in range(3): + final.update({output.request_id: output for output in e.step() if output.finished}) + # Mid-generation: prefill and at least one speculative submission have run. + assert "a" not in final and e.speculative_runner.stats.rounds + assert e.abort_request("a").finish_reason == "abort" + assert "a" not in e.speculative_runner.pipeline_states + e.add_request("a", [6, 2], params(model, 5)) + final.update(drain(e)) + assert final["a"].token_ids == greedy_reference(model, [6, 2], 5)[2:] + assert final["b"].token_ids == greedy_reference(model, [9, 5], 6)[2:] + assert e.cache_manager.num_used_blocks == 0 + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_failed_round_releases_requests_and_allows_id_reuse(mode, monkeypatch): + model = dory(2) + e = engine(model, 3, 1, mode) + e.add_request("a", [4, 7, 3], params(model, 8)) + runner = e.speculative_runner + original = runner._coda + + def fail(*args, **kwargs): + original(*args, **kwargs) + raise RuntimeError("injected coda failure") + + monkeypatch.setattr(runner, "_coda", fail) + with pytest.raises(RuntimeError, match="injected coda failure"): + drain(e) + assert not runner.pipeline_states + assert e.cache_manager.num_used_blocks == 0 + monkeypatch.setattr(runner, "_coda", original) + e.add_request("a", [5], params(model, 3)) + assert drain(e)["a"].token_ids == greedy_reference(model, [5], 3)[1:] + + +def test_block_sampling_is_seeded_and_isolated_between_requests(): + model = dory(2) + + def run(batch): + e = engine(model, 3, 1) + for rid, seed in batch: + e.add_request(rid, [4, 7, 3], params(model, 9, temperature=0.8, top_k=8, seed=seed)) + final = drain(e) + assert e.cache_manager.num_used_blocks == 0 + return {rid: final[rid].token_ids for rid, _ in batch} + + together = run([("a", 11), ("b", 12)]) + assert together == run([("a", 11), ("b", 12)]) + assert together["a"] == run([("a", 11)])["a"] + assert together["a"] != together["b"] + + +def test_block_sampling_recovers_the_target_distribution(): + """Rejection sampling over draft-depth proposals must sample the target model. + + The second output token is the first one a speculative round commits, by + accepting the draft candidate or sampling the residual. Its exact law is + the mixture over the first token of the full-depth distributions, which + the dense oracle gives in closed form. + """ + # Large residual steps make one loop a visibly different model from two. + model, prompt = dory(2, ngpt_alpha_init=1.0), [4, 7, 3] + sampling = dict(temperature=1.0, top_k=3) + probe = params(model, 3, **sampling) + + def distribution(tokens): + logits = dense_dory_reference(model, torch.tensor(tokens))["logits"][-1] + return Sampler.probabilities(logits, probe) + + first = distribution(prompt) + expected = torch.zeros(model.config.vocab_size) + for token in first.nonzero().flatten().tolist(): + expected += first[token] * distribution([*prompt, token]) + e = engine(model, 2, 1, scheduler_config=SchedulerConfig(max_num_seqs=8)) + samples = 1600 + counts = torch.zeros(model.config.vocab_size) + for start in range(0, samples, 8): + for seed in range(start, start + 8): + e.add_request(str(seed), prompt, params(model, 3, seed=seed, **sampling)) + for output in drain(e).values(): + counts[output.token_ids[1]] += 1 + stats = e.speculative_runner.stats + # The draft is a different model: both outcomes of the test must occur. + assert 0 < stats.accepted_tokens < stats.drafted_tokens + assert not (counts > 0)[expected == 0].any() + sigma = (expected * (1 - expected) / samples).sqrt() + assert ((counts / samples - expected).abs() <= 4.5 * sigma + 1e-9).all() + + +def test_unqualified_speculative_combinations_fail_at_construction(): + model = dory(2) + with pytest.raises(ValueError, match="synchronous"): + engine(model, 2, 1, execution_config=ExecutionConfig(async_scheduling=True)) + with pytest.raises(ValueError, match="CUDA graphs"): + engine(model, 2, 1, execution_config=ExecutionConfig(cuda_graphs=True)) + with pytest.raises(ValueError, match="without CUDA graphs or preemption"): + engine( + model, + 2, + 1, + scheduler_config=SchedulerConfig(max_num_seqs=4, enable_preemption=True), + ) + with pytest.raises(ValueError, match="target_loops"): + LLMEngine(model, speculative_config=SpeculativeConfig(2, draft_loops=1, target_loops=4)) + shared = dory(2, recurrent_kv_cache_mode="last_loop") + with pytest.raises(ValueError, match="last_exited"): + LLMEngine( + shared, + cache_config=CacheConfig(num_blocks=256, block_size=2, layout="shared"), + speculative_config=SpeculativeConfig(2, draft_loops=1, target_loops=2), + ) + pipeline = engine(model, 2, 1, "pipeline") + with pytest.raises(ValueError, match="greedy"): + pipeline.add_request("r", [4], params(model, 2, temperature=0.7)) + block = engine(model, 2, 1) + with pytest.raises(ValueError, match="min_loops and max_loops"): + block.add_request("r", [4], SamplingParams(max_tokens=2, min_loops=1, max_loops=2)) diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index fe0a1c90..a34d8567 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -87,7 +87,15 @@ def __init__( "boundary-KV models require synchronous scheduling without prefill UVA" ) if speculative_config is not None: - raise ValueError("boundary-KV speculative decoding is not yet supported") + if not getattr(model, "supports_self_speculation", False): + raise ValueError("boundary-KV speculative decoding is not yet supported") + # Qualified: synchronous eager rounds. Graph replay and resumable + # preemption of draft/target boundary state are separate work. + if self.execution_config.cuda_graphs or scheduler_config.enable_preemption: + raise ValueError( + "boundary-KV speculative decoding requires synchronous eager " + "execution without CUDA graphs or preemption" + ) if not getattr(model, "supports_cuda_graphs", True) and self.execution_config.cuda_graphs: raise ValueError("this model does not support CUDA graphs yet") self.speculative_config = speculative_config diff --git a/vllm_rlt/models/dory.py b/vllm_rlt/models/dory.py index 6639345e..8e6a388e 100644 --- a/vllm_rlt/models/dory.py +++ b/vllm_rlt/models/dory.py @@ -488,6 +488,9 @@ class DoryForCausalLM(nn.Module): supports_async_scheduling = True # Selection must reject a backend that would silently attend full history. requires_sliding_window = True + # Synchronous self-speculation with per-loop KV: a draft's loops are the + # target's first loops, and boundary KV is committed by the target pass. + supports_self_speculation = True def __init__(self, config: DoryConfig) -> None: super().__init__() diff --git a/vllm_rlt/worker/speculative.py b/vllm_rlt/worker/speculative.py index 85f0e235..a61f7561 100644 --- a/vllm_rlt/worker/speculative.py +++ b/vllm_rlt/worker/speculative.py @@ -2,6 +2,14 @@ Temporary tensors belong to a submission until collection. Requests expose only committed outputs; Scheduler applies returned tokens and KV commits; Engine coordinates cleanup. + +Boundary-KV models (attention in PRELUDE and CODA) keep one boundary history at +depth 0. PRELUDE KV depends only on the input tokens, so a candidate's prelude +is the target's whenever the candidate is accepted. CODA KV depends on the +state it reads: a draft CODA stores draft-depth KV at its own position, which is +never below the committed frontier, and the target CODA of that position stores +over it before any target row attends it. Target rows therefore read target +history only; a rejected suffix is cut by ``truncate_suffix``. """ from contextlib import nullcontext @@ -154,6 +162,9 @@ def __init__( compute_stream=None, ): self.model, self.cache, self.config = model, cache, config + self.boundary_kv = getattr(model, "requires_boundary_kv", False) + # Windowed attention needs one query row per token. + self.row_queries = getattr(model, "requires_sliding_window", False) self.metadata = cache.metadata_builder self.runtime = runtime if runtime is not None else ModelRuntime(model, cache) self.pipeline_states: dict[str, PipelineState] = {} @@ -175,7 +186,12 @@ def __init__( def _packed(self, packed): capabilities = self.cache.attention_capabilities - return packed and capabilities.packed_prefill and capabilities.packed_matches_decode + return ( + packed + and not self.row_queries + and capabilities.packed_prefill + and capabilities.packed_matches_decode + ) def _prepare_inputs(self, batch): """Plan all host-known inputs, then publish one snapshot per dtype.""" @@ -202,7 +218,21 @@ def metadata(ids, positions, depths, *, packed=False): active = [i for i, span in enumerate(requests) if offset < span.token_count] ids = [requests[i].request.request_id for i in active] positions = [requests[i].token_start + offset for i in active] - steps.append((active, metadata(ids, positions, range(self.config.draft_loops)))) + layouts = metadata(ids, positions, range(self.config.draft_loops)) + # Depth 0 addresses the boundary history: the prelude of these rows + # reuses the first loop's view, the draft CODA covers drafting rows. + coda = None + if self.boundary_kv: + drafting = [ + row for row, i in enumerate(active) if offset + 1 < requests[i].token_count + ] + if len(drafting) == len(active): + coda = layouts[0] + elif drafting: + coda = metadata( + [ids[row] for row in drafting], [positions[row] for row in drafting], (0,) + )[0] + steps.append((active, layouts, coda)) ids, positions = [], [] for span in requests: ids.extend([span.request.request_id] * span.token_count) @@ -210,16 +240,26 @@ def metadata(ids, positions, depths, *, packed=False): verification = metadata( ids, positions, range(self.config.draft_loops, self.config.target_loops), packed=True ) + target_coda = metadata(ids, positions, (0,))[0] if self.boundary_kv else None uploaded = {dtype: buffer.upload() for dtype, buffer in self.input_buffers.items()} tokens = uploaded[tokens.dtype].as_strided( tokens.shape, tokens.stride(), tokens.storage_offset() ) + + def bind(kv): + return None if kv is None else self.metadata.bind(kv, uploaded) + steps = [ - (active, tuple(self.metadata.bind(kv, uploaded) for kv in layouts)) - for active, layouts in steps + (active, tuple(bind(kv) for kv in layouts), bind(coda)) + for active, layouts, coda in steps ] - verification = tuple(self.metadata.bind(kv, uploaded) for kv in verification) - return tokens, steps, verification + verification = tuple(bind(kv) for kv in verification) + return tokens, steps, verification, bind(target_coda) + + def _prelude(self, tokens, boundary=None): + return self.runtime.forward( + ForwardBatch(Operation.PRELUDE, tokens=tokens, prelude_coda_kv_metadata=boundary) + ).hidden def _recurrent(self, hidden, metadata): forward = ForwardBatch( @@ -235,12 +275,13 @@ def _acquire_generator(self, request, logits): """ return self.rng.acquire(request.request_id, request.sampling_params, logits.device) - def _coda(self, hidden): + def _coda(self, hidden, boundary=None): return self.runtime.forward( self.runtime.prepare( ForwardBatch( Operation.CODA, hidden=hidden, + prelude_coda_kv_metadata=boundary, graphs=self.coda_graphs, ) ) @@ -260,8 +301,7 @@ def submit(self, batch): if self.config.mode == "pipeline": submission = self._submit_pipeline(batch) else: - inputs, steps, verification = self._prepare_inputs(batch) - submission = self._submit_round(batch, inputs, steps, verification) + submission = self._submit_round(batch, *self._prepare_inputs(batch)) self.submissions.complete(submission.event) return submission except Exception: @@ -321,38 +361,69 @@ def _submit_pipeline(self, batch): ids.extend([request.request_id] * len(state.tokens)) for buffer in self.input_buffers.values(): buffer.reset() - new = [t for t in tokens if t.hidden is None] - input_view = self.input_buffers[torch.int64].append([t.token for t in new]) + new = [i for i, t in enumerate(tokens) if t.hidden is None] + input_view = self.input_buffers[torch.int64].append([tokens[i].token for i in new]) + + def stage(values, dtype): + return self.input_buffers[dtype].append(values) + + positions = [t.position for t in tokens] layouts = self.metadata.build( ids, [[t.depth + offset for t in tokens] for offset in range(self.config.draft_loops)], - [t.position for t in tokens], + positions, packed=False, ) - layouts = stage_metadata( - layouts, lambda values, dtype: self.input_buffers[dtype].append(values) - ) + layouts = stage_metadata(layouts, stage) + # Rows that read a draft or target distribution after this step's loops. + endpoints = [ + i + for i, t in enumerate(tokens) + if t.depth + self.config.draft_loops + in (self.config.draft_loops, self.config.target_loops) + ] + prelude_kv = coda_kv = None + if self.boundary_kv: + # Boundary history lives at depth 0. A request's rows stay ordered by + # position and every row stores before any row attends, so a draft + # row reads the target KV its predecessor stores in the same call. + def boundary(rows): + if not rows: + return None + return stage_metadata( + self.metadata.build( + [ids[i] for i in rows], + [[0] * len(rows)], + [positions[i] for i in rows], + packed=False, + ), + stage, + )[0] + + prelude_kv, coda_kv = boundary(new), boundary(endpoints) uploaded = {dtype: buffer.upload() for dtype, buffer in self.input_buffers.items()} if new: inputs = uploaded[input_view.dtype].as_strided( input_view.shape, input_view.stride(), input_view.storage_offset() ) - hidden = self.runtime.forward(ForwardBatch(Operation.PRELUDE, tokens=inputs)).hidden - for token, row in zip(new, hidden): - token.hidden = row + hidden = self._prelude( + inputs, None if prelude_kv is None else self.metadata.bind(prelude_kv, uploaded) + ) + for i, row in zip(new, hidden): + tokens[i].hidden = row hidden = self._recurrent( torch.stack([t.hidden for t in tokens]), tuple(self.metadata.bind(kv, uploaded) for kv in layouts), ) - endpoints = [] - for i, (token, row) in enumerate(zip(tokens, hidden)): + for token, row in zip(tokens, hidden): token.hidden = row token.depth += self.config.draft_loops - if token.depth in (self.config.draft_loops, self.config.target_loops): - endpoints.append(i) # Synchronous execution reads decisions in collect(), before the next step. choices = ( - self._coda(hidden[endpoints]).argmax(-1) + self._coda( + hidden[endpoints], + None if coda_kv is None else self.metadata.bind(coda_kv, uploaded), + ).argmax(-1) if endpoints else hidden.new_empty(0, dtype=torch.long) ) @@ -415,7 +486,7 @@ def _collect_pipeline(self, submission): results.append(SpeculativeResult(emitted, accepted, kv_frontier=frontier)) return results - def _submit_round(self, batch, inputs, steps, verification): + def _submit_round(self, batch, inputs, steps, verification, target_coda=None): scheduled_requests = batch.requests proposals = [[] for _ in scheduled_requests] states = [[] for _ in scheduled_requests] @@ -423,8 +494,8 @@ def _submit_round(self, batch, inputs, steps, verification): # prelude input, and all of them are read back once after verification. drafted = [] # Each step consumes the previous step's GPU-resident sampled IDs. - for offset, (active, metadata) in enumerate(steps): - hidden = self.runtime.forward(ForwardBatch(Operation.PRELUDE, tokens=inputs)).hidden + for offset, (active, metadata, draft_coda) in enumerate(steps): + hidden = self._prelude(inputs, metadata[0] if self.boundary_kv else None) hidden = self._recurrent(hidden, metadata) drafting = [] for row, i in enumerate(active): @@ -433,7 +504,7 @@ def _submit_round(self, batch, inputs, steps, verification): drafting.append((row, i)) if not drafting: break - logits = self._coda(torch.stack([hidden[row] for row, _ in drafting])) + logits = self._coda(torch.stack([hidden[row] for row, _ in drafting]), draft_coda) inputs = logits.argmax(-1) for row, (_, i) in enumerate(drafting): request = scheduled_requests[i].request @@ -446,7 +517,7 @@ def _submit_round(self, batch, inputs, steps, verification): # Verification rows are contiguous by request, matching prepared metadata. hidden = torch.stack([h for request_states in states for h in request_states]) hidden = self._recurrent(hidden, verification) - logits = self._coda(hidden) + logits = self._coda(hidden, target_coda) event = None if self.device.type == "cuda": event = torch.cuda.Event() From d90e376b9cb6c8894b05a848fd0272644fdaa2a4 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 12 Nov 2026 14:59:51 +0100 Subject: [PATCH 2/5] docs(dory): record self-speculation validation and measurements Signed-off-by: 0z5a --- docs/dory.md | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 2 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index 23aab486..ac6b1e98 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -96,7 +96,9 @@ These fail fast rather than degrade silently: - WCPB, mixed or staged prefill; packed prefill metadata (`prefill_uva`); PD. - Prefix caching. -- Self-speculation. +- Self-speculation combined with CUDA Graphs, asynchronous scheduling, + preemption or `last_loop` KV, and sampled token-pipeline speculation. The + synchronous eager form is supported; see below. - Per-request loop counts other than the configured full depth: Dory has no trained exit gate, so the runtime refuses to invent one, including via `min_loops` with `exit_threshold=0`. @@ -113,6 +115,43 @@ construction instead of attending full history. Masking accepts host-planned, graph-private and device-owned metadata; storage reclamation of out-of-window pages stays with 2B. +## Synchronous self-speculation + +`SpeculativeConfig(K, draft_loops=d, target_loops=T, mode=...)` drafts with the +first `d` recurrent loops and verifies with all `T`, where `T` is the loop count +the model was loaded with. It needs `per_loop` KV, synchronous eager execution +and fixed-depth requests (`min_loops == max_loops == T`). Qualified depth pairs +are 1→2, 1→4 and 2→4. + +| Mode | Sampling | Round | +| --- | --- | --- | +| `block` | greedy, or temperature/top-k/top-p with rejection sampling | `K` draft tokens one after another, then one verification of `K + 1` positions | +| `pipeline` | greedy | every submission advances each retained token by `d` loops; `d` must divide `T` | + +What a draft shares with the target, and what it does not: + +- **Input stage.** Prelude KV depends only on the input tokens. A candidate's + prelude is the target's prelude whenever the candidate is accepted, so it is + computed once. +- **Recurrent loops.** With `per_loop` KV, loop `i` of a position reads only + loop `i` of earlier positions. The draft's loops `0..d-1` are the target's + first `d` loops; verification continues from the retained state with loops + `d..T-1`. +- **Output stage.** CODA attention reads the state it is given, so draft CODA + KV (after `d` loops) differs from target CODA KV (after `T` loops) at the + same position. A draft CODA stores at its own position, which is never below + the committed frontier, and the target CODA of that position stores over it + before any target row attends it. Target rows therefore read target history + only. Draft rows read target history for committed positions and draft + history for positions still in flight; that affects acceptance, never the + committed output. +- **Rollback.** After a rejection the scheduler cuts every plane, boundary + layers included, back to the committed frontier (`truncate_suffix`); the next + round stores over the stale slots. + +Sliding-window attention takes one query row per token, so verification uses +per-row attention rather than a packed call. + ## Validation status `tests/test_dory.py` and `tests/test_dory_gpu.py` hold the regression suites; @@ -206,14 +245,94 @@ cohorts into one decode wave and BF16 greedy tokens depend on batch composition. FP32 is therefore the exact check; BF16 agreement is not promised across batch shapes. +### Self-speculation + +`tests/test_dory_speculative.py` (54 CPU cases) checks block and token-pipeline +speculation for the depth pairs 1→2, 1→4 and 2→4 against the dense oracle: +greedy tokens for ragged batches, a forced rejection at every draft position, +all-accepted rounds with the bonus token, EOS inside an accepted prefix, output +and context limits, cancellation with ID reuse, cleanup after a failed round, +seeded sampling replay, the rejection-sampling output distribution, and the +rejected combinations. After every round the committed KV equals the serial +target history and nothing is written above the committed frontier. On one +GH200 the file passes together with `tests/test_dory_gpu.py` and +`tests/test_speculative.py` (251 passed; the only failures are 12 +`flash_attn_4` cases whose optional package is not installed there). + +Real checkpoint, Triton, synchronous eager, greedy, WikiText prompts, one GH200 +(torch 2.14.0+cu130). Speculative tokens are compared with the non-speculative +baseline of the same process: + +| Dtype | Loops | Shape (requests × prompt + generated) | Arms | Requests with identical tokens | +| --- | --- | --- | --- | --- | +| FP32 | 2 | 8 × 512 + 64 | block 1→2 K=1/3/4, pipeline 1→2 K=1/3 | 8/8 every arm | +| FP32 | 4 | 8 × 512 + 64 | block and pipeline, 1→4 and 2→4, K=3 | 8/8 every arm | +| FP32 | 2 | 32 × 937 + 128 | block 1→2 K=3, pipeline 1→2 K=1/3 | 32/32 every arm | +| FP32 | 4 | 16 × 512 + 96 | block and pipeline, 1→4 and 2→4, K=3 | 16/16 every arm | +| BF16 | 2 and 4 | 8 × 512 + 64 | same arms as the first two rows | 3–6 of 8 per arm | + +BF16 is not token-exact, and that is not specific to speculation: serving the +same eight BF16 requests without speculation in waves of four or one instead of +eight also leaves only 2–4 of 8 requests token-identical, while the FP32 +baseline is identical across wave sizes (32/32). Speculation changes the batch +shapes of the same per-row arithmetic, BF16 rounding then differs, and greedy +near-ties flip; every later token of that request diverges. FP32 is therefore +the exact check, and BF16 output is a valid greedy continuation up to rounding, +not a bit-reproduction of the baseline. + +Acceptance (accepted / drafted, BF16): 1→2 0.78–0.93, 2→4 0.90–0.98, 1→4 +0.72–0.93. + +Whole-run wall time in seconds (all requests submitted at once, prefill +included; BF16; one pinned host core; arms interleaved, mean of two repetitions +except where marked †; decode-phase committed tokens/s in parentheses): + +| Loops | Shape | Baseline | Block d=1 K=3 | Pipeline d=1 K=3 | Pipeline d=1 K=1 | +| --- | --- | --- | --- | --- | --- | +| 2 | 128 × 468 + 128 | 34.5 (697) | 31.9 (1133) | 23.0 (1249) | 27.1 (948) | +| 2 | 64 × 937 + 128 | 37.6 (330) | 34.8 (618) | 24.0 (678) | 28.2 (488) | +| 2 | 32 × 1875 + 128 | 39.5 (171) | 38.4 (298) | 25.4 (359) | 29.9 (256) | +| 2 | 8 × 7500 + 128 | 41.3 (45) | 31.9 (83) | 29.8 (85) | 33.0 (66) | +| 2 | 1 × 8000 + 128 | 18.3 (8.3) | 17.6 (8.9) | 12.1 (13.2) | 14.8 (10.3) | +| 2 | 2 × 32000 + 96 | 60.1 (12.9) | 52.6 (19.9) | 49.7 (24.8) | 52.2 (18.6) | + +| Loops | Shape | Baseline | Block d=1 K=3 † | Block d=2 K=3 | Pipeline d=1 K=3 † | Pipeline d=2 K=3 | +| --- | --- | --- | --- | --- | --- | --- | +| 4 | 64 × 937 + 96 | 49.0 (197) | 39.8 (496) | 47.1 (412) | 33.0 (397) | 32.1 (470) | +| 4 | 8 × 7500 + 96 | 59.9 (25.5) | 44.9 (58) | 46.8 (53) | 45.3 (50) | 42.8 (61) | +| 4 | 1 × 8000 + 96 | 23.3 (5.0) | 14.5 (9.2) | 17.7 (7.1) | 15.9 (8.1) | 15.4 (8.4) | + +Reading the numbers: + +- The pipeline is the faster mode in every two-loop shape (wall time −25 % to + −39 % up to 8K prompts). Block speculation gains little wall time at high + concurrency (−3 % to −8 % for 32–128 requests): its first round runs K draft + steps before the first token is delivered, which adds 5–11 s to the time until + every request has a first token, so its decode-phase rate overstates the + benefit and wall time is the figure to use. +- At 2 × 32000 the run is dominated by ~42 s of prompt processing that + speculation does not touch; the generated part is short (96 tokens). +- Single runs vary: the two single-request block repetitions were 21.2 s and + 13.9 s, the baseline 20.1 s and 16.4 s. Eager Dory is bound by host-side + per-step cost, which is why fewer engine steps per committed token helps at + all; these ratios should not be carried over to a graph-captured path. +- The longest context exercised is 32,096 tokens (two requests). Peak allocated + memory was 74–82 GiB with the automatically sized KV pool; every arm released + its KV blocks. + ### Not yet performed - A runtime-to-runtime comparison against vLLM once a Dory-capable build is available. - Long-context behaviour beyond 32K tokens: the longest measured prompt is 32,256 tokens, far below the artifact's 256K configuration limit, which is - not a qualified deployment length. + not a qualified deployment length. Dense-oracle logit parity is still only + qualified on the 5-token prompt above. - Windowed Flash-backed kernels. - Graph-captured prefill, open-loop arrival traces and sampled decoding on the real checkpoint in Graph or async mode. The measurements above are closed-loop, greedy, and their time to first token is eager prefill. +- Self-speculation with CUDA graphs, asynchronous scheduling, preemption, + sampled token-pipeline, adaptive exit, or a draft width above the fixed K. +- Sampled speculation on the real checkpoint (the rejection-sampling contract is + qualified on the tiny CPU model only). From c6c1add5e7d92781abd49cb69e971b9ea4269294 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Sat, 14 Nov 2026 13:21:09 +0100 Subject: [PATCH 3/5] fix(dory): keep async self-speculation rejected after 2D With 2D merged, Dory declares supports_async_scheduling, so the generic boundary-KV async check no longer rejects an async engine. Self-speculation for boundary-KV models is only qualified for synchronous eager rounds, so its own gate now names async scheduling next to CUDA Graphs and preemption. test_unqualified_speculative_combinations_fail_at_construction covers it. Signed-off-by: 0z5a --- vllm_rlt/engine/stage_engine.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index a34d8567..e16f2e07 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -89,9 +89,13 @@ def __init__( if speculative_config is not None: if not getattr(model, "supports_self_speculation", False): raise ValueError("boundary-KV speculative decoding is not yet supported") - # Qualified: synchronous eager rounds. Graph replay and resumable - # preemption of draft/target boundary state are separate work. - if self.execution_config.cuda_graphs or scheduler_config.enable_preemption: + # Qualified: synchronous eager rounds. Async rounds, Graph replay and + # resumable preemption of draft/target boundary state are separate work. + if ( + self.execution_config.async_scheduling + or self.execution_config.cuda_graphs + or scheduler_config.enable_preemption + ): raise ValueError( "boundary-KV speculative decoding requires synchronous eager " "execution without CUDA graphs or preemption" From a7b573bafb4fc98c42c95c06838859bb14bc54a5 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Sat, 14 Nov 2026 13:21:09 +0100 Subject: [PATCH 4/5] refactor(speculative): select every round view from one layout source (#182) Block and pipeline preparation built several related metadata objects independently: per-offset recurrent layouts, draft CODA metadata, target verification metadata and target CODA metadata. They are row/depth selections of the same request-position layout, so they now come from one. - MetadataBuilder.select_rows() selects arbitrary host rows of a built KVLayout before staging: positions, write addresses and allocation identities are taken from the layout, attention values are re-planned for the selection (a narrower table for a subset, rebuilt sequence boundaries for packed queries). The staged-metadata slice_rows() is unchanged. - Block rounds build one layout over all round rows at every target depth and stage only selections of it: each offset's rows at the draft depths (depth 0 also feeds the boundary PRELUDE), the drafting subset at depth 0 for the draft CODA, all rows at the remaining depths for verification (packed when the backend allows) and at depth 0 for the target CODA. When every row of an offset drafts, the CODA reuses that offset's depth-0 view, as before. - Pipeline steps add depth 0 to their single build and select the new rows (PRELUDE) and endpoint rows (CODA) from it instead of building twice. - _prepare_inputs returns a PreparedRound; each DraftStep names its request indices and the coda_rows that select both the CODA hidden rows and their KV metadata. _submit_round no longer recomputes which rows draft. Staging, the single upload per dtype, RNG consumption, acceptance, KV commit/rollback and the supported execution modes are unchanged. A new test checks that a selection equals a layout built for those rows, for unequal request lengths, partial and empty subsets, caller order and packed subsets. Another drives packed verification end to end on CPU through a reference packed prefill and compares greedy tokens with native decoding. docs/dory.md describes the row order and the lifetime of a prepared round. Signed-off-by: 0z5a --- docs/dory.md | 22 +++ tests/test_input_preparation.py | 43 ++++++ tests/test_speculative.py | 51 +++++++ vllm_rlt/worker/metadata.py | 46 +++++++ vllm_rlt/worker/speculative.py | 236 ++++++++++++++++++-------------- 5 files changed, 296 insertions(+), 102 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index ac6b1e98..9b4aec05 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -152,6 +152,28 @@ What a draft shares with the target, and what it does not: Sliding-window attention takes one query row per token, so verification uses per-row attention rather than a packed call. +**Round metadata.** A block round prepares its metadata on the host before any +GPU work. One `MetadataBuilder.build` covers every row of the round (each +request's committed token plus its `K` candidates, grouped by request in +position order) at every target depth. Each operation stages a selection of +that layout (`MetadataBuilder.select_rows`): + +- draft offset `j`: the requests still drafting at `j`, at depths `0..d-1` + (depth 0 also serves the prelude); +- draft CODA: the subset that emits a candidate at `j`, at depth 0. The same + `coda_rows` index both the CODA hidden rows and their KV metadata; +- verification: every row at depths `d..T-1`, packed when the attention backend + declares `packed_matches_decode` and the model allows packed queries; +- target CODA: every row at depth 0. + +Selections keep the caller's row order and take positions and write addresses +from the source layout; attention tables are re-planned for the rows selected. +All selections go into one upload per dtype, and `_prepare_inputs` returns them +as a `PreparedRound`. Its tensors are views into the runner's input buffers and +stay valid until the next round is prepared, which happens only after the +current round has been read back. The pipeline mode uses one build per +submission and selects its prelude and CODA rows from it the same way. + ## Validation status `tests/test_dory.py` and `tests/test_dory_gpu.py` hold the regression suites; diff --git a/tests/test_input_preparation.py b/tests/test_input_preparation.py index 6f580135..6091a703 100644 --- a/tests/test_input_preparation.py +++ b/tests/test_input_preparation.py @@ -1,5 +1,7 @@ """CPU/GPU input preparation, reusable metadata and forward boundaries.""" +from dataclasses import replace + import pytest import torch @@ -621,3 +623,44 @@ def checked_execute(scheduled, prepared): assert operations == {Operation.PRELUDE, Operation.RECURRENT, Operation.CODA} finally: e.close() + + +@pytest.mark.parametrize( + "selection,packed", + [ + ((0, 1, 2, 3, 4, 5, 6), False), # every row + ((1, 3, 6), False), # one offset across unequal requests + ((3, 6), False), # partial CODA subset, last request skipped + ((5, 3, 0), False), # caller order is preserved + ((), False), # empty subset + ((0, 1, 2, 3, 4, 5, 6), True), # packed verification rows + ((1, 2, 4, 5), True), # packed subset rebuilds sequence boundaries + ], +) +def test_selected_rows_match_a_layout_built_for_those_rows(monkeypatch, selection, packed): + cache = KVCacheManager(1, 1, 8, 32, 4, 4, layout="last_exited") + monkeypatch.setattr( + type(cache), + "attention_capabilities", + property(lambda self: replace(self.attention.capabilities, packed_prefill=True)), + ) + for name, length in (("a", 9), ("b", 6), ("c", 11)): + assert cache.allocate(name, length) + # Rows grouped by request in position order, as speculative rounds lay them out. + ids = ["a", "a", "a", "b", "b", "c", "c"] + positions = [6, 7, 8, 4, 5, 9, 10] + builder = cache.metadata_builder + source = builder.build(ids, [[depth] * len(ids) for depth in range(4)], positions) + for depth in range(4): + selected = builder.select_rows(source[depth], selection, packed=packed) + if not selection: + assert selected is None + continue + expected = builder.build( + [ids[row] for row in selection], + [[depth] * len(selection)], + [positions[row] for row in selection], + packed=packed, + )[0] + assert selected == expected + assert source[0].positions == positions # the source's shared lists are untouched diff --git a/tests/test_speculative.py b/tests/test_speculative.py index b2593c83..7efb6992 100644 --- a/tests/test_speculative.py +++ b/tests/test_speculative.py @@ -44,6 +44,57 @@ def drain(e): pytest.fail("speculative scheduler did not drain") +@pytest.mark.parametrize("k", [1, 3]) +def test_packed_verification_matches_native_greedy(k, monkeypatch): + """Packed verification rows, boundaries and tables reproduce native greedy decoding. + + On GPU only FA4 routes verification through packed queries; here a reference + packed prefill expands each sequence into bottom-right aligned single rows. + """ + from vllm_rlt.attention import BackendCapabilities + from vllm_rlt.core.kv_cache_manager import KVCacheManager + from vllm_rlt.kernels.paged_attention import TorchPagedAttention, torch_paged_attention + from vllm_rlt.worker.metadata import MetadataBuilder + + def reference_prefill(self, q, key_cache, value_cache, tables, lengths, cu_seqlens, _): + rows, row_lengths = [], [] + for seq, (start, end) in enumerate(zip(cu_seqlens.tolist(), cu_seqlens[1:].tolist())): + for row in range(start, end): + rows.append(seq) + row_lengths.append(int(lengths[seq]) - (end - 1 - row)) + return torch_paged_attention( + q, key_cache, value_cache, tables[rows], torch.tensor(row_lengths, dtype=torch.int32) + ) + + m = model() + prompts = [[2], [3, 4, 5, 6, 7], [9, 3], [5, 6, 7]] + params = [SamplingParams(max_tokens=n, ignore_eos=True) for n in [1, 13, 8, 11]] + expected = LLM(m).generate(prompts, params) + monkeypatch.setattr( + KVCacheManager, + "attention_capabilities", + property(lambda self: BackendCapabilities(packed_prefill=True, packed_matches_decode=True)), + ) + monkeypatch.setattr(TorchPagedAttention, "prefill", reference_prefill, raising=False) + packed_rows = [] + select = MetadataBuilder.select_rows + + def counted(layout, rows, *, packed=False): + selected = select(layout, rows, packed=packed) + if packed and selected is not None: + assert selected.attention.cu_seqlens_q is not None + packed_rows.append(len(selected.rows)) + return selected + + monkeypatch.setattr(MetadataBuilder, "select_rows", staticmethod(counted)) + llm = LLM(m, speculative_config=SpeculativeConfig(k), cache_config=CacheConfig(256, 2)) + assert llm.engine.speculative_runner._packed(True) + actual = llm.generate(prompts, params) + assert packed_rows + assert [o.token_ids for o in actual] == [o.token_ids for o in expected] + assert llm.engine.cache_manager.num_used_blocks == 0 + + @pytest.mark.parametrize( "generation,packed,matches_decode,expected", # Triton packs prompt prefill, but its packed kernel is not bitwise equal to diff --git a/vllm_rlt/worker/metadata.py b/vllm_rlt/worker/metadata.py index 8f2c020b..95fde7bb 100644 --- a/vllm_rlt/worker/metadata.py +++ b/vllm_rlt/worker/metadata.py @@ -228,6 +228,52 @@ def build( ) return tuple(layouts) + @staticmethod + def select_rows(layout, rows, *, packed=False): + """Select host rows of a built layout, in the given order, before staging. + + ``rows`` indexes ``layout.rows``. Positions, write addresses and + allocation identities are taken from the layout, so the selection + describes exactly the rows a caller selects from the matching hidden + tensor. Attention values are re-planned for the selection: a subset may + need a narrower table, and packed queries need their own sequence + boundaries (selected rows of one request/depth must stay contiguous). + The result is a new layout; the source and its shared lists are not + modified. An empty selection returns ``None``. + """ + if not rows: + return None + owner = layout.owner + if packed and ( + owner.layout != "last_exited" or not owner.attention_capabilities.packed_prefill + ): + raise ValueError("packed queries require a compatible attention backend and KV layout") + selected = tuple(layout.rows[row] for row in rows) + positions = [layout.positions[row] for row in rows] + names = {id(allocation): name for name, allocation in layout.allocations} + allocations = {} + for allocation, _, _ in selected: + allocations.setdefault(names[id(allocation)], allocation) + attention = plan_attention_metadata( + AttentionRows( + block_tables=[a.block_tables[owner._plane(d)] for a, d, _ in selected], + positions=positions, + sequence_keys=[(id(a), d) for a, d, _ in selected] if packed else None, + ), + block_size=owner.block_size, + packed_prefill=packed, + ) + return KVLayout( + owner=owner, + rows=selected, + allocations=tuple(allocations.items()), + positions=positions, + write_blocks=[layout.write_blocks[row] for row in rows], + write_offsets=[layout.write_offsets[row] for row in rows], + attention=attention, + writable=layout.writable, + ) + @staticmethod def bind(metadata, storage): """Bind a host-planned layout to its uploaded typed storage.""" diff --git a/vllm_rlt/worker/speculative.py b/vllm_rlt/worker/speculative.py index a61f7561..6961ece3 100644 --- a/vllm_rlt/worker/speculative.py +++ b/vllm_rlt/worker/speculative.py @@ -141,6 +141,40 @@ def to(self, device): ) +@dataclass(frozen=True) +class DraftStep: + """Prepared inputs of one draft offset in a block round. + + ``requests`` holds batch request indices; row ``r`` of this step's hidden + tensor belongs to ``requests[r]``. ``coda_rows`` holds indices into those + rows, in order, for the requests that draft another token; the draft CODA + reads exactly these hidden rows and ``coda`` describes exactly these rows. + ``layouts`` are the step's recurrent depths; ``layouts[0]`` is depth 0, + which boundary-KV models also use for the PRELUDE. + """ + + requests: tuple[int, ...] + coda_rows: tuple[int, ...] + layouts: tuple + coda: object | None = None + + +@dataclass(frozen=True) +class PreparedRound: + """Bound inputs of one block round, borrowed from the runner's input buffers. + + Every tensor views the single upload per dtype made by ``_prepare_inputs`` + and stays valid until the next preparation resets those buffers, i.e. until + the round's submission has been collected. Verification and target CODA rows + are all round rows grouped by request, in position order. + """ + + tokens: torch.Tensor + steps: tuple[DraftStep, ...] + verification: tuple + target_coda: object | None = None + + @dataclass class PipelineStep: states: list[PipelineState] @@ -193,69 +227,90 @@ def _packed(self, packed): and capabilities.packed_matches_decode ) + def _stage(self, values, dtype): + return self.input_buffers[dtype].append(values) + + def _view(self, source, rows, depths, *, packed=False): + """Stage one selection of ``source``: the same rows at each depth index. + + ``rows=None`` selects every row in source order without re-planning. + Returns ``None`` for an empty selection. + """ + if rows is None and not packed: + layouts = [source[d] for d in depths] + else: + rows = range(len(source[0].rows)) if rows is None else rows + layouts = [self.metadata.select_rows(source[d], rows, packed=packed) for d in depths] + if layouts[0] is None: + return None + return stage_metadata(layouts, self._stage) + + def _boundary(self, source, rows, depth): + """Stage the depth-0 boundary view of ``rows``; ``None`` when empty.""" + view = self._view(source, rows, (depth,)) + return None if view is None else view[0] + + def _bind(self, metadata, uploaded): + return None if metadata is None else self.metadata.bind(metadata, uploaded) + def _prepare_inputs(self, batch): - """Plan all host-known inputs, then publish one snapshot per dtype.""" + """Plan every round input from one layout source, then upload once per dtype. + + The source covers all round rows (request order, then position) at + every target depth. Each operation stages a row/depth selection of it; + nothing is selected, uploaded or read back inside the execution loop. + """ for buffer in self.input_buffers.values(): buffer.reset() requests = batch.requests - tokens = self.input_buffers[torch.int64].append( - [span.request.input_token_id for span in requests] - ) - - def metadata(ids, positions, depths, *, packed=False): - layouts = self.metadata.build( - ids, - [[depth] * len(ids) for depth in depths], - positions, - packed=self._packed(packed), - ) - return stage_metadata( - layouts, lambda values, dtype: self.input_buffers[dtype].append(values) - ) - - steps = [] - for offset in range(max(span.token_count for span in requests)): - active = [i for i, span in enumerate(requests) if offset < span.token_count] - ids = [requests[i].request.request_id for i in active] - positions = [requests[i].token_start + offset for i in active] - layouts = metadata(ids, positions, range(self.config.draft_loops)) - # Depth 0 addresses the boundary history: the prelude of these rows - # reuses the first loop's view, the draft CODA covers drafting rows. - coda = None - if self.boundary_kv: - drafting = [ - row for row, i in enumerate(active) if offset + 1 < requests[i].token_count - ] - if len(drafting) == len(active): - coda = layouts[0] - elif drafting: - coda = metadata( - [ids[row] for row in drafting], [positions[row] for row in drafting], (0,) - )[0] - steps.append((active, layouts, coda)) - ids, positions = [], [] + tokens = self._stage([span.request.input_token_id for span in requests], torch.int64) + ids, positions, starts = [], [], [] for span in requests: + starts.append(len(ids)) ids.extend([span.request.request_id] * span.token_count) positions.extend(range(span.token_start, span.token_start + span.token_count)) - verification = metadata( - ids, positions, range(self.config.draft_loops, self.config.target_loops), packed=True + draft, target = self.config.draft_loops, self.config.target_loops + source = self.metadata.build( + ids, [[depth] * len(ids) for depth in range(target)], positions ) - target_coda = metadata(ids, positions, (0,))[0] if self.boundary_kv else None + steps = [] + for offset in range(max(span.token_count for span in requests)): + active = tuple(i for i, span in enumerate(requests) if offset < span.token_count) + layouts = self._view(source, [starts[i] + offset for i in active], range(draft)) + coda_rows = tuple( + row for row, i in enumerate(active) if offset + 1 < requests[i].token_count + ) + coda = None + if self.boundary_kv and coda_rows: + # Depth 0 is the boundary history. When every row drafts, the + # CODA selection is the step's own depth-0 view. + coda = ( + layouts[0] + if len(coda_rows) == len(active) + else self._boundary( + source, [starts[active[row]] + offset for row in coda_rows], 0 + ) + ) + steps.append(DraftStep(active, coda_rows, layouts, coda)) + verification = self._view(source, None, range(draft, target), packed=self._packed(True)) + target_coda = self._boundary(source, None, 0) if self.boundary_kv else None uploaded = {dtype: buffer.upload() for dtype, buffer in self.input_buffers.items()} - tokens = uploaded[tokens.dtype].as_strided( - tokens.shape, tokens.stride(), tokens.storage_offset() + return PreparedRound( + tokens=uploaded[tokens.dtype].as_strided( + tokens.shape, tokens.stride(), tokens.storage_offset() + ), + steps=tuple( + replace( + step, + layouts=tuple(self._bind(kv, uploaded) for kv in step.layouts), + coda=self._bind(step.coda, uploaded), + ) + for step in steps + ), + verification=tuple(self._bind(kv, uploaded) for kv in verification), + target_coda=self._bind(target_coda, uploaded), ) - def bind(kv): - return None if kv is None else self.metadata.bind(kv, uploaded) - - steps = [ - (active, tuple(bind(kv) for kv in layouts), bind(coda)) - for active, layouts, coda in steps - ] - verification = tuple(bind(kv) for kv in verification) - return tokens, steps, verification, bind(target_coda) - def _prelude(self, tokens, boundary=None): return self.runtime.forward( ForwardBatch(Operation.PRELUDE, tokens=tokens, prelude_coda_kv_metadata=boundary) @@ -301,7 +356,7 @@ def submit(self, batch): if self.config.mode == "pipeline": submission = self._submit_pipeline(batch) else: - submission = self._submit_round(batch, *self._prepare_inputs(batch)) + submission = self._submit_round(batch, self._prepare_inputs(batch)) self.submissions.complete(submission.event) return submission except Exception: @@ -364,51 +419,32 @@ def _submit_pipeline(self, batch): new = [i for i, t in enumerate(tokens) if t.hidden is None] input_view = self.input_buffers[torch.int64].append([tokens[i].token for i in new]) - def stage(values, dtype): - return self.input_buffers[dtype].append(values) - positions = [t.position for t in tokens] - layouts = self.metadata.build( - ids, - [[t.depth + offset for t in tokens] for offset in range(self.config.draft_loops)], - positions, - packed=False, - ) - layouts = stage_metadata(layouts, stage) + draft = self.config.draft_loops + # Depth sets per row: this step's loops, then (boundary KV) depth 0. + depth_sets = [[t.depth + offset for t in tokens] for offset in range(draft)] + if self.boundary_kv: + depth_sets.append([0] * len(tokens)) + source = self.metadata.build(ids, depth_sets, positions, packed=False) + layouts = self._view(source, None, range(draft)) # Rows that read a draft or target distribution after this step's loops. endpoints = [ - i - for i, t in enumerate(tokens) - if t.depth + self.config.draft_loops - in (self.config.draft_loops, self.config.target_loops) + i for i, t in enumerate(tokens) if t.depth + draft in (draft, self.config.target_loops) ] prelude_kv = coda_kv = None if self.boundary_kv: # Boundary history lives at depth 0. A request's rows stay ordered by # position and every row stores before any row attends, so a draft # row reads the target KV its predecessor stores in the same call. - def boundary(rows): - if not rows: - return None - return stage_metadata( - self.metadata.build( - [ids[i] for i in rows], - [[0] * len(rows)], - [positions[i] for i in rows], - packed=False, - ), - stage, - )[0] - - prelude_kv, coda_kv = boundary(new), boundary(endpoints) + # PRELUDE runs for new rows only; CODA reads the endpoint rows. + prelude_kv = self._boundary(source, new, draft) + coda_kv = self._boundary(source, endpoints, draft) uploaded = {dtype: buffer.upload() for dtype, buffer in self.input_buffers.items()} if new: inputs = uploaded[input_view.dtype].as_strided( input_view.shape, input_view.stride(), input_view.storage_offset() ) - hidden = self._prelude( - inputs, None if prelude_kv is None else self.metadata.bind(prelude_kv, uploaded) - ) + hidden = self._prelude(inputs, self._bind(prelude_kv, uploaded)) for i, row in zip(new, hidden): tokens[i].hidden = row hidden = self._recurrent( @@ -420,10 +456,7 @@ def boundary(rows): token.depth += self.config.draft_loops # Synchronous execution reads decisions in collect(), before the next step. choices = ( - self._coda( - hidden[endpoints], - None if coda_kv is None else self.metadata.bind(coda_kv, uploaded), - ).argmax(-1) + self._coda(hidden[endpoints], self._bind(coda_kv, uploaded)).argmax(-1) if endpoints else hidden.new_empty(0, dtype=torch.long) ) @@ -486,38 +519,37 @@ def _collect_pipeline(self, submission): results.append(SpeculativeResult(emitted, accepted, kv_frontier=frontier)) return results - def _submit_round(self, batch, inputs, steps, verification, target_coda=None): + def _submit_round(self, batch, prepared): scheduled_requests = batch.requests proposals = [[] for _ in scheduled_requests] states = [[] for _ in scheduled_requests] # Draft IDs stay on the device: each offset's IDs are the next offset's # prelude input, and all of them are read back once after verification. drafted = [] + inputs = prepared.tokens # Each step consumes the previous step's GPU-resident sampled IDs. - for offset, (active, metadata, draft_coda) in enumerate(steps): - hidden = self._prelude(inputs, metadata[0] if self.boundary_kv else None) - hidden = self._recurrent(hidden, metadata) - drafting = [] - for row, i in enumerate(active): + for step in prepared.steps: + hidden = self._prelude(inputs, step.layouts[0] if self.boundary_kv else None) + hidden = self._recurrent(hidden, step.layouts) + for row, i in enumerate(step.requests): states[i].append(hidden[row]) - if offset + 1 < scheduled_requests[i].token_count: - drafting.append((row, i)) - if not drafting: + if not step.coda_rows: break - logits = self._coda(torch.stack([hidden[row] for row, _ in drafting]), draft_coda) + logits = self._coda(torch.stack([hidden[row] for row in step.coda_rows]), step.coda) inputs = logits.argmax(-1) - for row, (_, i) in enumerate(drafting): + drafting = [step.requests[row] for row in step.coda_rows] + for row, i in enumerate(drafting): request = scheduled_requests[i].request if request.sampling_params.temperature != 0: q = Sampler.probabilities(logits[row], request.sampling_params) proposals[i].append(q) # Per-request RNG consumption order is unchanged; only the read moves. inputs[row] = Sampler.draw(q, self._acquire_generator(request, logits[row])) - drafted.append(([i for _, i in drafting], inputs)) + drafted.append((drafting, inputs)) # Verification rows are contiguous by request, matching prepared metadata. hidden = torch.stack([h for request_states in states for h in request_states]) - hidden = self._recurrent(hidden, verification) - logits = self._coda(hidden, target_coda) + hidden = self._recurrent(hidden, prepared.verification) + logits = self._coda(hidden, prepared.target_coda) event = None if self.device.type == "cuda": event = torch.cuda.Event() From f2cb7e3974570779bb1e73bb1d36cbede682f8e5 Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Mon, 16 Nov 2026 02:57:32 +0000 Subject: [PATCH 5/5] refactor(speculative): unify stage metadata in layouts Store all prepared KV metadata in one layout table. Draft and target operations reference layouts by index, preserving shared logical row identity and reusing identical selections across prelude, recurrent and coda. Signed-off-by: bjf-frz --- docs/dory.md | 16 +++- tests/test_dory_speculative.py | 53 ++++++++++++ vllm_rlt/worker/speculative.py | 143 ++++++++++++++++++--------------- 3 files changed, 142 insertions(+), 70 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index 9b4aec05..d53dfe69 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -169,10 +169,18 @@ that layout (`MetadataBuilder.select_rows`): Selections keep the caller's row order and take positions and write addresses from the source layout; attention tables are re-planned for the rows selected. All selections go into one upload per dtype, and `_prepare_inputs` returns them -as a `PreparedRound`. Its tensors are views into the runner's input buffers and -stay valid until the next round is prepared, which happens only after the -current round has been read back. The pipeline mode uses one build per -submission and selects its prelude and CODA rows from it the same way. +as a `PreparedRound`. Only `PreparedRound.layouts` stores bound KV metadata. +Each draft step's `views` maps PRELUDE, RECURRENT and CODA to indices in that +store; `target` uses the same index contract for verification and target CODA. +There are no separately owned CODA or verification metadata fields. Repeated +row/depth/packed selections reuse the same layout index, including a full-row +draft CODA and its PRELUDE/recurrent depth-0 view. Empty selections have no +indices. `coda_rows` still selects the matching hidden rows. + +Layout tensors borrow the runner's typed upload snapshots until the submission's +final consumer completes. The pipeline mode uses the same preparation-local +layout store and operation indices for its new-token, recurrent and endpoint +views; all views are bound before execution. ## Validation status diff --git a/tests/test_dory_speculative.py b/tests/test_dory_speculative.py index b8881777..e52b2832 100644 --- a/tests/test_dory_speculative.py +++ b/tests/test_dory_speculative.py @@ -17,6 +17,7 @@ ) from vllm_rlt.engine.llm_engine import LLMEngine from vllm_rlt.models import DoryForCausalLM +from vllm_rlt.request import Operation from vllm_rlt.worker.sampler import Sampler DEPTH_PAIRS = [(1, 2), (1, 4), (2, 4)] @@ -63,6 +64,58 @@ def greedy_reference(model, prompt, count): return tokens +def test_round_layouts_align_partial_coda_rows_and_upload_once(monkeypatch): + model = dory(4) + e = engine(model, 3, 2, scheduler_config=SchedulerConfig(max_num_seqs=2)) + runner = e.speculative_runner + prepare = runner._prepare_inputs + uploads = dict.fromkeys(runner.input_buffers, 0) + subsets = set() + for dtype, buffer in runner.input_buffers.items(): + upload = buffer.upload + + def counted(dtype=dtype, upload=upload): + uploads[dtype] += 1 + return upload() + + monkeypatch.setattr(buffer, "upload", counted) + + def checked(batch): + before = uploads.copy() + prepared = prepare(batch) + assert all(uploads[dtype] == before[dtype] + 1 for dtype in uploads) + for step in prepared.steps: + recurrent = prepared.layouts[step.views[Operation.RECURRENT][0]] + assert prepared.layouts[step.views[Operation.PRELUDE][0]] is recurrent + indices = step.views[Operation.CODA] + if not step.coda_rows: + subsets.add("empty") + assert not indices + continue + coda = prepared.layouts[indices[0]] + assert coda.rows == tuple(recurrent.rows[row] for row in step.coda_rows) + if len(step.coda_rows) == len(step.requests): + subsets.add("full") + assert coda is recurrent + else: + subsets.add("partial") + target = prepared.layouts[prepared.target[Operation.CODA][0]] + assert len(target.rows) == sum(span.token_count for span in batch.requests) + return prepared + + monkeypatch.setattr(runner, "_prepare_inputs", checked) + try: + for name, prompt, count in (("a", [4, 7], 3), ("b", [9, 2, 3], 5)): + e.add_request(name, prompt, params(model, count)) + actual = drain(e) + assert actual["a"].token_ids == greedy_reference(model, [4, 7], 3)[-3:] + assert actual["b"].token_ids == greedy_reference(model, [9, 2, 3], 5)[-5:] + assert subsets == {"empty", "full", "partial"} + assert e.cache_manager.num_used_blocks == 0 + finally: + e.close() + + def serial_cache(model, tokens): """KV of the reference schedule: one token per step at the full depth.""" cache = make_cache(model) diff --git a/vllm_rlt/worker/speculative.py b/vllm_rlt/worker/speculative.py index 6961ece3..d292e422 100644 --- a/vllm_rlt/worker/speculative.py +++ b/vllm_rlt/worker/speculative.py @@ -22,6 +22,7 @@ from vllm_rlt.worker.buffers import stage_metadata from vllm_rlt.worker.cuda_graph import CodaGraphs, RecurrentGraphs from vllm_rlt.worker.executor import ForwardBatch, ModelRuntime +from vllm_rlt.worker.metadata import KVMetadata from vllm_rlt.worker.output import ModelRunnerOutput, SpeculativeResult from vllm_rlt.worker.sampler import RngRegistry, Sampler from vllm_rlt.worker.submission_queue import SubmissionQueue @@ -148,31 +149,30 @@ class DraftStep: ``requests`` holds batch request indices; row ``r`` of this step's hidden tensor belongs to ``requests[r]``. ``coda_rows`` holds indices into those rows, in order, for the requests that draft another token; the draft CODA - reads exactly these hidden rows and ``coda`` describes exactly these rows. - ``layouts`` are the step's recurrent depths; ``layouts[0]`` is depth 0, - which boundary-KV models also use for the PRELUDE. + reads exactly these hidden rows. ``views`` maps each model operation to + indices in ``PreparedRound.layouts``; it never owns separate KV metadata. + PRELUDE and a full-row CODA reuse the recurrent depth-0 layout index. """ requests: tuple[int, ...] coda_rows: tuple[int, ...] - layouts: tuple - coda: object | None = None + views: dict[Operation, tuple[int, ...]] @dataclass(frozen=True) class PreparedRound: """Bound inputs of one block round, borrowed from the runner's input buffers. - Every tensor views the single upload per dtype made by ``_prepare_inputs`` - and stays valid until the next preparation resets those buffers, i.e. until - the round's submission has been collected. Verification and target CODA rows - are all round rows grouped by request, in position order. + ``layouts`` is the only metadata store. Draft steps and target operations + hold indices into it; the same row/depth selection is staged and bound once. + Its tensors borrow the typed upload snapshots until this submission's final + consumer completes. Target rows are grouped by request, in position order. """ tokens: torch.Tensor + layouts: tuple[KVMetadata, ...] steps: tuple[DraftStep, ...] - verification: tuple - target_coda: object | None = None + target: dict[Operation, tuple[int, ...]] @dataclass @@ -230,28 +230,35 @@ def _packed(self, packed): def _stage(self, values, dtype): return self.input_buffers[dtype].append(values) - def _view(self, source, rows, depths, *, packed=False): - """Stage one selection of ``source``: the same rows at each depth index. + def _view(self, source, rows, depths, layouts, *, packed=False): + """Register a row/depth selection and return indices into the layout store. - ``rows=None`` selects every row in source order without re-planning. - Returns ``None`` for an empty selection. + The preparation-local store maps (source rows, depth index, packed) to + (index, staged metadata). Repeated selections reuse their index. Missing + depths are staged together, preserving shared positions and lengths. + ``rows=None`` selects every source row; an empty selection returns (). """ - if rows is None and not packed: - layouts = [source[d] for d in depths] - else: - rows = range(len(source[0].rows)) if rows is None else rows - layouts = [self.metadata.select_rows(source[d], rows, packed=packed) for d in depths] - if layouts[0] is None: - return None - return stage_metadata(layouts, self._stage) - - def _boundary(self, source, rows, depth): - """Stage the depth-0 boundary view of ``rows``; ``None`` when empty.""" - view = self._view(source, rows, (depth,)) - return None if view is None else view[0] + selected = tuple(range(len(source[0].rows))) if rows is None else tuple(rows) + if not selected: + return () + keys = tuple((selected, depth, packed) for depth in depths) + missing = [key for key in keys if key not in layouts] + if missing: + values = [ + source[depth] + if rows is None and not packed + else self.metadata.select_rows(source[depth], selected, packed=packed) + for _, depth, _ in missing + ] + for key, metadata in zip(missing, stage_metadata(values, self._stage)): + layouts[key] = (len(layouts), metadata) + return tuple(layouts[key][0] for key in keys) - def _bind(self, metadata, uploaded): - return None if metadata is None else self.metadata.bind(metadata, uploaded) + @staticmethod + def _boundary(layouts, views, operation): + """Resolve an optional boundary operation's already prepared layout.""" + indices = views.get(operation, ()) + return layouts[indices[0]] if indices else None def _prepare_inputs(self, batch): """Plan every round input from one layout source, then upload once per dtype. @@ -273,42 +280,37 @@ def _prepare_inputs(self, batch): source = self.metadata.build( ids, [[depth] * len(ids) for depth in range(target)], positions ) - steps = [] + layouts, steps = {}, [] for offset in range(max(span.token_count for span in requests)): active = tuple(i for i, span in enumerate(requests) if offset < span.token_count) - layouts = self._view(source, [starts[i] + offset for i in active], range(draft)) + rows = tuple(starts[i] + offset for i in active) + views = {Operation.RECURRENT: self._view(source, rows, range(draft), layouts)} coda_rows = tuple( row for row, i in enumerate(active) if offset + 1 < requests[i].token_count ) - coda = None - if self.boundary_kv and coda_rows: - # Depth 0 is the boundary history. When every row drafts, the - # CODA selection is the step's own depth-0 view. - coda = ( - layouts[0] - if len(coda_rows) == len(active) - else self._boundary( - source, [starts[active[row]] + offset for row in coda_rows], 0 - ) + if self.boundary_kv: + views[Operation.PRELUDE] = views[Operation.RECURRENT][:1] + views[Operation.CODA] = self._view( + source, tuple(rows[row] for row in coda_rows), (0,), layouts ) - steps.append(DraftStep(active, coda_rows, layouts, coda)) - verification = self._view(source, None, range(draft, target), packed=self._packed(True)) - target_coda = self._boundary(source, None, 0) if self.boundary_kv else None + steps.append(DraftStep(active, coda_rows, views)) + target_views = { + Operation.RECURRENT: self._view( + source, None, range(draft, target), layouts, packed=self._packed(True) + ) + } + if self.boundary_kv: + target_views[Operation.CODA] = self._view(source, None, (0,), layouts) uploaded = {dtype: buffer.upload() for dtype, buffer in self.input_buffers.items()} return PreparedRound( tokens=uploaded[tokens.dtype].as_strided( tokens.shape, tokens.stride(), tokens.storage_offset() ), - steps=tuple( - replace( - step, - layouts=tuple(self._bind(kv, uploaded) for kv in step.layouts), - coda=self._bind(step.coda, uploaded), - ) - for step in steps + layouts=tuple( + self.metadata.bind(metadata, uploaded) for _, metadata in layouts.values() ), - verification=tuple(self._bind(kv, uploaded) for kv in verification), - target_coda=self._bind(target_coda, uploaded), + steps=tuple(steps), + target=target_views, ) def _prelude(self, tokens, boundary=None): @@ -426,37 +428,38 @@ def _submit_pipeline(self, batch): if self.boundary_kv: depth_sets.append([0] * len(tokens)) source = self.metadata.build(ids, depth_sets, positions, packed=False) - layouts = self._view(source, None, range(draft)) + layouts = {} + views = {Operation.RECURRENT: self._view(source, None, range(draft), layouts)} # Rows that read a draft or target distribution after this step's loops. endpoints = [ i for i, t in enumerate(tokens) if t.depth + draft in (draft, self.config.target_loops) ] - prelude_kv = coda_kv = None if self.boundary_kv: # Boundary history lives at depth 0. A request's rows stay ordered by # position and every row stores before any row attends, so a draft # row reads the target KV its predecessor stores in the same call. # PRELUDE runs for new rows only; CODA reads the endpoint rows. - prelude_kv = self._boundary(source, new, draft) - coda_kv = self._boundary(source, endpoints, draft) + views[Operation.PRELUDE] = self._view(source, new, (draft,), layouts) + views[Operation.CODA] = self._view(source, endpoints, (draft,), layouts) uploaded = {dtype: buffer.upload() for dtype, buffer in self.input_buffers.items()} + layouts = tuple(self.metadata.bind(metadata, uploaded) for _, metadata in layouts.values()) if new: inputs = uploaded[input_view.dtype].as_strided( input_view.shape, input_view.stride(), input_view.storage_offset() ) - hidden = self._prelude(inputs, self._bind(prelude_kv, uploaded)) + hidden = self._prelude(inputs, self._boundary(layouts, views, Operation.PRELUDE)) for i, row in zip(new, hidden): tokens[i].hidden = row hidden = self._recurrent( torch.stack([t.hidden for t in tokens]), - tuple(self.metadata.bind(kv, uploaded) for kv in layouts), + tuple(layouts[index] for index in views[Operation.RECURRENT]), ) for token, row in zip(tokens, hidden): token.hidden = row token.depth += self.config.draft_loops # Synchronous execution reads decisions in collect(), before the next step. choices = ( - self._coda(hidden[endpoints], self._bind(coda_kv, uploaded)).argmax(-1) + self._coda(hidden[endpoints], self._boundary(layouts, views, Operation.CODA)).argmax(-1) if endpoints else hidden.new_empty(0, dtype=torch.long) ) @@ -527,15 +530,21 @@ def _submit_round(self, batch, prepared): # prelude input, and all of them are read back once after verification. drafted = [] inputs = prepared.tokens + layouts = prepared.layouts # Each step consumes the previous step's GPU-resident sampled IDs. for step in prepared.steps: - hidden = self._prelude(inputs, step.layouts[0] if self.boundary_kv else None) - hidden = self._recurrent(hidden, step.layouts) + hidden = self._prelude(inputs, self._boundary(layouts, step.views, Operation.PRELUDE)) + hidden = self._recurrent( + hidden, tuple(layouts[index] for index in step.views[Operation.RECURRENT]) + ) for row, i in enumerate(step.requests): states[i].append(hidden[row]) if not step.coda_rows: break - logits = self._coda(torch.stack([hidden[row] for row in step.coda_rows]), step.coda) + logits = self._coda( + torch.stack([hidden[row] for row in step.coda_rows]), + self._boundary(layouts, step.views, Operation.CODA), + ) inputs = logits.argmax(-1) drafting = [step.requests[row] for row in step.coda_rows] for row, i in enumerate(drafting): @@ -548,8 +557,10 @@ def _submit_round(self, batch, prepared): drafted.append((drafting, inputs)) # Verification rows are contiguous by request, matching prepared metadata. hidden = torch.stack([h for request_states in states for h in request_states]) - hidden = self._recurrent(hidden, prepared.verification) - logits = self._coda(hidden, prepared.target_coda) + hidden = self._recurrent( + hidden, tuple(layouts[index] for index in prepared.target[Operation.RECURRENT]) + ) + logits = self._coda(hidden, self._boundary(layouts, prepared.target, Operation.CODA)) event = None if self.device.type == "cuda": event = torch.cuda.Event()