Skip to content

Map SGLang's prefix cache and retraction counters #544

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dennis-upbound merged 1 commit into from
Oct 20, 2026
Merged

Map SGLang's prefix cache and retraction counters #544

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions docs/content/platform/telemetry.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,8 +275,13 @@ spec:
--collect-tokens-histogram
```

SGLang doesn't publish request queue time or preemption counters, so only vLLM
populates `modelplane_request_queue_seconds` and `modelplane_requests_preempted_total`.
SGLang's prefix cache and retraction counters populate
`modelplane_prefix_cache_hits_total`, `modelplane_prefix_cache_lookups_total` and
`modelplane_requests_preempted_total`, so a prefix cache hit rate compares across both
engines. SGLang counts a request when it finishes, where vLLM counts it when it's
scheduled, so an aborted request is missing from SGLang's counts. SGLang's queue time
histogram uses different buckets from vLLM's, so only vLLM populates
`modelplane_request_queue_seconds`.

Because the gateway, not the engine, measures the `modelplane_frontend_*` series, they
work without configuration for an engine Modelplane has never seen.
Expand Down
16 changes: 10 additions & 6 deletions functions/compose-serving-stack/function/stacks/metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,13 @@
"vllm:prefix_cache_queries_total": "modelplane_prefix_cache_lookups_total",
}

# SGLang publishes none of the queue-latency or preemption measurements vLLM
# does, so there is nothing here to rename onto those names.
# sglang:avg_request_queue_latency is the nearest thing to a queue time and is
# not the same measurement - a gauge holding the mean over the last batch,
# where modelplane_request_queue_seconds is a per-request histogram - and one
# name holding both would make a quantile over the fleet meaningless.
# SGLang's prefix-cache and retraction counters count what vLLM's do, in the
# same units: prefix-cache hits and lookups in prompt tokens, so hits over
# lookups is a hit rate for either engine. SGLang counts when a request
# finishes where vLLM counts when it's scheduled, and labels its hits with a
# cache_source (device, host, storage). sglang:queue_time_seconds is a
# per-request histogram like vLLM's queue time, but its buckets differ from
# vLLM's and between SGLang versions, so it isn't renamed (#523).
# The two token histograms need --collect-tokens-histogram as well as
# --enable-metrics; the engine publishes only the _total counters without it.
_SGLANG = {
Expand All @@ -71,6 +72,9 @@
"sglang:token_usage": "modelplane_kv_cache_utilization_ratio",
"sglang:prompt_tokens_histogram": "modelplane_request_input_tokens",
"sglang:generation_tokens_histogram": "modelplane_request_output_tokens",
"sglang:num_retracted_requests_total": "modelplane_requests_preempted_total",
"sglang:cached_tokens_total": "modelplane_prefix_cache_hits_total",
"sglang:prompt_tokens_total": "modelplane_prefix_cache_lookups_total",
}

# The endpoint picker. A router's queue is a different measurement from an
Expand Down
41 changes: 22 additions & 19 deletions functions/compose-serving-stack/tests/test_collector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,9 @@ def _config(
'set(metric.name, "modelplane_kv_cache_utilization_ratio") where metric.name == "sglang:token_usage"',
'set(metric.name, "modelplane_request_input_tokens") where metric.name == "sglang:prompt_tokens_histogram"',
'set(metric.name, "modelplane_request_output_tokens") where metric.name == "sglang:generation_tokens_histogram"',
'set(metric.name, "modelplane_requests_preempted_total") where metric.name == "sglang:num_retracted_requests_total"',
'set(metric.name, "modelplane_prefix_cache_hits_total") where metric.name == "sglang:cached_tokens_total"',
'set(metric.name, "modelplane_prefix_cache_lookups_total") where metric.name == "sglang:prompt_tokens_total"',
'set(metric.name, "modelplane_route_decision_seconds") where metric.name == "llm_d_epp_scheduler_e2e_duration_seconds"',
'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "DCGM_FI_DEV_FB_USED"',
'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "DCGM_FI_PROF_GR_ENGINE_ACTIVE"',
Expand Down Expand Up @@ -723,6 +726,9 @@ def test_port_rewrite(case: PortRewriteCase) -> None:
'set(metric.name, "modelplane_kv_cache_utilization_ratio") where metric.name == "sglang:token_usage"',
'set(metric.name, "modelplane_request_input_tokens") where metric.name == "sglang:prompt_tokens_histogram"',
'set(metric.name, "modelplane_request_output_tokens") where metric.name == "sglang:generation_tokens_histogram"',
'set(metric.name, "modelplane_requests_preempted_total") where metric.name == "sglang:num_retracted_requests_total"',
'set(metric.name, "modelplane_prefix_cache_hits_total") where metric.name == "sglang:cached_tokens_total"',
'set(metric.name, "modelplane_prefix_cache_lookups_total") where metric.name == "sglang:prompt_tokens_total"',
'set(metric.name, "modelplane_route_decision_seconds") where metric.name == "llm_d_epp_scheduler_e2e_duration_seconds"',
'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "DCGM_FI_DEV_FB_USED"',
'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "DCGM_FI_PROF_GR_ENGINE_ACTIVE"',
Expand Down Expand Up @@ -931,7 +937,7 @@ def test_transform(case: TransformCase) -> None:
(
"collector",
_deployment(
config_hash="ad2c3f990baaeb4c",
config_hash="55bc4445ed790fa2",
volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}],
volume_mounts=[{"name": "config", "mountPath": "/conf"}],
env_from=None,
Expand Down Expand Up @@ -976,7 +982,7 @@ def test_transform(case: TransformCase) -> None:
(
"collector",
_deployment(
config_hash="9f072b028dc68ea5",
config_hash="5b3619d55150aa79",
volumes=[
{"name": "config", "configMap": {"name": "modelplane-collector"}},
{"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}},
Expand Down Expand Up @@ -1043,7 +1049,7 @@ def test_transform(case: TransformCase) -> None:
(
"collector",
_deployment(
config_hash="176c01ff260e4716",
config_hash="5097b903785819b1",
volumes=[
{"name": "config", "configMap": {"name": "modelplane-collector"}},
{"name": "credentials-vendor", "secret": {"secretName": "vendor-token"}},
Expand Down Expand Up @@ -1195,26 +1201,23 @@ def test_metric_name_pattern() -> None:


def test_sglang_mappings() -> None:
"""The SGLang built-in renames nothing onto queue time or preemption."""
# SGLang publishes neither. Checked against a running SGLang v0.4.9.post2:
# it has no per-request queue-time metric and no retraction counters at
# all. The nearest thing, sglang:avg_request_queue_latency, is a gauge of
# the mean over the last batch - a different measurement from vLLM's
# per-request histogram, and one name holding both makes a fleet quantile
# meaningless. The built-in mappings are read from the stacks package,
# because they're what this checks.
"""The SGLang built-in renames its counters onto vLLM's names, but not its queue time."""
# Checked against SGLang v0.5.10.post1 and v0.5.12.post1, the versions the
# repo pins. Counters have no buckets, so they share vLLM's names.
# sglang:queue_time_seconds is a histogram whose buckets match neither
# vLLM's nor each other's across those versions, so a quantile over one
# name holding them would be wrong (#523). The built-in mappings are read
# from the stacks package, because they're what this checks.
sglang = {
m.from_: m.to
for mapping in stacks.BUILTIN_MAPPINGS
for m in mapping.spec.metrics
if m.from_.startswith("sglang:")
}
assert sglang, "the SGLang built-in went missing"
unpublished = {
source: target
for source, target in sglang.items()
if target in {"modelplane_request_queue_seconds", "modelplane_requests_preempted_total"}
or "retracted" in source
or "queue_time" in source
counters = {
"sglang:num_retracted_requests_total": "modelplane_requests_preempted_total",
"sglang:cached_tokens_total": "modelplane_prefix_cache_hits_total",
"sglang:prompt_tokens_total": "modelplane_prefix_cache_lookups_total",
}
assert unpublished == {}, "SGLang publishes no queue time or preemption to rename"
assert {s: sglang.get(s) for s in counters} == counters, "SGLang's counters map onto vLLM's names"
assert "sglang:queue_time_seconds" not in sglang, "SGLang's queue time keeps its own buckets"
13 changes: 8 additions & 5 deletions functions/compose-serving-stack/tests/test_fn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1505,6 +1505,9 @@ def _collector_config(
'set(metric.name, "modelplane_kv_cache_utilization_ratio") where metric.name == "sglang:token_usage"',
'set(metric.name, "modelplane_request_input_tokens") where metric.name == "sglang:prompt_tokens_histogram"',
'set(metric.name, "modelplane_request_output_tokens") where metric.name == "sglang:generation_tokens_histogram"',
'set(metric.name, "modelplane_requests_preempted_total") where metric.name == "sglang:num_retracted_requests_total"',
'set(metric.name, "modelplane_prefix_cache_hits_total") where metric.name == "sglang:cached_tokens_total"',
'set(metric.name, "modelplane_prefix_cache_lookups_total") where metric.name == "sglang:prompt_tokens_total"',
'set(metric.name, "modelplane_route_decision_seconds") where metric.name == "llm_d_epp_scheduler_e2e_duration_seconds"',
'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "DCGM_FI_DEV_FB_USED"',
'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "DCGM_FI_PROF_GR_ENGINE_ACTIVE"',
Expand Down Expand Up @@ -4336,7 +4339,7 @@ def _to_dict(msg: message.Message) -> dict:
service_extensions=None,
),
"collector": _collector(
config_hash="80bb34e74b135d14",
config_hash="0cf483d8c47017af",
volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}],
volume_mounts=[{"name": "config", "mountPath": "/conf"}],
env_from=None,
Expand Down Expand Up @@ -4693,7 +4696,7 @@ def _to_dict(msg: message.Message) -> dict:
ready=fnv1.READY_TRUE,
),
"collector": _collector(
config_hash="173c93b5f6048d87",
config_hash="47b8fbf904b68d9c",
volumes=[
{"name": "config", "configMap": {"name": "modelplane-collector"}},
{"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}},
Expand Down Expand Up @@ -5018,7 +5021,7 @@ def _to_dict(msg: message.Message) -> dict:
service_extensions=None,
),
"collector": _collector(
config_hash="5676f43101e4b6b1",
config_hash="6b40121d65a36ddc",
volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}],
volume_mounts=[{"name": "config", "mountPath": "/conf"}],
env_from=None,
Expand Down Expand Up @@ -5319,7 +5322,7 @@ def _to_dict(msg: message.Message) -> dict:
service_extensions=None,
),
"collector": _collector(
config_hash="460bb87917962297",
config_hash="517d32fcc11ffd0d",
volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}],
volume_mounts=[{"name": "config", "mountPath": "/conf"}],
env_from=None,
Expand Down Expand Up @@ -5648,7 +5651,7 @@ def _to_dict(msg: message.Message) -> dict:
service_extensions=None,
),
"collector": _collector(
config_hash="80bb34e74b135d14",
config_hash="0cf483d8c47017af",
volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}],
volume_mounts=[{"name": "config", "mountPath": "/conf"}],
env_from=None,
Expand Down
Loading