From 495c9fffa5acefb954a39a99408951cef28bab6c Mon Sep 17 00:00:00 2001 From: Dennis Ramdass Date: Tue, 20 Oct 2026 09:10:30 -0700 Subject: [PATCH] Map SGLang's prefix cache and retraction counters The built-in SGLang mapping renamed running and waiting requests, KV usage and the token histograms, but none of SGLang's prefix cache or retraction counters. modelplane_prefix_cache_hits_total, modelplane_prefix_cache_lookups_total and modelplane_requests_preempted_total were empty for every SGLang deployment, so a prefix cache hit rate across vLLM and SGLang showed SGLang at nothing. SGLang v0.5.10.post1 and v0.5.12.post1, the versions the repo pins, publish cached_tokens_total, prompt_tokens_total and num_retracted_requests_total, which count what vLLM's prefix cache hits, lookups and preemptions count, in the same units. Counters have no buckets, so they share vLLM's names. SGLang counts when a request finishes where vLLM counts when it's scheduled, and labels its hits with a cache_source. The telemetry docs say so. sglang:queue_time_seconds stays unmapped: its buckets differ from vLLM's and between the two pinned SGLang versions (#523). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Dennis Ramdass --- docs/content/platform/telemetry.md | 9 +++- .../function/stacks/metrics.py | 16 +++++--- .../tests/test_collector.py | 41 ++++++++++--------- .../compose-serving-stack/tests/test_fn.py | 13 +++--- 4 files changed, 47 insertions(+), 32 deletions(-) diff --git a/docs/content/platform/telemetry.md b/docs/content/platform/telemetry.md index 2af8b2d86..346f61f21 100644 --- a/docs/content/platform/telemetry.md +++ b/docs/content/platform/telemetry.md @@ -275,8 +275,13 @@ spec: --collect-tokens-histogram ``` -SGLang doesn't publish request queue time or preemption counters, so only vLLM -populates `modelplane_request_queue_seconds` and `modelplane_requests_preempted_total`. +SGLang's prefix cache and retraction counters populate +`modelplane_prefix_cache_hits_total`, `modelplane_prefix_cache_lookups_total` and +`modelplane_requests_preempted_total`, so a prefix cache hit rate compares across both +engines. SGLang counts a request when it finishes, where vLLM counts it when it's +scheduled, so an aborted request is missing from SGLang's counts. SGLang's queue time +histogram uses different buckets from vLLM's, so only vLLM populates +`modelplane_request_queue_seconds`. Because the gateway, not the engine, measures the `modelplane_frontend_*` series, they work without configuration for an engine Modelplane has never seen. diff --git a/functions/compose-serving-stack/function/stacks/metrics.py b/functions/compose-serving-stack/function/stacks/metrics.py index 66dac9f22..77fc92567 100644 --- a/functions/compose-serving-stack/function/stacks/metrics.py +++ b/functions/compose-serving-stack/function/stacks/metrics.py @@ -57,12 +57,13 @@ "vllm:prefix_cache_queries_total": "modelplane_prefix_cache_lookups_total", } -# SGLang publishes none of the queue-latency or preemption measurements vLLM -# does, so there is nothing here to rename onto those names. -# sglang:avg_request_queue_latency is the nearest thing to a queue time and is -# not the same measurement - a gauge holding the mean over the last batch, -# where modelplane_request_queue_seconds is a per-request histogram - and one -# name holding both would make a quantile over the fleet meaningless. +# SGLang's prefix-cache and retraction counters count what vLLM's do, in the +# same units: prefix-cache hits and lookups in prompt tokens, so hits over +# lookups is a hit rate for either engine. SGLang counts when a request +# finishes where vLLM counts when it's scheduled, and labels its hits with a +# cache_source (device, host, storage). sglang:queue_time_seconds is a +# per-request histogram like vLLM's queue time, but its buckets differ from +# vLLM's and between SGLang versions, so it isn't renamed (#523). # The two token histograms need --collect-tokens-histogram as well as # --enable-metrics; the engine publishes only the _total counters without it. _SGLANG = { @@ -71,6 +72,9 @@ "sglang:token_usage": "modelplane_kv_cache_utilization_ratio", "sglang:prompt_tokens_histogram": "modelplane_request_input_tokens", "sglang:generation_tokens_histogram": "modelplane_request_output_tokens", + "sglang:num_retracted_requests_total": "modelplane_requests_preempted_total", + "sglang:cached_tokens_total": "modelplane_prefix_cache_hits_total", + "sglang:prompt_tokens_total": "modelplane_prefix_cache_lookups_total", } # The endpoint picker. A router's queue is a different measurement from an diff --git a/functions/compose-serving-stack/tests/test_collector.py b/functions/compose-serving-stack/tests/test_collector.py index 406a3bceb..5f5947431 100644 --- a/functions/compose-serving-stack/tests/test_collector.py +++ b/functions/compose-serving-stack/tests/test_collector.py @@ -384,6 +384,9 @@ def _config( 'set(metric.name, "modelplane_kv_cache_utilization_ratio") where metric.name == "sglang:token_usage"', 'set(metric.name, "modelplane_request_input_tokens") where metric.name == "sglang:prompt_tokens_histogram"', 'set(metric.name, "modelplane_request_output_tokens") where metric.name == "sglang:generation_tokens_histogram"', + 'set(metric.name, "modelplane_requests_preempted_total") where metric.name == "sglang:num_retracted_requests_total"', + 'set(metric.name, "modelplane_prefix_cache_hits_total") where metric.name == "sglang:cached_tokens_total"', + 'set(metric.name, "modelplane_prefix_cache_lookups_total") where metric.name == "sglang:prompt_tokens_total"', 'set(metric.name, "modelplane_route_decision_seconds") where metric.name == "llm_d_epp_scheduler_e2e_duration_seconds"', 'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "DCGM_FI_DEV_FB_USED"', 'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "DCGM_FI_PROF_GR_ENGINE_ACTIVE"', @@ -723,6 +726,9 @@ def test_port_rewrite(case: PortRewriteCase) -> None: 'set(metric.name, "modelplane_kv_cache_utilization_ratio") where metric.name == "sglang:token_usage"', 'set(metric.name, "modelplane_request_input_tokens") where metric.name == "sglang:prompt_tokens_histogram"', 'set(metric.name, "modelplane_request_output_tokens") where metric.name == "sglang:generation_tokens_histogram"', + 'set(metric.name, "modelplane_requests_preempted_total") where metric.name == "sglang:num_retracted_requests_total"', + 'set(metric.name, "modelplane_prefix_cache_hits_total") where metric.name == "sglang:cached_tokens_total"', + 'set(metric.name, "modelplane_prefix_cache_lookups_total") where metric.name == "sglang:prompt_tokens_total"', 'set(metric.name, "modelplane_route_decision_seconds") where metric.name == "llm_d_epp_scheduler_e2e_duration_seconds"', 'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "DCGM_FI_DEV_FB_USED"', 'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "DCGM_FI_PROF_GR_ENGINE_ACTIVE"', @@ -931,7 +937,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="ad2c3f990baaeb4c", + config_hash="55bc4445ed790fa2", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -976,7 +982,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="9f072b028dc68ea5", + config_hash="5b3619d55150aa79", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}}, @@ -1043,7 +1049,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="176c01ff260e4716", + config_hash="5097b903785819b1", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-vendor", "secret": {"secretName": "vendor-token"}}, @@ -1195,26 +1201,23 @@ def test_metric_name_pattern() -> None: def test_sglang_mappings() -> None: - """The SGLang built-in renames nothing onto queue time or preemption.""" - # SGLang publishes neither. Checked against a running SGLang v0.4.9.post2: - # it has no per-request queue-time metric and no retraction counters at - # all. The nearest thing, sglang:avg_request_queue_latency, is a gauge of - # the mean over the last batch - a different measurement from vLLM's - # per-request histogram, and one name holding both makes a fleet quantile - # meaningless. The built-in mappings are read from the stacks package, - # because they're what this checks. + """The SGLang built-in renames its counters onto vLLM's names, but not its queue time.""" + # Checked against SGLang v0.5.10.post1 and v0.5.12.post1, the versions the + # repo pins. Counters have no buckets, so they share vLLM's names. + # sglang:queue_time_seconds is a histogram whose buckets match neither + # vLLM's nor each other's across those versions, so a quantile over one + # name holding them would be wrong (#523). The built-in mappings are read + # from the stacks package, because they're what this checks. sglang = { m.from_: m.to for mapping in stacks.BUILTIN_MAPPINGS for m in mapping.spec.metrics if m.from_.startswith("sglang:") } - assert sglang, "the SGLang built-in went missing" - unpublished = { - source: target - for source, target in sglang.items() - if target in {"modelplane_request_queue_seconds", "modelplane_requests_preempted_total"} - or "retracted" in source - or "queue_time" in source + counters = { + "sglang:num_retracted_requests_total": "modelplane_requests_preempted_total", + "sglang:cached_tokens_total": "modelplane_prefix_cache_hits_total", + "sglang:prompt_tokens_total": "modelplane_prefix_cache_lookups_total", } - assert unpublished == {}, "SGLang publishes no queue time or preemption to rename" + assert {s: sglang.get(s) for s in counters} == counters, "SGLang's counters map onto vLLM's names" + assert "sglang:queue_time_seconds" not in sglang, "SGLang's queue time keeps its own buckets" diff --git a/functions/compose-serving-stack/tests/test_fn.py b/functions/compose-serving-stack/tests/test_fn.py index fa9a76625..6019d1878 100644 --- a/functions/compose-serving-stack/tests/test_fn.py +++ b/functions/compose-serving-stack/tests/test_fn.py @@ -1505,6 +1505,9 @@ def _collector_config( 'set(metric.name, "modelplane_kv_cache_utilization_ratio") where metric.name == "sglang:token_usage"', 'set(metric.name, "modelplane_request_input_tokens") where metric.name == "sglang:prompt_tokens_histogram"', 'set(metric.name, "modelplane_request_output_tokens") where metric.name == "sglang:generation_tokens_histogram"', + 'set(metric.name, "modelplane_requests_preempted_total") where metric.name == "sglang:num_retracted_requests_total"', + 'set(metric.name, "modelplane_prefix_cache_hits_total") where metric.name == "sglang:cached_tokens_total"', + 'set(metric.name, "modelplane_prefix_cache_lookups_total") where metric.name == "sglang:prompt_tokens_total"', 'set(metric.name, "modelplane_route_decision_seconds") where metric.name == "llm_d_epp_scheduler_e2e_duration_seconds"', 'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "DCGM_FI_DEV_FB_USED"', 'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "DCGM_FI_PROF_GR_ENGINE_ACTIVE"', @@ -4336,7 +4339,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="80bb34e74b135d14", + config_hash="0cf483d8c47017af", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -4693,7 +4696,7 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, ), "collector": _collector( - config_hash="173c93b5f6048d87", + config_hash="47b8fbf904b68d9c", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}}, @@ -5018,7 +5021,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="5676f43101e4b6b1", + config_hash="6b40121d65a36ddc", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5319,7 +5322,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="460bb87917962297", + config_hash="517d32fcc11ffd0d", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5648,7 +5651,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="80bb34e74b135d14", + config_hash="0cf483d8c47017af", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None,