Qualify SampleProfiler CPU evidence and preserve loss warnings

benchmarks/Invoke-TrackDInvestigation.ps1

@@ -950,20 +950,22 @@ function Test-FiniteJsonNumber([object] $Value) {
 
 function ConvertTo-ValidatedSamplingCount(
     [object] $Value,
-    [string] $Owner) {
+    [string] $Owner,
+    [int] $SchemaVersion) {
+    [string] $provenanceError = "$Owner returned incompatible schema $SchemaVersion CPU sampling provenance."
     if (
         -not (Test-FiniteJsonNumber $Value) -or
         [double]$Value -lt 0 -or
         [double]$Value -ne [Math]::Truncate([double]$Value)
     ) {
-        throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+        throw $provenanceError
     }
 
     try {
         return [Convert]::ToInt64($Value, [Globalization.CultureInfo]::InvariantCulture)
     }
     catch {
-        throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+        throw $provenanceError
     }
 }
 
@@ -1027,12 +1029,13 @@ function Get-ValidatedCpuSampling(
         }
     }
 
+    [string] $provenanceError = "$Owner returned incompatible schema $SchemaVersion CPU sampling provenance."
     [object] $cpuSamplingProperty = $Container.PSObject.Properties['cpuSampling']
     if (
         $null -eq $cpuSamplingProperty -or
         $cpuSamplingProperty.Value -isnot [pscustomobject]
     ) {
-        throw "$Owner omitted schema 17 CPU sampling provenance."
+        throw "$Owner omitted schema $SchemaVersion CPU sampling provenance."
     }
     [object] $cpuSampling = $cpuSamplingProperty.Value
     [object] $weightUnitProperty = $cpuSampling.PSObject.Properties['weightUnit']
@@ -1055,16 +1058,17 @@ function Get-ValidatedCpuSampling(
         $intervalsProperty.Value.Count -gt 32 -or
         $ExpectedRecordCount -lt 0
     ) {
-        throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+        throw $provenanceError
     }
 
     [long] $unknownIntervalSampleCount = ConvertTo-ValidatedSamplingCount `
         $unknownCountProperty.Value `
-        $Owner
+        $Owner `
+        $SchemaVersion
     [long] $retainedIntervalSampleCount = 0
     foreach ($interval in $intervalsProperty.Value) {
         if ($null -eq $interval) {
-            throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+            throw $provenanceError
         }
 
         [object] $intervalProperty = $interval.PSObject.Properties['intervalMSec']
@@ -1075,18 +1079,19 @@ function Get-ValidatedCpuSampling(
             [double]$intervalProperty.Value -le 0 -or
             $null -eq $sampleCountProperty
         ) {
-            throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+            throw $provenanceError
         }
 
         [long] $sampleCount = ConvertTo-ValidatedSamplingCount `
             $sampleCountProperty.Value `
-            $Owner
+            $Owner `
+            $SchemaVersion
         if (
             $sampleCount -le 0 -or
             $sampleCount -gt $ExpectedRecordCount -or
             $retainedIntervalSampleCount -gt $ExpectedRecordCount - $sampleCount
         ) {
-            throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+            throw $provenanceError
         }
         $retainedIntervalSampleCount += $sampleCount
     }
@@ -1098,13 +1103,13 @@ function Get-ValidatedCpuSampling(
         0
     }
     else {
-        ConvertTo-ValidatedSamplingCount $omittedSegmentsProperty.Value $Owner
+        ConvertTo-ValidatedSamplingCount $omittedSegmentsProperty.Value $Owner $SchemaVersion
     }
     [long] $omittedIntervalSampleCount = if ($null -eq $omittedSamplesProperty) {
         0
     }
     else {
-        ConvertTo-ValidatedSamplingCount $omittedSamplesProperty.Value $Owner
+        ConvertTo-ValidatedSamplingCount $omittedSamplesProperty.Value $Owner $SchemaVersion
     }
     [bool] $intervalsTruncated = if ($null -eq $truncatedProperty) {
         $false
@@ -1113,14 +1118,14 @@ function Get-ValidatedCpuSampling(
         $truncatedProperty.Value
     }
     else {
-        throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+        throw $provenanceError
     }
     if (
         $intervalsTruncated -ne ($omittedIntervalSegmentCount -gt 0) -or
         $omittedIntervalSampleCount -lt $omittedIntervalSegmentCount -or
         ($omittedIntervalSegmentCount -eq 0 -and $omittedIntervalSampleCount -ne 0)
     ) {
-        throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+        throw $provenanceError
     }
 
     [string] $source = [string]$sourceProperty.Value
@@ -1132,16 +1137,27 @@ function Get-ValidatedCpuSampling(
             $source -cnotin @(
                 'unavailable',
                 'etw-perfinfo',
+                'sampleprofiler',
+                'mixed-etw-sampleprofiler',
                 'speedscope-profile-declared-sample-weights') -or
-            ($source -cin @('unavailable', 'speedscope-profile-declared-sample-weights') -and
+            ($source -cin @(
+                'unavailable', 'sampleprofiler',
+                'speedscope-profile-declared-sample-weights') -and
                 ($intervalCount -ne 0 -or $intervalsTruncated -or
-                    $unknownIntervalSampleCount -ne $ExpectedRecordCount)) -or
+                    $unknownIntervalSampleCount -ne $ExpectedRecordCount -or
+                    ($source -ceq 'sampleprofiler' -and $ExpectedRecordCount -eq 0))) -or
             ($source -ceq 'etw-perfinfo' -and
                 ($intervalCount -eq 0 -or
                     $unknownIntervalSampleCount -le 0 -or
-                    $unknownIntervalSampleCount -gt $ExpectedRecordCount))
+                    $unknownIntervalSampleCount -gt $ExpectedRecordCount)) -or
+            ($source -ceq 'mixed-etw-sampleprofiler' -and
+                ($ExpectedRecordCount -lt 2 -or
+                    $unknownIntervalSampleCount -le 0 -or
+                    ($intervalCount -eq 0 -and
+                        ($omittedIntervalSegmentCount -ne 0 -or
+                            $unknownIntervalSampleCount -ne $ExpectedRecordCount))))
         ) {
-            throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+            throw $provenanceError
         }
     }
     elseif (
@@ -1152,10 +1168,10 @@ function Get-ValidatedCpuSampling(
         ($source -ceq 'speedscope-profile-declared-time-weights' -and
             ($intervalCount -ne 0 -or $intervalsTruncated))
     ) {
-        throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+        throw $provenanceError
     }
 
-    if ($source -ceq 'etw-perfinfo') {
+    if ($source -cin @('etw-perfinfo', 'mixed-etw-sampleprofiler')) {
         if (
             $unknownIntervalSampleCount -gt $ExpectedRecordCount -or
             $retainedIntervalSampleCount -gt $ExpectedRecordCount - $unknownIntervalSampleCount -or
@@ -1164,7 +1180,7 @@ function Get-ValidatedCpuSampling(
             $retainedIntervalSampleCount + $omittedIntervalSampleCount +
                 $unknownIntervalSampleCount -ne $ExpectedRecordCount
         ) {
-            throw "$Owner returned incompatible schema 17 CPU sampling provenance."
+            throw $provenanceError
         }
     }
 
@@ -1533,7 +1549,8 @@ function Get-AnalysisEvidence(
             }
             $retainedSampleCount = ConvertTo-ValidatedSamplingCount `
                 $sampleCountProperty.Value `
-                "Profile analysis '$AnalysisName' info"
+                "Profile analysis '$AnalysisName' info" `
+                $schemaVersion
             [object] $infoCpuSamplingProperty =
                 $resultProperty.Value.PSObject.Properties['cpuSampling']
             [object] $infoWeightUnitProperty = if (

docs/workflow.md

@@ -715,6 +715,20 @@ cleanly and stays cheap in tokens.
   scope. CPU units are analyzer-version dependent: call them milliseconds only when
   recorded interval provenance and interval-aware weighting are established; otherwise
   use qualified sample counts/weights. Never derive CPU time from wall clock times share.
+  SampleProfiler thread-stack samples in EventPipe captures may include waiting or
+  native execution; `cpuSampling.source: sampleprofiler` identifies their raw counts,
+  not on-core CPU time or a blocked-time percentage. A result mixing those samples
+  with ETW sampled-profile records uses counts for both (`mixed-etw-sampleprofiler`),
+  even when the ETW subset has a recorded interval. CPU-derived CLI and MCP results
+  warn on the selected SampleProfiler evidence and qualify CPU-focused hints. Use
+  ETW PerfInfo sampling when the question requires on-core CPU time; do not equate
+  SampleProfiler `External` events with blocked time.
+  Counts from a mixed-provider result need not be comparable across samplers; do
+  not treat their combined share as a CPU-time comparison.
+  Bounded manifest batch/diff cases retain lost-event and SampleProfiler evidence
+  (combined with arm labels in diff) plus root-mismatch and per-operation
+  diagnostics before lower-priority warnings. Inspect a direct trace when the
+  four-warning case budget omits further quality detail.
 - Keep counts separate from weight. `trace_info.sampleCount` describes the loaded
   whole trace after process/activity/time filters; it does not establish that a
   narrower root/method/file query is well sampled. Stack rankings and callers expose

src/Filtrace.Core/Output/SteeringHints.cs

@@ -5,6 +5,7 @@
 using System.Globalization;
 using Filtrace.Tracing;
 using Filtrace.Tracing.Providers;
+using Filtrace.Tracing.Readers;
 
 namespace Filtrace.Output;
 
@@ -144,7 +145,12 @@ public static IReadOnlyList<string> ForTraceInfo(TraceInfo info)
         // keeps manually constructed legacy TraceInfo objects useful, but labels those
         // routes as format-supported because they carry no capture evidence.
         List<string> routes = [];
-        if (analyses.Contains("cpu")) { routes.Add("CPU-bound -> cpu"); }
+        if (analyses.Contains("cpu"))
+        {
+            routes.Add(CpuSampleEvidence.ContainsSampleProfiler(info.CpuSampling)
+                ? "sampled thread stacks (not on-core CPU time) -> cpu"
+                : "CPU-bound -> cpu");
+        }
 
         List<string> blocked = [];
         if (analyses.Contains("contention")) { blocked.Add("contention"); }
@@ -285,7 +291,32 @@ public static IReadOnlyList<string> ForRanking(
         ScopeRequest? scope = null,
         string? path = null,
         string? symbols = null,
-        bool nativeSymbols = false)
+        bool nativeSymbols = false) =>
+            ForRanking(ranking, metric, scope, path, symbols, nativeSymbols, cpuSampling: null);
+
+    /// <summary>
+    ///  The next-step hints for a ranking with its CPU sample provenance, preserving
+    ///  its metric and scope without calling thread-stack samples on-core CPU time.
+    /// </summary>
+    /// <param name="ranking">The ranking the hints steer from.</param>
+    /// <param name="metric">The metric the ranking carries.</param>
+    /// <param name="scope">Optional process, activity, and time scope of the ranking.</param>
+    /// <param name="path">The trace path for a complete quality follow-up.</param>
+    /// <param name="symbols">The local symbol directory, if any.</param>
+    /// <param name="nativeSymbols">Whether native symbol resolution was enabled.</param>
+    /// <param name="cpuSampling">The CPU sample provenance, if applicable.</param>
+    /// <returns>The hints and complete follow-ups, never <see langword="null"/>.</returns>
+    /// <exception cref="ArgumentNullException">
+    ///  <paramref name="ranking"/> or <paramref name="metric"/> is <see langword="null"/>.
+    /// </exception>
+    public static IReadOnlyList<string> ForRanking(
+        RankingResult ranking,
+        MetricInfo metric,
+        ScopeRequest? scope,
+        string? path,
+        string? symbols,
+        bool nativeSymbols,
+        CpuSampleProvenance? cpuSampling)
     {
         ArgumentNullException.ThrowIfNull(ranking);
         ArgumentNullException.ThrowIfNull(metric);
@@ -305,8 +336,12 @@ public static IReadOnlyList<string> ForRanking(
 
         if (scope?.ActivityName is not null || scope?.Window is not null)
         {
+            string kind = CpuSampleEvidence.ContainsSampleProfiler(cpuSampling)
+                ? "sampled-thread stack"
+                : "CPU";
+
             string reason =
-                "this CPU ranking is activity/time-scoped; callers, lines, heatmap, and tree cannot preserve that slice - refine it with self/inclusive measure or root in rank";
+                $"this {kind} ranking is activity/time-scoped; callers, lines, heatmap, and tree cannot preserve that slice - refine it with self/inclusive measure or root in rank";
 
             if (IsUnresolvedFrame(ranking.Rows[0].Frame))
             {
@@ -356,7 +391,10 @@ public static IReadOnlyList<string> ForRanking(
                 ]);
         }
 
-        string hint = $"drill into the hot frame with: callers {ranking.Rows[0].Frame}";
+        string hint = CpuSampleEvidence.ContainsSampleProfiler(cpuSampling)
+            ? $"drill into the most sampled frame with: callers {ranking.Rows[0].Frame}"
+            : $"drill into the hot frame with: callers {ranking.Rows[0].Frame}";
+
         string message = PreserveLocalSymbols(
             PreserveCpuScope(hint, ranking.RootFrame, scope), symbols);
 
@@ -796,7 +834,9 @@ public static IReadOnlyList<string> ForTimeline(TimelineResult timeline)
             string window = $"{FormatMs(timeline.FromMs)}-{FormatMs(timeline.ToMs)} ms";
             if (snapshot.Cpu.SampleCount > 0)
             {
-                return SnapshotDrillGuidance("CPU work", "cpu", timeline, window);
+                return SnapshotDrillGuidance(
+                    timeline.CpuSampleWarning is null ? "CPU work" : "sampled thread stacks",
+                    "cpu", timeline, window);
             }
 
             if (snapshot.Alloc.Types.Count > 0)
@@ -817,7 +857,9 @@ public static IReadOnlyList<string> ForTimeline(TimelineResult timeline)
         // nothing to point at, so it is skipped.
         if (TryPeakBucket(timeline.Cpu, static bucket => bucket.SampleCount, out int cpuIndex))
         {
-            return DrillWindowGuidance("CPU", "cpu", timeline, cpuIndex);
+            return DrillWindowGuidance(
+                timeline.CpuSampleWarning is null ? "CPU" : "thread-stack sample",
+                "cpu", timeline, cpuIndex);
         }
 
         if (TryPeakBucket(timeline.Alloc, static bucket => bucket.Count, out int allocIndex))