feat(payload): v1.0 (idx/ETP) trace agent payload - hardening, wire-format validation, and fingerprint corpus

CHANGELOG.md

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+- Trace-agent payload generation for `/v1.0/traces` (idx/ETP), with a service
+  graph, streaming string interning, error injection, span links and events,
+  and an exact `chunks_per_payload` batch size.
+
+### Fixed
+- Block-cache construction now fails with an actionable error after 1024
+  consecutive rejected blocks instead of retrying indefinitely.
+
 ## [0.33.0]
 ### Changed
 - **Breaking Change**: Datadog blackhole now records received series under the

Cargo.toml

@@ -14,6 +14,7 @@ members = [
 [workspace.dependencies]
 anyhow = "1.0"
 arc-swap = { version = "1.8" }
+base64 = { version = "0.22", default-features = false }
 arrow-array = { version = "57.0", default-features = false }
 arrow-buffer = { version = "57.0", default-features = false }
 arrow-schema = { version = "57.0", default-features = false }
@@ -29,6 +30,7 @@ metrics-util = "0.20"
 metrics-exporter-prometheus = { version = "0.18", default-features = false, features = [
   "uds-listener",
 ] }
+rmp = { version = "0.8", default-features = false }
 prost = "0.14"
 prost-build = "0.14"
 rand = { version = "0.10", default-features = false }

ci/fingerprints/trace_agent_v1/fingerprint.txt

@@ -0,0 +1 @@
+trace_agent_v1_tcp: 061ca1a27fb3649bb2e946774d53cee37f4d67d7f161bd9752e04fcc6c1cdc0e entropy=6.6435

ci/fingerprints/trace_agent_v1/lading.yaml

@@ -0,0 +1,60 @@
+generator:
+  - id: "trace_agent_v1_tcp"
+    tcp:
+      seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]
+      addr: "127.0.0.1:8126"
+      variant:
+        trace_agent:
+          v1.0:
+            timestamp_mode:
+              fixed:
+                anchor_unix_nanos: 1735689600000000000
+            error_rate: 0.1
+            chunks_per_payload: 1
+            language_name: "go"
+            language_version: "1.24.2"
+            tracer_version: "2.5.0"
+            env: "production"
+            app_version: "1.0.0"
+            priority: 1
+            services:
+              - name: "api-gateway"
+                operations:
+                  - id: "get-product"
+                    name: "http.request"
+                    resource: "GET /api/v1/products/{id}"
+                    span_type: "web"
+                    kind: 2
+                    component: "net/http"
+                    attributes:
+                      http.method: "GET"
+                    suboperations:
+                      - to: "product-service/get-product"
+                        rate: 1.0
+              - name: "product-service"
+                operations:
+                  - id: "get-product"
+                    name: "grpc.server.request"
+                    resource: "/ProductService/GetProduct"
+                    span_type: "web"
+                    kind: 2
+                    component: "grpc"
+                    suboperations:
+                      - to: "product-cache/get-product-by-id"
+                        rate: 1.0
+              - name: "product-cache"
+                operations:
+                  - id: "get-product-by-id"
+                    name: "redis.command"
+                    resource: "GET products:by_id:$1"
+                    span_type: "redis"
+                    kind: 3
+                    component: "redis"
+      bytes_per_second: "100 Mb"
+      maximum_block_size: "1 Mb"
+      maximum_prebuild_cache_size_bytes: "10 Mb"
+      parallel_connections: 1
+
+blackhole:
+  - http:
+      binding_addr: "127.0.0.1:9999"

examples/trace-agent-v1.yaml

@@ -0,0 +1,72 @@
+generator:
+  - id: "trace_agent_v1"
+    trace_agent:
+      seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137]
+      target_uri: "http://localhost:8126"
+      backoff_behavior:
+        obey:
+          max_retries: 3
+      variant:
+        v1.0:
+          timestamp_mode:
+            fixed:
+              anchor_unix_nanos: 1767225600000000000 # 2026-01-01T00:00:00Z; deterministic across runs
+          error_rate: 0.05
+          chunks_per_payload: 1 # Exact chunks per HTTP request; not a minimum.
+          language_name: "go"
+          language_version: "1.24.2"
+          tracer_version: "2.5.0"
+          env: "production"
+          app_version: "1.0.0"
+          priority: 1
+          services:
+            - name: "api-gateway"
+              operations:
+                - id: "get-product"
+                  name: "http.request"
+                  resource: "GET /api/v1/products/{id}"
+                  span_type: "web"
+                  kind: 2
+                  component: "net/http"
+                  attributes:
+                    http.method: "GET"
+                  suboperations:
+                    - to: "product-service/get-product"
+                      rate: 1.0
+            - name: "product-service"
+              operations:
+                - id: "get-product"
+                  name: "grpc.server.request"
+                  resource: "/ProductService/GetProduct"
+                  span_type: "web"
+                  kind: 2
+                  component: "grpc"
+                  suboperations:
+                    - to: "product-cache/get-product-by-id"
+                      rate: 0.9
+                    - to: "product-db/select-product-by-id"
+                      rate: 0.5
+            - name: "product-cache"
+              operations:
+                - id: "get-product-by-id"
+                  name: "redis.command"
+                  resource: "GET products:by_id:$1"
+                  span_type: "redis"
+                  kind: 3
+                  component: "redis"
+            - name: "product-db"
+              operations:
+                - id: "select-product-by-id"
+                  name: "postgres.query"
+                  resource: "SELECT * FROM products WHERE id = $1"
+                  span_type: "sql"
+                  kind: 3
+                  component: "postgresql"
+      bytes_per_second: "50 MiB"
+      maximum_block_size: "1 MiB"
+      maximum_prebuild_cache_size_bytes: "256 MiB"
+      parallel_connections: 1
+
+blackhole:
+  - http:
+      binding_addr: "0.0.0.0:8126"

lading/src/bin/payloadtool.rs

@@ -496,7 +496,7 @@ fn check_generator(config: &generator::Config, args: &Args) -> Result<Option<Fin
             let total_bytes =
                 generator::trace_agent::validate_cache_size(g.maximum_prebuild_cache_size_bytes)
                     .map_err(|e| anyhow::anyhow!("Cache size validation failed: {e}"))?;
-            let conf = lading_payload::Config::TraceAgent(g.variant);
+            let conf = lading_payload::Config::TraceAgent(g.variant.clone());
             generate_and_check(&conf, g.seed, total_bytes, g.maximum_block_size, args)
         }
     }

lading/src/generator/trace_agent.rs

@@ -284,7 +284,7 @@ impl TraceAgent {
         let maximum_prebuild_cache_size_bytes =
             validate_cache_size(config.maximum_prebuild_cache_size_bytes)?;
         let maximum_block_size = config.maximum_block_size.as_u128();
-        let block_config = lading_payload::Config::TraceAgent(config.variant);
+        let block_config = lading_payload::Config::TraceAgent(config.variant.clone());
         let block_cache = Arc::new(match config.block_cache_method {
             block::CacheMethod::Fixed => block::Cache::fixed_with_max_overhead(
                 &mut rng,

lading_payload/Cargo.toml

@@ -20,6 +20,7 @@ opentelemetry-proto = { workspace = true }
 prost = { workspace = true }
 rand = { workspace = true, features = ["std", "std_rng"] }
 rustc-hash = { workspace = true }
+rmp = { workspace = true }
 rmp-serde = { version = "1.3", default-features = false }
 serde = { workspace = true }
 serde_json = { workspace = true }
@@ -33,6 +34,7 @@ arbitrary = { version = "1", optional = true, features = ["derive"] }
 enum_dispatch = { workspace = true }
 
 [dev-dependencies]
+base64 = { workspace = true, features = ["std"] }
 proptest = { workspace = true }
 proptest-derive = { workspace = true }
 criterion = { version = "0.8", features = ["html_reports"] }

lading_payload/proptest-regressions/trace_agent/v1/test.txt

@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc c42a453f2f561ead27c37e3d089901a423111fd9bca0ec478d34849bcba6b4be # shrinks to seed = 6659423045877441987, max_bytes = 27301