diff --git a/api/src/main/java/org/apache/flink/agents/api/context/RunnerContext.java b/api/src/main/java/org/apache/flink/agents/api/context/RunnerContext.java
index a61dc0d18b..76f8b11a1f 100644
--- a/api/src/main/java/org/apache/flink/agents/api/context/RunnerContext.java
+++ b/api/src/main/java/org/apache/flink/agents/api/context/RunnerContext.java
@@ -72,6 +72,8 @@ public interface RunnerContext {
* from inside a {@link #durableExecute} or {@link #durableExecuteAsync} callable, which runs on
* a separate thread pool.
*
+ *
Names used by built-in Agent metrics are reserved in this group.
+ *
* @return the metric group shared across all actions.
*/
FlinkAgentsMetricGroup getAgentMetricGroup();
@@ -83,6 +85,8 @@ public interface RunnerContext {
* from inside a {@link #durableExecute} or {@link #durableExecuteAsync} callable, which runs on
* a separate thread pool.
*
+ *
Names used by built-in Action metrics are reserved in this group.
+ *
* @return the individual metric group specific to the current action.
*/
FlinkAgentsMetricGroup getActionMetricGroup();
diff --git a/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionLifecycleEvents.java b/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionLifecycleEvents.java
index dd00a59907..004aa2f595 100644
--- a/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionLifecycleEvents.java
+++ b/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionLifecycleEvents.java
@@ -31,11 +31,13 @@
/** Event factory for execution lifecycle reports in the trace model. */
public final class ExecutionLifecycleEvents {
+ public static final String EXECUTION_CREATED_EVENT_TYPE = "_execution_created_event";
public static final String EXECUTION_STARTED_EVENT_TYPE = "_execution_started_event";
public static final String EXECUTION_FINISHED_EVENT_TYPE = "_execution_finished_event";
public static final String EXECUTION_FAILED_EVENT_TYPE = "_execution_failed_event";
public static final String EXECUTION_REUSED_EVENT_TYPE = "_execution_reused_event";
+ public static final String STATUS_CREATED = "created";
public static final String STATUS_STARTED = "started";
public static final String STATUS_SUCCESS = "success";
public static final String STATUS_FAILED = "failed";
@@ -45,6 +47,10 @@ public final class ExecutionLifecycleEvents {
private ExecutionLifecycleEvents() {}
+ public static Event executionCreated() {
+ return eventWithStatus(EXECUTION_CREATED_EVENT_TYPE, STATUS_CREATED);
+ }
+
public static Event executionStarted() {
return eventWithStatus(EXECUTION_STARTED_EVENT_TYPE, STATUS_STARTED);
}
@@ -59,7 +65,8 @@ public static Event executionReused() {
/** Returns whether the given type identifies an execution lifecycle event. */
public static boolean isExecutionLifecycleEvent(String eventType) {
- return EXECUTION_STARTED_EVENT_TYPE.equals(eventType)
+ return EXECUTION_CREATED_EVENT_TYPE.equals(eventType)
+ || EXECUTION_STARTED_EVENT_TYPE.equals(eventType)
|| EXECUTION_FINISHED_EVENT_TYPE.equals(eventType)
|| EXECUTION_FAILED_EVENT_TYPE.equals(eventType)
|| EXECUTION_REUSED_EVENT_TYPE.equals(eventType);
diff --git a/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionReporter.java b/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionReporter.java
index a3fb6f11b3..bb049f0aa8 100644
--- a/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionReporter.java
+++ b/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionReporter.java
@@ -26,7 +26,7 @@
*
*
Implementations decide how reports are consumed or ignored. Callers should provide stable
* entity type/name pairs and keep metadata small, structured, serializable, and stable for equality
- * matching between the start and terminal reports of the same logical execution.
+ * matching between lifecycle reports of the same logical execution.
*/
public interface ExecutionReporter {
@@ -50,6 +50,22 @@ final class ProblemCategories {
private ProblemCategories() {}
}
+ /**
+ * Reports that a logical execution has been created but has not necessarily started.
+ *
+ *
This is an optional lifecycle phase for executions whose admission and invocation are
+ * observably separate. Implementations that do not consume it may keep the default no-op. A
+ * later start or terminal report is not guaranteed, so consumers must not infer whether the
+ * underlying invocation ran from the absence of either report.
+ *
+ * @param entityType stable category of the reported execution, such as LLM, parser, or tool
+ * @param entityName stable name of the reported execution, such as model or tool name
+ * @param entityMetadata small structured metadata used to match subsequent lifecycle reports
+ */
+ default void reportExecutionCreated(
+ String entityType, String entityName, Map entityMetadata)
+ throws Exception {}
+
/**
* Reports that a logical execution started within the current action.
*
@@ -63,7 +79,22 @@ void reportExecutionStarted(
throws Exception;
/**
- * Reports that a previously started logical execution completed successfully.
+ * Reports that a logical execution started at the given occurrence timestamp.
+ *
+ * The default implementation delegates to {@link #reportExecutionStarted(String, String,
+ * Map)}, so reporters that do not retain occurrence timestamps may use their observation time.
+ */
+ default void reportExecutionStartedAt(
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ String timestamp)
+ throws Exception {
+ reportExecutionStarted(entityType, entityName, entityMetadata);
+ }
+
+ /**
+ * Reports that a logical execution completed successfully.
*
* The entity type/name/metadata should match the corresponding start report when one was
* reported.
@@ -72,6 +103,21 @@ void reportExecutionSucceeded(
String entityType, String entityName, Map entityMetadata)
throws Exception;
+ /**
+ * Reports that a logical execution completed successfully at the given occurrence timestamp.
+ *
+ * The default implementation delegates to {@link #reportExecutionSucceeded(String, String,
+ * Map)}, so reporters that do not retain occurrence timestamps may use their observation time.
+ */
+ default void reportExecutionSucceededAt(
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ String timestamp)
+ throws Exception {
+ reportExecutionSucceeded(entityType, entityName, entityMetadata);
+ }
+
/**
* Reports that a logical execution failed.
*
@@ -85,4 +131,22 @@ void reportExecutionFailed(
Throwable error,
@Nullable String problemCategory)
throws Exception;
+
+ /**
+ * Reports that a logical execution failed at the given occurrence timestamp.
+ *
+ * The default implementation delegates to {@link #reportExecutionFailed(String, String, Map,
+ * Throwable, String)}, so reporters that do not retain occurrence timestamps may use their
+ * observation time.
+ */
+ default void reportExecutionFailedAt(
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ Throwable error,
+ @Nullable String problemCategory,
+ String timestamp)
+ throws Exception {
+ reportExecutionFailed(entityType, entityName, entityMetadata, error, problemCategory);
+ }
}
diff --git a/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionReporters.java b/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionReporters.java
index 87ca12445e..2a0b3cd021 100644
--- a/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionReporters.java
+++ b/api/src/main/java/org/apache/flink/agents/api/trace/ExecutionReporters.java
@@ -41,6 +41,21 @@ public final class ExecutionReporters {
private ExecutionReporters() {}
+ public static void created(RunnerContext ctx, String entityType, String entityName) {
+ created(ctx, entityType, entityName, EMPTY_METADATA);
+ }
+
+ public static void created(
+ RunnerContext ctx,
+ String entityType,
+ String entityName,
+ Map entityMetadata) {
+ report(
+ ctx,
+ reporter -> reporter.reportExecutionCreated(entityType, entityName, entityMetadata),
+ null);
+ }
+
public static void started(RunnerContext ctx, String entityType, String entityName) {
started(ctx, entityType, entityName, EMPTY_METADATA);
}
@@ -56,6 +71,20 @@ public static void started(
null);
}
+ public static void startedAt(
+ RunnerContext ctx,
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ String timestamp) {
+ report(
+ ctx,
+ reporter ->
+ reporter.reportExecutionStartedAt(
+ entityType, entityName, entityMetadata, timestamp),
+ null);
+ }
+
public static void succeeded(RunnerContext ctx, String entityType, String entityName) {
succeeded(ctx, entityType, entityName, EMPTY_METADATA);
}
@@ -72,6 +101,20 @@ public static void succeeded(
null);
}
+ public static void succeededAt(
+ RunnerContext ctx,
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ String timestamp) {
+ report(
+ ctx,
+ reporter ->
+ reporter.reportExecutionSucceededAt(
+ entityType, entityName, entityMetadata, timestamp),
+ null);
+ }
+
public static void failed(
RunnerContext ctx,
String entityType,
@@ -96,6 +139,27 @@ public static void failed(
error);
}
+ public static void failedAt(
+ RunnerContext ctx,
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ Throwable error,
+ @Nullable String problemCategory,
+ String timestamp) {
+ report(
+ ctx,
+ reporter ->
+ reporter.reportExecutionFailedAt(
+ entityType,
+ entityName,
+ entityMetadata,
+ error,
+ problemCategory,
+ timestamp),
+ error);
+ }
+
private static void report(
RunnerContext ctx, ReporterCall reporterCall, @Nullable Throwable businessError) {
if (ctx instanceof ExecutionReporter) {
diff --git a/api/src/main/java/org/apache/flink/agents/api/trace/ToolExecutionMetadataKeys.java b/api/src/main/java/org/apache/flink/agents/api/trace/ToolExecutionMetadataKeys.java
index 85f910c9fa..acc6d96c7f 100644
--- a/api/src/main/java/org/apache/flink/agents/api/trace/ToolExecutionMetadataKeys.java
+++ b/api/src/main/java/org/apache/flink/agents/api/trace/ToolExecutionMetadataKeys.java
@@ -26,6 +26,7 @@ public final class ToolExecutionMetadataKeys {
public static final String TOOL_TYPE = "toolType";
public static final String MCP_SERVER = "mcpServer";
public static final String SKILL_NAME = "skillName";
+ public static final String SKILL_REGISTERED = "skillRegistered";
public static final String SKILL_RESOURCE_PATH = "skillResourcePath";
private ToolExecutionMetadataKeys() {}
diff --git a/api/src/test/java/org/apache/flink/agents/api/trace/ExecutionLifecycleEventsTest.java b/api/src/test/java/org/apache/flink/agents/api/trace/ExecutionLifecycleEventsTest.java
index bec417e392..870fec45a6 100644
--- a/api/src/test/java/org/apache/flink/agents/api/trace/ExecutionLifecycleEventsTest.java
+++ b/api/src/test/java/org/apache/flink/agents/api/trace/ExecutionLifecycleEventsTest.java
@@ -28,6 +28,17 @@
/** Tests for {@link ExecutionLifecycleEvents}. */
class ExecutionLifecycleEventsTest {
+ @Test
+ void executionCreatedUsesReservedLifecycleShape() {
+ Event event = ExecutionLifecycleEvents.executionCreated();
+
+ assertThat(event.getType())
+ .isEqualTo(ExecutionLifecycleEvents.EXECUTION_CREATED_EVENT_TYPE);
+ assertThat(event.getAttr(ExecutionLifecycleEvents.STATUS_ATTRIBUTE))
+ .isEqualTo(ExecutionLifecycleEvents.STATUS_CREATED);
+ assertThat(ExecutionLifecycleEvents.isExecutionLifecycleEvent(event.getType())).isTrue();
+ }
+
@Test
void executionFailedUsesDeepestCause() {
IllegalArgumentException root = new IllegalArgumentException("root");
diff --git a/api/src/test/java/org/apache/flink/agents/api/trace/ExecutionReportersTest.java b/api/src/test/java/org/apache/flink/agents/api/trace/ExecutionReportersTest.java
index 4699036341..e0dad3a6e5 100644
--- a/api/src/test/java/org/apache/flink/agents/api/trace/ExecutionReportersTest.java
+++ b/api/src/test/java/org/apache/flink/agents/api/trace/ExecutionReportersTest.java
@@ -36,11 +36,16 @@
class ExecutionReportersTest {
@Test
- void startedAndSucceededIgnoreReporterFailures() throws Exception {
+ void createdStartedAndSucceededIgnoreReporterFailures() throws Exception {
RunnerContext ctx = mockReportingContext();
ExecutionReporter reporter = (ExecutionReporter) ctx;
+ Exception createdError = new Exception("created failed");
Exception startedError = new Exception("started failed");
Exception succeededError = new Exception("succeeded failed");
+ doThrow(createdError)
+ .when(reporter)
+ .reportExecutionCreated(
+ eq(ExecutionReporter.EntityTypes.LLM), eq("model-a"), anyMap());
doThrow(startedError)
.when(reporter)
.reportExecutionStarted(
@@ -50,6 +55,11 @@ void startedAndSucceededIgnoreReporterFailures() throws Exception {
.reportExecutionSucceeded(
eq(ExecutionReporter.EntityTypes.LLM), eq("model-a"), anyMap());
+ assertThatCode(
+ () ->
+ ExecutionReporters.created(
+ ctx, ExecutionReporter.EntityTypes.LLM, "model-a"))
+ .doesNotThrowAnyException();
assertThatCode(
() ->
ExecutionReporters.started(
@@ -61,6 +71,9 @@ void startedAndSucceededIgnoreReporterFailures() throws Exception {
ctx, ExecutionReporter.EntityTypes.LLM, "model-a"))
.doesNotThrowAnyException();
+ verify(reporter)
+ .reportExecutionCreated(
+ eq(ExecutionReporter.EntityTypes.LLM), eq("model-a"), anyMap());
verify(reporter)
.reportExecutionStarted(
eq(ExecutionReporter.EntityTypes.LLM), eq("model-a"), anyMap());
@@ -103,6 +116,11 @@ void helpersIgnoreContextsWithoutExecutionReporter() {
RunnerContext ctx = mock(RunnerContext.class);
RuntimeException businessError = new RuntimeException("business failed");
+ assertThatCode(
+ () ->
+ ExecutionReporters.created(
+ ctx, ExecutionReporter.EntityTypes.LLM, "model-a"))
+ .doesNotThrowAnyException();
assertThatCode(
() ->
ExecutionReporters.started(
diff --git a/docs/content/docs/operations/configuration.md b/docs/content/docs/operations/configuration.md
index 4cc35929eb..452ede9e12 100644
--- a/docs/content/docs/operations/configuration.md
+++ b/docs/content/docs/operations/configuration.md
@@ -130,7 +130,7 @@ Here is the list of all built-in core configuration options.
| `action.trigger-condition.evaluate-failure-strategy` | `WARN_AND_SKIP` | ConditionEvaluationFailureStrategy | Handles event-time failures while preparing variables for or evaluating a compiled condition, including a dynamic non-Boolean result. `WARN_AND_SKIP` (default): log a warning, treat that condition as false, and continue with later OR conditions. `FAIL`: throw `IllegalStateException` and fail the Flink task; recovery follows the job's restart configuration. Plan-validation failures and runtime compilation or static type-check failures occur during initialization and are not handled by this option. |
| `error-handling-strategy` | ErrorHandlingStrategy.FAIL | ErrorHandlingStrategy | Strategy for handling errors during model requests, include timeout and unexpected output schema. The option value could be: `ErrorHandlingStrategy.FAIL` `ErrorHandlingStrategy.RETRY` `ErrorHandlingStrategy.IGNORE` |
| `max-retries` | 3 | int | Number of retries when using `ErrorHandlingStrategy.RETRY`. |
-| `retry-wait-interval` | 1 | int | Base wait interval in seconds between retries when using `ErrorHandlingStrategy.RETRY`. Uses exponential backoff: the actual wait time for the Nth retry is `retry-wait-interval * 2^(N-1)` seconds. For example, with default 1s, waits are 1s, 2s, 4s, etc. Retry count and total wait time are reported in `ChatResponseEvent` and recorded as metrics (`retryCount`, `retryWaitSec`) under the connection name. |
+| `retry-wait-interval` | 1 | int | Base wait interval in seconds between retries when using `ErrorHandlingStrategy.RETRY`. Uses exponential backoff: the actual wait time for the Nth retry is `retry-wait-interval * 2^(N-1)` seconds. For example, with default 1s, waits are 1s, 2s, 4s, etc. Retry count and total wait time are reported in `ChatResponseEvent` and recorded as metrics (`retryCount`, `retryWaitSec`) under the configured ChatModel resource name. |
| `chat.async` | true | boolean | Whether chat asynchronously for built-in chat action. |
| `tool-call.async` | true | boolean | Whether the built-in tool-call action runs each tool via durable async execution. |
| `tool-call.parallelism` | os cpu count | int | In-flight concurrency for tool calls from one `ToolRequestEvent` batch when `tool-call.async` is enabled. `1` runs tools serially; values greater than `1` run a parallel durable batch with a sliding window of at most that many concurrent tool calls. On **Java**, concurrent in-batch execution requires **JDK 21+** (Continuation API); below JDK 21 the batch still runs but tool calls execute serially. **Python** uses the shared async `ThreadPoolExecutor` and runs batches concurrently regardless of JDK version. Increases in-flight external calls; after failover, unfinished tools may be submitted again — side-effecting tools should be idempotent or provide a reconciler. {{< hint warning >}}**Default is parallel** (`os cpu count`). Chat, RAG, and tool batches share one `num-async-threads` pool **per operator subtask** (all keys on that subtask). Built-in actions for a single key run one at a time, so chat and a tool batch on the **same key** do not overlap in the usual chat → tool path; delay shows up mainly **across keys** on the same subtask. With defaults (`num-async-threads = 2× cores`, `tool-call.parallelism = cores`), one batch can use up to half the pool; several busy keys can still saturate it. Lower this value or increase `num-async-threads` on hot subtasks. {{< /hint >}} |
diff --git a/docs/content/docs/operations/monitoring.md b/docs/content/docs/operations/monitoring.md
index 652c9bfe90..357fff51b7 100644
--- a/docs/content/docs/operations/monitoring.md
+++ b/docs/content/docs/operations/monitoring.md
@@ -26,7 +26,7 @@ under the License.
### Built-in Metrics
-We offer data monitoring for built-in metrics, which includes events, actions, and token usage.
+We offer data monitoring for built-in metrics, including input runs, events, actions, execution health, and token usage.
#### Event and Action Metrics
@@ -36,11 +36,64 @@ We offer data monitoring for built-in metrics, which includes events, actions, a
| **Agent** | numOfEventProcessedPerSec | The number of Events this operator has processed per second. | Meter |
| **Agent** | numOfActionsExecuted | The total number of actions this operator has executed. | Count |
| **Agent** | numOfActionsExecutedPerSec | The number of actions this operator has executed per second. | Meter |
+| **Agent** | numOfInputRunsSucceeded | The number of input runs that reached the run-completion boundary. | Count |
+| **Agent** | numOfInputRunsFailed | The number of input runs terminated by an unhandled exception. | Count |
+| **Agent** | inputRunLatencyMs | End-to-end input-run latency from entering the agent operator to completion or failure, including time queued behind another input with the same key. | Histogram |
+| **Agent** | inputRunQueueLatencyMs | Time from entering the agent operator until the input run starts processing. | Histogram |
+| **Agent** | inputRunProcessingLatencyMs | Time from the input-run start boundary until completion or failure. | Histogram |
+| **Agent** | numOfPendingInputEvents | Current number of input Events buffered behind an active run with the same key. | Gauge |
+| **Agent** | numOfActiveInputRuns | Current number of logical input runs that are processing or waiting for asynchronous work. | Gauge |
| **Action** | action.\.numOfActionsExecuted | The total number of actions this operator has executed for a specific action name. | Count |
| **Action** | action.\.numOfActionsExecutedPerSec | The number of actions this operator has executed per second for a specific action name. | Meter |
+| **Action** | action.\.actionSchedulingLatencyMs | Time from enqueuing the initial Action task until it is selected for execution. | Histogram |
+| **Action** | action.\.actionExecutionLatencyMs | End-to-end latency of one logical Action execution, including asynchronous waits and continuations. | Histogram |
+| **Action** | action.\.numOfPendingActionTasks | Current number of physical Action task segments waiting to run, including continuations. | Gauge |
+| **Action** | action.\.numOfActiveActionExecutions | Current number of logical Action executions that have started but have not reached a terminal state. | Gauge |
| **Agent** | eventLogTruncatedEvents | Number of event log records whose payload was truncated at `STANDARD` level. Increments once per event, regardless of how many fields inside it were truncated. Use this to decide whether to raise truncation thresholds or move specific event types to `VERBOSE`. | Count |
| **Agent** | eventLogWriteFailures | Number of Event Log write attempts for which `append`, `flush`, or both failed. Event Log writes are best-effort and do not fail the job. | Count |
+For a locally observed input run, `inputRunLatencyMs` is split into queueing and processing time at the input-run start boundary. `numOfPendingInputEvents` counts buffered inputs, while `numOfActiveInputRuns` counts logical runs; an asynchronous run remains active while it is waiting for its continuation.
+
+An Action execution can be active while one of its continuation tasks is pending, so `numOfActiveActionExecutions` and `numOfPendingActionTasks` are independent. Action scheduling latency is recorded only for the initial task; continuation queueing does not create another scheduling sample.
+
+Input-run outcomes and all latency samples are process-local. Runs or Action executions already in flight when a task is restored do not produce latency samples because their original timestamps are unavailable. An input Event restored from the pending queue can still produce an outcome and processing-latency sample after it starts in the new task attempt, but it does not produce queue or end-to-end latency. Current-count gauges are rebuilt from Flink state after restore.
+
+#### Execution Metrics
+
+LLM and Tool outcome and latency metrics are derived from execution lifecycle Events. A Tool execution is created once its call identity and metadata are available. This happens before a preparation failure is reported and, for an invocable call, before submission to the durable execution path. The Tool callable then records its own start and completion timestamps. Start and terminal Events may be delivered after a parallel batch completes, but retain each call's occurrence timestamps rather than using the batch duration. The durable execution `Outcome` indicates whether the invocation returned or raised; a returned `ToolResponse` independently indicates whether the Tool operation succeeded or failed. Metrics preserve both layers without redefining the existing durable-persistence semantics. Event publication is independent of response aggregation, so a later response-processing failure does not repeat or discard reports for calls with available Outcomes. The `model_resource`, `tool`, `skill`, and `mcp_server` scopes are independent key-value scopes directly under an Action; none is nested under another. The existing `model` scope remains dedicated to model usage metrics.
+
+| Scope | Metrics | Description | Type |
+|-------|---------|-------------|------|
+| **Model Resource** | action.\.model_resource.\.numOfLlmCallsSucceeded | The number of framework-observed model invocations that returned successfully. | Count |
+| **Model Resource** | action.\.model_resource.\.numOfLlmCallsFailed | The number of framework-observed model invocations that failed. | Count |
+| **Model Resource** | action.\.model_resource.\.llmCallLatencyMs | Latency of each framework-observed model invocation, excluding structured-output parsing and retry wait time. | Histogram |
+| **Model Resource** | action.\.model_resource.\.retryCount | The number of additional model invocations initiated when `ErrorHandlingStrategy.RETRY` is configured. Only recorded when at least one retry occurs. See [retry-wait-interval]({{< ref "docs/operations/configuration#core-options" >}}). | Count |
+| **Model Resource** | action.\.model_resource.\.retryWaitSec | The total backoff time, in seconds, accumulated when `ErrorHandlingStrategy.RETRY` is configured. Only recorded when at least one retry occurs. | Count |
+| **Tool** | action.\.tool.\.numOfToolCallsSucceeded | The number of successful calls to the Tool. | Count |
+| **Tool** | action.\.tool.\.numOfToolCallsFailed | The number of failed calls to the Tool. | Count |
+| **Tool** | action.\.tool.\.toolCallLatencyMs | Time spent invoking the individual Tool, excluding time waiting for other calls in the same parallel batch. | Histogram |
+| **Skill** | action.\.skill.\.numOfSkillLoads | The number of terminal explicit `load_skill` calls attributed to the Skill, regardless of outcome. | Count |
+| **Skill** | action.\.skill.\.skillLoadLatencyMs | Time spent invoking an explicit `load_skill` call. | Histogram |
+| **MCP Server** | action.\.mcp_server.\.numOfMcpToolCallsSucceeded | The number of successful Tool calls served by the MCP Server. | Count |
+| **MCP Server** | action.\.mcp_server.\.numOfMcpToolCallsFailed | The number of failed Tool calls served by the MCP Server. | Count |
+| **MCP Server** | action.\.mcp_server.\.mcpToolCallLatencyMs | Individual Tool invocation latency aggregated across the MCP Server. | Histogram |
+
+An LLM metric represents one framework invocation of `ChatModel`. A framework retry that calls the model again produces another LLM outcome and latency sample; retries hidden inside a provider or connection are not observed. Every named Tool execution emits Tool metrics. Skill metrics are emitted only for explicit `load_skill` calls; subsequent Tool calls are not inferred to belong to a Skill. MCP metrics aggregate only Tool executions carrying an explicit MCP Server resource name. A `load_skill` or MCP Tool execution therefore contributes to both its Tool scope and the corresponding Skill or MCP Server scope.
+
+Execution metrics currently inherit Agent Trace's durable-replay behavior. During fine-grained recovery, a cached durable child result is reported as a new execution because child cache reuse is not exposed to execution reporting. A cached LLM result is reported as successful. A cached Tool result retains its normalized Tool outcome: an explicit `ToolResponse.error(...)` is reported as failed, while a successful `ToolResponse` or a raw Python return is reported as successful. A cached LLM result may produce a near-zero latency sample; a cached Tool result produces no latency sample because the Tool callable was not invoked and no execution duration was measured. Distinguishing reused child executions is follow-up work.
+
+Tool names that are not registered runtime resources are aggregated under the fixed `tool=unknown` scope. Requested Skill names that do not resolve in the runtime registry are similarly aggregated under `skill=unknown`. The original requested names remain available in Agent Trace records, while Metric scope cardinality remains bounded.
+
+Tool outcomes follow the same two-layer contract in Java and Python. Resource preparation and invocation exceptions are failures. A returned `ToolResponse.error(...)` is also a failed Tool execution even though the durable invocation returned normally. Existing Python Tools may continue returning arbitrary raw values, which are treated as successful without inspecting their payload. A durable-persistence exception is reflected as a Tool failure only when the existing durable execution path exposes it to `ToolCallAction`.
+
+`numOfSkillLoads` counts terminal calls rather than successful loads. In both runtimes, `load_skill` reports an unavailable manager, missing Skill, or missing resource through `ToolResponse.error(...)`; the Tool outcome is therefore failed while the Skill load counter still increments. MCP protocol-level errors follow the same failed Tool outcome contract.
+
+Execution latency tracking is process-local and uses the occurrence timestamps in matching start and terminal Events. Creation Events do not start latency measurement. Both start and terminal Events must be observed in the same task attempt. A Tool latency sample additionally requires the Tool callable to start no later than the Action observes its durable result; queueing and parallel-batch fan-in are excluded. LLM and Tool terminal counters are still updated when no matching start Event is available.
+
+A request timeout does not necessarily stop a running Tool. `ToolCallAction` records when the durable call returns or raises, before processing responses or publishing terminal Events. If a Tool has not finished by that time, its terminal Event uses this fixed observation time; a later Tool completion cannot extend it or produce another terminal Event. This excludes response-processing and Event-publication delays, but is not the execution framework's exact timeout-decision time: any delay before the durable result reaches the Action remains included. A callable that starts after this observation retains its creation and timeout terminal Events without a start Event or latency sample, even if it later completes in the background. Cached results and failures before invocation follow the same creation-plus-terminal shape. If a batch aborts with an ordinary Exception without returning per-call Outcomes, every prepared call retains its creation Event and known starts may also be reported, but no terminal execution Events are inferred from timestamps or the batch exception. If a Java batch instead propagates an `Error`, directly or wrapped by a `CompletionException`, unresolved prepared calls are reported as failed and the original `Error` is rethrown. These failures denote a fatal batch abort, not that every Tool body was invoked. Existing business `ToolResponseEvent` error handling is unchanged. These observation rules do not change the execution framework's timeout, failure, or durable-persistence behavior.
+
+In previous releases, `retryCount` and `retryWaitSec` used the `model.` scope. They now use `model_resource.` so retries are attributed to the configured ChatModel resource. Existing queries and dashboards for these two metrics must use the new scope.
+
#### Token Usage Metrics
Token usage metrics are automatically recorded when chat models are invoked through `ChatModelConnection`. These metrics help track LLM API usage and costs.
@@ -49,13 +102,13 @@ Token usage metrics are automatically recorded when chat models are invoked thro
|-----------|--------------------------------------------------------------|--------------------------------------------------------------------------------|-------|
| **Model** | action.\.model.\.promptTokens | The total number of prompt tokens consumed by the model within an action. | Count |
| **Model** | action.\.model.\.completionTokens | The total number of completion tokens generated by the model within an action. | Count |
-| **Model** | action.\.model.\.retryCount | The total number of retries performed for model requests when using `ErrorHandlingStrategy.RETRY`. Only recorded when at least one retry occurs. See [retry-wait-interval]({{< ref "docs/operations/configuration#core-options" >}}). | Count |
-| **Model** | action.\.model.\.retryWaitSec | The total wait time (in seconds) spent across retries for model requests when using `ErrorHandlingStrategy.RETRY`. | Count |
### How to add custom metrics
In Flink Agents, users implement their logic by defining custom Actions that respond to various Events throughout the Agent lifecycle. To support user-defined metrics, we introduce two new properties: `agent_metric_group` and `action_metric_group` in the RunnerContext. These properties allow users to create or update global metrics and independent metrics for actions. For an introduction to metric types, please refer to the [Metric types documentation](https://nightlies.apache.org/flink/flink-docs-release-1.20/docs/ops/metrics/#metric-types).
+Metric names listed in the built-in tables above are reserved in their corresponding scopes. Custom metrics must use different names within the same scope.
+
Here is the user case example:
{{< tabs "Custom Metrics" >}}
@@ -116,7 +169,7 @@ public class MyAgent extends Agent {
### How to check the metrics with Flink executor
-Flink agents enable the reporting of metrics to external systems by creating a metric identifier prefix in the format `.taskmanager....`. Agent-specific metrics use key-value metric groups (e.g., `action.`, `model.`) which are exposed as dimensions/labels in reporters that support them (such as Prometheus). Please refer to [Flink Metric Reporters](https://nightlies.apache.org/flink/flink-docs-release-1.20/docs/deployment/metric_reporters/) for more details.
+Flink agents enable the reporting of metrics to external systems by creating a metric identifier prefix in the format `.taskmanager....`. For an agent operator, `` is the agent name. If the Agent name is unavailable, the operator retains the previous `action-execute-operator` value as a fallback. This changes only the value of the existing `` scope; the Agent-specific metric hierarchy is unchanged. Queries and dashboards that filter on `operator_name=action-execute-operator` must use the Agent name after upgrading. Agent-specific metrics use key-value metric groups (e.g., `action.`, `model.`) which are exposed as dimensions/labels in reporters that support them (such as Prometheus). Please refer to [Flink Metric Reporters](https://nightlies.apache.org/flink/flink-docs-release-1.20/docs/deployment/metric_reporters/) for more details.
Additionally, we can check the metric results in the Flink Job WebUI using the metric identifier prefix `.`.
@@ -185,7 +238,17 @@ Each record is a flat JSON object. Framework-owned field names use camelCase, co
Agent Trace persistence is disabled by default. Set `event-log.trace.enabled: true` to add trace context to business Event records and persist Action, LLM, Parser, and Tool lifecycle Events. When Trace persistence is disabled, business Events continue to be logged without the trace fields shown below.
-After fine-grained recovery, a cached durable LLM or Tool result is currently recorded as a new successful execution because cache reuse is not exposed to execution reporting. Distinguishing reused child executions is follow-up work.
+After fine-grained recovery, a cached durable child result is recorded as a new execution because cache reuse is not exposed to execution reporting. Cached LLM results are recorded as successful; cached Tool results retain the success or failure represented by their normalized Tool response. Distinguishing reused child executions is follow-up work.
+
+Tool executions use an optional creation phase because the runtime can identify a call before its callable starts. LLM and Parser executions currently begin directly with a started Event. All lifecycle Events for one Tool call use the same `executionId`.
+
+| Event | Occurrence time | Publication time |
+|-------|-----------------|------------------|
+| `_execution_created_event` | After the call identity and metadata are available, before a preparation failure is reported or an invocable call is submitted for durable execution. | Immediately at that boundary. |
+| `_execution_started_event` | When an invocable Tool enters its callable. | After the durable call or parallel batch returns or raises, while retaining the callable-entry timestamp. |
+| Terminal Event: `_execution_finished_event` or `_execution_failed_event` | When the callable exits, or when the Action observes an outcome without observing a completed invocation at that boundary, such as a preparation failure, durable cache hit, or timeout. | Immediately for a preparation failure; otherwise after the durable call or parallel batch returns or raises. |
+
+Because started and terminal Events can be published after their occurrences, a missing Event only means that its report was not published. In particular, a Tool represented only by a created Event may still be queued, or it may have started or completed before the batch blocked or the task exited. Its invocation state cannot be inferred from the created Event alone. Tool latency is recorded only when matching started and terminal Events are both available, and is calculated from their occurrence timestamps rather than publication times.
Example Trace record:
@@ -216,7 +279,7 @@ is the requested identifier and is not a provider-confirmed model identity.
### Trace Tree Reconstruction
-The `flink-agents-trace-tree` command is installed with the Flink Agents Python wheel. It rebuilds InputEvent-rooted Trace Trees from business Events in a saved File Event Log and ignores execution lifecycle Events. The four lifecycle types `_execution_started_event`, `_execution_finished_event`, `_execution_failed_event`, and `_execution_reused_event` are reserved for the framework. A record is ignored only when its type, status, and execution identity match the corresponding framework lifecycle shape. A business Event that uses a reserved type without that shape is retained and reported with a reconstruction warning. The reader accepts both the current flat record shape and the previous nested `event` shape, including files that contain both formats. Pass either one log file for text output or a log directory for Trace Tree JSON:
+The `flink-agents-trace-tree` command is installed with the Flink Agents Python wheel. It rebuilds InputEvent-rooted Trace Trees from business Events in a saved File Event Log and ignores execution lifecycle Events. The five lifecycle types `_execution_created_event`, `_execution_started_event`, `_execution_finished_event`, `_execution_failed_event`, and `_execution_reused_event` are reserved for the framework. A record is ignored only when its type, status, and execution identity match the corresponding framework lifecycle shape. A business Event that uses a reserved type without that shape is retained and reported with a reconstruction warning. The reader accepts both the current flat record shape and the previous nested `event` shape, including files that contain both formats. Pass either one log file for text output or a log directory for Trace Tree JSON:
```bash
flink-agents-trace-tree /path/to/events-job-task-0.log
@@ -398,6 +461,7 @@ You can override the level for individual event types using the `event-log.type.
| `ToolResponseEvent` | `_tool_response_event` |
| `ContextRetrievalRequestEvent` | `_context_retrieval_request_event` |
| `ContextRetrievalResponseEvent` | `_context_retrieval_response_event` |
+| Execution lifecycle: created | `_execution_created_event` |
| Execution lifecycle: started | `_execution_started_event` |
| Execution lifecycle: finished | `_execution_finished_event` |
| Execution lifecycle: failed | `_execution_failed_event` |
diff --git a/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/TokenMetricsE2ETest.java b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/TokenMetricsE2ETest.java
index 5b0982b29b..7e4006e414 100644
--- a/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/TokenMetricsE2ETest.java
+++ b/e2e-test/flink-agents-end-to-end-tests-integration/src/test/java/org/apache/flink/agents/integration/test/TokenMetricsE2ETest.java
@@ -69,12 +69,16 @@ class TokenMetricsE2ETest {
+ "\"finish_reason\":\"stop\"}],"
+ "\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":5,\"total_tokens\":15}}";
+ private static final String OPERATOR_NAME = TokenMetricsE2EAgent.class.getSimpleName();
+
private static final Pattern PREFIX_PATTERN =
Pattern.compile(
- "^\\.taskmanager\\.([a-f0-9-]+)\\.Flink Streaming Job\\.action-execute-operator\\.0\\.");
+ "^\\.taskmanager\\.([a-f0-9-]+)\\.Flink Streaming Job\\."
+ + Pattern.quote(OPERATOR_NAME)
+ + "\\.0\\.");
private static final String PREFIX_TEMPLATE =
- ".taskmanager.%s.Flink Streaming Job.action-execute-operator.0.";
+ ".taskmanager.%s.Flink Streaming Job." + OPERATOR_NAME + ".0.";
/**
* Expected agent Counter metrics. Each key is the deterministic suffix after the prefix in the
diff --git a/plan/src/main/java/org/apache/flink/agents/plan/actions/ChatModelAction.java b/plan/src/main/java/org/apache/flink/agents/plan/actions/ChatModelAction.java
index 50f62e6d6d..482392e179 100644
--- a/plan/src/main/java/org/apache/flink/agents/plan/actions/ChatModelAction.java
+++ b/plan/src/main/java/org/apache/flink/agents/plan/actions/ChatModelAction.java
@@ -208,15 +208,16 @@ private static Map getRetryStats(MemoryObject sensoryMem, UUID ini
}
private static void recordRetryMetrics(
- RunnerContext ctx, String model, int retryCount, int totalRetryWaitSec) {
+ RunnerContext ctx, String modelResource, int retryCount, int totalRetryWaitSec) {
if (retryCount <= 0) {
return;
}
FlinkAgentsMetricGroup metricGroup = ctx.getActionMetricGroup();
if (metricGroup != null) {
- FlinkAgentsMetricGroup modelGroup = metricGroup.getSubGroup("model", model);
- modelGroup.getCounter("retryCount").inc(retryCount);
- modelGroup.getCounter("retryWaitSec").inc(totalRetryWaitSec);
+ FlinkAgentsMetricGroup modelResourceGroup =
+ metricGroup.getSubGroup("model_resource", modelResource);
+ modelResourceGroup.getCounter("retryCount").inc(retryCount);
+ modelResourceGroup.getCounter("retryWaitSec").inc(totalRetryWaitSec);
}
}
@@ -407,7 +408,7 @@ private static void chat(
recordAttemptRetryStats(
ctx,
initialRequestId,
- result.chatModel,
+ result.model,
result.retryCount,
result.totalRetryWaitSec);
if (selection.isRouter()) {
@@ -476,7 +477,7 @@ private static void chat(
return;
} catch (ChatModelInvoker.ChatAttemptFailed e) {
recordAttemptRetryStats(
- ctx, initialRequestId, e.chatModel, e.retryCount, e.totalRetryWaitSec);
+ ctx, initialRequestId, e.model, e.retryCount, e.totalRetryWaitSec);
// Keep every candidate's failure: chain the previous error into the new one so
// exhaustion surfaces A's and B's errors as suppressed of C's, not just C's.
if (lastError != null && lastError != e.error) {
@@ -520,7 +521,7 @@ private static void chat(
public static void recordAttemptRetryStats(
RunnerContext ctx,
UUID initialRequestId,
- BaseChatModelSetup chatModel,
+ String modelResource,
int retryCount,
int retryWaitSec)
throws Exception {
@@ -528,12 +529,7 @@ public static void recordAttemptRetryStats(
return;
}
accumulateRetryStats(ctx.getSensoryMemory(), initialRequestId, retryCount, retryWaitSec);
- String metricModel = chatModel == null ? null : chatModel.getConnectionName();
- recordRetryMetrics(
- ctx,
- metricModel == null || metricModel.isEmpty() ? "unknown" : metricModel,
- retryCount,
- retryWaitSec);
+ recordRetryMetrics(ctx, modelResource, retryCount, retryWaitSec);
}
/**
diff --git a/plan/src/main/java/org/apache/flink/agents/plan/actions/ToolCallAction.java b/plan/src/main/java/org/apache/flink/agents/plan/actions/ToolCallAction.java
index e9ea957314..724c2ff8fc 100644
--- a/plan/src/main/java/org/apache/flink/agents/plan/actions/ToolCallAction.java
+++ b/plan/src/main/java/org/apache/flink/agents/plan/actions/ToolCallAction.java
@@ -42,12 +42,14 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.CompletionException;
/** Built-in action for processing tool call. */
public class ToolCallAction {
@@ -138,9 +140,8 @@ private static List buildToolCallExecutions(
name,
tool,
metadataParameters);
- ExecutionReporters.started(
+ ExecutionReporters.created(
ctx, ExecutionReporter.EntityTypes.TOOL, name, entityMetadata);
-
if (tool == null || preparationError != null) {
Exception failure =
preparationError != null
@@ -174,6 +175,7 @@ private static List buildToolCallExecutions(
final Tool toolRef = tool;
final Map callArguments = mergedArguments;
+ ToolCallOccurrence occurrence = new ToolCallOccurrence();
DurableCallable callable =
new DurableCallable<>() {
@Override
@@ -188,10 +190,15 @@ public Class getResultClass() {
@Override
public ToolResponse call() throws Exception {
- return toolRef.call(new ToolParameters(callArguments));
+ occurrence.markStarted();
+ try {
+ return toolRef.call(new ToolParameters(callArguments));
+ } finally {
+ occurrence.markFinished();
+ }
}
};
- executions.add(new ToolCallExecution(id, name, callable, entityMetadata));
+ executions.add(new ToolCallExecution(id, name, callable, entityMetadata, occurrence));
}
return executions;
}
@@ -207,10 +214,14 @@ private static void executeParallel(
for (ToolCallExecution execution : executions) {
callables.add(execution.callable);
}
+ List> outcomes = List.of();
+ Instant resultObservedAt = null;
+ Error fatalError = null;
try {
- List> outcomes = ctx.durableExecuteAllAsync(callables);
+ outcomes = ctx.durableExecuteAllAsync(callables);
+ resultObservedAt = Instant.now();
for (int i = 0; i < outcomes.size(); i++) {
- recordOutcome(executions.get(i), outcomes.get(i), ctx, success, error, responses);
+ recordOutcome(executions.get(i), outcomes.get(i), success, error, responses);
}
} catch (InterruptedException e) {
// A cancellation signal, not a batch failure: propagate immediately instead of
@@ -219,20 +230,31 @@ private static void executeParallel(
Thread.currentThread().interrupt();
throw e;
} catch (Exception e) {
+ if (resultObservedAt == null) {
+ resultObservedAt = Instant.now();
+ }
+ if (e instanceof CompletionException && e.getCause() instanceof Error) {
+ fatalError = (Error) e.getCause();
+ throw fatalError;
+ }
for (ToolCallExecution execution : executions) {
recordExecutionException(execution, e, success, error, responses);
}
} catch (Error e) {
- for (ToolCallExecution execution : executions) {
- ExecutionReporters.failed(
- ctx,
- ExecutionReporter.EntityTypes.TOOL,
- execution.name,
- execution.entityMetadata,
- e,
- ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED);
+ if (resultObservedAt == null) {
+ resultObservedAt = Instant.now();
}
+ fatalError = e;
throw e;
+ } finally {
+ for (int i = 0; i < executions.size(); i++) {
+ reportExecution(
+ executions.get(i),
+ ctx,
+ i < outcomes.size() ? outcomes.get(i) : null,
+ fatalError,
+ resultObservedAt);
+ }
}
}
@@ -245,27 +267,17 @@ private static void executeSequentially(
Map responses)
throws InterruptedException {
for (ToolCallExecution execution : executions) {
+ Outcome outcome = null;
+ Instant resultObservedAt = null;
+ Error fatalError = null;
try {
ToolResponse response =
toolCallAsync
? ctx.durableExecuteAsync(execution.callable)
: ctx.durableExecute(execution.callable);
+ resultObservedAt = Instant.now();
+ outcome = Outcome.success(response);
recordToolResponse(execution.id, response, success, error, responses);
- if (response.isError()) {
- ExecutionReporters.failed(
- ctx,
- ExecutionReporter.EntityTypes.TOOL,
- execution.name,
- execution.entityMetadata,
- new RuntimeException(response.getError()),
- ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED);
- } else {
- ExecutionReporters.succeeded(
- ctx,
- ExecutionReporter.EntityTypes.TOOL,
- execution.name,
- execution.entityMetadata);
- }
} catch (InterruptedException e) {
// A cancellation signal, not a tool failure: propagate immediately instead of
// recording it as a tool error and letting the loop move on to (or past) the
@@ -273,23 +285,19 @@ private static void executeSequentially(
Thread.currentThread().interrupt();
throw e;
} catch (Exception e) {
+ if (resultObservedAt == null) {
+ resultObservedAt = Instant.now();
+ }
+ outcome = Outcome.failure(e);
recordExecutionException(execution, e, success, error, responses);
- ExecutionReporters.failed(
- ctx,
- ExecutionReporter.EntityTypes.TOOL,
- execution.name,
- execution.entityMetadata,
- e,
- ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED);
} catch (Error e) {
- ExecutionReporters.failed(
- ctx,
- ExecutionReporter.EntityTypes.TOOL,
- execution.name,
- execution.entityMetadata,
- e,
- ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED);
+ if (resultObservedAt == null) {
+ resultObservedAt = Instant.now();
+ }
+ fatalError = e;
throw e;
+ } finally {
+ reportExecution(execution, ctx, outcome, fatalError, resultObservedAt);
}
}
}
@@ -297,38 +305,70 @@ private static void executeSequentially(
private static void recordOutcome(
ToolCallExecution execution,
Outcome outcome,
- RunnerContext ctx,
Map success,
Map error,
Map responses) {
if (outcome.isFailure()) {
recordExecutionException(execution, outcome.getError(), success, error, responses);
- ExecutionReporters.failed(
+ } else {
+ recordToolResponse(execution.id, outcome.getValue(), success, error, responses);
+ }
+ }
+
+ private static void reportExecution(
+ ToolCallExecution execution,
+ RunnerContext ctx,
+ Outcome outcome,
+ Error fatalError,
+ Instant resultObservedAt) {
+ Instant finishedAt = execution.occurrence.finishedAt;
+ Instant startedAt = execution.occurrence.startedAt;
+ if (startedAt != null && (outcome == null || !startedAt.isAfter(resultObservedAt))) {
+ ExecutionReporters.startedAt(
ctx,
ExecutionReporter.EntityTypes.TOOL,
execution.name,
execution.entityMetadata,
- outcome.getError(),
- ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED);
+ startedAt.toString());
+ }
+ if (outcome == null && fatalError == null) {
+ return;
+ }
+ // A timed-out callable may finish after the Action already received its failure.
+ if (finishedAt == null || finishedAt.isAfter(resultObservedAt)) {
+ finishedAt = resultObservedAt;
+ }
+ Throwable failure =
+ outcome == null
+ ? fatalError
+ : outcome.isFailure()
+ ? outcome.getError()
+ : toolResponseFailure(outcome.getValue());
+
+ if (failure == null) {
+ ExecutionReporters.succeededAt(
+ ctx,
+ ExecutionReporter.EntityTypes.TOOL,
+ execution.name,
+ execution.entityMetadata,
+ finishedAt.toString());
} else {
- ToolResponse response = outcome.getValue();
- recordToolResponse(execution.id, response, success, error, responses);
- if (response.isError()) {
- ExecutionReporters.failed(
- ctx,
- ExecutionReporter.EntityTypes.TOOL,
- execution.name,
- execution.entityMetadata,
- new RuntimeException(response.getError()),
- ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED);
- } else {
- ExecutionReporters.succeeded(
- ctx,
- ExecutionReporter.EntityTypes.TOOL,
- execution.name,
- execution.entityMetadata);
- }
+ ExecutionReporters.failedAt(
+ ctx,
+ ExecutionReporter.EntityTypes.TOOL,
+ execution.name,
+ execution.entityMetadata,
+ failure,
+ ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED,
+ finishedAt.toString());
+ }
+ }
+
+ private static Throwable toolResponseFailure(ToolResponse response) {
+ if (response == null) {
+ return new IllegalStateException("Tool returned a null response.");
}
+ return response.isError() ? new RuntimeException(response.getError()) : null;
}
private static void recordInlineResponse(
@@ -375,16 +415,32 @@ private static final class ToolCallExecution {
private final String name;
private final DurableCallable callable;
private final Map entityMetadata;
+ private final ToolCallOccurrence occurrence;
private ToolCallExecution(
String id,
String name,
DurableCallable callable,
- Map entityMetadata) {
+ Map entityMetadata,
+ ToolCallOccurrence occurrence) {
this.id = id;
this.name = name;
this.callable = callable;
this.entityMetadata = entityMetadata;
+ this.occurrence = occurrence;
+ }
+ }
+
+ private static final class ToolCallOccurrence {
+ private volatile Instant startedAt;
+ private volatile Instant finishedAt;
+
+ private void markStarted() {
+ startedAt = Instant.now();
+ }
+
+ private void markFinished() {
+ finishedAt = Instant.now();
}
}
diff --git a/plan/src/main/java/org/apache/flink/agents/plan/routing/LlmJudgeRoutingExecutor.java b/plan/src/main/java/org/apache/flink/agents/plan/routing/LlmJudgeRoutingExecutor.java
index 3cf4e79fbf..36f5e9435f 100644
--- a/plan/src/main/java/org/apache/flink/agents/plan/routing/LlmJudgeRoutingExecutor.java
+++ b/plan/src/main/java/org/apache/flink/agents/plan/routing/LlmJudgeRoutingExecutor.java
@@ -127,7 +127,7 @@ public RoutingDecision route(
ChatModelAction.recordAttemptRetryStats(
ctx,
context.getRequestId(),
- judgeResult.chatModel,
+ judgeResult.model,
judgeResult.retryCount,
judgeResult.totalRetryWaitSec);
ChatMessage reply = judgeResult.response;
@@ -150,7 +150,7 @@ public RoutingDecision route(
ChatModelAction.recordAttemptRetryStats(
ctx,
context.getRequestId(),
- failure.chatModel,
+ failure.model,
failure.retryCount,
failure.totalRetryWaitSec);
// Cancellation surfacing from inside the judge attempt (the invoker wraps every
diff --git a/plan/src/test/java/org/apache/flink/agents/plan/actions/ChatModelActionRetryTest.java b/plan/src/test/java/org/apache/flink/agents/plan/actions/ChatModelActionRetryTest.java
index 450ef22dc1..8b6046a10f 100644
--- a/plan/src/test/java/org/apache/flink/agents/plan/actions/ChatModelActionRetryTest.java
+++ b/plan/src/test/java/org/apache/flink/agents/plan/actions/ChatModelActionRetryTest.java
@@ -330,8 +330,8 @@ void chatRetriesWithExponentialBackoff() throws Exception {
assertThat(responseEvent.getTotalRetryWaitSec()).isEqualTo(1);
assertThat(elapsed).isGreaterThanOrEqualTo(1000L);
- // Verify metrics recorded under connection name
- verify(mockActionMetricGroup).getSubGroup("model", mockChatModel.getConnectionName());
+ // Retry health belongs to the ChatModel resource, not the provider connection or model.
+ verify(mockActionMetricGroup).getSubGroup("model_resource", "test-model");
verify(mockRetryCountCounter).inc(1);
verify(mockRetryWaitSecCounter).inc(1);
}
@@ -358,6 +358,9 @@ void chatExhaustsRetriesAndThrows() {
.hasMessage("persistent error");
assertThat(sentEvents).isEmpty();
+ verify(mockActionMetricGroup).getSubGroup("model_resource", "test-model");
+ verify(mockRetryCountCounter).inc(2);
+ verify(mockRetryWaitSecCounter).inc(0);
}
@Test
diff --git a/plan/src/test/java/org/apache/flink/agents/plan/actions/ChatModelActionRoutingTest.java b/plan/src/test/java/org/apache/flink/agents/plan/actions/ChatModelActionRoutingTest.java
index ce74ead9a3..a42b5775b2 100644
--- a/plan/src/test/java/org/apache/flink/agents/plan/actions/ChatModelActionRoutingTest.java
+++ b/plan/src/test/java/org/apache/flink/agents/plan/actions/ChatModelActionRoutingTest.java
@@ -47,6 +47,8 @@
import org.apache.flink.agents.api.resource.ResourceType;
import org.apache.flink.agents.api.tools.ToolResponse;
import org.apache.flink.agents.plan.AgentConfiguration;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Histogram;
import org.junit.jupiter.api.Test;
import java.util.ArrayDeque;
@@ -61,6 +63,9 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
/** Integration tests for model routing inside {@link ChatModelAction}. */
public class ChatModelActionRoutingTest {
@@ -118,6 +123,7 @@ static class FakeRunnerContext implements RunnerContext {
private final ModelRouter router;
private final MemoryObject sensoryMemory = new FakeMemoryObject(new HashMap<>());
private final AgentConfiguration config = new AgentConfiguration(Map.of());
+ private FlinkAgentsMetricGroup actionMetricGroup;
FakeRunnerContext(ModelRouter router) {
this.router = router;
@@ -145,6 +151,11 @@ FakeRunnerContext withRetryBudget(int maxRetries, int waitIntervalSec) {
return this;
}
+ FakeRunnerContext withActionMetricGroup(FlinkAgentsMetricGroup actionMetricGroup) {
+ this.actionMetricGroup = actionMetricGroup;
+ return this;
+ }
+
@Override
public boolean hasResource(String name, ResourceType type) {
return type == ResourceType.MODEL_ROUTER && "router".equals(name) && router != null;
@@ -192,7 +203,7 @@ public FlinkAgentsMetricGroup getAgentMetricGroup() {
@Override
public FlinkAgentsMetricGroup getActionMetricGroup() {
- return null;
+ return actionMetricGroup;
}
@Override
@@ -515,6 +526,15 @@ void llmJudgeVerdictOutsideCandidatesAbstainsToDefault() throws Exception {
void llmJudgeUsesTheRetryBudget() throws Exception {
// v1 review lesson: every routing test ran with numRetries == 0, so the retry path was
// never exercised. Judge fails once, retries, then delivers a verdict.
+ FlinkAgentsMetricGroup actionMetricGroup = mock(FlinkAgentsMetricGroup.class);
+ FlinkAgentsMetricGroup judgeMetricGroup = mock(FlinkAgentsMetricGroup.class);
+ Counter retryCount = mock(Counter.class);
+ Counter retryWaitSec = mock(Counter.class);
+ when(actionMetricGroup.getHistogram("routingDecisionLatencyMs"))
+ .thenReturn(mock(Histogram.class));
+ when(actionMetricGroup.getSubGroup("model_resource", "judge")).thenReturn(judgeMetricGroup);
+ when(judgeMetricGroup.getCounter("retryCount")).thenReturn(retryCount);
+ when(judgeMetricGroup.getCounter("retryWaitSec")).thenReturn(retryWaitSec);
ModelRouter router =
new ModelRouter(
ModelRouter.of("small", "big")
@@ -526,6 +546,7 @@ void llmJudgeUsesTheRetryBudget() throws Exception {
new FakeRunnerContext(router)
.withErrorHandling(Agent.ErrorHandlingStrategy.RETRY)
.withRetryBudget(1, 0)
+ .withActionMetricGroup(actionMetricGroup)
.register(
"judge",
new FakeChatModel(
@@ -541,6 +562,9 @@ void llmJudgeUsesTheRetryBudget() throws Exception {
ModelRoutingEvent event = ctx.routingEvent();
assertThat(event.getSelectedModel()).isEqualTo("big");
assertThat(event.getDecisionSource()).isEqualTo(ModelRoutingEvent.SOURCE_LLM_JUDGE);
+ verify(actionMetricGroup).getSubGroup("model_resource", "judge");
+ verify(retryCount).inc(1);
+ verify(retryWaitSec).inc(0);
}
@Test
@@ -631,6 +655,17 @@ void routedRequestUsesRoutedDurableCallIds() throws Exception {
@Test
void retryBudgetRunsBeforeFallback() throws Exception {
+ FlinkAgentsMetricGroup actionMetricGroup = mock(FlinkAgentsMetricGroup.class);
+ FlinkAgentsMetricGroup modelResourceMetricGroup = mock(FlinkAgentsMetricGroup.class);
+ Counter retryCount = mock(Counter.class);
+ Counter retryWaitSec = mock(Counter.class);
+ when(actionMetricGroup.getHistogram("routingDecisionLatencyMs"))
+ .thenReturn(mock(Histogram.class));
+ when(actionMetricGroup.getSubGroup("model_resource", "big"))
+ .thenReturn(modelResourceMetricGroup);
+ when(modelResourceMetricGroup.getCounter("retryCount")).thenReturn(retryCount);
+ when(modelResourceMetricGroup.getCounter("retryWaitSec")).thenReturn(retryWaitSec);
+
ModelRouter router =
new ModelRouter(
ModelRouter.of("small", "big")
@@ -643,6 +678,7 @@ void retryBudgetRunsBeforeFallback() throws Exception {
new FakeRunnerContext(router)
.withErrorHandling(Agent.ErrorHandlingStrategy.RETRY)
.withRetryBudget(1, 0)
+ .withActionMetricGroup(actionMetricGroup)
.register(
"big",
new FakeChatModel(
@@ -659,6 +695,9 @@ void retryBudgetRunsBeforeFallback() throws Exception {
assertThat(ctx.chatResponse().getResponse().getContent()).isEqualTo("recovered on retry");
assertThat(ctx.resolvedChatModels).containsExactly("big");
assertThat(ctx.routingEventCount()).isEqualTo(1L);
+ verify(actionMetricGroup).getSubGroup("model_resource", "big");
+ verify(retryCount).inc(1);
+ verify(retryWaitSec).inc(0);
}
@Test
diff --git a/plan/src/test/java/org/apache/flink/agents/plan/actions/ToolCallActionReportTest.java b/plan/src/test/java/org/apache/flink/agents/plan/actions/ToolCallActionReportTest.java
index 08348327b4..af2d56aaf6 100644
--- a/plan/src/test/java/org/apache/flink/agents/plan/actions/ToolCallActionReportTest.java
+++ b/plan/src/test/java/org/apache/flink/agents/plan/actions/ToolCallActionReportTest.java
@@ -20,6 +20,7 @@
import org.apache.flink.agents.api.Event;
import org.apache.flink.agents.api.agents.AgentExecutionOptions;
import org.apache.flink.agents.api.context.DurableCallable;
+import org.apache.flink.agents.api.context.Outcome;
import org.apache.flink.agents.api.context.RunnerContext;
import org.apache.flink.agents.api.event.ToolRequestEvent;
import org.apache.flink.agents.api.event.ToolResponseEvent;
@@ -33,18 +34,39 @@
import org.apache.flink.agents.api.trace.ExecutionReporter;
import org.apache.flink.agents.api.trace.ToolExecutionMetadataKeys;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.ArgumentCaptor;
+import org.mockito.MockedStatic;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.withSettings;
@@ -87,9 +109,23 @@ void processToolRequestReportsEachToolCall() throws Exception {
metadata.put(ToolExecutionMetadataKeys.MCP_SERVER, "search-server");
ExecutionReporter reporter = (ExecutionReporter) ctx;
verify(reporter)
- .reportExecutionStarted(ExecutionReporter.EntityTypes.TOOL, "search", metadata);
+ .reportExecutionCreated(ExecutionReporter.EntityTypes.TOOL, "search", metadata);
+ ArgumentCaptor startedAt = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor finishedAt = ArgumentCaptor.forClass(String.class);
verify(reporter)
- .reportExecutionSucceeded(ExecutionReporter.EntityTypes.TOOL, "search", metadata);
+ .reportExecutionStartedAt(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("search"),
+ eq(metadata),
+ startedAt.capture());
+ verify(reporter)
+ .reportExecutionSucceededAt(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("search"),
+ eq(metadata),
+ finishedAt.capture());
+ assertThat(Instant.parse(finishedAt.getValue()))
+ .isAfterOrEqualTo(Instant.parse(startedAt.getValue()));
assertThat(sentEvents).hasSize(1);
assertThat(sentEvents.get(0)).isInstanceOf(ToolResponseEvent.class);
@@ -123,20 +159,700 @@ void processToolRequestMarksErrorResponseAsFailed() throws Exception {
metadata.put(ToolExecutionMetadataKeys.TOOL_CALL_ID, "call-1");
ExecutionReporter reporter = (ExecutionReporter) ctx;
verify(reporter)
- .reportExecutionFailed(
+ .reportExecutionFailedAt(
eq(ExecutionReporter.EntityTypes.TOOL),
eq("search"),
eq(metadata),
any(Throwable.class),
- eq(ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED));
+ eq(ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED),
+ anyString());
verify(reporter, never())
- .reportExecutionSucceeded(ExecutionReporter.EntityTypes.TOOL, "search", metadata);
+ .reportExecutionSucceededAt(anyString(), anyString(), anyMap(), anyString());
ToolResponseEvent responseEvent = (ToolResponseEvent) sentEvents.get(0);
assertThat(responseEvent.getSuccess()).containsEntry("call-1", false);
assertThat(responseEvent.getError()).containsEntry("call-1", "tool rejected request");
}
+ @Test
+ void sequentialToolErrorReportsFailureAndRethrows() throws Exception {
+ RunnerContext ctx =
+ mock(RunnerContext.class, withSettings().extraInterfaces(ExecutionReporter.class));
+ Tool tool = mock(Tool.class);
+ AssertionError failure = new AssertionError("tool died");
+ when(tool.call(any())).thenThrow(failure);
+ when(ctx.getResource("search", ResourceType.TOOL)).thenReturn(tool);
+ when(ctx.getConfig()).thenReturn(toolCallConfig());
+ when(ctx.durableExecute(any()))
+ .thenAnswer(inv -> inv.>getArgument(0).call());
+
+ assertThatThrownBy(
+ () ->
+ ToolCallAction.processToolRequest(
+ new ToolRequestEvent(
+ "test-model", List.of(toolCall("call-1"))),
+ ctx))
+ .isSameAs(failure);
+
+ assertReports(ctx, List.of("call-1"), List.of(), List.of("call-1"));
+ verify((ExecutionReporter) ctx)
+ .reportExecutionFailedAt(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("search"),
+ anyMap(),
+ eq(failure),
+ eq(ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED),
+ anyString());
+ verify(ctx, never()).sendEvent(any());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void parallelToolErrorReportsUnresolvedCallsAndRethrows(boolean wrappedByFuture)
+ throws Exception {
+ Tool tool = mock(Tool.class);
+ AssertionError failure = new AssertionError("tool died");
+ when(tool.call(any())).thenThrow(failure);
+ RunnerContext ctx = parallelContext(tool);
+ if (wrappedByFuture) {
+ doAnswer(
+ invocation -> {
+ List> callables =
+ invocation.getArgument(0);
+ CompletableFuture> first =
+ CompletableFuture.supplyAsync(
+ () -> {
+ try {
+ return Outcome.success(
+ callables.get(0).call());
+ } catch (Exception e) {
+ return Outcome.failure(e);
+ }
+ });
+ return List.of(first.join());
+ })
+ .when(ctx)
+ .durableExecuteAllAsync(any());
+ }
+
+ assertThatThrownBy(() -> ToolCallAction.processToolRequest(parallelRequest(), ctx))
+ .isSameAs(failure);
+
+ assertReports(ctx, List.of("call-1"), List.of(), List.of("call-1", "call-2", "call-3"));
+ verify((ExecutionReporter) ctx, times(3))
+ .reportExecutionFailedAt(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("search"),
+ anyMap(),
+ eq(failure),
+ eq(ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED),
+ anyString());
+ verify(ctx, never()).sendEvent(any());
+ }
+
+ @Test
+ void missingToolReportsCreationAndFailureWithoutStart() throws Exception {
+ RunnerContext ctx =
+ mock(RunnerContext.class, withSettings().extraInterfaces(ExecutionReporter.class));
+ when(ctx.getResource("missing", ResourceType.TOOL))
+ .thenThrow(new IllegalArgumentException("Tool does not exist."));
+ when(ctx.getConfig()).thenReturn(toolCallConfig());
+ Map function = new LinkedHashMap<>();
+ function.put("name", "missing");
+ function.put("arguments", Map.of());
+ Map toolCall = new LinkedHashMap<>();
+ toolCall.put("id", "call-1");
+ toolCall.put("function", function);
+
+ ToolCallAction.processToolRequest(
+ new ToolRequestEvent("test-model", List.of(toolCall)), ctx);
+
+ ExecutionReporter reporter = (ExecutionReporter) ctx;
+ ArgumentCaptor> createdMetadata = ArgumentCaptor.forClass(Map.class);
+ verify(reporter)
+ .reportExecutionCreated(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("missing"),
+ createdMetadata.capture());
+ verify(reporter)
+ .reportExecutionFailed(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("missing"),
+ eq(createdMetadata.getValue()),
+ any(Throwable.class),
+ eq(ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED));
+ verify(reporter, never())
+ .reportExecutionStartedAt(anyString(), anyString(), anyMap(), anyString());
+ }
+
+ @Test
+ void parallelToolCallsReportIndependentOutcomes() throws Exception {
+ RunnerContext ctx =
+ mock(RunnerContext.class, withSettings().extraInterfaces(ExecutionReporter.class));
+ List sentEvents = new ArrayList<>();
+ Tool tool = mock(Tool.class);
+ when(tool.call(any()))
+ .thenAnswer(
+ invocation -> {
+ String query =
+ invocation
+ .getArgument(0)
+ .getParameter("query", String.class);
+ if ("call-2".equals(query)) {
+ throw new IllegalStateException("call-2 failed");
+ }
+ if ("call-3".equals(query)) {
+ return ToolResponse.error("call-3 rejected");
+ }
+ return ToolResponse.success("ok");
+ });
+ when(ctx.getResource("search", ResourceType.TOOL)).thenReturn(tool);
+ when(ctx.getConfig()).thenReturn(toolCallConfig(true, 3));
+ when(ctx.durableExecuteAllAsync(any()))
+ .thenAnswer(
+ invocation -> {
+ List> callables =
+ invocation.getArgument(0);
+ List> outcomes = new ArrayList<>();
+ for (DurableCallable callable : callables) {
+ try {
+ outcomes.add(Outcome.success(callable.call()));
+ } catch (Exception e) {
+ outcomes.add(Outcome.failure(e));
+ }
+ }
+ return outcomes;
+ });
+ doAnswer(inv -> sentEvents.add(inv.getArgument(0))).when(ctx).sendEvent(any());
+
+ ToolCallAction.processToolRequest(
+ new ToolRequestEvent(
+ "test-model",
+ List.of(toolCall("call-1"), toolCall("call-2"), toolCall("call-3"))),
+ ctx);
+
+ ExecutionReporter reporter = (ExecutionReporter) ctx;
+ verify(reporter, times(3))
+ .reportExecutionCreated(
+ eq(ExecutionReporter.EntityTypes.TOOL), eq("search"), anyMap());
+ verify(reporter, times(3))
+ .reportExecutionStartedAt(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("search"),
+ anyMap(),
+ anyString());
+ verify(reporter)
+ .reportExecutionSucceededAt(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("search"),
+ anyMap(),
+ anyString());
+ verify(reporter, times(2))
+ .reportExecutionFailedAt(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("search"),
+ anyMap(),
+ any(Throwable.class),
+ eq(ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED),
+ anyString());
+
+ ToolResponseEvent response = (ToolResponseEvent) sentEvents.get(0);
+ assertThat(response.getSuccess())
+ .containsEntry("call-1", true)
+ .containsEntry("call-2", false)
+ .containsEntry("call-3", false);
+ assertThat(response.getError())
+ .containsEntry("call-2", "call-2 failed")
+ .containsEntry("call-3", "call-3 rejected");
+ }
+
+ @Test
+ void parallelToolCallsReportTheirOwnCompletionTimestamps() throws Exception {
+ Instant base = Instant.parse("2026-01-01T00:00:00Z");
+ Instant firstFinishedAt = base.plusMillis(20);
+ Instant secondFinishedAt = base.plusMillis(150);
+ AtomicReference now = new AtomicReference<>(base);
+ Tool tool = mock(Tool.class);
+ when(tool.call(any()))
+ .thenAnswer(
+ invocation -> {
+ String callId =
+ invocation
+ .getArgument(0)
+ .getParameter("query", String.class);
+ now.set("call-1".equals(callId) ? firstFinishedAt : secondFinishedAt);
+ return ToolResponse.success("ok");
+ });
+ RunnerContext ctx = parallelContext(tool);
+ when(ctx.getConfig()).thenReturn(toolCallConfig(true, 2));
+ doAnswer(
+ invocation -> {
+ List> callables =
+ invocation.getArgument(0);
+ now.set(base);
+ ToolResponse first = callables.get(0).call();
+ now.set(base.plusMillis(100));
+ ToolResponse second = callables.get(1).call();
+ return List.of(Outcome.success(first), Outcome.success(second));
+ })
+ .when(ctx)
+ .durableExecuteAllAsync(any());
+
+ try (MockedStatic clock = mockStatic(Instant.class)) {
+ clock.when(Instant::now).thenAnswer(invocation -> now.get());
+ ToolCallAction.processToolRequest(
+ new ToolRequestEvent(
+ "test-model", List.of(toolCall("call-1"), toolCall("call-2"))),
+ ctx);
+ }
+
+ ArgumentCaptor> metadata = ArgumentCaptor.forClass(Map.class);
+ ArgumentCaptor timestamp = ArgumentCaptor.forClass(String.class);
+ verify((ExecutionReporter) ctx, times(2))
+ .reportExecutionSucceededAt(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("search"),
+ metadata.capture(),
+ timestamp.capture());
+ Map terminalTimestamps = new LinkedHashMap<>();
+ for (int i = 0; i < metadata.getAllValues().size(); i++) {
+ terminalTimestamps.put(
+ String.valueOf(
+ metadata.getAllValues()
+ .get(i)
+ .get(ToolExecutionMetadataKeys.TOOL_CALL_ID)),
+ timestamp.getAllValues().get(i));
+ }
+ assertThat(terminalTimestamps)
+ .containsEntry("call-1", firstFinishedAt.toString())
+ .containsEntry("call-2", secondFinishedAt.toString());
+ }
+
+ @Test
+ void responseProcessingFailureDoesNotRepeatCompletedOccurrences() throws Exception {
+ Tool tool = mock(Tool.class);
+ when(tool.call(any()))
+ .thenAnswer(
+ invocation ->
+ "call-2"
+ .equals(
+ invocation
+ .getArgument(0)
+ .getParameter(
+ "query", String.class))
+ ? null
+ : ToolResponse.success("ok"));
+ RunnerContext ctx = parallelContext(tool);
+
+ ToolCallAction.processToolRequest(parallelRequest(), ctx);
+
+ assertReports(
+ ctx,
+ List.of("call-1", "call-2", "call-3"),
+ List.of("call-1", "call-3"),
+ List.of("call-2"));
+ assertBusinessResponsesFailed(ctx);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void durableFailureIsReportedAsToolFailure(boolean async) throws Exception {
+ IllegalStateException failure = new IllegalStateException("persist failed");
+ Tool tool = mock(Tool.class);
+ when(tool.call(any())).thenReturn(ToolResponse.success("ok"));
+ RunnerContext ctx = parallelContext(tool);
+ when(ctx.getConfig()).thenReturn(toolCallConfig(async, 1));
+ when(ctx.durableExecute(any()))
+ .thenAnswer(
+ invocation -> {
+ invocation.>getArgument(0).call();
+ throw failure;
+ });
+ when(ctx.durableExecuteAsync(any()))
+ .thenAnswer(
+ invocation -> {
+ invocation.>getArgument(0).call();
+ throw failure;
+ });
+
+ ToolCallAction.processToolRequest(parallelRequest(), ctx);
+
+ assertReports(
+ ctx,
+ List.of("call-1", "call-2", "call-3"),
+ List.of(),
+ List.of("call-1", "call-2", "call-3"));
+ verify((ExecutionReporter) ctx, times(3))
+ .reportExecutionFailedAt(
+ anyString(), anyString(), anyMap(), eq(failure), anyString(), anyString());
+ assertBusinessResponsesFailed(ctx);
+ }
+
+ @Test
+ void parallelDurableFailureIsReportedForItsToolCall() throws Exception {
+ IllegalStateException failure = new IllegalStateException("persist failed");
+ Tool tool = mock(Tool.class);
+ when(tool.call(any())).thenReturn(ToolResponse.success("ok"));
+ RunnerContext ctx = parallelContext(tool);
+ doAnswer(
+ invocation -> {
+ List> callables =
+ invocation.getArgument(0);
+ List> outcomes = new ArrayList<>();
+ for (DurableCallable callable : callables) {
+ outcomes.add(Outcome.success(callable.call()));
+ }
+ outcomes.set(1, Outcome.failure(failure));
+ return outcomes;
+ })
+ .when(ctx)
+ .durableExecuteAllAsync(any());
+
+ ToolCallAction.processToolRequest(parallelRequest(), ctx);
+
+ assertReports(
+ ctx,
+ List.of("call-1", "call-2", "call-3"),
+ List.of("call-1", "call-3"),
+ List.of("call-2"));
+ verify((ExecutionReporter) ctx)
+ .reportExecutionFailedAt(
+ anyString(), anyString(), anyMap(), eq(failure), anyString(), anyString());
+ ArgumentCaptor event = ArgumentCaptor.forClass(Event.class);
+ verify(ctx).sendEvent(event.capture());
+ assertThat(((ToolResponseEvent) event.getValue()).getSuccess())
+ .containsEntry("call-1", true)
+ .containsEntry("call-2", false)
+ .containsEntry("call-3", true);
+ }
+
+ @Test
+ void parallelBatchFailureBeforeInvocationRetainsCreatedExecutions() throws Exception {
+ Tool tool = mock(Tool.class);
+ RunnerContext ctx = parallelContext(tool);
+ doThrow(new IllegalStateException("batch failed before invocation"))
+ .when(ctx)
+ .durableExecuteAllAsync(any());
+
+ ToolCallAction.processToolRequest(parallelRequest(), ctx);
+
+ ArgumentCaptor> metadata = ArgumentCaptor.forClass(Map.class);
+ verify((ExecutionReporter) ctx, times(3))
+ .reportExecutionCreated(
+ eq(ExecutionReporter.EntityTypes.TOOL), eq("search"), metadata.capture());
+ assertThat(metadata.getAllValues())
+ .extracting(value -> value.get(ToolExecutionMetadataKeys.TOOL_CALL_ID))
+ .containsExactlyInAnyOrder("call-1", "call-2", "call-3");
+ verify((ExecutionReporter) ctx, never())
+ .reportExecutionStartedAt(anyString(), anyString(), anyMap(), anyString());
+ verify((ExecutionReporter) ctx, never())
+ .reportExecutionSucceededAt(anyString(), anyString(), anyMap(), anyString());
+ verify((ExecutionReporter) ctx, never())
+ .reportExecutionFailedAt(
+ anyString(),
+ anyString(),
+ anyMap(),
+ any(Throwable.class),
+ anyString(),
+ anyString());
+ verify(tool, never()).call(any());
+ }
+
+ @Test
+ void timeoutIsReportedAsFailureWithoutRepeatingOnLateCompletion() throws Exception {
+ TimeoutException failure = new TimeoutException("request timed out");
+ CountDownLatch started = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ ExecutorService worker = Executors.newSingleThreadExecutor();
+ AtomicReference> pending = new AtomicReference<>();
+ AtomicReference reportingStartedAt = new AtomicReference<>();
+ Tool tool = mock(Tool.class);
+ when(tool.call(any()))
+ .thenAnswer(
+ invocation -> {
+ started.countDown();
+ assertThat(release.await(5, TimeUnit.SECONDS)).isTrue();
+ return ToolResponse.success("ok");
+ });
+ RunnerContext ctx = parallelContext(tool);
+ when(ctx.getConfig()).thenReturn(toolCallConfig(true, 1));
+ when(ctx.durableExecuteAsync(any()))
+ .thenAnswer(
+ invocation -> {
+ DurableCallable callable = invocation.getArgument(0);
+ pending.set(worker.submit(callable::call));
+ assertThat(started.await(5, TimeUnit.SECONDS)).isTrue();
+ throw failure;
+ });
+ doAnswer(
+ invocation -> {
+ reportingStartedAt.set(Instant.now());
+ return null;
+ })
+ .when((ExecutionReporter) ctx)
+ .reportExecutionStartedAt(anyString(), anyString(), anyMap(), anyString());
+
+ try {
+ ToolCallAction.processToolRequest(
+ new ToolRequestEvent("test-model", List.of(toolCall("call-1"))), ctx);
+ assertReports(ctx, List.of("call-1"), List.of(), List.of("call-1"));
+ ArgumentCaptor finishedAt = ArgumentCaptor.forClass(String.class);
+ verify((ExecutionReporter) ctx)
+ .reportExecutionFailedAt(
+ anyString(),
+ anyString(),
+ anyMap(),
+ eq(failure),
+ anyString(),
+ finishedAt.capture());
+ assertThat(Instant.parse(finishedAt.getValue()))
+ .isBeforeOrEqualTo(reportingStartedAt.get());
+ assertBusinessResponsesFailed(ctx);
+ } finally {
+ release.countDown();
+ worker.shutdown();
+ assertThat(worker.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
+ }
+ assertThat(pending.get().get().isSuccess()).isTrue();
+ assertReports(ctx, List.of("call-1"), List.of(), List.of("call-1"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void parallelTimeoutTimestampPrecedesResponseProcessingAndReporting(
+ boolean completeDuringReporting) throws Exception {
+ TimeoutException failure = new TimeoutException("batch timed out");
+ CountDownLatch started = new CountDownLatch(2);
+ CountDownLatch release = new CountDownLatch(1);
+ ExecutorService workers = Executors.newFixedThreadPool(2);
+ List> pending = new ArrayList<>();
+ AtomicReference responseProcessingStartedAt = new AtomicReference<>();
+ ToolResponse firstResponse = mock(ToolResponse.class);
+ when(firstResponse.isSuccess())
+ .thenAnswer(
+ invocation -> {
+ responseProcessingStartedAt.compareAndSet(null, Instant.now());
+ return true;
+ });
+ Tool tool = mock(Tool.class);
+ when(tool.call(any()))
+ .thenAnswer(
+ invocation -> {
+ ToolParameters parameters = invocation.getArgument(0);
+ if ("call-1".equals(parameters.getParameter("query"))) {
+ return firstResponse;
+ }
+ started.countDown();
+ assertThat(release.await(5, TimeUnit.SECONDS)).isTrue();
+ return ToolResponse.success("late result");
+ });
+ RunnerContext ctx = parallelContext(tool);
+ doAnswer(
+ invocation -> {
+ List> callables =
+ invocation.getArgument(0);
+ ToolResponse response = callables.get(0).call();
+ pending.add(workers.submit(callables.get(1)::call));
+ pending.add(workers.submit(callables.get(2)::call));
+ assertThat(started.await(5, TimeUnit.SECONDS)).isTrue();
+ return List.of(
+ Outcome.success(response),
+ Outcome.failure(failure),
+ Outcome.failure(failure));
+ })
+ .when(ctx)
+ .durableExecuteAllAsync(any());
+ doAnswer(
+ invocation -> {
+ if (completeDuringReporting) {
+ release.countDown();
+ for (Future future : pending) {
+ future.get(5, TimeUnit.SECONDS);
+ }
+ }
+ return null;
+ })
+ .when((ExecutionReporter) ctx)
+ .reportExecutionStartedAt(anyString(), anyString(), anyMap(), anyString());
+
+ try {
+ ToolCallAction.processToolRequest(parallelRequest(), ctx);
+ ArgumentCaptor finishedAt = ArgumentCaptor.forClass(String.class);
+ verify((ExecutionReporter) ctx, times(2))
+ .reportExecutionFailedAt(
+ anyString(),
+ anyString(),
+ anyMap(),
+ eq(failure),
+ anyString(),
+ finishedAt.capture());
+ assertThat(finishedAt.getAllValues().get(0))
+ .isEqualTo(finishedAt.getAllValues().get(1));
+ assertThat(Instant.parse(finishedAt.getValue()))
+ .isBeforeOrEqualTo(responseProcessingStartedAt.get());
+ } finally {
+ release.countDown();
+ workers.shutdown();
+ assertThat(workers.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
+ }
+ assertReports(
+ ctx,
+ List.of("call-1", "call-2", "call-3"),
+ List.of("call-1"),
+ List.of("call-2", "call-3"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void parallelTimeoutOmitsStartsAfterResultObservation(boolean startsAfterObservation)
+ throws Exception {
+ TimeoutException failure = new TimeoutException("batch timed out");
+ Instant base = Instant.parse("2026-01-01T00:00:00Z");
+ Instant observedAt = base.plusSeconds(1);
+ Instant delayedStart = startsAfterObservation ? observedAt.plusSeconds(1) : observedAt;
+ AtomicReference now = new AtomicReference<>(base);
+ List> delayed = new ArrayList<>();
+ Tool tool = mock(Tool.class);
+ when(tool.call(any())).thenReturn(ToolResponse.success("ok"));
+ RunnerContext ctx = parallelContext(tool);
+ doAnswer(
+ invocation -> {
+ List> callables =
+ invocation.getArgument(0);
+ ToolResponse first = callables.get(0).call();
+ delayed.addAll(callables.subList(1, callables.size()));
+ now.set(observedAt);
+ return List.of(
+ Outcome.success(first),
+ Outcome.failure(failure),
+ Outcome.failure(failure));
+ })
+ .when(ctx)
+ .durableExecuteAllAsync(any());
+ doAnswer(
+ invocation -> {
+ Map metadata = invocation.getArgument(2);
+ if ("call-1"
+ .equals(metadata.get(ToolExecutionMetadataKeys.TOOL_CALL_ID))) {
+ // The delayed calls enter after the Action has observed the
+ // timeout.
+ now.set(delayedStart);
+ for (DurableCallable callable : delayed) {
+ callable.call();
+ }
+ }
+ return null;
+ })
+ .when((ExecutionReporter) ctx)
+ .reportExecutionStartedAt(anyString(), anyString(), anyMap(), anyString());
+
+ try (MockedStatic clock = mockStatic(Instant.class)) {
+ clock.when(Instant::now).thenAnswer(invocation -> now.get());
+ ToolCallAction.processToolRequest(parallelRequest(), ctx);
+ }
+
+ assertReports(
+ ctx,
+ startsAfterObservation ? List.of("call-1") : List.of("call-1", "call-2", "call-3"),
+ List.of("call-1"),
+ List.of("call-2", "call-3"));
+ verify(tool, times(3)).call(any());
+ ArgumentCaptor finishedAt = ArgumentCaptor.forClass(String.class);
+ verify((ExecutionReporter) ctx, times(2))
+ .reportExecutionFailedAt(
+ anyString(),
+ anyString(),
+ anyMap(),
+ eq(failure),
+ anyString(),
+ finishedAt.capture());
+ assertThat(finishedAt.getAllValues())
+ .containsExactly(observedAt.toString(), observedAt.toString());
+ ArgumentCaptor response = ArgumentCaptor.forClass(Event.class);
+ verify(ctx).sendEvent(response.capture());
+ assertThat(((ToolResponseEvent) response.getValue()).getSuccess())
+ .containsExactlyInAnyOrderEntriesOf(
+ Map.of("call-1", true, "call-2", false, "call-3", false));
+ }
+
+ private static RunnerContext parallelContext(Tool tool) throws Exception {
+ RunnerContext ctx =
+ mock(RunnerContext.class, withSettings().extraInterfaces(ExecutionReporter.class));
+ when(ctx.getResource("search", ResourceType.TOOL)).thenReturn(tool);
+ when(ctx.getConfig()).thenReturn(toolCallConfig(true, 3));
+ when(ctx.durableExecuteAllAsync(any()))
+ .thenAnswer(
+ invocation -> {
+ List> callables =
+ invocation.getArgument(0);
+ List> outcomes = new ArrayList<>();
+ for (DurableCallable callable : callables) {
+ try {
+ outcomes.add(Outcome.success(callable.call()));
+ } catch (Exception e) {
+ outcomes.add(Outcome.failure(e));
+ }
+ }
+ return outcomes;
+ });
+ return ctx;
+ }
+
+ private static ToolRequestEvent parallelRequest() {
+ return new ToolRequestEvent(
+ "test-model", List.of(toolCall("call-1"), toolCall("call-2"), toolCall("call-3")));
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private static void assertReports(
+ RunnerContext ctx, List started, List succeeded, List failed)
+ throws Exception {
+ ExecutionReporter reporter = (ExecutionReporter) ctx;
+ Set expectedCreated = new LinkedHashSet<>();
+ expectedCreated.addAll(started);
+ expectedCreated.addAll(succeeded);
+ expectedCreated.addAll(failed);
+ ArgumentCaptor creations = ArgumentCaptor.forClass(Map.class);
+ ArgumentCaptor starts = ArgumentCaptor.forClass(Map.class);
+ ArgumentCaptor successes = ArgumentCaptor.forClass(Map.class);
+ ArgumentCaptor failures = ArgumentCaptor.forClass(Map.class);
+ verify(reporter, times(expectedCreated.size()))
+ .reportExecutionCreated(anyString(), anyString(), creations.capture());
+ verify(reporter, times(started.size()))
+ .reportExecutionStartedAt(anyString(), anyString(), starts.capture(), anyString());
+ verify(reporter, times(succeeded.size()))
+ .reportExecutionSucceededAt(
+ anyString(), anyString(), successes.capture(), anyString());
+ verify(reporter, times(failed.size()))
+ .reportExecutionFailedAt(
+ anyString(),
+ anyString(),
+ failures.capture(),
+ any(Throwable.class),
+ anyString(),
+ anyString());
+ assertThat(creations.getAllValues())
+ .extracting(m -> m.get(ToolExecutionMetadataKeys.TOOL_CALL_ID))
+ .containsExactlyInAnyOrderElementsOf(expectedCreated);
+ assertThat(starts.getAllValues())
+ .extracting(m -> m.get(ToolExecutionMetadataKeys.TOOL_CALL_ID))
+ .containsExactlyInAnyOrderElementsOf(started);
+ assertThat(successes.getAllValues())
+ .extracting(m -> m.get(ToolExecutionMetadataKeys.TOOL_CALL_ID))
+ .containsExactlyInAnyOrderElementsOf(succeeded);
+ assertThat(failures.getAllValues())
+ .extracting(m -> m.get(ToolExecutionMetadataKeys.TOOL_CALL_ID))
+ .containsExactlyInAnyOrderElementsOf(failed);
+ }
+
+ private static void assertBusinessResponsesFailed(RunnerContext ctx) {
+ ArgumentCaptor event = ArgumentCaptor.forClass(Event.class);
+ verify(ctx).sendEvent(event.capture());
+ assertThat(((ToolResponseEvent) event.getValue()).getSuccess().values())
+ .isNotEmpty()
+ .containsOnly(false);
+ }
+
@Test
void processLoadSkillToolRequestAddsSkillMetadata() throws Exception {
RunnerContext ctx =
@@ -148,7 +864,9 @@ void processLoadSkillToolRequestAddsSkillMetadata() throws Exception {
ToolExecutionMetadataKeys.SKILL_NAME,
"math-calculator",
ToolExecutionMetadataKeys.SKILL_RESOURCE_PATH,
- "README.md"),
+ "README.md",
+ ToolExecutionMetadataKeys.SKILL_REGISTERED,
+ true),
ToolResponse.success("skill content"));
when(ctx.getResource("load_skill", ResourceType.TOOL)).thenReturn(tool);
when(ctx.getConfig()).thenReturn(toolCallConfig());
@@ -171,8 +889,110 @@ void processLoadSkillToolRequestAddsSkillMetadata() throws Exception {
metadata.put(ToolExecutionMetadataKeys.TOOL_TYPE, ToolType.FUNCTION.getValue());
metadata.put(ToolExecutionMetadataKeys.SKILL_NAME, "math-calculator");
metadata.put(ToolExecutionMetadataKeys.SKILL_RESOURCE_PATH, "README.md");
+ metadata.put(ToolExecutionMetadataKeys.SKILL_REGISTERED, true);
verify((ExecutionReporter) ctx)
- .reportExecutionStarted(ExecutionReporter.EntityTypes.TOOL, "load_skill", metadata);
+ .reportExecutionStartedAt(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("load_skill"),
+ eq(metadata),
+ anyString());
+ }
+
+ @Test
+ void durableCacheHitDoesNotRecordToolCallLatency() throws Exception {
+ RunnerContext ctx =
+ mock(RunnerContext.class, withSettings().extraInterfaces(ExecutionReporter.class));
+ Tool tool = mock(Tool.class);
+ when(ctx.getResource("search", ResourceType.TOOL)).thenReturn(tool);
+ when(ctx.getConfig()).thenReturn(toolCallConfig());
+ when(ctx.durableExecute(any())).thenReturn(ToolResponse.success("cached"));
+ Map function = new LinkedHashMap<>();
+ function.put("name", "search");
+ function.put("arguments", Map.of("query", "flink"));
+ Map toolCall = new LinkedHashMap<>();
+ toolCall.put("id", "call-1");
+ toolCall.put("function", function);
+
+ ToolCallAction.processToolRequest(
+ new ToolRequestEvent("test-model", List.of(toolCall)), ctx);
+
+ verify(tool, never()).call(any());
+ ExecutionReporter reporter = (ExecutionReporter) ctx;
+ verify(reporter, never())
+ .reportExecutionStartedAt(anyString(), anyString(), anyMap(), anyString());
+ verify(reporter)
+ .reportExecutionSucceededAt(
+ eq(ExecutionReporter.EntityTypes.TOOL),
+ eq("search"),
+ anyMap(),
+ anyString());
+ }
+
+ @Test
+ void interruptionBeforeToolInvocationReportsCreatedWithoutTerminal() throws Exception {
+ RunnerContext ctx =
+ mock(RunnerContext.class, withSettings().extraInterfaces(ExecutionReporter.class));
+ Tool tool = mock(Tool.class);
+ when(ctx.getResource("search", ResourceType.TOOL)).thenReturn(tool);
+ when(ctx.getConfig()).thenReturn(toolCallConfig());
+ when(ctx.durableExecute(any()))
+ .thenThrow(new InterruptedException("cancelled"));
+
+ Thread.interrupted();
+ assertThatThrownBy(
+ () ->
+ ToolCallAction.processToolRequest(
+ new ToolRequestEvent(
+ "test-model", List.of(toolCall("call-1"))),
+ ctx))
+ .isInstanceOf(InterruptedException.class);
+ Thread.interrupted();
+
+ ExecutionReporter reporter = (ExecutionReporter) ctx;
+ verify(reporter)
+ .reportExecutionCreated(
+ eq(ExecutionReporter.EntityTypes.TOOL), eq("search"), anyMap());
+ verify(reporter, never())
+ .reportExecutionStartedAt(anyString(), anyString(), anyMap(), anyString());
+ verifyNoTerminalReport(reporter);
+ }
+
+ @Test
+ void parallelInterruptionReportsStartedToolsWithoutTerminal() throws Exception {
+ RunnerContext ctx =
+ mock(RunnerContext.class, withSettings().extraInterfaces(ExecutionReporter.class));
+ Tool tool = mock(Tool.class);
+ when(ctx.getResource("search", ResourceType.TOOL)).thenReturn(tool);
+ when(ctx.getConfig()).thenReturn(toolCallConfig(true, 2));
+ when(tool.call(any())).thenReturn(ToolResponse.success("ok"));
+ when(ctx.durableExecuteAllAsync(any()))
+ .thenAnswer(
+ inv -> {
+ List> callables = inv.getArgument(0);
+ for (DurableCallable callable : callables) {
+ callable.call();
+ }
+ throw new InterruptedException("cancelled");
+ });
+
+ Thread.interrupted();
+ assertThatThrownBy(
+ () ->
+ ToolCallAction.processToolRequest(
+ new ToolRequestEvent(
+ "test-model",
+ List.of(toolCall("call-1"), toolCall("call-2"))),
+ ctx))
+ .isInstanceOf(InterruptedException.class);
+ Thread.interrupted();
+
+ ExecutionReporter reporter = (ExecutionReporter) ctx;
+ verify(reporter, times(2))
+ .reportExecutionCreated(
+ eq(ExecutionReporter.EntityTypes.TOOL), eq("search"), anyMap());
+ verify(reporter, times(2))
+ .reportExecutionStartedAt(anyString(), anyString(), anyMap(), anyString());
+ verifyNoTerminalReport(reporter);
}
@Test
@@ -202,12 +1022,20 @@ void executionMetadataCannotMutateToolCallParameters() throws Exception {
private static org.apache.flink.agents.api.configuration.ReadableConfiguration
toolCallConfig() {
+ return toolCallConfig(false, 1);
+ }
+
+ private static org.apache.flink.agents.api.configuration.ReadableConfiguration toolCallConfig(
+ boolean async, int parallelism) {
return new org.apache.flink.agents.api.configuration.ReadableConfiguration() {
@Override
@SuppressWarnings("unchecked")
public T get(org.apache.flink.agents.api.configuration.ConfigOption option) {
if (option == AgentExecutionOptions.TOOL_CALL_ASYNC) {
- return (T) Boolean.FALSE;
+ return (T) Boolean.valueOf(async);
+ }
+ if (option == AgentExecutionOptions.TOOL_CALL_PARALLELISM) {
+ return (T) Integer.valueOf(parallelism);
}
return option.getDefaultValue();
}
@@ -244,6 +1072,24 @@ public String getStr(String key, String defaultValue) {
};
}
+ private static Map toolCall(String id) {
+ Map function = new LinkedHashMap<>();
+ function.put("name", "search");
+ function.put("arguments", Map.of("query", id));
+ Map toolCall = new LinkedHashMap<>();
+ toolCall.put("id", id);
+ toolCall.put("function", function);
+ return toolCall;
+ }
+
+ private static void verifyNoTerminalReport(ExecutionReporter reporter) throws Exception {
+ verify(reporter, never())
+ .reportExecutionSucceededAt(anyString(), anyString(), anyMap(), anyString());
+ verify(reporter, never())
+ .reportExecutionFailedAt(
+ anyString(), anyString(), anyMap(), any(), anyString(), anyString());
+ }
+
private static final class ReportingTool extends Tool implements ToolExecutionMetadataProvider {
private final ToolType toolType;
private final Map entityMetadata;
diff --git a/python/flink_agents/api/tests/test_execution_reporter.py b/python/flink_agents/api/tests/test_execution_reporter.py
index a318104847..ac39cb5137 100644
--- a/python/flink_agents/api/tests/test_execution_reporter.py
+++ b/python/flink_agents/api/tests/test_execution_reporter.py
@@ -48,9 +48,79 @@ def test_failed_reporter_uses_metadata_before_error() -> None:
)
+def test_created_reporter_forwards_identity() -> None:
+ ctx = MagicMock(spec=ExecutionReporter)
+ metadata = {"toolCallId": "call-1"}
+
+ ExecutionReporters.created(
+ ctx,
+ ExecutionEntityTypes.TOOL,
+ "search",
+ metadata,
+ )
+
+ ctx.report_execution_created.assert_called_once_with(
+ ExecutionEntityTypes.TOOL,
+ "search",
+ metadata,
+ )
+
+
+def test_timestamped_reporters_forward_occurrence_timestamps() -> None:
+ ctx = MagicMock(spec=ExecutionReporter)
+ metadata = {"toolCallId": "call-1"}
+ error = RuntimeError("boom")
+
+ ExecutionReporters.started_at(
+ ctx,
+ ExecutionEntityTypes.TOOL,
+ "search",
+ metadata,
+ "2026-01-01T00:00:00.001Z",
+ )
+ ExecutionReporters.succeeded_at(
+ ctx,
+ ExecutionEntityTypes.TOOL,
+ "search",
+ metadata,
+ "2026-01-01T00:00:00.025Z",
+ )
+ ExecutionReporters.failed_at(
+ ctx,
+ ExecutionEntityTypes.TOOL,
+ "search",
+ metadata,
+ error,
+ ExecutionProblemCategories.TOOL_CALL_FAILED,
+ "2026-01-01T00:00:00.030Z",
+ )
+
+ ctx.report_execution_started_at.assert_called_once_with(
+ ExecutionEntityTypes.TOOL,
+ "search",
+ metadata,
+ "2026-01-01T00:00:00.001Z",
+ )
+ ctx.report_execution_succeeded_at.assert_called_once_with(
+ ExecutionEntityTypes.TOOL,
+ "search",
+ metadata,
+ "2026-01-01T00:00:00.025Z",
+ )
+ ctx.report_execution_failed_at.assert_called_once_with(
+ ExecutionEntityTypes.TOOL,
+ "search",
+ metadata,
+ error,
+ ExecutionProblemCategories.TOOL_CALL_FAILED,
+ "2026-01-01T00:00:00.030Z",
+ )
+
+
def test_reporters_ignore_context_without_execution_reporter() -> None:
ctx = MagicMock()
+ ExecutionReporters.created(ctx, ExecutionEntityTypes.LLM, "model")
ExecutionReporters.started(ctx, ExecutionEntityTypes.LLM, "model")
ExecutionReporters.succeeded(ctx, ExecutionEntityTypes.LLM, "model")
ExecutionReporters.failed(
@@ -62,6 +132,7 @@ def test_reporters_ignore_context_without_execution_reporter() -> None:
ExecutionProblemCategories.MODEL_CALL_FAILED,
)
+ ctx.report_execution_created.assert_not_called()
ctx.report_execution_started.assert_not_called()
ctx.report_execution_succeeded.assert_not_called()
ctx.report_execution_failed.assert_not_called()
diff --git a/python/flink_agents/api/trace/execution_lifecycle_events.py b/python/flink_agents/api/trace/execution_lifecycle_events.py
index ae32791f0e..61df56a08a 100644
--- a/python/flink_agents/api/trace/execution_lifecycle_events.py
+++ b/python/flink_agents/api/trace/execution_lifecycle_events.py
@@ -22,17 +22,20 @@
class ExecutionLifecycleEvents:
"""Framework-owned execution lifecycle Event types and statuses."""
+ EXECUTION_CREATED_EVENT_TYPE = "_execution_created_event"
EXECUTION_STARTED_EVENT_TYPE = "_execution_started_event"
EXECUTION_FINISHED_EVENT_TYPE = "_execution_finished_event"
EXECUTION_FAILED_EVENT_TYPE = "_execution_failed_event"
EXECUTION_REUSED_EVENT_TYPE = "_execution_reused_event"
+ STATUS_CREATED = "created"
STATUS_STARTED = "started"
STATUS_SUCCESS = "success"
STATUS_FAILED = "failed"
STATUS_REUSED = "reused"
_EXPECTED_STATUS_BY_EVENT_TYPE: ClassVar[dict[str, str]] = {
+ EXECUTION_CREATED_EVENT_TYPE: STATUS_CREATED,
EXECUTION_STARTED_EVENT_TYPE: STATUS_STARTED,
EXECUTION_FINISHED_EVENT_TYPE: STATUS_SUCCESS,
EXECUTION_FAILED_EVENT_TYPE: STATUS_FAILED,
diff --git a/python/flink_agents/api/trace/execution_reporter.py b/python/flink_agents/api/trace/execution_reporter.py
index 926f1ae393..93972ed9cb 100644
--- a/python/flink_agents/api/trace/execution_reporter.py
+++ b/python/flink_agents/api/trace/execution_reporter.py
@@ -49,6 +49,19 @@ class ExecutionProblemCategories:
class ExecutionReporter(ABC):
"""Optional capability for reporting executions nested inside an action."""
+ def report_execution_created(
+ self,
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None = None,
+ ) -> None:
+ """Report that a logical execution exists but has not necessarily started.
+
+ A later start or terminal report is not guaranteed. Their absence does not
+ show whether the underlying invocation ran.
+ """
+ return None
+
@abstractmethod
def report_execution_started(
self,
@@ -58,6 +71,16 @@ def report_execution_started(
) -> None:
"""Report that a logical execution started."""
+ def report_execution_started_at(
+ self,
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None,
+ timestamp: str,
+ ) -> None:
+ """Report that a logical execution started at an occurrence timestamp."""
+ self.report_execution_started(entity_type, entity_name, entity_metadata)
+
@abstractmethod
def report_execution_succeeded(
self,
@@ -67,6 +90,16 @@ def report_execution_succeeded(
) -> None:
"""Report that a logical execution completed successfully."""
+ def report_execution_succeeded_at(
+ self,
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None,
+ timestamp: str,
+ ) -> None:
+ """Report successful completion at an occurrence timestamp."""
+ self.report_execution_succeeded(entity_type, entity_name, entity_metadata)
+
@abstractmethod
def report_execution_failed(
self,
@@ -78,10 +111,43 @@ def report_execution_failed(
) -> None:
"""Report that a logical execution failed."""
+ def report_execution_failed_at(
+ self,
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None,
+ error: BaseException,
+ problem_category: str | None,
+ timestamp: str,
+ ) -> None:
+ """Report failed completion at an occurrence timestamp."""
+ self.report_execution_failed(
+ entity_type,
+ entity_name,
+ entity_metadata,
+ error,
+ problem_category,
+ )
+
class ExecutionReporters:
"""Best-effort helpers for contexts that implement ExecutionReporter."""
+ @staticmethod
+ def created(
+ ctx: "RunnerContext",
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None = None,
+ ) -> None:
+ """Report creation of a nested execution if the context supports it."""
+ ExecutionReporters._report(
+ ctx,
+ lambda reporter: reporter.report_execution_created(
+ entity_type, entity_name, entity_metadata or _EMPTY_METADATA
+ ),
+ )
+
@staticmethod
def started(
ctx: "RunnerContext",
@@ -97,6 +163,25 @@ def started(
),
)
+ @staticmethod
+ def started_at(
+ ctx: "RunnerContext",
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None,
+ timestamp: str,
+ ) -> None:
+ """Report a start occurrence if the context supports it."""
+ ExecutionReporters._report(
+ ctx,
+ lambda reporter: reporter.report_execution_started_at(
+ entity_type,
+ entity_name,
+ entity_metadata or _EMPTY_METADATA,
+ timestamp,
+ ),
+ )
+
@staticmethod
def succeeded(
ctx: "RunnerContext",
@@ -112,6 +197,25 @@ def succeeded(
),
)
+ @staticmethod
+ def succeeded_at(
+ ctx: "RunnerContext",
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None,
+ timestamp: str,
+ ) -> None:
+ """Report a successful occurrence if the context supports it."""
+ ExecutionReporters._report(
+ ctx,
+ lambda reporter: reporter.report_execution_succeeded_at(
+ entity_type,
+ entity_name,
+ entity_metadata or _EMPTY_METADATA,
+ timestamp,
+ ),
+ )
+
@staticmethod
def failed(
ctx: "RunnerContext",
@@ -133,6 +237,29 @@ def failed(
),
)
+ @staticmethod
+ def failed_at(
+ ctx: "RunnerContext",
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None,
+ error: BaseException,
+ problem_category: str | None,
+ timestamp: str,
+ ) -> None:
+ """Report a failed occurrence if the context supports it."""
+ ExecutionReporters._report(
+ ctx,
+ lambda reporter: reporter.report_execution_failed_at(
+ entity_type,
+ entity_name,
+ entity_metadata or _EMPTY_METADATA,
+ error,
+ problem_category,
+ timestamp,
+ ),
+ )
+
@staticmethod
def _report(
ctx: "RunnerContext", report: Callable[[ExecutionReporter], None]
diff --git a/python/flink_agents/api/trace/tool_execution_metadata_keys.py b/python/flink_agents/api/trace/tool_execution_metadata_keys.py
index 1308cb7bed..7549a096cb 100644
--- a/python/flink_agents/api/trace/tool_execution_metadata_keys.py
+++ b/python/flink_agents/api/trace/tool_execution_metadata_keys.py
@@ -27,4 +27,5 @@ class ToolExecutionMetadataKeys:
TOOL_TYPE = "toolType"
MCP_SERVER = "mcpServer"
SKILL_NAME = "skillName"
+ SKILL_REGISTERED = "skillRegistered"
SKILL_RESOURCE_PATH = "skillResourcePath"
diff --git a/python/flink_agents/cli/tests/test_trace_tree.py b/python/flink_agents/cli/tests/test_trace_tree.py
index 5f3915d85e..1e07753e18 100644
--- a/python/flink_agents/cli/tests/test_trace_tree.py
+++ b/python/flink_agents/cli/tests/test_trace_tree.py
@@ -246,6 +246,11 @@ def test_reader_ignores_execution_lifecycle_records(tmp_path: Path) -> None:
log_path,
[
_record("root", "_input_event"),
+ _execution_record(
+ "created",
+ ExecutionLifecycleEvents.EXECUTION_CREATED_EVENT_TYPE,
+ ExecutionLifecycleEvents.STATUS_CREATED,
+ ),
_execution_record(
"started",
ExecutionLifecycleEvents.EXECUTION_STARTED_EVENT_TYPE,
diff --git a/python/flink_agents/plan/actions/chat_model_action.py b/python/flink_agents/plan/actions/chat_model_action.py
index b802fda3ba..15d2a822f1 100644
--- a/python/flink_agents/plan/actions/chat_model_action.py
+++ b/python/flink_agents/plan/actions/chat_model_action.py
@@ -185,16 +185,21 @@ def _get_retry_stats(
def _record_retry_metrics(
- ctx: RunnerContext, model: str, retry_count: int, total_retry_wait_sec: int
+ ctx: RunnerContext,
+ model_resource: str,
+ retry_count: int,
+ total_retry_wait_sec: int,
) -> None:
- """Record retry metrics under the connection name if retries occurred."""
+ """Record retry metrics under the ChatModel resource if retries occurred."""
if retry_count <= 0:
return
metric_group = ctx.action_metric_group
if metric_group is not None:
- model_group = metric_group.get_sub_group("model", model)
- model_group.get_counter("retryCount").inc(retry_count)
- model_group.get_counter("retryWaitSec").inc(total_retry_wait_sec)
+ model_resource_group = metric_group.get_sub_group(
+ "model_resource", model_resource
+ )
+ model_resource_group.get_counter("retryCount").inc(retry_count)
+ model_resource_group.get_counter("retryWaitSec").inc(total_retry_wait_sec)
def _inject_bash_tool_args(
@@ -387,82 +392,85 @@ async def chat(
total_wait_time_sec = 0
llm_metadata = {LLMExecutionMetadataKeys.MODEL: chat_model.model}
- for attempt in range(num_retries + 1):
- try:
- ExecutionReporters.started(
- ctx, ExecutionEntityTypes.LLM, model, llm_metadata
- )
+ try:
+ for attempt in range(num_retries + 1):
try:
- if chat_async:
- response = await ctx.durable_execute_async(
- chat_model.chat, messages, prompt_args=prompt_args
- )
- else:
- response = ctx.durable_execute(
- chat_model.chat, messages, prompt_args=prompt_args
+ ExecutionReporters.started(
+ ctx, ExecutionEntityTypes.LLM, model, llm_metadata
+ )
+ try:
+ if chat_async:
+ response = await ctx.durable_execute_async(
+ chat_model.chat, messages, prompt_args=prompt_args
+ )
+ else:
+ response = ctx.durable_execute(
+ chat_model.chat, messages, prompt_args=prompt_args
+ )
+ response = _require_model_response(response)
+ except Exception as model_error:
+ ExecutionReporters.failed(
+ ctx,
+ ExecutionEntityTypes.LLM,
+ model,
+ llm_metadata,
+ model_error,
+ ExecutionProblemCategories.MODEL_CALL_FAILED,
)
- response = _require_model_response(response)
- except Exception as model_error:
- ExecutionReporters.failed(
- ctx,
- ExecutionEntityTypes.LLM,
- model,
- llm_metadata,
- model_error,
- ExecutionProblemCategories.MODEL_CALL_FAILED,
+ raise
+ ExecutionReporters.succeeded(
+ ctx, ExecutionEntityTypes.LLM, model, llm_metadata
)
- raise
- ExecutionReporters.succeeded(
- ctx, ExecutionEntityTypes.LLM, model, llm_metadata
- )
- if (
- request_metric_group is not None
- and response.extra_args.get("model_name")
- and response.extra_args.get("promptTokens")
- and response.extra_args.get("completionTokens")
- ):
- chat_model._record_token_metrics(
- response.extra_args["model_name"],
- response.extra_args["promptTokens"],
- response.extra_args["completionTokens"],
- request_metric_group,
- )
- # A truncated response consumed its full token budget, so the token
- # metrics above are recorded before this rejects and abandons the
- # response.
- _reject_incomplete_response(response)
- if output_schema is not None and len(response.tool_calls) == 0:
- response = _generate_structured_output_with_report(
- ctx, response, output_schema
- )
- break
- except Exception as e:
- if error_handling_strategy == ErrorHandlingStrategy.IGNORE:
- _logger.warning(
- f"Chat request {initial_request_id} failed with error: {e}, ignored."
- )
- return
- elif error_handling_strategy == ErrorHandlingStrategy.RETRY:
- if attempt == num_retries:
+ if (
+ request_metric_group is not None
+ and response.extra_args.get("model_name")
+ and response.extra_args.get("promptTokens")
+ and response.extra_args.get("completionTokens")
+ ):
+ chat_model._record_token_metrics(
+ response.extra_args["model_name"],
+ response.extra_args["promptTokens"],
+ response.extra_args["completionTokens"],
+ request_metric_group,
+ )
+ # A truncated response consumed its full token budget, so the token
+ # metrics above are recorded before this rejects and abandons the
+ # response.
+ _reject_incomplete_response(response)
+ if output_schema is not None and len(response.tool_calls) == 0:
+ response = _generate_structured_output_with_report(
+ ctx, response, output_schema
+ )
+ break
+ except Exception as e:
+ if error_handling_strategy == ErrorHandlingStrategy.IGNORE:
+ _logger.warning(
+ f"Chat request {initial_request_id} failed with error: {e}, ignored."
+ )
+ return
+ elif error_handling_strategy == ErrorHandlingStrategy.RETRY:
+ if attempt == num_retries:
+ raise
+ actual_retry_count = attempt + 1
+ current_wait_sec = retry_wait_interval_sec * (
+ 1 << (actual_retry_count - 1)
+ )
+ _logger.warning(
+ f"Chat request {initial_request_id} failed with error: {e}, "
+ f"retrying {actual_retry_count} / {num_retries}, "
+ f"waiting {current_wait_sec} s."
+ )
+ if current_wait_sec > 0:
+ time.sleep(current_wait_sec)
+ total_wait_time_sec += current_wait_sec
+ else:
+ _logger.debug(
+ f"Chat request {initial_request_id} failed, the input chat messages are {messages}."
+ )
raise
- actual_retry_count = attempt + 1
- current_wait_sec = retry_wait_interval_sec * (
- 1 << (actual_retry_count - 1)
- )
- _logger.warning(
- f"Chat request {initial_request_id} failed with error: {e}, "
- f"retrying {actual_retry_count} / {num_retries}, "
- f"waiting {current_wait_sec} s."
- )
- if current_wait_sec > 0:
- time.sleep(current_wait_sec)
- total_wait_time_sec += current_wait_sec
- else:
- _logger.debug(
- f"Chat request {initial_request_id} failed, the input chat messages are {messages}."
- )
- raise
+ finally:
+ _record_retry_metrics(ctx, model, actual_retry_count, total_wait_time_sec)
if actual_retry_count > 0:
_accumulate_retry_stats(
@@ -490,10 +498,6 @@ async def chat(
total_retry_count = retry_stats["total_retry_count"]
total_retry_wait_sec = retry_stats["total_retry_wait_sec"]
- _record_retry_metrics(
- ctx, chat_model.connection, total_retry_count, total_retry_wait_sec
- )
-
ctx.send_event(
ChatResponseEvent(
request_id=initial_request_id,
diff --git a/python/flink_agents/plan/actions/tool_call_action.py b/python/flink_agents/plan/actions/tool_call_action.py
index 4952bfd8dc..8b529fc671 100644
--- a/python/flink_agents/plan/actions/tool_call_action.py
+++ b/python/flink_agents/plan/actions/tool_call_action.py
@@ -16,7 +16,10 @@
# limitations under the License.
#################################################################################
import logging
+from collections.abc import Callable
from dataclasses import dataclass
+from datetime import datetime, timezone
+from functools import wraps
from typing import Any
from flink_agents.api.core_options import AgentExecutionOptions
@@ -78,12 +81,30 @@ def _tool_entity_metadata(
return metadata
+@dataclass
+class _ToolCallOccurrence:
+ started_at: datetime | None = None
+ finished_at: datetime | None = None
+
+ def wrap(self, func: Callable[..., Any]) -> Callable[..., Any]:
+ @wraps(func)
+ def observed_call(*args: Any, **kwargs: Any) -> Any:
+ self.started_at = datetime.now(timezone.utc)
+ try:
+ return func(*args, **kwargs)
+ finally:
+ self.finished_at = datetime.now(timezone.utc)
+
+ return observed_call
+
+
@dataclass(frozen=True)
class _ToolCallExecution:
id: str
name: str
durable_call: DurableCall
entity_metadata: dict[str, Any]
+ occurrence: _ToolCallOccurrence
async def process_tool_request(event: Event, ctx: RunnerContext) -> None:
@@ -166,10 +187,12 @@ def _build_tool_call_executions(
entity_metadata = _tool_entity_metadata(
event.id, call_id, external_id, name, tool, call_kwargs
)
- ExecutionReporters.started(
- ctx, ExecutionEntityTypes.TOOL, name, entity_metadata
+ ExecutionReporters.created(
+ ctx,
+ ExecutionEntityTypes.TOOL,
+ name,
+ entity_metadata,
)
-
if not tool or preparation_error is not None:
failure = preparation_error or RuntimeError(
f"Tool `{name}` does not exist."
@@ -191,15 +214,17 @@ def _build_tool_call_executions(
)
continue
+ occurrence = _ToolCallOccurrence()
executions.append(
_ToolCallExecution(
id=call_id,
name=name,
durable_call=DurableCall(
- func=tool.call,
+ func=occurrence.wrap(tool.call),
kwargs=call_kwargs,
),
entity_metadata=entity_metadata,
+ occurrence=occurrence,
)
)
return executions
@@ -212,15 +237,28 @@ async def _execute_parallel(
success: dict,
error: dict,
) -> None:
+ outcomes: list[Outcome] = []
+ result_observed_at = None
try:
outcomes = await ctx.durable_execute_all_async(
[execution.durable_call for execution in executions]
)
+ result_observed_at = datetime.now(timezone.utc)
for execution, outcome in zip(executions, outcomes, strict=True):
- _record_outcome(execution, outcome, ctx, responses, success, error)
+ _record_outcome(execution, outcome, responses, success, error)
except Exception as e:
+ if result_observed_at is None:
+ result_observed_at = datetime.now(timezone.utc)
for execution in executions:
- _record_execution_exception(execution, e, ctx, responses, success, error)
+ _record_execution_exception(execution, e, responses, success, error)
+ finally:
+ for index, execution in enumerate(executions):
+ _report_execution(
+ execution,
+ ctx,
+ outcomes[index] if index < len(outcomes) else None,
+ result_observed_at,
+ )
async def _execute_sequentially(
@@ -233,6 +271,8 @@ async def _execute_sequentially(
error: dict,
) -> None:
for execution in executions:
+ outcome = None
+ result_observed_at = None
try:
call = execution.durable_call
if tool_call_async:
@@ -247,31 +287,34 @@ async def _execute_sequentially(
*call.args,
**(call.kwargs or {}),
)
- _record_tool_response(execution, response, ctx, responses, success, error)
- except Exception as e: # noqa: PERF203
- _record_execution_exception(execution, e, ctx, responses, success, error)
+ result_observed_at = datetime.now(timezone.utc)
+ outcome = Outcome.success(response)
+ _record_tool_response(execution, response, responses, success, error)
+ except Exception as e:
+ if result_observed_at is None:
+ result_observed_at = datetime.now(timezone.utc)
+ outcome = Outcome.failure(e)
+ _record_execution_exception(execution, e, responses, success, error)
+ finally:
+ _report_execution(execution, ctx, outcome, result_observed_at)
def _record_outcome(
execution: _ToolCallExecution,
outcome: Outcome,
- ctx: RunnerContext,
responses: dict,
success: dict,
error: dict,
) -> None:
if outcome.is_failure():
- _record_execution_exception(
- execution, outcome.error, ctx, responses, success, error
- )
+ _record_execution_exception(execution, outcome.error, responses, success, error)
else:
- _record_tool_response(execution, outcome.value, ctx, responses, success, error)
+ _record_tool_response(execution, outcome.value, responses, success, error)
def _record_tool_response(
execution: _ToolCallExecution,
value: Any,
- ctx: RunnerContext,
responses: dict,
success: dict,
error: dict,
@@ -282,30 +325,15 @@ def _record_tool_response(
responses[execution.id] = message
success[execution.id] = False
error[execution.id] = message
- ExecutionReporters.failed(
- ctx,
- ExecutionEntityTypes.TOOL,
- execution.name,
- execution.entity_metadata,
- RuntimeError(message),
- ExecutionProblemCategories.TOOL_CALL_FAILED,
- )
return
responses[execution.id] = response.result
success[execution.id] = True
- ExecutionReporters.succeeded(
- ctx,
- ExecutionEntityTypes.TOOL,
- execution.name,
- execution.entity_metadata,
- )
def _record_execution_exception(
execution: _ToolCallExecution,
exception: BaseException,
- ctx: RunnerContext,
responses: dict,
success: dict,
error: dict,
@@ -313,14 +341,57 @@ def _record_execution_exception(
responses[execution.id] = f"Tool `{execution.name}` execute failed."
success[execution.id] = False
error[execution.id] = str(exception)
- ExecutionReporters.failed(
- ctx,
- ExecutionEntityTypes.TOOL,
- execution.name,
- execution.entity_metadata,
- exception,
- ExecutionProblemCategories.TOOL_CALL_FAILED,
+
+
+def _report_execution(
+ execution: _ToolCallExecution,
+ ctx: RunnerContext,
+ outcome: Outcome | None,
+ result_observed_at: datetime | None,
+) -> None:
+ finished_at = execution.occurrence.finished_at
+ started_at = execution.occurrence.started_at
+ if started_at is not None and (outcome is None or started_at <= result_observed_at):
+ ExecutionReporters.started_at(
+ ctx,
+ ExecutionEntityTypes.TOOL,
+ execution.name,
+ execution.entity_metadata,
+ started_at.isoformat().replace("+00:00", "Z"),
+ )
+ if outcome is None:
+ return
+ # A timed-out callable may finish after the Action already received its failure.
+ if finished_at is None or finished_at > result_observed_at:
+ finished_at = result_observed_at
+ finished_timestamp = finished_at.isoformat().replace("+00:00", "Z")
+ failure = (
+ outcome.error if outcome.is_failure() else _tool_response_failure(outcome.value)
)
+ if failure is None:
+ ExecutionReporters.succeeded_at(
+ ctx,
+ ExecutionEntityTypes.TOOL,
+ execution.name,
+ execution.entity_metadata,
+ finished_timestamp,
+ )
+ else:
+ ExecutionReporters.failed_at(
+ ctx,
+ ExecutionEntityTypes.TOOL,
+ execution.name,
+ execution.entity_metadata,
+ failure,
+ ExecutionProblemCategories.TOOL_CALL_FAILED,
+ finished_timestamp,
+ )
+
+
+def _tool_response_failure(value: Any) -> BaseException | None:
+ if isinstance(value, ToolResponse) and value.is_error():
+ return RuntimeError(value.error_message)
+ return None
def _resolve_injected_arguments(tool: object, ctx: RunnerContext) -> dict:
diff --git a/python/flink_agents/plan/tests/actions/test_chat_model_action_retry.py b/python/flink_agents/plan/tests/actions/test_chat_model_action_retry.py
index a5cad1a340..f5e4ce9643 100644
--- a/python/flink_agents/plan/tests/actions/test_chat_model_action_retry.py
+++ b/python/flink_agents/plan/tests/actions/test_chat_model_action_retry.py
@@ -246,10 +246,12 @@ def mock_chat(messages: Sequence[ChatMessage], **kwargs: Any) -> ChatMessage:
assert event.total_retry_wait_sec == 1
assert elapsed >= 1.0
- # Verify metrics recorded under connection name
- model_group = metric_group.get_sub_group("model", chat_model.connection)
- assert model_group.get_counter("retryCount").get_count() == 1
- assert model_group.get_counter("retryWaitSec").get_count() == 1
+ # Retry health belongs to the ChatModel resource.
+ model_resource_group = metric_group.get_sub_group(
+ "model_resource", "test-model"
+ )
+ assert model_resource_group.get_counter("retryCount").get_count() == 1
+ assert model_resource_group.get_counter("retryWaitSec").get_count() == 1
assert ctx.report_execution_started.call_count == 2
ctx.report_execution_failed.assert_called_once()
failed_args = ctx.report_execution_failed.call_args.args
@@ -267,7 +269,7 @@ def test_chat_exhausts_retries_and_raises(self) -> None:
chat_model = MagicMock()
chat_model.chat = MagicMock(side_effect=RuntimeError("persistent error"))
- ctx, sent_events, _, _ = _create_mock_runner_context(
+ ctx, sent_events, metric_group, _ = _create_mock_runner_context(
chat_model, max_retries=2, retry_wait_interval_sec=0
)
request_id = uuid4()
@@ -292,6 +294,11 @@ def test_chat_exhausts_retries_and_raises(self) -> None:
assert failed_call.args[2] == _LLM_METADATA
assert failed_call.args[-1] == ExecutionProblemCategories.MODEL_CALL_FAILED
ctx.report_execution_succeeded.assert_not_called()
+ model_resource_group = metric_group.get_sub_group(
+ "model_resource", "test-model"
+ )
+ assert model_resource_group.get_counter("retryCount").get_count() == 2
+ assert model_resource_group.get_counter("retryWaitSec").get_count() == 0
def test_structured_output_parse_error_retries_without_failing_llm(
self,
diff --git a/python/flink_agents/plan/tests/actions/test_tool_call_action.py b/python/flink_agents/plan/tests/actions/test_tool_call_action.py
index f119e31e78..8b56448ab6 100644
--- a/python/flink_agents/plan/tests/actions/test_tool_call_action.py
+++ b/python/flink_agents/plan/tests/actions/test_tool_call_action.py
@@ -16,8 +16,13 @@
# limitations under the License.
#################################################################################
import asyncio
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timedelta, timezone
from typing import Any
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
+
+import pytest
from flink_agents.api.core_options import AgentExecutionOptions
from flink_agents.api.events.tool_event import ToolRequestEvent, ToolResponseEvent
@@ -36,6 +41,7 @@
ExecutionReporter,
ToolExecutionMetadataKeys,
)
+from flink_agents.plan.actions import tool_call_action
from flink_agents.plan.actions.tool_call_action import process_tool_request
from flink_agents.plan.configuration import AgentConfiguration
from flink_agents.plan.function import PythonFunction
@@ -369,6 +375,507 @@ def test_tool_call_action_uses_parallel_batch_for_multiple_tools() -> None:
assert ctx.durable_execute_async_calls == []
+def test_parallel_tool_calls_report_independent_occurrences() -> None:
+ config = AgentConfiguration({"tenant_id": "tenant-1"})
+ config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, True)
+ config.set(AgentExecutionOptions.TOOL_CALL_PARALLELISM, 4)
+ tool = MagicMock()
+ tool.tool_type.return_value = ToolType.FUNCTION
+
+ def call_tool(**kwargs: Any) -> str | ToolResponse:
+ query = kwargs["query"]
+ if query == "call-2":
+ message = "call-2 failed"
+ raise RuntimeError(message)
+ if query == "call-3":
+ return ToolResponse.error("call-3 rejected")
+ return "ok"
+
+ tool.call = MagicMock(side_effect=call_tool)
+ ctx = MagicMock(spec=ExecutionReporter)
+ ctx.config = config
+ ctx.get_resource = MagicMock(return_value=tool)
+ sent_events = []
+ ctx.send_event = MagicMock(side_effect=sent_events.append)
+
+ async def execute_all(callables: list[Any]) -> list[Outcome]:
+ outcomes = []
+ for durable_call in callables:
+ try:
+ outcomes.append(
+ Outcome.success(
+ durable_call.func(
+ *durable_call.args, **(durable_call.kwargs or {})
+ )
+ )
+ )
+ except Exception as error: # noqa: PERF203
+ outcomes.append(Outcome.failure(error))
+ return outcomes
+
+ ctx.durable_execute_all_async = execute_all
+
+ request = ToolRequestEvent(
+ model="model-a",
+ tool_calls=[
+ {
+ "id": call_id,
+ "function": {
+ "name": "search",
+ "arguments": {"query": call_id},
+ },
+ }
+ for call_id in ("call-1", "call-2", "call-3")
+ ],
+ )
+
+ asyncio.run(process_tool_request(request, ctx))
+
+ assert ctx.report_execution_created.call_count == 3
+ assert ctx.report_execution_started_at.call_count == 3
+ assert ctx.report_execution_succeeded_at.call_count == 1
+ assert ctx.report_execution_failed_at.call_count == 2
+ terminal_call_ids = {
+ report.args[2][ToolExecutionMetadataKeys.TOOL_CALL_ID]
+ for report in (
+ ctx.report_execution_succeeded_at.call_args_list
+ + ctx.report_execution_failed_at.call_args_list
+ )
+ }
+ assert terminal_call_ids == {"call-1", "call-2", "call-3"}
+
+ response = ToolResponseEvent.from_event(sent_events[0])
+ assert response.success == {
+ "call-1": True,
+ "call-2": False,
+ "call-3": False,
+ }
+ assert response.error == {
+ "call-2": "call-2 failed",
+ "call-3": "call-3 rejected",
+ }
+
+
+def test_parallel_tool_calls_report_their_own_completion_timestamps() -> None:
+ base = datetime(2026, 1, 1, tzinfo=timezone.utc)
+ first_finished_at = base + timedelta(milliseconds=20)
+ second_finished_at = base + timedelta(milliseconds=150)
+ clock = [base]
+ tool = MagicMock()
+ tool.tool_type.return_value = ToolType.FUNCTION
+
+ def call_tool(**kwargs: Any) -> str:
+ clock[0] = (
+ first_finished_at if kwargs["query"] == "call-1" else second_finished_at
+ )
+ return "ok"
+
+ tool.call.side_effect = call_tool
+ ctx, _ = trace_context(tool)
+ ctx.config = AgentConfiguration({})
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, True)
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_PARALLELISM, 2)
+
+ async def execute_all(callables: list[Any]) -> list[Outcome]:
+ clock[0] = base
+ first = callables[0].func(*callables[0].args, **(callables[0].kwargs or {}))
+ clock[0] = base + timedelta(milliseconds=100)
+ second = callables[1].func(*callables[1].args, **(callables[1].kwargs or {}))
+ return [Outcome.success(first), Outcome.success(second)]
+
+ ctx.durable_execute_all_async = execute_all
+ request = ToolRequestEvent(
+ model="model-a",
+ tool_calls=[
+ {
+ "id": call_id,
+ "function": {
+ "name": "search",
+ "arguments": {"query": call_id},
+ },
+ }
+ for call_id in ("call-1", "call-2")
+ ],
+ )
+
+ with patch.object(tool_call_action, "datetime") as datetime_mock:
+ datetime_mock.now.side_effect = lambda tz: clock[0]
+ asyncio.run(process_tool_request(request, ctx))
+
+ terminal_timestamps = {
+ call.args[2][ToolExecutionMetadataKeys.TOOL_CALL_ID]: call.args[-1]
+ for call in ctx.report_execution_succeeded_at.call_args_list
+ }
+ assert terminal_timestamps == {
+ "call-1": "2026-01-01T00:00:00.020000Z",
+ "call-2": "2026-01-01T00:00:00.150000Z",
+ }
+
+
+def test_response_processing_failure_does_not_repeat_occurrences() -> None:
+ tool = MagicMock()
+ tool.tool_type.return_value = ToolType.FUNCTION
+ tool.call.return_value = "ok"
+ ctx, sent_events = trace_context(tool)
+ ctx.config = AgentConfiguration({})
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, True)
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_PARALLELISM, 3)
+
+ async def execute_all(callables: list[Any]) -> list[Outcome]:
+ return [
+ Outcome.success(call.func(*call.args, **(call.kwargs or {})))
+ for call in callables
+ ]
+
+ ctx.durable_execute_all_async = execute_all
+ with patch(
+ "flink_agents.plan.actions.tool_call_action._record_outcome",
+ side_effect=[None, RuntimeError("response processing failed")],
+ ):
+ asyncio.run(process_tool_request(parallel_trace_request(), ctx))
+
+ assert_occurrence_reports(
+ ctx, ["call-1", "call-2", "call-3"], ["call-1", "call-2", "call-3"], []
+ )
+ assert sent_events[0].success == dict.fromkeys(
+ ["call-1", "call-2", "call-3"], False
+ )
+
+
+@pytest.mark.parametrize("mode", ["sync", "serial_async", "parallel"])
+def test_durable_failure_is_reported_as_tool_failure(mode: str) -> None:
+ failure = RuntimeError("persist failed")
+ tool = MagicMock()
+ tool.tool_type.return_value = ToolType.FUNCTION
+ tool.call.return_value = "ok"
+ ctx, sent_events = trace_context(tool)
+ ctx.config = AgentConfiguration({})
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, mode != "sync")
+ ctx.config.set(
+ AgentExecutionOptions.TOOL_CALL_PARALLELISM, 3 if mode == "parallel" else 1
+ )
+
+ def execute(func: Any, **kwargs: Any) -> Any:
+ func(**kwargs)
+ raise failure
+
+ async def execute_async(func: Any, **kwargs: Any) -> Any:
+ return execute(func, **kwargs)
+
+ async def execute_all(callables: list[Any]) -> list[Outcome]:
+ outcomes = []
+ for call in callables:
+ result = call.func(*call.args, **(call.kwargs or {}))
+ outcomes.append(
+ Outcome.failure(failure)
+ if call.kwargs["query"] == "call-2"
+ else Outcome.success(result)
+ )
+ return outcomes
+
+ ctx.durable_execute = execute
+ ctx.durable_execute_async = execute_async
+ ctx.durable_execute_all_async = execute_all
+
+ asyncio.run(process_tool_request(parallel_trace_request(), ctx))
+
+ assert_occurrence_reports(
+ ctx,
+ ["call-1", "call-2", "call-3"],
+ ["call-1", "call-3"] if mode == "parallel" else [],
+ ["call-2"] if mode == "parallel" else ["call-1", "call-2", "call-3"],
+ )
+ assert all(
+ call.args[3] is failure
+ for call in ctx.report_execution_failed_at.call_args_list
+ )
+ assert sent_events[0].success["call-2"] is False
+ assert sent_events[0].error["call-2"] == "persist failed"
+
+
+def test_parallel_batch_failure_before_invocation_retains_created_executions() -> None:
+ tool = MagicMock()
+ tool.tool_type.return_value = ToolType.FUNCTION
+ ctx, _ = trace_context(tool)
+ ctx.config = AgentConfiguration({})
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, True)
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_PARALLELISM, 3)
+ failure_message = "batch failed before invocation"
+
+ async def execute_all(callables: list[Any]) -> list[Outcome]:
+ raise RuntimeError(failure_message)
+
+ ctx.durable_execute_all_async = execute_all
+
+ asyncio.run(process_tool_request(parallel_trace_request(), ctx))
+
+ created_call_ids = {
+ call.args[2][ToolExecutionMetadataKeys.TOOL_CALL_ID]
+ for call in ctx.report_execution_created.call_args_list
+ }
+ assert created_call_ids == {"call-1", "call-2", "call-3"}
+ ctx.report_execution_started_at.assert_not_called()
+ ctx.report_execution_succeeded_at.assert_not_called()
+ ctx.report_execution_failed_at.assert_not_called()
+ tool.call.assert_not_called()
+
+
+def test_timeout_reports_failure_without_repeating_on_late_completion() -> None:
+ failure = TimeoutError("request timed out")
+ started = threading.Event()
+ release = threading.Event()
+ tool = MagicMock()
+ tool.tool_type.return_value = ToolType.FUNCTION
+
+ def call_tool(**kwargs: Any) -> str:
+ started.set()
+ assert release.wait(5)
+ return "ok"
+
+ tool.call.side_effect = call_tool
+ ctx, sent_events = trace_context(tool)
+ ctx.config = AgentConfiguration({})
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, True)
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_PARALLELISM, 1)
+ worker = ThreadPoolExecutor(max_workers=1)
+ pending = []
+ reporting_started_at = []
+ ctx.report_execution_started_at.side_effect = (
+ lambda *args: reporting_started_at.append(datetime.now(timezone.utc))
+ )
+
+ async def execute_async(func: Any, **kwargs: Any) -> Any:
+ pending.append(worker.submit(func, **kwargs))
+ assert started.wait(5)
+ raise failure
+
+ ctx.durable_execute_async = execute_async
+ try:
+ asyncio.run(
+ process_tool_request(
+ ToolRequestEvent(model="model-a", tool_calls=[trace_tool_call()]), ctx
+ )
+ )
+ assert_occurrence_reports(ctx, ["call-1"], [], ["call-1"])
+ assert ctx.report_execution_failed_at.call_args.args[3] is failure
+ finished_at = ctx.report_execution_failed_at.call_args.args[-1]
+ assert (
+ datetime.fromisoformat(finished_at.replace("Z", "+00:00"))
+ <= (reporting_started_at[0])
+ )
+ assert sent_events[0].success["call-1"] is False
+ finally:
+ release.set()
+ worker.shutdown(wait=True)
+
+ assert pending[0].result() == "ok"
+ assert_occurrence_reports(ctx, ["call-1"], [], ["call-1"])
+
+
+@pytest.mark.parametrize("complete_during_reporting", [False, True])
+def test_parallel_timeout_timestamp_precedes_response_processing_and_reporting(
+ complete_during_reporting: bool,
+) -> None:
+ failure = TimeoutError("batch timed out")
+ release = threading.Event()
+ started = {call_id: threading.Event() for call_id in ("call-2", "call-3")}
+ base = datetime(2026, 1, 1, tzinfo=timezone.utc)
+ clock = [base]
+ observed_at = base + timedelta(seconds=1)
+ tool = MagicMock()
+ tool.tool_type.return_value = ToolType.FUNCTION
+
+ def call_tool(**kwargs: Any) -> str:
+ call_id = kwargs["query"]
+ if call_id != "call-1":
+ started[call_id].set()
+ assert release.wait(5)
+ return "ok"
+
+ tool.call.side_effect = call_tool
+ ctx, sent_events = trace_context(tool)
+ ctx.config = AgentConfiguration({})
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, True)
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_PARALLELISM, 3)
+ pending = []
+ record_outcome = tool_call_action._record_outcome
+
+ def process_response(*args: Any) -> None:
+ clock[0] = base + timedelta(seconds=2)
+ record_outcome(*args)
+
+ def report_started(*args: Any) -> None:
+ clock[0] = base + timedelta(seconds=3)
+ if complete_during_reporting:
+ release.set()
+ for future in pending:
+ future.result(timeout=5)
+
+ ctx.report_execution_started_at.side_effect = report_started
+ with (
+ ThreadPoolExecutor(max_workers=2) as workers,
+ patch.object(tool_call_action, "datetime") as datetime_mock,
+ patch.object(tool_call_action, "_record_outcome", side_effect=process_response),
+ ):
+ datetime_mock.now.side_effect = lambda tz: clock[0]
+
+ async def execute_all(callables: list[Any]) -> list[Outcome]:
+ first = callables[0]
+ result = first.func(*first.args, **(first.kwargs or {}))
+ pending.extend(
+ workers.submit(call.func, *call.args, **(call.kwargs or {}))
+ for call in callables[1:]
+ )
+ assert all(event.wait(5) for event in started.values())
+ clock[0] = observed_at
+ return [
+ Outcome.success(result),
+ Outcome.failure(failure),
+ Outcome.failure(failure),
+ ]
+
+ ctx.durable_execute_all_async = execute_all
+ try:
+ asyncio.run(process_tool_request(parallel_trace_request(), ctx))
+ timestamps = [
+ call.args[-1] for call in ctx.report_execution_failed_at.call_args_list
+ ]
+ assert timestamps == ["2026-01-01T00:00:01Z"] * 2
+ assert sent_events[0].success == {
+ "call-1": True,
+ "call-2": False,
+ "call-3": False,
+ }
+ finally:
+ release.set()
+ for future in pending:
+ future.result(timeout=5)
+
+ assert_occurrence_reports(
+ ctx, ["call-1", "call-2", "call-3"], ["call-1"], ["call-2", "call-3"]
+ )
+
+
+@pytest.mark.parametrize("starts_after_observation", [False, True])
+def test_parallel_timeout_omits_starts_after_result_observation(
+ starts_after_observation: bool,
+) -> None:
+ failure = TimeoutError("batch timed out")
+ base = datetime(2026, 1, 1, tzinfo=timezone.utc)
+ observed_at = base + timedelta(seconds=1)
+ delayed_start = (
+ observed_at + timedelta(seconds=1) if starts_after_observation else observed_at
+ )
+ clock = [base]
+ delayed = []
+ tool = MagicMock()
+ tool.tool_type.return_value = ToolType.FUNCTION
+ tool.call.return_value = "ok"
+ ctx, sent_events = trace_context(tool)
+ ctx.config = AgentConfiguration({})
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, True)
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_PARALLELISM, 3)
+
+ async def execute_all(callables: list[Any]) -> list[Outcome]:
+ first = callables[0]
+ result = first.func(*first.args, **(first.kwargs or {}))
+ delayed.extend(callables[1:])
+ clock[0] = observed_at
+ return [
+ Outcome.success(result),
+ Outcome.failure(failure),
+ Outcome.failure(failure),
+ ]
+
+ def report_started(*args: Any) -> None:
+ if args[2][ToolExecutionMetadataKeys.TOOL_CALL_ID] == "call-1":
+ # The delayed calls enter after the Action has observed the timeout.
+ clock[0] = delayed_start
+ for call in delayed:
+ call.func(*call.args, **(call.kwargs or {}))
+
+ ctx.durable_execute_all_async = execute_all
+ ctx.report_execution_started_at.side_effect = report_started
+ with patch.object(tool_call_action, "datetime") as datetime_mock:
+ datetime_mock.now.side_effect = lambda tz: clock[0]
+ asyncio.run(process_tool_request(parallel_trace_request(), ctx))
+
+ starts = ["call-1"] if starts_after_observation else ["call-1", "call-2", "call-3"]
+ assert_occurrence_reports(ctx, starts, ["call-1"], ["call-2", "call-3"])
+ assert tool.call.call_count == 3
+ assert all(
+ call.args[3] is failure and call.args[-1] == "2026-01-01T00:00:01Z"
+ for call in ctx.report_execution_failed_at.call_args_list
+ )
+ assert sent_events[0].success == {
+ "call-1": True,
+ "call-2": False,
+ "call-3": False,
+ }
+
+
+def test_partial_cache_replay_only_reports_start_for_invoked_tool() -> None:
+ tool = MagicMock()
+ tool.tool_type.return_value = ToolType.FUNCTION
+ tool.call.return_value = "ok"
+ ctx, sent_events = trace_context(tool)
+ ctx.config = AgentConfiguration({})
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, True)
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_PARALLELISM, 3)
+
+ async def execute_all(callables: list[Any]) -> list[Outcome]:
+ call = callables[1]
+ result = call.func(*call.args, **(call.kwargs or {}))
+ return [
+ Outcome.success("cached"),
+ Outcome.success(result),
+ Outcome.success("cached"),
+ ]
+
+ ctx.durable_execute_all_async = execute_all
+
+ asyncio.run(process_tool_request(parallel_trace_request(), ctx))
+
+ assert_occurrence_reports(ctx, ["call-2"], ["call-1", "call-2", "call-3"], [])
+ assert all(sent_events[0].success.values())
+ tool.call.assert_called_once()
+
+
+def parallel_trace_request() -> ToolRequestEvent:
+ return ToolRequestEvent(
+ model="model-a",
+ tool_calls=[
+ {
+ "id": call_id,
+ "function": {"name": "search", "arguments": {"query": call_id}},
+ }
+ for call_id in ("call-1", "call-2", "call-3")
+ ],
+ )
+
+
+def assert_occurrence_reports(
+ ctx: MagicMock, started: list[str], succeeded: list[str], failed: list[str]
+) -> None:
+ expected_created = set(started + succeeded + failed)
+ actual_created = {
+ call.args[2][ToolExecutionMetadataKeys.TOOL_CALL_ID]
+ for call in ctx.report_execution_created.call_args_list
+ }
+ assert ctx.report_execution_created.call_count == len(expected_created)
+ assert actual_created == expected_created
+ for method, expected in (
+ (ctx.report_execution_started_at, started),
+ (ctx.report_execution_succeeded_at, succeeded),
+ (ctx.report_execution_failed_at, failed),
+ ):
+ actual = [
+ call.args[2][ToolExecutionMetadataKeys.TOOL_CALL_ID]
+ for call in method.call_args_list
+ ]
+ assert sorted(actual) == sorted(expected)
+
+
def test_tool_call_action_uses_serial_async_when_parallelism_is_one() -> None:
config = AgentConfiguration({"tenant_id": "tenant-1"})
config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, True)
@@ -539,13 +1046,19 @@ def test_tool_call_reports_started_and_succeeded() -> None:
ToolExecutionMetadataKeys.EXTERNAL_ID: "external-call-1",
ToolExecutionMetadataKeys.TOOL_TYPE: "function",
}
- ctx.report_execution_started.assert_called_once_with(
+ ctx.report_execution_created.assert_called_once_with(
ExecutionEntityTypes.TOOL, "search", metadata
)
- ctx.report_execution_succeeded.assert_called_once_with(
- ExecutionEntityTypes.TOOL, "search", metadata
+ ctx.report_execution_started_at.assert_called_once()
+ started_args = ctx.report_execution_started_at.call_args.args
+ assert started_args[:3] == (ExecutionEntityTypes.TOOL, "search", metadata)
+ ctx.report_execution_succeeded_at.assert_called_once()
+ succeeded_args = ctx.report_execution_succeeded_at.call_args.args
+ assert succeeded_args[:3] == (ExecutionEntityTypes.TOOL, "search", metadata)
+ assert datetime.fromisoformat(succeeded_args[3].replace("Z", "+00:00")) >= (
+ datetime.fromisoformat(started_args[3].replace("Z", "+00:00"))
)
- ctx.report_execution_failed.assert_not_called()
+ ctx.report_execution_failed_at.assert_not_called()
def test_tool_call_reports_failed() -> None:
@@ -557,13 +1070,14 @@ def test_tool_call_reports_failed() -> None:
asyncio.run(process_tool_request(request, ctx))
- ctx.report_execution_failed.assert_called_once()
- args = ctx.report_execution_failed.call_args.args
+ ctx.report_execution_failed_at.assert_called_once()
+ args = ctx.report_execution_failed_at.call_args.args
assert args[0] == ExecutionEntityTypes.TOOL
assert args[1] == "search"
assert args[2][ToolExecutionMetadataKeys.TOOL_CALL_ID] == "call-1"
assert isinstance(args[3], RuntimeError)
assert args[4] == ExecutionProblemCategories.TOOL_CALL_FAILED
+ assert datetime.fromisoformat(args[5].replace("Z", "+00:00"))
def test_tool_call_reports_explicit_tool_response_failure() -> None:
@@ -579,8 +1093,8 @@ def test_tool_call_reports_explicit_tool_response_failure() -> None:
assert response.responses["call-1"] == "business failure"
assert response.success["call-1"] is False
assert response.error["call-1"] == "business failure"
- ctx.report_execution_failed.assert_called_once()
- ctx.report_execution_succeeded.assert_not_called()
+ ctx.report_execution_failed_at.assert_called_once()
+ ctx.report_execution_succeeded_at.assert_not_called()
def test_tool_call_preserves_empty_tool_response_error() -> None:
@@ -598,6 +1112,32 @@ def test_tool_call_preserves_empty_tool_response_error() -> None:
assert response.error["call-1"] == ""
+def test_missing_tool_reports_creation_and_failure_without_start() -> None:
+ ctx = MagicMock(spec=ExecutionReporter)
+ ctx.config = AgentConfiguration({})
+ ctx.config.set(AgentExecutionOptions.TOOL_CALL_ASYNC, False)
+ ctx.get_resource = MagicMock(side_effect=ValueError("Tool does not exist."))
+ ctx.send_event = MagicMock()
+
+ asyncio.run(
+ process_tool_request(
+ ToolRequestEvent(model="model-a", tool_calls=[trace_tool_call()]), ctx
+ )
+ )
+
+ created_metadata = ctx.report_execution_created.call_args.args[2]
+ ctx.report_execution_created.assert_called_once_with(
+ ExecutionEntityTypes.TOOL, "search", created_metadata
+ )
+ assert created_metadata[ToolExecutionMetadataKeys.TOOL_CALL_ID] == "call-1"
+ assert ctx.report_execution_failed.call_args.args[:3] == (
+ ExecutionEntityTypes.TOOL,
+ "search",
+ created_metadata,
+ )
+ ctx.report_execution_started_at.assert_not_called()
+
+
def test_tool_call_includes_provider_metadata() -> None:
class MetadataTool(ToolExecutionMetadataProvider):
@staticmethod
@@ -618,10 +1158,60 @@ def get_tool_execution_metadata(
asyncio.run(process_tool_request(request, ctx))
- metadata = ctx.report_execution_started.call_args.args[2]
+ metadata = ctx.report_execution_started_at.call_args.args[2]
assert metadata[ToolExecutionMetadataKeys.MCP_SERVER] == "search_server"
+def test_tool_call_reports_registered_skill_metadata() -> None:
+ class SkillTool(ToolExecutionMetadataProvider):
+ @staticmethod
+ def tool_type() -> ToolType:
+ return ToolType.FUNCTION
+
+ @staticmethod
+ def call(**kwargs: object) -> str:
+ return "skill content"
+
+ def get_tool_execution_metadata(
+ self, parameters: dict[str, object]
+ ) -> dict[str, object]:
+ return {
+ ToolExecutionMetadataKeys.SKILL_NAME: "calculator",
+ ToolExecutionMetadataKeys.SKILL_REGISTERED: True,
+ }
+
+ ctx, _ = trace_context(SkillTool())
+
+ asyncio.run(
+ process_tool_request(
+ ToolRequestEvent(model="model-a", tool_calls=[trace_tool_call()]), ctx
+ )
+ )
+
+ metadata = ctx.report_execution_started_at.call_args.args[2]
+ assert metadata[ToolExecutionMetadataKeys.SKILL_NAME] == "calculator"
+ assert metadata[ToolExecutionMetadataKeys.SKILL_REGISTERED] is True
+
+
+def test_durable_cache_hit_does_not_record_tool_call_latency() -> None:
+ tool = MagicMock()
+ tool.tool_type.return_value = ToolType.FUNCTION
+ tool.call = MagicMock(return_value="uncached")
+ ctx, _ = trace_context(tool)
+ ctx.durable_execute = MagicMock(return_value="cached")
+
+ asyncio.run(
+ process_tool_request(
+ ToolRequestEvent(model="model-a", tool_calls=[trace_tool_call()]), ctx
+ )
+ )
+
+ tool.call.assert_not_called()
+ ctx.report_execution_created.assert_called_once()
+ ctx.report_execution_started_at.assert_not_called()
+ ctx.report_execution_succeeded_at.assert_called_once()
+
+
def test_tool_execution_metadata_cannot_mutate_call_arguments() -> None:
class MutatingMetadataTool(ToolExecutionMetadataProvider):
@staticmethod
diff --git a/python/flink_agents/runtime/flink_runner_context.py b/python/flink_agents/runtime/flink_runner_context.py
index 8fab1e7f3f..fc3c1ebeec 100644
--- a/python/flink_agents/runtime/flink_runner_context.py
+++ b/python/flink_agents/runtime/flink_runner_context.py
@@ -689,6 +689,19 @@ def action_metric_group(self) -> FlinkMetricGroup:
"""
return FlinkMetricGroup(self._j_runner_context.getActionMetricGroup())
+ @override
+ def report_execution_created(
+ self,
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None = None,
+ ) -> None:
+ self._j_runner_context.reportExecutionCreatedJson(
+ entity_type,
+ entity_name,
+ self._entity_metadata_json(entity_metadata),
+ )
+
@override
def report_execution_started(
self,
@@ -702,6 +715,21 @@ def report_execution_started(
self._entity_metadata_json(entity_metadata),
)
+ @override
+ def report_execution_started_at(
+ self,
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None,
+ timestamp: str,
+ ) -> None:
+ self._j_runner_context.reportExecutionStartedAtJson(
+ entity_type,
+ entity_name,
+ self._entity_metadata_json(entity_metadata),
+ timestamp,
+ )
+
@override
def report_execution_succeeded(
self,
@@ -715,6 +743,21 @@ def report_execution_succeeded(
self._entity_metadata_json(entity_metadata),
)
+ @override
+ def report_execution_succeeded_at(
+ self,
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None,
+ timestamp: str,
+ ) -> None:
+ self._j_runner_context.reportExecutionSucceededAtJson(
+ entity_type,
+ entity_name,
+ self._entity_metadata_json(entity_metadata),
+ timestamp,
+ )
+
@override
def report_execution_failed(
self,
@@ -735,6 +778,28 @@ def report_execution_failed(
problem_category,
)
+ @override
+ def report_execution_failed_at(
+ self,
+ entity_type: str,
+ entity_name: str,
+ entity_metadata: Mapping[str, Any] | None,
+ error: BaseException,
+ problem_category: str | None,
+ timestamp: str,
+ ) -> None:
+ root_error = _root_cause(error)
+ error_message = str(root_error)
+ self._j_runner_context.reportExecutionFailedAtJson(
+ entity_type,
+ entity_name,
+ self._entity_metadata_json(entity_metadata),
+ _error_type(root_error),
+ error_message or None,
+ problem_category,
+ timestamp,
+ )
+
@staticmethod
def _entity_metadata_json(entity_metadata: Mapping[str, Any] | None) -> str:
return json.dumps(dict(entity_metadata or {}))
diff --git a/python/flink_agents/runtime/skill/skill_tools.py b/python/flink_agents/runtime/skill/skill_tools.py
index f262212464..931252e82e 100644
--- a/python/flink_agents/runtime/skill/skill_tools.py
+++ b/python/flink_agents/runtime/skill/skill_tools.py
@@ -92,7 +92,11 @@ def get_tool_execution_metadata(
"""Describe the requested skill resource for execution tracing."""
metadata = {}
if "name" in parameters:
- metadata[ToolExecutionMetadataKeys.SKILL_NAME] = str(parameters["name"])
+ skill_name = str(parameters["name"])
+ metadata[ToolExecutionMetadataKeys.SKILL_NAME] = skill_name
+ metadata[ToolExecutionMetadataKeys.SKILL_REGISTERED] = (
+ self._is_registered_skill(skill_name)
+ )
metadata[ToolExecutionMetadataKeys.SKILL_RESOURCE_PATH] = (
_normalize_skill_resource_path(
parameters.get("path"), missing="path" not in parameters
@@ -157,6 +161,13 @@ def call(self, *args: Any, **kwargs: Any) -> str | ToolResponse:
)
return content
+ def _is_registered_skill(self, skill_name: str) -> bool:
+ try:
+ manager = self._get_skill_manager()
+ return manager is not None and skill_name in manager.get_all_skill_names()
+ except Exception:
+ return False
+
def _get_skill_manager(self) -> SkillManager | None:
from flink_agents.runtime.resource_context import ResourceContextImpl
diff --git a/python/flink_agents/runtime/skill/tests/test_load_skill.py b/python/flink_agents/runtime/skill/tests/test_load_skill.py
index e428f8e701..8d8716e339 100644
--- a/python/flink_agents/runtime/skill/tests/test_load_skill.py
+++ b/python/flink_agents/runtime/skill/tests/test_load_skill.py
@@ -98,6 +98,7 @@ def test_execution_metadata_describes_requested_resource(
{"name": "github", "path": "README.md"}
)
assert metadata[ToolExecutionMetadataKeys.SKILL_NAME] == "github"
+ assert metadata[ToolExecutionMetadataKeys.SKILL_REGISTERED] is True
assert metadata[ToolExecutionMetadataKeys.SKILL_RESOURCE_PATH] == "README.md"
def test_execution_metadata_normalizes_omitted_path(
@@ -125,6 +126,13 @@ def test_skill_not_found(self, tool: LoadSkillTool) -> None:
assert "github" in result.error_message
assert "nano-banana-pro" in result.error_message
+ def test_execution_metadata_marks_unknown_skill_as_unregistered(
+ self, tool: LoadSkillTool
+ ) -> None:
+ metadata = tool.get_tool_execution_metadata({"name": "nonexistent-skill"})
+ assert metadata[ToolExecutionMetadataKeys.SKILL_NAME] == "nonexistent-skill"
+ assert metadata[ToolExecutionMetadataKeys.SKILL_REGISTERED] is False
+
# -- no skill manager ----------------------------------------------------
def test_no_skill_manager(self) -> None:
diff --git a/python/flink_agents/runtime/tests/test_flink_runner_context_trace.py b/python/flink_agents/runtime/tests/test_flink_runner_context_trace.py
index 083a3ef737..0be6880598 100644
--- a/python/flink_agents/runtime/tests/test_flink_runner_context_trace.py
+++ b/python/flink_agents/runtime/tests/test_flink_runner_context_trace.py
@@ -30,6 +30,48 @@ def test_flink_runner_context_is_execution_reporter() -> None:
assert issubclass(FlinkRunnerContext, ExecutionReporter)
+def test_timestamped_execution_reports_forward_to_java_context() -> None:
+ java_context = MagicMock()
+ ctx = FlinkRunnerContext.__new__(FlinkRunnerContext)
+ ctx._j_runner_context = java_context
+
+ ctx.report_execution_created(
+ ExecutionEntityTypes.TOOL,
+ "search",
+ {"toolCallId": "call-1"},
+ )
+ ctx.report_execution_started_at(
+ ExecutionEntityTypes.TOOL,
+ "search",
+ {"toolCallId": "call-1"},
+ "2026-01-01T00:00:00.001Z",
+ )
+ ctx.report_execution_succeeded_at(
+ ExecutionEntityTypes.TOOL,
+ "search",
+ {"toolCallId": "call-1"},
+ "2026-01-01T00:00:00.025Z",
+ )
+
+ java_context.reportExecutionCreatedJson.assert_called_once_with(
+ ExecutionEntityTypes.TOOL,
+ "search",
+ '{"toolCallId": "call-1"}',
+ )
+ java_context.reportExecutionStartedAtJson.assert_called_once_with(
+ ExecutionEntityTypes.TOOL,
+ "search",
+ '{"toolCallId": "call-1"}',
+ "2026-01-01T00:00:00.001Z",
+ )
+ java_context.reportExecutionSucceededAtJson.assert_called_once_with(
+ ExecutionEntityTypes.TOOL,
+ "search",
+ '{"toolCallId": "call-1"}',
+ "2026-01-01T00:00:00.025Z",
+ )
+
+
def test_failed_execution_reports_deepest_explicit_cause() -> None:
java_context = MagicMock()
ctx = FlinkRunnerContext.__new__(FlinkRunnerContext)
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/CompileUtils.java b/runtime/src/main/java/org/apache/flink/agents/runtime/CompileUtils.java
index 65e998cb80..d8e123ca11 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/CompileUtils.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/CompileUtils.java
@@ -32,6 +32,7 @@
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.typeinfo.python.PickledByteArrayTypeInfo;
import org.apache.flink.types.Row;
+import org.apache.flink.util.StringUtils;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkState;
@@ -41,6 +42,7 @@ public class CompileUtils {
private static final int PYTHON_KEY_FIELD_INDEX = 0;
private static final int PYTHON_VALUE_FIELD_INDEX = 1;
+ private static final String DEFAULT_OPERATOR_NAME = "action-execute-operator";
// ============================ invoke by python ====================================
public static DataStream connectToAgent(
@@ -102,10 +104,13 @@ private static DataStream connectToAgent(
boolean inputIsJava,
boolean pythonKeyIsPickled) {
checkBatchStateBackendCompatibility(keyedInputStream);
+ String agentName = agentPlan.getAgentName();
+ String operatorName =
+ StringUtils.isNullOrWhitespaceOnly(agentName) ? DEFAULT_OPERATOR_NAME : agentName;
return (DataStream)
keyedInputStream
.transform(
- "action-execute-operator",
+ operatorName,
outTypeInformation,
new ActionExecutionOperatorFactory(
agentPlan, inputIsJava, pythonKeyIsPickled))
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/context/RunnerContextImpl.java b/runtime/src/main/java/org/apache/flink/agents/runtime/context/RunnerContextImpl.java
index e626afaaef..f26ce9983e 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/context/RunnerContextImpl.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/context/RunnerContextImpl.java
@@ -22,6 +22,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
import org.apache.flink.agents.api.configuration.ReadableConfiguration;
import org.apache.flink.agents.api.context.DurableCallable;
import org.apache.flink.agents.api.context.MemoryObject;
@@ -344,6 +345,17 @@ public List getShortTermMemoryUpdates() {
return List.copyOf(memoryContext.getShortTermMemoryUpdates());
}
+ @Override
+ public void reportExecutionCreated(
+ String entityType, String entityName, Map entityMetadata)
+ throws Exception {
+ reportChildExecution(
+ entityType,
+ entityName,
+ entityMetadata,
+ ExecutionLifecycleEvents.executionCreated());
+ }
+
@Override
public void reportExecutionStarted(
String entityType, String entityName, Map entityMetadata)
@@ -355,6 +367,22 @@ public void reportExecutionStarted(
ExecutionLifecycleEvents.executionStarted());
}
+ @Override
+ public void reportExecutionStartedAt(
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ String timestamp)
+ throws Exception {
+ Event event = ExecutionLifecycleEvents.executionStarted();
+ reportChildExecution(
+ entityType,
+ entityName,
+ entityMetadata,
+ new EventContext(event.getType(), timestamp),
+ event);
+ }
+
@Override
public void reportExecutionSucceeded(
String entityType, String entityName, Map entityMetadata)
@@ -366,6 +394,22 @@ public void reportExecutionSucceeded(
ExecutionLifecycleEvents.executionFinished());
}
+ @Override
+ public void reportExecutionSucceededAt(
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ String timestamp)
+ throws Exception {
+ Event event = ExecutionLifecycleEvents.executionFinished();
+ reportChildExecution(
+ entityType,
+ entityName,
+ entityMetadata,
+ new EventContext(event.getType(), timestamp),
+ event);
+ }
+
@Override
public void reportExecutionFailed(
String entityType,
@@ -381,19 +425,48 @@ public void reportExecutionFailed(
ExecutionLifecycleEvents.executionFailed(error, problemCategory));
}
+ @Override
+ public void reportExecutionFailedAt(
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ Throwable error,
+ @Nullable String problemCategory,
+ String timestamp)
+ throws Exception {
+ Event event = ExecutionLifecycleEvents.executionFailed(error, problemCategory);
+ reportChildExecution(
+ entityType,
+ entityName,
+ entityMetadata,
+ new EventContext(event.getType(), timestamp),
+ event);
+ }
+
/**
* Fans the report out to the current action execution's component listeners best-effort: a
* listener that throws is logged and skipped, so reporting never fails the caller.
*/
protected void reportChildExecution(
String entityType, String entityName, Map entityMetadata, Event event) {
+ reportChildExecution(
+ entityType, entityName, entityMetadata, new EventContext(event), event);
+ }
+
+ protected void reportChildExecution(
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ EventContext eventContext,
+ Event event) {
mailboxThreadChecker.run();
if (componentExecutionListeners == null) {
return;
}
for (ComponentExecutionListener listener : componentExecutionListeners) {
try {
- listener.onComponentExecution(entityType, entityName, entityMetadata, event);
+ listener.onComponentExecution(
+ entityType, entityName, entityMetadata, eventContext, event);
} catch (Exception | LinkageError e) {
LOG.warn(
"Component execution listener {} failed on a report for action '{}' ({})",
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/eventlog/EventLogWriter.java b/runtime/src/main/java/org/apache/flink/agents/runtime/eventlog/EventLogWriter.java
index abffc76d14..2ef843534d 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/eventlog/EventLogWriter.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/eventlog/EventLogWriter.java
@@ -99,10 +99,16 @@ public void appendBusinessEventAndFlush(
/** Appends and flushes an execution lifecycle Event when Trace recording is enabled. */
public void appendExecutionEventAndFlush(Event event, ExecutionTraceContext traceContext) {
+ appendExecutionEventAndFlush(new EventContext(event), event, traceContext);
+ }
+
+ /** Appends and flushes an execution lifecycle Event with its occurrence context. */
+ public void appendExecutionEventAndFlush(
+ EventContext eventContext, Event event, ExecutionTraceContext traceContext) {
if (!traceEnabled) {
return;
}
- appendAndFlush(new EventContext(event), event, traceContext);
+ appendAndFlush(eventContext, event, traceContext);
}
private void appendAndFlush(
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/lifecycle/ComponentExecutionListener.java b/runtime/src/main/java/org/apache/flink/agents/runtime/lifecycle/ComponentExecutionListener.java
index e7099fb66a..3a21b979d3 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/lifecycle/ComponentExecutionListener.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/lifecycle/ComponentExecutionListener.java
@@ -19,6 +19,7 @@
package org.apache.flink.agents.runtime.lifecycle;
import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
import java.util.Map;
@@ -52,8 +53,13 @@ public interface ComponentExecutionListener {
* org.apache.flink.agents.api.trace.ExecutionReporter.EntityTypes}.
* @param entityName the component entity name.
* @param entityMetadata the entity metadata reported with the execution.
+ * @param eventContext the occurrence context of the lifecycle event.
* @param event the lifecycle event, one of those produced by {@link ExecutionLifecycleEvents}.
*/
void onComponentExecution(
- String entityType, String entityName, Map entityMetadata, Event event);
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ EventContext eventContext,
+ Event event);
}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInActionMetrics.java b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInActionMetrics.java
index 2a518ef34d..84d908bf43 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInActionMetrics.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInActionMetrics.java
@@ -19,26 +19,124 @@
package org.apache.flink.agents.runtime.metrics;
+import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Histogram;
import org.apache.flink.metrics.Meter;
-/**
- * ActionMetricGroup class extends FlinkAgentsMetricGroupImpl and is used to monitor and measure the
- * performance metrics of executing actions. It provides metrics for the total number of actions
- * executed and the number of actions executed per second.
- */
+import java.util.HashMap;
+import java.util.Map;
+import java.util.OptionalLong;
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongSupplier;
+
+/** Tracks execution rate, scheduling latency, and current task/execution counts for one Action. */
public class BuiltInActionMetrics {
+ static final String ACTION_SCHEDULING_LATENCY_MS = "actionSchedulingLatencyMs";
+ static final String ACTION_EXECUTION_LATENCY_MS = "actionExecutionLatencyMs";
+ static final String NUM_PENDING_ACTION_TASKS = "numOfPendingActionTasks";
+ static final String NUM_ACTIVE_ACTION_EXECUTIONS = "numOfActiveActionExecutions";
+
private final Meter numOfActionsExecutedPerSec;
+ private final Histogram schedulingLatencyHistogram;
+ private final Histogram executionLatencyHistogram;
+ private final CurrentCountGauge pendingActionTasks;
+ private final CurrentCountGauge activeActionExecutions;
+ private final LongSupplier nanoTime;
+
+ private final Map initialTaskEnqueueNanos = new HashMap<>();
+ private final Map activeExecutions = new HashMap<>();
public BuiltInActionMetrics(FlinkAgentsMetricGroupImpl parentMetricGroup) {
+ this(parentMetricGroup, System::nanoTime);
+ }
+
+ BuiltInActionMetrics(FlinkAgentsMetricGroupImpl parentMetricGroup, LongSupplier nanoTime) {
Counter numOfActionsExecuted = parentMetricGroup.getCounter("numOfActionsExecuted");
this.numOfActionsExecutedPerSec =
parentMetricGroup.getMeter("numOfActionsExecutedPerSec", numOfActionsExecuted);
+ this.schedulingLatencyHistogram =
+ parentMetricGroup.getHistogram(ACTION_SCHEDULING_LATENCY_MS);
+ this.executionLatencyHistogram =
+ parentMetricGroup.getHistogram(ACTION_EXECUTION_LATENCY_MS);
+ this.pendingActionTasks =
+ new CurrentCountGauge(parentMetricGroup, NUM_PENDING_ACTION_TASKS);
+ this.activeActionExecutions =
+ new CurrentCountGauge(parentMetricGroup, NUM_ACTIVE_ACTION_EXECUTIONS);
+ this.nanoTime = nanoTime;
}
/** Marks that an action has finished executing. */
public void markActionExecuted() {
numOfActionsExecutedPerSec.markEvent();
}
+
+ void actionTaskEnqueued(String executionId, boolean executionStarted) {
+ pendingActionTasks.increment();
+ if (!executionStarted && !isBlank(executionId)) {
+ initialTaskEnqueueNanos.putIfAbsent(executionId, nanoTime.getAsLong());
+ }
+ }
+
+ void actionTaskDequeued(String executionId, boolean executionStarted) {
+ pendingActionTasks.decrement();
+ if (executionStarted || isBlank(executionId)) {
+ return;
+ }
+
+ Long enqueueNanos = initialTaskEnqueueNanos.remove(executionId);
+ if (enqueueNanos != null) {
+ schedulingLatencyHistogram.update(
+ TimeUnit.NANOSECONDS.toMillis(
+ Math.max(0L, nanoTime.getAsLong() - enqueueNanos)));
+ }
+ }
+
+ void restoreActionTask(String executionId, boolean executionStarted) {
+ pendingActionTasks.increment();
+ if (executionStarted
+ && !isBlank(executionId)
+ && activeExecutions.putIfAbsent(executionId, OptionalLong.empty()) == null) {
+ activeActionExecutions.increment();
+ }
+ }
+
+ void executionEventObserved(Event event, ExecutionTraceContext traceContext) {
+ String executionId = traceContext.getExecutionId();
+ if (isBlank(executionId)) {
+ return;
+ }
+
+ if (ExecutionLifecycleEvents.EXECUTION_STARTED_EVENT_TYPE.equals(event.getType())) {
+ if (activeExecutions.putIfAbsent(executionId, OptionalLong.of(nanoTime.getAsLong()))
+ == null) {
+ activeActionExecutions.increment();
+ }
+ return;
+ }
+
+ if (!ExecutionLifecycleEvents.EXECUTION_FINISHED_EVENT_TYPE.equals(event.getType())
+ && !ExecutionLifecycleEvents.EXECUTION_FAILED_EVENT_TYPE.equals(event.getType())
+ && !ExecutionLifecycleEvents.EXECUTION_REUSED_EVENT_TYPE.equals(event.getType())) {
+ return;
+ }
+
+ OptionalLong startNanos = activeExecutions.remove(executionId);
+ if (startNanos == null) {
+ return;
+ }
+ activeActionExecutions.decrement();
+ if (startNanos.isPresent()) {
+ executionLatencyHistogram.update(
+ TimeUnit.NANOSECONDS.toMillis(
+ Math.max(0L, nanoTime.getAsLong() - startNanos.getAsLong())));
+ }
+ }
+
+ private static boolean isBlank(String value) {
+ return value == null || value.isBlank();
+ }
}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInExecutionMetrics.java b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInExecutionMetrics.java
new file mode 100644
index 0000000000..4bb0378e75
--- /dev/null
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInExecutionMetrics.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
+import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.format.DateTimeParseException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Predicate;
+
+/** Derives built-in LLM and Tool metrics from execution lifecycle events. */
+final class BuiltInExecutionMetrics {
+
+ private final FlinkAgentsMetricGroupImpl agentMetricGroup;
+ private final Map metricRecordersByEntityType;
+ private final Map> startTimesByActionExecutionId = new HashMap<>();
+
+ BuiltInExecutionMetrics(
+ FlinkAgentsMetricGroupImpl agentMetricGroup, Predicate isRegisteredTool) {
+ this.agentMetricGroup = agentMetricGroup;
+ ExecutionMetricRecorder llmMetricRecorder = new LlmExecutionMetricRecorder();
+ ExecutionMetricRecorder toolMetricRecorder =
+ new ToolExecutionMetricRecorder(isRegisteredTool);
+ this.metricRecordersByEntityType =
+ Map.of(
+ llmMetricRecorder.entityType(),
+ llmMetricRecorder,
+ toolMetricRecorder.entityType(),
+ toolMetricRecorder);
+ }
+
+ void executionEventObserved(
+ String actionName,
+ EventContext eventContext,
+ Event event,
+ ExecutionTraceContext traceContext) {
+ ExecutionMetricRecorder recorder =
+ metricRecordersByEntityType.get(traceContext.getEntityType());
+ if (isBlank(actionName) || recorder == null) {
+ return;
+ }
+
+ String executionId = traceContext.getExecutionId();
+ String actionExecutionId = traceContext.getParentExecutionId();
+ if (ExecutionLifecycleEvents.EXECUTION_STARTED_EVENT_TYPE.equals(event.getType())) {
+ Instant startTime = parseTimestamp(eventContext);
+ if (startTime != null && !isBlank(actionExecutionId) && !isBlank(executionId)) {
+ startTimesByActionExecutionId
+ .computeIfAbsent(actionExecutionId, ignored -> new HashMap<>())
+ .putIfAbsent(executionId, startTime);
+ }
+ return;
+ }
+
+ boolean succeeded =
+ ExecutionLifecycleEvents.EXECUTION_FINISHED_EVENT_TYPE.equals(event.getType());
+ boolean failed =
+ ExecutionLifecycleEvents.EXECUTION_FAILED_EVENT_TYPE.equals(event.getType());
+ if (!succeeded && !failed) {
+ return;
+ }
+
+ Instant startTime = removeExecutionStart(actionExecutionId, executionId);
+ Instant terminalTime = parseTimestamp(eventContext);
+ Long latencyMs = latencyBetween(startTime, terminalTime);
+
+ FlinkAgentsMetricGroupImpl actionMetricGroup =
+ agentMetricGroup.getSubGroup("action", actionName);
+ ExecutionMetricRecorder.Outcome outcome =
+ succeeded
+ ? ExecutionMetricRecorder.Outcome.SUCCEEDED
+ : ExecutionMetricRecorder.Outcome.FAILED;
+ recorder.record(actionMetricGroup, traceContext, outcome, latencyMs);
+ }
+
+ void actionExecutionTerminated(String actionExecutionId) {
+ if (!isBlank(actionExecutionId)) {
+ startTimesByActionExecutionId.remove(actionExecutionId);
+ }
+ }
+
+ private Instant removeExecutionStart(String actionExecutionId, String executionId) {
+ if (isBlank(actionExecutionId) || isBlank(executionId)) {
+ return null;
+ }
+
+ Map actionExecutionStarts =
+ startTimesByActionExecutionId.get(actionExecutionId);
+ if (actionExecutionStarts == null) {
+ return null;
+ }
+
+ Instant startTime = actionExecutionStarts.remove(executionId);
+ if (actionExecutionStarts.isEmpty()) {
+ startTimesByActionExecutionId.remove(actionExecutionId);
+ }
+ return startTime;
+ }
+
+ private static Instant parseTimestamp(EventContext eventContext) {
+ try {
+ return Instant.parse(eventContext.getTimestamp());
+ } catch (DateTimeParseException | NullPointerException ignored) {
+ return null;
+ }
+ }
+
+ private static Long latencyBetween(Instant startTime, Instant terminalTime) {
+ if (startTime == null || terminalTime == null) {
+ return null;
+ }
+ return Math.max(0L, Duration.between(startTime, terminalTime).toMillis());
+ }
+
+ private static boolean isBlank(String value) {
+ return value == null || value.isBlank();
+ }
+}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInInputRunMetrics.java b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInInputRunMetrics.java
new file mode 100644
index 0000000000..0b50f7c34b
--- /dev/null
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInInputRunMetrics.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Histogram;
+
+import javax.annotation.Nullable;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongSupplier;
+
+/**
+ * Tracks input-run outcomes and latency on the operator mailbox thread.
+ *
+ * A run is successful when it reaches the operator's run-completion boundary and failed when an
+ * unhandled exception terminates it. Handled or recovered execution failures do not change the
+ * final run outcome. End-to-end latency starts when the input enters the operator and is split into
+ * queueing and processing latency at the input-run start boundary.
+ *
+ *
Tracking state is intentionally process-local because Flink metrics reset when the task
+ * restarts. Latency and outcome samples for runs already in flight when a task is restored are
+ * excluded because their original timestamps are unavailable. Current-count gauges are rebuilt from
+ * operator state.
+ */
+final class BuiltInInputRunMetrics {
+
+ static final String NUM_INPUT_RUNS_SUCCEEDED = "numOfInputRunsSucceeded";
+ static final String NUM_INPUT_RUNS_FAILED = "numOfInputRunsFailed";
+ static final String INPUT_RUN_LATENCY_MS = "inputRunLatencyMs";
+ static final String INPUT_RUN_QUEUE_LATENCY_MS = "inputRunQueueLatencyMs";
+ static final String INPUT_RUN_PROCESSING_LATENCY_MS = "inputRunProcessingLatencyMs";
+ static final String NUM_PENDING_INPUT_EVENTS = "numOfPendingInputEvents";
+ static final String NUM_ACTIVE_INPUT_RUNS = "numOfActiveInputRuns";
+
+ private final Counter succeededCounter;
+ private final Counter failedCounter;
+ private final Histogram latencyHistogram;
+ private final Histogram queueLatencyHistogram;
+ private final Histogram processingLatencyHistogram;
+ private final CurrentCountGauge pendingInputEvents;
+ private final CurrentCountGauge activeInputRuns;
+ private final LongSupplier nanoTime;
+
+ private final Map receivedInputNanos = new HashMap<>();
+ private final Map activeRunTimings = new HashMap<>();
+ private final Set activeInputRunIds = new HashSet<>();
+ private long unidentifiedRestoredActiveRuns;
+
+ BuiltInInputRunMetrics(FlinkAgentsMetricGroupImpl metricGroup, LongSupplier nanoTime) {
+ this.succeededCounter = metricGroup.getCounter(NUM_INPUT_RUNS_SUCCEEDED);
+ this.failedCounter = metricGroup.getCounter(NUM_INPUT_RUNS_FAILED);
+ this.latencyHistogram = metricGroup.getHistogram(INPUT_RUN_LATENCY_MS);
+ this.queueLatencyHistogram = metricGroup.getHistogram(INPUT_RUN_QUEUE_LATENCY_MS);
+ this.processingLatencyHistogram = metricGroup.getHistogram(INPUT_RUN_PROCESSING_LATENCY_MS);
+ this.pendingInputEvents = new CurrentCountGauge(metricGroup, NUM_PENDING_INPUT_EVENTS);
+ this.activeInputRuns = new CurrentCountGauge(metricGroup, NUM_ACTIVE_INPUT_RUNS);
+ this.nanoTime = nanoTime;
+ }
+
+ void inputEventReceived(Event inputEvent) {
+ receivedInputNanos.putIfAbsent(eventId(inputEvent), nanoTime.getAsLong());
+ }
+
+ void inputEventFailed(Event inputEvent) {
+ String inputEventId = eventId(inputEvent);
+ Long receivedNanos = receivedInputNanos.remove(inputEventId);
+ if (receivedNanos != null) {
+ recordTerminal(true, receivedNanos);
+ }
+ }
+
+ void inputRunStarted(Event inputEvent, ExecutionTraceContext traceContext) {
+ String inputRunId = traceContext.getInputRunId();
+ if (inputRunId == null || !activeInputRunIds.add(inputRunId)) {
+ return;
+ }
+
+ activeInputRuns.increment();
+ long startedNanos = nanoTime.getAsLong();
+ String inputEventId = eventId(inputEvent);
+ Long receivedNanos = receivedInputNanos.remove(inputEventId);
+ if (receivedNanos != null) {
+ queueLatencyHistogram.update(elapsedMillis(receivedNanos, startedNanos));
+ }
+ activeRunTimings.put(inputRunId, new RunTiming(receivedNanos, startedNanos));
+ }
+
+ void inputRunCompleted(String inputRunId) {
+ finish(inputRunId, false);
+ }
+
+ void inputRunFailed(String inputRunId) {
+ finish(inputRunId, true);
+ }
+
+ void pendingInputEventEnqueued() {
+ pendingInputEvents.increment();
+ }
+
+ void pendingInputEventDequeued() {
+ pendingInputEvents.decrement();
+ }
+
+ void restorePendingInputEvents(long count) {
+ pendingInputEvents.set(count);
+ }
+
+ void restoreActiveInputRuns(long count) {
+ unidentifiedRestoredActiveRuns = Math.max(0L, count);
+ activeInputRuns.set(unidentifiedRestoredActiveRuns + activeInputRunIds.size());
+ }
+
+ void identifyRestoredActiveInputRun(String inputRunId) {
+ if (inputRunId != null
+ && activeInputRunIds.add(inputRunId)
+ && unidentifiedRestoredActiveRuns > 0L) {
+ unidentifiedRestoredActiveRuns--;
+ }
+ }
+
+ private void finish(String inputRunId, boolean failed) {
+ boolean activeRunFinished = inputRunId != null && activeInputRunIds.remove(inputRunId);
+ if (!activeRunFinished && inputRunId == null && unidentifiedRestoredActiveRuns > 0L) {
+ unidentifiedRestoredActiveRuns--;
+ activeRunFinished = true;
+ }
+ if (activeRunFinished) {
+ activeInputRuns.decrement();
+ }
+
+ RunTiming timing = inputRunId == null ? null : activeRunTimings.remove(inputRunId);
+ if (timing == null) {
+ return;
+ }
+
+ long terminalNanos = nanoTime.getAsLong();
+ recordOutcome(failed);
+ if (timing.receivedNanos != null) {
+ latencyHistogram.update(elapsedMillis(timing.receivedNanos, terminalNanos));
+ }
+ processingLatencyHistogram.update(elapsedMillis(timing.startedNanos, terminalNanos));
+ }
+
+ private void recordTerminal(boolean failed, long startNanos) {
+ recordOutcome(failed);
+ latencyHistogram.update(elapsedMillis(startNanos, nanoTime.getAsLong()));
+ }
+
+ private void recordOutcome(boolean failed) {
+ if (failed) {
+ failedCounter.inc();
+ } else {
+ succeededCounter.inc();
+ }
+ }
+
+ private static String eventId(Event event) {
+ return event.getId().toString();
+ }
+
+ private static long elapsedMillis(long startNanos, long endNanos) {
+ return TimeUnit.NANOSECONDS.toMillis(Math.max(0L, endNanos - startNanos));
+ }
+
+ private static final class RunTiming {
+ @Nullable private final Long receivedNanos;
+ private final long startedNanos;
+
+ private RunTiming(@Nullable Long receivedNanos, long startedNanos) {
+ this.receivedNanos = receivedNanos;
+ this.startedNanos = startedNanos;
+ }
+ }
+}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInMetrics.java b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInMetrics.java
index 8e3a17cb81..7e39c89fd5 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInMetrics.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/BuiltInMetrics.java
@@ -19,20 +19,28 @@
package org.apache.flink.agents.runtime.metrics;
+import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
+import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
+import org.apache.flink.agents.api.trace.ExecutionReporter;
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
import org.apache.flink.agents.plan.AgentPlan;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Meter;
import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Predicate;
/**
* Represents a group of built-in metrics for monitoring the performance and behavior of a flink
- * agent job. This class is responsible for collecting and managing various metrics such as the
- * number of events processed, the number of actions being executed, and the number of actions
- * executed per second.
+ * agent job. This class is responsible for collecting and managing input-run, event, and action
+ * metrics.
*/
public class BuiltInMetrics {
+ private final FlinkAgentsMetricGroupImpl parentMetricGroup;
+
private final Meter numOfEventProcessedPerSec;
private final Meter numOfActionsExecutedPerSec;
@@ -41,9 +49,17 @@ public class BuiltInMetrics {
private final Counter eventLogWriteFailures;
- private final HashMap actionMetricGroups;
+ private final BuiltInInputRunMetrics inputRunMetrics;
+
+ private final BuiltInExecutionMetrics executionMetrics;
+
+ private final Map actionMetricGroups;
- public BuiltInMetrics(FlinkAgentsMetricGroupImpl parentMetricGroup, AgentPlan agentPlan) {
+ public BuiltInMetrics(
+ FlinkAgentsMetricGroupImpl parentMetricGroup,
+ AgentPlan agentPlan,
+ Predicate isRegisteredTool) {
+ this.parentMetricGroup = parentMetricGroup;
Counter numOfEventsProcessed = parentMetricGroup.getCounter("numOfEventProcessed");
this.numOfEventProcessedPerSec =
parentMetricGroup.getMeter("numOfEventProcessedPerSec", numOfEventsProcessed);
@@ -54,12 +70,12 @@ public BuiltInMetrics(FlinkAgentsMetricGroupImpl parentMetricGroup, AgentPlan ag
this.eventLogTruncatedEvents = parentMetricGroup.getCounter("eventLogTruncatedEvents");
this.eventLogWriteFailures = parentMetricGroup.getCounter("eventLogWriteFailures");
+ this.inputRunMetrics = new BuiltInInputRunMetrics(parentMetricGroup, System::nanoTime);
+ this.executionMetrics = new BuiltInExecutionMetrics(parentMetricGroup, isRegisteredTool);
this.actionMetricGroups = new HashMap<>();
for (String actionName : agentPlan.getActions().keySet()) {
- actionMetricGroups.put(
- actionName,
- new BuiltInActionMetrics(parentMetricGroup.getSubGroup("action", actionName)));
+ actionMetricGroups.put(actionName, createActionMetrics(actionName));
}
}
@@ -68,13 +84,84 @@ public void markEventProcessed() {
numOfEventProcessedPerSec.markEvent();
}
- /**
- * Marks that an action has finished executing. Decrements the executing actions counter and
- * marks an event on the executed meter.
- */
+ /** Marks that an action has finished executing. */
public void markActionExecuted(String actionName) {
numOfActionsExecutedPerSec.markEvent();
- actionMetricGroups.get(actionName).markActionExecuted();
+ actionMetrics(actionName).markActionExecuted();
+ }
+
+ public void markInputEventReceived(Event inputEvent) {
+ inputRunMetrics.inputEventReceived(inputEvent);
+ }
+
+ public void markInputEventFailed(Event inputEvent) {
+ inputRunMetrics.inputEventFailed(inputEvent);
+ }
+
+ public void markInputRunStarted(Event inputEvent, ExecutionTraceContext traceContext) {
+ inputRunMetrics.inputRunStarted(inputEvent, traceContext);
+ }
+
+ public void markInputRunCompleted(String inputRunId) {
+ inputRunMetrics.inputRunCompleted(inputRunId);
+ }
+
+ public void markInputRunFailed(String inputRunId) {
+ inputRunMetrics.inputRunFailed(inputRunId);
+ }
+
+ public void markPendingInputEventEnqueued() {
+ inputRunMetrics.pendingInputEventEnqueued();
+ }
+
+ public void markPendingInputEventDequeued() {
+ inputRunMetrics.pendingInputEventDequeued();
+ }
+
+ public void restorePendingInputEvents(long count) {
+ inputRunMetrics.restorePendingInputEvents(count);
+ }
+
+ public void restoreActiveInputRuns(long count) {
+ inputRunMetrics.restoreActiveInputRuns(count);
+ }
+
+ public void markActionTaskEnqueued(
+ ExecutionTraceContext traceContext, boolean executionStarted) {
+ actionMetrics(traceContext.getEntityName())
+ .actionTaskEnqueued(traceContext.getExecutionId(), executionStarted);
+ }
+
+ public void markActionTaskDequeued(
+ ExecutionTraceContext traceContext, boolean executionStarted) {
+ actionMetrics(traceContext.getEntityName())
+ .actionTaskDequeued(traceContext.getExecutionId(), executionStarted);
+ }
+
+ public void restoreActionTask(ExecutionTraceContext traceContext, boolean executionStarted) {
+ inputRunMetrics.identifyRestoredActiveInputRun(traceContext.getInputRunId());
+ restoredActionMetrics(traceContext.getEntityName())
+ .restoreActionTask(traceContext.getExecutionId(), executionStarted);
+ }
+
+ public void markExecutionEvent(
+ String actionName, Event event, ExecutionTraceContext traceContext) {
+ markExecutionEvent(actionName, new EventContext(event), event, traceContext);
+ }
+
+ public void markExecutionEvent(
+ String actionName,
+ EventContext eventContext,
+ Event event,
+ ExecutionTraceContext traceContext) {
+ if (ExecutionReporter.EntityTypes.ACTION.equals(traceContext.getEntityType())) {
+ actionMetrics(actionName).executionEventObserved(event, traceContext);
+ if (isTerminalExecutionEvent(event)) {
+ executionMetrics.actionExecutionTerminated(traceContext.getExecutionId());
+ }
+ } else {
+ executionMetrics.executionEventObserved(actionName, eventContext, event, traceContext);
+ }
}
/** Returns the counter tracking event log truncation occurrences. */
@@ -86,4 +173,27 @@ public Counter getEventLogTruncatedEventsCounter() {
public Counter getEventLogWriteFailuresCounter() {
return eventLogWriteFailures;
}
+
+ private BuiltInActionMetrics actionMetrics(String actionName) {
+ BuiltInActionMetrics actionMetrics = actionMetricGroups.get(actionName);
+ if (actionMetrics == null) {
+ throw new IllegalArgumentException("Unknown action: " + actionName);
+ }
+ return actionMetrics;
+ }
+
+ private BuiltInActionMetrics restoredActionMetrics(String actionName) {
+ return actionMetricGroups.computeIfAbsent(actionName, this::createActionMetrics);
+ }
+
+ private BuiltInActionMetrics createActionMetrics(String actionName) {
+ return new BuiltInActionMetrics(
+ parentMetricGroup.getSubGroup("action", actionName), System::nanoTime);
+ }
+
+ private static boolean isTerminalExecutionEvent(Event event) {
+ return ExecutionLifecycleEvents.EXECUTION_FINISHED_EVENT_TYPE.equals(event.getType())
+ || ExecutionLifecycleEvents.EXECUTION_FAILED_EVENT_TYPE.equals(event.getType())
+ || ExecutionLifecycleEvents.EXECUTION_REUSED_EVENT_TYPE.equals(event.getType());
+ }
}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/CurrentCountGauge.java b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/CurrentCountGauge.java
new file mode 100644
index 0000000000..3deb668861
--- /dev/null
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/CurrentCountGauge.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+/** Maintains a non-negative current-count gauge on the operator mailbox thread. */
+final class CurrentCountGauge {
+
+ private final UpdatableGaugeImpl gauge;
+ private long value;
+
+ @SuppressWarnings("unchecked")
+ CurrentCountGauge(FlinkAgentsMetricGroupImpl metricGroup, String name) {
+ this.gauge = (UpdatableGaugeImpl) metricGroup.getGauge(name);
+ update(0L);
+ }
+
+ void increment() {
+ update(value + 1L);
+ }
+
+ void decrement() {
+ update(Math.max(0L, value - 1L));
+ }
+
+ void set(long value) {
+ update(Math.max(0L, value));
+ }
+
+ private void update(long value) {
+ this.value = value;
+ gauge.update(value);
+ }
+}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/ExecutionMetricRecorder.java b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/ExecutionMetricRecorder.java
new file mode 100644
index 0000000000..dc68d704e6
--- /dev/null
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/ExecutionMetricRecorder.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
+
+import javax.annotation.Nullable;
+
+/** Records metrics for one execution entity type from a terminal lifecycle sample. */
+interface ExecutionMetricRecorder {
+
+ enum Outcome {
+ SUCCEEDED,
+ FAILED
+ }
+
+ /** Returns the execution entity type consumed by this recorder. */
+ String entityType();
+
+ /** Records a terminal execution whose lifecycle has already been resolved by the caller. */
+ void record(
+ FlinkAgentsMetricGroupImpl actionMetricGroup,
+ ExecutionTraceContext traceContext,
+ Outcome outcome,
+ @Nullable Long latencyMs);
+}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/LlmExecutionMetricRecorder.java b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/LlmExecutionMetricRecorder.java
new file mode 100644
index 0000000000..7347697b4c
--- /dev/null
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/LlmExecutionMetricRecorder.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+import org.apache.flink.agents.api.trace.ExecutionReporter;
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
+import org.apache.flink.metrics.Histogram;
+
+/** Records framework-observed LLM invocation outcomes and latency by ChatModel resource. */
+final class LlmExecutionMetricRecorder implements ExecutionMetricRecorder {
+
+ static final String NUM_LLM_CALLS_SUCCEEDED = "numOfLlmCallsSucceeded";
+ static final String NUM_LLM_CALLS_FAILED = "numOfLlmCallsFailed";
+ static final String LLM_CALL_LATENCY_MS = "llmCallLatencyMs";
+
+ @Override
+ public String entityType() {
+ return ExecutionReporter.EntityTypes.LLM;
+ }
+
+ @Override
+ public void record(
+ FlinkAgentsMetricGroupImpl actionMetricGroup,
+ ExecutionTraceContext traceContext,
+ Outcome outcome,
+ Long latencyMs) {
+ String entityName = traceContext.getEntityName();
+ if (entityName == null || entityName.isBlank()) {
+ return;
+ }
+ FlinkAgentsMetricGroupImpl modelResourceMetricGroup =
+ actionMetricGroup.getSubGroup("model_resource", entityName);
+ modelResourceMetricGroup
+ .getCounter(
+ outcome == Outcome.SUCCEEDED
+ ? NUM_LLM_CALLS_SUCCEEDED
+ : NUM_LLM_CALLS_FAILED)
+ .inc();
+ Histogram latencyHistogram = modelResourceMetricGroup.getHistogram(LLM_CALL_LATENCY_MS);
+ if (latencyMs != null) {
+ latencyHistogram.update(latencyMs);
+ }
+ }
+}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/ToolExecutionMetricRecorder.java b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/ToolExecutionMetricRecorder.java
new file mode 100644
index 0000000000..f2b0bc7eba
--- /dev/null
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/metrics/ToolExecutionMetricRecorder.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+import org.apache.flink.agents.api.trace.ExecutionReporter;
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
+import org.apache.flink.agents.api.trace.ToolExecutionMetadataKeys;
+import org.apache.flink.metrics.Histogram;
+
+import java.util.Objects;
+import java.util.function.Predicate;
+
+/** Records Tool metrics and additional Skill and MCP projections. */
+final class ToolExecutionMetricRecorder implements ExecutionMetricRecorder {
+
+ static final String UNKNOWN_TOOL_NAME = "unknown";
+ static final String UNKNOWN_SKILL_NAME = "unknown";
+
+ static final String NUM_TOOL_CALLS_SUCCEEDED = "numOfToolCallsSucceeded";
+ static final String NUM_TOOL_CALLS_FAILED = "numOfToolCallsFailed";
+ static final String TOOL_CALL_LATENCY_MS = "toolCallLatencyMs";
+
+ static final String NUM_SKILL_LOADS = "numOfSkillLoads";
+ static final String SKILL_LOAD_LATENCY_MS = "skillLoadLatencyMs";
+
+ static final String NUM_MCP_TOOL_CALLS_SUCCEEDED = "numOfMcpToolCallsSucceeded";
+ static final String NUM_MCP_TOOL_CALLS_FAILED = "numOfMcpToolCallsFailed";
+ static final String MCP_TOOL_CALL_LATENCY_MS = "mcpToolCallLatencyMs";
+
+ private final Predicate isRegisteredTool;
+
+ ToolExecutionMetricRecorder(Predicate isRegisteredTool) {
+ this.isRegisteredTool = Objects.requireNonNull(isRegisteredTool);
+ }
+
+ @Override
+ public String entityType() {
+ return ExecutionReporter.EntityTypes.TOOL;
+ }
+
+ @Override
+ public void record(
+ FlinkAgentsMetricGroupImpl actionMetricGroup,
+ ExecutionTraceContext traceContext,
+ Outcome outcome,
+ Long latencyMs) {
+ String requestedToolName = traceContext.getEntityName();
+ String metricToolName =
+ !isBlank(requestedToolName) && isRegisteredTool.test(requestedToolName)
+ ? requestedToolName
+ : UNKNOWN_TOOL_NAME;
+ recordOutcome(
+ actionMetricGroup.getSubGroup("tool", metricToolName),
+ outcome,
+ NUM_TOOL_CALLS_SUCCEEDED,
+ NUM_TOOL_CALLS_FAILED,
+ TOOL_CALL_LATENCY_MS,
+ latencyMs);
+
+ String skillName = metadataValue(traceContext, ToolExecutionMetadataKeys.SKILL_NAME);
+ if (!isBlank(skillName)) {
+ String metricSkillName =
+ isRegisteredSkill(traceContext) ? skillName : UNKNOWN_SKILL_NAME;
+ FlinkAgentsMetricGroupImpl skillMetricGroup =
+ actionMetricGroup.getSubGroup("skill", metricSkillName);
+ skillMetricGroup.getCounter(NUM_SKILL_LOADS).inc();
+ updateLatency(skillMetricGroup.getHistogram(SKILL_LOAD_LATENCY_MS), latencyMs);
+ }
+
+ String mcpServer = metadataValue(traceContext, ToolExecutionMetadataKeys.MCP_SERVER);
+ if (!isBlank(mcpServer)) {
+ recordOutcome(
+ actionMetricGroup.getSubGroup("mcp_server", mcpServer),
+ outcome,
+ NUM_MCP_TOOL_CALLS_SUCCEEDED,
+ NUM_MCP_TOOL_CALLS_FAILED,
+ MCP_TOOL_CALL_LATENCY_MS,
+ latencyMs);
+ }
+ }
+
+ private static void recordOutcome(
+ FlinkAgentsMetricGroupImpl metricGroup,
+ Outcome outcome,
+ String succeededCounter,
+ String failedCounter,
+ String latencyHistogram,
+ Long latencyMs) {
+ metricGroup
+ .getCounter(outcome == Outcome.SUCCEEDED ? succeededCounter : failedCounter)
+ .inc();
+ updateLatency(metricGroup.getHistogram(latencyHistogram), latencyMs);
+ }
+
+ private static void updateLatency(Histogram histogram, Long latencyMs) {
+ if (latencyMs != null) {
+ histogram.update(latencyMs);
+ }
+ }
+
+ private static String metadataValue(ExecutionTraceContext traceContext, String metadataKey) {
+ Object value = traceContext.getEntityMetadata().get(metadataKey);
+ return value == null ? null : String.valueOf(value);
+ }
+
+ private static boolean isRegisteredSkill(ExecutionTraceContext traceContext) {
+ return Boolean.TRUE.equals(
+ traceContext.getEntityMetadata().get(ToolExecutionMetadataKeys.SKILL_REGISTERED));
+ }
+
+ private static boolean isBlank(String value) {
+ return value == null || value.isBlank();
+ }
+}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/operator/ActionExecutionOperator.java b/runtime/src/main/java/org/apache/flink/agents/runtime/operator/ActionExecutionOperator.java
index 4d8d5d84f3..ae24256f26 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/operator/ActionExecutionOperator.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/operator/ActionExecutionOperator.java
@@ -18,6 +18,7 @@
package org.apache.flink.agents.runtime.operator;
import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
import org.apache.flink.agents.api.OutputEvent;
import org.apache.flink.agents.api.agents.AgentExecutionOptions;
import org.apache.flink.agents.api.event.AgentRunBeginEvent;
@@ -211,7 +212,11 @@ public void open() throws Exception {
getRuntimeContext().getUserCodeClassLoader());
metricGroup = new FlinkAgentsMetricGroupImpl(getMetricGroup());
- builtInMetrics = new BuiltInMetrics(metricGroup, agentPlan);
+ builtInMetrics =
+ new BuiltInMetrics(
+ metricGroup,
+ agentPlan,
+ toolName -> resourceCache.hasResource(toolName, ResourceType.TOOL));
eventRouter.open(builtInMetrics);
@@ -246,7 +251,7 @@ public void open() throws Exception {
componentExecutionListeners = new ArrayList<>();
}
- registerEventLogListeners();
+ registerBuiltInLifecycleListeners();
registerSubagentSetups();
// init context manager for runner context creation and memory contexts
@@ -286,35 +291,47 @@ public void processElement(StreamRecord record) throws Exception {
if (record.hasTimestamp()) {
inputEvent.setSourceTimestamp(record.getTimestamp());
}
+ builtInMetrics.markInputEventReceived(inputEvent);
- eventRouter.getKeySegmentQueue().addKeyToLastSegment(getCurrentKey());
+ Object key = getCurrentKey();
+ try {
+ eventRouter.getKeySegmentQueue().addKeyToLastSegment(key);
- if (stateManager.hasMoreActionTasks()) {
- // If there are already actions being processed for the current key, the newly incoming
- // event should be queued and processed later. Therefore, we add it to
- // pendingInputEventsState.
- stateManager.addPendingInputEvent(inputEvent);
- } else {
- // Otherwise, the new event is processed immediately.
- processInputEvent(getCurrentKey(), inputEvent);
+ if (stateManager.hasMoreActionTasks()) {
+ // If there are already actions being processed for the current key, the newly
+ // incoming event should be queued and processed later. Therefore, we add it to
+ // pendingInputEventsState.
+ enqueuePendingInputEvent(inputEvent);
+ return;
+ }
+ } catch (Exception e) {
+ builtInMetrics.markInputEventFailed(inputEvent);
+ throw e;
}
+
+ // Otherwise, the new event is processed immediately. Its failures are attributed to the
+ // input run created by processInputEvent.
+ processInputEvent(key, inputEvent);
}
/** Resolves one context key for an input and reuses it for the entire agent run. */
private void processInputEvent(Object key, Event inputEvent) throws Exception {
- processEvent(key, resolveContextKey(key), inputEvent);
- }
-
- /**
- * Processes an incoming event for the given key and may submit a new mail
- * `tryProcessActionTaskForKey` to continue processing.
- */
- private void processEvent(Object key, String contextKey, Event event) throws Exception {
- processEvent(
- key,
- contextKey,
- event,
- ExecutionTraceContext.forInputRun(contextKey, agentPlan.getAgentName()));
+ final String contextKey;
+ final ExecutionTraceContext traceContext;
+ try {
+ contextKey = resolveContextKey(key);
+ traceContext = ExecutionTraceContext.forInputRun(contextKey, agentPlan.getAgentName());
+ } catch (Exception e) {
+ builtInMetrics.markInputEventFailed(inputEvent);
+ throw e;
+ }
+ builtInMetrics.markInputRunStarted(inputEvent, traceContext);
+ try {
+ processEvent(key, contextKey, inputEvent, traceContext);
+ } catch (Exception e) {
+ builtInMetrics.markInputRunFailed(traceContext.getInputRunId());
+ throw e;
+ }
}
private void processEvent(
@@ -355,7 +372,7 @@ private void processEvent(
List triggerActions = eventRouter.getActionsTriggeredBy(event);
if (triggerActions != null && !triggerActions.isEmpty()) {
for (Action triggerAction : triggerActions) {
- stateManager.addActionTask(
+ enqueueActionTask(
createActionTask(
key,
triggerAction,
@@ -373,7 +390,8 @@ private void processEvent(
if (isInputEvent) {
// If the event is an InputEvent, we submit a new mail to try processing the actions.
mailboxExecutor.submit(
- () -> tryProcessActionTaskForKey(key, contextKey), "process action task");
+ () -> tryProcessActionTaskForKey(key, contextKey, traceContext.getInputRunId()),
+ "process action task");
}
}
@@ -423,9 +441,10 @@ private void tryEmitAgentRunBeginEvent(
processEvent(key, contextKey, beginEvent, traceContext);
}
- private void tryProcessActionTaskForKey(Object key, String contextKey) {
+ private void tryProcessActionTaskForKey(
+ Object key, String contextKey, @Nullable String inputRunId) {
try {
- processActionTaskForKey(key, contextKey);
+ processActionTaskForKey(key, contextKey, inputRunId);
} catch (Throwable t) {
// MailboxExecutor.submit() stores task failures in its Future. Catch Throwable and
// rethrow via execute() so Errors fail the task instead of leaving the key in-flight.
@@ -438,26 +457,39 @@ private void tryProcessActionTaskForKey(Object key, String contextKey) {
}
}
- private void processActionTaskForKey(Object key, String contextKey) throws Exception {
- // 1. Get an action task for the key.
- setCurrentKey(key);
-
- ActionTask actionTask = stateManager.pollNextActionTask();
- if (actionTask == null) {
- int removedCount = stateManager.removeProcessingKey(key);
- checkState(
- removedCount == 1,
- "Current processing key count for key "
- + key
- + " should be 1, but got "
- + removedCount);
- checkState(
- eventRouter.getKeySegmentQueue().removeKey(key),
- "Current key" + key + " is missing from the segmentedQueue.");
- eventRouter.processEligibleWatermarks(super::processWatermark);
- return;
+ private void processActionTaskForKey(Object key, String contextKey, @Nullable String inputRunId)
+ throws Exception {
+ String currentInputRunId = inputRunId;
+ try {
+ // 1. Get an action task for the key.
+ setCurrentKey(key);
+
+ ActionTask actionTask = pollNextActionTask();
+ if (actionTask == null) {
+ int removedCount = stateManager.removeProcessingKey(key);
+ checkState(
+ removedCount == 1,
+ "Current processing key count for key "
+ + key
+ + " should be 1, but got "
+ + removedCount);
+ checkState(
+ eventRouter.getKeySegmentQueue().removeKey(key),
+ "Current key" + key + " is missing from the segmentedQueue.");
+ eventRouter.processEligibleWatermarks(super::processWatermark);
+ builtInMetrics.markInputRunCompleted(currentInputRunId);
+ return;
+ }
+ currentInputRunId = actionTask.getTraceContext().getInputRunId();
+ processActionTask(key, contextKey, actionTask);
+ } catch (Exception e) {
+ builtInMetrics.markInputRunFailed(currentInputRunId);
+ throw e;
}
+ }
+ private void processActionTask(Object key, String contextKey, ActionTask actionTask)
+ throws Exception {
// 2. Invoke the action task.
contextManager.createAndSetRunnerContext(
actionTask,
@@ -592,7 +624,7 @@ private void processActionTaskForKey(Object key, String contextKey) throws Excep
contextManager.transferContexts(actionTask, generatedActionTask, durableExecManager);
notifyActionTransferred(actionTask, generatedActionTask);
- stateManager.addActionTask(generatedActionTask);
+ enqueueActionTask(generatedActionTask);
}
// 3. Process the next InputEvent or next action task
@@ -615,7 +647,8 @@ private void processActionTaskForKey(Object key, String contextKey) throws Excep
eventRouter.getKeySegmentQueue().removeKey(key),
"Current key" + key + " is missing from the segmentedQueue.");
eventRouter.processEligibleWatermarks(super::processWatermark);
- Event pendingInputEvent = stateManager.pollNextPendingInputEvent();
+ builtInMetrics.markInputRunCompleted(actionTask.getTraceContext().getInputRunId());
+ Event pendingInputEvent = pollNextPendingInputEvent();
if (pendingInputEvent != null) {
processInputEvent(key, pendingInputEvent);
}
@@ -623,7 +656,10 @@ private void processActionTaskForKey(Object key, String contextKey) throws Excep
// If the current key has additional action tasks remaining, we should submit a new mail
// to continue processing them.
mailboxExecutor.submit(
- () -> tryProcessActionTaskForKey(key, contextKey), "process action task");
+ () ->
+ tryProcessActionTaskForKey(
+ key, contextKey, actionTask.getTraceContext().getInputRunId()),
+ "process action task");
}
}
@@ -762,8 +798,15 @@ private void notifyActionStarted(ActionTask actionTask) {
actionTask.markExecutionStartedEventEmitted();
}
- private void registerEventLogListeners() {
- addTaskLifecycleListener(new EventLogTaskLifecycleListener(executionEventLogger));
+ private void registerBuiltInLifecycleListeners() {
+ addTaskLifecycleListener(
+ new EventLogTaskLifecycleListener(
+ (eventContext, event, traceContext) ->
+ observeExecutionEvent(
+ traceContext.getEntityName(),
+ eventContext,
+ event,
+ traceContext)));
}
/**
@@ -774,11 +817,26 @@ private List createComponentListeners(ActionTask act
List listeners = new ArrayList<>();
listeners.add(
new EventLogComponentExecutionListener(
- actionTask.getTraceContext(), executionEventLogger));
+ actionTask.getTraceContext(),
+ (eventContext, event, traceContext) ->
+ observeExecutionEvent(
+ actionTask.getAction().getName(),
+ eventContext,
+ event,
+ traceContext)));
listeners.addAll(componentExecutionListeners);
return listeners;
}
+ private void observeExecutionEvent(
+ String actionName,
+ EventContext eventContext,
+ Event event,
+ ExecutionTraceContext traceContext) {
+ executionEventLogger.emit(eventContext, event, traceContext);
+ builtInMetrics.markExecutionEvent(actionName, eventContext, event, traceContext);
+ }
+
/**
* Materializes every sub-agent setup, in either language, and registers the ones that observe
* the task lifecycle. A Java setup joins this operator's listeners directly; a Python setup
@@ -919,8 +977,39 @@ static String resolveContextKey(
return pythonActionExecutor.resolveKeyText(key, pythonKeyIsPickled);
}
+ private void enqueuePendingInputEvent(Event event) throws Exception {
+ stateManager.addPendingInputEvent(event);
+ builtInMetrics.markPendingInputEventEnqueued();
+ }
+
+ @Nullable
+ private Event pollNextPendingInputEvent() throws Exception {
+ Event event = stateManager.pollNextPendingInputEvent();
+ if (event != null) {
+ builtInMetrics.markPendingInputEventDequeued();
+ }
+ return event;
+ }
+
+ private void enqueueActionTask(ActionTask actionTask) throws Exception {
+ stateManager.addActionTask(actionTask);
+ builtInMetrics.markActionTaskEnqueued(
+ actionTask.getTraceContext(), actionTask.hasExecutionStartedEventEmitted());
+ }
+
+ @Nullable
+ private ActionTask pollNextActionTask() throws Exception {
+ ActionTask actionTask = stateManager.pollNextActionTask();
+ if (actionTask != null) {
+ builtInMetrics.markActionTaskDequeued(
+ actionTask.getTraceContext(), actionTask.hasExecutionStartedEventEmitted());
+ }
+ return actionTask;
+ }
+
private void tryResumeProcessActionTasks() throws Exception {
Iterable keys = stateManager.getProcessingKeys();
+ long activeInputRuns = 0L;
if (keys != null) {
int maxParallelism = getRuntimeContext().getTaskInfo().getMaxNumberOfParallelSubtasks();
KeyGroupRange currentSubtaskKeyGroupRange =
@@ -941,20 +1030,34 @@ private void tryResumeProcessActionTasks() throws Exception {
// round so listeners observe a paired start/finished bracket as well.
notifyRecordStart(key);
mailboxExecutor.submit(
- () -> tryProcessActionTaskForKey(key, contextKey), "process action task");
+ () -> tryProcessActionTaskForKey(key, contextKey, null),
+ "process action task");
}
stateManager.replaceProcessingKeys(new ArrayList<>(ownedKeys));
+ activeInputRuns = ownedKeys.size();
}
+ builtInMetrics.restoreActiveInputRuns(activeInputRuns);
+
+ stateManager.forEachActionTaskKey(
+ getKeyedStateBackend(),
+ (key, state) -> {
+ for (ActionTask actionTask : state.get()) {
+ builtInMetrics.restoreActionTask(
+ actionTask.getTraceContext(),
+ actionTask.hasExecutionStartedEventEmitted());
+ }
+ });
+ long[] pendingInputEvents = {0L};
stateManager.forEachPendingInputEventKey(
getKeyedStateBackend(),
- (key, state) ->
- state.get()
- .forEach(
- event ->
- eventRouter
- .getKeySegmentQueue()
- .addKeyToLastSegment(key)));
+ (key, state) -> {
+ for (Event ignored : state.get()) {
+ eventRouter.getKeySegmentQueue().addKeyToLastSegment(key);
+ pendingInputEvents[0]++;
+ }
+ });
+ builtInMetrics.restorePendingInputEvents(pendingInputEvents[0]);
}
@VisibleForTesting
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/operator/OperatorStateManager.java b/runtime/src/main/java/org/apache/flink/agents/runtime/operator/OperatorStateManager.java
index 19f1f67cf6..ea0c23f21c 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/operator/OperatorStateManager.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/operator/OperatorStateManager.java
@@ -76,6 +76,7 @@
class OperatorStateManager {
static final String MESSAGE_SEQUENCE_NUMBER_STATE_NAME = "messageSequenceNumber";
+ static final String ACTION_TASK_STATE_NAME = "actionTasks";
static final String PENDING_INPUT_EVENT_STATE_NAME = "pendingInputEvents";
private ListState actionTasksKState;
@@ -126,7 +127,7 @@ void initializeKeyedStates(
actionTasksKState =
runtimeContext.getListState(
new ListStateDescriptor<>(
- "actionTasks", TypeInformation.of(ActionTask.class)));
+ ACTION_TASK_STATE_NAME, TypeInformation.of(ActionTask.class)));
pendingInputEventsKState =
runtimeContext.getListState(
new ListStateDescriptor<>(
@@ -327,4 +328,19 @@ void forEachPendingInputEventKey(
PENDING_INPUT_EVENT_STATE_NAME, TypeInformation.of(Event.class)),
function);
}
+
+ /** Applies a function to the pending-action-task list state for every key in the backend. */
+ @SuppressWarnings("unchecked")
+ void forEachActionTaskKey(
+ KeyedStateBackend> keyedStateBackend,
+ KeyedStateFunction> function)
+ throws Exception {
+ ((KeyedStateBackend) keyedStateBackend)
+ .applyToAllKeys(
+ VoidNamespace.INSTANCE,
+ VoidNamespaceSerializer.INSTANCE,
+ new ListStateDescriptor<>(
+ ACTION_TASK_STATE_NAME, TypeInformation.of(ActionTask.class)),
+ function);
+ }
}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/python/context/PythonRunnerContextImpl.java b/runtime/src/main/java/org/apache/flink/agents/runtime/python/context/PythonRunnerContextImpl.java
index 6335387258..679e43d45e 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/python/context/PythonRunnerContextImpl.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/python/context/PythonRunnerContextImpl.java
@@ -21,6 +21,7 @@
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
import org.apache.flink.agents.plan.AgentPlan;
import org.apache.flink.agents.runtime.ResourceCache;
@@ -71,16 +72,35 @@ public void sendEventJson(String eventJson) throws IOException {
sendEvent(event);
}
+ public void reportExecutionCreatedJson(
+ String entityType, String entityName, String entityMetadataJson) throws Exception {
+ reportExecutionCreated(entityType, entityName, parseEntityMetadata(entityMetadataJson));
+ }
+
public void reportExecutionStartedJson(
String entityType, String entityName, String entityMetadataJson) throws Exception {
reportExecutionStarted(entityType, entityName, parseEntityMetadata(entityMetadataJson));
}
+ public void reportExecutionStartedAtJson(
+ String entityType, String entityName, String entityMetadataJson, String timestamp)
+ throws Exception {
+ reportExecutionStartedAt(
+ entityType, entityName, parseEntityMetadata(entityMetadataJson), timestamp);
+ }
+
public void reportExecutionSucceededJson(
String entityType, String entityName, String entityMetadataJson) throws Exception {
reportExecutionSucceeded(entityType, entityName, parseEntityMetadata(entityMetadataJson));
}
+ public void reportExecutionSucceededAtJson(
+ String entityType, String entityName, String entityMetadataJson, String timestamp)
+ throws Exception {
+ reportExecutionSucceededAt(
+ entityType, entityName, parseEntityMetadata(entityMetadataJson), timestamp);
+ }
+
public void reportExecutionFailedJson(
String entityType,
String entityName,
@@ -96,6 +116,25 @@ public void reportExecutionFailedJson(
ExecutionLifecycleEvents.executionFailed(errorType, errorMessage, problemCategory));
}
+ public void reportExecutionFailedAtJson(
+ String entityType,
+ String entityName,
+ String entityMetadataJson,
+ String errorType,
+ String errorMessage,
+ String problemCategory,
+ String timestamp)
+ throws Exception {
+ Event event =
+ ExecutionLifecycleEvents.executionFailed(errorType, errorMessage, problemCategory);
+ reportChildExecution(
+ entityType,
+ entityName,
+ parseEntityMetadata(entityMetadataJson),
+ new EventContext(event.getType(), timestamp),
+ event);
+ }
+
public void checkMailboxThread() {
// this method will be invoked by PythonActionExecutor's python interpreter.
this.mailboxThreadChecker.run();
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/skill/LoadSkillTool.java b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/LoadSkillTool.java
index 7891780f6f..d7424cc380 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/skill/LoadSkillTool.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/skill/LoadSkillTool.java
@@ -69,9 +69,9 @@ public ToolType getToolType() {
public Map getToolExecutionMetadata(ToolParameters parameters) {
Map metadata = new LinkedHashMap<>();
if (parameters.hasParameter("name")) {
- metadata.put(
- ToolExecutionMetadataKeys.SKILL_NAME,
- String.valueOf(parameters.getParameter("name")));
+ String skillName = String.valueOf(parameters.getParameter("name"));
+ metadata.put(ToolExecutionMetadataKeys.SKILL_NAME, skillName);
+ metadata.put(ToolExecutionMetadataKeys.SKILL_REGISTERED, isRegisteredSkill(skillName));
}
metadata.put(
ToolExecutionMetadataKeys.SKILL_RESOURCE_PATH, normalizeResourcePath(parameters));
@@ -158,6 +158,15 @@ private SkillManager resolveSkillManager() throws Exception {
return null;
}
+ private boolean isRegisteredSkill(String skillName) {
+ try {
+ SkillManager manager = resolveSkillManager();
+ return manager != null && manager.getAllSkillNames().contains(skillName);
+ } catch (Exception ignored) {
+ return false;
+ }
+ }
+
private static String normalizeResourcePath(ToolParameters parameters) {
if (parameters == null || !parameters.hasParameter("path")) {
return "SKILL.md";
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/trace/EventLogComponentExecutionListener.java b/runtime/src/main/java/org/apache/flink/agents/runtime/trace/EventLogComponentExecutionListener.java
index 30f9e18e92..7c53d6fd28 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/trace/EventLogComponentExecutionListener.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/trace/EventLogComponentExecutionListener.java
@@ -19,6 +19,7 @@
package org.apache.flink.agents.runtime.trace;
import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
import org.apache.flink.agents.api.trace.ExecutionTraceContext;
import org.apache.flink.agents.runtime.lifecycle.ComponentExecutionListener;
@@ -32,8 +33,8 @@
/**
* Per-action-execution adapter that turns component execution reports into event log records under
* the action's trace context. Its bookkeeping never leaks across actions because each execution
- * gets its own instance, and the start/terminal pairing survives continuation task transfers
- * because the adapter is tied to the action execution rather than the individual task.
+ * gets its own instance, and lifecycle pairing survives continuation task transfers because the
+ * adapter is tied to the action execution rather than the individual task.
*/
@Internal
public final class EventLogComponentExecutionListener implements ComponentExecutionListener {
@@ -54,25 +55,37 @@ public EventLogComponentExecutionListener(
@Override
public void onComponentExecution(
- String entityType, String entityName, Map entityMetadata, Event event) {
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ EventContext eventContext,
+ Event event) {
ReportedExecutionKey key = new ReportedExecutionKey(entityType, entityName, entityMetadata);
ExecutionTraceContext reportTraceContext;
- if (ExecutionLifecycleEvents.EXECUTION_STARTED_EVENT_TYPE.equals(event.getType())) {
+ if (ExecutionLifecycleEvents.EXECUTION_CREATED_EVENT_TYPE.equals(event.getType())) {
reportTraceContext =
actionTraceContext.childExecution(
entityType, entityName, key.getEntityMetadata());
ExecutionTraceContext previous = activeReportedExecutions.put(key, reportTraceContext);
if (previous != null) {
LOG.debug(
- "Execution start report for {}:{} replaced an active report with the same metadata.",
+ "Execution creation report for {}:{} replaced an active report with the same metadata.",
entityType,
entityName);
}
+ } else if (ExecutionLifecycleEvents.EXECUTION_STARTED_EVENT_TYPE.equals(event.getType())) {
+ reportTraceContext = activeReportedExecutions.get(key);
+ if (reportTraceContext == null) {
+ reportTraceContext =
+ actionTraceContext.childExecution(
+ entityType, entityName, key.getEntityMetadata());
+ activeReportedExecutions.put(key, reportTraceContext);
+ }
} else {
reportTraceContext = activeReportedExecutions.remove(key);
if (reportTraceContext == null) {
LOG.debug(
- "Execution terminal report for {}:{} has no matching start report; emitting it with a new execution id.",
+ "Execution terminal report for {}:{} has no matching creation or start report; emitting it with a new execution id.",
entityType,
entityName);
reportTraceContext =
@@ -81,6 +94,6 @@ public void onComponentExecution(
}
}
- executionEventSink.emit(event, reportTraceContext);
+ executionEventSink.emit(eventContext, event, reportTraceContext);
}
}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/trace/EventLogTaskLifecycleListener.java b/runtime/src/main/java/org/apache/flink/agents/runtime/trace/EventLogTaskLifecycleListener.java
index 376d27f59c..7286826bb3 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/trace/EventLogTaskLifecycleListener.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/trace/EventLogTaskLifecycleListener.java
@@ -18,6 +18,8 @@
package org.apache.flink.agents.runtime.trace;
+import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
import org.apache.flink.agents.api.trace.ExecutionReporter;
import org.apache.flink.agents.runtime.lifecycle.TaskLifecycleListener;
@@ -39,26 +41,28 @@ public EventLogTaskLifecycleListener(ExecutionEventSink executionEventSink) {
@Override
public void onActionStarted(ActionTask task) {
- executionEventSink.emit(
- ExecutionLifecycleEvents.executionStarted(), task.getTraceContext());
+ emit(ExecutionLifecycleEvents.executionStarted(), task);
}
@Override
public void onActionReused(ActionTask task) {
- executionEventSink.emit(ExecutionLifecycleEvents.executionReused(), task.getTraceContext());
+ emit(ExecutionLifecycleEvents.executionReused(), task);
}
@Override
public void onActionFinished(ActionTask task) {
- executionEventSink.emit(
- ExecutionLifecycleEvents.executionFinished(), task.getTraceContext());
+ emit(ExecutionLifecycleEvents.executionFinished(), task);
}
@Override
public void onActionFailed(ActionTask task, Throwable error) {
- executionEventSink.emit(
+ emit(
ExecutionLifecycleEvents.executionFailed(
error, ExecutionReporter.ProblemCategories.ACTION_EXECUTION_FAILED),
- task.getTraceContext());
+ task);
+ }
+
+ private void emit(Event event, ActionTask task) {
+ executionEventSink.emit(new EventContext(event), event, task.getTraceContext());
}
}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/trace/ExecutionEventLogger.java b/runtime/src/main/java/org/apache/flink/agents/runtime/trace/ExecutionEventLogger.java
index 21e0921147..33253accd1 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/trace/ExecutionEventLogger.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/trace/ExecutionEventLogger.java
@@ -18,6 +18,7 @@
package org.apache.flink.agents.runtime.trace;
import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
import org.apache.flink.agents.api.trace.ExecutionTraceContext;
import org.apache.flink.agents.runtime.eventlog.EventLogWriter;
import org.apache.flink.annotation.Internal;
@@ -37,7 +38,7 @@ private ExecutionEventLogger(EventLogWriter eventLogWriter) {
}
@Override
- public void emit(Event event, ExecutionTraceContext traceContext) {
- eventLogWriter.appendExecutionEventAndFlush(event, traceContext);
+ public void emit(EventContext eventContext, Event event, ExecutionTraceContext traceContext) {
+ eventLogWriter.appendExecutionEventAndFlush(eventContext, event, traceContext);
}
}
diff --git a/runtime/src/main/java/org/apache/flink/agents/runtime/trace/ExecutionEventSink.java b/runtime/src/main/java/org/apache/flink/agents/runtime/trace/ExecutionEventSink.java
index 3eee1b24f2..6af5a2a05e 100644
--- a/runtime/src/main/java/org/apache/flink/agents/runtime/trace/ExecutionEventSink.java
+++ b/runtime/src/main/java/org/apache/flink/agents/runtime/trace/ExecutionEventSink.java
@@ -18,12 +18,13 @@
package org.apache.flink.agents.runtime.trace;
import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
import org.apache.flink.agents.api.trace.ExecutionTraceContext;
import org.apache.flink.annotation.Internal;
-/** Runtime bridge for emitting execution lifecycle events to the event log pipeline. */
+/** Runtime bridge for emitting execution lifecycle events to runtime observability consumers. */
@Internal
@FunctionalInterface
public interface ExecutionEventSink {
- void emit(Event event, ExecutionTraceContext traceContext);
+ void emit(EventContext eventContext, Event event, ExecutionTraceContext traceContext);
}
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/CompileUtilsTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/CompileUtilsTest.java
index b2dba2ea2d..c21cc6508d 100644
--- a/runtime/src/test/java/org/apache/flink/agents/runtime/CompileUtilsTest.java
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/CompileUtilsTest.java
@@ -208,6 +208,23 @@ void rejectsMalformedPythonInputType() {
.hasMessageContaining("contain 2 fields");
}
+ @Test
+ void testAgentNameUsedAsOperatorName() {
+ AgentPlan namedAgentPlan =
+ new AgentPlan(
+ TEST_AGENT_PLAN.getActions(),
+ TEST_AGENT_PLAN.getResourceProviders(),
+ TEST_AGENT_PLAN.getConfig(),
+ "test-agent");
+
+ assertThat(compileOperatorName(namedAgentPlan)).isEqualTo("test-agent");
+ }
+
+ @Test
+ void testMissingAgentNameUsesDefaultOperatorName() {
+ assertThat(compileOperatorName(TEST_AGENT_PLAN)).isEqualTo("action-execute-operator");
+ }
+
private static KeyedStream createPythonInputStream(TypeInformation> valueType) {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
TypeInformation inputType =
@@ -227,6 +244,14 @@ private static List getTestSequence() {
return testSequence;
}
+ private static String compileOperatorName(AgentPlan agentPlan) {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ KeyedStream keyedInputStream = env.fromData(1L).keyBy(value -> value);
+ return CompileUtils.connectToAgent(keyedInputStream, agentPlan)
+ .getTransformation()
+ .getName();
+ }
+
private static void checkResult(List resultList) {
List expectedResultList =
testSequence.stream().map(x -> (x + 1) * 2 + 1).collect(Collectors.toList());
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/context/RunnerContextImplExecutionReporterTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/context/RunnerContextImplExecutionReporterTest.java
index 68857cad01..687116cb75 100644
--- a/runtime/src/test/java/org/apache/flink/agents/runtime/context/RunnerContextImplExecutionReporterTest.java
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/context/RunnerContextImplExecutionReporterTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.agents.runtime.context;
import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
import org.apache.flink.agents.api.trace.ExecutionReporter;
import org.apache.flink.agents.plan.AgentPlan;
@@ -45,19 +46,35 @@ void reportsFanOutToComponentExecutionListeners() throws Exception {
new RunnerContextImpl(null, () -> {}, emptyAgentPlan(), null, "job");
switchToChatModelAction(runnerContext, List.of(listener));
- runnerContext.reportExecutionStarted(
- ExecutionReporter.EntityTypes.LLM, "model-a", Map.of("temperature", 0.7));
- runnerContext.reportExecutionSucceeded(
+ runnerContext.reportExecutionCreated(
ExecutionReporter.EntityTypes.LLM, "model-a", Map.of("temperature", 0.7));
+ runnerContext.reportExecutionStartedAt(
+ ExecutionReporter.EntityTypes.LLM,
+ "model-a",
+ Map.of("temperature", 0.7),
+ "2026-01-01T00:00:00.001Z");
+ runnerContext.reportExecutionSucceededAt(
+ ExecutionReporter.EntityTypes.LLM,
+ "model-a",
+ Map.of("temperature", 0.7),
+ "2026-01-01T00:00:00.025Z");
+ assertThat(listener.created).hasSize(1);
+ assertThat(listener.created.get(0).identity)
+ .containsExactly(
+ ExecutionReporter.EntityTypes.LLM, "model-a", Map.of("temperature", 0.7));
assertThat(listener.started).hasSize(1);
- assertThat(listener.started.get(0))
+ assertThat(listener.started.get(0).identity)
.containsExactly(
ExecutionReporter.EntityTypes.LLM, "model-a", Map.of("temperature", 0.7));
+ assertThat(listener.started.get(0).eventContext.getTimestamp())
+ .isEqualTo("2026-01-01T00:00:00.001Z");
assertThat(listener.succeeded).hasSize(1);
- assertThat(listener.succeeded.get(0))
+ assertThat(listener.succeeded.get(0).identity)
.containsExactly(
ExecutionReporter.EntityTypes.LLM, "model-a", Map.of("temperature", 0.7));
+ assertThat(listener.succeeded.get(0).eventContext.getTimestamp())
+ .isEqualTo("2026-01-01T00:00:00.025Z");
}
@Test
@@ -89,7 +106,7 @@ void failedReportResolvesRootCauseTypeAndMessage() throws Exception {
void throwingListenerNeverFailsTheReportingCall() throws Exception {
RecordingComponentListener receiver = new RecordingComponentListener();
ComponentExecutionListener thrower =
- (entityType, entityName, entityMetadata, event) -> {
+ (entityType, entityName, entityMetadata, eventContext, event) -> {
throw new IllegalStateException("listener boom");
};
RunnerContextImpl runnerContext =
@@ -149,18 +166,26 @@ void pythonReporterBridgePreservesMetadataAndPythonErrorFields() throws Exceptio
List.of(listener));
String metadata = "{\"toolCallId\":\"call-1\",\"toolType\":\"function\"}";
- runnerContext.reportExecutionStartedJson(
+ runnerContext.reportExecutionCreatedJson(
ExecutionReporter.EntityTypes.TOOL, "search", metadata);
- runnerContext.reportExecutionFailedJson(
+ runnerContext.reportExecutionStartedAtJson(
+ ExecutionReporter.EntityTypes.TOOL, "search", metadata, "2026-01-01T00:00:01.001Z");
+ runnerContext.reportExecutionFailedAtJson(
ExecutionReporter.EntityTypes.TOOL,
"search",
metadata,
"builtins.ValueError",
"bad response",
- ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED);
+ ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED,
+ "2026-01-01T00:00:01.125Z");
+ assertThat(listener.created).hasSize(1);
+ assertThat(listener.created.get(0).identity.get(2))
+ .asInstanceOf(org.assertj.core.api.InstanceOfAssertFactories.MAP)
+ .containsEntry("toolCallId", "call-1")
+ .containsEntry("toolType", "function");
assertThat(listener.started).hasSize(1);
- assertThat(listener.started.get(0).get(2))
+ assertThat(listener.started.get(0).identity.get(2))
.asInstanceOf(org.assertj.core.api.InstanceOfAssertFactories.MAP)
.containsEntry("toolCallId", "call-1")
.containsEntry("toolType", "function");
@@ -172,6 +197,9 @@ void pythonReporterBridgePreservesMetadataAndPythonErrorFields() throws Exceptio
assertThat(failure.errorMessage).isEqualTo("bad response");
assertThat(failure.problemCategory)
.isEqualTo(ExecutionReporter.ProblemCategories.TOOL_CALL_FAILED);
+ assertThat(listener.started.get(0).eventContext.getTimestamp())
+ .isEqualTo("2026-01-01T00:00:01.001Z");
+ assertThat(failure.eventContext.getTimestamp()).isEqualTo("2026-01-01T00:00:01.125Z");
}
private static void switchToChatModelAction(
@@ -192,8 +220,9 @@ private static AgentPlan emptyAgentPlan() {
/** Records the raw arguments of every component report it receives. */
private static final class RecordingComponentListener implements ComponentExecutionListener {
- private final List> started = new ArrayList<>();
- private final List> succeeded = new ArrayList<>();
+ private final List created = new ArrayList<>();
+ private final List started = new ArrayList<>();
+ private final List succeeded = new ArrayList<>();
private final List failed = new ArrayList<>();
@Override
@@ -201,16 +230,28 @@ public void onComponentExecution(
String entityType,
String entityName,
Map entityMetadata,
+ EventContext eventContext,
Event event) {
switch (event.getType()) {
+ case ExecutionLifecycleEvents.EXECUTION_CREATED_EVENT_TYPE:
+ created.add(
+ new RecordedComponentReport(
+ entityType, entityName, entityMetadata, eventContext));
+ break;
case ExecutionLifecycleEvents.EXECUTION_STARTED_EVENT_TYPE:
- started.add(List.of(entityType, entityName, entityMetadata));
+ started.add(
+ new RecordedComponentReport(
+ entityType, entityName, entityMetadata, eventContext));
break;
case ExecutionLifecycleEvents.EXECUTION_FINISHED_EVENT_TYPE:
- succeeded.add(List.of(entityType, entityName, entityMetadata));
+ succeeded.add(
+ new RecordedComponentReport(
+ entityType, entityName, entityMetadata, eventContext));
break;
case ExecutionLifecycleEvents.EXECUTION_FAILED_EVENT_TYPE:
- failed.add(new RecordedFailure(entityType, entityName, entityMetadata, event));
+ failed.add(
+ new RecordedFailure(
+ entityType, entityName, entityMetadata, eventContext, event));
break;
default:
throw new AssertionError("Unexpected event type " + event.getType());
@@ -218,10 +259,25 @@ public void onComponentExecution(
}
}
+ private static final class RecordedComponentReport {
+ private final List identity;
+ private final EventContext eventContext;
+
+ private RecordedComponentReport(
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ EventContext eventContext) {
+ this.identity = List.of(entityType, entityName, entityMetadata);
+ this.eventContext = eventContext;
+ }
+ }
+
private static final class RecordedFailure {
private final String entityType;
private final String entityName;
private final Map entityMetadata;
+ private final EventContext eventContext;
private final String errorType;
@Nullable private final String errorMessage;
@Nullable private final String problemCategory;
@@ -230,10 +286,12 @@ private RecordedFailure(
String entityType,
String entityName,
Map entityMetadata,
+ EventContext eventContext,
Event event) {
this.entityType = entityType;
this.entityName = entityName;
this.entityMetadata = entityMetadata;
+ this.eventContext = eventContext;
this.errorType = (String) event.getAttr("errorType");
this.errorMessage = (String) event.getAttr("errorMessage");
this.problemCategory =
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/eventlog/EventLogWriterTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/eventlog/EventLogWriterTest.java
index e682e35d7f..ca51d822e4 100644
--- a/runtime/src/test/java/org/apache/flink/agents/runtime/eventlog/EventLogWriterTest.java
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/eventlog/EventLogWriterTest.java
@@ -131,10 +131,12 @@ void traceEnabledWritesExecutionEventWithTraceContext() throws Exception {
ExecutionTraceContext.forInputRun("business-key", "agent")
.childExecution("action", "action1");
Event executionEvent = ExecutionLifecycleEvents.executionStarted();
+ EventContext eventContext =
+ new EventContext(executionEvent.getType(), "2026-01-01T00:00:00.123Z");
- writer.appendExecutionEventAndFlush(executionEvent, traceContext);
+ writer.appendExecutionEventAndFlush(eventContext, executionEvent, traceContext);
- verify(mockLogger).append(any(EventContext.class), eq(executionEvent), eq(traceContext));
+ verify(mockLogger).append(eq(eventContext), eq(executionEvent), eq(traceContext));
verify(mockLogger).flush();
}
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInActionMetricsTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInActionMetricsTest.java
new file mode 100644
index 0000000000..9e0e23c57e
--- /dev/null
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInActionMetricsTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
+import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class BuiltInActionMetricsTest {
+
+ private final AtomicLong nanoTime = new AtomicLong();
+ private FlinkAgentsMetricGroupImpl metricGroup;
+ private BuiltInActionMetrics metrics;
+
+ @BeforeEach
+ void setUp() {
+ metricGroup =
+ new FlinkAgentsMetricGroupImpl(
+ UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup());
+ metrics = new BuiltInActionMetrics(metricGroup, nanoTime::get);
+ }
+
+ @Test
+ void initialTaskRecordsSchedulingLatencyAndPendingCount() {
+ metrics.actionTaskEnqueued("execution", false);
+ assertThat(gauge(BuiltInActionMetrics.NUM_PENDING_ACTION_TASKS)).isEqualTo(1L);
+
+ nanoTime.set(25_000_000L);
+ metrics.actionTaskDequeued("execution", false);
+
+ assertThat(gauge(BuiltInActionMetrics.NUM_PENDING_ACTION_TASKS)).isZero();
+ assertThat(
+ metricGroup
+ .getHistogram(BuiltInActionMetrics.ACTION_SCHEDULING_LATENCY_MS)
+ .getStatistics()
+ .getMax())
+ .isEqualTo(25L);
+ }
+
+ @Test
+ void actionLifecycleRecordsExecutionLatency() {
+ ExecutionTraceContext action = actionExecution();
+
+ metrics.executionEventObserved(ExecutionLifecycleEvents.executionStarted(), action);
+ nanoTime.set(35_000_000L);
+ metrics.executionEventObserved(ExecutionLifecycleEvents.executionFinished(), action);
+
+ assertThat(
+ metricGroup
+ .getHistogram(BuiltInActionMetrics.ACTION_EXECUTION_LATENCY_MS)
+ .getStatistics()
+ .getMax())
+ .isEqualTo(35L);
+ assertThat(gauge(BuiltInActionMetrics.NUM_ACTIVE_ACTION_EXECUTIONS)).isZero();
+ }
+
+ @Test
+ void continuationCanBePendingWhileLogicalExecutionIsActive() {
+ ExecutionTraceContext action = actionExecution();
+ String executionId = action.getExecutionId();
+
+ metrics.executionEventObserved(ExecutionLifecycleEvents.executionStarted(), action);
+ metrics.actionTaskEnqueued(executionId, true);
+
+ assertThat(gauge(BuiltInActionMetrics.NUM_ACTIVE_ACTION_EXECUTIONS)).isEqualTo(1L);
+ assertThat(gauge(BuiltInActionMetrics.NUM_PENDING_ACTION_TASKS)).isEqualTo(1L);
+
+ metrics.actionTaskDequeued(executionId, true);
+ metrics.executionEventObserved(ExecutionLifecycleEvents.executionFinished(), action);
+
+ assertThat(gauge(BuiltInActionMetrics.NUM_ACTIVE_ACTION_EXECUTIONS)).isZero();
+ assertThat(gauge(BuiltInActionMetrics.NUM_PENDING_ACTION_TASKS)).isZero();
+ assertThat(
+ metricGroup
+ .getHistogram(BuiltInActionMetrics.ACTION_SCHEDULING_LATENCY_MS)
+ .getCount())
+ .isZero();
+ }
+
+ @Test
+ void restoredContinuationRebuildsPendingAndActiveGauges() {
+ ExecutionTraceContext action = actionExecution();
+
+ metrics.restoreActionTask(action.getExecutionId(), true);
+ assertThat(gauge(BuiltInActionMetrics.NUM_PENDING_ACTION_TASKS)).isEqualTo(1L);
+ assertThat(gauge(BuiltInActionMetrics.NUM_ACTIVE_ACTION_EXECUTIONS)).isEqualTo(1L);
+
+ metrics.actionTaskDequeued(action.getExecutionId(), true);
+ metrics.executionEventObserved(ExecutionLifecycleEvents.executionFinished(), action);
+
+ assertThat(gauge(BuiltInActionMetrics.NUM_PENDING_ACTION_TASKS)).isZero();
+ assertThat(gauge(BuiltInActionMetrics.NUM_ACTIVE_ACTION_EXECUTIONS)).isZero();
+ assertThat(
+ metricGroup
+ .getHistogram(BuiltInActionMetrics.ACTION_EXECUTION_LATENCY_MS)
+ .getCount())
+ .isZero();
+ }
+
+ @Test
+ void reusedRestoredExecutionEndsActiveGaugeWithoutLatency() {
+ ExecutionTraceContext action = actionExecution();
+
+ metrics.restoreActionTask(action.getExecutionId(), true);
+ metrics.actionTaskDequeued(action.getExecutionId(), true);
+ metrics.executionEventObserved(ExecutionLifecycleEvents.executionReused(), action);
+
+ assertThat(gauge(BuiltInActionMetrics.NUM_PENDING_ACTION_TASKS)).isZero();
+ assertThat(gauge(BuiltInActionMetrics.NUM_ACTIVE_ACTION_EXECUTIONS)).isZero();
+ assertThat(
+ metricGroup
+ .getHistogram(BuiltInActionMetrics.ACTION_EXECUTION_LATENCY_MS)
+ .getCount())
+ .isZero();
+ }
+
+ private long gauge(String name) {
+ return (Long) metricGroup.getGauge(name).getValue();
+ }
+
+ private static ExecutionTraceContext actionExecution() {
+ return ExecutionTraceContext.forAction(
+ ExecutionTraceContext.forInputRun("key", "agent"), "action");
+ }
+}
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInExecutionMetricsTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInExecutionMetricsTest.java
new file mode 100644
index 0000000000..69e471483e
--- /dev/null
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInExecutionMetricsTest.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
+import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
+import org.apache.flink.agents.api.trace.ExecutionReporter;
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
+import org.apache.flink.agents.api.trace.ToolExecutionMetadataKeys;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.Map;
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class BuiltInExecutionMetricsTest {
+
+ private static final String ACTION_NAME = "chat_model_action";
+
+ private FlinkAgentsMetricGroupImpl metricGroup;
+ private BuiltInExecutionMetrics metrics;
+
+ @BeforeEach
+ void setUp() {
+ MetricGroup parentMetricGroup =
+ UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
+ metricGroup = new FlinkAgentsMetricGroupImpl(parentMetricGroup);
+ Set registeredTools = Set.of("search", "fetch", "load_skill");
+ metrics = new BuiltInExecutionMetrics(metricGroup, registeredTools::contains);
+ }
+
+ @Test
+ void recordsLlmOutcomeByModelResource() {
+ ExecutionTraceContext success =
+ execution(ExecutionReporter.EntityTypes.LLM, "primary_model", Map.of());
+ observe(ExecutionLifecycleEvents.executionStarted(), success, 0);
+ observe(ExecutionLifecycleEvents.executionFinished(), success, 25);
+
+ ExecutionTraceContext failure =
+ execution(ExecutionReporter.EntityTypes.LLM, "primary_model", Map.of());
+ observe(ExecutionLifecycleEvents.executionStarted(), failure, 100);
+ observe(
+ ExecutionLifecycleEvents.executionFailed(new RuntimeException("failed")),
+ failure,
+ 140);
+
+ FlinkAgentsMetricGroupImpl modelResource =
+ actionMetricGroup().getSubGroup("model_resource", "primary_model");
+ assertThat(
+ modelResource
+ .getCounter(LlmExecutionMetricRecorder.NUM_LLM_CALLS_SUCCEEDED)
+ .getCount())
+ .isEqualTo(1);
+ assertThat(
+ modelResource
+ .getCounter(LlmExecutionMetricRecorder.NUM_LLM_CALLS_FAILED)
+ .getCount())
+ .isEqualTo(1);
+ assertThat(
+ modelResource
+ .getHistogram(LlmExecutionMetricRecorder.LLM_CALL_LATENCY_MS)
+ .getCount())
+ .isEqualTo(2);
+ }
+
+ @Test
+ void recordsToolOutcomeByToolName() {
+ ExecutionTraceContext success =
+ execution(ExecutionReporter.EntityTypes.TOOL, "search", Map.of());
+ observe(ExecutionLifecycleEvents.executionStarted(), success, 0);
+ observe(ExecutionLifecycleEvents.executionFinished(), success, 15);
+
+ ExecutionTraceContext failure =
+ execution(ExecutionReporter.EntityTypes.TOOL, "search", Map.of());
+ observe(ExecutionLifecycleEvents.executionStarted(), failure, 5);
+ observe(
+ ExecutionLifecycleEvents.executionFailed(new RuntimeException("failed")),
+ failure,
+ 25);
+
+ FlinkAgentsMetricGroupImpl tool = actionMetricGroup().getSubGroup("tool", "search");
+ assertThat(tool.getCounter(ToolExecutionMetricRecorder.NUM_TOOL_CALLS_SUCCEEDED).getCount())
+ .isEqualTo(1);
+ assertThat(tool.getCounter(ToolExecutionMetricRecorder.NUM_TOOL_CALLS_FAILED).getCount())
+ .isEqualTo(1);
+ assertThat(tool.getHistogram(ToolExecutionMetricRecorder.TOOL_CALL_LATENCY_MS).getCount())
+ .isEqualTo(2);
+ assertThat(
+ tool.getHistogram(ToolExecutionMetricRecorder.TOOL_CALL_LATENCY_MS)
+ .getStatistics()
+ .getMax())
+ .isEqualTo(20L);
+ }
+
+ @Test
+ void aggregatesUnregisteredToolNamesIntoUnknownScope() {
+ ExecutionTraceContext first =
+ execution(ExecutionReporter.EntityTypes.TOOL, "hallucinated_one", Map.of());
+ ExecutionTraceContext second =
+ execution(ExecutionReporter.EntityTypes.TOOL, "hallucinated_two", Map.of());
+
+ observe(
+ ExecutionLifecycleEvents.executionFailed(new RuntimeException("missing")),
+ first,
+ 0);
+ observe(
+ ExecutionLifecycleEvents.executionFailed(new RuntimeException("missing")),
+ second,
+ 1);
+
+ FlinkAgentsMetricGroupImpl unknown =
+ actionMetricGroup()
+ .getSubGroup("tool", ToolExecutionMetricRecorder.UNKNOWN_TOOL_NAME);
+ assertThat(unknown.getCounter(ToolExecutionMetricRecorder.NUM_TOOL_CALLS_FAILED).getCount())
+ .isEqualTo(2);
+ }
+
+ @Test
+ void recordsExplicitSkillLoads() {
+ ExecutionTraceContext loadSkill =
+ execution(
+ ExecutionReporter.EntityTypes.TOOL,
+ "load_skill",
+ Map.of(
+ ToolExecutionMetadataKeys.SKILL_NAME,
+ "calculator",
+ ToolExecutionMetadataKeys.SKILL_REGISTERED,
+ true));
+ observe(ExecutionLifecycleEvents.executionStarted(), loadSkill, 0);
+ observe(ExecutionLifecycleEvents.executionFinished(), loadSkill, 12);
+
+ FlinkAgentsMetricGroupImpl skill = actionMetricGroup().getSubGroup("skill", "calculator");
+ assertThat(skill.getCounter(ToolExecutionMetricRecorder.NUM_SKILL_LOADS).getCount())
+ .isEqualTo(1);
+ assertThat(
+ skill.getHistogram(ToolExecutionMetricRecorder.SKILL_LOAD_LATENCY_MS)
+ .getStatistics()
+ .getMax())
+ .isEqualTo(12L);
+
+ FlinkAgentsMetricGroupImpl tool = actionMetricGroup().getSubGroup("tool", "load_skill");
+ assertThat(tool.getCounter(ToolExecutionMetricRecorder.NUM_TOOL_CALLS_SUCCEEDED).getCount())
+ .isEqualTo(1);
+ }
+
+ @Test
+ void aggregatesUnregisteredSkillNamesIntoUnknownScope() {
+ ExecutionTraceContext first =
+ execution(
+ ExecutionReporter.EntityTypes.TOOL,
+ "load_skill",
+ Map.of(
+ ToolExecutionMetadataKeys.SKILL_NAME,
+ "hallucinated_one",
+ ToolExecutionMetadataKeys.SKILL_REGISTERED,
+ false));
+ ExecutionTraceContext second =
+ execution(
+ ExecutionReporter.EntityTypes.TOOL,
+ "load_skill",
+ Map.of(
+ ToolExecutionMetadataKeys.SKILL_NAME,
+ "hallucinated_two",
+ ToolExecutionMetadataKeys.SKILL_REGISTERED,
+ false));
+
+ observe(ExecutionLifecycleEvents.executionFinished(), first, 0);
+ observe(ExecutionLifecycleEvents.executionFinished(), second, 1);
+
+ FlinkAgentsMetricGroupImpl unknown =
+ actionMetricGroup()
+ .getSubGroup("skill", ToolExecutionMetricRecorder.UNKNOWN_SKILL_NAME);
+ assertThat(unknown.getCounter(ToolExecutionMetricRecorder.NUM_SKILL_LOADS).getCount())
+ .isEqualTo(2);
+ }
+
+ @Test
+ void aggregatesMcpToolOutcomesByServer() {
+ ExecutionTraceContext success =
+ execution(
+ ExecutionReporter.EntityTypes.TOOL,
+ "search",
+ Map.of(ToolExecutionMetadataKeys.MCP_SERVER, "search_server"));
+ observe(ExecutionLifecycleEvents.executionStarted(), success, 0);
+ observe(ExecutionLifecycleEvents.executionFinished(), success, 30);
+
+ ExecutionTraceContext failure =
+ execution(
+ ExecutionReporter.EntityTypes.TOOL,
+ "fetch",
+ Map.of(ToolExecutionMetadataKeys.MCP_SERVER, "search_server"));
+ observe(ExecutionLifecycleEvents.executionStarted(), failure, 10);
+ observe(
+ ExecutionLifecycleEvents.executionFailed(new RuntimeException("failed")),
+ failure,
+ 60);
+
+ FlinkAgentsMetricGroupImpl mcpServer =
+ actionMetricGroup().getSubGroup("mcp_server", "search_server");
+ assertThat(
+ mcpServer
+ .getCounter(
+ ToolExecutionMetricRecorder.NUM_MCP_TOOL_CALLS_SUCCEEDED)
+ .getCount())
+ .isEqualTo(1);
+ assertThat(
+ mcpServer
+ .getCounter(ToolExecutionMetricRecorder.NUM_MCP_TOOL_CALLS_FAILED)
+ .getCount())
+ .isEqualTo(1);
+ assertThat(
+ mcpServer
+ .getHistogram(ToolExecutionMetricRecorder.MCP_TOOL_CALL_LATENCY_MS)
+ .getCount())
+ .isEqualTo(2);
+ }
+
+ @Test
+ void terminalEventWithoutLocalStartDoesNotRecordLatency() {
+ ExecutionTraceContext llm =
+ execution(ExecutionReporter.EntityTypes.LLM, "restored_model", Map.of());
+ observe(ExecutionLifecycleEvents.executionFinished(), llm, 0);
+
+ FlinkAgentsMetricGroupImpl modelResource =
+ actionMetricGroup().getSubGroup("model_resource", "restored_model");
+ assertThat(
+ modelResource
+ .getCounter(LlmExecutionMetricRecorder.NUM_LLM_CALLS_SUCCEEDED)
+ .getCount())
+ .isEqualTo(1);
+ assertThat(
+ modelResource
+ .getHistogram(LlmExecutionMetricRecorder.LLM_CALL_LATENCY_MS)
+ .getCount())
+ .isZero();
+ }
+
+ @Test
+ void toolFailureAfterCreationWithoutStartCountsFailureWithoutLatency() {
+ ExecutionTraceContext traceContext =
+ execution(ExecutionReporter.EntityTypes.TOOL, "search", Map.of());
+ observe(ExecutionLifecycleEvents.executionCreated(), traceContext, 0);
+ observe(
+ ExecutionLifecycleEvents.executionFailed(new RuntimeException("timed out")),
+ traceContext,
+ 1000);
+
+ FlinkAgentsMetricGroupImpl tool = actionMetricGroup().getSubGroup("tool", "search");
+ assertThat(tool.getCounter(ToolExecutionMetricRecorder.NUM_TOOL_CALLS_FAILED).getCount())
+ .isEqualTo(1);
+ assertThat(tool.getHistogram(ToolExecutionMetricRecorder.TOOL_CALL_LATENCY_MS).getCount())
+ .isZero();
+ }
+
+ private void observe(Event event, ExecutionTraceContext traceContext, long timestampMillis) {
+ metrics.executionEventObserved(
+ ACTION_NAME,
+ new EventContext(
+ event.getType(), Instant.EPOCH.plusMillis(timestampMillis).toString()),
+ event,
+ traceContext);
+ }
+
+ private FlinkAgentsMetricGroupImpl actionMetricGroup() {
+ return metricGroup.getSubGroup("action", ACTION_NAME);
+ }
+
+ private static ExecutionTraceContext execution(
+ String entityType, String entityName, Map metadata) {
+ ExecutionTraceContext action =
+ ExecutionTraceContext.forAction(
+ ExecutionTraceContext.forInputRun("key", "agent"), ACTION_NAME);
+ return action.childExecution(entityType, entityName, metadata);
+ }
+}
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInInputRunMetricsTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInInputRunMetricsTest.java
new file mode 100644
index 0000000000..168364d022
--- /dev/null
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInInputRunMetricsTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.InputEvent;
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Histogram;
+import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.assertj.core.api.Assertions.assertThat;
+
+class BuiltInInputRunMetricsTest {
+
+ private final AtomicLong nanoTime = new AtomicLong();
+
+ private FlinkAgentsMetricGroupImpl metricGroup;
+ private BuiltInInputRunMetrics metrics;
+
+ @BeforeEach
+ void setUp() {
+ metricGroup =
+ new FlinkAgentsMetricGroupImpl(
+ UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup());
+ metrics = new BuiltInInputRunMetrics(metricGroup, nanoTime::get);
+ }
+
+ @Test
+ void completedRunIsSuccessfulAndLatencyIncludesQueueTime() {
+ setTimeMillis(100L);
+ Event inputEvent = new InputEvent("input");
+ ExecutionTraceContext inputRun = ExecutionTraceContext.forInputRun("key", "agent");
+
+ metrics.inputEventReceived(inputEvent);
+ setTimeMillis(130L);
+ metrics.inputRunStarted(inputEvent, inputRun);
+ setTimeMillis(200L);
+ metrics.inputRunCompleted(inputRun.getInputRunId());
+
+ assertThat(counter(BuiltInInputRunMetrics.NUM_INPUT_RUNS_SUCCEEDED).getCount())
+ .isEqualTo(1L);
+ assertThat(counter(BuiltInInputRunMetrics.NUM_INPUT_RUNS_FAILED).getCount()).isZero();
+ assertHistogram(BuiltInInputRunMetrics.INPUT_RUN_LATENCY_MS, 1L, 100L);
+ assertHistogram(BuiltInInputRunMetrics.INPUT_RUN_QUEUE_LATENCY_MS, 1L, 30L);
+ assertHistogram(BuiltInInputRunMetrics.INPUT_RUN_PROCESSING_LATENCY_MS, 1L, 70L);
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_ACTIVE_INPUT_RUNS)).isZero();
+ }
+
+ @Test
+ void inputProcessingFailureBeforeRunStartIsRecorded() {
+ setTimeMillis(100L);
+ Event inputEvent = new InputEvent("input");
+
+ metrics.inputEventReceived(inputEvent);
+ setTimeMillis(125L);
+ metrics.inputEventFailed(inputEvent);
+
+ assertThat(counter(BuiltInInputRunMetrics.NUM_INPUT_RUNS_FAILED).getCount()).isEqualTo(1L);
+ assertHistogram(BuiltInInputRunMetrics.INPUT_RUN_LATENCY_MS, 1L, 25L);
+ assertThat(histogram(BuiltInInputRunMetrics.INPUT_RUN_QUEUE_LATENCY_MS).getCount())
+ .isZero();
+ assertThat(histogram(BuiltInInputRunMetrics.INPUT_RUN_PROCESSING_LATENCY_MS).getCount())
+ .isZero();
+ }
+
+ @Test
+ void restoredRunRebuildsActiveGaugeWithoutRecordingHistoricalSamples() {
+ setTimeMillis(300L);
+ ExecutionTraceContext inputRun = ExecutionTraceContext.forInputRun("key", "agent");
+
+ metrics.restoreActiveInputRuns(1L);
+ metrics.identifyRestoredActiveInputRun(inputRun.getInputRunId());
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_ACTIVE_INPUT_RUNS)).isEqualTo(1L);
+
+ setTimeMillis(375L);
+ metrics.inputRunCompleted(inputRun.getInputRunId());
+
+ assertThat(counter(BuiltInInputRunMetrics.NUM_INPUT_RUNS_SUCCEEDED).getCount()).isZero();
+ assertThat(counter(BuiltInInputRunMetrics.NUM_INPUT_RUNS_FAILED).getCount()).isZero();
+ assertThat(histogram(BuiltInInputRunMetrics.INPUT_RUN_LATENCY_MS).getCount()).isZero();
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_ACTIVE_INPUT_RUNS)).isZero();
+ }
+
+ @Test
+ void restoredPendingInputRecordsLocallyObservedOutcomeAndProcessingLatency() {
+ Event inputEvent = new InputEvent("restored-pending");
+ ExecutionTraceContext inputRun = ExecutionTraceContext.forInputRun("key", "agent");
+
+ metrics.restorePendingInputEvents(1L);
+ metrics.pendingInputEventDequeued();
+ setTimeMillis(300L);
+ metrics.inputRunStarted(inputEvent, inputRun);
+ setTimeMillis(375L);
+ metrics.inputRunCompleted(inputRun.getInputRunId());
+
+ assertThat(counter(BuiltInInputRunMetrics.NUM_INPUT_RUNS_SUCCEEDED).getCount())
+ .isEqualTo(1L);
+ assertThat(histogram(BuiltInInputRunMetrics.INPUT_RUN_LATENCY_MS).getCount()).isZero();
+ assertThat(histogram(BuiltInInputRunMetrics.INPUT_RUN_QUEUE_LATENCY_MS).getCount())
+ .isZero();
+ assertHistogram(BuiltInInputRunMetrics.INPUT_RUN_PROCESSING_LATENCY_MS, 1L, 75L);
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_PENDING_INPUT_EVENTS)).isZero();
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_ACTIVE_INPUT_RUNS)).isZero();
+ }
+
+ @Test
+ void duplicateTerminalForIdentifiedRestoredRunDoesNotConsumeAnonymousRun() {
+ ExecutionTraceContext identifiedRun = ExecutionTraceContext.forInputRun("key-1", "agent");
+
+ metrics.restoreActiveInputRuns(2L);
+ metrics.identifyRestoredActiveInputRun(identifiedRun.getInputRunId());
+ metrics.inputRunCompleted(identifiedRun.getInputRunId());
+ metrics.inputRunCompleted(identifiedRun.getInputRunId());
+
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_ACTIVE_INPUT_RUNS)).isEqualTo(1L);
+
+ metrics.inputRunCompleted(null);
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_ACTIVE_INPUT_RUNS)).isZero();
+ }
+
+ @Test
+ void terminalFailureIsAttributedToMatchingRunForSameKey() {
+ setTimeMillis(100L);
+ Event firstInput = new InputEvent("first");
+ ExecutionTraceContext firstRun = ExecutionTraceContext.forInputRun("key", "agent");
+ metrics.inputEventReceived(firstInput);
+ metrics.inputRunStarted(firstInput, firstRun);
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_ACTIVE_INPUT_RUNS)).isEqualTo(1L);
+
+ setTimeMillis(110L);
+ Event secondInput = new InputEvent("second");
+ ExecutionTraceContext secondRun = ExecutionTraceContext.forInputRun("key", "agent");
+ metrics.inputEventReceived(secondInput);
+ metrics.inputRunStarted(secondInput, secondRun);
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_ACTIVE_INPUT_RUNS)).isEqualTo(2L);
+
+ setTimeMillis(150L);
+ metrics.inputRunCompleted(firstRun.getInputRunId());
+ setTimeMillis(180L);
+ metrics.inputRunFailed(secondRun.getInputRunId());
+
+ assertThat(counter(BuiltInInputRunMetrics.NUM_INPUT_RUNS_SUCCEEDED).getCount())
+ .isEqualTo(1L);
+ assertThat(counter(BuiltInInputRunMetrics.NUM_INPUT_RUNS_FAILED).getCount()).isEqualTo(1L);
+ assertThat(histogram(BuiltInInputRunMetrics.INPUT_RUN_LATENCY_MS).getCount()).isEqualTo(2L);
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_ACTIVE_INPUT_RUNS)).isZero();
+ }
+
+ @Test
+ void pendingInputGaugeTracksQueueAndRestore() {
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_PENDING_INPUT_EVENTS)).isZero();
+
+ metrics.pendingInputEventEnqueued();
+ metrics.pendingInputEventEnqueued();
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_PENDING_INPUT_EVENTS)).isEqualTo(2L);
+
+ metrics.pendingInputEventDequeued();
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_PENDING_INPUT_EVENTS)).isEqualTo(1L);
+
+ metrics.restorePendingInputEvents(3L);
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_PENDING_INPUT_EVENTS)).isEqualTo(3L);
+ }
+
+ @Test
+ void duplicateTerminalNotificationDoesNotUnderflowActiveGauge() {
+ Event inputEvent = new InputEvent("input");
+ ExecutionTraceContext inputRun = ExecutionTraceContext.forInputRun("key", "agent");
+ metrics.inputEventReceived(inputEvent);
+ metrics.inputRunStarted(inputEvent, inputRun);
+
+ metrics.inputRunCompleted(inputRun.getInputRunId());
+ metrics.inputRunCompleted(inputRun.getInputRunId());
+
+ assertThat(gauge(BuiltInInputRunMetrics.NUM_ACTIVE_INPUT_RUNS)).isZero();
+ assertThat(counter(BuiltInInputRunMetrics.NUM_INPUT_RUNS_SUCCEEDED).getCount())
+ .isEqualTo(1L);
+ }
+
+ private Counter counter(String name) {
+ return metricGroup.getCounter(name);
+ }
+
+ private Histogram histogram(String name) {
+ return metricGroup.getHistogram(name);
+ }
+
+ private long gauge(String name) {
+ return (Long) metricGroup.getGauge(name).getValue();
+ }
+
+ private void setTimeMillis(long millis) {
+ nanoTime.set(MILLISECONDS.toNanos(millis));
+ }
+
+ private void assertHistogram(String name, long count, long max) {
+ Histogram histogram = histogram(name);
+ assertThat(histogram.getCount()).isEqualTo(count);
+ assertThat(histogram.getStatistics().getMax()).isEqualTo(max);
+ }
+}
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInMetricsTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInMetricsTest.java
new file mode 100644
index 0000000000..ff59e4cbdc
--- /dev/null
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/BuiltInMetricsTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
+import org.apache.flink.agents.api.trace.ExecutionReporter;
+import org.apache.flink.agents.api.trace.ExecutionTraceContext;
+import org.apache.flink.agents.plan.AgentPlan;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class BuiltInMetricsTest {
+
+ @Test
+ void restoredActionMissingFromCurrentPlanKeepsItsMetricLifecycle() {
+ MetricGroup parentMetricGroup =
+ UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
+ FlinkAgentsMetricGroupImpl metricGroup = new FlinkAgentsMetricGroupImpl(parentMetricGroup);
+ BuiltInMetrics metrics =
+ new BuiltInMetrics(metricGroup, new AgentPlan(Map.of()), ignored -> false);
+ ExecutionTraceContext restoredAction =
+ ExecutionTraceContext.forAction(
+ ExecutionTraceContext.forInputRun("key", "agent"), "restored_action");
+
+ metrics.restoreActionTask(restoredAction, true);
+ metrics.markActionTaskDequeued(restoredAction, true);
+ metrics.markExecutionEvent(
+ "restored_action", ExecutionLifecycleEvents.executionReused(), restoredAction);
+ metrics.markActionExecuted("restored_action");
+
+ FlinkAgentsMetricGroupImpl actionMetricGroup =
+ metricGroup.getSubGroup("action", "restored_action");
+ assertThat(
+ actionMetricGroup
+ .getGauge(BuiltInActionMetrics.NUM_PENDING_ACTION_TASKS)
+ .getValue())
+ .isEqualTo(0L);
+ assertThat(
+ actionMetricGroup
+ .getGauge(BuiltInActionMetrics.NUM_ACTIVE_ACTION_EXECUTIONS)
+ .getValue())
+ .isEqualTo(0L);
+ assertThat(actionMetricGroup.getCounter("numOfActionsExecuted").getCount()).isEqualTo(1L);
+ }
+
+ @Test
+ void actionTerminalDropsOnlyItsUnpairedChildLatencyState() {
+ MetricGroup parentMetricGroup =
+ UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
+ FlinkAgentsMetricGroupImpl metricGroup = new FlinkAgentsMetricGroupImpl(parentMetricGroup);
+ BuiltInMetrics metrics =
+ new BuiltInMetrics(metricGroup, new AgentPlan(Map.of()), ignored -> false);
+ ExecutionTraceContext inputRun = ExecutionTraceContext.forInputRun("key", "agent");
+ ExecutionTraceContext completedAction =
+ ExecutionTraceContext.forAction(inputRun, "restored_action");
+ ExecutionTraceContext activeAction =
+ ExecutionTraceContext.forAction(inputRun, "restored_action");
+ ExecutionTraceContext completedActionLlm =
+ completedAction.childExecution(ExecutionReporter.EntityTypes.LLM, "primary_model");
+ ExecutionTraceContext activeActionLlm =
+ activeAction.childExecution(ExecutionReporter.EntityTypes.LLM, "secondary_model");
+ metrics.restoreActionTask(completedAction, false);
+
+ metrics.markExecutionEvent(
+ "restored_action", ExecutionLifecycleEvents.executionStarted(), completedActionLlm);
+ metrics.markExecutionEvent(
+ "restored_action", ExecutionLifecycleEvents.executionStarted(), activeActionLlm);
+ metrics.markExecutionEvent(
+ "restored_action", ExecutionLifecycleEvents.executionFinished(), completedAction);
+ metrics.markExecutionEvent(
+ "restored_action",
+ ExecutionLifecycleEvents.executionFinished(),
+ completedActionLlm);
+ metrics.markExecutionEvent(
+ "restored_action", ExecutionLifecycleEvents.executionFinished(), activeActionLlm);
+
+ assertThat(
+ metricGroup
+ .getSubGroup("action", "restored_action")
+ .getSubGroup("model_resource", "primary_model")
+ .getHistogram(LlmExecutionMetricRecorder.LLM_CALL_LATENCY_MS)
+ .getCount())
+ .isZero();
+ assertThat(
+ metricGroup
+ .getSubGroup("action", "restored_action")
+ .getSubGroup("model_resource", "secondary_model")
+ .getHistogram(LlmExecutionMetricRecorder.LLM_CALL_LATENCY_MS)
+ .getCount())
+ .isEqualTo(1L);
+ }
+}
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/CurrentCountGaugeTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/CurrentCountGaugeTest.java
new file mode 100644
index 0000000000..ff9b6c32a9
--- /dev/null
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/metrics/CurrentCountGaugeTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.agents.runtime.metrics;
+
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class CurrentCountGaugeTest {
+
+ private static final String GAUGE_NAME = "current";
+
+ private FlinkAgentsMetricGroupImpl metricGroup;
+ private CurrentCountGauge gauge;
+
+ @BeforeEach
+ void setUp() {
+ MetricGroup parentMetricGroup =
+ UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
+ metricGroup = new FlinkAgentsMetricGroupImpl(parentMetricGroup);
+ gauge = new CurrentCountGauge(metricGroup, GAUGE_NAME);
+ }
+
+ @Test
+ void decrementAtZeroRemainsZero() {
+ gauge.decrement();
+
+ assertThat(value()).isZero();
+ }
+
+ @Test
+ void negativeSetValueIsClampedToZero() {
+ gauge.set(-1L);
+
+ assertThat(value()).isZero();
+ }
+
+ private long value() {
+ return (Long) metricGroup.getGauge(GAUGE_NAME).getValue();
+ }
+}
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/operator/ActionExecutionOperatorTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/operator/ActionExecutionOperatorTest.java
index a135af77fa..500634bc53 100644
--- a/runtime/src/test/java/org/apache/flink/agents/runtime/operator/ActionExecutionOperatorTest.java
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/operator/ActionExecutionOperatorTest.java
@@ -61,6 +61,7 @@
import org.apache.flink.agents.runtime.eventlog.FileEventLogger;
import org.apache.flink.agents.runtime.eventlog.Slf4jEventLogger;
import org.apache.flink.agents.runtime.memory.Mem0LongTermMemory;
+import org.apache.flink.agents.runtime.metrics.FlinkAgentsMetricGroupImpl;
import org.apache.flink.api.common.serialization.SerializerConfigImpl;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.base.LongSerializer;
@@ -228,6 +229,29 @@ void testJavaEventAttachmentsAreOffloadedAndResolvedBetweenActions() throws Exce
}
}
+ @Test
+ void contextKeyResolutionFailureMarksInputEventFailed() throws Exception {
+ try (KeyedOneInputStreamOperatorTestHarness testHarness =
+ new KeyedOneInputStreamOperatorTestHarness<>(
+ new ActionExecutionOperatorFactory(TestAgent.getAgentPlan(false), true),
+ (KeySelector) value -> FailingContextKey.INSTANCE,
+ TypeInformation.of(FailingContextKey.class))) {
+ testHarness.open();
+ ActionExecutionOperator operator =
+ (ActionExecutionOperator) testHarness.getOperator();
+
+ assertThatThrownBy(() -> testHarness.processElement(new StreamRecord<>(1L)))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("context key conversion failed");
+
+ Field metricGroupField = ActionExecutionOperator.class.getDeclaredField("metricGroup");
+ metricGroupField.setAccessible(true);
+ FlinkAgentsMetricGroupImpl metricGroup =
+ (FlinkAgentsMetricGroupImpl) metricGroupField.get(operator);
+ assertThat(metricGroup.getCounter("numOfInputRunsFailed").getCount()).isEqualTo(1L);
+ }
+ }
+
@Test
void testSameKeyDataAreProcessedInOrder() throws Exception {
try (KeyedOneInputStreamOperatorTestHarness testHarness =
@@ -547,7 +571,7 @@ void testMailboxSubmittedActionTaskPropagatesErrorAndClosesActionLifecycle() thr
}
@Test
- void testToolLinkageErrorEmitsFailedLifecycleBeforeActionFailure() throws Exception {
+ void testToolLinkageErrorReportsFailedExecutionAndPropagates() throws Exception {
AgentPlan basePlan = TestAgent.getLinkageErrorToolAgentPlan();
AgentPlan agentPlan =
new AgentPlan(
@@ -578,17 +602,37 @@ void testToolLinkageErrorEmitsFailedLifecycleBeforeActionFailure() throws Except
ExecutionLifecycleEvents.EXECUTION_STARTED_EVENT_TYPE,
"linkageErrorTool",
ExecutionLifecycleEvents.STATUS_STARTED);
- RecordedEvent failed =
+ RecordedEvent toolFailed =
findRecordedLifecycleEvent(
ExecutionLifecycleEvents.EXECUTION_FAILED_EVENT_TYPE,
"linkageErrorTool",
ExecutionLifecycleEvents.STATUS_FAILED);
+ RecordedEvent actionFailed =
+ findRecordedLifecycleEvent(
+ ExecutionLifecycleEvents.EXECUTION_FAILED_EVENT_TYPE,
+ "tool_call_action",
+ ExecutionLifecycleEvents.STATUS_FAILED);
assertThat(started.traceContext().getEntityType())
.isEqualTo(ExecutionReporter.EntityTypes.TOOL);
- assertThat(failed.traceContext().getExecutionId())
+ assertThat(toolFailed.traceContext().getExecutionId())
.isEqualTo(started.traceContext().getExecutionId());
- assertThat(failed.event.getAttr("errorType"))
+ assertThat(actionFailed.traceContext().getExecutionId())
+ .isEqualTo(started.traceContext().getParentExecutionId());
+ assertThat(toolFailed.event.getAttr("errorType"))
.isEqualTo(NoClassDefFoundError.class.getName());
+ assertThat(actionFailed.event.getAttr("errorType"))
+ .isEqualTo(NoClassDefFoundError.class.getName());
+ assertThat(RecordingEventLogger.events())
+ .filteredOn(
+ record ->
+ started.traceContext()
+ .getExecutionId()
+ .equals(record.traceContext().getExecutionId()))
+ .extracting(record -> record.event.getType())
+ .containsExactly(
+ ExecutionLifecycleEvents.EXECUTION_CREATED_EVENT_TYPE,
+ ExecutionLifecycleEvents.EXECUTION_STARTED_EVENT_TYPE,
+ ExecutionLifecycleEvents.EXECUTION_FAILED_EVENT_TYPE);
}
@Test
@@ -1519,6 +1563,23 @@ record ->
.isEqualTo(
ExecutionLifecycleEvents
.EXECUTION_STARTED_EVENT_TYPE));
+
+ Field metricGroupField = ActionExecutionOperator.class.getDeclaredField("metricGroup");
+ metricGroupField.setAccessible(true);
+ FlinkAgentsMetricGroupImpl metricGroup =
+ (FlinkAgentsMetricGroupImpl) metricGroupField.get(operator);
+ assertThat(
+ metricGroup
+ .getSubGroup("action", "action1")
+ .getHistogram("actionExecutionLatencyMs")
+ .getCount())
+ .isEqualTo(1L);
+ assertThat(
+ metricGroup
+ .getSubGroup("action", "action2")
+ .getHistogram("actionExecutionLatencyMs")
+ .getCount())
+ .isEqualTo(1L);
}
assertThat(RecordingEventLogger.closeCount()).isEqualTo(1);
@@ -3978,6 +4039,15 @@ private Map getCompletedStateBytes() {
}
}
+ private enum FailingContextKey {
+ INSTANCE;
+
+ @Override
+ public String toString() {
+ throw new IllegalStateException("context key conversion failed");
+ }
+ }
+
private static void assertMailboxSizeAndRun(TaskMailbox mailbox, int expectedSize)
throws Exception {
assertThat(mailbox.size()).isEqualTo(expectedSize);
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/operator/ActionTaskContextManagerTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/operator/ActionTaskContextManagerTest.java
index 827daf1173..d95ab33fdd 100644
--- a/runtime/src/test/java/org/apache/flink/agents/runtime/operator/ActionTaskContextManagerTest.java
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/operator/ActionTaskContextManagerTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.agents.runtime.operator;
import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.EventContext;
import org.apache.flink.agents.api.InputEvent;
import org.apache.flink.agents.api.trace.ExecutionLifecycleEvents;
import org.apache.flink.agents.api.trace.ExecutionReporter;
@@ -477,6 +478,7 @@ public void onComponentExecution(
String entityType,
String entityName,
Map entityMetadata,
+ EventContext eventContext,
Event event) {
if (ExecutionLifecycleEvents.EXECUTION_STARTED_EVENT_TYPE.equals(event.getType())) {
started.add(entityName);
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/operator/EventRouterTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/operator/EventRouterTest.java
index 1cba936225..e781879a49 100644
--- a/runtime/src/test/java/org/apache/flink/agents/runtime/operator/EventRouterTest.java
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/operator/EventRouterTest.java
@@ -288,6 +288,6 @@ private static BuiltInMetrics makeMetrics() {
FlinkAgentsMetricGroupImpl metricGroup =
mock(FlinkAgentsMetricGroupImpl.class, RETURNS_DEEP_STUBS);
AgentPlan plan = new AgentPlan(new HashMap<>(), new HashMap<>());
- return new BuiltInMetrics(metricGroup, plan);
+ return new BuiltInMetrics(metricGroup, plan, ignored -> false);
}
}
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/skill/LoadSkillToolTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/skill/LoadSkillToolTest.java
index 71d6ff6e3e..0125c7854f 100644
--- a/runtime/src/test/java/org/apache/flink/agents/runtime/skill/LoadSkillToolTest.java
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/skill/LoadSkillToolTest.java
@@ -104,9 +104,19 @@ void executionMetadataDescribesRequestedSkillResource() {
tool(contextWithSkills()).getToolExecutionMetadata(args("github", "README.md"));
assertEquals("github", metadata.get(ToolExecutionMetadataKeys.SKILL_NAME));
+ assertEquals(true, metadata.get(ToolExecutionMetadataKeys.SKILL_REGISTERED));
assertEquals("README.md", metadata.get(ToolExecutionMetadataKeys.SKILL_RESOURCE_PATH));
}
+ @Test
+ void executionMetadataMarksUnknownSkillAsUnregistered() {
+ Map metadata =
+ tool(contextWithSkills()).getToolExecutionMetadata(args("does-not-exist", null));
+
+ assertEquals("does-not-exist", metadata.get(ToolExecutionMetadataKeys.SKILL_NAME));
+ assertEquals(false, metadata.get(ToolExecutionMetadataKeys.SKILL_REGISTERED));
+ }
+
@Test
void executionMetadataNormalizesOmittedPathToSkillMd() {
Map metadata =
diff --git a/runtime/src/test/java/org/apache/flink/agents/runtime/trace/EventLogComponentExecutionListenerTest.java b/runtime/src/test/java/org/apache/flink/agents/runtime/trace/EventLogComponentExecutionListenerTest.java
index eea2fb0747..91c2de3d1b 100644
--- a/runtime/src/test/java/org/apache/flink/agents/runtime/trace/EventLogComponentExecutionListenerTest.java
+++ b/runtime/src/test/java/org/apache/flink/agents/runtime/trace/EventLogComponentExecutionListenerTest.java
@@ -41,6 +41,69 @@
*/
class EventLogComponentExecutionListenerTest {
+ @Test
+ void createdStartAndTerminalReportsShareOneChildExecution() {
+ CapturingEventLogger logger = new CapturingEventLogger();
+ EventLogComponentExecutionListener listener =
+ new EventLogComponentExecutionListener(actionTraceContext(), sink(logger));
+ Map metadata = Map.of("toolCallId", "call-1");
+
+ report(
+ listener,
+ ExecutionReporter.EntityTypes.TOOL,
+ "search",
+ metadata,
+ ExecutionLifecycleEvents.executionCreated());
+ report(
+ listener,
+ ExecutionReporter.EntityTypes.TOOL,
+ "search",
+ metadata,
+ ExecutionLifecycleEvents.executionStarted());
+ report(
+ listener,
+ ExecutionReporter.EntityTypes.TOOL,
+ "search",
+ metadata,
+ ExecutionLifecycleEvents.executionFinished());
+
+ assertThat(logger.records)
+ .extracting(record -> record.event.getType())
+ .containsExactly(
+ ExecutionLifecycleEvents.EXECUTION_CREATED_EVENT_TYPE,
+ ExecutionLifecycleEvents.EXECUTION_STARTED_EVENT_TYPE,
+ ExecutionLifecycleEvents.EXECUTION_FINISHED_EVENT_TYPE);
+ assertThat(logger.records)
+ .extracting(record -> record.traceContext.getExecutionId())
+ .containsOnly(logger.records.get(0).traceContext.getExecutionId());
+ }
+
+ @Test
+ void terminalBeforeStartPairsWithItsCreation() {
+ CapturingEventLogger logger = new CapturingEventLogger();
+ EventLogComponentExecutionListener listener =
+ new EventLogComponentExecutionListener(actionTraceContext(), sink(logger));
+ Map metadata = Map.of("toolCallId", "call-1");
+
+ report(
+ listener,
+ ExecutionReporter.EntityTypes.TOOL,
+ "search",
+ metadata,
+ ExecutionLifecycleEvents.executionCreated());
+ report(
+ listener,
+ ExecutionReporter.EntityTypes.TOOL,
+ "search",
+ metadata,
+ ExecutionLifecycleEvents.executionFailed(
+ new IllegalStateException("failed before invocation")));
+
+ assertThat(logger.records).hasSize(2);
+ assertThat(logger.records.get(1).traceContext.getExecutionId())
+ .isEqualTo(logger.records.get(0).traceContext.getExecutionId());
+ }
+
@Test
void startAndTerminalReportsShareOneChildExecution() {
CapturingEventLogger logger = new CapturingEventLogger();
@@ -48,12 +111,14 @@ void startAndTerminalReportsShareOneChildExecution() {
EventLogComponentExecutionListener listener =
new EventLogComponentExecutionListener(actionContext, sink(logger));
- listener.onComponentExecution(
+ report(
+ listener,
ExecutionReporter.EntityTypes.LLM,
"model-a",
Map.of(),
ExecutionLifecycleEvents.executionStarted());
- listener.onComponentExecution(
+ report(
+ listener,
ExecutionReporter.EntityTypes.LLM,
"model-a",
Map.of(),
@@ -86,12 +151,14 @@ void pairingSurvivesWhenReportsUseSeparateListenerAccesses() {
// Mirrors a continuation: the start is reported first, the terminal arrives later
// through the same per-execution listener instance.
- listener.onComponentExecution(
+ report(
+ listener,
ExecutionReporter.EntityTypes.TOOL,
"search",
metadata,
ExecutionLifecycleEvents.executionStarted());
- listener.onComponentExecution(
+ report(
+ listener,
ExecutionReporter.EntityTypes.TOOL,
"search",
metadata,
@@ -121,7 +188,8 @@ void terminalReportWithoutStartGetsAFreshExecutionId() {
EventLogComponentExecutionListener listener =
new EventLogComponentExecutionListener(actionContext, sink(logger));
- listener.onComponentExecution(
+ report(
+ listener,
ExecutionReporter.EntityTypes.PARSER,
"json-parser",
Map.of(),
@@ -134,32 +202,34 @@ void terminalReportWithoutStartGetsAFreshExecutionId() {
}
@Test
- void repeatedStartReportReplacesTheActiveReport() {
+ void repeatedStartReportReusesTheActiveExecution() {
CapturingEventLogger logger = new CapturingEventLogger();
EventLogComponentExecutionListener listener =
new EventLogComponentExecutionListener(actionTraceContext(), sink(logger));
- listener.onComponentExecution(
+ report(
+ listener,
ExecutionReporter.EntityTypes.LLM,
"model-a",
Map.of(),
ExecutionLifecycleEvents.executionStarted());
- listener.onComponentExecution(
+ report(
+ listener,
ExecutionReporter.EntityTypes.LLM,
"model-a",
Map.of(),
ExecutionLifecycleEvents.executionStarted());
- listener.onComponentExecution(
+ report(
+ listener,
ExecutionReporter.EntityTypes.LLM,
"model-a",
Map.of(),
ExecutionLifecycleEvents.executionFinished());
assertThat(logger.records).hasSize(3);
- // The terminal pairs with the second start; the first start stays unpaired.
- assertThat(logger.records.get(2).traceContext.getExecutionId())
- .isEqualTo(logger.records.get(1).traceContext.getExecutionId())
- .isNotEqualTo(logger.records.get(0).traceContext.getExecutionId());
+ assertThat(logger.records)
+ .extracting(record -> record.traceContext.getExecutionId())
+ .containsOnly(logger.records.get(0).traceContext.getExecutionId());
}
@Test
@@ -171,7 +241,8 @@ void disabledTraceSwitchSuppressesExecutionRecords() {
ExecutionEventLogger.forEventLogWriter(
EventLogWriter.forEventLogger(logger, false)));
- listener.onComponentExecution(
+ report(
+ listener,
ExecutionReporter.EntityTypes.LLM,
"model-a",
Map.of(),
@@ -185,6 +256,16 @@ private static ExecutionTraceContext actionTraceContext() {
.childExecution("action", "chat_model_action");
}
+ private static void report(
+ EventLogComponentExecutionListener listener,
+ String entityType,
+ String entityName,
+ Map entityMetadata,
+ Event event) {
+ listener.onComponentExecution(
+ entityType, entityName, entityMetadata, new EventContext(event), event);
+ }
+
private static ExecutionEventSink sink(EventLogger logger) {
return ExecutionEventLogger.forEventLogWriter(EventLogWriter.forEventLogger(logger));
}