Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 7 additions & 1 deletion docs/monitoring.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ sup.monitors().add(Tracer).await;

### ActorMonitor

Tracks actor lifecycle and overflow counts. Clone it to query from any thread:
Tracks actor lifecycle and per-actor event flow metrics. Clone it to query from any thread:

```rust
use maiko::monitors::ActorMonitor;
Expand All @@ -89,6 +89,12 @@ sup.monitors().add(monitor).await;
// Later, from any thread:
query.is_alive(&actor_id);
query.overflow_count(&actor_id);
query.dispatched_count(&actor_id);
query.delivered_count(&actor_id);
query.handled_count(&actor_id);
query.error_count(&actor_id);
query.queue_depth(&actor_id);
query.state(&actor_id);
query.actors(); // snapshot of active actor IDs
query.stopped_actors(); // snapshot of stopped actor IDs
```
Expand Down
1 change: 1 addition & 0 deletions maiko/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ serde = ["dep:serde", "dep:serde_json"]
test-harness = ["monitoring"]

[dependencies]
dashmap = "6"
futures-util = { version = "0.3", default-features = false, features = ["alloc"] }
maiko-macros = { version = "=0.3.1", optional = true, path = "../maiko-macros/" }
serde = { version = "1.0", features = ["derive", "rc"], optional = true }
Expand Down
189 changes: 165 additions & 24 deletions maiko/src/internal/actor_controller.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use std::future;
use std::sync::Arc;

use tokio::{
Expand Down Expand Up @@ -28,22 +29,46 @@ pub(crate) struct ActorController<A: Actor, T: Topic<A::Event>> {

impl<A: Actor, T: Topic<A::Event>> ActorController<A, T> {
pub async fn run(&mut self) -> Result {
// `select!` decides what should happen next without running actor work
// directly inside branch futures. The selected action is executed below
// in ordinary sequential code, which keeps borrow scopes simple.
enum Next<E> {
Continue,
Stop,
HandleEvent(Arc<Envelope<E>>),
RunStep { clear_backoff: bool },
}

self.actor.on_start().await?;
let mut step_handler = StepHandler::default();
loop {
select! {
let next = select! {
biased;

cmd_res = self.command_rx.recv() => match cmd_res {
Ok(cmd) => match cmd {
Command::StopActor(ref id) if id == self.ctx.actor_id() => break,
Command::StopRuntime => break,
_ => {}
Command::StopActor(ref id) if id == self.ctx.actor_id() => Next::Stop,
Command::StopRuntime => Next::Stop,
_ => Next::Continue,
}
Err(e) => return Err(Error::internal(e))
Err(e) => return Err(Error::internal(e)),
},

Some(event) = self.receiver.recv() => {
Some(event) = self.receiver.recv() => Next::HandleEvent(event),

_ = async {
if let Some(backoff_sleep) = step_handler.backoff.as_mut() {
backoff_sleep.as_mut().await;
}
}, if step_handler.is_delayed() => Next::RunStep { clear_backoff: true },

_ = future::ready(()), if step_handler.can_step() => Next::RunStep { clear_backoff: false },
};

match next {
Next::Continue => continue,
Next::Stop => break,
Next::HandleEvent(event) => {
self.handle_incoming_event(event).await?;

let mut cnt = 1;
Expand All @@ -58,36 +83,29 @@ impl<A: Actor, T: Topic<A::Event>> ActorController<A, T> {
step_handler.pause = StepPause::None;
}
}

_ = async {
if let Some(backoff_sleep) = step_handler.backoff.as_mut() {
backoff_sleep.as_mut().await;
Next::RunStep { clear_backoff } => {
if clear_backoff {
let _ = step_handler.backoff.take();
}
}, if step_handler.is_delayed() => {
let _ = step_handler.backoff.take();

#[cfg(feature = "monitoring")]
self.notify_step_enter();

match self.actor.step().await {
Ok(action) => handle_step_action(action, &mut step_handler).await,
Err(e) => {
Ok(action) => {
#[cfg(feature = "monitoring")]
self.notify_error(&e);
self.notify_step_exit(action);

self.actor.on_error(e)?;
step_handler.reset();
handle_step_action(action, &mut step_handler).await
}
}
}

res = self.actor.step(), if step_handler.can_step() => {
match res {
Ok(action) => handle_step_action(action, &mut step_handler).await,
Err(e) => {
#[cfg(feature = "monitoring")]
self.notify_error(&e);

self.actor.on_error(e)?;
step_handler.reset();
}
}
}
}
}
}
Expand Down Expand Up @@ -189,6 +207,24 @@ impl<A: Actor, T: Topic<A::Event>> ActorController<A, T> {
.send(MonitoringEvent::ActorStopped(self.ctx.actor_id().clone()));
}
}

#[inline]
fn notify_step_enter(&self) {
if self.monitoring.is_active() {
self.monitoring
.send(MonitoringEvent::StepEnter(self.ctx.actor_id().clone()));
}
}

#[inline]
fn notify_step_exit(&self, step_action: StepAction) {
if self.monitoring.is_active() {
self.monitoring.send(MonitoringEvent::StepExit(
step_action,
self.ctx.actor_id().clone(),
));
}
}
}

#[cfg(all(test, feature = "test-harness"))]
Expand Down Expand Up @@ -289,3 +325,108 @@ mod tests {
Ok(())
}
}

#[cfg(all(test, feature = "monitoring"))]
mod monitoring_tests {
//! Monitoring-specific regression tests for [`crate::internal::ActorController`].
//!
//! These cover runtime signals that are only emitted when the
//! `monitoring` feature is enabled, such as step enter/exit callbacks.

use crate::{
Actor, ActorId, DefaultTopic, Envelope, Event, StepAction, Subscribe, Supervisor,
monitoring::Monitor,
};
use std::{
sync::{Arc, Mutex},
time::Duration,
};
use tokio::time;

#[derive(Clone, Debug)]
struct TestEvent;
impl Event for TestEvent {}

#[derive(Debug, Clone, PartialEq, Eq)]
enum StepRecord {
Enter(ActorId),
Exit(StepAction, ActorId),
}

struct StepMonitor {
records: Arc<Mutex<Vec<StepRecord>>>,
}

impl Monitor<TestEvent, DefaultTopic> for StepMonitor {
fn on_step_enter(&self, actor_id: &ActorId) {
self.records
.lock()
.unwrap()
.push(StepRecord::Enter(actor_id.clone()));
}

fn on_step_exit(&self, step_action: &StepAction, actor_id: &ActorId) {
self.records
.lock()
.unwrap()
.push(StepRecord::Exit(*step_action, actor_id.clone()));
}
}

struct BackoffThenAwaitActor {
steps: usize,
}

impl Actor for BackoffThenAwaitActor {
type Event = TestEvent;

async fn handle_event(&mut self, _event: &Envelope<Self::Event>) -> crate::Result {
Ok(())
}

async fn step(&mut self) -> crate::Result<StepAction> {
self.steps += 1;
Ok(match self.steps {
1 => StepAction::Backoff(Duration::from_millis(1)),
_ => StepAction::AwaitEvent,
})
}
}

#[tokio::test]
async fn test_monitoring_emits_step_enter_and_exit() -> crate::Result<()> {
let mut sup = Supervisor::<TestEvent>::default();
let records = Arc::new(Mutex::new(Vec::new()));
let handle = sup
.monitors()
.add(StepMonitor {
records: records.clone(),
})
.await;

sup.add_actor(
"stepper",
|_| BackoffThenAwaitActor { steps: 0 },
Subscribe::none(),
)?;
sup.start().await?;

time::sleep(Duration::from_millis(10)).await;
handle.flush(Duration::from_millis(1)).await;

let stepper = ActorId::new("stepper");
let expected = vec![
StepRecord::Enter(stepper.clone()),
StepRecord::Exit(
StepAction::Backoff(Duration::from_millis(1)),
stepper.clone(),
),
StepRecord::Enter(stepper.clone()),
StepRecord::Exit(StepAction::AwaitEvent, stepper),
];

assert_eq!(*records.lock().unwrap(), expected);

sup.stop().await
}
}
6 changes: 6 additions & 0 deletions maiko/src/monitoring/dispatcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,12 @@ impl<E: Event, T: Topic<E>> MonitorDispatcher<E, T> {
EventHandled(envelope, topic, actor_id) => {
self.notify(|m| m.on_event_handled(&envelope, &topic, &actor_id));
}
StepEnter(actor_id) => {
self.notify(|m| m.on_step_enter(&actor_id));
}
StepExit(step_action, actor_id) => {
self.notify(|m| m.on_step_exit(&step_action, &actor_id));
}
Overflow(envelope, topic, actor_id, policy) => {
self.notify(|m| m.on_overflow(&envelope, &topic, &actor_id, policy));
}
Expand Down
4 changes: 3 additions & 1 deletion maiko/src/monitoring/monitoring_event.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
use std::sync::Arc;

use crate::{ActorId, Envelope, OverflowPolicy};
use crate::{ActorId, Envelope, OverflowPolicy, StepAction};

pub(crate) enum MonitoringEvent<E, T> {
EventDispatched(Arc<Envelope<E>>, Arc<T>, ActorId),
EventDelivered(Arc<Envelope<E>>, Arc<T>, ActorId),
EventHandled(Arc<Envelope<E>>, Arc<T>, ActorId),
StepEnter(ActorId),
StepExit(StepAction, ActorId),
Overflow(Arc<Envelope<E>>, Arc<T>, ActorId, OverflowPolicy),
ActorRegistered(ActorId),
ActorStopped(ActorId),
Expand Down
Loading
Loading