Add token/s on dashboard and reporting token/s in the metrics endpoint

Makefile

@@ -59,6 +59,9 @@ target/release/paddler_gui: $(PADDLER_SOURCES) esbuild-meta.json
 target/vulkan/release/paddler: $(PADDLER_SOURCES) esbuild-meta.json
 	cargo build --release -p paddler_cli --features vulkan,web_admin_panel --target-dir target/vulkan
 
+target/vulkan/release/paddler_gui: $(PADDLER_SOURCES) esbuild-meta.json
+	cargo build --release -p paddler_gui --features vulkan,web_admin_panel --target-dir target/vulkan
+
 # -----------------------------------------------------------------------------
 # Phony targets
 # -----------------------------------------------------------------------------

paddler_agent/src/continuous_batch_arbiter.rs

@@ -52,6 +52,7 @@ pub struct ContinuousBatchArbiter {
     pub agent_name: Option<String>,
     pub chat_template_override: Option<ChatTemplate>,
     pub desired_slots_total: i32,
+    pub gpu_devices: Vec<usize>,
     pub inference_parameters: InferenceParameters,
     pub multimodal_projection_path: Option<PathBuf>,
     pub model_metadata_holder: Arc<ModelMetadataHolder>,
@@ -66,6 +67,7 @@ impl ContinuousBatchArbiter {
         applicable_state: AgentApplicableState,
         agent_name: Option<String>,
         desired_slots_total: i32,
+        gpu_devices: Vec<usize>,
         model_metadata_holder: Arc<ModelMetadataHolder>,
         slot_aggregated_status_manager: Arc<SlotAggregatedStatusManager>,
     ) -> ContinuousBatchArbiterBuildOutcome {
@@ -79,6 +81,7 @@ impl ContinuousBatchArbiter {
             agent_name,
             chat_template_override: applicable_state.chat_template_override,
             desired_slots_total,
+            gpu_devices,
             inference_parameters: applicable_state.inference_parameters,
             multimodal_projection_path: applicable_state.multimodal_projection_path,
             model_metadata_holder,
@@ -108,6 +111,7 @@ impl ContinuousBatchArbiter {
 
         let agent_name_clone = self.agent_name.clone();
         let desired_slots_total = self.desired_slots_total;
+        let gpu_devices = self.gpu_devices.clone();
         let inference_parameters = self.inference_parameters.clone();
         let model_metadata_holder = self.model_metadata_holder.clone();
         let multimodal_projection_path = self.multimodal_projection_path.clone();
@@ -148,14 +152,18 @@ impl ContinuousBatchArbiter {
                         .to_llama_kv_cache_dtype(),
                 );
 
+            let mut model_params =
+                LlamaModelParams::default().with_n_gpu_layers(inference_parameters.n_gpu_layers);
+
+            if !gpu_devices.is_empty() {
+                model_params = model_params
+                    .with_devices(&gpu_devices)
+                    .context("Invalid --gpu-devices index")?;
+            }
+
             let model = Arc::new(
-                LlamaModel::load_from_file(
-                    &llama_backend,
-                    model_path.clone(),
-                    &LlamaModelParams::default()
-                        .with_n_gpu_layers(inference_parameters.n_gpu_layers),
-                )
-                .context("Unable to load model from file")?,
+                LlamaModel::load_from_file(&llama_backend, model_path.clone(), &model_params)
+                    .context("Unable to load model from file")?,
             );
 
             send_startup_signal(
@@ -306,6 +314,7 @@ impl ContinuousBatchArbiter {
                 inference_parameters,
                 model_path: model_path.clone(),
                 multimodal_context,
+                slot_aggregated_status: slot_aggregated_status_manager.slot_aggregated_status.clone(),
                 token_bos_str: model.token_to_piece(
                     &SampledToken::Content(model.token_bos()),
                     &mut special_token_decoder,

paddler_agent/src/continuous_batch_scheduler/advance_generating_phase.rs

@@ -49,7 +49,13 @@ impl AdvanceGeneratingPhase<'_> {
         })
         .run(request, batch_index)
         {
-            SampleOutcome::Sampled(token) => token,
+            SampleOutcome::Sampled(token) => {
+                self.scheduler_context
+                    .slot_aggregated_status
+                    .record_generated_token();
+
+                token
+            }
             SampleOutcome::AllCandidatesEliminated => {
                 error!(
                     "{:?}: sequence {} sampling exhausted candidates",

paddler_agent/src/continuous_batch_scheduler_context.rs

@@ -6,6 +6,7 @@ use llama_cpp_bindings::mtmd::MtmdContext;
 use paddler_messaging::inference_parameters::InferenceParameters;
 
 use crate::chat_template_renderer::ChatTemplateRenderer;
+use crate::slot_aggregated_status::SlotAggregatedStatus;
 
 pub struct ContinuousBatchSchedulerContext {
     pub agent_name: Option<String>,
@@ -15,6 +16,7 @@ pub struct ContinuousBatchSchedulerContext {
     pub model: Arc<LlamaModel>,
     pub model_path: PathBuf,
     pub multimodal_context: Option<Arc<MtmdContext>>,
+    pub slot_aggregated_status: Arc<SlotAggregatedStatus>,
     pub token_bos_str: String,
     pub token_eos_str: String,
     pub token_nl_str: String,

paddler_agent/src/lib.rs

@@ -32,6 +32,7 @@ pub mod embedding_input_tokenized;
 mod from_request_params;
 pub mod generate_embedding_batch_request;
 pub mod grammar_sampler;
+pub mod list_gpu_devices;
 pub mod llamacpp_arbiter_service;
 pub mod management_socket_client_service;
 pub mod model_metadata_holder;
@@ -59,6 +60,7 @@ pub mod slot_aggregated_status;
 pub mod slot_aggregated_status_download_progress;
 pub mod slot_aggregated_status_manager;
 pub mod slot_guard;
+pub mod token_throughput_meter;
 pub mod tool_call_buffer;
 pub mod tool_call_event;
 pub mod tool_call_pipeline;

paddler_agent/src/list_gpu_devices.rs

@@ -0,0 +1,18 @@
+use anyhow::Context;
+use anyhow::Result;
+use llama_cpp_bindings::llama_backend::LlamaBackend;
+
+pub use llama_cpp_bindings::LlamaBackendDevice;
+pub use llama_cpp_bindings::LlamaBackendDeviceType;
+
+/// Lists every backend device (GPU, integrated GPU, accelerator, or CPU) that llama.cpp can see
+/// on this machine, in the same index order accepted by `--gpu-devices`.
+///
+/// # Errors
+/// Returns an error if the llama.cpp backend fails to initialize.
+pub fn list_gpu_devices() -> Result<Vec<LlamaBackendDevice>> {
+    let _llama_backend =
+        LlamaBackend::init().context("Unable to initialize llama.cpp backend")?;
+
+    Ok(llama_cpp_bindings::list_llama_ggml_backend_devices())
+}

paddler_agent/src/llamacpp_arbiter_service.rs

@@ -33,6 +33,7 @@ async fn apply_state(
     agent_applicable_state: Option<&AgentApplicableState>,
     agent_name: Option<&str>,
     desired_slots_total: i32,
+    gpu_devices: &[usize],
     model_metadata_holder: &Arc<ModelMetadataHolder>,
     slot_aggregated_status_manager: &Arc<SlotAggregatedStatusManager>,
     continuous_batch_arbiter_handle: &mut Option<ContinuousBatchArbiterHandle>,
@@ -52,6 +53,7 @@ async fn apply_state(
             applicable_state,
             agent_name.map(str::to_owned),
             desired_slots_total,
+            gpu_devices.to_vec(),
             model_metadata_holder.clone(),
             slot_aggregated_status_manager.clone(),
         ) {
@@ -110,6 +112,7 @@ async fn try_to_apply_state(
     agent_applicable_state: Option<&AgentApplicableState>,
     agent_name: Option<&str>,
     desired_slots_total: i32,
+    gpu_devices: &[usize],
     model_metadata_holder: &Arc<ModelMetadataHolder>,
     slot_aggregated_status_manager: &Arc<SlotAggregatedStatusManager>,
     continuous_batch_arbiter_handle: &mut Option<ContinuousBatchArbiterHandle>,
@@ -119,6 +122,7 @@ async fn try_to_apply_state(
         agent_applicable_state,
         agent_name,
         desired_slots_total,
+        gpu_devices,
         model_metadata_holder,
         slot_aggregated_status_manager,
         continuous_batch_arbiter_handle,
@@ -150,6 +154,7 @@ pub struct LlamaCppArbiterService {
     pub continue_from_raw_prompt_request_rx: mpsc::UnboundedReceiver<ContinueFromRawPromptRequest>,
     pub desired_slots_total: i32,
     pub generate_embedding_batch_request_rx: mpsc::UnboundedReceiver<GenerateEmbeddingBatchRequest>,
+    pub gpu_devices: Vec<usize>,
     pub continuous_batch_arbiter_handle: Option<ContinuousBatchArbiterHandle>,
     pub model_metadata_holder: Arc<ModelMetadataHolder>,
     pub slot_aggregated_status_manager: Arc<SlotAggregatedStatusManager>,
@@ -170,6 +175,7 @@ impl Service for LlamaCppArbiterService {
             mut continue_from_raw_prompt_request_rx,
             desired_slots_total,
             mut generate_embedding_batch_request_rx,
+            gpu_devices,
             mut continuous_batch_arbiter_handle,
             model_metadata_holder,
             slot_aggregated_status_manager,
@@ -203,6 +209,7 @@ impl Service for LlamaCppArbiterService {
                             agent_applicable_state.as_ref(),
                             agent_name.as_deref(),
                             desired_slots_total,
+                            &gpu_devices,
                             &model_metadata_holder,
                             &slot_aggregated_status_manager,
                             &mut continuous_batch_arbiter_handle,
@@ -220,6 +227,7 @@ impl Service for LlamaCppArbiterService {
                         agent_applicable_state.as_ref(),
                         agent_name.as_deref(),
                         desired_slots_total,
+                        &gpu_devices,
                         &model_metadata_holder,
                         &slot_aggregated_status_manager,
                         &mut continuous_batch_arbiter_handle,
@@ -353,6 +361,7 @@ mod tests {
             None,
             None,
             1,
+            &[],
             &model_metadata_holder,
             &slot_aggregated_status_manager,
             &mut continuous_batch_arbiter_handle,
@@ -421,6 +430,7 @@ mod tests {
             continue_from_raw_prompt_request_rx,
             desired_slots_total: 1,
             generate_embedding_batch_request_rx,
+            gpu_devices: Vec::new(),
             continuous_batch_arbiter_handle: None,
             model_metadata_holder: Arc::new(ModelMetadataHolder::default()),
             slot_aggregated_status_manager: Arc::new(SlotAggregatedStatusManager::new(1)),

paddler_agent/src/slot_aggregated_status.rs

@@ -12,6 +12,7 @@ use tokio::sync::watch;
 
 use crate::agent_issue_fix::AgentIssueFix;
 use crate::dispenses_slots::DispensesSlots;
+use crate::token_throughput_meter::TokenThroughputMeter;
 use paddler_messaging::atomic_value::AtomicValue;
 use paddler_messaging::produces_snapshot::ProducesSnapshot;
 use paddler_messaging::subscribes_to_updates::SubscribesToUpdates;
@@ -27,6 +28,7 @@ pub struct SlotAggregatedStatus {
     slots_processing: AtomicValue<AtomicI32>,
     slots_total: AtomicValue<AtomicI32>,
     state_application_status_code: AtomicValue<AtomicI32>,
+    token_throughput_meter: TokenThroughputMeter,
     update_tx: watch::Sender<()>,
     uses_chat_template_override: AtomicValue<AtomicBool>,
     version: AtomicValue<AtomicI32>,
@@ -50,6 +52,7 @@ impl SlotAggregatedStatus {
             ),
             slots_processing: AtomicValue::<AtomicI32>::new(0),
             slots_total: AtomicValue::<AtomicI32>::new(0),
+            token_throughput_meter: TokenThroughputMeter::new(),
             update_tx,
             uses_chat_template_override: AtomicValue::<AtomicBool>::new(false),
             version: AtomicValue::<AtomicI32>::new(0),
@@ -174,6 +177,21 @@ impl SlotAggregatedStatus {
     pub fn slots_processing_count(&self) -> i32 {
         self.slots_processing.get()
     }
+
+    /// Records that this agent has just generated one token, contributing to
+    /// the tokens-per-second rate reported in status snapshots.
+    ///
+    /// This does not bump `version` or notify subscribers: it happens once
+    /// per generated token, far too often to treat as a state change worth
+    /// pushing immediately. The periodic status update (sent once a second
+    /// regardless of `version`) is what picks the latest rate up.
+    pub fn record_generated_token(&self) {
+        self.token_throughput_meter.record_token();
+    }
+
+    pub fn tokens_per_second(&self) -> f64 {
+        self.token_throughput_meter.tokens_per_second()
+    }
 }
 
 impl DispensesSlots for SlotAggregatedStatus {
@@ -211,6 +229,7 @@ impl ProducesSnapshot for SlotAggregatedStatus {
             slots_processing: self.slots_processing.get(),
             slots_total: self.slots_total.get(),
             state_application_status: self.state_application_status_code.get().try_into()?,
+            tokens_per_second: self.tokens_per_second(),
             uses_chat_template_override: self.uses_chat_template_override.get(),
             version: self.version.get(),
         })
@@ -341,6 +360,36 @@ mod tests {
         );
     }
 
+    #[test]
+    fn make_snapshot_reports_zero_tokens_per_second_before_any_generation() {
+        let status = SlotAggregatedStatus::new(1);
+
+        let snapshot = status.make_snapshot().unwrap();
+
+        assert_eq!(snapshot.tokens_per_second, 0.0);
+    }
+
+    #[test]
+    fn record_generated_token_is_reflected_in_snapshot_after_a_window_closes() {
+        let status = SlotAggregatedStatus::new(1);
+
+        for _ in 0..5 {
+            status.record_generated_token();
+        }
+
+        std::thread::sleep(std::time::Duration::from_millis(1050));
+
+        status.record_generated_token();
+
+        let snapshot = status.make_snapshot().unwrap();
+
+        assert!(
+            snapshot.tokens_per_second > 0.0,
+            "expected a positive tokens_per_second, got {}",
+            snapshot.tokens_per_second
+        );
+    }
+
     #[test]
     fn get_state_application_status_reflects_set_value() {
         let status = SlotAggregatedStatus::new(2);