Add read-only Loki log discovery and queries

docs/_static/install-builder.mjs

@@ -2,18 +2,23 @@
 // SPDX-License-Identifier: Apache-2.0
 
 const chart = 'oci://nvcr.io/0948643769302270/afoa-release/mosaic-stack';
-const modules = {
-  kubernetes: 'Kubernetes', observability: 'Prometheus and Alertmanager', grafana: 'Grafana',
+export const modules = {
+  kubernetes: 'Kubernetes', prometheus: 'Prometheus metrics', alertmanager: 'Alertmanager alerts', loki: 'Loki logs', grafana: 'Grafana',
   bcm: 'Base Command Manager', slurm: 'Slurm', diagnostics: 'Hardware diagnostics',
   research: 'Research', terminal: 'Browser terminal', edit: 'Guarded Kubernetes edits', clusters: 'Cluster monitoring',
 };
+const observabilityServices = ['prometheus', 'alertmanager', 'loki'];
 const fields = {
   namespace: ['Namespace', 'mosaic'], pullSecret: ['NGC pull Secret', 'nvcr-image-pull-secret'],
   version: ['Pinned chart version (optional)', '', 'e.g. 0.0.1'],
   baseUrl: ['Inference API base URL', '', 'https://inference.example.com/v1'],
   model: ['Inference model', '', 'Your served model name'], llmSecret: ['Inference Secret (apiKey)', 'mosaic-external-llm'],
   prometheusUrl: ['Prometheus URL', '', 'http://prometheus:9090'],
   alertmanagerUrl: ['Alertmanager URL', '', 'http://alertmanager:9093'],
+  lokiUrl: ['Loki URL', '', 'https://logs.example.com'],
+  lokiTenant: ['Loki tenant ID (optional)', ''],
+  lokiSecret: ['Loki Authorization Secret (optional)', ''],
+  lokiKey: ['Loki Secret key', 'authorization'],
   grafanaUrl: ['Grafana URL, including serving path', '', 'https://grafana.example.com/grafana'],
   grafanaUid: ['Prometheus datasource UID', 'prometheus'], grafanaSecret: ['Grafana Secret (username, password)', 'mosaic-grafana-auth'],
   bcmHead: ['BCM SSH head host', '', 'user@bcm-head.example.com'], bcmSecret: ['BCM SSH Secret (id_ecdsa)', 'bcm-host-ssh-key'],
@@ -25,6 +30,7 @@ export function enabledModules(state) {
   const enabled = new Set(state.modules || []);
   if (enabled.has('diagnostics') || (enabled.has('slurm') && state.slurmBackend !== 'vanilla')) enabled.add('bcm');
   if (['terminal', 'edit', 'clusters'].some(name => enabled.has(name))) enabled.add('kubernetes');
+  if ([...observabilityServices, 'grafana'].some(name => enabled.has(name))) enabled.add('observability');
   return enabled;
 }
 const quote = value => `'${String(value).replaceAll("'", "'\\''")}'`;
@@ -39,10 +45,15 @@ export function buildCommand(state) {
     set('llm.external.baseUrl', value('baseUrl'), true); set('llm.external.model', value('model'), true);
     set('llm.external.existingSecret', value('llmSecret'), true);
   }
-  for (const name of ['ui', 'execution', ...Object.keys(modules)]) set(`modules.${name}.enabled`, ['ui', 'execution'].includes(name) || enabled.has(name));
-  if (enabled.has('observability')) {
-    set('observability.prometheusUrl', value('prometheusUrl'), true);
-    set('observability.alertmanagerUrl', value('alertmanagerUrl'), true);
+  for (const name of ['ui', 'execution', 'observability', ...Object.keys(modules).filter(name => !observabilityServices.includes(name))]) set(`modules.${name}.enabled`, ['ui', 'execution'].includes(name) || enabled.has(name));
+  for (const service of ['prometheus', 'alertmanager']) {
+    set(`observability.${service}Url`, enabled.has(service) ? value(`${service}Url`) : '', true);
+  }
+  set('observability.loki.url', enabled.has('loki') ? value('lokiUrl') : '', true);
+  if (enabled.has('loki')) {
+    for (const [key, field] of Object.entries({tenantId:'lokiTenant', 'auth.existingSecret':'lokiSecret', 'auth.authorizationKey':'lokiKey'})) {
+      set(`observability.loki.${key}`, state[field]?.trim() || fields[field][1], true);
+    }
   }
   if (enabled.has('grafana')) {
     set('observability.grafanaUrl', value('grafanaUrl'), true); set('observability.grafanaDatasourceUid', value('grafanaUid'), true);
@@ -120,7 +131,9 @@ if (root) {
       settingsBody.replaceChildren();
       const keys = ['namespace', 'pullSecret', 'version'];
       if (state.inference === 'external') keys.push('baseUrl', 'model', 'llmSecret');
-      if (enabled.has('observability')) keys.push('prometheusUrl', 'alertmanagerUrl');
+      if (enabled.has('prometheus')) keys.push('prometheusUrl');
+      if (enabled.has('alertmanager')) keys.push('alertmanagerUrl');
+      if (enabled.has('loki')) keys.push('lokiUrl', 'lokiTenant', 'lokiSecret', 'lokiKey');
       if (enabled.has('grafana')) keys.push('grafanaUrl', 'grafanaUid', 'grafanaSecret');
       if (enabled.has('slurm')) settingsBody.append(select('slurmBackend', 'Slurm integration', [['bcm', 'Through BCM'], ['vanilla', 'Standalone Slurm']]));
       if (enabled.has('bcm')) keys.push('bcmHead', 'bcmSecret');
@@ -136,6 +149,10 @@ if (root) {
     }
     const notes = ['Complete namespace and NGC access setup first. Replace all <placeholders> before running.'];
     if (state.inference === 'external') notes.push(`Create ${state.llmSecret || fields.llmSecret[1]} with key apiKey.`);
+    if (enabled.has('loki')) {
+      notes.push('Connect to an existing Loki endpoint; no Loki server or collector is installed.');
+      if (state.lokiUrl?.trim() && state.lokiSecret?.trim()) notes.push(`Create ${state.lokiSecret.trim()} in the release namespace with key ${state.lokiKey?.trim() || fields.lokiKey[1]} containing the complete Authorization header. Do not enter credentials in this form.`);
+    }
     if (enabled.has('grafana')) notes.push(`Create ${state.grafanaSecret || fields.grafanaSecret[1]} with keys username and password.`);
     if (enabled.has('bcm')) notes.push(`Create ${state.bcmSecret || fields.bcmSecret[1]} with key id_ecdsa; its SSH identity must bootstrap the read-only CMSH user.`);
     if (enabled.has('research')) notes.push('Create iraop-secrets with NVIDIA_API_KEY, NVIDIA_CHAT_API_KEY and IRAOP_API_KEY; prepare the corpus directory.');

docs/installation.md

@@ -216,6 +216,26 @@ helm upgrade mosaic "$MOSAIC_CHART" \
   --timeout 12m
 ```
 
+### Read-only Loki logs
+
+To connect Mosaic to an existing Loki service, run:
+
+```bash
+export LOKI_URL='https://logs.example.com'
+helm upgrade mosaic "$MOSAIC_CHART" \
+  --devel \
+  -n mosaic \
+  --reuse-values \
+  --set modules.observability.enabled=true \
+  --set-string observability.loki.url="$LOKI_URL" \
+  --wait \
+  --timeout 12m
+```
+
+Mosaic queries existing logs; it does not install Loki or a collector. For endpoints
+requiring authentication or a tenant ID, see [Loki connection options](loki.md#connection-options).
+Verify access by asking Mosaic to show 10 log entries from the past 15 minutes.
+
 ### Alertmanager
 
 AI Factory Operations Agent defaults to the Alertmanager service installed by NMC at `http://kube-prometheus-stack-alertmanager.prometheus.svc.cluster.local:9093`. To use another Alertmanager endpoint, run:

docs/loki.md

@@ -0,0 +1,66 @@
+# Loki query reference
+
+For connection setup and verification, see [Read-only Loki logs](installation.md#read-only-loki-logs).
+
+## Connection options
+
+If Loki requires authentication, provision a Secret in Mosaic's namespace using
+your secret-management process:
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: loki-readonly
+  namespace: mosaic
+type: Opaque
+stringData:
+  authorization: "Bearer <your-read-only-token>"
+```
+
+For basic authentication, use `Basic <base64(username:password)>` instead.
+Keep credentials out of Helm values and source control.
+
+Add these options to the installation command, replacing the Secret name and
+tenant ID with your values:
+
+```bash
+--set-string observability.loki.auth.existingSecret=loki-readonly \
+--set-string observability.loki.tenantId=your-tenant-id
+```
+
+Omit the Secret option if authentication is not required, and omit the tenant
+option if your Loki service does not require a tenant ID. The Secret's default
+key is `authorization`; use `observability.loki.auth.authorizationKey` if yours
+differs. The Secret must exist in the release namespace before upgrading.
+
+## Ask about your logs
+
+In View mode, ask Mosaic to find logs, count matching events, or show how activity
+changes over time. For example:
+
+- “Which services have logs available?”
+- “Show errors from the scheduler over the past 15 minutes.”
+- “How many errors did that service log in the last hour?”
+- “Show its error rate over the last hour.”
+
+Include the service or host and the time range you want to investigate.
+
+## Query limits
+
+| Query setting | Default | Maximum |
+| --- | --- | --- |
+| Range query time window | Past hour | 24 hours |
+| Returned entries | 100 | 1,000 |
+| Upstream response size | — | 2 MiB |
+| Request duration | — | 15 seconds |
+
+If a query reaches a limit, narrow the source labels or time window.
+
+Connection behavior:
+
+- Use HTTPS for remote endpoints. Internal HTTP service URLs are supported.
+- Redirects are rejected; configure the endpoint directly.
+- `tenantId` supplies the `X-Scope-OrgID` header. It does not replace authentication.
+- An empty `observability.loki.url` disables log queries. Apply connection changes
+  with a Helm upgrade.

docs/slurm_rca.md

@@ -17,3 +17,14 @@ Default evidence candidates:
 - `/var/log/journal`
 
 Sites with non-standard Slurm paths can adjust `slurmEvidenceCollector.roots`; the default install already attempts the common locations above.
+
+## Job logs in Loki
+
+To investigate Slurm jobs through Loki without head-node access, configure your
+logging platform to collect job output and Slurm daemon logs. Include job IDs and
+node names as labels or searchable fields so Mosaic can correlate logs with job
+accounting and node metrics. Collecting Kubernetes logs alone does not provide
+Slurm job logs.
+
+[Connect Mosaic to Loki](installation.md#read-only-loki-logs), then verify access
+by asking for logs from a known job ID and its execution time range.

helm/mosaic-stack/files/openclaw-seed/extensions/automation-context.ts

@@ -65,6 +65,8 @@ const READ_ONLY_TOOLS = new Set([
   "observability_metric_names",
   "observability_query",
   "observability_range_query",
+  "observability_log_labels",
+  "observability_logs",
   "pdf",
   "read",
   "run_kubectl",

helm/mosaic-stack/files/openclaw-seed/extensions/grafana/index.ts

@@ -8,8 +8,9 @@ declare const Buffer: { from(value: string): { toString(encoding: string): strin
 type PanelSpec = {
   title: string;
   query: string;
+  datasource?: { type: "prometheus" | "loki"; uid: string };
   unit?: string;
-  visualization?: "timeseries" | "stat" | "gauge" | "table";
+  visualization?: "timeseries" | "stat" | "gauge" | "table" | "logs";
   legend?: string;
   description?: string;
 };
@@ -120,7 +121,9 @@ function normalizePanel(raw: unknown, index: number): PanelSpec | null {
   const value = raw as Record<string, unknown>;
   const query = stringParam(value.query);
   if (!query) return null;
+  const datasource = value.datasource as PanelSpec["datasource"];
   return {
+    datasource,
     title: stringParam(value.title) || `Panel ${index + 1}`,
     query,
     unit: stringParam(value.unit),
@@ -159,8 +162,9 @@ function buildGrafanaQueryPayload(panel: PanelSpec, rangeMinutes: number, dataso
     queries: [
       {
         refId: "A",
-        datasource: { type: "prometheus", uid: datasourceUid },
+        datasource: panel.datasource || { type: "prometheus", uid: datasourceUid },
         expr: panel.query,
+        ...(panel.datasource?.type === "loki" ? {queryType: "range", maxLines: 1000} : {}),
         range: true,
         instant: false,
         intervalMs: 15000,
@@ -175,8 +179,9 @@ function buildGrafanaQueryPayload(panel: PanelSpec, rangeMinutes: number, dataso
 function grafanaFrameCount(payload: Record<string, unknown>) {
   const results = payload.results as Record<string, unknown> | undefined;
   const first = results?.A as Record<string, unknown> | undefined;
+  if (first?.error) throw new Error(String(first.error));
   const frames = Array.isArray(first?.frames) ? first.frames : [];
-  return frames.length;
+  return frames.filter((frame: any) => frame.data?.values?.some((values: unknown[]) => values.length > 0)).length;
 }
 
 function authHeaders(pluginConfig: unknown) {
@@ -213,14 +218,24 @@ async function validatePanels(
   const issues: ValidationIssue[] = [];
   const results: Array<{ title: string; query: string; seriesCount: number; grafanaFrameCount: number; sample?: unknown }> = [];
 
+  const datasources = await fetchJson(`${grafanaUrl}/api/datasources`, {headers}) as unknown as Array<{uid: string; type: string; name: string}>;
   for (const panel of panels) {
+    const selected = panel.datasource || {type: "prometheus", uid: datasourceUid};
+    if (!["prometheus", "loki"].includes(selected.type) || !datasources.some(source => source.uid === selected.uid && source.type === selected.type)) {
+      issues.push({panelTitle: panel.title, query: panel.query, severity: "error", message: "Panel datasource UID or type is missing or unavailable. Select the intended datasource from availableDatasources and retry."});
+      continue;
+    }
+    if (panel.visualization === "logs" && selected.type !== "loki") {
+      issues.push({panelTitle: panel.title, query: panel.query, severity: "error", message: "Log panels require a Loki datasource. Select one from availableDatasources."});
+      continue;
+    }
     const url = new URL(`${prometheusUrl}/api/v1/query`);
     url.searchParams.set("query", panel.query);
     try {
-      const payload = await fetchJson(url.toString());
+      const payload = selected.type === "prometheus" ? await fetchJson(url.toString()) : {};
       const data = payload.data as { result?: unknown[] } | undefined;
       const result = Array.isArray(data?.result) ? data.result : [];
-      if (requireData && result.length === 0) {
+      if (requireData && selected.type === "prometheus" && result.length === 0) {
         issues.push({
           panelTitle: panel.title,
           query: panel.query,
@@ -290,6 +305,7 @@ async function validatePanels(
     ok: issues.filter((issue) => issue.severity === "error").length === 0,
     issues,
     results,
+    availableDatasources: datasources.map(({uid, type, name}) => ({uid, type, name})),
   };
 }
 
@@ -310,8 +326,12 @@ function pathFromGrafanaUrl(value: string) {
   return value.startsWith("/") ? value : `/${value}`;
 }
 
-function ensureGrafanaSubPath(path: string) {
-  return path.startsWith(GRAFANA_PROXY_PREFIX) ? path : `${GRAFANA_PROXY_PREFIX}${path}`;
+function ensureGrafanaSubPath(path: string, grafanaUrl: string) {
+  if (path.startsWith(GRAFANA_PROXY_PREFIX)) return path;
+  const pathname = new URL(grafanaUrl).pathname;
+  const base = pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
+  const relative = base && path.startsWith(`${base}/`) ? path.slice(base.length) : path;
+  return `${GRAFANA_PROXY_PREFIX}${relative}`;
 }
 
 function appendTimeRange(path: string, rangeMinutes: number) {
@@ -327,7 +347,7 @@ function buildPanel(panel: PanelSpec, index: number, datasourceUid: string) {
     type,
     title: panel.title,
     description: panel.description,
-    datasource: { type: "prometheus", uid: datasourceUid },
+    datasource: panel.datasource || { type: "prometheus", uid: datasourceUid },
     gridPos: {
       h: type === "stat" || type === "gauge" ? 6 : 8,
       w: width,
@@ -338,6 +358,7 @@ function buildPanel(panel: PanelSpec, index: number, datasourceUid: string) {
       {
         refId: "A",
         expr: panel.query,
+        ...(panel.datasource?.type === "loki" ? {queryType: "range", maxLines: 1000} : {}),
         range: true,
         legendFormat: panel.legend || "{{Hostname}} {{gpu}} {{node}} {{device}} {{port}}",
       },
@@ -414,25 +435,25 @@ export default definePluginEntry({
     registerTool({
       name: "grafana_dashboard_presets",
       label: "Grafana Dashboard Presets",
-      description: "List dashboard presets the agent can use as starting points.",
+      description: "Discover Grafana datasource names, types, and UIDs and list dashboard presets.",
       parameters: { type: "object", additionalProperties: false, properties: {} },
       async execute() {
-        return jsonToolResult({ presets: PRESETS });
+        return jsonToolResult({ presets: PRESETS, datasources: await fetchJson(`${grafanaUrl}/api/datasources`, {headers}) });
       },
     });
 
     registerTool({
       name: "grafana_dashboard_validate",
       label: "Validate Grafana Dashboard Queries",
-      description: "Validate dashboard panel PromQL against observability Prometheus before creating a dashboard.",
+      description: "Validate dashboard panel PromQL or LogQL against the selected Grafana datasource before creating a dashboard.",
       parameters: {
         type: "object",
         additionalProperties: false,
         properties: {
           preset: { type: "string", description: `Optional preset: ${Object.keys(PRESETS).join(", ")}.` },
           panels: {
             type: "array",
-            description: "Panel specs with title/query/unit/visualization.",
+            description: "Panel specs: title, query (PromQL or LogQL), datasource {type: prometheus or loki, uid: discovered Grafana UID}, visualization (timeseries, stat, gauge, table, logs), optional unit/legend/description. Omit datasource only to use the configured Prometheus default.",
             items: { type: "object" },
           },
           requireData: { type: "boolean", description: "If true, empty query results are validation errors. Default true." },
@@ -460,7 +481,7 @@ export default definePluginEntry({
     registerTool({
       name: "grafana_dashboard_create",
       label: "Create Grafana Dashboard",
-      description: "Create a Grafana dashboard after validating every panel query, then open it in the UI Grafana tab.",
+      description: "Create a Grafana dashboard after validating every Prometheus or Loki panel query, then open it in the UI Grafana tab.",
       parameters: {
         type: "object",
         additionalProperties: false,
@@ -469,7 +490,7 @@ export default definePluginEntry({
           preset: { type: "string", description: `Optional preset: ${Object.keys(PRESETS).join(", ")}.` },
           panels: {
             type: "array",
-            description: "Panel specs with title/query/unit/visualization.",
+            description: "Panel specs: title, query (PromQL or LogQL), datasource {type: prometheus or loki, uid: discovered Grafana UID}, visualization (timeseries, stat, gauge, table, logs), optional unit/legend/description. Omit datasource only to use the configured Prometheus default.",
             items: { type: "object" },
           },
           rangeMinutes: { type: "number", description: "Dashboard time range in minutes. Default 60." },
@@ -504,9 +525,9 @@ export default definePluginEntry({
 
         const uid = stringParam(createPayload.uid) || stringParam((dashboard as Record<string, unknown>).uid) || "";
         const url = stringParam(createPayload.url) || `/d/${uid}/${slugify(title)}`;
-        const grafanaPath = ensureGrafanaSubPath(pathFromGrafanaUrl(url));
+        const grafanaPath = ensureGrafanaSubPath(pathFromGrafanaUrl(url), grafanaUrl);
         const iframeUrl = appendTimeRange(grafanaPath, rangeMinutes);
-        const dashboardUrl = `${grafanaUrl}${grafanaPath}`;
+        const dashboardUrl = `${grafanaUrl}${grafanaPath.slice(GRAFANA_PROXY_PREFIX.length)}`;
         let uiNotification: unknown = null;
 
         if (boolParam(rawParams.openInUi, true)) {
@@ -559,7 +580,7 @@ export default definePluginEntry({
         if (!uid) throw new Error("uid is required");
         const title = stringParam(rawParams.title) || uid;
         const rangeMinutes = Math.max(1, Math.min(10080, numberParam(rawParams.rangeMinutes, 60)));
-        const path = ensureGrafanaSubPath(`/d/${uid}/${slugify(title)}`);
+        const path = ensureGrafanaSubPath(`/d/${uid}/${slugify(title)}`, grafanaUrl);
         const iframeUrl = appendTimeRange(path, rangeMinutes);
         const uiNotification = await fetchJson(`${uiUrl}/api/ui/actions`, {
           method: "POST",
@@ -569,7 +590,7 @@ export default definePluginEntry({
             title,
             iframeUrl,
             dashboardUid: uid,
-            dashboardUrl: `${grafanaUrl}${path}`,
+            dashboardUrl: `${grafanaUrl}${path.slice(GRAFANA_PROXY_PREFIX.length)}`,
           }),
         });
         return jsonToolResult({ opened: true, uid, iframeUrl, uiNotification });