diff --git a/eng/dashboard/dashboard.html b/eng/dashboard/dashboard.html index b3e631e1c2..ababffaa67 100644 --- a/eng/dashboard/dashboard.html +++ b/eng/dashboard/dashboard.html @@ -245,6 +245,11 @@ background: var(--surface); color: var(--text); border: 1px solid var(--border); border-radius: 6px; padding: 6px 8px; font-size: 13px; margin-left: 4px; } + .sv-controls input[type="search"] { + background: var(--surface); color: var(--text); border: 1px solid var(--border); + border-radius: 6px; padding: 6px 8px; font-size: 13px; margin-left: 4px; + } + .sv-controls input[type="search"]:focus-visible { outline: 2px solid var(--skilled); outline-offset: 2px; } .sv-sub { display: block; color: var(--text-muted); font-size: 11px; font-weight: 400; margin-top: 2px; } .sv-controls .sv-sub { display: inline; margin: 0; } .sv-table td { vertical-align: top; } @@ -260,6 +265,10 @@ .sv-btn { background: var(--surface); color: var(--text); border: 1px solid var(--border); border-radius: 6px; padding: 6px 10px; font-size: 12px; cursor: pointer; } .sv-btn:hover { border-color: var(--skilled); color: var(--skilled); } /* Model leaf rows carry the real numbers, so undo the token-table level-2 muting. */ + .sv-table { overflow: visible; } + .sv-table thead th { position: sticky; top: 0; z-index: 1; background: var(--surface); } + .sv-table thead th:first-child { border-top-left-radius: 8px; } + .sv-table thead th:last-child { border-top-right-radius: 8px; } .sv-table tr.level-2 td { color: var(--text); } .sv-table tr.sv-detail td { padding-left: 88px; } diff --git a/eng/dashboard/skill-value.js b/eng/dashboard/skill-value.js index 981c7b9c0c..ebc3b1845c 100644 --- a/eng/dashboard/skill-value.js +++ b/eng/dashboard/skill-value.js @@ -646,17 +646,26 @@ if (row.activationContract && (row.activationContract.evaluated !== true || row. container.innerHTML = `
` + + `` + `` + `` + `` + `` + `Grouped Plugin → Skill → Model. Every headline value and suggestion comes from the same latest run and commit. Up to ${TRAILING_RUNS} earlier runs appear only as separate context in the drilldown. Reliability N / CI shows the number of runs and the likely range of the pass-rate difference for the latest run. Strong install or avoid recommendations require consistent side-by-side task results. Tokens = input+output. Models and commits are never blended.` + `
` + + `` + `
`; + const search = container.querySelector('#sv-search'); + const empty = container.querySelector('#sv-search-empty'); const modelSel = container.querySelector('#sv-model'); const judgeSel = container.querySelector('#sv-judge'); const wrap = container.querySelector('#sv-table-wrap'); + const expand = container.querySelector('#sv-expand'); + const collapse = container.querySelector('#sv-collapse'); + let tree = []; + let savedTreeState = null; + let searchScheduled = false; // A Plugin/Skill group-header row. Metric columns are intentionally blank: // token/time deltas are RELATIVE to each model's own baseline, so they cannot @@ -691,6 +700,7 @@ if (row.activationContract && (row.activationContract.evaluated !== true || row. } function setAll(open) { + if (search.value.trim()) return; // Reveal/hide plugin, skill and model rows. Model drill-down rows (sv-detail) // stay closed on "Expand all" — opening every drill-down at once buries the // table; the viewer opens a specific model to see its arms. @@ -709,6 +719,7 @@ if (row.activationContract && (row.activationContract.evaluated !== true || row. function wireTree() { wrap.querySelectorAll('.expandable').forEach(tr => { tr.addEventListener('click', () => { + if (search.value.trim() && !tr.classList.contains('level-2')) return; const tid = tr.dataset.toggle; const icon = wrap.querySelector('#icon-' + tid); if (icon && icon.classList.contains('expanded')) { @@ -723,10 +734,84 @@ if (row.activationContract && (row.activationContract.evaluated !== true || row. wrap.querySelectorAll('tr.level-0.expandable, tr.level-1.expandable').forEach(tr => openRow(tr.dataset.toggle)); } + function applySearch() { + searchScheduled = false; + const query = search.value.trim().toLowerCase(); + expand.disabled = collapse.disabled = !!query; + if (!query) { + if (savedTreeState) { + for (const plugin of tree) { + for (const node of [plugin, ...plugin.skills, ...plugin.skills.flatMap(skill => skill.models)]) { + const state = savedTreeState.get(node.key); + if (!state) continue; + node.element.style.display = state.display; + node.icon?.classList.toggle('expanded', state.expanded); + if (node.detail) node.detail.style.display = state.detailDisplay; + } + if (!plugin.icon?.classList.contains('expanded')) { + collapseChildren(plugin.element.dataset.toggle); + } else { + for (const skill of plugin.skills) { + if (!skill.icon?.classList.contains('expanded')) collapseChildren(skill.element.dataset.toggle); + } + } + } + savedTreeState = null; + } + empty.style.display = 'none'; + return; + } + + if (!savedTreeState) savedTreeState = new Map(); + for (const plugin of tree) { + for (const node of [plugin, ...plugin.skills, ...plugin.skills.flatMap(skill => skill.models)]) { + if (!savedTreeState.has(node.key)) { + savedTreeState.set(node.key, { + display: node.element.style.display, + expanded: node.icon?.classList.contains('expanded'), + detailDisplay: node.detail?.style.display, + }); + } + } + } + + let found = false; + for (const plugin of tree) { + const pluginMatches = plugin.name.includes(query); + let visibleSkills = 0; + for (const skill of plugin.skills) { + const show = pluginMatches || skill.name.includes(query); + skill.element.style.display = show ? '' : 'none'; + skill.icon?.classList.toggle('expanded', show); + for (const model of skill.models) { + model.element.style.display = show ? '' : 'none'; + if (model.detail) model.detail.style.display = 'none'; + model.icon?.classList.remove('expanded'); + } + if (show) visibleSkills++; + } + plugin.element.style.display = visibleSkills ? '' : 'none'; + plugin.icon?.classList.toggle('expanded', !!visibleSkills); + if (visibleSkills) found = true; + } + empty.style.display = found ? 'none' : ''; + } + + function scheduleSearch() { + if (searchScheduled) return; + searchScheduled = true; + window.requestAnimationFrame(applySearch); + } + function draw() { + tree = []; const fm = modelSel.value, fj = judgeSel.value; const rows = allRows.filter(r => (!fm || r.model === fm) && (!fj || r.judge === fj)); - if (rows.length === 0) { wrap.innerHTML = '

No skills match the selected filters.

'; return; } + if (rows.length === 0) { + wrap.innerHTML = '

No skills match the selected filters.

'; + empty.style.display = 'none'; + return; + } // Group into Plugin → Skill → model rows. const byPlugin = new Map(); @@ -742,9 +827,13 @@ if (row.activationContract && (row.activationContract.evaluated !== true || row. `Time ΔReliability N / CIReliability pass rate (base→skill)What the results suggest`; let uid = 0; - for (const plugin of [...byPlugin.keys()].sort()) { + const groups = []; + for (const plugin of [...byPlugin.keys()].sort((a, b) => + (a === 'agentic-workflows') - (b === 'agentic-workflows') || (a < b ? -1 : a > b ? 1 : 0))) { const skills = byPlugin.get(plugin); const pid = `svp${uid++}`; + const pluginGroup = { name: plugin.toLowerCase(), key: JSON.stringify(['plugin', plugin]), id: pid, skills: [] }; + groups.push(pluginGroup); const pRows = []; for (const arr of skills.values()) { for (const r of arr) pRows.push(r); } // One model in the whole plugin (e.g. filtered to a single executor+judge): @@ -758,6 +847,8 @@ if (row.activationContract && (row.activationContract.evaluated !== true || row. const modelRows = skills.get(skill).slice() .sort((a, b) => a.model.localeCompare(b.model) || a.judge.localeCompare(b.judge)); const sid = `svs${uid++}`; + const skillGroup = { name: skill.toLowerCase(), key: JSON.stringify(['skill', plugin, skill]), id: sid, models: [] }; + pluginGroup.skills.push(skillGroup); const sRollup = modelRows.length === 1 ? singleModelRollup(modelRows[0]) : countRollup(modelRows); @@ -765,6 +856,7 @@ if (row.activationContract && (row.activationContract.evaluated !== true || row. for (const row of modelRows) { const mid = `svm${uid++}`; + skillGroup.models.push({ id: mid, key: JSON.stringify(['model', plugin, skill, row.model, row.judge]) }); const v = valueSentence(row); const costAvailable = costMultiplier(row) != null; const diluted = row.activation != null && row.activation < ACTIVATION_MIN; @@ -777,19 +869,42 @@ if (row.activationContract && (row.activationContract.evaluated !== true || row. metricCell(row) + failureCell(row) + `${v.text}` + - `${drilldown(row)}`; + `${drilldown(row)}`; } } } html += ''; wrap.innerHTML = html; wireTree(); + const elements = new Map([...wrap.querySelectorAll('.expandable')].map(tr => [tr.dataset.toggle, tr])); + const details = new Map([...wrap.querySelectorAll('.sv-detail')].map(tr => [tr.dataset.parent, tr])); + const node = id => ({ + element: elements.get(id), + icon: wrap.querySelector('#icon-' + id), + }); + tree = groups.map(plugin => ({ + name: plugin.name, + key: plugin.key, + ...node(plugin.id), + skills: plugin.skills.map(skill => ({ + name: skill.name, + key: skill.key, + ...node(skill.id), + models: skill.models.map(model => ({ + key: model.key, + ...node(model.id), + detail: details.get(model.id), + })), + })), + })); + applySearch(); } - container.querySelector('#sv-expand').addEventListener('click', () => setAll(true)); - container.querySelector('#sv-collapse').addEventListener('click', () => setAll(false)); + expand.addEventListener('click', () => setAll(true)); + collapse.addEventListener('click', () => setAll(false)); modelSel.addEventListener('change', draw); judgeSel.addEventListener('change', draw); + search.addEventListener('input', scheduleSearch); draw(); } diff --git a/eng/dashboard/skill-value.test.js b/eng/dashboard/skill-value.test.js index e9b5bce42e..19156d3fe2 100644 --- a/eng/dashboard/skill-value.test.js +++ b/eng/dashboard/skill-value.test.js @@ -1,5 +1,215 @@ const test = require('node:test'); const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +test('skill value column headers stay visible during page scroll', () => { + const html = fs.readFileSync(path.join(__dirname, 'dashboard.html'), 'utf8'); + assert.match(html, /\.sv-table\s*\{\s*overflow:\s*visible;/); + assert.match(html, /\.sv-table thead th\s*\{\s*position:\s*sticky;\s*top:\s*0;/); +}); + +function dashboardElement(className = '', dataset = {}) { + const element = { + className, dataset, style: { display: '' }, value: '', disabled: false, + listeners: new Map(), + addEventListener(event, callback) { this.listeners.set(event, callback); }, + dispatch(event) { this.listeners.get(event)?.(); }, + }; + element.classList = { + contains: name => element.className.split(' ').includes(name), + add: name => { if (!element.classList.contains(name)) element.className += ` ${name}`; }, + remove: name => { element.className = element.className.split(' ').filter(part => part !== name).join(' '); }, + toggle: (name, force) => { + const enabled = force ?? !element.classList.contains(name); + if (enabled) element.classList.add(name); + else element.classList.remove(name); + }, + }; + return element; +} + +function dashboardTable() { + const table = { + renders: 0, rows: [], icons: new Map(), html: '', + set innerHTML(html) { + this.html = html; + this.renders++; + this.rows = []; + this.icons.clear(); + for (const match of html.matchAll(/]*)>([\s\S]*?)<\/tr>/g)) { + const attributes = match[2]; + const toggle = attributes.match(/data-toggle="([^"]+)"/)?.[1]; + const parent = attributes.match(/data-parent="([^"]+)"/)?.[1]; + const row = dashboardElement(match[1], { toggle, parent }); + row.style.display = attributes.includes('display:none') ? 'none' : ''; + row.html = match[3]; + this.rows.push(row); + if (toggle) this.icons.set(toggle, dashboardElement('expand-icon')); + } + }, + get innerHTML() { return this.html; }, + querySelector(selector) { return this.querySelectorAll(selector)[0] || null; }, + querySelectorAll(selector) { + if (selector.startsWith('#icon-')) { + const icon = this.icons.get(selector.slice(6)); + return icon ? [icon] : []; + } + if (selector === '[class*="child-of-"]') return this.rows.filter(row => row.className.includes('child-of-')); + if (selector.startsWith('.child-of-')) return this.rows.filter(row => row.classList.contains(selector.slice(1))); + return this.rows.filter(row => selector.split(',').some(part => { + const classes = part.trim().split('.'); + return classes.slice(1).every(name => row.classList.contains(name)); + })); + }, + }; + return table; +} + +async function renderSearchDashboard(t, entries) { + const previous = { document: globalThis.document, window: globalThis.window, fetch: globalThis.fetch }; + const present = Object.fromEntries(Object.keys(previous).map(name => [name, Object.hasOwn(globalThis, name)])); + const modulePath = require.resolve('./skill-value.js'); + t.after(() => { + for (const name of Object.keys(previous)) { + if (present[name]) globalThis[name] = previous[name]; + else delete globalThis[name]; + } + delete require.cache[modulePath]; + }); + const wrap = dashboardTable(); + const controls = new Map([ + ['#sv-table-wrap', wrap], + ...['#sv-search', '#sv-search-empty', '#sv-model', '#sv-judge', '#sv-expand', '#sv-collapse'] + .map(id => [id, dashboardElement()]), + ]); + const container = { + innerHTML: '', + querySelector: selector => controls.get(selector), + }; + const frames = []; + globalThis.document = { + getElementById: () => container, + createElement: () => ({ + innerHTML: '', + set textContent(value) { + this.innerHTML = String(value).replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>'); + }, + }), + }; + globalThis.window = { requestAnimationFrame: callback => frames.push(callback) }; + globalThis.fetch = async () => ({ ok: true, json: async () => ({ entries }) }); + delete require.cache[modulePath]; + require(modulePath); + await window.initSkillValue(); + return { wrap, controls, frames }; +} + +function visibleRows(wrap, level) { + return wrap.rows.filter(row => row.classList.contains(`level-${level}`) && row.style.display !== 'none'); +} + +test('search retains full matched hierarchy without rebuilding the table and pins agentic-workflows last', async t => { + const skill = name => ({ + skill: name, + baseline: { n: 5, timeMs: 1000, tokens: 100 }, + treatment: { n: 5, timeMs: 900, tokens: 90 }, + }); + const entries = [ + { plugin: 'agentic-workflows', model: 'model-a', judgeModel: 'judge-a', date: 1, skills: [skill('workflow')] }, + { plugin: 'zzz-plugin', model: 'model-a', judgeModel: 'judge-a', date: 1, skills: [skill('needle-two')] }, + { plugin: 'alpha-plugin', model: 'model-a', judgeModel: 'judge-a', date: 1, skills: [skill('needle-one'), skill('other')] }, + { plugin: 'alpha-plugin', model: 'model-b', judgeModel: 'judge-b', date: 1, skills: [skill('needle-one'), skill('other')] }, + ]; + const { wrap, controls, frames } = await renderSearchDashboard(t, entries); + const search = controls.get('#sv-search'); + const empty = controls.get('#sv-search-empty'); + const pluginNames = () => visibleRows(wrap, 0).map(row => row.html.match(/<\/span>([^<]+)/)?.[1].trim()); + const skillNames = () => visibleRows(wrap, 1).map(row => row.html.match(/<\/span>([^<]+)/)?.[1].trim()); + const submit = value => { + search.value = value; + search.dispatch('input'); + assert.equal(frames.length, 1, 'rapid input is coalesced into a single frame'); + frames.shift()(); + }; + + assert.deepEqual(pluginNames(), ['alpha-plugin', 'zzz-plugin', 'agentic-workflows']); + controls.get('#sv-collapse').dispatch('click'); + assert.equal(visibleRows(wrap, 1).length, 0); + search.value = 'n'; + search.dispatch('input'); + search.value = 'NEEDLE'; + search.dispatch('input'); + assert.equal(frames.length, 1); + frames.shift()(); + assert.deepEqual(pluginNames(), ['alpha-plugin', 'zzz-plugin']); + assert.deepEqual(skillNames(), ['needle-one', 'needle-two']); + assert.equal(visibleRows(wrap, 2).length, 3, 'a matching skill includes every model'); + assert.equal(wrap.renders, 1, 'search changes visibility, not table markup'); + assert.equal(controls.get('#sv-expand').disabled, true); + + submit('ALPHA'); + assert.deepEqual(pluginNames(), ['alpha-plugin']); + assert.deepEqual(skillNames(), ['needle-one', 'other'], 'a plugin match shows all its skills'); + assert.equal(visibleRows(wrap, 2).length, 4); + + submit('not-present'); + assert.equal(visibleRows(wrap, 0).length, 0); + assert.equal(empty.style.display, ''); + submit(''); + assert.deepEqual(pluginNames(), ['alpha-plugin', 'zzz-plugin', 'agentic-workflows']); + assert.equal(visibleRows(wrap, 1).length, 0, 'clearing search restores collapsed rows'); + assert.equal(empty.style.display, 'none'); + assert.equal(controls.get('#sv-expand').disabled, false); + assert.equal(wrap.renders, 1); + + submit('needle'); + controls.get('#sv-model').value = 'model-b'; + controls.get('#sv-model').dispatch('change'); + assert.deepEqual(pluginNames(), ['alpha-plugin']); + assert.deepEqual(skillNames(), ['needle-one']); + assert.equal(visibleRows(wrap, 2).length, 1, 'model filter and search intersect'); + assert.equal(wrap.renders, 2, 'only model filter changes rebuild the table'); + submit(''); + assert.equal(visibleRows(wrap, 1).length, 0, 'clearing search after a model redraw restores collapsed skills'); + assert.equal(visibleRows(wrap, 2).length, 0); + + controls.get('#sv-model').value = ''; + controls.get('#sv-model').dispatch('change'); + controls.get('#sv-collapse').dispatch('click'); + const alpha = wrap.rows.find(row => row.classList.contains('level-0') && row.html.includes('alpha-plugin')); + alpha.dispatch('click'); + assert.equal(visibleRows(wrap, 1).length, 2); + submit('needle'); + controls.get('#sv-judge').value = 'judge-b'; + controls.get('#sv-judge').dispatch('change'); + submit(''); + assert.deepEqual(pluginNames(), ['alpha-plugin']); + assert.equal(visibleRows(wrap, 1).length, 2, 'judge redraw restores the open plugin'); + assert.equal(visibleRows(wrap, 2).length, 0, 'judge redraw restores collapsed skills'); + + controls.get('#sv-judge').value = ''; + controls.get('#sv-judge').dispatch('change'); + controls.get('#sv-model').value = 'model-a'; + controls.get('#sv-model').dispatch('change'); + controls.get('#sv-collapse').dispatch('click'); + submit('needle'); + controls.get('#sv-model').value = ''; + controls.get('#sv-model').dispatch('change'); + submit(''); + assert.equal(visibleRows(wrap, 1).length, 0, 'widening executor filter keeps skills collapsed'); + assert.equal(visibleRows(wrap, 2).length, 0, 'new model rows stay hidden beneath collapsed skills'); + + controls.get('#sv-judge').value = 'judge-a'; + controls.get('#sv-judge').dispatch('change'); + controls.get('#sv-collapse').dispatch('click'); + submit('needle'); + controls.get('#sv-judge').value = ''; + controls.get('#sv-judge').dispatch('change'); + submit(''); + assert.equal(visibleRows(wrap, 1).length, 0, 'widening judge filter keeps skills collapsed'); + assert.equal(visibleRows(wrap, 2).length, 0, 'new judge rows stay hidden beneath collapsed skills'); +}); test('skill value table uses preference evidence for install guidance', async (t) => { const previousGlobals = { @@ -37,6 +247,7 @@ test('skill value table uses preference evidence for install guidance', async (t if (!elements.has(selector)) { elements.set(selector, { value: '', + style: {}, addEventListener: () => {}, }); } diff --git a/tests/dotnet-msbuild/troubleshoot-binlog-mcp/eval.yaml b/tests/dotnet-msbuild/troubleshoot-binlog-mcp/eval.yaml index a700f2ad7f..aca5468c22 100644 --- a/tests/dotnet-msbuild/troubleshoot-binlog-mcp/eval.yaml +++ b/tests/dotnet-msbuild/troubleshoot-binlog-mcp/eval.yaml @@ -303,12 +303,12 @@ stimuli: - Separates current connectivity from future uncached package availability - name: Leave recorded build errors to build diagnosis - expect_activation: false prompt: |- My .NET build wrote a binlog containing CS0246 for Newtonsoft.Json in Services.csproj. I already confirmed the binlog reader connects and works. What kind of project change would address that compiler error? Advice only. tags: + expect_activation: "false" capability: recorded-build-error-boundary risk: hijacking-build-diagnosis journey: fix-project-dependency @@ -334,12 +334,12 @@ stimuli: - Does not recommend reinstalling a working reader or changing its feeds - name: Leave binlog capture to build logging - expect_activation: false prompt: |- Show me a PowerShell command that builds a .NET project and captures a uniquely named binary log. I am not asking to install or connect any reader. Do not execute the command. tags: + expect_activation: "false" capability: binlog-capture-boundary risk: unnecessary-server-repair journey: capture-build-log @@ -367,12 +367,12 @@ stimuli: - Does not initiate package lookup, installation, or MCP connection troubleshooting - name: Do not apply binlog-specific diagnosis to another server - expect_activation: false prompt: |- My PostgreSQL MCP server reports CONNECTION_CLOSED. It launches a Node.js database adapter; this workspace does not use .NET. What startup evidence should I collect before diagnosing it? Advice only. tags: + expect_activation: "false" capability: unrelated-server-boundary risk: wrong-runtime-diagnosis journey: investigate-database-adapter