Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion packages/cli/src/intent/evaluate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,16 @@ export function suggestionFromAnswers(
}
}

const CRITERION_TEXT_MAX = 80

export function criteria(moves: readonly TreeMove[]): Record<string, string> {
return Object.fromEntries(
moves.map((move) => [String(move.elementIndex), `${move.role} ${move.label}`.trim()])
moves.map((move) => {
const name = `${move.role} ${move.label}`.trim()
// Code points, so a cap on an emoji name does not end on a lone surrogate.
const capped = [...name].slice(0, CRITERION_TEXT_MAX).join('')
return [String(move.elementIndex), capped]
})
)
}

Expand Down
44 changes: 31 additions & 13 deletions packages/cli/src/intent/tree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,23 +51,40 @@ function isHeaderOrFooter(line: string): boolean {
return false
}

function roleAndLabel(body: string): { role: string; label: string } {
const lower = body.toLowerCase()
// Darwin, Linux, and Windows append these after the visible name.
// Darwin writes a leading space and no comma on Placeholder when name and value are absent.
const METADATA_MARK = /, (?:Value|Placeholder|Description|Text|Secondary Actions):| Placeholder:/

function visibleControlText(body: string): string {
const newline = body.indexOf('\n')
const firstLine = newline === -1 ? body : body.slice(0, newline)
const mark = firstLine.search(METADATA_MARK)
const visible = mark === -1 ? firstLine : firstLine.slice(0, mark)
return visible.trim()
}

const DISABLED_TRAIT = /\((?:[^)]*,\s*)?disabled(?:\s*,[^)]*)?\)/
const SETTABLE_TRAIT = /\((?:[^)]*,\s*)?settable(?:\s*,[^)]*)?\)/

function splitRole(visible: string): { role: string; label: string } {
const lower = visible.toLowerCase()
const multi = MULTI_WORD_ROLES.find(
(role) =>
lower.startsWith(role) && (body.length === role.length || /[\s,(]/.test(body[role.length]!))
lower.startsWith(role) &&
(visible.length === role.length || /[\s,(]/.test(visible[role.length]!))
)
if (multi !== undefined) {
return {
role: body.slice(0, multi.length),
label: body.slice(multi.length).replace(/^[\s,]+/, '')
role: visible.slice(0, multi.length),
label: visible
.slice(multi.length)
.replace(/^[\s,]+/, '')
.trim()
}
}
const comma = body.indexOf(',')
const head = (comma === -1 ? body : body.slice(0, comma)).trim()
const space = head.indexOf(' ')
if (space === -1) return { role: head, label: '' }
return { role: head.slice(0, space), label: head.slice(space + 1).trim() }
const space = visible.indexOf(' ')
if (space === -1) return { role: visible, label: '' }
return { role: visible.slice(0, space), label: visible.slice(space + 1).trim() }
}

export function parseTreeMoves(treeText: string): TreeMove[] {
Expand All @@ -82,10 +99,11 @@ export function parseTreeMoves(treeText: string): TreeMove[] {
if (previous !== undefined && !isHeaderOrFooter(line)) previous.body += `\n${line}`
}
return rows.map((row) => {
const { role, label } = roleAndLabel(row.body)
const disabled = /\(disabled\)/.test(row.body)
const visible = visibleControlText(row.body)
const { role, label } = splitRole(visible)
const disabled = DISABLED_TRAIT.test(visible)
const roleKey = role.toLowerCase()
const settable = /\(settable\)/.test(row.body) || SETTABLE_ROLE.has(roleKey)
const settable = SETTABLE_TRAIT.test(visible) || SETTABLE_ROLE.has(roleKey)
const clickable = !disabled && (CLICKABLE_ROLE.has(roleKey) || settable)
return {
elementIndex: row.elementIndex,
Expand Down
142 changes: 140 additions & 2 deletions tests/intent/evaluate.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
import { afterEach, describe, expect, it, vi } from 'vitest'

import { liveEvaluator } from '../../packages/cli/src/intent/evaluate.js'
import { clickableMoves, parseTreeMoves } from '../../packages/cli/src/intent/tree.js'
import {
criteria,
liveEvaluator,
suggestionFromAnswers
} from '../../packages/cli/src/intent/evaluate.js'
import {
clickableMoves,
parseTreeMoves,
type TreeMove
} from '../../packages/cli/src/intent/tree.js'

const moves = clickableMoves(
parseTreeMoves(`0 standard window Notes
Expand All @@ -13,6 +21,97 @@ afterEach(() => {
vi.unstubAllGlobals()
})

function move(elementIndex: number, role: string, label: string): TreeMove {
return {
elementIndex,
role,
label,
line: `${elementIndex} ${role} ${label}`,
disabled: false,
clickable: true,
settable: role === 'text field' || role === 'text area'
}
}

describe('criteria', () => {
it('keeps a short control name', () => {
expect(criteria([move(68, 'button', 'New Note')])).toEqual({ '68': 'button New Note' })
})

it('caps the joined name at 80 code points', () => {
const posted = criteria([
move(1, 'button', 'N'.repeat(72)),
move(2, 'button', 'N'.repeat(73)),
move(3, 'button', 'N'.repeat(74)),
move(4, 'button', 'N'.repeat(5000))
])
expect(posted['1']).toBe(`button ${'N'.repeat(72)}`)
expect([...(posted['1'] ?? '')]).toHaveLength(79)
expect(posted['2']).toBe(`button ${'N'.repeat(73)}`)
expect([...(posted['2'] ?? '')]).toHaveLength(80)
expect(posted['3']).toBe(`button ${'N'.repeat(73)}`)
expect([...(posted['3'] ?? '')]).toHaveLength(80)
expect(posted['4']).toBe(`button ${'N'.repeat(73)}`)
expect([...(posted['4'] ?? '')]).toHaveLength(80)
})

it('does not split an emoji name into a lone surrogate', () => {
const posted = criteria([move(1, 'button', '😀'.repeat(100))])['1'] ?? ''
expect([...posted]).toHaveLength(80)
expect(posted).toBe([...`button ${'😀'.repeat(100)}`].slice(0, 80).join(''))
expect(posted).not.toMatch(/[\uD800-\uDBFF]$/)
})

it('leaves renderer metadata on a label the parser already stored', () => {
expect(criteria([move(40, 'text field', 'Compose, Value: CANARY-ESSAY')])).toEqual({
'40': 'text field Compose, Value: CANARY-ESSAY'
})
})

it('posts the parsed visible name and caps only that name', () => {
const essay = `CANARY-ESSAY ${'word '.repeat(40)}`
const hint = `CANARY-HINT ${'h'.repeat(80)}`
const tree = [
'0 standard window Chat',
'\t13 button Send',
`\t40 text field Compose, Value: ${essay}`,
'\t42 text field (selected, settable) Compose, Value: the essay',
'\t7 button (selected, expanded) New Note, Secondary Actions: press',
`\t3 text field Placeholder: ${hint}`,
'\t43 text field Address, Placeholder: Search',
'\t44 button Save, Description: CANARY-SUB',
'\t41 text area Note',
'the rest of the note is CANARY-AREA and has no comma',
`\t12 button ${'N'.repeat(90)}`,
`\t9 text ${'m'.repeat(200)}`
].join('\n')
const parsed = parseTreeMoves(tree)
expect(criteria(clickableMoves(parsed))).toEqual({
'13': 'button Send',
'40': 'text field Compose',
'42': 'text field (selected, settable) Compose',
'7': 'button (selected, expanded) New Note',
'3': 'text field',
'43': 'text field Address',
'44': 'button Save',
'41': 'text area Note',
'12': `button ${'N'.repeat(73)}`
})
expect(criteria(parsed)['9']).toBe(`text ${'m'.repeat(75)}`)
expect(JSON.stringify(criteria(parsed))).not.toContain('CANARY')
})

it('keeps the full visible name on the suggestion', () => {
const name = 'N'.repeat(200)
const parsed = parseTreeMoves(`12 button ${name}, Value: CANARY-ESSAY`)
expect(suggestionFromAnswers('snap', parsed, { move: 'click', clickWhich: '12' }).label).toBe(
`button ${name}`
)
expect(criteria(parsed)['12']).toBe(`button ${'N'.repeat(73)}`)
expect(JSON.stringify(criteria(parsed))).not.toContain('CANARY')
})
})

describe('liveEvaluator', () => {
it('returns http size and timing stats without echoing the goal', async () => {
const payload = JSON.stringify({
Expand Down Expand Up @@ -90,6 +189,45 @@ describe('liveEvaluator', () => {
expect(JSON.stringify(ok.http)).not.toContain('Make a new note')
})

it('sends the parsed name in the state and in both choice questions', async () => {
const fetchMock = vi.fn(async () => new Response('{"answers":{}}', { status: 200 }))
vi.stubGlobal('fetch', fetchMock)
const essay = `CANARY-ESSAY ${'word '.repeat(40)}`
const ranked = clickableMoves(
parseTreeMoves(
[
'68 button New Note',
`40 text field Compose, Value: ${essay}`,
'7 button (selected, expanded) New Note'
].join('\n')
)
)
await liveEvaluator('sk-test')({
goal: 'Open the compose field.',
app: 'com.github.Electron',
moves: ranked
})
const request = fetchMock.mock.calls[0]?.[1] as RequestInit
const body = JSON.parse(String(request.body)) as {
state: { goal: string; app: string; clickable: Record<string, string> }
questions: {
click_which: { criteria: Record<string, string> }
setvalue_which: { criteria: Record<string, string> }
}
}
const posted = {
'68': 'button New Note',
'40': 'text field Compose',
'7': 'button (selected, expanded) New Note'
}
expect(body.state.goal).toBe('Open the compose field.')
expect(body.state.app).toBe('com.github.Electron')
expect(body.state.clickable).toEqual(posted)
expect(body.questions.click_which.criteria).toEqual(posted)
expect(body.questions.setvalue_which.criteria).toEqual(posted)
expect(JSON.stringify(body)).not.toContain('CANARY-ESSAY')
})

it('records a network failure as status 0', async () => {
vi.stubGlobal(
'fetch',
Expand Down
Loading
Loading