From c9be16af9c4497a35c1461c83db0bf2c96834657 Mon Sep 17 00:00:00 2001 From: PatrickJ Date: Sat, 3 Oct 2026 09:25:53 +0200 Subject: [PATCH] feat(skill): generate-test-data skill for tables that already exist A second skill in the nanook plugin: start from an existing workbook, inspect it, write and register the generators it calls, generate one JSON fixture per test case, check the count and use the fixtures in Vitest or Playwright tests. - inspect-workbook.mts: lists tables, Execute/NeverExecute/Multiplicity, filters and the minimum fixture count; reports unregistered generators and filter processors, unknown faker paths and text instructions, broken references and test cases dropped by NeverExecute - generate-fixtures.mts: registers SimpleArrayFilter and SimpleArrayIgnoreFilter, so tables with a FilterSection run; the text generator caches per instruction, so several gen::text cells in one test case no longer share one value - create-equivalence-class-table: x in the ExecuteSection is false, not true; points to the new skill for existing tables - plugin 0.4.0, README, test that keeps the script copies in sync Co-Authored-By: Claude Opus 5.5 --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 10 +- .../commands/createEquivalenceClassTable.md | 2 + .claude/skills/generate-test-data | 1 + README.md | 18 +- .../create-equivalence-class-table/SKILL.md | 21 +- .../scripts/generate-fixtures.mts | 15 + skills/generate-test-data/SKILL.md | 308 ++++++++++++++++++ .../scripts/generate-fixtures.mts | 160 +++++++++ .../scripts/inspect-workbook.mts | 263 +++++++++++++++ tests/skills/scriptsInSync.test.ts | 42 +++ 11 files changed, 824 insertions(+), 18 deletions(-) create mode 120000 .claude/skills/generate-test-data create mode 100644 skills/generate-test-data/SKILL.md create mode 100644 skills/generate-test-data/scripts/generate-fixtures.mts create mode 100644 skills/generate-test-data/scripts/inspect-workbook.mts create mode 100644 tests/skills/scriptsInSync.test.ts diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 1f9c73b..f87eb74 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ { "name": "nanook", "source": "./", - "description": "nanook.xhub: draft decision tables as XLSX, check the coverage, generate test data." + "description": "nanook.xhub: draft decision tables as XLSX, check the coverage, generate test data from new or existing tables." } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index ea55071..9d0ea62 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "nanook", - "description": "nanook.xhub: draft decision tables (equivalence classes, CASCADE coverage) as XLSX for a form, page or API, check the coverage and generate test data with @xhubio/nanook-table.", - "version": "0.3.0", + "description": "nanook.xhub: draft decision tables (equivalence classes, CASCADE coverage) as XLSX for a form, page or API, check the coverage, generate test data from new or existing tables and use it in tests with @xhubio/nanook-table.", + "version": "0.4.0", "author": { "name": "xhub.io", "url": "https://xhub.io" @@ -14,9 +14,11 @@ "test-data", "equivalence-classes", "decision-table", - "xlsx" + "xlsx", + "fixtures" ], "skills": [ - "./skills/create-equivalence-class-table" + "./skills/create-equivalence-class-table", + "./skills/generate-test-data" ] } diff --git a/.claude/commands/createEquivalenceClassTable.md b/.claude/commands/createEquivalenceClassTable.md index dc8f81c..d7036d6 100644 --- a/.claude/commands/createEquivalenceClassTable.md +++ b/.claude/commands/createEquivalenceClassTable.md @@ -41,6 +41,8 @@ After the table is created: 1. Open the workbook in a spreadsheet app and check markers and coverage 2. Register further generators in `scripts/generate-fixtures.mts` if the table calls any 3. `node scripts/generate-fixtures.mts resources/-tests.xlsx` to generate the fixtures +4. For own generators, a count that does not match and data-driven tests from the fixtures, use the + skill `generate-test-data` (`/nanook:generate-test-data` in the plugin) --- diff --git a/.claude/skills/generate-test-data b/.claude/skills/generate-test-data new file mode 120000 index 0000000..5ebf6ac --- /dev/null +++ b/.claude/skills/generate-test-data @@ -0,0 +1 @@ +../../skills/generate-test-data \ No newline at end of file diff --git a/README.md b/README.md index 94bf023..85047f2 100644 --- a/README.md +++ b/README.md @@ -177,10 +177,15 @@ See the [docs/](docs/) directory for detailed guides, API reference, and tutoria [![skills.sh](https://skills.sh/b/xhubio/nanook-table)](https://skills.sh/xhubio/nanook-table) -The nanook.xhub skill `create-equivalence-class-table` drafts a decision table for a form, page or API -as a formatted XLSX, checks its coverage and generates the test data. It lives in -[`skills/`](skills/create-equivalence-class-table/SKILL.md) and ships in the npm package. It is listed on -[skills.sh](https://skills.sh/xhubio/nanook-table/create-equivalence-class-table), and the docs are +Two nanook.xhub skills live in [`skills/`](skills/) and ship in the npm package: + +- [`create-equivalence-class-table`](skills/create-equivalence-class-table/SKILL.md) drafts a decision + table for a form, page or API as a formatted XLSX, checks its coverage and generates the test data. +- [`generate-test-data`](skills/generate-test-data/SKILL.md) starts from a table that already exists: + it inspects the workbook, writes and registers the generators it calls, generates one JSON fixture + per test case, checks the count and wires the fixtures into Vitest or Playwright tests. + +They are listed on [skills.sh](https://skills.sh/xhubio/nanook-table), and the docs are indexed on [Context7](https://context7.com/xhubio/nanook-table). **Claude Code** (CLI, desktop, web, IDE), as a plugin: @@ -189,6 +194,7 @@ indexed on [Context7](https://context7.com/xhubio/nanook-table). /plugin marketplace add xhubio/nanook-table /plugin install nanook@nanook /nanook:create-equivalence-class-table Login form with email and password +/nanook:generate-test-data resources/login-tests.xlsx ``` **Other agents** (Codex, Cursor, Copilot, Gemini CLI and others that read @@ -196,13 +202,15 @@ indexed on [Context7](https://context7.com/xhubio/nanook-table). ```bash npx skills add xhubio/nanook-table --skill create-equivalence-class-table +npx skills add xhubio/nanook-table --skill generate-test-data ``` **Any agent that reads `AGENTS.md`**: paste the block from [docs/agents-snippet.md](docs/agents-snippet.md) into your project's `AGENTS.md`. It points the agent to the Markdown docs in `node_modules/@xhubio/nanook-table/docs/`, which match the installed version. -The generated script needs `exceljs` in your project (`npm install -D exceljs`). Setup for each agent, +The script that `create-equivalence-class-table` generates needs `exceljs` in your project +(`npm install -D exceljs`); `generate-test-data` does not. Setup for each agent, the docs as plain text (`llms.txt`) and how to check what the agent produced: [nanook.xhub.io/docs/guide/use-with-ai](https://nanook.xhub.io/docs/guide/use-with-ai). diff --git a/skills/create-equivalence-class-table/SKILL.md b/skills/create-equivalence-class-table/SKILL.md index 728dd5b..9ac6eed 100644 --- a/skills/create-equivalence-class-table/SKILL.md +++ b/skills/create-equivalence-class-table/SKILL.md @@ -1,15 +1,16 @@ --- name: create-equivalence-class-table description: > - nanook.xhub: Draft a decision table (equivalence classes, test cases, CASCADE coverage) + nanook.xhub: Draft a new decision table (equivalence classes, test cases, CASCADE coverage) as a formatted XLSX for a form, page or API, check its coverage and generate test - data from it with @xhubio/nanook-table. Use when asked to create an equivalence - class table, a decision table, a test case table or test data for a form or API - with Nanook. Also: "equivalence class table", "decision table", + data from it with @xhubio/nanook-table. Use when asked to create or extend an equivalence + class table, a decision table or a test case table for a form or API with Nanook. + For a table that already exists and only needs test data, fixtures or data-driven tests, + use generate-test-data instead. Also: "equivalence class table", "decision table", "test data table", "nanook table". license: MIT metadata: - version: "0.3.0" + version: "0.4.0" --- # nanook.xhub: Create a decision table @@ -51,7 +52,7 @@ node scripts/generate-fixtures.mts resources/-tests.xlsx fixtures/ | Script | What it does | Exit 1 if | |---|---|---| | `check-classes.mts` | reads the markers from the cells (not the formula values), recomputes combinations, column products and coverage, reports every class without its own `x` | coverage < 100 %, class without `x`, field without marker, no decision table | -| `generate-fixtures.mts` | Nanook reads the workbook, generators `faker` and `text` are registered, one JSON per test case, count per table | Nanook logged errors (Nanook does not throw, it logs and carries on) | +| `generate-fixtures.mts` | Nanook reads the workbook, generators `faker` and `text` and the filter processors `SimpleArrayFilter` and `SimpleArrayIgnoreFilter` are registered, one JSON per test case, count per table | Nanook logged errors (Nanook does not throw, it logs and carries on) | The `text` generator (in `generate-fixtures.mts`) provides the edge cases `faker` cannot: `gen::text:empty` (empty string), `gen::text:spaces:N`, `gen::text:alpha:N`, @@ -92,6 +93,8 @@ TCs needed for 100% CASCADE = (sum of all non-preferred EqClasses) + 1 happy - Run the script → generate the Excel file - Open it in a spreadsheet → check colours, formulas, markers - Run Nanook generate → check the fixtures +- Own generators, a run with fewer test cases than expected, using the fixtures in Vitest or + Playwright: the `generate-test-data` skill covers that part ## Column layout (ParserDecision) @@ -145,9 +148,9 @@ Category ← TagSection with "negative"/"valid" rows | GeneratorSwitchSection | Multi-Row | Switch off specific generators per TC | ### ExecuteSection values -- **True**: `x`, `1`, `y`, `j`, `yes`, `ja`, `si`, `true`, `ok`, `T` (case-insensitive) +- **True**: `T`, `1`, `y`, `j`, `yes`, `ja`, `si`, `true`, `ok` (case-insensitive) - **False**: `F` or any other value -- **CAUTION**: 'x' is recognised as TRUE! Always use 'F' for sub-tables +- **CAUTION**: `x` is recognised as FALSE! Write `T` for tables that generate, `F` for sub-tables ## Marker system @@ -554,6 +557,8 @@ Category | TagSection | | further element of a range reference. Fewer means: a generator failed. 5. Tell the user to open the workbook in a spreadsheet application (colours, formulas, summary row), and name the assumptions that did not come from their request (lengths, error codes). +6. For the next steps (generators the table calls that are not registered yet, using the fixtures + in tests), point to the `generate-test-data` skill. ## References between tables (Nanook's core feature) diff --git a/skills/create-equivalence-class-table/scripts/generate-fixtures.mts b/skills/create-equivalence-class-table/scripts/generate-fixtures.mts index 9ba84a5..125237e 100644 --- a/skills/create-equivalence-class-table/scripts/generate-fixtures.mts +++ b/skills/create-equivalence-class-table/scripts/generate-fixtures.mts @@ -8,6 +8,10 @@ * text gen::text:empty | spaces:N | alpha:N | email:N * Register your own generators below if the table calls others. * + * Filter processors registered (FilterSection, column B of a row = processor name): + * SimpleArrayFilter keep the test case if one of the tags is in the expression (a;b;c) + * SimpleArrayIgnoreFilter drop the test case if one of the tags is in the expression + * * Nanook does not throw on a failed generator or an unresolved reference; it logs and goes on, * and the test case is missing. So the logger is checked after the run: exit 1 on any error. * Compare the count per table with the number of test-case columns (plus one per extra element @@ -21,6 +25,8 @@ import { GeneratorFaker, DataGeneratorBase, DataGeneratorRegistry, + SimpleArrayFilterProcessor, + SimpleArrayIgnoreFilterProcessor, createDefaultFileProcessor } from '@xhubio/nanook-table' import type { @@ -51,6 +57,13 @@ function randomAlpha(n: number): string { } class GeneratorText extends DataGeneratorBase { + // DataGeneratorBase caches per instance id only, and gen::text:… has the same id for every + // field of a test case: all of them would get the first value. Key on the instruction too. + public generate(request: DataGeneratorGenerateRequest): Promise { + const config = request.generatorDirective?.config ?? '' + return super.generate({ ...request, instanceId: `${request.instanceId}:${config}` }) + } + protected doGenerate(request: DataGeneratorGenerateRequest): Promise { const config = request.generatorDirective?.config ?? '' const [kind, arg] = config.split(':') @@ -119,6 +132,8 @@ async function main() { const writer = new FixtureWriter(logger) const processor = new TestcaseProcessor({ logger, generatorRegistry: registry, writer: [writer], tables }) + processor.addFilterProcessor(new SimpleArrayFilterProcessor({ name: 'SimpleArrayFilter' })) + processor.addFilterProcessor(new SimpleArrayIgnoreFilterProcessor({ name: 'SimpleArrayIgnoreFilter' })) await processor.process() const perTable: Record = {} diff --git a/skills/generate-test-data/SKILL.md b/skills/generate-test-data/SKILL.md new file mode 100644 index 0000000..7b438b7 --- /dev/null +++ b/skills/generate-test-data/SKILL.md @@ -0,0 +1,308 @@ +--- +name: generate-test-data +description: > + nanook.xhub: Generate test data from an existing Nanook decision table (XLSX/XLS) with + @xhubio/nanook-table: inspect the workbook, register or write the generators it calls, + write one JSON fixture per test case, check the count and use the fixtures in Vitest or + Playwright tests. Use when a table already exists and the user wants test data, fixtures or + data-driven tests from it, or when a Nanook run yields fewer test cases than expected, + missing fields or "no generator registered". To draft a new table, use + create-equivalence-class-table instead. Also: "generate test data", "fixtures from table", + "nanook generate", "Testdaten generieren". +license: MIT +metadata: + version: "0.4.0" +--- + +# nanook.xhub: Generate test data from an existing table + +Takes a workbook with Nanook decision tables that already exists, hand-made, from an older +Nanook version or from the `create-equivalence-class-table` skill, and turns it into one JSON +fixture per test case that tests can read. Nanook does not throw on a failed generator or an +unresolved reference, it logs and carries on, so every step here ends with a check. + +## Environment and bundled scripts + +- Project with `@xhubio/nanook-table` (3.1.0 or later, ESM). No `exceljs` needed. +- Node.js 22.18 or later runs `.ts`/`.mts` directly; older 22.x need + `--experimental-strip-types`, `npx tsx` works everywhere. +- Default locations: `resources/-tests.xlsx`, `fixtures//`, own generators in + `scripts/generators/`. If the user names other folders, use theirs. +- **Output language**: write comments, messages and test names in the language of the user's + request, not in the language of these instructions. + +Two finished, tested scripts are in the `scripts/` folder next to this `SKILL.md` and, from +`@xhubio/nanook-table` 3.2.0, in the project under +`node_modules/@xhubio/nanook-table/skills/generate-test-data/scripts/`. +**Copy them from there**: that folder is inside the project, the plugin folder is often outside it and +then not readable. Run them only inside the project: Node resolves imports relative to the script and +does not strip TypeScript types under `node_modules`. If `cp` is not allowed, read the file +and write it into the project **unchanged**. **Never rewrite them**: the scripts are +tested, a rewrite is not. The one place meant to be edited is the generator registration in +`generate-fixtures.mts` (step 3). If neither source can be found, tell the user so instead of +building a substitute. + +``` +cp node_modules/@xhubio/nanook-table/skills/generate-test-data/scripts/*.mts scripts/ +node scripts/inspect-workbook.mts resources/-tests.xlsx +node scripts/generate-fixtures.mts resources/-tests.xlsx fixtures/ +``` + +**With 3.1.x** the package has no `generate-test-data` folder. Copy `generate-fixtures.mts` from +`node_modules/@xhubio/nanook-table/skills/create-equivalence-class-table/scripts/`; there is no +`inspect-workbook.mts` then. Skip step 1, tell the user that upgrading to 3.2.0 adds the check, +and count by hand in step 5: one fixture per column with a true Execute value, times +Multiplicity, plus one per further element of a range reference. + +| Script | What it does | Exit 1 if | +|---|---|---| +| `inspect-workbook.mts` | reads the workbook with Nanook's parser; lists every table with its columns, Execute, NeverExecute, Multiplicity, filters and the minimum fixture count; lists every `gen:` grouped by generator and checks every `ref:` | a generator or filter processor is not registered in `generate-fixtures.mts` (read from the file next to it), a faker path does not exist, a `text` instruction is unknown, a reference in an executed column points to a missing table, field or test case, the loader logged an error | +| `generate-fixtures.mts` | Nanook reads the workbook, generators `faker` and `text` and the filter processors `SimpleArrayFilter` and `SimpleArrayIgnoreFilter` are registered, one JSON per test case (`..json`), count per table; `--verbose` prints Nanook's log | Nanook logged errors | + +The `text` generator provides the edge cases `faker` cannot: `gen::text:empty`, +`gen::text:spaces:N`, `gen::text:alpha:N`, `gen::text:email:N` (N letters + `@example.com`). + +## Workflow + +### Step 1: Inspect the workbook +Run `inspect-workbook.mts` and read the output **before** generating anything. Note: +- which tables are executed (`execute`) and how many fixtures each must yield at least, +- columns marked `+range` (one fixture per element of the range, name gets `-1`, `-2`, …), +- columns marked `dropped` (they reference a test case with NeverExecute; Nanook skips them), +- columns marked `filter` (not in the minimum: the filter keeps them only if a tag in their call + tree matches; with a range it decides per element), +- every generator marked `NOT REGISTERED`, every filter processor reported as not registered. +- Warnings about references in columns that are not executed matter only if another column + references that column. + +If the user only gave a folder, look for `*.xlsx`/`*.xls` there and ask which workbook is meant +when there are several. Tables referenced across workbooks are not supported by the scripts: +they load one file. + +### Step 2: Resolve the generators +For every unregistered generator decide with the user, in this order: +1. **The cell is wrong**: a typo (`fakr`), a faker call with arguments (`string.alpha(5)`, + use `gen::text:alpha:5`), a path from an old faker version (`name.firstName` is + `person.firstName` today, `address.city` is `location.city`), a generator from an old project + that faker or `text` covers → propose the corrected cell, the user changes the workbook. +2. **The project already has the generator** (search for `extends DataGeneratorBase`) → + import and register it. +3. **It must be written** → step 3. + +Never invent what an unknown generator returns. Ask what `gen::kundennr:` is meant to produce +(format, uniqueness, relation to other fields) before writing it. + +### Step 3: Write and register own generators +Put each generator in its own file under `scripts/generators/`, import it into +`generate-fixtures.mts` and register it next to `faker` and `text`; `inspect-workbook.mts` +reads the registrations from there. + +```ts +// scripts/generators/GeneratorCustomerNo.ts +import { DataGeneratorBase } from '@xhubio/nanook-table' +import type { DataGeneratorGenerateRequest } from '@xhubio/nanook-table' + +// gen::customerNo: e.g. gen:1:customerNo:K -> K-000001, unique per run +export class GeneratorCustomerNo extends DataGeneratorBase { + private counter = 0 + + // DataGeneratorBase caches per instance id only; gen::customerNo:A and gen::customerNo:B in + // one test case share that id. Key on the config too, unless one record SHOULD be shared. + public generate(request: DataGeneratorGenerateRequest): Promise { + const config = request.generatorDirective?.config ?? '' + return super.generate({ ...request, instanceId: `${request.instanceId}:${config}` }) + } + + protected doGenerate(request: DataGeneratorGenerateRequest): Promise { + const prefix = request.generatorDirective?.config || 'K' + this.counter++ + return Promise.resolve(`${prefix}-${String(this.counter).padStart(6, '0')}`) + } +} +``` + +```ts +// generate-fixtures.mts, after the two existing registrations +import { GeneratorCustomerNo } from './generators/GeneratorCustomerNo.ts' +registry.registerGenerator('customerNo', new GeneratorCustomerNo({ generatorRegistry: registry, name: 'customerNo', logger })) +``` + +Rules for generators: +- `doGenerate` gets the part after the third colon as `request.generatorDirective.config` + (colons inside it are kept). Parse it, reject unknown configs with an error: + `DataGeneratorBase` logs it and the field stays empty, `generate-fixtures.mts` exits 1. +- **Same instance id = same entity.** `gen:1:person:firstName` and `gen:1:person:email` + in one test case get the same `request.instanceId`; a generator that builds one record per + id and returns a field of it keeps them consistent (see `docs/tutorials/create-generator.md`). + `gen::…` (empty suffix) also shares one id across all fields of the test case, hence the + `generate()` override above for generators where each cell is independent. +- **A field built from other fields** returns `undefined` while one of them is missing; + Nanook calls it again after the other directives ran. The record of the test case is + `request.testcaseData.data[
][]`: + +```ts +// gen::template:{first-name}.{last-name}@example.com +protected doGenerate(request: DataGeneratorGenerateRequest): Promise { + const directive = request.generatorDirective + const template = directive?.config ?? '' + const nodeId = (directive as { node?: { instanceId: string } } | undefined)?.node?.instanceId ?? '' + const record: Record = + request.testcaseData?.data?.[directive?.testcaseMeta.tableName ?? '']?.[nodeId] ?? {} + const names = [...template.matchAll(/\{([^}]+)\}/g)].map((m) => m[1]) + if (names.some((name) => record[name] === undefined)) return Promise.resolve(undefined) + return Promise.resolve(template.replace(/\{([^}]+)\}/g, (_, name: string) => String(record[name]))) +} +``` + +- **Uniqueness** (`uniqueSet` of `DataGeneratorBase`, or a counter as above) is only tested + when the test forces the collision: fixed inputs, several calls, different results. +- **Store values across runs** only with `useStore: true` and a `varDir`; otherwise every run + starts fresh. + +Run `inspect-workbook.mts` again: no `NOT REGISTERED` left. + +### Step 4: Generate +``` +node scripts/generate-fixtures.mts resources/-tests.xlsx fixtures/ +``` +The script deletes the `*.json` files in the target folder first, nothing else. Rerun with +`--verbose` to see Nanook's log when something is off. + +### Step 5: Check the count +Compare the count per table with step 1: at least the minimum, more only for `+range` +columns. **Fewer means a test case was dropped** even if the exit code is 0 for some reason; +see Troubleshooting. Open two or three fixtures and check that every field has a value +and that values meant to differ (empty, too long, invalid) do differ. + +The output is random where the table leaves a choice (faker; several `e` markers in one +column, of which Nanook picks one at random). Do not commit fixtures as a golden master unless +the user wants exactly that; generate them in the test run or in CI instead. + +### Step 6: Use the fixtures in tests +Ask which test runner the project uses, then write a data-driven test that reads the folder. +Keep the expected result in the table (an `Expected Result` MultiRowSection, see below) +instead of hard-coding it per test case, so a new column in the table needs no change in the test. + +**Vitest** +```ts +import fs from 'node:fs' +import path from 'node:path' +import { describe, it, expect } from 'vitest' +import type { TestcaseDataInterface } from '@xhubio/nanook-table' + +const DIR = path.join(import.meta.dirname, '../fixtures/') +const fixtures: TestcaseDataInterface[] = fs + .readdirSync(DIR) + .filter((f) => f.endsWith('.json')) + .map((f) => JSON.parse(fs.readFileSync(path.join(DIR, f), 'utf8'))) + +// the record of the test case's own table (see "The fixture" when it has none) +const recordOf = (tc: TestcaseDataInterface) => tc.data[tc.tableName]?.[tc.instanceId] ?? {} + +describe.each(fixtures)('$tableName $name', (tc) => { + it('is handled as the table expects', async () => { + const { 'Expected Result': expected, ...input } = recordOf(tc) + const result = await register(input) // the code under test + expect(result.errorCode ?? 'valid').toBe(expected[0].key) + }) +}) +``` + +**Playwright** +```ts +import fs from 'node:fs' +import path from 'node:path' +import { test, expect } from '@playwright/test' + +const DIR = path.join(import.meta.dirname, '../fixtures/') +// fixtures are named
..json: take the table this page test is about +for (const file of fs.readdirSync(DIR).filter((f) => f.startsWith('
.') && f.endsWith('.json'))) { + const tc = JSON.parse(fs.readFileSync(path.join(DIR, file), 'utf8')) + const input = tc.data[tc.tableName]?.[tc.instanceId] ?? {} + const [expected] = input['Expected Result'] + test(`${tc.tableName} ${tc.name}`, async ({ page }) => { + await page.goto('/register') + await page.getByLabel('Email').fill(input.email) + // … one fill per field + await page.getByRole('button', { name: 'Register' }).click() + if (expected.key === 'valid') await expect(page).toHaveURL(/welcome/) + else await expect(page.getByRole('alert')).toHaveText(expected.other) + }) +} +``` + +Field names are the names in column A of the table, exactly as written there (`first-name`, +`E-Mail`). A MultiRowSection such as `Expected Result` arrives as an array of the marked rows, +`[{ key: , other: , comment: }]`; check its name and columns in +the workbook before writing the test. Generate the fixtures before the test run, e.g. `"pretest": "node scripts/generate-fixtures.mts …"`. + +## The fixture + +``` +{ + tableName, name, instanceId, + data: {
: { : { : } } }, + callTree: { tableName, testcaseName, instanceId, neverExecute, tags, children: [...] }, + postProcessDirectives: [] +} +``` + +- `data[tableName][instanceId]` is the record of the test case itself: one entry per field, + MultiRowSections as arrays of `{ key, comment, other }`. Use the top-level `instanceId`: + `callTree.instanceId` of the root is a different id (Nanook renews the ids after building + the call tree). +- **A table without a record of its own**: when every field of the test case is a reference to a + whole record, `data[tableName]` does not exist. The referenced records are reached through the + call tree, whose children ids do match `data`: + `tc.callTree.children.map((c) => tc.data[c.tableName][c.instanceId])`. +- **References**: the referenced records are in `data` under their own table and instance id. + A reference with a field name also copies that value into the referencing field; a reference + to a whole record (`ref::
::`) adds no field, read the record from `data`. +- `callTree` mirrors the references; `tags` comes from the TagSection of each table in it. +- **Names**: `` normally; `-1`, `-2`, … when a range reference fans out; + `.1`, `.2`, … with Multiplicity. + +## Directives in a cell + +Full reference: `docs/guide/directives.md` in the package. + +| Cell | Meaning | +|---|---| +| `DE`, `100`, `not-an-email` | static value, taken as it is (a string) | +| `gen:::` | value from a registered generator; same `` = same entity within the test case | +| `gen:1:faker:person.firstName` | faker, **a path only, no arguments** | +| `ref::
::` | value of `` in `` of `
` | +| `ref::
::` | the whole record of that test case | +| `ref::
::[tc2-tc4]` | a range: one fixture per element; a range must not have an `` | +| `ref::::` | self-reference: another field of the same test case (2.1.3 or later) | + +ExecuteSection: `t y j 1 yes ja si true ok` (any case) is **true**, everything else is false, +**`x` included**. A table without an ExecuteSection is executed completely. + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| `There was no generator registered with the name 'x'` | the generator is not registered; the whole test case is dropped | step 2 | +| fewer fixtures than the minimum, no error | a referenced test case has NeverExecute (`dropped` in step 1), or a FilterSection excluded it | intended? otherwise remove the NeverExecute or the filter | +| fewer fixtures, error in the log | `faker` threw (a path that does not exist); the test case is dropped | `inspect-workbook.mts` names the cell | +| a field is missing in the fixture, `Error in generating data` or the generator's message | an own generator rejected; the field stays empty, the test case is written | `--verbose`, fix the generator or the config | +| a field is missing in the fixture, `Could not resolve all the fields` | a reference or a generator never got its value (cycle, missing field, generator returns `undefined` forever) | check the reference target; a composite generator must name existing fields | +| `The targetTable 'X' does not exists` | the table name in `ref:` does not match a sheet name (case and spaces count) | correct the reference | +| every `ref:` fails in an own script | `tables` passed as the array from `fileProcessor.tables` | key them by `tableName` (as `generate-fixtures.mts` does) | +| several fields of one test case have the same random value | an own generator without the `generate()` override, all `gen::` cells share one instance id | step 3 | +| a table yields nothing | no column has a true Execute value (`x` is false) | `T` / `1` / `yes` | +| a second sheet with the same name replaced the first | Nanook keys tables by name, a duplicate overwrites (warning in step 1) | rename one of them | +| `Method not implemented` in `before()` | an own script uses `createDefaultWriter()`; it is a stub | use the writer from `generate-fixtures.mts` | +| `A filterProcessor with the name 'X' does not exists. Filter is ignored` | the FilterSection names a processor `generate-fixtures.mts` does not register | use `SimpleArrayFilter`/`SimpleArrayIgnoreFilter` in the table, or write one (`name` and `filter(tags, expression)`) and add it with `processor.addFilterProcessor(new X({ name: 'X' }))` next to the two existing ones | + +## After generating + +Tell the user: +- the count per table and the total, against the minimum from step 1, +- which generators were written or changed and what they produce, +- which cells you proposed to change in the workbook and why, +- where the fixtures are and how the test reads them. + +The workbook stays the source: changes to test cases go into the table, not into the fixtures. +To change or extend the table itself, use the `create-equivalence-class-table` skill. diff --git a/skills/generate-test-data/scripts/generate-fixtures.mts b/skills/generate-test-data/scripts/generate-fixtures.mts new file mode 100644 index 0000000..125237e --- /dev/null +++ b/skills/generate-test-data/scripts/generate-fixtures.mts @@ -0,0 +1,160 @@ +/** + * Lets Nanook read a workbook and writes one JSON fixture per generated test case. + * + * Usage: node scripts/generate-fixtures.mts resources/-tests.xlsx [fixtures/] + * + * Generators registered: + * faker gen::faker:. e.g. gen:1:faker:internet.email (a path, no arguments) + * text gen::text:empty | spaces:N | alpha:N | email:N + * Register your own generators below if the table calls others. + * + * Filter processors registered (FilterSection, column B of a row = processor name): + * SimpleArrayFilter keep the test case if one of the tags is in the expression (a;b;c) + * SimpleArrayIgnoreFilter drop the test case if one of the tags is in the expression + * + * Nanook does not throw on a failed generator or an unresolved reference; it logs and goes on, + * and the test case is missing. So the logger is checked after the run: exit 1 on any error. + * Compare the count per table with the number of test-case columns (plus one per extra element + * of a range reference). + */ +import path from 'node:path' +import fs from 'node:fs/promises' +import { + LoggerMemory, + TestcaseProcessor, + GeneratorFaker, + DataGeneratorBase, + DataGeneratorRegistry, + SimpleArrayFilterProcessor, + SimpleArrayIgnoreFilterProcessor, + createDefaultFileProcessor +} from '@xhubio/nanook-table' +import type { + InterfaceWriter, + TestcaseDataInterface, + DataGeneratorGenerateRequest, + TableInterface +} from '@xhubio/nanook-table' + +if (!process.argv[2]) { + console.error('usage: node scripts/generate-fixtures.mts [out-dir]') + process.exit(1) +} +const XLSX_FILE = path.resolve(process.argv[2]) +const OUT_DIR = path.resolve( + process.argv[3] ?? path.join('fixtures', path.basename(XLSX_FILE, '.xlsx').replace(/-tests$/, '')) +) + +// ------------------------------------------------------------------ +// Generator 'text': edge cases faker cannot produce without arguments +// ------------------------------------------------------------------ + +function randomAlpha(n: number): string { + const letters = 'abcdefghijklmnopqrstuvwxyz' + let s = '' + for (let i = 0; i < n; i++) s += letters[Math.floor(Math.random() * letters.length)] + return s +} + +class GeneratorText extends DataGeneratorBase { + // DataGeneratorBase caches per instance id only, and gen::text:… has the same id for every + // field of a test case: all of them would get the first value. Key on the instruction too. + public generate(request: DataGeneratorGenerateRequest): Promise { + const config = request.generatorDirective?.config ?? '' + return super.generate({ ...request, instanceId: `${request.instanceId}:${config}` }) + } + + protected doGenerate(request: DataGeneratorGenerateRequest): Promise { + const config = request.generatorDirective?.config ?? '' + const [kind, arg] = config.split(':') + const n = Number.parseInt(arg ?? '0', 10) + switch (kind) { + case 'empty': + return Promise.resolve('') + case 'spaces': + return Promise.resolve(' '.repeat(n)) + case 'alpha': + return Promise.resolve(randomAlpha(n)) + case 'email': + return Promise.resolve(`${randomAlpha(n)}@example.com`) + default: + return Promise.reject(new Error(`generator 'text': unknown instruction '${config}'`)) + } + } +} + +// ------------------------------------------------------------------ +// Writer: one JSON per test case. Inline on purpose: in 3.0.1 the default +// writer throws in before(). +// ------------------------------------------------------------------ + +class FixtureWriter implements InterfaceWriter { + logger: LoggerMemory + written: TestcaseDataInterface[] = [] + + constructor(logger: LoggerMemory) { + this.logger = logger + } + + async before(): Promise { + await fs.mkdir(OUT_DIR, { recursive: true }) + // remove only our own output, keep the directory + for (const entry of await fs.readdir(OUT_DIR)) { + if (entry.endsWith('.json')) await fs.rm(path.join(OUT_DIR, entry)) + } + } + + async write(testcaseData: TestcaseDataInterface): Promise { + const file = path.join(OUT_DIR, `${testcaseData.tableName}.${testcaseData.name}.json`) + await fs.writeFile(file, JSON.stringify(testcaseData, null, 2)) + this.written.push(testcaseData) + } + + after(): Promise { + return Promise.resolve() + } +} + +async function main() { + const logger = new LoggerMemory({ writeConsole: process.argv.includes('--verbose') }) + + const fileProcessor = createDefaultFileProcessor(logger) + await fileProcessor.load([XLSX_FILE]) + // TestcaseProcessor wants the tables keyed by name; with the array every ref: fails + const tables: Record = {} + for (const table of fileProcessor.tables) tables[table.tableName] = table + if (Object.keys(tables).length === 0) throw new Error(`no table loaded from ${XLSX_FILE}`) + console.log(`loaded: ${Object.keys(tables).join(', ')}`) + + const registry = new DataGeneratorRegistry() + registry.registerGenerator('faker', new GeneratorFaker({ generatorRegistry: registry, name: 'faker', logger })) + registry.registerGenerator('text', new GeneratorText({ generatorRegistry: registry, name: 'text', logger })) + + const writer = new FixtureWriter(logger) + const processor = new TestcaseProcessor({ logger, generatorRegistry: registry, writer: [writer], tables }) + processor.addFilterProcessor(new SimpleArrayFilterProcessor({ name: 'SimpleArrayFilter' })) + processor.addFilterProcessor(new SimpleArrayIgnoreFilterProcessor({ name: 'SimpleArrayIgnoreFilter' })) + await processor.process() + + const perTable: Record = {} + for (const tc of writer.written) perTable[tc.tableName] = (perTable[tc.tableName] ?? 0) + 1 + for (const [table, n] of Object.entries(perTable)) console.log(` ${table}: ${n} test case(s)`) + + const warnings = logger.entries.warning + const errors = logger.entries.error + if (warnings.length > 0) { + console.log(`\n${warnings.length} warning(s):`) + for (const w of warnings) console.log(JSON.stringify(w)) + } + if (errors.length > 0) { + console.error(`\n${errors.length} error(s) while generating:`) + for (const e of errors) console.error(JSON.stringify(e, null, 2)) + process.exit(1) + } + console.log(`\n${writer.written.length} fixture(s) written to ${path.relative(process.cwd(), OUT_DIR) || '.'}`) +} + +main().catch((err) => { + console.error(err instanceof Error ? (err.stack ?? err.message) : err) + process.exit(1) +}) diff --git a/skills/generate-test-data/scripts/inspect-workbook.mts b/skills/generate-test-data/scripts/inspect-workbook.mts new file mode 100644 index 0000000..f704788 --- /dev/null +++ b/skills/generate-test-data/scripts/inspect-workbook.mts @@ -0,0 +1,263 @@ +/** + * Reads a workbook with Nanook's own parser and reports what a generator run will need, + * before anything is generated. + * + * - every table: type, test-case columns, Execute / NeverExecute / Multiplicity + * - the minimum number of fixtures per table (executed columns times multiplicity, without the + * columns that reference a NeverExecute test case); a column with a range reference is marked + * `+range`, it yields one fixture per element of the range + * - every gen: directive grouped by generator; a generator that generate-fixtures.mts does not + * register, a faker path faker does not have and an unknown text instruction are errors + * - every ref: directive; a missing target table, field or test case is an error in an executed + * column and a warning in any other; a target with NeverExecute is a warning (Nanook drops the + * whole test case) + * - every FilterSection row a column uses; such a column is marked `filter` and not counted in + * the minimum (the filter decides at run time); a filter processor generate-fixtures.mts does + * not register is an error + * - every error the loader logged + * + * Usage: node scripts/inspect-workbook.mts resources/-tests.xlsx + * The registered generators and filter processors are read from generate-fixtures.mts next to + * this script. + * Exit 1 on an unknown generator or filter processor, a broken reference, a load error or no + * table at all. + */ +import path from 'node:path' +import fs from 'node:fs/promises' +import { fileURLToPath } from 'node:url' +import { + LoggerMemory, + GeneratorFaker, + DataGeneratorRegistry, + FieldSectionDefinition, + TableDecision, + createDefaultFileProcessor +} from '@xhubio/nanook-table' +import type { GeneratorDirectiveInterface } from '@xhubio/nanook-table' + +if (!process.argv[2]) { + console.error('usage: node scripts/inspect-workbook.mts ') + process.exit(1) +} +const XLSX_FILE = path.resolve(process.argv[2]) +const FIXTURE_SCRIPT = path.join(path.dirname(fileURLToPath(import.meta.url)), 'generate-fixtures.mts') +const TEXT_INSTRUCTION = /^(empty|spaces:\d+|alpha:\d+|email:\d+)$/ + +interface Field { + name: string + // row id -> content of the generator column + tdgs: Record +} + +const errors: string[] = [] +const warnings: string[] = [] + +async function registeredInFixtureScript(): Promise<{ generators: string[]; filters: string[] }> { + try { + const source = await fs.readFile(FIXTURE_SCRIPT, 'utf8') + return { + generators: [...source.matchAll(/registerGenerator\(\s*['"]([^'"]+)['"]/g)].map((m) => m[1]), + filters: [...source.matchAll(/addFilterProcessor\(\s*new\s+\w+\(\s*\{\s*name:\s*['"]([^'"]+)['"]/g)].map( + (m) => m[1] + ) + } + } catch { + warnings.push(`${path.basename(FIXTURE_SCRIPT)} not found next to this script, assuming its defaults`) + return { generators: ['faker', 'text'], filters: ['SimpleArrayFilter', 'SimpleArrayIgnoreFilter'] } + } +} + +function fieldsOf(table: TableDecision): Field[] { + const fields: Field[] = [] + for (const sectionId of table.sectionOrder) { + const section = table.sections[sectionId] + if (!(section instanceof FieldSectionDefinition)) continue + for (const sub of Object.values(section.subSections)) { + if (sub.name !== undefined) fields.push({ name: sub.name, tdgs: sub.tdgs }) + } + } + return fields +} + +// the cells a test-case column can pick for a field, as Nanook picks them: +// x/a win over any other marker, '' and i are never picked, one of the rest at random +function usedCells(field: Field, data: Record): string[] { + const preferred: string[] = [] + const others: string[] = [] + for (const [rowId, tdg] of Object.entries(field.tdgs)) { + const marker = (data[rowId] ?? '').toLowerCase() + if (marker === '' || marker === 'i' || !tdg) continue + if (marker === 'x' || marker === 'a') preferred.push(tdg) + else others.push(tdg) + } + return preferred.length > 0 ? preferred : others +} + +const fakerLogger = new LoggerMemory() +const fakerRegistry = new DataGeneratorRegistry() +const faker = new GeneratorFaker({ generatorRegistry: fakerRegistry, name: 'faker', logger: fakerLogger }) + +async function fakerPathExists(config: string): Promise { + try { + const value = await faker.generate({ + instanceId: `inspect:${config}`, + testcaseData: { name: 'inspect' }, + generatorDirective: { config } as GeneratorDirectiveInterface + }) + return value !== undefined + } catch { + return false + } +} + +async function main() { + const logger = new LoggerMemory() + const fileProcessor = createDefaultFileProcessor(logger) + await fileProcessor.load([XLSX_FILE]) + const tables = fileProcessor.tables + if (tables.length === 0) errors.push(`no table loaded from ${XLSX_FILE}`) + + const { generators: registered, filters: registeredFilters } = await registeredInFixtureScript() + const decisionTables: Record = {} + for (const table of tables) { + if (table instanceof TableDecision) decisionTables[table.tableName] = table + } + + // generator name -> directive -> where it is used + const generators: Record>> = {} + let minimum = 0 + + console.log(`workbook: ${path.relative(process.cwd(), XLSX_FILE)}`) + console.log(`registered in ${path.basename(FIXTURE_SCRIPT)}:`) + console.log(` generators: ${registered.join(', ') || 'none'}`) + console.log(` filter processors: ${registeredFilters.join(', ') || 'none'}\n`) + + for (const table of tables) { + if (!(table instanceof TableDecision)) { + console.log(`${table.tableName} [${table.tableType}] not inspected, only decision tables are`) + continue + } + const fields = fieldsOf(table) + const testcases = table.testcaseOrder.map((id) => table.testcases[id]) + const executed = testcases.filter((tc) => tc.execute) + let tableMinimum = 0 + const lines: string[] = [] + + for (const tc of testcases) { + const flags = [ + tc.execute ? 'execute' : 'no execute', + tc.neverExecute ? 'NeverExecute' : '', + tc.multiplicity > 1 ? `x${tc.multiplicity}` : '' + ].filter(Boolean) + let fansOut = false + let dropped = false + + for (const field of fields) { + for (const cell of usedCells(field, tc.data)) { + const where = `${table.tableName}.${tc.testcaseName}.${field.name}` + const lower = cell.toLowerCase() + if (lower.startsWith('gen:')) { + const name = cell.split(':')[2] ?? '' + generators[name] ??= {} + generators[name][cell] ??= new Set() + generators[name][cell].add(where) + } else if (lower.startsWith('ref:')) { + // a column that is not executed only matters when another column references it + const report = tc.execute ? errors : warnings + const [, suffix = '', targetTableName = '', targetField = '', targetTc = ''] = cell.split(':') + const targetTable = decisionTables[targetTableName || table.tableName] + if (targetTable === undefined) { + report.push(`${where}: '${cell}' points to table '${targetTableName}', which is not in this workbook`) + continue + } + // without a field name the reference takes the whole record + if (targetField !== '' && !fieldsOf(targetTable).some((f) => f.name === targetField)) { + report.push(`${where}: '${cell}' points to field '${targetField}', which table '${targetTable.tableName}' does not have`) + } + if (targetTc === '') continue + const names = targetTable.processRanges(targetTc) + if (names.length > 1) { + fansOut = true + if (suffix !== '') report.push(`${where}: '${cell}' combines a range with an instance id suffix`) + } + const targetTcs = Object.values(targetTable.testcases) + for (const name of names) { + const target = targetTcs.find((t) => t.testcaseName === name) + if (target === undefined) { + report.push(`${where}: '${cell}' points to test case '${name}', which table '${targetTable.tableName}' does not have`) + } else if (target.neverExecute && tc.execute) { + if (names.length === 1) dropped = true + warnings.push(`${where}: '${cell}' points to '${name}' with NeverExecute, Nanook drops that test case`) + } + } + } + } + } + + // only the filters of the executed column count, those of referenced columns are not run + const filters = tc.execute ? tc.createFilter() : [] + for (const filter of filters) { + if (!registeredFilters.includes(filter.filterProcessorName)) { + errors.push( + `${table.tableName}.${tc.testcaseName}: filter processor '${filter.filterProcessorName}' is not registered in ${path.basename(FIXTURE_SCRIPT)}` + ) + } + } + + if (tc.execute && !dropped && filters.length === 0) tableMinimum += tc.multiplicity + if (fansOut) flags.push('+range') + if (tc.execute && dropped) flags.push('dropped') + if (filters.length > 0) { + flags.push(`filter ${filters.map((f) => `${f.filterProcessorName}(${f.expression})`).join(' and ')}`) + } + lines.push(` ${tc.testcaseName}: ${flags.join(', ')}`) + } + + minimum += tableMinimum + console.log(`${table.tableName}: ${testcases.length} column(s), ${executed.length} executed, at least ${tableMinimum} fixture(s)`) + for (const line of lines) console.log(line) + } + + console.log('\ngenerators:') + for (const [name, directives] of Object.entries(generators)) { + const known = registered.includes(name) + console.log(` ${name}${known ? '' : ' NOT REGISTERED'}`) + for (const [directive, uses] of Object.entries(directives)) { + console.log(` ${directive} (${uses.size}x)`) + const config = directive.split(':').slice(3).join(':') + const first = [...uses][0] + if (name === 'faker' && !(await fakerPathExists(config))) { + errors.push(`${first}: '${directive}' is no faker function (a path like person.firstName, no arguments)`) + } + if (name === 'text' && known && !TEXT_INSTRUCTION.test(config)) { + errors.push(`${first}: '${directive}' is no text instruction (empty, spaces:N, alpha:N, email:N)`) + } + } + if (!known) { + errors.push(`generator '${name}' is used in ${[...Object.values(directives)].reduce((n, s) => n + s.size, 0)} cell(s) but not registered in ${path.basename(FIXTURE_SCRIPT)}`) + } + } + if (Object.keys(generators).length === 0) console.log(' none') + + for (const entry of logger.entries.error) errors.push(`loader: ${JSON.stringify(entry)}`) + for (const entry of logger.entries.warning) warnings.push(`loader: ${JSON.stringify(entry)}`) + + console.log( + `\nexpected: at least ${minimum} fixture(s), more for columns marked +range, between 0 and all for columns marked filter` + ) + if (warnings.length > 0) { + console.log(`\n${warnings.length} warning(s):`) + for (const w of warnings) console.log(` ${w}`) + } + if (errors.length > 0) { + console.error(`\n${errors.length} error(s):`) + for (const e of errors) console.error(` ${e}`) + process.exit(1) + } + console.log('\nno errors') +} + +main().catch((err) => { + console.error(err instanceof Error ? (err.stack ?? err.message) : err) + process.exit(1) +}) diff --git a/tests/skills/scriptsInSync.test.ts b/tests/skills/scriptsInSync.test.ts new file mode 100644 index 0000000..642b2ef --- /dev/null +++ b/tests/skills/scriptsInSync.test.ts @@ -0,0 +1,42 @@ +import fs from 'node:fs' +import path from 'node:path' +import { test, expect } from 'vitest' + +// Each skill is installed on its own (npx skills add --skill …), so each folder carries its +// own copy of the scripts it uses. The copies must not drift apart. +const SKILLS_DIR = path.join(import.meta.dirname, '..', '..', 'skills') +const SHARED_SCRIPTS = ['generate-fixtures.mts'] +const SKILLS = ['create-equivalence-class-table', 'generate-test-data'] + +test.each(SHARED_SCRIPTS)('%s is the same in every skill', (script) => { + const [first, ...others] = SKILLS.map((skill) => + fs.readFileSync(path.join(SKILLS_DIR, skill, 'scripts', script), 'utf8') + ) + for (const other of others) { + expect(other).toBe(first) + } +}) + +test.each(SKILLS)('plugin.json lists the skill %s', (skill) => { + const plugin = JSON.parse( + fs.readFileSync( + path.join(SKILLS_DIR, '..', '.claude-plugin', 'plugin.json'), + 'utf8' + ) + ) + expect(plugin.skills).toContain(`./skills/${skill}`) +}) + +test.each(SKILLS)('%s has the version of plugin.json', (skill) => { + const plugin = JSON.parse( + fs.readFileSync( + path.join(SKILLS_DIR, '..', '.claude-plugin', 'plugin.json'), + 'utf8' + ) + ) + const skillMd = fs.readFileSync( + path.join(SKILLS_DIR, skill, 'SKILL.md'), + 'utf8' + ) + expect(skillMd).toContain(`version: "${plugin.version}"`) +})