diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 1f9c73b..f87eb74 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ { "name": "nanook", "source": "./", - "description": "nanook.xhub: draft decision tables as XLSX, check the coverage, generate test data." + "description": "nanook.xhub: draft decision tables as XLSX, check the coverage, generate test data from new or existing tables." } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index ea55071..9d0ea62 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "nanook", - "description": "nanook.xhub: draft decision tables (equivalence classes, CASCADE coverage) as XLSX for a form, page or API, check the coverage and generate test data with @xhubio/nanook-table.", - "version": "0.3.0", + "description": "nanook.xhub: draft decision tables (equivalence classes, CASCADE coverage) as XLSX for a form, page or API, check the coverage, generate test data from new or existing tables and use it in tests with @xhubio/nanook-table.", + "version": "0.4.0", "author": { "name": "xhub.io", "url": "https://xhub.io" @@ -14,9 +14,11 @@ "test-data", "equivalence-classes", "decision-table", - "xlsx" + "xlsx", + "fixtures" ], "skills": [ - "./skills/create-equivalence-class-table" + "./skills/create-equivalence-class-table", + "./skills/generate-test-data" ] } diff --git a/.claude/commands/createEquivalenceClassTable.md b/.claude/commands/createEquivalenceClassTable.md index dc8f81c..d7036d6 100644 --- a/.claude/commands/createEquivalenceClassTable.md +++ b/.claude/commands/createEquivalenceClassTable.md @@ -41,6 +41,8 @@ After the table is created: 1. Open the workbook in a spreadsheet app and check markers and coverage 2. Register further generators in `scripts/generate-fixtures.mts` if the table calls any 3. `node scripts/generate-fixtures.mts resources/-tests.xlsx` to generate the fixtures +4. For own generators, a count that does not match and data-driven tests from the fixtures, use the + skill `generate-test-data` (`/nanook:generate-test-data` in the plugin) --- diff --git a/.claude/skills/generate-test-data b/.claude/skills/generate-test-data new file mode 120000 index 0000000..5ebf6ac --- /dev/null +++ b/.claude/skills/generate-test-data @@ -0,0 +1 @@ +../../skills/generate-test-data \ No newline at end of file diff --git a/README.md b/README.md index 94bf023..85047f2 100644 --- a/README.md +++ b/README.md @@ -177,10 +177,15 @@ See the [docs/](docs/) directory for detailed guides, API reference, and tutoria [![skills.sh](https://skills.sh/b/xhubio/nanook-table)](https://skills.sh/xhubio/nanook-table) -The nanook.xhub skill `create-equivalence-class-table` drafts a decision table for a form, page or API -as a formatted XLSX, checks its coverage and generates the test data. It lives in -[`skills/`](skills/create-equivalence-class-table/SKILL.md) and ships in the npm package. It is listed on -[skills.sh](https://skills.sh/xhubio/nanook-table/create-equivalence-class-table), and the docs are +Two nanook.xhub skills live in [`skills/`](skills/) and ship in the npm package: + +- [`create-equivalence-class-table`](skills/create-equivalence-class-table/SKILL.md) drafts a decision + table for a form, page or API as a formatted XLSX, checks its coverage and generates the test data. +- [`generate-test-data`](skills/generate-test-data/SKILL.md) starts from a table that already exists: + it inspects the workbook, writes and registers the generators it calls, generates one JSON fixture + per test case, checks the count and wires the fixtures into Vitest or Playwright tests. + +They are listed on [skills.sh](https://skills.sh/xhubio/nanook-table), and the docs are indexed on [Context7](https://context7.com/xhubio/nanook-table). **Claude Code** (CLI, desktop, web, IDE), as a plugin: @@ -189,6 +194,7 @@ indexed on [Context7](https://context7.com/xhubio/nanook-table). /plugin marketplace add xhubio/nanook-table /plugin install nanook@nanook /nanook:create-equivalence-class-table Login form with email and password +/nanook:generate-test-data resources/login-tests.xlsx ``` **Other agents** (Codex, Cursor, Copilot, Gemini CLI and others that read @@ -196,13 +202,15 @@ indexed on [Context7](https://context7.com/xhubio/nanook-table). ```bash npx skills add xhubio/nanook-table --skill create-equivalence-class-table +npx skills add xhubio/nanook-table --skill generate-test-data ``` **Any agent that reads `AGENTS.md`**: paste the block from [docs/agents-snippet.md](docs/agents-snippet.md) into your project's `AGENTS.md`. It points the agent to the Markdown docs in `node_modules/@xhubio/nanook-table/docs/`, which match the installed version. -The generated script needs `exceljs` in your project (`npm install -D exceljs`). Setup for each agent, +The script that `create-equivalence-class-table` generates needs `exceljs` in your project +(`npm install -D exceljs`); `generate-test-data` does not. Setup for each agent, the docs as plain text (`llms.txt`) and how to check what the agent produced: [nanook.xhub.io/docs/guide/use-with-ai](https://nanook.xhub.io/docs/guide/use-with-ai). diff --git a/skills/create-equivalence-class-table/SKILL.md b/skills/create-equivalence-class-table/SKILL.md index 728dd5b..9ac6eed 100644 --- a/skills/create-equivalence-class-table/SKILL.md +++ b/skills/create-equivalence-class-table/SKILL.md @@ -1,15 +1,16 @@ --- name: create-equivalence-class-table description: > - nanook.xhub: Draft a decision table (equivalence classes, test cases, CASCADE coverage) + nanook.xhub: Draft a new decision table (equivalence classes, test cases, CASCADE coverage) as a formatted XLSX for a form, page or API, check its coverage and generate test - data from it with @xhubio/nanook-table. Use when asked to create an equivalence - class table, a decision table, a test case table or test data for a form or API - with Nanook. Also: "equivalence class table", "decision table", + data from it with @xhubio/nanook-table. Use when asked to create or extend an equivalence + class table, a decision table or a test case table for a form or API with Nanook. + For a table that already exists and only needs test data, fixtures or data-driven tests, + use generate-test-data instead. Also: "equivalence class table", "decision table", "test data table", "nanook table". license: MIT metadata: - version: "0.3.0" + version: "0.4.0" --- # nanook.xhub: Create a decision table @@ -51,7 +52,7 @@ node scripts/generate-fixtures.mts resources/-tests.xlsx fixtures/ | Script | What it does | Exit 1 if | |---|---|---| | `check-classes.mts` | reads the markers from the cells (not the formula values), recomputes combinations, column products and coverage, reports every class without its own `x` | coverage < 100 %, class without `x`, field without marker, no decision table | -| `generate-fixtures.mts` | Nanook reads the workbook, generators `faker` and `text` are registered, one JSON per test case, count per table | Nanook logged errors (Nanook does not throw, it logs and carries on) | +| `generate-fixtures.mts` | Nanook reads the workbook, generators `faker` and `text` and the filter processors `SimpleArrayFilter` and `SimpleArrayIgnoreFilter` are registered, one JSON per test case, count per table | Nanook logged errors (Nanook does not throw, it logs and carries on) | The `text` generator (in `generate-fixtures.mts`) provides the edge cases `faker` cannot: `gen::text:empty` (empty string), `gen::text:spaces:N`, `gen::text:alpha:N`, @@ -92,6 +93,8 @@ TCs needed for 100% CASCADE = (sum of all non-preferred EqClasses) + 1 happy - Run the script → generate the Excel file - Open it in a spreadsheet → check colours, formulas, markers - Run Nanook generate → check the fixtures +- Own generators, a run with fewer test cases than expected, using the fixtures in Vitest or + Playwright: the `generate-test-data` skill covers that part ## Column layout (ParserDecision) @@ -145,9 +148,9 @@ Category ← TagSection with "negative"/"valid" rows | GeneratorSwitchSection | Multi-Row | Switch off specific generators per TC | ### ExecuteSection values -- **True**: `x`, `1`, `y`, `j`, `yes`, `ja`, `si`, `true`, `ok`, `T` (case-insensitive) +- **True**: `T`, `1`, `y`, `j`, `yes`, `ja`, `si`, `true`, `ok` (case-insensitive) - **False**: `F` or any other value -- **CAUTION**: 'x' is recognised as TRUE! Always use 'F' for sub-tables +- **CAUTION**: `x` is recognised as FALSE! Write `T` for tables that generate, `F` for sub-tables ## Marker system @@ -554,6 +557,8 @@ Category | TagSection | | further element of a range reference. Fewer means: a generator failed. 5. Tell the user to open the workbook in a spreadsheet application (colours, formulas, summary row), and name the assumptions that did not come from their request (lengths, error codes). +6. For the next steps (generators the table calls that are not registered yet, using the fixtures + in tests), point to the `generate-test-data` skill. ## References between tables (Nanook's core feature) diff --git a/skills/create-equivalence-class-table/scripts/generate-fixtures.mts b/skills/create-equivalence-class-table/scripts/generate-fixtures.mts index 9ba84a5..125237e 100644 --- a/skills/create-equivalence-class-table/scripts/generate-fixtures.mts +++ b/skills/create-equivalence-class-table/scripts/generate-fixtures.mts @@ -8,6 +8,10 @@ * text gen::text:empty | spaces:N | alpha:N | email:N * Register your own generators below if the table calls others. * + * Filter processors registered (FilterSection, column B of a row = processor name): + * SimpleArrayFilter keep the test case if one of the tags is in the expression (a;b;c) + * SimpleArrayIgnoreFilter drop the test case if one of the tags is in the expression + * * Nanook does not throw on a failed generator or an unresolved reference; it logs and goes on, * and the test case is missing. So the logger is checked after the run: exit 1 on any error. * Compare the count per table with the number of test-case columns (plus one per extra element @@ -21,6 +25,8 @@ import { GeneratorFaker, DataGeneratorBase, DataGeneratorRegistry, + SimpleArrayFilterProcessor, + SimpleArrayIgnoreFilterProcessor, createDefaultFileProcessor } from '@xhubio/nanook-table' import type { @@ -51,6 +57,13 @@ function randomAlpha(n: number): string { } class GeneratorText extends DataGeneratorBase { + // DataGeneratorBase caches per instance id only, and gen::text:… has the same id for every + // field of a test case: all of them would get the first value. Key on the instruction too. + public generate(request: DataGeneratorGenerateRequest): Promise { + const config = request.generatorDirective?.config ?? '' + return super.generate({ ...request, instanceId: `${request.instanceId}:${config}` }) + } + protected doGenerate(request: DataGeneratorGenerateRequest): Promise { const config = request.generatorDirective?.config ?? '' const [kind, arg] = config.split(':') @@ -119,6 +132,8 @@ async function main() { const writer = new FixtureWriter(logger) const processor = new TestcaseProcessor({ logger, generatorRegistry: registry, writer: [writer], tables }) + processor.addFilterProcessor(new SimpleArrayFilterProcessor({ name: 'SimpleArrayFilter' })) + processor.addFilterProcessor(new SimpleArrayIgnoreFilterProcessor({ name: 'SimpleArrayIgnoreFilter' })) await processor.process() const perTable: Record = {} diff --git a/skills/generate-test-data/SKILL.md b/skills/generate-test-data/SKILL.md new file mode 100644 index 0000000..7b438b7 --- /dev/null +++ b/skills/generate-test-data/SKILL.md @@ -0,0 +1,308 @@ +--- +name: generate-test-data +description: > + nanook.xhub: Generate test data from an existing Nanook decision table (XLSX/XLS) with + @xhubio/nanook-table: inspect the workbook, register or write the generators it calls, + write one JSON fixture per test case, check the count and use the fixtures in Vitest or + Playwright tests. Use when a table already exists and the user wants test data, fixtures or + data-driven tests from it, or when a Nanook run yields fewer test cases than expected, + missing fields or "no generator registered". To draft a new table, use + create-equivalence-class-table instead. Also: "generate test data", "fixtures from table", + "nanook generate", "Testdaten generieren". +license: MIT +metadata: + version: "0.4.0" +--- + +# nanook.xhub: Generate test data from an existing table + +Takes a workbook with Nanook decision tables that already exists, hand-made, from an older +Nanook version or from the `create-equivalence-class-table` skill, and turns it into one JSON +fixture per test case that tests can read. Nanook does not throw on a failed generator or an +unresolved reference, it logs and carries on, so every step here ends with a check. + +## Environment and bundled scripts + +- Project with `@xhubio/nanook-table` (3.1.0 or later, ESM). No `exceljs` needed. +- Node.js 22.18 or later runs `.ts`/`.mts` directly; older 22.x need + `--experimental-strip-types`, `npx tsx` works everywhere. +- Default locations: `resources/-tests.xlsx`, `fixtures//`, own generators in + `scripts/generators/`. If the user names other folders, use theirs. +- **Output language**: write comments, messages and test names in the language of the user's + request, not in the language of these instructions. + +Two finished, tested scripts are in the `scripts/` folder next to this `SKILL.md` and, from +`@xhubio/nanook-table` 3.2.0, in the project under +`node_modules/@xhubio/nanook-table/skills/generate-test-data/scripts/`. +**Copy them from there**: that folder is inside the project, the plugin folder is often outside it and +then not readable. Run them only inside the project: Node resolves imports relative to the script and +does not strip TypeScript types under `node_modules`. If `cp` is not allowed, read the file +and write it into the project **unchanged**. **Never rewrite them**: the scripts are +tested, a rewrite is not. The one place meant to be edited is the generator registration in +`generate-fixtures.mts` (step 3). If neither source can be found, tell the user so instead of +building a substitute. + +``` +cp node_modules/@xhubio/nanook-table/skills/generate-test-data/scripts/*.mts scripts/ +node scripts/inspect-workbook.mts resources/-tests.xlsx +node scripts/generate-fixtures.mts resources/-tests.xlsx fixtures/ +``` + +**With 3.1.x** the package has no `generate-test-data` folder. Copy `generate-fixtures.mts` from +`node_modules/@xhubio/nanook-table/skills/create-equivalence-class-table/scripts/`; there is no +`inspect-workbook.mts` then. Skip step 1, tell the user that upgrading to 3.2.0 adds the check, +and count by hand in step 5: one fixture per column with a true Execute value, times +Multiplicity, plus one per further element of a range reference. + +| Script | What it does | Exit 1 if | +|---|---|---| +| `inspect-workbook.mts` | reads the workbook with Nanook's parser; lists every table with its columns, Execute, NeverExecute, Multiplicity, filters and the minimum fixture count; lists every `gen:` grouped by generator and checks every `ref:` | a generator or filter processor is not registered in `generate-fixtures.mts` (read from the file next to it), a faker path does not exist, a `text` instruction is unknown, a reference in an executed column points to a missing table, field or test case, the loader logged an error | +| `generate-fixtures.mts` | Nanook reads the workbook, generators `faker` and `text` and the filter processors `SimpleArrayFilter` and `SimpleArrayIgnoreFilter` are registered, one JSON per test case (`..json`), count per table; `--verbose` prints Nanook's log | Nanook logged errors | + +The `text` generator provides the edge cases `faker` cannot: `gen::text:empty`, +`gen::text:spaces:N`, `gen::text:alpha:N`, `gen::text:email:N` (N letters + `@example.com`). + +## Workflow + +### Step 1: Inspect the workbook +Run `inspect-workbook.mts` and read the output **before** generating anything. Note: +- which tables are executed (`execute`) and how many fixtures each must yield at least, +- columns marked `+range` (one fixture per element of the range, name gets `-1`, `-2`, …), +- columns marked `dropped` (they reference a test case with NeverExecute; Nanook skips them), +- columns marked `filter` (not in the minimum: the filter keeps them only if a tag in their call + tree matches; with a range it decides per element), +- every generator marked `NOT REGISTERED`, every filter processor reported as not registered. +- Warnings about references in columns that are not executed matter only if another column + references that column. + +If the user only gave a folder, look for `*.xlsx`/`*.xls` there and ask which workbook is meant +when there are several. Tables referenced across workbooks are not supported by the scripts: +they load one file. + +### Step 2: Resolve the generators +For every unregistered generator decide with the user, in this order: +1. **The cell is wrong**: a typo (`fakr`), a faker call with arguments (`string.alpha(5)`, + use `gen::text:alpha:5`), a path from an old faker version (`name.firstName` is + `person.firstName` today, `address.city` is `location.city`), a generator from an old project + that faker or `text` covers → propose the corrected cell, the user changes the workbook. +2. **The project already has the generator** (search for `extends DataGeneratorBase`) → + import and register it. +3. **It must be written** → step 3. + +Never invent what an unknown generator returns. Ask what `gen::kundennr:` is meant to produce +(format, uniqueness, relation to other fields) before writing it. + +### Step 3: Write and register own generators +Put each generator in its own file under `scripts/generators/`, import it into +`generate-fixtures.mts` and register it next to `faker` and `text`; `inspect-workbook.mts` +reads the registrations from there. + +```ts +// scripts/generators/GeneratorCustomerNo.ts +import { DataGeneratorBase } from '@xhubio/nanook-table' +import type { DataGeneratorGenerateRequest } from '@xhubio/nanook-table' + +// gen::customerNo: e.g. gen:1:customerNo:K -> K-000001, unique per run +export class GeneratorCustomerNo extends DataGeneratorBase { + private counter = 0 + + // DataGeneratorBase caches per instance id only; gen::customerNo:A and gen::customerNo:B in + // one test case share that id. Key on the config too, unless one record SHOULD be shared. + public generate(request: DataGeneratorGenerateRequest): Promise { + const config = request.generatorDirective?.config ?? '' + return super.generate({ ...request, instanceId: `${request.instanceId}:${config}` }) + } + + protected doGenerate(request: DataGeneratorGenerateRequest): Promise { + const prefix = request.generatorDirective?.config || 'K' + this.counter++ + return Promise.resolve(`${prefix}-${String(this.counter).padStart(6, '0')}`) + } +} +``` + +```ts +// generate-fixtures.mts, after the two existing registrations +import { GeneratorCustomerNo } from './generators/GeneratorCustomerNo.ts' +registry.registerGenerator('customerNo', new GeneratorCustomerNo({ generatorRegistry: registry, name: 'customerNo', logger })) +``` + +Rules for generators: +- `doGenerate` gets the part after the third colon as `request.generatorDirective.config` + (colons inside it are kept). Parse it, reject unknown configs with an error: + `DataGeneratorBase` logs it and the field stays empty, `generate-fixtures.mts` exits 1. +- **Same instance id = same entity.** `gen:1:person:firstName` and `gen:1:person:email` + in one test case get the same `request.instanceId`; a generator that builds one record per + id and returns a field of it keeps them consistent (see `docs/tutorials/create-generator.md`). + `gen::…` (empty suffix) also shares one id across all fields of the test case, hence the + `generate()` override above for generators where each cell is independent. +- **A field built from other fields** returns `undefined` while one of them is missing; + Nanook calls it again after the other directives ran. The record of the test case is + `request.testcaseData.data[
][]`: + +```ts +// gen::template:{first-name}.{last-name}@example.com +protected doGenerate(request: DataGeneratorGenerateRequest): Promise { + const directive = request.generatorDirective + const template = directive?.config ?? '' + const nodeId = (directive as { node?: { instanceId: string } } | undefined)?.node?.instanceId ?? '' + const record: Record = + request.testcaseData?.data?.[directive?.testcaseMeta.tableName ?? '']?.[nodeId] ?? {} + const names = [...template.matchAll(/\{([^}]+)\}/g)].map((m) => m[1]) + if (names.some((name) => record[name] === undefined)) return Promise.resolve(undefined) + return Promise.resolve(template.replace(/\{([^}]+)\}/g, (_, name: string) => String(record[name]))) +} +``` + +- **Uniqueness** (`uniqueSet` of `DataGeneratorBase`, or a counter as above) is only tested + when the test forces the collision: fixed inputs, several calls, different results. +- **Store values across runs** only with `useStore: true` and a `varDir`; otherwise every run + starts fresh. + +Run `inspect-workbook.mts` again: no `NOT REGISTERED` left. + +### Step 4: Generate +``` +node scripts/generate-fixtures.mts resources/-tests.xlsx fixtures/ +``` +The script deletes the `*.json` files in the target folder first, nothing else. Rerun with +`--verbose` to see Nanook's log when something is off. + +### Step 5: Check the count +Compare the count per table with step 1: at least the minimum, more only for `+range` +columns. **Fewer means a test case was dropped** even if the exit code is 0 for some reason; +see Troubleshooting. Open two or three fixtures and check that every field has a value +and that values meant to differ (empty, too long, invalid) do differ. + +The output is random where the table leaves a choice (faker; several `e` markers in one +column, of which Nanook picks one at random). Do not commit fixtures as a golden master unless +the user wants exactly that; generate them in the test run or in CI instead. + +### Step 6: Use the fixtures in tests +Ask which test runner the project uses, then write a data-driven test that reads the folder. +Keep the expected result in the table (an `Expected Result` MultiRowSection, see below) +instead of hard-coding it per test case, so a new column in the table needs no change in the test. + +**Vitest** +```ts +import fs from 'node:fs' +import path from 'node:path' +import { describe, it, expect } from 'vitest' +import type { TestcaseDataInterface } from '@xhubio/nanook-table' + +const DIR = path.join(import.meta.dirname, '../fixtures/') +const fixtures: TestcaseDataInterface[] = fs + .readdirSync(DIR) + .filter((f) => f.endsWith('.json')) + .map((f) => JSON.parse(fs.readFileSync(path.join(DIR, f), 'utf8'))) + +// the record of the test case's own table (see "The fixture" when it has none) +const recordOf = (tc: TestcaseDataInterface) => tc.data[tc.tableName]?.[tc.instanceId] ?? {} + +describe.each(fixtures)('$tableName $name', (tc) => { + it('is handled as the table expects', async () => { + const { 'Expected Result': expected, ...input } = recordOf(tc) + const result = await register(input) // the code under test + expect(result.errorCode ?? 'valid').toBe(expected[0].key) + }) +}) +``` + +**Playwright** +```ts +import fs from 'node:fs' +import path from 'node:path' +import { test, expect } from '@playwright/test' + +const DIR = path.join(import.meta.dirname, '../fixtures/') +// fixtures are named
..json: take the table this page test is about +for (const file of fs.readdirSync(DIR).filter((f) => f.startsWith('
.') && f.endsWith('.json'))) { + const tc = JSON.parse(fs.readFileSync(path.join(DIR, file), 'utf8')) + const input = tc.data[tc.tableName]?.[tc.instanceId] ?? {} + const [expected] = input['Expected Result'] + test(`${tc.tableName} ${tc.name}`, async ({ page }) => { + await page.goto('/register') + await page.getByLabel('Email').fill(input.email) + // … one fill per field + await page.getByRole('button', { name: 'Register' }).click() + if (expected.key === 'valid') await expect(page).toHaveURL(/welcome/) + else await expect(page.getByRole('alert')).toHaveText(expected.other) + }) +} +``` + +Field names are the names in column A of the table, exactly as written there (`first-name`, +`E-Mail`). A MultiRowSection such as `Expected Result` arrives as an array of the marked rows, +`[{ key: , other: , comment: }]`; check its name and columns in +the workbook before writing the test. Generate the fixtures before the test run, e.g. `"pretest": "node scripts/generate-fixtures.mts …"`. + +## The fixture + +``` +{ + tableName, name, instanceId, + data: {
: { : { : } } }, + callTree: { tableName, testcaseName, instanceId, neverExecute, tags, children: [...] }, + postProcessDirectives: [] +} +``` + +- `data[tableName][instanceId]` is the record of the test case itself: one entry per field, + MultiRowSections as arrays of `{ key, comment, other }`. Use the top-level `instanceId`: + `callTree.instanceId` of the root is a different id (Nanook renews the ids after building + the call tree). +- **A table without a record of its own**: when every field of the test case is a reference to a + whole record, `data[tableName]` does not exist. The referenced records are reached through the + call tree, whose children ids do match `data`: + `tc.callTree.children.map((c) => tc.data[c.tableName][c.instanceId])`. +- **References**: the referenced records are in `data` under their own table and instance id. + A reference with a field name also copies that value into the referencing field; a reference + to a whole record (`ref::
::`) adds no field, read the record from `data`. +- `callTree` mirrors the references; `tags` comes from the TagSection of each table in it. +- **Names**: `` normally; `-1`, `-2`, … when a range reference fans out; + `.1`, `.2`, … with Multiplicity. + +## Directives in a cell + +Full reference: `docs/guide/directives.md` in the package. + +| Cell | Meaning | +|---|---| +| `DE`, `100`, `not-an-email` | static value, taken as it is (a string) | +| `gen:::` | value from a registered generator; same `` = same entity within the test case | +| `gen:1:faker:person.firstName` | faker, **a path only, no arguments** | +| `ref::
::` | value of `` in `` of `
` | +| `ref::
::` | the whole record of that test case | +| `ref::
::[tc2-tc4]` | a range: one fixture per element; a range must not have an `` | +| `ref::::` | self-reference: another field of the same test case (2.1.3 or later) | + +ExecuteSection: `t y j 1 yes ja si true ok` (any case) is **true**, everything else is false, +**`x` included**. A table without an ExecuteSection is executed completely. + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| `There was no generator registered with the name 'x'` | the generator is not registered; the whole test case is dropped | step 2 | +| fewer fixtures than the minimum, no error | a referenced test case has NeverExecute (`dropped` in step 1), or a FilterSection excluded it | intended? otherwise remove the NeverExecute or the filter | +| fewer fixtures, error in the log | `faker` threw (a path that does not exist); the test case is dropped | `inspect-workbook.mts` names the cell | +| a field is missing in the fixture, `Error in generating data` or the generator's message | an own generator rejected; the field stays empty, the test case is written | `--verbose`, fix the generator or the config | +| a field is missing in the fixture, `Could not resolve all the fields` | a reference or a generator never got its value (cycle, missing field, generator returns `undefined` forever) | check the reference target; a composite generator must name existing fields | +| `The targetTable 'X' does not exists` | the table name in `ref:` does not match a sheet name (case and spaces count) | correct the reference | +| every `ref:` fails in an own script | `tables` passed as the array from `fileProcessor.tables` | key them by `tableName` (as `generate-fixtures.mts` does) | +| several fields of one test case have the same random value | an own generator without the `generate()` override, all `gen::` cells share one instance id | step 3 | +| a table yields nothing | no column has a true Execute value (`x` is false) | `T` / `1` / `yes` | +| a second sheet with the same name replaced the first | Nanook keys tables by name, a duplicate overwrites (warning in step 1) | rename one of them | +| `Method not implemented` in `before()` | an own script uses `createDefaultWriter()`; it is a stub | use the writer from `generate-fixtures.mts` | +| `A filterProcessor with the name 'X' does not exists. Filter is ignored` | the FilterSection names a processor `generate-fixtures.mts` does not register | use `SimpleArrayFilter`/`SimpleArrayIgnoreFilter` in the table, or write one (`name` and `filter(tags, expression)`) and add it with `processor.addFilterProcessor(new X({ name: 'X' }))` next to the two existing ones | + +## After generating + +Tell the user: +- the count per table and the total, against the minimum from step 1, +- which generators were written or changed and what they produce, +- which cells you proposed to change in the workbook and why, +- where the fixtures are and how the test reads them. + +The workbook stays the source: changes to test cases go into the table, not into the fixtures. +To change or extend the table itself, use the `create-equivalence-class-table` skill. diff --git a/skills/generate-test-data/scripts/generate-fixtures.mts b/skills/generate-test-data/scripts/generate-fixtures.mts new file mode 100644 index 0000000..125237e --- /dev/null +++ b/skills/generate-test-data/scripts/generate-fixtures.mts @@ -0,0 +1,160 @@ +/** + * Lets Nanook read a workbook and writes one JSON fixture per generated test case. + * + * Usage: node scripts/generate-fixtures.mts resources/-tests.xlsx [fixtures/] + * + * Generators registered: + * faker gen::faker:. e.g. gen:1:faker:internet.email (a path, no arguments) + * text gen::text:empty | spaces:N | alpha:N | email:N + * Register your own generators below if the table calls others. + * + * Filter processors registered (FilterSection, column B of a row = processor name): + * SimpleArrayFilter keep the test case if one of the tags is in the expression (a;b;c) + * SimpleArrayIgnoreFilter drop the test case if one of the tags is in the expression + * + * Nanook does not throw on a failed generator or an unresolved reference; it logs and goes on, + * and the test case is missing. So the logger is checked after the run: exit 1 on any error. + * Compare the count per table with the number of test-case columns (plus one per extra element + * of a range reference). + */ +import path from 'node:path' +import fs from 'node:fs/promises' +import { + LoggerMemory, + TestcaseProcessor, + GeneratorFaker, + DataGeneratorBase, + DataGeneratorRegistry, + SimpleArrayFilterProcessor, + SimpleArrayIgnoreFilterProcessor, + createDefaultFileProcessor +} from '@xhubio/nanook-table' +import type { + InterfaceWriter, + TestcaseDataInterface, + DataGeneratorGenerateRequest, + TableInterface +} from '@xhubio/nanook-table' + +if (!process.argv[2]) { + console.error('usage: node scripts/generate-fixtures.mts [out-dir]') + process.exit(1) +} +const XLSX_FILE = path.resolve(process.argv[2]) +const OUT_DIR = path.resolve( + process.argv[3] ?? path.join('fixtures', path.basename(XLSX_FILE, '.xlsx').replace(/-tests$/, '')) +) + +// ------------------------------------------------------------------ +// Generator 'text': edge cases faker cannot produce without arguments +// ------------------------------------------------------------------ + +function randomAlpha(n: number): string { + const letters = 'abcdefghijklmnopqrstuvwxyz' + let s = '' + for (let i = 0; i < n; i++) s += letters[Math.floor(Math.random() * letters.length)] + return s +} + +class GeneratorText extends DataGeneratorBase { + // DataGeneratorBase caches per instance id only, and gen::text:… has the same id for every + // field of a test case: all of them would get the first value. Key on the instruction too. + public generate(request: DataGeneratorGenerateRequest): Promise { + const config = request.generatorDirective?.config ?? '' + return super.generate({ ...request, instanceId: `${request.instanceId}:${config}` }) + } + + protected doGenerate(request: DataGeneratorGenerateRequest): Promise { + const config = request.generatorDirective?.config ?? '' + const [kind, arg] = config.split(':') + const n = Number.parseInt(arg ?? '0', 10) + switch (kind) { + case 'empty': + return Promise.resolve('') + case 'spaces': + return Promise.resolve(' '.repeat(n)) + case 'alpha': + return Promise.resolve(randomAlpha(n)) + case 'email': + return Promise.resolve(`${randomAlpha(n)}@example.com`) + default: + return Promise.reject(new Error(`generator 'text': unknown instruction '${config}'`)) + } + } +} + +// ------------------------------------------------------------------ +// Writer: one JSON per test case. Inline on purpose: in 3.0.1 the default +// writer throws in before(). +// ------------------------------------------------------------------ + +class FixtureWriter implements InterfaceWriter { + logger: LoggerMemory + written: TestcaseDataInterface[] = [] + + constructor(logger: LoggerMemory) { + this.logger = logger + } + + async before(): Promise { + await fs.mkdir(OUT_DIR, { recursive: true }) + // remove only our own output, keep the directory + for (const entry of await fs.readdir(OUT_DIR)) { + if (entry.endsWith('.json')) await fs.rm(path.join(OUT_DIR, entry)) + } + } + + async write(testcaseData: TestcaseDataInterface): Promise { + const file = path.join(OUT_DIR, `${testcaseData.tableName}.${testcaseData.name}.json`) + await fs.writeFile(file, JSON.stringify(testcaseData, null, 2)) + this.written.push(testcaseData) + } + + after(): Promise { + return Promise.resolve() + } +} + +async function main() { + const logger = new LoggerMemory({ writeConsole: process.argv.includes('--verbose') }) + + const fileProcessor = createDefaultFileProcessor(logger) + await fileProcessor.load([XLSX_FILE]) + // TestcaseProcessor wants the tables keyed by name; with the array every ref: fails + const tables: Record = {} + for (const table of fileProcessor.tables) tables[table.tableName] = table + if (Object.keys(tables).length === 0) throw new Error(`no table loaded from ${XLSX_FILE}`) + console.log(`loaded: ${Object.keys(tables).join(', ')}`) + + const registry = new DataGeneratorRegistry() + registry.registerGenerator('faker', new GeneratorFaker({ generatorRegistry: registry, name: 'faker', logger })) + registry.registerGenerator('text', new GeneratorText({ generatorRegistry: registry, name: 'text', logger })) + + const writer = new FixtureWriter(logger) + const processor = new TestcaseProcessor({ logger, generatorRegistry: registry, writer: [writer], tables }) + processor.addFilterProcessor(new SimpleArrayFilterProcessor({ name: 'SimpleArrayFilter' })) + processor.addFilterProcessor(new SimpleArrayIgnoreFilterProcessor({ name: 'SimpleArrayIgnoreFilter' })) + await processor.process() + + const perTable: Record = {} + for (const tc of writer.written) perTable[tc.tableName] = (perTable[tc.tableName] ?? 0) + 1 + for (const [table, n] of Object.entries(perTable)) console.log(` ${table}: ${n} test case(s)`) + + const warnings = logger.entries.warning + const errors = logger.entries.error + if (warnings.length > 0) { + console.log(`\n${warnings.length} warning(s):`) + for (const w of warnings) console.log(JSON.stringify(w)) + } + if (errors.length > 0) { + console.error(`\n${errors.length} error(s) while generating:`) + for (const e of errors) console.error(JSON.stringify(e, null, 2)) + process.exit(1) + } + console.log(`\n${writer.written.length} fixture(s) written to ${path.relative(process.cwd(), OUT_DIR) || '.'}`) +} + +main().catch((err) => { + console.error(err instanceof Error ? (err.stack ?? err.message) : err) + process.exit(1) +}) diff --git a/skills/generate-test-data/scripts/inspect-workbook.mts b/skills/generate-test-data/scripts/inspect-workbook.mts new file mode 100644 index 0000000..f704788 --- /dev/null +++ b/skills/generate-test-data/scripts/inspect-workbook.mts @@ -0,0 +1,263 @@ +/** + * Reads a workbook with Nanook's own parser and reports what a generator run will need, + * before anything is generated. + * + * - every table: type, test-case columns, Execute / NeverExecute / Multiplicity + * - the minimum number of fixtures per table (executed columns times multiplicity, without the + * columns that reference a NeverExecute test case); a column with a range reference is marked + * `+range`, it yields one fixture per element of the range + * - every gen: directive grouped by generator; a generator that generate-fixtures.mts does not + * register, a faker path faker does not have and an unknown text instruction are errors + * - every ref: directive; a missing target table, field or test case is an error in an executed + * column and a warning in any other; a target with NeverExecute is a warning (Nanook drops the + * whole test case) + * - every FilterSection row a column uses; such a column is marked `filter` and not counted in + * the minimum (the filter decides at run time); a filter processor generate-fixtures.mts does + * not register is an error + * - every error the loader logged + * + * Usage: node scripts/inspect-workbook.mts resources/-tests.xlsx + * The registered generators and filter processors are read from generate-fixtures.mts next to + * this script. + * Exit 1 on an unknown generator or filter processor, a broken reference, a load error or no + * table at all. + */ +import path from 'node:path' +import fs from 'node:fs/promises' +import { fileURLToPath } from 'node:url' +import { + LoggerMemory, + GeneratorFaker, + DataGeneratorRegistry, + FieldSectionDefinition, + TableDecision, + createDefaultFileProcessor +} from '@xhubio/nanook-table' +import type { GeneratorDirectiveInterface } from '@xhubio/nanook-table' + +if (!process.argv[2]) { + console.error('usage: node scripts/inspect-workbook.mts ') + process.exit(1) +} +const XLSX_FILE = path.resolve(process.argv[2]) +const FIXTURE_SCRIPT = path.join(path.dirname(fileURLToPath(import.meta.url)), 'generate-fixtures.mts') +const TEXT_INSTRUCTION = /^(empty|spaces:\d+|alpha:\d+|email:\d+)$/ + +interface Field { + name: string + // row id -> content of the generator column + tdgs: Record +} + +const errors: string[] = [] +const warnings: string[] = [] + +async function registeredInFixtureScript(): Promise<{ generators: string[]; filters: string[] }> { + try { + const source = await fs.readFile(FIXTURE_SCRIPT, 'utf8') + return { + generators: [...source.matchAll(/registerGenerator\(\s*['"]([^'"]+)['"]/g)].map((m) => m[1]), + filters: [...source.matchAll(/addFilterProcessor\(\s*new\s+\w+\(\s*\{\s*name:\s*['"]([^'"]+)['"]/g)].map( + (m) => m[1] + ) + } + } catch { + warnings.push(`${path.basename(FIXTURE_SCRIPT)} not found next to this script, assuming its defaults`) + return { generators: ['faker', 'text'], filters: ['SimpleArrayFilter', 'SimpleArrayIgnoreFilter'] } + } +} + +function fieldsOf(table: TableDecision): Field[] { + const fields: Field[] = [] + for (const sectionId of table.sectionOrder) { + const section = table.sections[sectionId] + if (!(section instanceof FieldSectionDefinition)) continue + for (const sub of Object.values(section.subSections)) { + if (sub.name !== undefined) fields.push({ name: sub.name, tdgs: sub.tdgs }) + } + } + return fields +} + +// the cells a test-case column can pick for a field, as Nanook picks them: +// x/a win over any other marker, '' and i are never picked, one of the rest at random +function usedCells(field: Field, data: Record): string[] { + const preferred: string[] = [] + const others: string[] = [] + for (const [rowId, tdg] of Object.entries(field.tdgs)) { + const marker = (data[rowId] ?? '').toLowerCase() + if (marker === '' || marker === 'i' || !tdg) continue + if (marker === 'x' || marker === 'a') preferred.push(tdg) + else others.push(tdg) + } + return preferred.length > 0 ? preferred : others +} + +const fakerLogger = new LoggerMemory() +const fakerRegistry = new DataGeneratorRegistry() +const faker = new GeneratorFaker({ generatorRegistry: fakerRegistry, name: 'faker', logger: fakerLogger }) + +async function fakerPathExists(config: string): Promise { + try { + const value = await faker.generate({ + instanceId: `inspect:${config}`, + testcaseData: { name: 'inspect' }, + generatorDirective: { config } as GeneratorDirectiveInterface + }) + return value !== undefined + } catch { + return false + } +} + +async function main() { + const logger = new LoggerMemory() + const fileProcessor = createDefaultFileProcessor(logger) + await fileProcessor.load([XLSX_FILE]) + const tables = fileProcessor.tables + if (tables.length === 0) errors.push(`no table loaded from ${XLSX_FILE}`) + + const { generators: registered, filters: registeredFilters } = await registeredInFixtureScript() + const decisionTables: Record = {} + for (const table of tables) { + if (table instanceof TableDecision) decisionTables[table.tableName] = table + } + + // generator name -> directive -> where it is used + const generators: Record>> = {} + let minimum = 0 + + console.log(`workbook: ${path.relative(process.cwd(), XLSX_FILE)}`) + console.log(`registered in ${path.basename(FIXTURE_SCRIPT)}:`) + console.log(` generators: ${registered.join(', ') || 'none'}`) + console.log(` filter processors: ${registeredFilters.join(', ') || 'none'}\n`) + + for (const table of tables) { + if (!(table instanceof TableDecision)) { + console.log(`${table.tableName} [${table.tableType}] not inspected, only decision tables are`) + continue + } + const fields = fieldsOf(table) + const testcases = table.testcaseOrder.map((id) => table.testcases[id]) + const executed = testcases.filter((tc) => tc.execute) + let tableMinimum = 0 + const lines: string[] = [] + + for (const tc of testcases) { + const flags = [ + tc.execute ? 'execute' : 'no execute', + tc.neverExecute ? 'NeverExecute' : '', + tc.multiplicity > 1 ? `x${tc.multiplicity}` : '' + ].filter(Boolean) + let fansOut = false + let dropped = false + + for (const field of fields) { + for (const cell of usedCells(field, tc.data)) { + const where = `${table.tableName}.${tc.testcaseName}.${field.name}` + const lower = cell.toLowerCase() + if (lower.startsWith('gen:')) { + const name = cell.split(':')[2] ?? '' + generators[name] ??= {} + generators[name][cell] ??= new Set() + generators[name][cell].add(where) + } else if (lower.startsWith('ref:')) { + // a column that is not executed only matters when another column references it + const report = tc.execute ? errors : warnings + const [, suffix = '', targetTableName = '', targetField = '', targetTc = ''] = cell.split(':') + const targetTable = decisionTables[targetTableName || table.tableName] + if (targetTable === undefined) { + report.push(`${where}: '${cell}' points to table '${targetTableName}', which is not in this workbook`) + continue + } + // without a field name the reference takes the whole record + if (targetField !== '' && !fieldsOf(targetTable).some((f) => f.name === targetField)) { + report.push(`${where}: '${cell}' points to field '${targetField}', which table '${targetTable.tableName}' does not have`) + } + if (targetTc === '') continue + const names = targetTable.processRanges(targetTc) + if (names.length > 1) { + fansOut = true + if (suffix !== '') report.push(`${where}: '${cell}' combines a range with an instance id suffix`) + } + const targetTcs = Object.values(targetTable.testcases) + for (const name of names) { + const target = targetTcs.find((t) => t.testcaseName === name) + if (target === undefined) { + report.push(`${where}: '${cell}' points to test case '${name}', which table '${targetTable.tableName}' does not have`) + } else if (target.neverExecute && tc.execute) { + if (names.length === 1) dropped = true + warnings.push(`${where}: '${cell}' points to '${name}' with NeverExecute, Nanook drops that test case`) + } + } + } + } + } + + // only the filters of the executed column count, those of referenced columns are not run + const filters = tc.execute ? tc.createFilter() : [] + for (const filter of filters) { + if (!registeredFilters.includes(filter.filterProcessorName)) { + errors.push( + `${table.tableName}.${tc.testcaseName}: filter processor '${filter.filterProcessorName}' is not registered in ${path.basename(FIXTURE_SCRIPT)}` + ) + } + } + + if (tc.execute && !dropped && filters.length === 0) tableMinimum += tc.multiplicity + if (fansOut) flags.push('+range') + if (tc.execute && dropped) flags.push('dropped') + if (filters.length > 0) { + flags.push(`filter ${filters.map((f) => `${f.filterProcessorName}(${f.expression})`).join(' and ')}`) + } + lines.push(` ${tc.testcaseName}: ${flags.join(', ')}`) + } + + minimum += tableMinimum + console.log(`${table.tableName}: ${testcases.length} column(s), ${executed.length} executed, at least ${tableMinimum} fixture(s)`) + for (const line of lines) console.log(line) + } + + console.log('\ngenerators:') + for (const [name, directives] of Object.entries(generators)) { + const known = registered.includes(name) + console.log(` ${name}${known ? '' : ' NOT REGISTERED'}`) + for (const [directive, uses] of Object.entries(directives)) { + console.log(` ${directive} (${uses.size}x)`) + const config = directive.split(':').slice(3).join(':') + const first = [...uses][0] + if (name === 'faker' && !(await fakerPathExists(config))) { + errors.push(`${first}: '${directive}' is no faker function (a path like person.firstName, no arguments)`) + } + if (name === 'text' && known && !TEXT_INSTRUCTION.test(config)) { + errors.push(`${first}: '${directive}' is no text instruction (empty, spaces:N, alpha:N, email:N)`) + } + } + if (!known) { + errors.push(`generator '${name}' is used in ${[...Object.values(directives)].reduce((n, s) => n + s.size, 0)} cell(s) but not registered in ${path.basename(FIXTURE_SCRIPT)}`) + } + } + if (Object.keys(generators).length === 0) console.log(' none') + + for (const entry of logger.entries.error) errors.push(`loader: ${JSON.stringify(entry)}`) + for (const entry of logger.entries.warning) warnings.push(`loader: ${JSON.stringify(entry)}`) + + console.log( + `\nexpected: at least ${minimum} fixture(s), more for columns marked +range, between 0 and all for columns marked filter` + ) + if (warnings.length > 0) { + console.log(`\n${warnings.length} warning(s):`) + for (const w of warnings) console.log(` ${w}`) + } + if (errors.length > 0) { + console.error(`\n${errors.length} error(s):`) + for (const e of errors) console.error(` ${e}`) + process.exit(1) + } + console.log('\nno errors') +} + +main().catch((err) => { + console.error(err instanceof Error ? (err.stack ?? err.message) : err) + process.exit(1) +}) diff --git a/tests/skills/scriptsInSync.test.ts b/tests/skills/scriptsInSync.test.ts new file mode 100644 index 0000000..642b2ef --- /dev/null +++ b/tests/skills/scriptsInSync.test.ts @@ -0,0 +1,42 @@ +import fs from 'node:fs' +import path from 'node:path' +import { test, expect } from 'vitest' + +// Each skill is installed on its own (npx skills add --skill …), so each folder carries its +// own copy of the scripts it uses. The copies must not drift apart. +const SKILLS_DIR = path.join(import.meta.dirname, '..', '..', 'skills') +const SHARED_SCRIPTS = ['generate-fixtures.mts'] +const SKILLS = ['create-equivalence-class-table', 'generate-test-data'] + +test.each(SHARED_SCRIPTS)('%s is the same in every skill', (script) => { + const [first, ...others] = SKILLS.map((skill) => + fs.readFileSync(path.join(SKILLS_DIR, skill, 'scripts', script), 'utf8') + ) + for (const other of others) { + expect(other).toBe(first) + } +}) + +test.each(SKILLS)('plugin.json lists the skill %s', (skill) => { + const plugin = JSON.parse( + fs.readFileSync( + path.join(SKILLS_DIR, '..', '.claude-plugin', 'plugin.json'), + 'utf8' + ) + ) + expect(plugin.skills).toContain(`./skills/${skill}`) +}) + +test.each(SKILLS)('%s has the version of plugin.json', (skill) => { + const plugin = JSON.parse( + fs.readFileSync( + path.join(SKILLS_DIR, '..', '.claude-plugin', 'plugin.json'), + 'utf8' + ) + ) + const skillMd = fs.readFileSync( + path.join(SKILLS_DIR, skill, 'SKILL.md'), + 'utf8' + ) + expect(skillMd).toContain(`version: "${plugin.version}"`) +})