From d9796a9aab1eb882c6991a1ca68539ce347921db Mon Sep 17 00:00:00 2001 From: mattreid Date: Fri, 30 Oct 2026 14:21:37 -0400 Subject: [PATCH 1/6] feat(release): add /rhdh-kb-theme-analysis for public KB theme analysis Adds a read-only skill that pulls Red Hat Customer Portal KB Solution articles for Red Hat Developer Hub and compares a rolling 90-day window against a rolling 365-day window. The taxonomy is data rather than code: the built-in clusters stay the default, --themes PATH applies a taxonomy proposed for the investigation, and --themes none hands the article corpus to the agent to group. The report and JSON carry the signals needed to judge a taxonomy: per-window unmatched counts, and a warning when a single article supplies most of a cluster's caseCount365. - dependency-free Node 18+ script; no build step, no dependencies - catalog entry, /ask-rhdh routing table, README row, install manifest - one generic installable-group test replaces per-skill assertions - 34 script tests covering windows, taxonomy modes, and report shapes Assisted-By: Codex --- .claude-plugin/marketplace.json | 3 +- README.md | 4 +- skills/meta/ask-rhdh/SKILL.md | 1 + .../setup-rhdh-skills/assets/catalog.json | 6 + .../release/rhdh-kb-theme-analysis/SKILL.md | 83 ++ .../rhdh-kb-theme-analysis/agents/openai.yaml | 3 + .../rhdh-kb-theme-analysis/references/api.md | 65 ++ .../references/interpretation.md | 110 +++ .../references/taxonomy.md | 105 +++ .../scripts/analyze-kb.mjs | 749 ++++++++++++++++++ .../tests/analyze-kb.test.mjs | 545 +++++++++++++ tests/unit/test_plugin_manifest.py | 15 + 12 files changed, 1686 insertions(+), 3 deletions(-) create mode 100644 skills/release/rhdh-kb-theme-analysis/SKILL.md create mode 100644 skills/release/rhdh-kb-theme-analysis/agents/openai.yaml create mode 100644 skills/release/rhdh-kb-theme-analysis/references/api.md create mode 100644 skills/release/rhdh-kb-theme-analysis/references/interpretation.md create mode 100644 skills/release/rhdh-kb-theme-analysis/references/taxonomy.md create mode 100644 skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs create mode 100644 skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c215d815..81fe2281 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -63,7 +63,8 @@ "./skills/release/rhdh-smoke-tests", "./skills/release/rhdh-translation-upload", "./skills/release/rhdh-translation-deploy", - "./skills/release/rhdh-test-plan-review" + "./skills/release/rhdh-test-plan-review", + "./skills/release/rhdh-kb-theme-analysis" ] }, { diff --git a/README.md b/README.md index 99fa5d05..c77923f1 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ one. It performs no work itself. and the single source of truth for membership. This file does not restate it. Three skills are human-invoked and never selected automatically: `/ask-rhdh`, -`/setup-rhdh-skills`, and `/clean-prose`. The other 68 are model-invoked, and can +`/setup-rhdh-skills`, and `/clean-prose`. The other 69 are model-invoked, and can also be called by name. Skills are grouped into seven folders: @@ -58,7 +58,7 @@ Skills are grouped into seven folders: | `jira/` | Creating, refining, updating, and reporting on RHIDP, RHDHPLAN, RHDHBUGS, and RHDHSUPP work, plus sprint ceremonies and linking PRs to issues. | | `plugins/` | Authoring, wiring, exporting, and fixing Backstage dynamic plugins; the overlays repository; local RHDH; opening and reviewing pull requests; testing rhdh-operator PRs on a cluster; midstream propagation; browser UI verification with Playwright. | | `ci/` | Prow job configuration and nightly triggers, Konflux and Tekton task updates, plugin catalog index status, release-data admission tags, base images, and Yarn bumps. | -| `release/` | Release status and readiness, candidate-feature capacity through Code Freeze, milestone schedules, freeze announcements, teams, fix-version sync across RHIDP/RHDHPLAN/RHDHBUGS, test-plan review, platform lifecycle, plugin CVE export, and RC/GA Helm+Operator smoke. | +| `release/` | Release status and readiness, candidate-feature capacity through Code Freeze, milestone schedules, freeze announcements, teams, fix-version sync across RHIDP/RHDHPLAN/RHDHBUGS, test-plan review, platform lifecycle, plugin CVE export, RC/GA Helm+Operator smoke, and public KB theme analysis for release planning. | | `refinement/` | The OpenSpec artifact-driven change workflow: propose, explore, implement, audit, verify, and archive a change. See [the workflow](docs/refinement/refinement-workflow.md). | | `reference/` | The reusable layer other skills invoke by name: repository and version context, the forge read seam, the write gate, the OpenSpec journal and RHDH's spec-driven schema, and the Jira and Backstage reference material. | | `meta/` | The three human-invoked entry points, plus skill authoring and repository agent-readiness. | diff --git a/skills/meta/ask-rhdh/SKILL.md b/skills/meta/ask-rhdh/SKILL.md index aad7c948..40b73c9e 100644 --- a/skills/meta/ask-rhdh/SKILL.md +++ b/skills/meta/ask-rhdh/SKILL.md @@ -74,6 +74,7 @@ competing inventory, which is what this table used to be. | Reads GitHub and GitLab and constructs unexecuted forge payloads for the other RHDH skills: parse an issue, pull request, or merge request reference, fetch issue detail, resolve a plugin workspace, inspect checks or pipelines, read repository files, or build the exact command for a GitHub pull request, GitLab merge request, comment, label, assignee, approval, or /publish write. Use for a forge URL, a bare #number, a !number merge request, "which workspace is this issue in", a stale statusCheckRollup, "why did that check fail", or safe gh, glab, and jq command construction. | `/rhdh-forge` | | Reads RHDH Jira and owns the mechanics every other RHDH Jira skill needs: `acli` flags and their traps, JQL for RHIDP, RHDHPLAN, RHDHBUGS and RHDHSUPP, custom field IDs, GraphQL and REST fallbacks, board and sprint IDs, the component catalog, and the workflow states with their exit criteria. Use to look up an issue such as RHIDP-1234, to write or debug a JQL query, to answer "which field is Story Points", "why is my search returning 30 rows", or "what does Release Pending require". Reading and query mechanics only — deciding what to file, judging readiness, or changing an issue belongs to the Jira skill that owns that verb. | `/rhdh-jira-api` | | Supplies the craft of writing good RHDH Jira work in RHIDP, RHDHPLAN, RHDHBUGS, and RHDHSUPP: the Feature, Epic, Story, Task, and Bug description templates with filled examples, the RHDH challenge matrix that stress-tests scope, sizing, acceptance criteria, and Epic independence, the T-shirt and Fibonacci sizing scales, keyword duplicate detection, and the tracer-bullet rules for breaking a Feature into Epics or an Epic into Stories. Use when drafting or judging the text and shape of an issue such as RHIDP-1234 — how big is this, is this AC testable, should this be one Epic or three, does this already exist. Prose and estimation craft only; it runs no Jira command. | `/rhdh-jira-authoring` | +| Analyzes Red Hat Customer Portal Knowledgebase Solution articles for Red Hat Developer Hub, compares rolling 90-day and 365-day windows, and turns public article metadata into support-theme, documentation, and roadmap signals. Use for RHDH KB trend analysis, support-theme reviews, or product friction scans. | `/rhdh-kb-theme-analysis` | | Owns the append-only interaction journal for an OpenSpec change — `openspec/changes//journal.jsonl` — and the fixed event vocabulary written to it (change.created, artifact.added/revised, mode.chosen, task.start/complete/blocked, verifier.result, decision, handoff, archive, skill.invoked, agent.spawned, context.compacted, turn.start/end). Invoked by name from the openspec-\* skills whenever one of those events happens; not a user-facing entry point on its own. Use for "log this to the journal", "emit a journal event", "what fields does task.complete need", or "why did the journal helper reject my write". | `/openspec-journal` | | Owns RHDH's project-local OpenSpec workflow definition — the `rhdh-spec-driven` schema (proposal -> {specs, design} -> tasks -> apply), its four artifact templates, the Canonical Touchpoints rule that ties a change back to `specifications/prd/`, `specifications/adr/`, and `openspec/specs//spec.md`, and the shared artifact-creation-loop mechanics every openspec-\* skill drives through the `openspec` CLI. Also installs `config.yaml` and `schemas/rhdh-spec-driven/` into a product repo's `openspec/` so the CLI can resolve that schema. Invoked by name from openspec-new-change, openspec-continue-change, openspec-ff-change, and openspec-onboard; not a standalone entry point. Use for "what does the spec-driven schema require", "install the rhdh-spec-driven schema", "what goes in Canonical Touchpoints", "how do I fill in an artifact template", or "why did an artifact instruction reject my capability name". | `/rhdh-spec-driven-schema` | | Scaffolds a new OpenSpec change directory and shows the first artifact's template without writing it. Use for "start a new change", "scaffold a change", "set up a change for X", or "what's the first artifact I need" — when the user wants the folder and the first template, not a drafted proposal. Creates the change directory and stops — it does not draft or write any artifact content, and it does not implement code. | `/openspec-new-change` | diff --git a/skills/meta/setup-rhdh-skills/assets/catalog.json b/skills/meta/setup-rhdh-skills/assets/catalog.json index 5b07ce9c..cd9b5986 100644 --- a/skills/meta/setup-rhdh-skills/assets/catalog.json +++ b/skills/meta/setup-rhdh-skills/assets/catalog.json @@ -452,6 +452,12 @@ "prose-editing" ] }, + { + "name": "rhdh-kb-theme-analysis", + "category": "release", + "invocation": "model", + "requiresSkills": [] + }, { "name": "openspec-journal", "category": "reference", diff --git a/skills/release/rhdh-kb-theme-analysis/SKILL.md b/skills/release/rhdh-kb-theme-analysis/SKILL.md new file mode 100644 index 00000000..de145116 --- /dev/null +++ b/skills/release/rhdh-kb-theme-analysis/SKILL.md @@ -0,0 +1,83 @@ +--- +name: rhdh-kb-theme-analysis +description: >- + Analyzes Red Hat Customer Portal Knowledgebase Solution articles for Red Hat + Developer Hub, compares rolling 90-day and 365-day windows, and turns public + article metadata into support-theme, documentation, and roadmap signals. Use + for RHDH KB trend analysis, support-theme reviews, or product friction scans. +compatibility: "Node 18+; no dependencies." +--- + +# RHDH KB theme analysis + +Turn public Red Hat Customer Portal KB metadata into a product-facing view of +RHDH support themes. The workflow is read-only and uses the public KB search +API; it does not open, read, or modify private support cases or issue-tracker +data. + +## Run + +Use the bundled script: + +```bash +node scripts/analyze-kb.mjs \ + --product "Red Hat Developer Hub" \ + --as-of YYYY-MM-DD \ + --format markdown +``` + +Defaults: + +- product: `Red Hat Developer Hub` +- recent window: trailing 90 days +- comparison window: trailing 365 days +- output: Markdown + +Pass explicit `--recent-since`, `--recent-until`, +`--comparison-since`, or `--comparison-until` values when the user asks for a +calendar quarter, calendar year, release window, or custom range. + +Choose the taxonomy before the run: + +- omitted: the built-in clusters, for a general support-theme scan; +- `--themes PATH`: a taxonomy proposed for this investigation; +- `--themes none`: no clustering, leaving the grouping to you. + +Read `references/taxonomy.md` before writing a taxonomy file. Use +`--format json` when the answer needs every article rather than the top 15, and +add `--include-abstracts` when the grouping will be done from article text. + +## Boundary with neighbouring skills + +- Use this skill for public KB article themes and release-planning signals. +- Use `/rhdh-release-status` for open Jira work, blockers, CVEs, and release + readiness. +- Use `/rhdh-feature-docs` to write or update documentation content. +- Use `/rhdh-platform-lifecycle` for vendor support phases and EOL dates. + +## Interpretation + +Read `references/interpretation.md` before turning script output into product, +documentation, or roadmap recommendations. It carries the taxonomy the run used +and the rule that both windows share one. + +Read `references/api.md` only when changing the query, troubleshooting the API, +or extending the script. + +## Failure mode + +If the KB API is unreachable or returns a non-2xx response, report that the +public KB data could not be fetched. Do not infer themes from memory or from a +previous run. Name the endpoint status if the script provides one. + +## Completion + +Complete when the answer names: + +- the product and run date; +- both window boundaries; +- recent and comparison article counts; +- the strongest theme signals; +- linked citations for every article named in the answer; +- a final “Product and documentation improvement areas” section that connects recent findings to candidate product and documentation work; +- the caveats from the interpretation reference. diff --git a/skills/release/rhdh-kb-theme-analysis/agents/openai.yaml b/skills/release/rhdh-kb-theme-analysis/agents/openai.yaml new file mode 100644 index 00000000..1c7d3990 --- /dev/null +++ b/skills/release/rhdh-kb-theme-analysis/agents/openai.yaml @@ -0,0 +1,3 @@ +interface: + display_name: "RHDH KB Theme Analysis" + short_description: "Compare RHDH KB article themes across rolling windows" diff --git a/skills/release/rhdh-kb-theme-analysis/references/api.md b/skills/release/rhdh-kb-theme-analysis/references/api.md new file mode 100644 index 00000000..7e2d499d --- /dev/null +++ b/skills/release/rhdh-kb-theme-analysis/references/api.md @@ -0,0 +1,65 @@ +# Red Hat KB search API + +The Customer Portal KB search page is client-rendered. Do not scrape the HTML. +Use the JSON endpoint behind it: + +```text +https://access.redhat.com/hydra/rest/search/platform/kbase +``` + +## Query shape + +The script sends: + +- `q=*` +- `fq=-id:Other` +- `fq=documentKind:(Solution Article)` +- `fq={!tag=productFilter}standard_product:"..."` +- `facet=true` +- `facet.mincount=1` +- `enableElevation=true` +- `f.standard_product.facet.limit=-1` +- `redhat_client=kb-search` +- `facet.field=documentKind` +- `facet.field={!ex=productFilter}standard_product` + +The Solr local params in `{!ex=...}` and `{!tag=...}` must remain URL-encoded. +`URLSearchParams` handles this correctly. + +## Pagination + +The API reports `response.numFound` and returns documents in +`response.docs`. The script requests 100 rows per page, uses a 30-second +request timeout, and follows `start` until it has `numFound` results or reaches +the 10,000-result safety cap. If the cap truncates the result, the report +prints an “at least” count rather than presenting the fetched count as the +total. + +## Useful fields + +Treat these fields as optional because Red Hat can change the public response. +The script accepts either string or array shapes for `category`, `component`, +and `tag`: + +- `id` +- `title` +- `view_uri` +- `createdDate` +- `lastModifiedDate` +- `category` +- `component` +- `tag` +- `standard_product` +- `issue` +- `caseCount` +- `caseCount_365` + +The analysis filters on the `YYYY-MM-DD` prefix of `createdDate`. Theme +matching reads `title`, `issue` (the abstract), `category`, and `component`, so +broad stems match abstract prose; `--include-abstracts` only controls whether an +abstract is echoed in the output, never whether it is matched. + +## Runtime + +`scripts/analyze-kb.mjs` is the committed, dependency-free runtime artifact. Do +not add a dependency or build step at skill runtime. diff --git a/skills/release/rhdh-kb-theme-analysis/references/interpretation.md b/skills/release/rhdh-kb-theme-analysis/references/interpretation.md new file mode 100644 index 00000000..75467241 --- /dev/null +++ b/skills/release/rhdh-kb-theme-analysis/references/interpretation.md @@ -0,0 +1,110 @@ +# Interpreting RHDH KB theme analysis + +## Time windows + +The default recent window is the trailing 90 days. The default comparison +window is the trailing 365 days. Both end on `--as-of` and are inclusive. + +When the recent window is nested inside the comparison window, compare: + +- article counts, +- each theme’s share of its window, +- each theme’s per-day rate, + +not raw counts alone. A theme can look large in the annual window simply +because the window is four times longer. + +For formal calendar periods, pass explicit start and end dates rather than +changing the default rolling behavior. If the windows are not nested, say so +and compare per-day rates and window shares rather than raw counts. + +## Theme signals + +The script emits heuristic, non-exclusive keyword clusters: + +- `upgrade-plugin-reliability` +- `authentication-identity` +- `operator-openshift-deployment` +- `rbac-permissions` +- `configuration-ui` +- `observability-diagnostics` +- `support-version-clarity` +- `catalog-database-scale` + +Use them as a starting point, not as official Red Hat categories. An article +can match more than one theme, so theme counts do not sum to the article total. + +Those eight clusters are the built-in taxonomy. A run can instead use a custom +taxonomy (`--themes PATH`) or none at all (`--themes none`); `taxonomy.md` covers +both. When the run did not use the built-in set, say which taxonomy produced the +table, and never compare theme counts against a run that used a different one. + +Prefer the objective `category`, `tag`, and `component` counts when they +conflict with a heuristic theme signal. + +## Product interpretation + +For each strong theme, report: + +- what customers appear to be hitting; +- whether it is configuration, troubleshooting, upgrade, or support-boundary + friction; +- the likely documentation impact; +- the likely product or roadmap impact; +- the confidence level. + +Look for themes that are: + +- new in the recent window, +- growing relative to the annual rate, +- persistent across both windows, +- high-touch by `caseCount_365`, +- concentrated around a release or plugin family. + +## Product and documentation recommendations + +End the final answer with a “Product and documentation improvement areas” +section. Connect the strongest recent themes to concrete candidate work, and +separate product improvements from documentation improvements. Frame them as +areas the team should consider funding, not as automatically committed roadmap +items. + +Candidate starting points, to be adjusted against the actual article evidence. +They are keyed to the built-in clusters; with a custom taxonomy, map your +clusters onto the closest problem area rather than reusing these names: + +- `upgrade-plugin-reliability`: upgrade preflight, plugin compatibility matrix, + plugin doctor, and release-critical upgrade documentation. +- `configuration-ui`: configuration linting, guided setup, stable customization + APIs, and task-based customization documentation. +- `observability-diagnostics`: actionable error codes, built-in diagnostics, a + support bundle, and symptom-based troubleshooting documentation. +- `operator-openshift-deployment`: install preflight, deployment profiles, + clearer Operator status, and an enterprise deployment guide. +- `rbac-permissions`: policy simulation, permission explainability, safer + defaults, scale work, and one RBAC mental-model guide. +- `authentication-identity`: guided provider setup, test-connection validation, + token lifecycle handling, and provider-specific recipes. +- `catalog-database-scale`: catalog/database status, ingestion diagnostics, + performance work, and scale guidance. +- `support-version-clarity`: generated compatibility and lifecycle views plus an + authoritative support matrix. + +## Article citations + +When naming an article in the final answer, link its title to the `url` from the +script output. Do not cite a KB article by title alone. + +## Caveats + +State these limitations in the final answer: + +- Article count is not case volume. +- `caseCount` and `caseCount_365` are article-level support-case signals, not + unique case counts. Theme totals sum these values across matching articles and + can be dominated by broad, multi-product articles. +- Public KB metadata does not confirm article authorship. +- The API may include articles tagged to multiple products. +- Theme clusters are heuristic. +- A spike can reflect release activity, documentation gaps, support process + changes, or product regressions; the KB alone does not prove causation. diff --git a/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md b/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md new file mode 100644 index 00000000..9b08d320 --- /dev/null +++ b/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md @@ -0,0 +1,105 @@ +# Theme taxonomies + +A taxonomy is the list of clusters the script counts articles into. Each cluster +is a name plus a case-insensitive keyword pattern, matched against the article +title, abstract, category, and component. Matching is non-exclusive, so one +article can land in several clusters and cluster counts do not sum to the +article total. + +## The built-in taxonomy + +`--themes` omitted uses the eight built-in clusters: + +- `upgrade-plugin-reliability` +- `authentication-identity` +- `operator-openshift-deployment` +- `rbac-permissions` +- `configuration-ui` +- `observability-diagnostics` +- `support-version-clarity` +- `catalog-database-scale` + +Keep it when the question is a general "what are customers hitting with RHDH" +scan. It is shaped by install, upgrade, and configuration support, which is where +most RHDH articles sit. + +## No taxonomy + +`--themes none` skips clustering entirely: the report carries `themes: null` and +the article list becomes the raw categorisation input. + +Use it when the user wants the data and will group it themselves, or when the +question does not fit any keyword scheme. Pair it with `--format json`, because +the Markdown report only lists the top 15 recent articles while JSON carries +every article. + +## A custom taxonomy + +`--themes PATH` loads a JSON array: + +```json +[ + { "theme": "techdocs-build", "pattern": "\\btechdocs?\\b|techdocs builder" }, + { "theme": "scaffolder-templates", "pattern": "\\bscaffolder\\b|software template" } +] +``` + +Rules the loader enforces: + +- the file is a non-empty JSON array of objects with a non-empty `theme` and `pattern`; +- theme names are unique; +- every `pattern` compiles as a regular expression. + +An unreadable file, a bad pattern, or a duplicate theme stops the run with a +message naming the entry, rather than silently producing an empty cluster. + +## Proposing one + +Propose the taxonomy from the article titles first, then run the script once to +apply it to both windows. Do not count clusters by hand per window — that is how +two windows end up measured differently. + +Guidance for the proposal: + +- Five to twelve clusters. Fewer loses signal; more turns the table into a title + index. +- Name clusters after the customer problem, not the component that failed. +- Anchor patterns on domain nouns (`techdocs`, `scaffolder`, `rbac`), not on + whole article titles. A pattern copied from one title fits one article and + measures nothing. +- Keep matching non-exclusive and leave out an "other" cluster; the article + count already tells you what did not match. + +## Checking the distribution + +Apply the taxonomy once and read the numbers before trusting it: + +- No cluster above roughly 40% of the window. A cluster that matches more than + a third of the articles is usually several themes wearing one name, and its + share tells you nothing. +- No empty cluster. A pattern that matches nothing in this corpus is either + miscast or already covered by a better cluster. +- Unmatched is small. The report prints `unmatched by taxonomy` per window; a + large number means the taxonomy misses a real theme. +- Case totals carry a concentration warning. When one article supplies more + than half of a cluster's `caseCount365`, the report says so; treat that total + as one article, not as demand for the whole theme. + +The matcher reads the abstract as well as the title. A stem that only ever +matches in long prose — `support`, `many`, `large`, bare `logs` — will pull in +articles that merely mention the word. When a cluster's matches are mostly +abstract-only, tighten the pattern rather than adding another cluster. + +## Comparability + +The recent and comparison windows must always use the same taxonomy. Theme +counts, shares, and per-day rates are only comparable across runs that used the +same one, so state which taxonomy produced a table whenever it is not the +built-in set. A run with `--themes none` has no theme table and must not be +compared against a clustered run. + +## Writing the file + +A taxonomy file is a file on the user's disk. Say where it will be written and +get agreement before creating it, then reuse that path for follow-up runs so the +windows stay comparable. diff --git a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs new file mode 100644 index 00000000..5c389c2b --- /dev/null +++ b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs @@ -0,0 +1,749 @@ +#!/usr/bin/env node +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; + +export const DEFAULT_PRODUCT = 'Red Hat Developer Hub'; +export const DEFAULT_RECENT_DAYS = 90; +export const DEFAULT_COMPARISON_DAYS = 365; +export const KB_ENDPOINT = 'https://access.redhat.com/hydra/rest/search/platform/kbase'; +export const MAX_RESULTS = 10000; +export const PAGE_SIZE = 100; +export const REQUEST_TIMEOUT_MS = 30000; + +const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/; + +const BUILT_IN_THEME_RULES = [ + { + theme: 'upgrade-plugin-reliability', + pattern: + /\b(?:upgrad(?:e|es|ed|ing)|plugins?|dynamic plugins?|install-dynamic-plugins|catalog indexes?|orchestrators?|modules?|dependenc(?:y|ies)|plugin versions?|version resolutions?|version mismatches?|compatibility matrix)\b/i, + }, + { + theme: 'authentication-identity', + pattern: + /\b(?:authentications?|authorizations?|auth|logins?|sso|saml|oidc|jwks|entra|azure|keycloaks?|gitlabs?|githubs?|mfa|tokens?|cookies?|redirects?)\b/i, + }, + { + theme: 'operator-openshift-deployment', + pattern: + /\b(?:operators?|openshifts?|ocp|helms?|deployments?|deploys?|deployed|deploying|regist(?:ry|ries)|images?|tls|certificates?|prox(?:y|ies)|networks?|disconnected|air-gapped|pods?|init containers?|init-containers?|crashloops?|storages?|pvcs?|namespaces?)\b/i, + }, + { + theme: 'rbac-permissions', + pattern: + /\b(?:rbac|permissions?|roles?|policies?|admins?|visibility|entit(?:y|ies)|owned|access(?:es)?|groups?)\b/i, + }, + { + theme: 'configuration-ui', + pattern: + /\b(?:configs?|configurations?|customizations?|custom|yamls?|schemas?|widgets?|forms?|templates?|ui|themes?|logos?|sidebars?|menus?|localizations?)\b/i, + }, + { + theme: 'observability-diagnostics', + pattern: + /\b(?:logs?|logging|debugs?|debugging|must-gathers?|audits?|auditing|monitors?|monitoring|performances?|slows?|crash(?:es|ed|ing)?|diagnostics?|traces?|tracing)\b/i, + }, + { + theme: 'support-version-clarity', + pattern: + /\b(?:supports?|supported|supportability|subscriptions?|lifecycles?|(?:in)?compatib(?:le|ility|ilities)|fips|azure devops|local supported|product supports?)\b/i, + }, + { + theme: 'catalog-database-scale', + pattern: + /\b(?:catalogs?|databases?|postgres(?:ql)?|replicas?|(?:auto|down)?scal(?:e|es|ed|ing)|large(?:r|st)?|many|ingestions?|schedulers?|scheduling|syncs?|syncing|synced|synchronizations?)\b/i, + }, +]; + +function asList(value) { + if (Array.isArray(value)) { + return value.filter((item) => typeof item === 'string' && item.length > 0); + } + if (typeof value === 'string' && value.length > 0) { + return [value]; + } + return []; +} + +export const ABSTRACT_LIMIT = 300; + +const BUILT_IN_TAXONOMY = { + mode: 'built-in', + name: 'built-in', + rules: BUILT_IN_THEME_RULES, +}; + +export function loadTaxonomy(filePath) { + let parsed; + try { + parsed = JSON.parse(readFileSync(filePath, 'utf8')); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Cannot read taxonomy file ${filePath}: ${message}`); + } + + if (!Array.isArray(parsed) || parsed.length === 0) { + throw new Error( + `Taxonomy file ${filePath} must be a non-empty JSON array of { "theme", "pattern" } objects`, + ); + } + + const seen = new Set(); + const rules = parsed.map((entry, index) => { + const theme = entry?.theme; + const pattern = entry?.pattern; + if (typeof theme !== 'string' || theme.length === 0) { + throw new Error(`Taxonomy entry ${index} in ${filePath} needs a non-empty "theme"`); + } + if (typeof pattern !== 'string' || pattern.length === 0) { + throw new Error(`Taxonomy entry "${theme}" in ${filePath} needs a non-empty "pattern"`); + } + if (seen.has(theme)) { + throw new Error(`Taxonomy file ${filePath} repeats the theme "${theme}"`); + } + seen.add(theme); + try { + return { theme, pattern: new RegExp(pattern, 'i') }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `Taxonomy pattern for "${theme}" in ${filePath} is not a valid regular expression: ${message}`, + ); + } + }); + + return { mode: 'custom', name: filePath, rules }; +} + +export function resolveTaxonomy(value) { + if (value === undefined) { + return BUILT_IN_TAXONOMY; + } + if (value === 'none') { + return { mode: 'none', name: 'none', rules: null }; + } + return loadTaxonomy(value); +} + +export function usage() { + return `Usage: analyze-kb.mjs [options] + +Analyze Red Hat Customer Portal KB Solution articles. + +Options: + --product NAME Product filter (default: "${DEFAULT_PRODUCT}") + --as-of YYYY-MM-DD Analysis date (default: today, local time) + --recent-days N Recent window length (default: ${DEFAULT_RECENT_DAYS}) + --comparison-days N Comparison window length (default: ${DEFAULT_COMPARISON_DAYS}) + --recent-since DATE Explicit recent window start + --recent-until DATE Explicit recent window end + --comparison-since DATE Explicit comparison window start + --comparison-until DATE Explicit comparison window end + --themes PATH|none Theme taxonomy JSON file, or "none" to skip clustering (default: built-in) + --include-abstracts Include a truncated abstract per article in JSON output + --format FORMAT Output format: markdown or json (default: markdown) + -h, --help Show this help + +Examples: + analyze-kb.mjs + analyze-kb.mjs --as-of 2026-10-30 + analyze-kb.mjs --recent-days 30 --comparison-days 180 + analyze-kb.mjs --recent-since 2026-07-01 --recent-until 2026-09-30 + analyze-kb.mjs --themes ./taxonomy.json --format json --include-abstracts + analyze-kb.mjs --themes none --format json +`; +} + +function nextValue(argv, index, flag) { + const nextIndex = index + 1; + if (nextIndex >= argv.length) { + throw new Error(`Missing value for ${flag}`); + } + return { value: argv[nextIndex], nextIndex }; +} + +export function parseArgs(argv) { + const args = { + product: DEFAULT_PRODUCT, + asOf: todayIso(), + format: 'markdown', + help: false, + includeAbstracts: false, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '-h' || arg === '--help') { + args.help = true; + continue; + } + if (arg === '--include-abstracts') { + args.includeAbstracts = true; + continue; + } + + const flagsWithValues = { + '--product': 'product', + '--as-of': 'asOf', + '--recent-days': 'recentDays', + '--comparison-days': 'comparisonDays', + '--recent-since': 'recentSince', + '--recent-until': 'recentUntil', + '--comparison-since': 'comparisonSince', + '--comparison-until': 'comparisonUntil', + '--themes': 'themes', + '--format': 'format', + }; + + if (arg in flagsWithValues) { + const { value, nextIndex } = nextValue(argv, i, arg); + const key = flagsWithValues[arg]; + if (key === 'recentDays' || key === 'comparisonDays') { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${arg} must be a positive integer`); + } + args[key] = parsed; + } else if (key === 'format') { + if (value !== 'json' && value !== 'markdown') { + throw new Error(`Invalid format: ${value}. Use json or markdown.`); + } + args.format = value; + } else { + args[key] = value; + } + i = nextIndex; + continue; + } + + throw new Error(`Unknown argument: ${arg}`); + } + + validateDate(args.asOf, '--as-of'); + for (const [flag, value] of Object.entries({ + '--recent-since': args.recentSince, + '--recent-until': args.recentUntil, + '--comparison-since': args.comparisonSince, + '--comparison-until': args.comparisonUntil, + })) { + if (value !== undefined) { + validateDate(value, flag); + } + } + + if (args.recentDays !== undefined && (args.recentSince !== undefined || args.recentUntil !== undefined)) { + throw new Error('--recent-days cannot be combined with --recent-since or --recent-until'); + } + if (args.comparisonDays !== undefined && (args.comparisonSince !== undefined || args.comparisonUntil !== undefined)) { + throw new Error('--comparison-days cannot be combined with --comparison-since or --comparison-until'); + } + if (args.recentUntil !== undefined && args.recentSince === undefined) { + throw new Error('--recent-until requires --recent-since'); + } + if (args.comparisonUntil !== undefined && args.comparisonSince === undefined) { + throw new Error('--comparison-until requires --comparison-since'); + } + + args.themes = resolveTaxonomy(args.themes); + + return args; +} + +export function todayIso() { + return new Date().toLocaleDateString('en-CA', { + year: 'numeric', + month: '2-digit', + day: '2-digit', + }); +} + +export function validateDate(value, flag) { + if (!ISO_DATE_RE.test(value)) { + throw new Error(`${flag} must use YYYY-MM-DD`); + } + const date = new Date(`${value}T00:00:00Z`); + if (Number.isNaN(date.getTime()) || date.toISOString().slice(0, 10) !== value) { + throw new Error(`${flag} is not a valid calendar date: ${value}`); + } +} + +export function addDaysIso(dateIso, days) { + const date = new Date(`${dateIso}T00:00:00Z`); + date.setUTCDate(date.getUTCDate() + days); + return date.toISOString().slice(0, 10); +} + +export function daysBetweenInclusive(start, end) { + const startTime = new Date(`${start}T00:00:00Z`).getTime(); + const endTime = new Date(`${end}T00:00:00Z`).getTime(); + return Math.round((endTime - startTime) / 86400000) + 1; +} + +function resolveWindow(asOf, days, since, until, label) { + if (since !== undefined) { + const end = until ?? asOf; + if (since > end) { + throw new Error(`${label} start must not be after ${label} end`); + } + return { start: since, end, days: daysBetweenInclusive(since, end) }; + } + + const length = days ?? (label === 'recent' ? DEFAULT_RECENT_DAYS : DEFAULT_COMPARISON_DAYS); + return { + start: addDaysIso(asOf, -(length - 1)), + end: asOf, + days: length, + }; +} + +export function resolveWindows(args) { + return { + recent: resolveWindow(args.asOf, args.recentDays, args.recentSince, args.recentUntil, 'recent'), + comparison: resolveWindow(args.asOf, args.comparisonDays, args.comparisonSince, args.comparisonUntil, 'comparison'), + }; +} + +export function buildQuery(product, start, rows) { + const params = new URLSearchParams({ + q: '*', + start: String(start), + rows: String(rows), + facet: 'true', + 'facet.mincount': '1', + enableElevation: 'true', + 'f.standard_product.facet.limit': '-1', + redhat_client: 'kb-search', + }); + params.append('fq', '-id:Other'); + params.append('fq', 'documentKind:(Solution Article)'); + params.append('facet.field', 'documentKind'); + params.append('facet.field', '{!ex=productFilter}standard_product'); + params.append('fq', `{!tag=productFilter}standard_product:"${product}"`); + + const url = new URL(KB_ENDPOINT); + url.search = params.toString(); + return url; +} + +export async function fetchKbDocs(product) { + const docs = []; + let numFound; + + while (true) { + const url = buildQuery(product, docs.length, PAGE_SIZE); + const response = await fetch(url, { + headers: { + Accept: 'application/json', + 'User-Agent': 'rhdh-kb-theme-analysis/1.0', + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!response.ok) { + throw new Error(`Red Hat KB request failed: ${response.status} ${response.statusText}`); + } + + const json = await response.json(); + const batch = Array.isArray(json.response?.docs) ? json.response.docs : []; + docs.push(...batch); + + const responseNumFound = json.response?.numFound; + if (responseNumFound !== undefined) { + numFound = Number(responseNumFound); + } else if (numFound === undefined) { + numFound = docs.length; + } + + if (batch.length === 0 || docs.length >= Math.min(numFound, MAX_RESULTS)) { + break; + } + } + + const fetched = docs.slice(0, MAX_RESULTS); + return { + docs: fetched, + numFound, + truncated: numFound > fetched.length, + }; +} + +function docCreatedDate(doc) { + const value = doc.createdDate; + if (typeof value !== 'string' || !ISO_DATE_RE.test(value.slice(0, 10))) { + return null; + } + return value.slice(0, 10); +} + +export function isInWindow(doc, window) { + const created = docCreatedDate(doc); + return created !== null && created >= window.start && created <= window.end; +} + +function countValues(docs, field) { + const counts = new Map(); + for (const doc of docs) { + for (const value of asList(doc[field])) { + counts.set(value, (counts.get(value) ?? 0) + 1); + } + } + return [...counts.entries()] + .map(([value, count]) => ({ value, count })) + .sort((a, b) => b.count - a.count || a.value.localeCompare(b.value)); +} + +function countByPeriod(docs, period) { + const counts = new Map(); + for (const doc of docs) { + const created = docCreatedDate(doc); + if (created === null) { + continue; + } + const key = period === 'month' ? created.slice(0, 7) : created.slice(0, 4); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return [...counts.entries()] + .map(([value, count]) => ({ value, count })) + .sort((a, b) => a.value.localeCompare(b.value)); +} + +function docText(doc) { + return [doc.title, ...asList(doc.issue), ...asList(doc.category), ...asList(doc.component)].join(' '); +} + +export function themeHints(doc, rules = BUILT_IN_THEME_RULES) { + const text = docText(doc); + return rules.filter((rule) => rule.pattern.test(text)).map((rule) => rule.theme); +} + +function abstractText(doc) { + const issue = asList(doc.issue).join(' '); + if (issue.length === 0) { + return undefined; + } + if (issue.length <= ABSTRACT_LIMIT) { + return issue; + } + return `${issue.slice(0, ABSTRACT_LIMIT).trimEnd()}\u2026`; +} + +function articleSummary(doc, rules, includeAbstracts) { + const summary = { + id: doc.id, + title: doc.title, + url: doc.view_uri ?? `https://access.redhat.com/solutions/${doc.id}`, + createdDate: docCreatedDate(doc) ?? 'unknown', + categories: asList(doc.category), + tags: asList(doc.tag), + components: asList(doc.component), + caseCount: doc.caseCount ?? 0, + caseCount365: doc.caseCount_365 ?? 0, + themes: rules === null ? null : themeHints(doc, rules), + }; + if (includeAbstracts) { + const abstract = abstractText(doc); + if (abstract !== undefined) { + summary.abstract = abstract; + } + } + return summary; +} + +function summarizeWindow(docs, window, rules) { + const windowDocs = docs.filter((doc) => isInWindow(doc, window)); + + return { + window, + articleCount: windowDocs.length, + categories: countValues(windowDocs, 'category'), + tags: countValues(windowDocs, 'tag'), + components: countValues(windowDocs, 'component'), + months: countByPeriod(windowDocs, 'month'), + years: countByPeriod(windowDocs, 'year'), + themes: rules === null ? null : countThemes(windowDocs, window, rules), + unmatched: + rules === null ? null : windowDocs.filter((doc) => themeHints(doc, rules).length === 0).length, + }; +} + +function countThemes(windowDocs, window, rules) { + const themes = new Map(); + + for (const doc of windowDocs) { + const cases = doc.caseCount_365 ?? 0; + for (const theme of themeHints(doc, rules)) { + const current = themes.get(theme) ?? { count: 0, caseCount365: 0, maxArticleCases: 0 }; + current.count += 1; + current.caseCount365 += cases; + current.maxArticleCases = Math.max(current.maxArticleCases, cases); + themes.set(theme, current); + } + } + + return [...themes.entries()] + .map(([theme, value]) => ({ + theme, + count: value.count, + caseCount365: value.caseCount365, + maxArticleCases: value.maxArticleCases, + topArticleShare: value.caseCount365 === 0 ? 0 : value.maxArticleCases / value.caseCount365, + perDay: value.count / window.days, + share: windowDocs.length === 0 ? 0 : value.count / windowDocs.length, + })) + .sort((a, b) => b.count - a.count || a.theme.localeCompare(b.theme)); +} + +export function summarize(fetchResult, args) { + const { docs, numFound, truncated } = fetchResult; + const { recent, comparison } = resolveWindows(args); + const taxonomy = args.themes ?? BUILT_IN_TAXONOMY; + const rules = taxonomy.rules ?? null; + const createdDates = docs + .map(docCreatedDate) + .filter((date) => date !== null) + .sort(); + + const articles = []; + for (const doc of docs) { + const windows = []; + if (isInWindow(doc, recent)) { + windows.push('recent'); + } + if (isInWindow(doc, comparison)) { + windows.push('comparison'); + } + articles.push({ ...articleSummary(doc, rules, args.includeAbstracts === true), windows }); + } + articles.sort( + (a, b) => b.createdDate.localeCompare(a.createdDate) || String(a.id).localeCompare(String(b.id)), + ); + + return { + product: args.product, + asOf: args.asOf, + taxonomy: + rules === null ? null : { mode: taxonomy.mode, name: taxonomy.name, themes: rules.map((rule) => rule.theme) }, + recent: summarizeWindow(docs, recent, rules), + comparison: summarizeWindow(docs, comparison, rules), + allTime: { + articleCount: numFound, + fetchedArticles: docs.length, + truncated, + firstCreated: createdDates[0] ?? null, + lastCreated: createdDates[createdDates.length - 1] ?? null, + }, + articles, + source: KB_ENDPOINT, + fetchedAt: new Date().toISOString(), + }; +} + +function formatPercent(value) { + return `${(value * 100).toFixed(1)}%`; +} + +function formatCountList(counts, limit = 10) { + if (counts.length === 0) { + return '- none'; + } + return counts + .slice(0, limit) + .map((item) => `- ${item.value}: ${item.count}`) + .join('\n'); +} + +function formatThemeCases(item) { + const base = `${item.caseCount365} linked support cases (365d, summed)`; + if (item.caseCount365 > 0 && item.topArticleShare > 0.5) { + return `${base}; one article supplies ${formatPercent(item.topArticleShare)}`; + } + return base; +} + +function formatThemeList(themes) { + if (themes.length === 0) { + return '- none'; + } + return themes + .map( + (item) => + `- ${item.theme}: ${formatArticleCount(item.count)}, ${formatPercent(item.share)} of window, ${item.perDay.toFixed(2)} per day, ${formatThemeCases(item)}`, + ) + .join('\n'); +} + +function formatArticles(articles, limit = 15) { + if (articles.length === 0) { + return '- none'; + } + return articles + .slice(0, limit) + .map( + (article) => + `- ${article.createdDate} — [${article.title}](${article.url}) (linked support cases, 365d: ${article.caseCount365})`, + ) + .join('\n'); +} + +function formatArticleCount(count) { + return `${count} ${count === 1 ? 'article' : 'articles'}`; +} + +function formatAllTime(report) { + if (report.allTime.truncated) { + return `≥ ${report.allTime.articleCount} (capped; fetched ${report.allTime.fetchedArticles})`; + } + return `${report.allTime.articleCount} (${report.allTime.firstCreated ?? 'unknown'} to ${report.allTime.lastCreated ?? 'unknown'})`; +} + +const CASE_TOTAL_CAVEAT = + '- Linked support-case totals are article-level caseCount_365 values summed across matching articles; they are not unique cases and may include multi-product articles.'; + +function caveatsFor(report) { + const mode = report.taxonomy?.mode ?? 'none'; + const lines = [ + '- Article count is not unique support-case volume.', + '- The public KB payload does not confirm article authorship.', + ]; + + if (mode === 'none') { + lines.push('- Theme signals are absent because no taxonomy was applied.'); + } else { + lines.push( + mode === 'custom' + ? '- Theme signals come from the supplied keyword taxonomy, not from official Red Hat categories.' + : '- Theme signals are heuristic keyword clusters, not official Red Hat categories.', + ); + lines.push(CASE_TOTAL_CAVEAT); + } + + lines.push(formatWindowCaveat(report)); + return lines.join('\n'); +} + +function formatWindowCaveat(report) { + const recent = report.recent.window; + const comparison = report.comparison.window; + const nested = recent.start >= comparison.start && recent.end <= comparison.end; + if (nested) { + return '- The recent window is a subset of the comparison window; use per-day rates and shares rather than raw counts alone.'; + } + return '- The recent and comparison windows are not nested; use per-day rates and window shares rather than raw counts alone.'; +} + +export function renderMarkdown(report) { + const recent = report.recent; + const comparison = report.comparison; + const recentArticles = report.articles.filter((article) => article.windows.includes('recent')); + const topRecentArticles = [...recentArticles] + .sort((a, b) => b.caseCount365 - a.caseCount365 || b.createdDate.localeCompare(a.createdDate)) + .slice(0, 15); + + const articleHeading = + recentArticles.length > 0 + ? `## Most-linked recent articles (top ${topRecentArticles.length} of ${recentArticles.length} by linked support cases)` + : '## Most-linked recent articles'; + const taxonomyLine = + report.taxonomy === null + ? '- Taxonomy: none (group the article list yourself)' + : `- Taxonomy: ${path.basename(report.taxonomy.name)} (${report.taxonomy.themes.length} clusters)`; + const unmatchedLines = + report.taxonomy === null + ? '' + : `\n- Recent unmatched by taxonomy: ${recent.unmatched}\n- Comparison unmatched by taxonomy: ${comparison.unmatched}`; + const taxonomyMode = report.taxonomy?.mode ?? 'none'; + const themeLabel = taxonomyMode === 'custom' ? 'Theme signals' : 'Heuristic theme signals'; + const themeSections = + taxonomyMode === 'none' + ? `## Theme signals + +No taxonomy applied. Group the article list below yourself, using one taxonomy across both windows, and say which taxonomy you used. + +` + : `## ${themeLabel} — recent + +${formatThemeList(recent.themes)} + +## ${themeLabel} — comparison + +${formatThemeList(comparison.themes)} + +`; + + return `# RHDH KB theme analysis + +- Product: ${report.product} +- Run date: ${report.asOf} +- Recent window: ${recent.window.start} to ${recent.window.end} (${recent.window.days} days) +- Comparison window: ${comparison.window.start} to ${comparison.window.end} (${comparison.window.days} days) +- All-time articles: ${formatAllTime(report)} +${taxonomyLine} + +## Volume + +- Recent articles: ${recent.articleCount} +- Comparison articles: ${comparison.articleCount} +- Recent daily rate: ${(recent.articleCount / recent.window.days).toFixed(2)} +- Comparison daily rate: ${(comparison.articleCount / comparison.window.days).toFixed(2)}${unmatchedLines} + +## Recent categories + +${formatCountList(recent.categories)} + +## Comparison categories + +${formatCountList(comparison.categories)} + +## Recent tags + +${formatCountList(recent.tags)} + +## Comparison tags + +${formatCountList(comparison.tags)} + +## Recent components + +${formatCountList(recent.components)} + +## Comparison components + +${formatCountList(comparison.components)} + +${themeSections}${articleHeading} + +${formatArticles(topRecentArticles)} + +## Caveats + +${caveatsFor(report)} +`; +} + +async function main() { + try { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + process.stdout.write(`${usage()}\n`); + return; + } + + const fetchResult = await fetchKbDocs(args.product); + const report = summarize(fetchResult, args); + if (args.format === 'json') { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + process.stdout.write(`${renderMarkdown(report)}\n`); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`Red Hat KB analysis failed: ${message}\n`); + process.exitCode = 1; + } +} + +const isDirectRun = + process.argv[1] !== undefined && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isDirectRun) { + await main(); +} diff --git a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs new file mode 100644 index 00000000..3402e672 --- /dev/null +++ b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs @@ -0,0 +1,545 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { + ABSTRACT_LIMIT, + DEFAULT_COMPARISON_DAYS, + DEFAULT_RECENT_DAYS, + buildQuery, + fetchKbDocs, + isInWindow, + parseArgs, + renderMarkdown, + resolveWindows, + summarize, + themeHints, +} from '../scripts/analyze-kb.mjs'; + +describe('parseArgs defaults', () => { + it('uses RHDH, Markdown, 90 days, and 365 days', () => { + const args = parseArgs(['--as-of', '2026-10-30']); + assert.equal(args.product, 'Red Hat Developer Hub'); + assert.equal(args.format, 'markdown'); + assert.equal(args.recentDays, undefined); + assert.equal(args.comparisonDays, undefined); + assert.equal(DEFAULT_RECENT_DAYS, 90); + assert.equal(DEFAULT_COMPARISON_DAYS, 365); + }); + + it('accepts JSON output', () => { + assert.equal(parseArgs(['--as-of', '2026-10-30', '--format', 'json']).format, 'json'); + }); +}); + +describe('resolveWindows', () => { + it('creates inclusive rolling windows', () => { + const args = parseArgs(['--as-of', '2026-10-30']); + const windows = resolveWindows(args); + assert.deepEqual(windows.recent, { + start: '2026-08-02', + end: '2026-10-30', + days: 90, + }); + assert.deepEqual(windows.comparison, { + start: '2025-10-31', + end: '2026-10-30', + days: 365, + }); + }); + + it('supports custom day lengths', () => { + const args = parseArgs([ + '--as-of', + '2026-10-30', + '--recent-days', + '30', + '--comparison-days', + '180', + ]); + const windows = resolveWindows(args); + assert.equal(windows.recent.start, '2026-10-01'); + assert.equal(windows.recent.days, 30); + assert.equal(windows.comparison.start, '2026-05-04'); + assert.equal(windows.comparison.days, 180); + }); + + it('supports explicit calendar windows', () => { + const args = parseArgs([ + '--as-of', + '2026-10-30', + '--recent-since', + '2026-07-01', + '--recent-until', + '2026-09-30', + '--comparison-since', + '2025-01-01', + '--comparison-until', + '2025-12-31', + ]); + const windows = resolveWindows(args); + assert.deepEqual(windows.recent, { + start: '2026-07-01', + end: '2026-09-30', + days: 92, + }); + assert.deepEqual(windows.comparison, { + start: '2025-01-01', + end: '2025-12-31', + days: 365, + }); + }); + + it('rejects conflicting day and date options', () => { + assert.throws( + () => parseArgs(['--as-of', '2026-10-30', '--recent-days', '30', '--recent-since', '2026-07-01']), + /cannot be combined/, + ); + assert.throws( + () => parseArgs(['--as-of', '2026-10-30', '--comparison-days', '365', '--comparison-since', '2025-01-01']), + /cannot be combined/, + ); + }); + + it('requires a start when an end is supplied', () => { + assert.throws( + () => parseArgs(['--as-of', '2026-10-30', '--recent-until', '2026-09-30']), + /requires/, + ); + }); +}); + +describe('buildQuery', () => { + it('encodes the product filter and Solr local params', () => { + const url = buildQuery('Red Hat Developer Hub', 20, 100); + assert.equal(url.origin, 'https://access.redhat.com'); + assert.equal(url.pathname, '/hydra/rest/search/platform/kbase'); + assert.equal(url.searchParams.get('start'), '20'); + assert.equal(url.searchParams.get('rows'), '100'); + assert.ok(url.searchParams.getAll('fq').includes('-id:Other')); + assert.ok(url.searchParams.getAll('fq').includes('documentKind:(Solution Article)')); + assert.ok( + url.searchParams + .getAll('fq') + .includes('{!tag=productFilter}standard_product:"Red Hat Developer Hub"'), + ); + assert.ok(url.searchParams.getAll('facet.field').includes('{!ex=productFilter}standard_product')); + }); +}); + +describe('analysis', () => { + const docs = [ + { + id: '1', + title: 'GitHub login fails after upgrade', + view_uri: 'https://access.redhat.com/solutions/1', + createdDate: '2026-10-30T10:00:00Z', + category: ['Troubleshoot'], + component: ['authentication'], + tag: ['authentication', 'rhdh'], + caseCount_365: 7, + }, + { + id: '2', + title: 'Dynamic plugin installation fails', + view_uri: 'https://access.redhat.com/solutions/2', + createdDate: '2026-08-02T10:00:00Z', + category: ['Configure'], + component: ['rhdh'], + tag: ['rhdh_configuration'], + caseCount_365: 3, + }, + { + id: '3', + title: 'Older Keycloak login issue', + view_uri: 'https://access.redhat.com/solutions/3', + createdDate: '2025-10-31T10:00:00Z', + category: ['Troubleshoot'], + component: ['keycloak'], + tag: ['authentication'], + caseCount_365: 2, + }, + ]; + + it('includes both window boundaries', () => { + const args = parseArgs(['--as-of', '2026-10-30']); + const windows = resolveWindows(args); + assert.equal(isInWindow(docs[0], windows.recent), true); + assert.equal(isInWindow(docs[1], windows.recent), true); + assert.equal(isInWindow(docs[2], windows.comparison), true); + }); + + it('emits non-exclusive theme hints and per-theme case totals', () => { + assert.ok(themeHints(docs[0]).includes('upgrade-plugin-reliability')); + assert.ok(themeHints(docs[0]).includes('authentication-identity')); + assert.ok(themeHints(docs[1]).includes('upgrade-plugin-reliability')); + + const report = summarize({ docs, numFound: docs.length, truncated: false }, parseArgs(['--as-of', '2026-10-30'])); + const upgrade = report.recent.themes.find((theme) => theme.theme === 'upgrade-plugin-reliability'); + assert.equal(upgrade.count, 2); + assert.equal(upgrade.caseCount365, 10); + }); + + it('does not match unrelated words inside theme stems', () => { + assert.deepEqual(themeHints({ title: 'How to change the RHDH logo' }), ['configuration-ui']); + assert.deepEqual(themeHints({ title: 'Authoring TechDocs' }), []); + assert.deepEqual(themeHints({ title: 'Accessibility statement for RHDH' }), []); + }); + + it('matches plural and derived theme forms', () => { + assert.ok(themeHints({ title: 'Upgrades fail' }).includes('upgrade-plugin-reliability')); + assert.ok(themeHints({ title: 'PostgreSQL 16' }).includes('catalog-database-scale')); + assert.ok(themeHints({ title: 'modules' }).includes('upgrade-plugin-reliability')); + assert.ok(themeHints({ title: 'operators' }).includes('operator-openshift-deployment')); + assert.ok(themeHints({ title: 'replicas' }).includes('catalog-database-scale')); + assert.ok(themeHints({ title: 'namespaces' }).includes('operator-openshift-deployment')); + assert.ok(themeHints({ title: 'configs' }).includes('configuration-ui')); + assert.ok(themeHints({ title: 'scales' }).includes('catalog-database-scale')); + assert.ok(themeHints({ title: 'autoscaling' }).includes('catalog-database-scale')); + + const incompatible = themeHints({ title: 'incompatible plugin' }); + assert.ok(incompatible.includes('upgrade-plugin-reliability')); + assert.ok(incompatible.includes('support-version-clarity')); + }); + + it('accepts scalar metadata fields without counting characters', () => { + const doc = { + id: 'scalar', + title: 'GitHub login fails after upgrade', + createdDate: '2026-10-30T10:00:00Z', + category: 'Troubleshoot', + component: 'authentication', + tag: 'rhdh', + caseCount_365: 4, + }; + const report = summarize({ docs: [doc], numFound: 1, truncated: false }, parseArgs(['--as-of', '2026-10-30'])); + assert.deepEqual(report.recent.categories, [{ value: 'Troubleshoot', count: 1 }]); + assert.deepEqual(report.recent.components, [{ value: 'authentication', count: 1 }]); + assert.deepEqual(report.recent.tags, [{ value: 'rhdh', count: 1 }]); + assert.match(renderMarkdown(report), /^- Recent articles: 1$/m); + assert.ok(renderMarkdown(report).includes('1 article,')); + }); + + it('keeps the first reported numFound when a later page omits it', async () => { + const originalFetch = globalThis.fetch; + const pages = [ + { response: { numFound: 3, docs: [{ id: '1', title: 'One' }] } }, + { response: { docs: [{ id: '2', title: 'Two' }] } }, + { response: { docs: [] } }, + ]; + let call = 0; + globalThis.fetch = async () => ({ + ok: true, + json: async () => pages[Math.min(call++, pages.length - 1)], + }); + try { + const result = await fetchKbDocs('Red Hat Developer Hub'); + assert.equal(result.docs.length, 2); + assert.equal(result.numFound, 3); + assert.equal(result.truncated, true); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it('falls back to fetched count when the API omits numFound', async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => ({ + ok: true, + json: async () => ({ response: { docs: [{ id: '1', title: 'Example' }] } }), + }); + try { + const result = await fetchKbDocs('Red Hat Developer Hub'); + assert.equal(result.numFound, 1); + assert.equal(result.truncated, false); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it('reports numFound as a floor when results are capped', () => { + const args = parseArgs(['--as-of', '2026-10-30']); + const report = summarize({ docs: docs.slice(0, 1), numFound: 2, truncated: true }, args); + assert.equal(report.allTime.articleCount, 2); + assert.equal(report.allTime.fetchedArticles, 1); + assert.equal(report.allTime.truncated, true); + const markdown = renderMarkdown(report); + assert.ok(markdown.includes('≥ 2 (capped; fetched 1)')); + assert.ok(!markdown.includes('unknown to unknown')); + }); + + it('summarizes recent and comparison windows', () => { + const args = parseArgs(['--as-of', '2026-10-30']); + const report = summarize({ docs, numFound: docs.length, truncated: false }, args); + assert.equal(report.recent.articleCount, 2); + assert.equal(report.comparison.articleCount, 3); + assert.equal(report.allTime.articleCount, 3); + assert.equal(report.allTime.firstCreated, '2025-10-31'); + assert.equal(report.allTime.lastCreated, '2026-10-30'); + assert.ok(renderMarkdown(report).includes('All-time articles: 3 (')); + assert.deepEqual( + report.recent.categories.map((item) => item.value).sort(), + ['Configure', 'Troubleshoot'], + ); + }); + + it('labels the high-signal article list when it is truncated', () => { + const manyDocs = Array.from({ length: 16 }, (_, index) => ({ + id: String(index + 1), + title: `Upgrades fail ${index + 1}`, + createdDate: '2026-10-30T10:00:00Z', + caseCount_365: index, + })); + const args = parseArgs(['--as-of', '2026-10-30']); + const markdown = renderMarkdown( + summarize({ docs: manyDocs, numFound: manyDocs.length, truncated: false }, args), + ); + assert.ok( + markdown.includes( + '## Most-linked recent articles (top 15 of 16 by linked support cases)', + ), + ); + }); + + it('renders the core Markdown sections and linked citations', () => { + const args = parseArgs(['--as-of', '2026-10-30']); + const markdown = renderMarkdown(summarize({ docs, numFound: docs.length, truncated: false }, args)); + assert.ok(markdown.includes('# RHDH KB theme analysis')); + assert.ok(markdown.includes('- Recent window: 2026-08-02 to 2026-10-30 (90 days)')); + assert.ok(markdown.includes('- Comparison window: 2025-10-31 to 2026-10-30 (365 days)')); + assert.ok(markdown.includes('## Heuristic theme signals — recent')); + assert.ok( + markdown.includes('[GitHub login fails after upgrade](https://access.redhat.com/solutions/1)'), + ); + assert.ok(!markdown.includes('Product improvement:')); + assert.ok(!markdown.includes('Documentation improvement:')); + assert.ok( + markdown.includes( + '## Most-linked recent articles (top 2 of 2 by linked support cases)', + ), + ); + assert.ok(markdown.includes('## Caveats')); + assert.ok(markdown.includes('The recent window is a subset of the comparison window')); + }); + + it('uses a non-nested caveat for disjoint custom windows', () => { + const args = parseArgs([ + '--as-of', + '2026-10-30', + '--recent-since', + '2026-07-01', + '--recent-until', + '2026-08-01', + '--comparison-since', + '2025-01-01', + '--comparison-until', + '2025-12-31', + ]); + const markdown = renderMarkdown(summarize({ docs: [], numFound: 0, truncated: false }, args)); + assert.ok(markdown.includes('The recent and comparison windows are not nested')); + assert.ok(markdown.includes('- none')); + }); +}); + +describe('taxonomy', () => { + const docs = [ + { id: '1', title: 'TechDocs build fails', createdDate: '2026-10-30T10:00:00Z', caseCount_365: 3 }, + { id: '2', title: 'Upgrade fails', createdDate: '2026-10-28T10:00:00Z', caseCount_365: 1 }, + { id: '3', title: 'Old TechDocs question', createdDate: '2025-11-01T10:00:00Z', caseCount_365: 2 }, + ]; + const argsFor = (...extra) => parseArgs(['--as-of', '2026-10-30', ...extra]); + const reportFor = (...extra) => + summarize({ docs, numFound: docs.length, truncated: false }, argsFor(...extra)); + + function writeTaxonomy(rules) { + const directory = mkdtempSync(join(tmpdir(), 'rhdh-kb-taxonomy-')); + const file = join(directory, 'taxonomy.json'); + writeFileSync(file, JSON.stringify(rules)); + return file; + } + + it('uses the built-in taxonomy by default', () => { + const report = reportFor(); + assert.equal(report.taxonomy.mode, 'built-in'); + assert.equal(report.taxonomy.themes.length, 8); + assert.ok(report.recent.themes.length > 0); + assert.ok(renderMarkdown(report).includes('## Heuristic theme signals — recent')); + }); + + it('skips clustering with --themes none', () => { + const report = reportFor('--themes', 'none'); + assert.equal(report.taxonomy, null); + assert.equal(report.recent.themes, null); + assert.equal(report.comparison.themes, null); + assert.equal(report.articles[0].themes, null); + + const markdown = renderMarkdown(report); + assert.ok(markdown.includes('- Taxonomy: none (group the article list yourself)')); + assert.ok(!markdown.includes('Heuristic theme signals')); + assert.ok(markdown.includes('No taxonomy applied')); + }); + + it('applies a custom taxonomy file to both windows', () => { + const file = writeTaxonomy([{ theme: 'techdocs', pattern: '\\btechdocs\\b' }]); + const report = reportFor('--themes', file); + + assert.equal(report.taxonomy.mode, 'custom'); + assert.deepEqual(report.taxonomy.themes, ['techdocs']); + assert.deepEqual( + report.recent.themes.map((item) => [item.theme, item.count]), + [['techdocs', 1]], + ); + assert.deepEqual( + report.comparison.themes.map((item) => [item.theme, item.count]), + [['techdocs', 2]], + ); + + const markdown = renderMarkdown(report); + assert.ok(markdown.includes('- Taxonomy: ' + basename(file))); + assert.ok(markdown.includes('## Theme signals — recent')); + assert.ok(!markdown.includes('Heuristic theme signals')); + }); + + it('rejects a taxonomy that is not a JSON array', () => { + const file = writeTaxonomy({ theme: 'x' }); + assert.throws(() => parseArgs(['--themes', file]), /must be a non-empty JSON array/); + }); + + it('rejects a taxonomy entry with an unusable pattern', () => { + const file = writeTaxonomy([{ theme: 'broken', pattern: '(' }]); + assert.throws(() => parseArgs(['--themes', file]), /not a valid regular expression/); + }); + + it('rejects a taxonomy with duplicate theme names', () => { + const file = writeTaxonomy([ + { theme: 'same', pattern: 'a' }, + { theme: 'same', pattern: 'b' }, + ]); + assert.throws(() => parseArgs(['--themes', file]), /repeats the theme/); + }); + + it('reports a missing taxonomy file', () => { + assert.throws( + () => parseArgs(['--themes', join(tmpdir(), 'no-such-taxonomy.json')]), + /Cannot read taxonomy file/, + ); + }); + + it('emits one article list carrying window membership', () => { + const report = reportFor(); + assert.equal('articles' in report.recent, false); + assert.equal('articles' in report.comparison, false); + assert.equal(report.articles.length, 3); + + const byId = new Map(report.articles.map((article) => [article.id, article.windows])); + assert.deepEqual(byId.get('1'), ['recent', 'comparison']); + assert.deepEqual(byId.get('2'), ['recent', 'comparison']); + assert.deepEqual(byId.get('3'), ['comparison']); + }); + + it('includes truncated abstracts only when asked', () => { + const long = 'x'.repeat(ABSTRACT_LIMIT + 200); + const withIssue = [{ id: '1', title: 't', createdDate: '2026-10-30', issue: [long] }]; + + const without = summarize( + { docs: withIssue, numFound: 1, truncated: false }, + argsFor(), + ); + assert.equal('abstract' in without.articles[0], false); + + const included = summarize( + { docs: withIssue, numFound: 1, truncated: false }, + argsFor('--include-abstracts'), + ); + assert.equal(included.articles[0].abstract.length, ABSTRACT_LIMIT + 1); + assert.ok(included.articles[0].abstract.endsWith('\u2026')); + }); +}); + +describe('report signals', () => { + const args = parseArgs(['--as-of', '2026-10-30']); + + function writeTaxonomy(rules) { + const directory = mkdtempSync(join(tmpdir(), 'rhdh-kb-signals-')); + const file = join(directory, 'taxonomy.json'); + writeFileSync(file, JSON.stringify(rules)); + return file; + } + + it('keeps out-of-window articles with empty window flags', () => { + const docs = [ + { id: 'old', title: 'Legacy question', createdDate: '2024-05-01T10:00:00Z' }, + { id: 'new', title: 'Upgrade fails', createdDate: '2026-10-30T10:00:00Z' }, + ]; + const report = summarize({ docs, numFound: 2, truncated: false }, args); + + const legacy = report.articles.find((article) => article.id === 'old'); + assert.deepEqual(legacy.windows, []); + assert.equal(report.articles.length, 2); + assert.equal(report.allTime.articleCount, 2); + assert.equal(report.recent.articleCount, 1); + }); + + it('counts articles the taxonomy leaves unmatched', () => { + const docs = [ + { id: '1', title: 'TechDocs build fails', createdDate: '2026-10-30T10:00:00Z' }, + { id: '2', title: 'Upgrade fails', createdDate: '2026-10-28T10:00:00Z' }, + { id: '3', title: 'Old TechDocs question', createdDate: '2025-11-01T10:00:00Z' }, + ]; + const file = writeTaxonomy([{ theme: 'upgrades', pattern: '\\bupgrad\\w*' }]); + const report = summarize( + { docs, numFound: docs.length, truncated: false }, + parseArgs(['--as-of', '2026-10-30', '--themes', file]), + ); + + assert.equal(report.recent.unmatched, 1); + assert.equal(report.comparison.unmatched, 2); + const markdown = renderMarkdown(report); + assert.ok(markdown.includes('- Recent unmatched by taxonomy: 1')); + assert.ok(markdown.includes('- Comparison unmatched by taxonomy: 2')); + }); + + it('reports no unmatched section when clustering is off', () => { + const report = summarize( + { docs: [{ id: '1', title: 'x', createdDate: '2026-10-30' }], numFound: 1, truncated: false }, + parseArgs(['--as-of', '2026-10-30', '--themes', 'none']), + ); + assert.equal(report.recent.unmatched, null); + assert.ok(!renderMarkdown(report).includes('unmatched by taxonomy')); + }); + + it('flags a theme whose case total comes from one article', () => { + const docs = [ + { id: 'spike', title: 'Upgrades hit a CVE', createdDate: '2026-10-01T10:00:00Z', caseCount_365: 90 }, + { id: 'small', title: 'Upgrade question', createdDate: '2026-10-02T10:00:00Z', caseCount_365: 1 }, + ]; + const file = writeTaxonomy([{ theme: 'upgrades', pattern: '\\bupgrad\\w*' }]); + const report = summarize( + { docs, numFound: 2, truncated: false }, + parseArgs(['--as-of', '2026-10-30', '--themes', file]), + ); + + const theme = report.recent.themes[0]; + assert.equal(theme.caseCount365, 91); + assert.equal(theme.maxArticleCases, 90); + assert.ok(theme.topArticleShare > 0.9); + assert.ok(renderMarkdown(report).includes('one article supplies')); + }); + + it('does not flag a theme with a spread of case counts', () => { + const docs = [ + { id: 'a', title: 'Upgrade one', createdDate: '2026-10-01T10:00:00Z', caseCount_365: 3 }, + { id: 'b', title: 'Upgrade two', createdDate: '2026-10-02T10:00:00Z', caseCount_365: 3 }, + ]; + const file = writeTaxonomy([{ theme: 'upgrades', pattern: '\\bupgrad\\w*' }]); + const report = summarize( + { docs, numFound: 2, truncated: false }, + parseArgs(['--as-of', '2026-10-30', '--themes', file]), + ); + + assert.ok(report.recent.themes[0].topArticleShare <= 0.5); + assert.ok(!renderMarkdown(report).includes('one article supplies')); + }); +}); diff --git a/tests/unit/test_plugin_manifest.py b/tests/unit/test_plugin_manifest.py index c36617b2..e01c1698 100644 --- a/tests/unit/test_plugin_manifest.py +++ b/tests/unit/test_plugin_manifest.py @@ -79,6 +79,21 @@ def test_third_party_installs_as_one_selectable_group_per_source(): assert by_name["vendor-two"]["skills"] == ["./skills/third-party/vendor-two/three"] +def test_every_non_reference_skill_is_installable_from_its_category_group(): + """No promoted, directly installable skill is stranded outside the manifest.""" + generator = load_generator() + catalog = generator.json.loads(generator.CATALOG_PATH.read_text(encoding="utf-8")) + manifest = generator.build_manifest(catalog) + listed = {path for plugin in manifest["plugins"] for path in plugin["skills"]} + + for entry in catalog["skills"]: + if entry["category"] in generator.HIDDEN_INSTALL_CATEGORIES: + continue + assert generator._skill_path(entry) in listed, ( + f"{entry['name']} is not reachable from an installable category group" + ) + + def test_reference_is_not_an_installer_group(): """Reference stays editorial; the installer must not offer a Reference group.""" generator = load_generator() From 036cb2b2b987e857102b1c3860e69b337727d759 Mon Sep 17 00:00:00 2001 From: mattreid Date: Fri, 6 Nov 2026 11:04:51 -0500 Subject: [PATCH 2/6] feat(release): measure KB demand by attributed cases The first cut answered "what did support publish recently" while reading as "what are customers hitting". Of 171 RHDH articles, the 16 created in the last 90 days carry 10 linked cases, and one 9-product CVE article holds 83 of the 187 corpus cases. Both distortions are now explicit. - demand view: whole corpus weighted by caseCount_365, not windowed by createdDate, since that count cannot be re-windowed below a year - casesRaw, casesAttributed (divided across tagged products), and casesDedicated (single-product only) per theme and per article - corpus concentration: largest article named, share of raw and attributed weight, warning above 25% - theme movement classified against a baseline that excludes the recent window, with a five-article guard that prints insufficient rather than a ratio - KCS state mix and refresh candidates for stale high-touch articles - component folding with --component-aliases for real synonyms - emergent terms mined from titles and tags, with ubiquitous-term and singleton floors - schemaVersion 2; existing JSON keys and report sections preserved - 53 script tests (was 34) Assisted-By: Codex --- .../release/rhdh-kb-theme-analysis/SKILL.md | 18 +- .../rhdh-kb-theme-analysis/agents/openai.yaml | 2 +- .../rhdh-kb-theme-analysis/references/api.md | 47 +- .../references/interpretation.md | 59 +- .../references/taxonomy.md | 4 +- .../scripts/analyze-kb.mjs | 657 +++++++++++++++++- .../tests/analyze-kb.test.mjs | 335 +++++++++ 7 files changed, 1075 insertions(+), 47 deletions(-) diff --git a/skills/release/rhdh-kb-theme-analysis/SKILL.md b/skills/release/rhdh-kb-theme-analysis/SKILL.md index de145116..c935a8ab 100644 --- a/skills/release/rhdh-kb-theme-analysis/SKILL.md +++ b/skills/release/rhdh-kb-theme-analysis/SKILL.md @@ -47,6 +47,20 @@ Read `references/taxonomy.md` before writing a taxonomy file. Use `--format json` when the answer needs every article rather than the top 15, and add `--include-abstracts` when the grouping will be done from article text. +`component` is unnormalized free text. The report folds case, whitespace, and +dashes by default; pass `--component-aliases PATH` with a JSON object of +`{"from": "to"}` pairs when a question needs real synonyms merged, and say that +you did. + +## Authoring and demand + +The report answers two questions. The `recent` and `comparison` windows filter on +`createdDate` and measure what support published. The `demand` view ignores +`createdDate` and weights the whole corpus by `caseCount_365`. Lead with demand +when the question is about customers; lead with authoring when it is about +publishing cadence or taxonomy coverage. Case weights are annual — a quarterly +case-volume answer is not derivable from this data. + ## Boundary with neighbouring skills - Use this skill for public KB article themes and release-planning signals. @@ -77,7 +91,9 @@ Complete when the answer names: - the product and run date; - both window boundaries; - recent and comparison article counts; -- the strongest theme signals; +- the strongest theme signals, as `movement` classes rather than adjectives; +- the demand view, its attributed case total, and any concentration warning; +- the stale, high-touch articles worth refreshing; - linked citations for every article named in the answer; - a final “Product and documentation improvement areas” section that connects recent findings to candidate product and documentation work; - the caveats from the interpretation reference. diff --git a/skills/release/rhdh-kb-theme-analysis/agents/openai.yaml b/skills/release/rhdh-kb-theme-analysis/agents/openai.yaml index 1c7d3990..e056ad21 100644 --- a/skills/release/rhdh-kb-theme-analysis/agents/openai.yaml +++ b/skills/release/rhdh-kb-theme-analysis/agents/openai.yaml @@ -1,3 +1,3 @@ interface: display_name: "RHDH KB Theme Analysis" - short_description: "Compare RHDH KB article themes across rolling windows" + short_description: "Compare RHDH KB themes and case-weighted demand" diff --git a/skills/release/rhdh-kb-theme-analysis/references/api.md b/skills/release/rhdh-kb-theme-analysis/references/api.md index 7e2d499d..049cbfc0 100644 --- a/skills/release/rhdh-kb-theme-analysis/references/api.md +++ b/skills/release/rhdh-kb-theme-analysis/references/api.md @@ -35,24 +35,41 @@ the 10,000-result safety cap. If the cap truncates the result, the report prints an “at least” count rather than presenting the fetched count as the total. -## Useful fields +## Fields the analysis reads Treat these fields as optional because Red Hat can change the public response. The script accepts either string or array shapes for `category`, `component`, -and `tag`: - -- `id` -- `title` -- `view_uri` -- `createdDate` -- `lastModifiedDate` -- `category` -- `component` -- `tag` -- `standard_product` -- `issue` -- `caseCount` -- `caseCount_365` +`tag`, and `standard_product`: + +- `id`, `title`, `view_uri` +- `createdDate` — publication date. The authoring windows filter on its + `YYYY-MM-DD` prefix. +- `lastModifiedDate` — last editorial change. Drives the refresh-candidate + signal. +- `kcsState`, with `publication_state` as a fallback — `verified`, `unverified`, + or `wip`. +- `category`, `component`, `tag` +- `standard_product` — every product the article is tagged to. More than one + entry means the article is shared, and its case weight is divided by that + count. +- `issue` — the abstract. The taxonomy matches against it, and it is mined for + emergent terms. +- `caseCount` — lifetime linked cases. +- `caseCount_365` — linked cases in a fixed trailing 365 days. + +## Case-count semantics + +`caseCount_365` is published by Red Hat and cannot be re-windowed. A query for a +90-day range still returns the trailing-year count, so no case-volume statement +shorter than a year is derivable from this API. The demand view in the report is +annual for that reason, whatever `--as-of` says. + +## Fields present but unused + +`kcsRateUp` and `kcsRateDown` carry article usefulness votes, and `sbr`, +`internalTags`, and `issueTag` carry extra classification. They are sparse in +the RHDH corpus — fewer than a dozen articles carry any vote — so the report +does not read them. Revisit if the vote coverage improves. The analysis filters on the `YYYY-MM-DD` prefix of `createdDate`. Theme matching reads `title`, `issue` (the abstract), `category`, and `component`, so diff --git a/skills/release/rhdh-kb-theme-analysis/references/interpretation.md b/skills/release/rhdh-kb-theme-analysis/references/interpretation.md index 75467241..131c1526 100644 --- a/skills/release/rhdh-kb-theme-analysis/references/interpretation.md +++ b/skills/release/rhdh-kb-theme-analysis/references/interpretation.md @@ -1,5 +1,19 @@ # Interpreting RHDH KB theme analysis +## Authoring versus demand + +The report answers two different questions and never mixes them. + +- **Authoring** is the `recent` and `comparison` views. They filter on + `createdDate`, so they measure how many articles support published in a + window, not how much pain customers reported. +- **Demand** is the `demand` view. It ignores `createdDate` and weights the whole + corpus by `caseCount_365`. + +Read the authoring view for publishing cadence and taxonomy coverage. Read the +demand view for what customers are hitting. A quarterly answer is an authoring +answer; the demand view is always annual. + ## Time windows The default recent window is the trailing 90 days. The default comparison @@ -53,13 +67,40 @@ For each strong theme, report: - the likely product or roadmap impact; - the confidence level. -Look for themes that are: +## Case-weight arithmetic + +Three case numbers appear per theme and per demand article, and they are not +interchangeable: + +- `casesRaw` — the plain sum of `caseCount_365`; +- `casesAttributed` — the sum after each article is divided by the number of + products it is tagged to. Use this one; +- `casesDedicated` — the sum over single-product articles only. + +A generic article tagged to nine products used to read as nine times the demand +of a product-specific one. Attribution removes that distortion without throwing +the shared article away. + +`demand.concentration` names the largest article and its share of raw and +attributed weight. When `warned` is true, at least a quarter of the corpus case +weight is one article: say so before quoting any total, and never present that +article's reach as demand for every theme it touches. + +Do not classify growth by eye. `movement.themes` already carries `class`, +`ratio`, and `deltaShare` for every theme, computed against the baseline window: + +- `new` — no baseline articles, at least three recent ones; +- `growing` — recent rate at least 1.5× the baseline rate; +- `declining` — recent rate at most 0.67× the baseline rate; +- `stable` — anything between; +- `insufficient` — fewer than five articles on either side, or no baseline. + +The baseline is *not* the comparison window. It is the comparison window minus +the recent window, so a theme is never compared against a period that contains +itself. `movement.baseline.window` carries the exact dates; quote them. -- new in the recent window, -- growing relative to the annual rate, -- persistent across both windows, -- high-touch by `caseCount_365`, -- concentrated around a release or plugin family. +Report an `insufficient` theme as unclassified. Never print a percentage or a +ratio for one, and never upgrade a 2-against-0 result into "emerging". ## Product and documentation recommendations @@ -108,3 +149,9 @@ State these limitations in the final answer: - Theme clusters are heuristic. - A spike can reflect release activity, documentation gaps, support process changes, or product regressions; the KB alone does not prove causation. +- `caseCount_365` is a fixed trailing-year count. Nothing shorter than a year + can be stated about case volume. +- Attributed cases are an estimate that assumes an article's cases split evenly + across its tagged products. +- Emergent terms are word frequencies, not themes. Raise one only when it maps + onto an article set you can cite. diff --git a/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md b/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md index 9b08d320..65d317a0 100644 --- a/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md +++ b/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md @@ -82,8 +82,10 @@ Apply the taxonomy once and read the numbers before trusting it: - Unmatched is small. The report prints `unmatched by taxonomy` per window; a large number means the taxonomy misses a real theme. - Case totals carry a concentration warning. When one article supplies more - than half of a cluster's `caseCount365`, the report says so; treat that total + than half of a cluster's raw case weight, the report says so; treat that total as one article, not as demand for the whole theme. +- Prefer `casesAttributed` over `casesRaw` when comparing clusters. Raw weight + rewards articles tagged to many products, and attribution removes that bias. The matcher reads the abstract as well as the title. A stem that only ever matches in long prose — `support`, `many`, `large`, bare `logs` — will pull in diff --git a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs index 5c389c2b..c320ee78 100644 --- a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs +++ b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs @@ -126,6 +126,195 @@ export function resolveTaxonomy(value) { return loadTaxonomy(value); } +export const SCHEMA_VERSION = 2; +export const STALE_AFTER_DAYS = 365; +export const DEMAND_TOP_ARTICLES = 20; +export const REFRESH_CANDIDATES = 15; +export const EMERGENT_TERMS = 10; +export const MIN_MOVEMENT_OBSERVATIONS = 5; +export const GROWING_RATIO = 1.5; +export const DECLINING_RATIO = 0.67; +export const CONCENTRATION_WARNING_SHARE = 0.25; +export const NEW_THEME_MIN_ARTICLES = 3; +export const MIN_TERM_ARTICLES = 3; +export const NEW_TERM_MIN_ARTICLES = 2; +export const RISING_TERM_RATIO = 2; +export const UBIQUITOUS_TERM_SHARE = 0.3; +export const UBIQUITOUS_TERM_MIN_ARTICLES = 10; + +const STOPWORDS = new Set( + `about above after again against also and any are because been before being below between both but can cannot could did does doing down during each few for from further had has have having here how into its itself just may might more most must not now off once only other out over own same should some such than that the their them then there these they this those through under until very was were what when where which while who whom why will with would your rhdh redhat even show shows showing shown use used uses using get gets getting set sets setting make makes making need needs needed way ways like want wants two three first second last`.split( + /\s+/, + ), +); + +export function normalizeComponentValue(value) { + return String(value).trim().replace(/[-_]+/g, ' ').replace(/\s+/g, ' ').toLowerCase(); +} + +export function loadComponentAliases(filePath) { + const aliases = new Map(); + if (filePath === undefined) { + return aliases; + } + + let parsed; + try { + parsed = JSON.parse(readFileSync(filePath, 'utf8')); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Cannot read component alias file ${filePath}: ${message}`); + } + + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error( + `Component alias file ${filePath} must be a JSON object of {"from": "to"} strings`, + ); + } + + for (const [from, to] of Object.entries(parsed)) { + if (typeof to !== 'string' || to.length === 0) { + throw new Error(`Component alias "${from}" in ${filePath} needs a non-empty string target`); + } + const key = normalizeComponentValue(from); + if (aliases.has(key) && aliases.get(key) !== to) { + throw new Error(`Component alias file ${filePath} maps "${key}" to two different targets`); + } + aliases.set(key, to); + } + + return aliases; +} + +export function productsOf(doc) { + const declared = asList(doc.standard_product); + if (declared.length > 0) { + return declared; + } + const fallback = asList(doc.product); + return fallback.length > 0 ? fallback : ['unspecified']; +} + +export function productCountOf(doc) { + return Math.max(1, productsOf(doc).length); +} + +export function casesRawOf(doc) { + const value = Number(doc.caseCount_365 ?? 0); + return Number.isFinite(value) && value > 0 ? value : 0; +} + +export function casesAttributedOf(doc) { + return casesRawOf(doc) / productCountOf(doc); +} + +export function roundTo(value, places) { + const factor = 10 ** places; + return Math.round(value * factor) / factor; +} + +export function lastModifiedOf(doc) { + const value = doc.lastModifiedDate; + if (typeof value !== 'string' || !ISO_DATE_RE.test(value.slice(0, 10))) { + return null; + } + return value.slice(0, 10); +} + +export function ageInDays(dateIso, asOf) { + if (dateIso === null) { + return null; + } + const then = new Date(`${dateIso}T00:00:00Z`).getTime(); + const now = new Date(`${asOf}T00:00:00Z`).getTime(); + if (Number.isNaN(then) || Number.isNaN(now)) { + return null; + } + return Math.round((now - then) / 86400000); +} + +export function docState(doc) { + if (typeof doc.kcsState === 'string' && doc.kcsState.length > 0) { + return doc.kcsState; + } + if (typeof doc.publication_state === 'string' && doc.publication_state.length > 0) { + return doc.publication_state; + } + return 'unknown'; +} + +function articleUrl(doc) { + return doc.view_uri ?? `https://access.redhat.com/solutions/${doc.id}`; +} + +export function tokensOf(doc) { + const text = [doc.title, ...asList(doc.tag)].join(' '); + const tokens = new Set(); + for (const token of text.toLowerCase().replace(/[^a-z0-9]+/g, ' ').split(' ')) { + if (token.length < 4 || /^\d+$/.test(token) || STOPWORDS.has(token)) { + continue; + } + tokens.add(token); + } + return tokens; +} + +function countComponents(docs, aliases = new Map()) { + const counts = new Map(); + const rawValues = new Set(); + for (const doc of docs) { + for (const value of asList(doc.component)) { + rawValues.add(value); + const folded = normalizeComponentValue(value); + const label = aliases.get(folded) ?? folded; + counts.set(label, (counts.get(label) ?? 0) + 1); + } + } + const sorted = [...counts.entries()] + .map(([value, count]) => ({ value, count })) + .sort((a, b) => b.count - a.count || a.value.localeCompare(b.value)); + return { + normalized: sorted, + variants: { rawValues: rawValues.size, foldedValues: sorted.length }, + }; +} + +function countStates(docs) { + const counts = new Map(); + for (const doc of docs) { + const state = docState(doc); + counts.set(state, (counts.get(state) ?? 0) + 1); + } + return [...counts.entries()] + .map(([value, count]) => ({ value, count })) + .sort((a, b) => b.count - a.count || a.value.localeCompare(b.value)); +} + +function movementClass(recentCount, baselineCount, ratio) { + if (recentCount === 0 && baselineCount === 0) { + return 'insufficient'; + } + if (baselineCount === 0) { + return recentCount >= NEW_THEME_MIN_ARTICLES ? 'new' : 'insufficient'; + } + if (recentCount === 0) { + return baselineCount >= MIN_MOVEMENT_OBSERVATIONS ? 'declining' : 'insufficient'; + } + if (recentCount < MIN_MOVEMENT_OBSERVATIONS || baselineCount < MIN_MOVEMENT_OBSERVATIONS) { + return 'insufficient'; + } + if (ratio === null) { + return 'insufficient'; + } + if (ratio >= GROWING_RATIO) { + return 'growing'; + } + if (ratio <= DECLINING_RATIO) { + return 'declining'; + } + return 'stable'; +} + export function usage() { return `Usage: analyze-kb.mjs [options] @@ -141,6 +330,7 @@ Options: --comparison-since DATE Explicit comparison window start --comparison-until DATE Explicit comparison window end --themes PATH|none Theme taxonomy JSON file, or "none" to skip clustering (default: built-in) + --component-aliases PATH JSON object mapping component names to canonical names --include-abstracts Include a truncated abstract per article in JSON output --format FORMAT Output format: markdown or json (default: markdown) -h, --help Show this help @@ -193,6 +383,7 @@ export function parseArgs(argv) { '--comparison-since': 'comparisonSince', '--comparison-until': 'comparisonUntil', '--themes': 'themes', + '--component-aliases': 'componentAliases', '--format': 'format', }; @@ -246,6 +437,7 @@ export function parseArgs(argv) { } args.themes = resolveTaxonomy(args.themes); + args.componentAliases = loadComponentAliases(args.componentAliases); return args; } @@ -431,13 +623,18 @@ function articleSummary(doc, rules, includeAbstracts) { const summary = { id: doc.id, title: doc.title, - url: doc.view_uri ?? `https://access.redhat.com/solutions/${doc.id}`, + url: articleUrl(doc), createdDate: docCreatedDate(doc) ?? 'unknown', + lastModifiedDate: lastModifiedOf(doc), + kcsState: docState(doc), categories: asList(doc.category), tags: asList(doc.tag), components: asList(doc.component), + products: productsOf(doc), + productCount: productCountOf(doc), caseCount: doc.caseCount ?? 0, - caseCount365: doc.caseCount_365 ?? 0, + caseCount365: roundTo(casesRawOf(doc), 1), + casesAttributed: roundTo(casesAttributedOf(doc), 2), themes: rules === null ? null : themeHints(doc, rules), }; if (includeAbstracts) { @@ -449,8 +646,9 @@ function articleSummary(doc, rules, includeAbstracts) { return summary; } -function summarizeWindow(docs, window, rules) { +function summarizeWindow(docs, window, rules, aliases = new Map()) { const windowDocs = docs.filter((doc) => isInWindow(doc, window)); + const components = countComponents(windowDocs, aliases); return { window, @@ -458,6 +656,9 @@ function summarizeWindow(docs, window, rules) { categories: countValues(windowDocs, 'category'), tags: countValues(windowDocs, 'tag'), components: countValues(windowDocs, 'component'), + componentsNormalized: components.normalized, + componentVariants: components.variants, + kcsStates: countStates(windowDocs), months: countByPeriod(windowDocs, 'month'), years: countByPeriod(windowDocs, 'year'), themes: rules === null ? null : countThemes(windowDocs, window, rules), @@ -470,12 +671,37 @@ function countThemes(windowDocs, window, rules) { const themes = new Map(); for (const doc of windowDocs) { - const cases = doc.caseCount_365 ?? 0; + const raw = casesRawOf(doc); + const attributed = casesAttributedOf(doc); + const dedicated = productCountOf(doc) === 1 ? raw : 0; for (const theme of themeHints(doc, rules)) { - const current = themes.get(theme) ?? { count: 0, caseCount365: 0, maxArticleCases: 0 }; + const current = + themes.get(theme) ?? + { + count: 0, + articlesWithCases: 0, + casesRaw: 0, + casesAttributed: 0, + casesDedicated: 0, + topArticle: null, + }; current.count += 1; - current.caseCount365 += cases; - current.maxArticleCases = Math.max(current.maxArticleCases, cases); + if (raw > 0) { + current.articlesWithCases += 1; + } + current.casesRaw += raw; + current.casesAttributed += attributed; + current.casesDedicated += dedicated; + if (raw > 0 && (current.topArticle === null || attributed > current.topArticle.attributed)) { + current.topArticle = { + id: doc.id, + title: doc.title, + url: articleUrl(doc), + raw: roundTo(raw, 1), + attributed: roundTo(attributed, 1), + productCount: productCountOf(doc), + }; + } themes.set(theme, current); } } @@ -484,15 +710,254 @@ function countThemes(windowDocs, window, rules) { .map(([theme, value]) => ({ theme, count: value.count, - caseCount365: value.caseCount365, - maxArticleCases: value.maxArticleCases, - topArticleShare: value.caseCount365 === 0 ? 0 : value.maxArticleCases / value.caseCount365, - perDay: value.count / window.days, - share: windowDocs.length === 0 ? 0 : value.count / windowDocs.length, + articlesWithCases: value.articlesWithCases, + casesRaw: roundTo(value.casesRaw, 1), + casesAttributed: roundTo(value.casesAttributed, 1), + casesDedicated: roundTo(value.casesDedicated, 1), + caseCount365: roundTo(value.casesRaw, 1), + maxArticleCases: value.topArticle === null ? 0 : value.topArticle.raw, + topArticleShare: + value.casesRaw === 0 ? 0 : roundTo((value.topArticle?.raw ?? 0) / value.casesRaw, 3), + topArticle: + value.topArticle === null + ? null + : { + id: value.topArticle.id, + title: value.topArticle.title, + url: value.topArticle.url, + casesRaw: value.topArticle.raw, + casesAttributed: value.topArticle.attributed, + productCount: value.topArticle.productCount, + }, + perDay: window && window.days ? roundTo(value.count / window.days, 3) : null, + share: windowDocs.length === 0 ? 0 : roundTo(value.count / windowDocs.length, 3), })) .sort((a, b) => b.count - a.count || a.theme.localeCompare(b.theme)); } +function summarizeDemand(docs, rules, aliases) { + const withCases = docs.filter((doc) => casesRawOf(doc) > 0); + const totalRaw = docs.reduce((sum, doc) => sum + casesRawOf(doc), 0); + const totalAttributed = docs.reduce((sum, doc) => sum + casesAttributedOf(doc), 0); + const ranked = [...withCases].sort( + (a, b) => casesAttributedOf(b) - casesAttributedOf(a) || casesRawOf(b) - casesRawOf(a), + ); + const topDoc = ranked[0] ?? null; + const topRaw = topDoc === null ? 0 : casesRawOf(topDoc); + const topAttributed = topDoc === null ? 0 : casesAttributedOf(topDoc); + const components = countComponents(docs, aliases); + const themes = rules === null ? null : countThemes(docs, { days: 365 }, rules); + + return { + basis: 'caseCount_365 over the whole corpus; not windowed by createdDate', + windowDays: 365, + articleCount: docs.length, + articlesWithCases: withCases.length, + casesRaw: roundTo(totalRaw, 1), + casesAttributed: roundTo(totalAttributed, 1), + concentration: { + topArticle: + topDoc === null + ? null + : { + id: topDoc.id, + title: topDoc.title, + url: articleUrl(topDoc), + casesRaw: roundTo(topRaw, 1), + casesAttributed: roundTo(topAttributed, 1), + productCount: productCountOf(topDoc), + }, + topShareRaw: totalRaw === 0 ? 0 : roundTo(topRaw / totalRaw, 3), + topShareAttributed: totalAttributed === 0 ? 0 : roundTo(topAttributed / totalAttributed, 3), + warned: totalRaw > 0 && topRaw / totalRaw > CONCENTRATION_WARNING_SHARE, + }, + themes: + themes === null + ? null + : [...themes].sort( + (a, b) => b.casesAttributed - a.casesAttributed || a.theme.localeCompare(b.theme), + ), + components: components.normalized, + topArticles: ranked + .slice(0, DEMAND_TOP_ARTICLES) + .map((doc) => ({ + id: doc.id, + title: doc.title, + url: articleUrl(doc), + createdDate: docCreatedDate(doc), + casesRaw: roundTo(casesRawOf(doc), 1), + casesAttributed: roundTo(casesAttributedOf(doc), 1), + productCount: productCountOf(doc), + kcsState: docState(doc), + themes: rules === null ? null : themeHints(doc, rules), + })), + }; +} + +function summarizeMovement(docs, recent, comparison, rules) { + const nested = recent.start >= comparison.start && recent.end <= comparison.end; + const baselineWindow = + nested && recent.start > comparison.start + ? { + start: comparison.start, + end: addDaysIso(recent.start, -1), + days: daysBetweenInclusive(comparison.start, addDaysIso(recent.start, -1)), + } + : { start: comparison.start, end: comparison.end, days: comparison.days }; + + const recentDocs = docs.filter((doc) => isInWindow(doc, recent)); + const baselineDocs = docs.filter((doc) => isInWindow(doc, baselineWindow)); + + const countMap = (list) => { + const counts = new Map(); + for (const doc of list) { + for (const theme of themeHints(doc, rules)) { + counts.set(theme, (counts.get(theme) ?? 0) + 1); + } + } + return counts; + }; + + const recentCounts = rules === null ? new Map() : countMap(recentDocs); + const baselineCounts = rules === null ? new Map() : countMap(baselineDocs); + + const themes = + rules === null + ? null + : rules + .map((rule) => { + const theme = rule.theme; + const recentCount = recentCounts.get(theme) ?? 0; + const baselineCount = baselineCounts.get(theme) ?? 0; + const recentPerDay = recentCount / recent.days; + const baselinePerDay = baselineCount / baselineWindow.days; + const ratio = baselinePerDay === 0 ? null : recentPerDay / baselinePerDay; + const shareRecent = recentDocs.length === 0 ? 0 : recentCount / recentDocs.length; + const shareBaseline = baselineDocs.length === 0 ? 0 : baselineCount / baselineDocs.length; + return { + theme, + recentCount, + baselineCount, + recentPerDay: roundTo(recentPerDay, 3), + baselinePerDay: roundTo(baselinePerDay, 3), + ratio: ratio === null ? null : roundTo(ratio, 2), + shareRecent: roundTo(shareRecent, 3), + shareBaseline: roundTo(shareBaseline, 3), + deltaShare: roundTo(shareRecent - shareBaseline, 3), + class: movementClass(recentCount, baselineCount, ratio), + }; + }) + .sort((a, b) => b.recentCount - a.recentCount || a.theme.localeCompare(b.theme)); + + return { + baseline: { + window: baselineWindow, + articleCount: baselineDocs.length, + nested, + minObservations: MIN_MOVEMENT_OBSERVATIONS, + }, + themes, + }; +} + +function summarizeKcs(docs, asOf) { + const candidates = docs + .filter((doc) => casesRawOf(doc) > 0) + .map((doc) => ({ doc, modified: lastModifiedOf(doc), age: ageInDays(lastModifiedOf(doc), asOf) })) + .filter((item) => item.age !== null && item.age > STALE_AFTER_DAYS) + .sort( + (a, b) => + casesAttributedOf(b.doc) - casesAttributedOf(a.doc) || + casesRawOf(b.doc) - casesRawOf(a.doc), + ) + .slice(0, REFRESH_CANDIDATES) + .map((item) => ({ + id: item.doc.id, + title: item.doc.title, + url: articleUrl(item.doc), + casesRaw: roundTo(casesRawOf(item.doc), 1), + casesAttributed: roundTo(casesAttributedOf(item.doc), 1), + productCount: productCountOf(item.doc), + lastModifiedDate: item.modified, + daysSinceModified: item.age, + kcsState: docState(item.doc), + })); + + return { + corpus: countStates(docs), + staleAfterDays: STALE_AFTER_DAYS, + refreshCandidates: candidates, + }; +} + +function summarizeEmergentTerms(docs, recent, baselineWindow) { + const recentDocs = docs.filter((doc) => isInWindow(doc, recent)); + const baselineDocs = docs.filter((doc) => isInWindow(doc, baselineWindow)); + + const index = (list) => { + const terms = new Map(); + for (const doc of list) { + for (const token of tokensOf(doc)) { + const entry = terms.get(token) ?? { count: 0, ids: [] }; + entry.count += 1; + if (entry.ids.length < 2) { + entry.ids.push(doc.id); + } + terms.set(token, entry); + } + } + return terms; + }; + + const recentTerms = index(recentDocs); + const baselineTerms = index(baselineDocs); + const rows = []; + + const ubiquitous = + recentDocs.length >= UBIQUITOUS_TERM_MIN_ARTICLES + ? Math.ceil(recentDocs.length * UBIQUITOUS_TERM_SHARE) + : Number.POSITIVE_INFINITY; + + for (const [term, entry] of recentTerms) { + if (entry.count >= ubiquitous) { + continue; + } + const baseline = baselineTerms.get(term) ?? { count: 0, ids: [] }; + const recentPerDay = entry.count / recent.days; + const baselinePerDay = baseline.count / baselineWindow.days; + const ratio = baselinePerDay === 0 ? null : recentPerDay / baselinePerDay; + let klass = null; + if (baseline.count === 0 && entry.count >= NEW_TERM_MIN_ARTICLES) { + klass = 'new'; + } else if (entry.count >= MIN_TERM_ARTICLES && baseline.count > 0 && ratio !== null && ratio >= RISING_TERM_RATIO) { + klass = 'rising'; + } + if (klass === null) { + continue; + } + rows.push({ + term, + class: klass, + recentArticles: entry.count, + baselineArticles: baseline.count, + ratio: ratio === null ? null : roundTo(ratio, 2), + exampleIds: entry.ids, + }); + } + + const pick = (klass) => + rows + .filter((row) => row.class === klass) + .sort((a, b) => b.recentArticles - a.recentArticles || a.term.localeCompare(b.term)) + .slice(0, EMERGENT_TERMS); + + return { + newTerms: pick('new'), + risingTerms: pick('rising'), + minArticles: MIN_TERM_ARTICLES, + }; +} + export function summarize(fetchResult, args) { const { docs, numFound, truncated } = fetchResult; const { recent, comparison } = resolveWindows(args); @@ -518,13 +983,23 @@ export function summarize(fetchResult, args) { (a, b) => b.createdDate.localeCompare(a.createdDate) || String(a.id).localeCompare(String(b.id)), ); + const demand = summarizeDemand(docs, rules, args.componentAliases ?? new Map()); + const movement = summarizeMovement(docs, recent, comparison, rules); + const kcs = summarizeKcs(docs, args.asOf); + const emergentTerms = summarizeEmergentTerms(docs, recent, movement.baseline.window); + return { + schemaVersion: SCHEMA_VERSION, product: args.product, asOf: args.asOf, taxonomy: rules === null ? null : { mode: taxonomy.mode, name: taxonomy.name, themes: rules.map((rule) => rule.theme) }, - recent: summarizeWindow(docs, recent, rules), - comparison: summarizeWindow(docs, comparison, rules), + recent: summarizeWindow(docs, recent, rules, args.componentAliases ?? new Map()), + comparison: summarizeWindow(docs, comparison, rules, args.componentAliases ?? new Map()), + demand, + movement, + kcs, + emergentTerms, allTime: { articleCount: numFound, fetchedArticles: docs.length, @@ -552,12 +1027,19 @@ function formatCountList(counts, limit = 10) { .join('\n'); } +function formatCases(count) { + return `${count} attributed ${count === 1 ? 'case' : 'cases'}`; +} + function formatThemeCases(item) { - const base = `${item.caseCount365} linked support cases (365d, summed)`; - if (item.caseCount365 > 0 && item.topArticleShare > 0.5) { - return `${base}; one article supplies ${formatPercent(item.topArticleShare)}`; + const parts = [formatCases(item.casesAttributed)]; + if (item.casesAttributed !== item.casesRaw || item.casesRaw !== item.casesDedicated) { + parts.push(`raw ${item.casesRaw}`, `single-product ${item.casesDedicated}`); } - return base; + if (item.articlesWithCases > 1 && item.casesRaw > 0 && item.topArticleShare > 0.5) { + parts.push(`one article supplies ${formatPercent(item.topArticleShare)} of raw weight`); + } + return parts.join('; '); } function formatThemeList(themes) { @@ -567,11 +1049,103 @@ function formatThemeList(themes) { return themes .map( (item) => - `- ${item.theme}: ${formatArticleCount(item.count)}, ${formatPercent(item.share)} of window, ${item.perDay.toFixed(2)} per day, ${formatThemeCases(item)}`, + `- ${item.theme}: ${formatArticleCount(item.count)} (${item.articlesWithCases} with cases), ${formatPercent(item.share)} of window, ${item.perDay.toFixed(2)} per day, ${formatThemeCases(item)}`, ) .join('\n'); } +function formatDemandThemes(themes, limit = 8) { + if (themes === null) { + return '- No taxonomy applied; rank the demand article list yourself.'; + } + if (themes.length === 0) { + return '- none'; + } + return themes + .slice(0, limit) + .map( + (item) => + `- ${item.theme}: ${formatCases(item.casesAttributed)} (raw ${item.casesRaw}, single-product ${item.casesDedicated}) across ${formatArticleCount(item.count)}, ${item.articlesWithCases} with linked cases`, + ) + .join('\n'); +} + +function formatConcentration(demand) { + const c = demand.concentration; + if (c.topArticle === null) { + return '- No article carries a linked support case in the current corpus.'; + } + const lines = [ + `- Largest article: [${c.topArticle.title}](${c.topArticle.url}) — ${formatCases(c.topArticle.casesAttributed)} (raw ${c.topArticle.casesRaw}), tagged to ${c.topArticle.productCount} product${c.topArticle.productCount === 1 ? '' : 's'}`, + `- Share of raw case weight: ${formatPercent(c.topShareRaw)}; share of attributed weight: ${formatPercent(c.topShareAttributed)}`, + ]; + if (c.warned) { + lines.push( + `- WARNING: a single article supplies more than ${formatPercent(CONCENTRATION_WARNING_SHARE)} of raw case weight. Treat corpus-level case totals as one article's reach, not as demand for every theme it touches.`, + ); + } + return lines.join('\n'); +} + +function formatKcs(kcs) { + const states = kcs.corpus.map((item) => `${item.value} ${item.count}`).join(', '); + const lines = [`- Corpus KCS state: ${states || 'unknown'}`]; + const candidates = kcs.refreshCandidates; + if (candidates.length === 0) { + lines.push(`- No article with linked cases has gone unedited for more than ${kcs.staleAfterDays} days.`); + return lines.join('\n'); + } + lines.push( + `- Refresh candidates (linked cases, last edited more than ${kcs.staleAfterDays} days ago): ${candidates.length}`, + ); + lines.push(''); + for (const item of candidates) { + lines.push( + `- [${item.title}](${item.url}) — ${formatCases(item.casesAttributed)}, last edited ${item.lastModifiedDate} (${item.daysSinceModified} days), ${item.kcsState}`, + ); + } + return lines.join('\n'); +} + +function formatMovement(movement) { + const baseline = movement.baseline; + const lines = [ + `- Baseline window: ${baseline.window.start} to ${baseline.window.end} (${baseline.window.days} days, ${baseline.articleCount} articles)`, + `- A theme needs at least ${baseline.minObservations} articles in both windows before its movement is classified; below that it reads \`insufficient\`.`, + ]; + if (movement.themes === null) { + lines.push('- No taxonomy applied; movement is not computed.'); + return lines.join('\n'); + } + lines.push(''); + for (const item of movement.themes) { + const ratio = item.ratio === null || item.class === 'insufficient' ? 'n/a' : `${item.ratio}x`; + lines.push( + `- ${item.theme}: recent ${item.recentCount}, baseline ${item.baselineCount}, rate ratio ${ratio}, ${item.class}`, + ); + } + return lines.join('\n'); +} + +function formatTerms(emergentTerms) { + const render = (rows) => + rows.length === 0 + ? '- none' + : rows + .map( + (row) => + `- ${row.term}: ${row.recentArticles} recent article${row.recentArticles === 1 ? '' : 's'} (baseline ${row.baselineArticles})`, + ) + .join('\n'); + return `### New terms + +${render(emergentTerms.newTerms)} + +### Rising terms + +${render(emergentTerms.risingTerms)}`; +} + function formatArticles(articles, limit = 15) { if (articles.length === 0) { return '- none'; @@ -597,7 +1171,7 @@ function formatAllTime(report) { } const CASE_TOTAL_CAVEAT = - '- Linked support-case totals are article-level caseCount_365 values summed across matching articles; they are not unique cases and may include multi-product articles.'; + '- Linked support-case totals are article-level caseCount_365 values summed across matching articles; they are not unique cases.'; function caveatsFor(report) { const mode = report.taxonomy?.mode ?? 'none'; @@ -618,6 +1192,17 @@ function caveatsFor(report) { } lines.push(formatWindowCaveat(report)); + lines.push( + '- The demand view weights the whole corpus by `caseCount_365`, a fixed trailing-365-day count published by Red Hat. It cannot be re-windowed, so a quarterly case-volume answer is not derivable from KB data.', + ); + lines.push( + '- Attributed cases divide each article\'s `caseCount_365` by the number of products it is tagged to; single-product cases exclude multi-product articles entirely.', + ); + if (report.demand.concentration.warned) { + lines.push( + '- One article dominates the corpus case weight; see Corpus concentration before quoting any case total.', + ); + } return lines.join('\n'); } @@ -634,6 +1219,7 @@ function formatWindowCaveat(report) { export function renderMarkdown(report) { const recent = report.recent; const comparison = report.comparison; + const demand = report.demand; const recentArticles = report.articles.filter((article) => article.windows.includes('recent')); const topRecentArticles = [...recentArticles] .sort((a, b) => b.caseCount365 - a.caseCount365 || b.createdDate.localeCompare(a.createdDate)) @@ -686,6 +1272,27 @@ ${taxonomyLine} - Recent daily rate: ${(recent.articleCount / recent.window.days).toFixed(2)} - Comparison daily rate: ${(comparison.articleCount / comparison.window.days).toFixed(2)}${unmatchedLines} +## Demand (trailing 365 days, case-weighted) + +- Basis: ${demand.basis} +- Articles with linked cases: ${demand.articlesWithCases} of ${demand.articleCount} +- Attributed cases: ${demand.casesAttributed} (raw ${demand.casesRaw}) +- Theme signals: + +${formatDemandThemes(demand.themes)} + +## Corpus concentration + +${formatConcentration(demand)} + +## KCS health and refresh candidates + +${formatKcs(report.kcs)} + +## Theme movement (recent vs prior baseline) + +${formatMovement(report.movement)} + ## Recent categories ${formatCountList(recent.categories)} @@ -704,13 +1311,17 @@ ${formatCountList(comparison.tags)} ## Recent components -${formatCountList(recent.components)} +${formatCountList(recent.componentsNormalized)} ## Comparison components -${formatCountList(comparison.components)} +${formatCountList(comparison.componentsNormalized)} + +${themeSections}## Emerging vocabulary + +${formatTerms(report.emergentTerms)} -${themeSections}${articleHeading} +${articleHeading} ${formatArticles(topRecentArticles)} diff --git a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs index 3402e672..23970d90 100644 --- a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs +++ b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs @@ -8,9 +8,13 @@ import { ABSTRACT_LIMIT, DEFAULT_COMPARISON_DAYS, DEFAULT_RECENT_DAYS, + SCHEMA_VERSION, buildQuery, + casesAttributedOf, fetchKbDocs, isInWindow, + loadComponentAliases, + normalizeComponentValue, parseArgs, renderMarkdown, resolveWindows, @@ -543,3 +547,334 @@ describe('report signals', () => { assert.ok(!renderMarkdown(report).includes('one article supplies')); }); }); + +describe('demand weighting', () => { + const args = parseArgs(['--as-of', '2026-10-30']); + + function writeTaxonomy(rules) { + const directory = mkdtempSync(join(tmpdir(), 'rhdh-kb-demand-')); + const file = join(directory, 'taxonomy.json'); + writeFileSync(file, JSON.stringify(rules)); + return file; + } + + it('divides a multi-product article across its products', () => { + const shared = { + id: 'cve', + title: 'Upgrade to check CVEs', + createdDate: '2026-10-01T10:00:00Z', + caseCount_365: 81, + standard_product: ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'], + }; + assert.equal(casesAttributedOf(shared), 9); + + const own = { + id: 'own', + title: 'Upgrade fails', + createdDate: '2026-10-02T10:00:00Z', + caseCount_365: 9, + standard_product: ['Red Hat Developer Hub'], + }; + + const file = writeTaxonomy([{ theme: 'upgrades', pattern: '\\bupgrad\\w*' }]); + const report = summarize( + { docs: [shared, own], numFound: 2, truncated: false }, + parseArgs(['--as-of', '2026-10-30', '--themes', file]), + ); + + const theme = report.demand.themes[0]; + assert.equal(theme.casesRaw, 90); + assert.equal(theme.casesAttributed, 18); + assert.equal(theme.casesDedicated, 9); + assert.equal(theme.topArticle.productCount, 9); + }); + + it('warns when one article holds most of the corpus weight', () => { + const docs = [ + { id: 'big', title: 'Upgrade CVE check', createdDate: '2026-10-01T10:00:00Z', caseCount_365: 90 }, + { id: 'small', title: 'Upgrade one', createdDate: '2026-10-02T10:00:00Z', caseCount_365: 2 }, + ]; + const report = summarize({ docs, numFound: 2, truncated: false }, args); + + assert.equal(report.demand.concentration.warned, true); + assert.equal(report.demand.concentration.topArticle.id, 'big'); + assert.ok(renderMarkdown(report).includes('WARNING: a single article supplies')); + }); + + it('does not warn when case weight is spread', () => { + const docs = [ + { id: 'a', title: 'Upgrade one', createdDate: '2026-10-01T10:00:00Z', caseCount_365: 3 }, + { id: 'b', title: 'Upgrade two', createdDate: '2026-10-02T10:00:00Z', caseCount_365: 3 }, + { id: 'c', title: 'Upgrade three', createdDate: '2026-10-03T10:00:00Z', caseCount_365: 3 }, + { id: 'd', title: 'Upgrade four', createdDate: '2026-10-04T10:00:00Z', caseCount_365: 3 }, + ]; + const report = summarize({ docs, numFound: 4, truncated: false }, args); + assert.equal(report.demand.concentration.warned, false); + assert.ok(!renderMarkdown(report).includes('WARNING: a single article supplies')); + }); + + it('never reports attributed weight above raw weight', () => { + const docs = [ + { id: 'a', title: 'Upgrade one', createdDate: '2026-10-01T10:00:00Z', caseCount_365: 12, standard_product: ['x', 'y', 'z'] }, + { id: 'b', title: 'Upgrade two', createdDate: '2026-10-02T10:00:00Z', caseCount_365: 4 }, + ]; + const report = summarize({ docs, numFound: 2, truncated: false }, args); + assert.ok(report.demand.casesAttributed <= report.demand.casesRaw); + for (const theme of report.demand.themes) { + assert.ok(theme.casesAttributed <= theme.casesRaw); + } + }); +}); + +describe('theme movement', () => { + function writeTaxonomy(rules) { + const directory = mkdtempSync(join(tmpdir(), 'rhdh-kb-move-')); + const file = join(directory, 'taxonomy.json'); + writeFileSync(file, JSON.stringify(rules)); + return file; + } + + function docsFor(theme, dates) { + return dates.map((date, index) => ({ + id: `${theme}-${index}`, + title: `${theme} question`, + createdDate: `${date}T10:00:00Z`, + })); + } + + const taxonomy = [ + { theme: 'alpha', pattern: '\\balpha\\b' }, + { theme: 'beta', pattern: '\\bbeta\\b' }, + { theme: 'gamma', pattern: '\\bgamma\\b' }, + { theme: 'delta', pattern: '\\bdelta\\b' }, + { theme: 'epsilon', pattern: '\\bepsilon\\b' }, + ]; + + it('compares the recent window against the previous baseline', () => { + const recentDate = '2026-10-01'; + const baselineDate = '2026-06-01'; + const docs = [ + ...docsFor('alpha', [recentDate, recentDate, recentDate, recentDate, recentDate, recentDate, recentDate, recentDate, recentDate, recentDate, recentDate, recentDate]), + ...docsFor('alpha', [baselineDate, baselineDate, baselineDate, baselineDate, baselineDate]), + ...docsFor('beta', [recentDate, recentDate, recentDate, recentDate]), + ...docsFor('beta', [baselineDate, baselineDate, baselineDate, baselineDate]), + ...docsFor('gamma', [recentDate, recentDate, recentDate]), + ...docsFor('delta', [baselineDate, baselineDate, baselineDate, baselineDate, baselineDate, baselineDate]), + ]; + const file = writeTaxonomy(taxonomy); + const report = summarize( + { docs, numFound: docs.length, truncated: false }, + parseArgs(['--as-of', '2026-10-30', '--recent-days', '101', '--comparison-days', '202', '--themes', file]), + ); + + const byTheme = Object.fromEntries(report.movement.themes.map((row) => [row.theme, row])); + assert.equal(byTheme.alpha.class, 'growing'); + assert.equal(byTheme.beta.class, 'insufficient'); + assert.equal(byTheme.gamma.class, 'new'); + assert.equal(byTheme.delta.class, 'declining'); + assert.equal(byTheme.epsilon.class, 'insufficient'); + + assert.equal(report.movement.baseline.window.end, '2026-07-21'); + assert.ok(report.movement.baseline.window.start >= report.comparison.window.start); + }); + + it('never prints a ratio below the observation guard', () => { + const docs = docsFor('alpha', ['2026-10-01', '2026-10-02']); + const file = writeTaxonomy(taxonomy); + const report = summarize( + { docs, numFound: docs.length, truncated: false }, + parseArgs(['--as-of', '2026-10-30', '--themes', file]), + ); + const alpha = report.movement.themes.find((row) => row.theme === 'alpha'); + assert.equal(alpha.class, 'insufficient'); + assert.ok(renderMarkdown(report).includes('alpha: recent 2, baseline 0')); + }); +}); + +describe('component normalization', () => { + it('folds case, whitespace, and dashes', () => { + assert.equal(normalizeComponentValue(' Dev-Console '), 'dev console'); + assert.equal(normalizeComponentValue('KEYCLOAK'), 'keycloak'); + }); + + it('reports folded counts and keeps raw variants visible', () => { + const docs = [ + { id: '1', title: 'x', createdDate: '2026-10-01T10:00:00Z', component: ['Authentication', 'authentication'] }, + { id: '2', title: 'y', createdDate: '2026-10-02T10:00:00Z', component: ['KEYCLOAK', 'key-cloak'] }, + ]; + const report = summarize( + { docs, numFound: 2, truncated: false }, + parseArgs(['--as-of', '2026-10-30']), + ); + const counts = Object.fromEntries(report.recent.componentsNormalized.map((item) => [item.value, item.count])); + assert.equal(counts.authentication, 2); + assert.equal(counts.keycloak, 1); + assert.equal(counts['key cloak'], 1); + assert.equal(report.recent.componentVariants.rawValues, 4); + assert.equal(report.recent.componentVariants.foldedValues, 3); + }); + + it('applies an alias file', () => { + const directory = mkdtempSync(join(tmpdir(), 'rhdh-kb-alias-')); + const aliases = join(directory, 'aliases.json'); + writeFileSync(aliases, JSON.stringify({ 'key-cloak': 'authentication' })); + + const docs = [ + { id: '1', title: 'x', createdDate: '2026-10-02T10:00:00Z', component: ['KEYCLOAK', 'key-cloak'] }, + ]; + const report = summarize( + { docs, numFound: 1, truncated: false }, + parseArgs(['--as-of', '2026-10-30', '--component-aliases', aliases]), + ); + const counts = Object.fromEntries(report.recent.componentsNormalized.map((item) => [item.value, item.count])); + assert.equal(counts.authentication, 1); + assert.equal(counts.keycloak, 1); + }); + + it('rejects an alias file that is not an object', () => { + const directory = mkdtempSync(join(tmpdir(), 'rhdh-kb-alias-bad-')); + const aliases = join(directory, 'aliases.json'); + writeFileSync(aliases, JSON.stringify(['keycloak'])); + assert.throws(() => loadComponentAliases(aliases), /must be a JSON object/); + }); + + it('rejects an alias with an empty target', () => { + const directory = mkdtempSync(join(tmpdir(), 'rhdh-kb-alias-empty-')); + const aliases = join(directory, 'aliases.json'); + writeFileSync(aliases, JSON.stringify({ keycloak: '' })); + assert.throws(() => loadComponentAliases(aliases), /needs a non-empty string target/); + }); +}); + +describe('KCS health', () => { + it('lists stale high-touch articles by attributed weight', () => { + const docs = [ + { id: 'stale-big', title: 'Upgrade is stuck', createdDate: '2024-01-01T10:00:00Z', lastModifiedDate: '2024-06-01T10:00:00Z', caseCount_365: 12 }, + { id: 'stale-small', title: 'Upgrade is slow', createdDate: '2024-01-01T10:00:00Z', lastModifiedDate: '2024-06-01T10:00:00Z', caseCount_365: 2 }, + { id: 'fresh', title: 'Upgrade works', createdDate: '2024-01-01T10:00:00Z', lastModifiedDate: '2026-10-01T10:00:00Z', caseCount_365: 40 }, + { id: 'quiet', title: 'Upgrade legend', createdDate: '2024-01-01T10:00:00Z', lastModifiedDate: '2024-06-01T10:00:00Z', caseCount_365: 0 }, + ]; + const report = summarize( + { docs, numFound: 4, truncated: false }, + parseArgs(['--as-of', '2026-10-30']), + ); + + const ids = report.kcs.refreshCandidates.map((item) => item.id); + assert.deepEqual(ids, ['stale-big', 'stale-small']); + assert.equal(report.kcs.refreshCandidates[0].daysSinceModified > 365, true); + }); + + it('counts KCS states in the corpus', () => { + const docs = [ + { id: '1', title: 'x', createdDate: '2026-10-01T10:00:00Z', kcsState: 'verified' }, + { id: '2', title: 'y', createdDate: '2026-10-02T10:00:00Z', kcsState: 'unverified' }, + { id: '3', title: 'z', createdDate: '2026-10-03T10:00:00Z' }, + ]; + const report = summarize( + { docs, numFound: 3, truncated: false }, + parseArgs(['--as-of', '2026-10-30']), + ); + const counts = Object.fromEntries(report.kcs.corpus.map((item) => [item.value, item.count])); + assert.deepEqual(counts, { verified: 1, unverified: 1, unknown: 1 }); + }); +}); + +describe('emergent vocabulary', () => { + it('finds terms that are new in the recent window', () => { + const docs = [ + { id: '1', title: 'Webhook delivery fails', createdDate: '2026-10-01T10:00:00Z' }, + { id: '2', title: 'Webhook retries stall', createdDate: '2026-10-02T10:00:00Z' }, + { id: '3', title: 'Webhook signature rejected', createdDate: '2026-10-03T10:00:00Z' }, + { id: '4', title: 'Upgrade fails', createdDate: '2025-12-01T10:00:00Z' }, + ]; + const report = summarize( + { docs, numFound: 4, truncated: false }, + parseArgs(['--as-of', '2026-10-30']), + ); + const terms = report.emergentTerms.newTerms.map((row) => row.term); + assert.ok(terms.includes('webhook')); + assert.ok(!terms.includes('upgrade')); + }); + + it('ignores abstract prose and terms present in most of the window', () => { + const docs = []; + for (let index = 0; index < 12; index++) { + docs.push({ + id: `u${index}`, + title: 'Upgrade question', + createdDate: '2026-10-01T10:00:00Z', + }); + } + docs.push({ id: 'w1', title: 'Webhook delivery fails', createdDate: '2026-10-02T10:00:00Z', issue: ['Webhook signature rejected'] }); + docs.push({ id: 'w2', title: 'Webhook retries stall', createdDate: '2026-10-03T10:00:00Z' }); + docs.push({ id: 'w3', title: 'Webhook timeout', createdDate: '2026-10-04T10:00:00Z' }); + + const report = summarize( + { docs, numFound: docs.length, truncated: false }, + parseArgs(['--as-of', '2026-10-30']), + ); + const terms = report.emergentTerms.newTerms.map((row) => row.term); + assert.ok(terms.includes('webhook')); + assert.ok(!terms.includes('upgrade'), 'ubiquitous terms are noise'); + }); + + it('does not mine the abstract for vocabulary', () => { + const docs = [ + { id: '1', title: 'Upgrade fails', createdDate: '2026-10-01T10:00:00Z', issue: ['Zookeeper ensemble is unhappy'] }, + { id: '2', title: 'Upgrade fails again', createdDate: '2026-10-02T10:00:00Z', issue: ['Zookeeper ensemble is unhappy'] }, + ]; + const report = summarize( + { docs, numFound: 2, truncated: false }, + parseArgs(['--as-of', '2026-10-30']), + ); + const terms = report.emergentTerms.newTerms.map((row) => row.term); + assert.ok(!terms.includes('zookeeper')); + }); + + it('keeps single mentions out of the term list', () => { + const docs = [{ id: '1', title: 'Webhook delivery fails', createdDate: '2026-10-01T10:00:00Z' }]; + const report = summarize( + { docs, numFound: 1, truncated: false }, + parseArgs(['--as-of', '2026-10-30']), + ); + assert.deepEqual(report.emergentTerms.newTerms, []); + assert.deepEqual(report.emergentTerms.risingTerms, []); + }); +}); + +describe('report schema', () => { + it('publishes schema version 2 with the new blocks', () => { + const docs = [ + { id: '1', title: 'Upgrade fails', createdDate: '2026-10-01T10:00:00Z', caseCount_365: 2 }, + ]; + const report = summarize( + { docs, numFound: 1, truncated: false }, + parseArgs(['--as-of', '2026-10-30']), + ); + assert.equal(SCHEMA_VERSION, 2); + assert.equal(report.schemaVersion, 2); + for (const key of ['demand', 'movement', 'kcs', 'emergentTerms', 'allTime', 'articles']) { + assert.ok(key in report, `missing ${key}`); + } + assert.ok(Array.isArray(report.articles)); + assert.ok('themes' in report.demand); + }); + + it('renders the new Markdown sections', () => { + const report = summarize( + { docs: [{ id: '1', title: 'Upgrade fails', createdDate: '2026-10-01T10:00:00Z' }], numFound: 1, truncated: false }, + parseArgs(['--as-of', '2026-10-30']), + ); + const markdown = renderMarkdown(report); + for (const heading of [ + '## Demand (trailing 365 days, case-weighted)', + '## Corpus concentration', + '## KCS health and refresh candidates', + '## Theme movement (recent vs prior baseline)', + '## Emerging vocabulary', + '## Caveats', + ]) { + assert.ok(markdown.includes(heading), `missing ${heading}`); + } + }); +}); From 0a9c7bf67ee739431c062069b38f02d82ed0e4d7 Mon Sep 17 00:00:00 2001 From: mattreid Date: Fri, 6 Nov 2026 12:15:07 -0500 Subject: [PATCH 3/6] fix(release): disclose the span an explicit recent window leaves uncovered An explicit --recent-since/--recent-until pair ends before the default comparison window does, so a calendar quarter's comparison silently included the following month. The caveat now names the uncovered span, counts its days, and points at --comparison-until. Assisted-By: Codex --- .../scripts/analyze-kb.mjs | 17 +++++++-- .../tests/analyze-kb.test.mjs | 37 +++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs index c320ee78..eeac56d1 100644 --- a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs +++ b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs @@ -1210,10 +1210,21 @@ function formatWindowCaveat(report) { const recent = report.recent.window; const comparison = report.comparison.window; const nested = recent.start >= comparison.start && recent.end <= comparison.end; - if (nested) { - return '- The recent window is a subset of the comparison window; use per-day rates and shares rather than raw counts alone.'; + if (!nested) { + return '- The recent and comparison windows are not nested; use per-day rates and window shares rather than raw counts alone.'; } - return '- The recent and comparison windows are not nested; use per-day rates and window shares rather than raw counts alone.'; + + const lines = [ + '- The recent window is a subset of the comparison window; use per-day rates and shares rather than raw counts alone.', + ]; + if (recent.end < comparison.end) { + const gapStart = addDaysIso(recent.end, 1); + const gapDays = daysBetweenInclusive(gapStart, comparison.end); + lines.push( + `- The recent window ends on ${recent.end} but the comparison window runs to ${comparison.end}: ${gapStart} to ${comparison.end} (${gapDays} days) counts toward the comparison total while sitting in neither the recent window nor the movement baseline. Pass \`--comparison-until ${recent.end}\` to align them.`, + ); + } + return lines.join('\n'); } export function renderMarkdown(report) { diff --git a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs index 23970d90..246285f8 100644 --- a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs +++ b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs @@ -878,3 +878,40 @@ describe('report schema', () => { } }); }); + +describe('window gap caveat', () => { + it('names the span the recent window leaves uncovered', () => { + const report = summarize( + { + docs: [{ id: '1', title: 'Upgrade fails', createdDate: '2026-08-01T10:00:00Z' }], + numFound: 1, + truncated: false, + }, + parseArgs([ + '--as-of', + '2026-10-30', + '--recent-since', + '2026-07-01', + '--recent-until', + '2026-09-30', + ]), + ); + + const markdown = renderMarkdown(report); + assert.ok(markdown.includes('2026-10-01 to 2026-10-30 (30 days)')); + assert.ok(markdown.includes('`--comparison-until 2026-09-30`')); + }); + + it('stays quiet when the recent window ends on the comparison end', () => { + const report = summarize( + { + docs: [{ id: '1', title: 'Upgrade fails', createdDate: '2026-10-01T10:00:00Z' }], + numFound: 1, + truncated: false, + }, + parseArgs(['--as-of', '2026-10-30']), + ); + + assert.ok(!renderMarkdown(report).includes('--comparison-until')); + }); +}); From 9271901ac03acd8f00cfaff7e69e8a5a23133a8f Mon Sep 17 00:00:00 2001 From: mattreid Date: Fri, 6 Nov 2026 13:48:41 -0500 Subject: [PATCH 4/6] fix(release): stop the vocabulary section reporting chance terms With 16 recent articles, any word used twice in the year has a 4.6% chance of landing only in the recent window, so a hundred candidate terms yield about four false 'new' hits per run. Raise the floor to three recent articles, drop symptom vocabulary from titles, and state the baseline length on rising rows so the rate claim is visible. Assisted-By: Codex --- .../references/interpretation.md | 6 ++- .../scripts/analyze-kb.mjs | 21 ++++---- .../tests/analyze-kb.test.mjs | 53 +++++++++++++++++++ 3 files changed, 69 insertions(+), 11 deletions(-) diff --git a/skills/release/rhdh-kb-theme-analysis/references/interpretation.md b/skills/release/rhdh-kb-theme-analysis/references/interpretation.md index 131c1526..530f6673 100644 --- a/skills/release/rhdh-kb-theme-analysis/references/interpretation.md +++ b/skills/release/rhdh-kb-theme-analysis/references/interpretation.md @@ -153,5 +153,7 @@ State these limitations in the final answer: can be stated about case volume. - Attributed cases are an estimate that assumes an article's cases split evenly across its tagged products. -- Emergent terms are word frequencies, not themes. Raise one only when it maps - onto an article set you can cite. +- Emergent terms are word frequencies, not themes. A term is `new` only with + three recent articles and no baseline use, and `rising` only with three recent + articles and twice the baseline rate. Those are floors, not evidence; raise a + term only when it maps onto an article set you can cite. diff --git a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs index eeac56d1..eaa842d7 100644 --- a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs +++ b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs @@ -137,13 +137,13 @@ export const DECLINING_RATIO = 0.67; export const CONCENTRATION_WARNING_SHARE = 0.25; export const NEW_THEME_MIN_ARTICLES = 3; export const MIN_TERM_ARTICLES = 3; -export const NEW_TERM_MIN_ARTICLES = 2; +export const NEW_TERM_MIN_ARTICLES = 3; export const RISING_TERM_RATIO = 2; export const UBIQUITOUS_TERM_SHARE = 0.3; export const UBIQUITOUS_TERM_MIN_ARTICLES = 10; const STOPWORDS = new Set( - `about above after again against also and any are because been before being below between both but can cannot could did does doing down during each few for from further had has have having here how into its itself just may might more most must not now off once only other out over own same should some such than that the their them then there these they this those through under until very was were what when where which while who whom why will with would your rhdh redhat even show shows showing shown use used uses using get gets getting set sets setting make makes making need needs needed way ways like want wants two three first second last`.split( + `about above after again against also and any are because been before being below between both but can cannot could did does doing down during each few for from further had has have having here how into its itself just may might more most must not now off once only other out over own same should some such than that the their them then there these they this those through under until very was were what when where which while who whom why will with would your rhdh redhat even show shows showing shown use used uses using get gets getting set sets setting make makes making need needs needed way ways like want wants two three first second last missing loading slow unable stuck issue issues problem problems`.split( /\s+/, ), ); @@ -940,6 +940,7 @@ function summarizeEmergentTerms(docs, recent, baselineWindow) { class: klass, recentArticles: entry.count, baselineArticles: baseline.count, + baselineWindowRatio: roundTo(baselineWindow.days / recent.days, 2), ratio: ratio === null ? null : roundTo(ratio, 2), exampleIds: entry.ids, }); @@ -1128,22 +1129,24 @@ function formatMovement(movement) { } function formatTerms(emergentTerms) { - const render = (rows) => + const render = (rows, withBaselineLength) => rows.length === 0 ? '- none' : rows - .map( - (row) => - `- ${row.term}: ${row.recentArticles} recent article${row.recentArticles === 1 ? '' : 's'} (baseline ${row.baselineArticles})`, - ) + .map((row) => { + const stem = `- ${row.term}: ${row.recentArticles} recent article${row.recentArticles === 1 ? '' : 's'}`; + return withBaselineLength + ? `${stem}, ${row.baselineArticles} in a baseline ${row.baselineWindowRatio}x as long` + : `${stem} (baseline ${row.baselineArticles})`; + }) .join('\n'); return `### New terms -${render(emergentTerms.newTerms)} +${render(emergentTerms.newTerms, false)} ### Rising terms -${render(emergentTerms.risingTerms)}`; +${render(emergentTerms.risingTerms, true)}`; } function formatArticles(articles, limit = 15) { diff --git a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs index 246285f8..c106d44f 100644 --- a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs +++ b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs @@ -20,6 +20,7 @@ import { resolveWindows, summarize, themeHints, + tokensOf, } from '../scripts/analyze-kb.mjs'; describe('parseArgs defaults', () => { @@ -915,3 +916,55 @@ describe('window gap caveat', () => { assert.ok(!renderMarkdown(report).includes('--comparison-until')); }); }); + +describe('emergent vocabulary floors', () => { + it('needs three recent articles before a term counts as new', () => { + const docs = [ + { id: '1', title: 'Webhook delivery fails', createdDate: '2026-10-01T10:00:00Z' }, + { id: '2', title: 'Webhook retries stall', createdDate: '2026-10-02T10:00:00Z' }, + ]; + const report = summarize( + { docs, numFound: 2, truncated: false }, + parseArgs(['--as-of', '2026-10-30']), + ); + assert.deepEqual(report.emergentTerms.newTerms, []); + }); + + it('states the baseline length on rising rows', () => { + const recent = ['2026-08-01', '2026-08-02', '2026-08-03'].map((date, index) => ({ + id: `r${index}`, + title: 'Zebra queue stalls', + createdDate: `${date}T10:00:00Z`, + })); + const baseline = ['2026-03-01', '2026-03-02', '2026-03-03'].map((date, index) => ({ + id: `b${index}`, + title: 'Zebra queue stalls', + createdDate: `${date}T10:00:00Z`, + })); + + const report = summarize( + { docs: [...recent, ...baseline], numFound: 6, truncated: false }, + parseArgs(['--as-of', '2026-10-30', '--recent-days', '100', '--comparison-days', '300']), + ); + + const markdown = renderMarkdown(report); + assert.ok( + markdown.includes('zebra: 3 recent articles, 3 in a baseline 2x as long'), + markdown.slice(markdown.indexOf('## Emerging vocabulary')), + ); + }); + + it('drops symptom vocabulary from the token stream', () => { + const tokens = [ + ...tokensOf({ + id: '1', + title: 'Performance issue with loading and missing permissions, problems', + createdDate: '2026-10-01T10:00:00Z', + }), + ]; + for (const word of ['issue', 'loading', 'missing', 'problems']) { + assert.ok(!tokens.includes(word), `${word} should be a stopword`); + } + assert.ok(tokens.includes('permissions')); + }); +}); From 91f3fd37e3192c6e651c4e439f0aaa828d8f92ee Mon Sep 17 00:00:00 2001 From: mattreid Date: Mon, 9 Nov 2026 15:15:33 -0500 Subject: [PATCH 5/6] feat(release): classify theme movement on share, not absolute rate The corpus published 46 articles in the prior half-year and 25 in the recent half. Rating themes on absolute rate turned that single corpus-wide slowdown into six 'declining' themes, two of which had gained share. Classes now read share ratio (1.25x / 0.8x) and the report states movement.corpusRateRatio so the absolute shift stays visible. Assisted-By: Codex --- .../references/interpretation.md | 12 +++- .../scripts/analyze-kb.mjs | 30 +++++++--- .../tests/analyze-kb.test.mjs | 55 +++++++++++++++++++ 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/skills/release/rhdh-kb-theme-analysis/references/interpretation.md b/skills/release/rhdh-kb-theme-analysis/references/interpretation.md index 530f6673..972b84a1 100644 --- a/skills/release/rhdh-kb-theme-analysis/references/interpretation.md +++ b/skills/release/rhdh-kb-theme-analysis/references/interpretation.md @@ -90,11 +90,19 @@ Do not classify growth by eye. `movement.themes` already carries `class`, `ratio`, and `deltaShare` for every theme, computed against the baseline window: - `new` — no baseline articles, at least three recent ones; -- `growing` — recent rate at least 1.5× the baseline rate; -- `declining` — recent rate at most 0.67× the baseline rate; +- `growing` — recent share at least 1.25× the baseline share; +- `declining` — recent share at most 0.8× the baseline share; - `stable` — anything between; - `insufficient` — fewer than five articles on either side, or no baseline. +Classes read **share**, not absolute rate. `movement.corpusRateRatio` states how +much the whole corpus sped up or slowed down between the windows; when it is far +from 1.0, every theme's absolute rate moves with it, and a rate-based reading +would report one corpus-wide shift as a trend in eight themes. Each row still +carries `ratio` so the absolute movement is visible, but the class answers the +narrower question: did this theme gain or lose ground relative to everything +else published in the same window. + The baseline is *not* the comparison window. It is the comparison window minus the recent window, so a theme is never compared against a period that contains itself. `movement.baseline.window` carries the exact dates; quote them. diff --git a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs index eaa842d7..9c2dd0ec 100644 --- a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs +++ b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs @@ -132,8 +132,8 @@ export const DEMAND_TOP_ARTICLES = 20; export const REFRESH_CANDIDATES = 15; export const EMERGENT_TERMS = 10; export const MIN_MOVEMENT_OBSERVATIONS = 5; -export const GROWING_RATIO = 1.5; -export const DECLINING_RATIO = 0.67; +export const GROWING_SHARE_RATIO = 1.25; +export const DECLINING_SHARE_RATIO = 0.8; export const CONCENTRATION_WARNING_SHARE = 0.25; export const NEW_THEME_MIN_ARTICLES = 3; export const MIN_TERM_ARTICLES = 3; @@ -290,7 +290,7 @@ function countStates(docs) { .sort((a, b) => b.count - a.count || a.value.localeCompare(b.value)); } -function movementClass(recentCount, baselineCount, ratio) { +function movementClass(recentCount, baselineCount, shareRatio) { if (recentCount === 0 && baselineCount === 0) { return 'insufficient'; } @@ -303,13 +303,13 @@ function movementClass(recentCount, baselineCount, ratio) { if (recentCount < MIN_MOVEMENT_OBSERVATIONS || baselineCount < MIN_MOVEMENT_OBSERVATIONS) { return 'insufficient'; } - if (ratio === null) { + if (shareRatio === null) { return 'insufficient'; } - if (ratio >= GROWING_RATIO) { + if (shareRatio >= GROWING_SHARE_RATIO) { return 'growing'; } - if (ratio <= DECLINING_RATIO) { + if (shareRatio <= DECLINING_SHARE_RATIO) { return 'declining'; } return 'stable'; @@ -834,6 +834,7 @@ function summarizeMovement(docs, recent, comparison, rules) { const ratio = baselinePerDay === 0 ? null : recentPerDay / baselinePerDay; const shareRecent = recentDocs.length === 0 ? 0 : recentCount / recentDocs.length; const shareBaseline = baselineDocs.length === 0 ? 0 : baselineCount / baselineDocs.length; + const shareRatio = shareBaseline === 0 ? null : shareRecent / shareBaseline; return { theme, recentCount, @@ -841,14 +842,18 @@ function summarizeMovement(docs, recent, comparison, rules) { recentPerDay: roundTo(recentPerDay, 3), baselinePerDay: roundTo(baselinePerDay, 3), ratio: ratio === null ? null : roundTo(ratio, 2), + shareRatio: shareRatio === null ? null : roundTo(shareRatio, 2), shareRecent: roundTo(shareRecent, 3), shareBaseline: roundTo(shareBaseline, 3), deltaShare: roundTo(shareRecent - shareBaseline, 3), - class: movementClass(recentCount, baselineCount, ratio), + class: movementClass(recentCount, baselineCount, shareRatio), }; }) .sort((a, b) => b.recentCount - a.recentCount || a.theme.localeCompare(b.theme)); + const corpusRecentPerDay = recentDocs.length / recent.days; + const corpusBaselinePerDay = baselineDocs.length / baselineWindow.days; + return { baseline: { window: baselineWindow, @@ -856,6 +861,8 @@ function summarizeMovement(docs, recent, comparison, rules) { nested, minObservations: MIN_MOVEMENT_OBSERVATIONS, }, + corpusRateRatio: + corpusBaselinePerDay === 0 ? null : roundTo(corpusRecentPerDay / corpusBaselinePerDay, 2), themes, }; } @@ -1112,6 +1119,9 @@ function formatMovement(movement) { const baseline = movement.baseline; const lines = [ `- Baseline window: ${baseline.window.start} to ${baseline.window.end} (${baseline.window.days} days, ${baseline.articleCount} articles)`, + movement.corpusRateRatio === null + ? '- Corpus publication rate: not comparable (no baseline articles).' + : `- Corpus publication rate: ${movement.corpusRateRatio}x the baseline. Themes are classified on share, so a corpus-wide shift does not read as a theme trend.`, `- A theme needs at least ${baseline.minObservations} articles in both windows before its movement is classified; below that it reads \`insufficient\`.`, ]; if (movement.themes === null) { @@ -1120,9 +1130,11 @@ function formatMovement(movement) { } lines.push(''); for (const item of movement.themes) { - const ratio = item.ratio === null || item.class === 'insufficient' ? 'n/a' : `${item.ratio}x`; + const unclassified = item.class === 'insufficient'; + const rate = item.ratio === null || unclassified ? 'n/a' : `${item.ratio}x`; + const share = item.shareRatio === null || unclassified ? 'n/a' : `${item.shareRatio}x`; lines.push( - `- ${item.theme}: recent ${item.recentCount}, baseline ${item.baselineCount}, rate ratio ${ratio}, ${item.class}`, + `- ${item.theme}: recent ${item.recentCount}, baseline ${item.baselineCount}, rate ${rate}, share ${share}, ${item.class}`, ); } return lines.join('\n'); diff --git a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs index c106d44f..4cf68d99 100644 --- a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs +++ b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs @@ -968,3 +968,58 @@ describe('emergent vocabulary floors', () => { assert.ok(tokens.includes('permissions')); }); }); + +describe('corpus-adjusted movement', () => { + const make = (prefix, count, date, title) => + Array.from({ length: count }, (_, index) => ({ + id: `${prefix}${index}`, + title, + createdDate: `${date}T10:00:00Z`, + })); + + function report() { + const docs = [ + ...make('r-alpha-', 5, '2026-10-01', 'alpha question'), + ...make('r-filler-', 5, '2026-10-02', 'zzz'), + ...make('b-alpha-', 10, '2026-08-01', 'alpha question'), + ...make('b-filler-', 30, '2026-08-02', 'zzz'), + ]; + const directory = mkdtempSync(join(tmpdir(), 'rhdh-kb-share-')); + const file = join(directory, 'taxonomy.json'); + writeFileSync(file, JSON.stringify([{ theme: 'alpha', pattern: '\\balpha\\b' }])); + + return summarize( + { docs, numFound: docs.length, truncated: false }, + parseArgs([ + '--as-of', + '2026-10-30', + '--recent-days', + '50', + '--comparison-days', + '100', + '--themes', + file, + ]), + ); + } + + it('classifies on share when the whole corpus slows down', () => { + const result = report(); + const alpha = result.movement.themes[0]; + + assert.equal(alpha.recentCount, 5); + assert.equal(alpha.baselineCount, 10); + assert.equal(alpha.ratio, 0.5, 'absolute rate halved'); + assert.equal(alpha.shareRatio, 2, 'share doubled'); + assert.equal(alpha.class, 'growing'); + }); + + it('reports the corpus-wide publication rate change', () => { + const result = report(); + assert.equal(result.movement.corpusRateRatio, 0.25); + + const markdown = renderMarkdown(result); + assert.ok(markdown.includes('Corpus publication rate: 0.25x')); + assert.ok(markdown.includes('rate 0.5x, share 2x, growing')); + }); +}); From 0668f242403df07a31af48e37959ca4ac9e5c676 Mon Sep 17 00:00:00 2001 From: mattreid Date: Mon, 9 Nov 2026 21:51:26 -0500 Subject: [PATCH 6/6] feat(release): tighten the built-in taxonomy to built-in-v2 Bare product nouns were doing the theme work. plugins? alone appeared in 40% of the corpus, taking upgrade-plugin-reliability to 66% of a window and inverting the demand ranking; catalogs? found articles about the software catalog rather than catalog scale, so catalog-database-scale claimed 39 articles and 22 attributed cases. Version 2 requires plugin-lifecycle context and catalog or data-layer context, and drops ui, custom, entity, large, and many. Shares now top out at 41%, unmatched rises from 1 to 5 of 73 comparison articles. The report names the version so runs are never compared across sets. Assisted-By: Codex --- .../references/taxonomy.md | 10 ++++ .../scripts/analyze-kb.mjs | 10 ++-- .../tests/analyze-kb.test.mjs | 54 ++++++++++++++++++- 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md b/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md index 65d317a0..00889b91 100644 --- a/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md +++ b/skills/release/rhdh-kb-theme-analysis/references/taxonomy.md @@ -23,6 +23,16 @@ Keep it when the question is a general "what are customers hitting with RHDH" scan. It is shaped by install, upgrade, and configuration support, which is where most RHDH articles sit. +The set is versioned, and the report names the version it used — currently +`built-in-v2`. Version 1 matched bare `plugins?`, `catalogs?`, `ui`, `custom`, +`entity`, `large`, and `many`. Those are the product's own nouns rather than a +theme: `plugins?` alone appeared in 40% of the corpus, so +`upgrade-plugin-reliability` matched two thirds of a window, and +`catalog-database-scale` was mostly finding articles about the software catalog. +Version 2 requires plugin-lifecycle context, and catalog or data-layer context. +Never compare counts across the two versions, or across any two runs whose +`taxonomy.name` differs. + ## No taxonomy `--themes none` skips clustering entirely: the report carries `themes: null` and diff --git a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs index 9c2dd0ec..00a38a12 100644 --- a/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs +++ b/skills/release/rhdh-kb-theme-analysis/scripts/analyze-kb.mjs @@ -17,7 +17,7 @@ const BUILT_IN_THEME_RULES = [ { theme: 'upgrade-plugin-reliability', pattern: - /\b(?:upgrad(?:e|es|ed|ing)|plugins?|dynamic plugins?|install-dynamic-plugins|catalog indexes?|orchestrators?|modules?|dependenc(?:y|ies)|plugin versions?|version resolutions?|version mismatches?|compatibility matrix)\b/i, + /\b(?:upgrad(?:e|es|ed|ing)|dynamic plugins?|install-dynamic-plugins|plugin versions?|plugin (?:dependency|dependencies|loading|installation)|catalog indexes?|orchestrators?|modules?|dependenc(?:y|ies)|version (?:resolutions?|mismatches?|compatibility)|compatibility matrix)\b/i, }, { theme: 'authentication-identity', @@ -32,12 +32,12 @@ const BUILT_IN_THEME_RULES = [ { theme: 'rbac-permissions', pattern: - /\b(?:rbac|permissions?|roles?|policies?|admins?|visibility|entit(?:y|ies)|owned|access(?:es)?|groups?)\b/i, + /\b(?:rbac|permissions?|roles?|policies?|admins?|visibility|owned|access(?:es)?|groups?)\b/i, }, { theme: 'configuration-ui', pattern: - /\b(?:configs?|configurations?|customizations?|custom|yamls?|schemas?|widgets?|forms?|templates?|ui|themes?|logos?|sidebars?|menus?|localizations?)\b/i, + /\b(?:configs?|configurations?|customizations?|yamls?|schemas?|widgets?|forms?|templates?|themes?|logos?|sidebars?|menus?|localizations?)\b/i, }, { theme: 'observability-diagnostics', @@ -52,7 +52,7 @@ const BUILT_IN_THEME_RULES = [ { theme: 'catalog-database-scale', pattern: - /\b(?:catalogs?|databases?|postgres(?:ql)?|replicas?|(?:auto|down)?scal(?:e|es|ed|ing)|large(?:r|st)?|many|ingestions?|schedulers?|scheduling|syncs?|syncing|synced|synchronizations?)\b/i, + /\b(?:databases?|postgres(?:ql)?|replicas?|ingestions?|(?:auto|down)?scal(?:e|es|ed|ing)|catalog (?:ingestion|processor|processing|providers?|sync|index)|(?:catalog|entity) providers?|(?:catalog|entity) (?:syncs?|syncing|synchron\w+))\b/i, }, ]; @@ -70,7 +70,7 @@ export const ABSTRACT_LIMIT = 300; const BUILT_IN_TAXONOMY = { mode: 'built-in', - name: 'built-in', + name: 'built-in-v2', rules: BUILT_IN_THEME_RULES, }; diff --git a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs index 4cf68d99..2494c88e 100644 --- a/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs +++ b/skills/release/rhdh-kb-theme-analysis/tests/analyze-kb.test.mjs @@ -204,7 +204,7 @@ describe('analysis', () => { assert.ok(themeHints({ title: 'scales' }).includes('catalog-database-scale')); assert.ok(themeHints({ title: 'autoscaling' }).includes('catalog-database-scale')); - const incompatible = themeHints({ title: 'incompatible plugin' }); + const incompatible = themeHints({ title: 'incompatible dynamic plugins' }); assert.ok(incompatible.includes('upgrade-plugin-reliability')); assert.ok(incompatible.includes('support-version-clarity')); }); @@ -1023,3 +1023,55 @@ describe('corpus-adjusted movement', () => { assert.ok(markdown.includes('rate 0.5x, share 2x, growing')); }); }); + +describe('built-in taxonomy v2', () => { + const args = parseArgs(['--as-of', '2026-10-30']); + + it('names the version that produced the numbers', () => { + const report = summarize( + { + docs: [{ id: '1', title: 'Upgrade fails', createdDate: '2026-10-01T10:00:00Z' }], + numFound: 1, + truncated: false, + }, + args, + ); + assert.equal(report.taxonomy.name, 'built-in-v2'); + assert.ok(renderMarkdown(report).includes('Taxonomy: built-in-v2')); + }); + + it('stops treating the product nouns and generic adjectives as a catalog theme', () => { + for (const title of [ + 'Large deployments with many users', + 'Catalog entity page shows no metadata', + 'Sync from Git fails', + ]) { + assert.ok( + !themeHints({ title }).includes('catalog-database-scale'), + `should not match catalog-database-scale: ${title}`, + ); + } + }); + + it('still matches catalog ingestion, scaling, and the data layer', () => { + for (const title of [ + 'Catalog ingestion fails for large entity sets', + 'PostgreSQL replica lag in the catalog database', + 'Entity provider sync stalls', + 'Autoscaling the catalog processor', + ]) { + assert.ok( + themeHints({ title }).includes('catalog-database-scale'), + `should match catalog-database-scale: ${title}`, + ); + } + }); + + it('stops treating a bare plugin or ui mention as its theme', () => { + assert.ok(!themeHints({ title: 'Community plugins overview' }).includes('upgrade-plugin-reliability')); + assert.ok(themeHints({ title: 'Dynamic plugins fail to load' }).includes('upgrade-plugin-reliability')); + assert.ok(!themeHints({ title: 'UI terminology conflict' }).includes('configuration-ui')); + assert.ok(!themeHints({ title: 'Custom image support' }).includes('configuration-ui')); + assert.ok(themeHints({ title: 'Widget layout is wrong' }).includes('configuration-ui')); + }); +});