From 45e3df9822d6444c5f0c0d0f47818ca52ea96e3a Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 27 Aug 2026 12:24:51 -0500 Subject: [PATCH 1/3] fix(web): support autocorrection of transposed text Fixes: #16398 Fixes: #12311 Note: will not handle transpositions that cross token boundaries. Also note: major changes will be needed for epic/boundary-correction, as it will discontinue use of the LegacyQuotientSpur type. Build-bot: skip release:web,android,ios --- .../src/main/correction/distance-modeler.ts | 12 ++- .../main/correction/legacy-quotient-spur.ts | 77 ++++++++++++++++-- .../worker-thread/src/main/predict-helpers.ts | 25 +++--- .../correction-search/getBestMatches.tests.ts | 6 +- .../prediction-helpers/auto-correct.tests.ts | 79 ------------------- 5 files changed, 97 insertions(+), 102 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index b69e6fd2d53..dac322e7670 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -103,6 +103,12 @@ export interface PartialSearchEdge { * more steps until valid search endpoints are reached. */ export class SearchNode { + /** + * Denotes any additional edit-cost components not modeled by the core edit-distance + * computation object. + */ + private addedEditCost = 0; + /** * The search-term keying method used by the active LexicalModel * @param str @@ -206,7 +212,7 @@ export class SearchNode { * by the current node. */ get editCount(): number { - return this.calculation.getHeuristicFinalCost() + this.deleteAfterInsertEditPairs; + return this.calculation.getHeuristicFinalCost() + this.deleteAfterInsertEditPairs + this.addedEditCost; } /** @@ -271,6 +277,10 @@ export class SearchNode { return EDIT_DISTANCE_COST_SCALE * this.editCount + this.inputSamplingCost; } + addEdit() { + this.addedEditCost++; + } + /** * Adds outbound paths from the current Node that model the insertion of a * character not seen in the input, as if the user accidentally skipped typing diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts index 89ce2e1b14f..9f7ddbe5054 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts @@ -9,9 +9,9 @@ */ import { LexicalModelTypes } from '@keymanapp/common-types'; -import { KMWString } from 'keyman/common/web-utils'; +import { KMWString, PriorityQueue } from 'keyman/common/web-utils'; -import { PathResult } from './correction-searchable.js'; +import { CORRECTION_QUEUE_COMPARATOR, PathResult } from './correction-searchable.js'; import { SearchNode } from './distance-modeler.js'; import { SearchQuotientNode, PathInputProperties } from './search-quotient-node.js'; import { SearchQuotientSpur } from './search-quotient-spur.js'; @@ -24,6 +24,9 @@ import Transform = LexicalModelTypes.Transform; // The set of search spaces corresponding to the same 'context' for search. // Whenever a wordbreak boundary is crossed, a new instance should be made. export class LegacyQuotientSpur extends SearchQuotientSpur { + private transposeQueue: PriorityQueue = new PriorityQueue(CORRECTION_QUEUE_COMPARATOR); + private incomingTransposeRootNodes: TokenResultMapping[] = []; + public readonly insertLength: number; public readonly leftDeleteLength: number; @@ -44,14 +47,22 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { super(space, inputs, inputSource, codepointLength); this.insertLength = insertLength; this.leftDeleteLength = inputSample.deleteLeft; - return; + + // Link to the grandparent node if it exists; transposes start construction rooted there. + const grandparentNode = this.parents[0].parents[0] + if(grandparentNode) { + this.incomingTransposeRootNodes = [...grandparentNode.previousResults]; + this.linkAndQueueFromParent(grandparentNode, this.incomingTransposeRootNodes); + } } construct(parentNode: SearchQuotientNode, inputs?: Distribution, inputSource?: PathInputProperties): this { return new LegacyQuotientSpur(parentNode, inputs, inputSource) as this; } - protected buildEdgesFromResults(priorResults: ReadonlyArray): SearchNode[] { + protected buildEdgesFromResults(priorResults: ReadonlyArray, inputs?: Distribution): SearchNode[] { + const edgeInputs = inputs ?? this.inputs; + // With a newly-available input, we can extend new input-dependent paths from // our previously-reached 'extractedResults' nodes. let outboundNodes = priorResults.map((result) => { @@ -61,9 +72,9 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { let deletionEdges: SearchNode[] = []; if(!substitutionsOnly) { - deletionEdges = result.buildDeletionEdges(this.inputs, this.spaceId); + deletionEdges = result.buildDeletionEdges(edgeInputs, this.spaceId); } - const substitutionEdges = result.buildSubstitutionEdges(this.inputs, this.spaceId); + const substitutionEdges = result.buildSubstitutionEdges(edgeInputs, this.spaceId); // Skip the queue for the first pass; there will ALWAYS be at least one pass, // and queue-enqueing does come with a cost - avoid unnecessary overhead here. @@ -73,6 +84,13 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { return outboundNodes; } + get currentCost() { + const defaultCost = super.currentCost; + const transposeCost = this.transposeQueue.peek()?.currentCost ?? Number.POSITIVE_INFINITY; + + return Math.min(transposeCost, defaultCost); + } + /** * Retrieves the lowest-cost / lowest-distance edge from the selection queue, * checks its validity as a correction to the input text, and reports on what @@ -80,6 +98,42 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { * @returns */ public handleNextNode(): PathResult { + this.processPendingRoots(); + const transposeCost = this.transposeQueue.peek()?.currentCost ?? Number.POSITIVE_INFINITY; + + // Handle transposition cases + if(transposeCost < super.currentCost) { + let currentNode = this.transposeQueue.dequeue(); + + let unmatchedResult: PathResult = { + type: 'intermediate', + cost: currentNode.currentCost + } + + // Stage 1: filter out nodes/edges we want to prune + + // Forbid a raw edit-distance of greater than 2. + // Note: .knownCost is not scaled, while its contribution to .currentCost _is_ scaled. + if(currentNode.editCount > 2) { + return unmatchedResult; + } + + // Stage 2: process subset further OR build remaining edges + + if(currentNode.hasPartialInput) { + // Re-use the current queue; the number of total inputs considered still holds. + this.transposeQueue.enqueueAll(currentNode.processSubsetEdge()); + return unmatchedResult; + } + + // If here, we've properly done the first half of a transpose. Now for the other half... + + // const transposeSecondHalfNodes = currentNode.buildSubstitutionEdges((this.parents[0] as LegacyQuotientSpur).inputs, this.spaceId); + const transposeSecondHalfNodes = this.buildEdgesFromResults([new TokenResultMapping(this, currentNode)], (this.parents[0] as LegacyQuotientSpur).inputs); + this.queueNodes(transposeSecondHalfNodes); + return unmatchedResult; + } + const result = super.handleNextNode(); if(result.type == 'complete') { @@ -95,4 +149,15 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { return result; } + + protected processPendingRoots(): void { + super.processPendingRoots(); + + while(this.incomingTransposeRootNodes.length > 0) { + // Build only substitution edges from these. + const transpositionFirstHalves = this.incomingTransposeRootNodes.pop().buildSubstitutionEdges(this.inputs, this.spaceId); + transpositionFirstHalves.forEach((n) => n.addEdit()); + this.transposeQueue.enqueueAll(transpositionFirstHalves); + } + } } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index aca78eaf599..f6ff401e123 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -662,7 +662,17 @@ export async function correctAndEnumerate( continue; } - if(match.editCount > 0 && !searchModules.find(s => s.correctionsEnabled)) { + // In the case of a backspace, we wipe out the original form of the search + // module and replace it with a format that also signals that corrections + // aren't enabled. + // + // To resolve this, we check the pre-transition form in order to check if + // corrections were enabled before a backspace. + const correctionsWereEnabled = transition.base.displayTokenization.tail.searchModule.correctionsEnabled; + if(match.editCount > 0 + && !searchModules.find(s => s.correctionsEnabled) + && !(TransformUtils.isBackspace(inputTransform) && correctionsWereEnabled) + ) { continue; } @@ -1078,19 +1088,6 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio return; } - // Find the highest probability for any correction that led to a valid prediction. - // No need to full-on re-sort everything, though. - const bestCorrection = suggestionDistribution.reduce( - (prev, current) => prev?.correction.p > current.correction.p ? prev : current, - null - ).correction; - if(bestCorrection.p > bestSuggestion.correction.p) { - // Here, the best suggestion didn't come from the best correction. - // Is it actually reasonable to auto-correct? We're probably just very - // biased toward its frequency. (Maybe a threshold should be considered?) - return; - } - // If we allow an option to allow same-key suggestions to replace context automatically // - such as replacing `cant` with `can't` if the latter is much more frequent - // we may wish to group matchLevel values below by 'mapping' them with an appropriate diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts index 71e844a07fd..4bccef2ce91 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts @@ -76,14 +76,16 @@ describe('Correction Searching', () => { // 't' -> 'b' (sub) 'beh', // '' -> 'c' (insertion) - 'tech' + 'tech', + // 'eh' -> 'he' (transposition) + 'the' ]; await checkBatch(thirdBatch, secondCost); // All replace the low-likelihood case for the third input. const fourthBatch = [ - 'the', 'thi', 'tho', 'thr', + 'thi', 'tho', 'thr', 'thu', 'tha' ]; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts index a6e97907dc9..9306371712c 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts @@ -564,85 +564,6 @@ describe('predictionAutoSelect', () => { assert.equal(autoselected, expectedSuggestion); }); - // The idea: avoid "over-correcting" when a potential correction has a - // super-high-frequency word. - it('does not auto-select suggestion if its root correction is not most likely', () => { - const keepSuggestion: CorrectionPredictionTuple= { - correction: { - sample: 'thi', - p: .7 - }, - prediction: { - sample: { - tag: 'keep', - transform: { // can be null / "mocked out" - insert: 'i', - deleteLeft: 0 - }, - displayAs: '"thi"', - matchesModel: false - }, - p: .05 - }, - totalProb: .035, - metadata: {...defaultMetadata} - } - - const highestCorrectionSuggestion: CorrectionPredictionTuple= { - correction: { - sample: 'thi', - p: .7 - }, - prediction: { - sample: { - transform: { // can be null / "mocked out" - insert: 'in', - deleteLeft: 0 - }, - displayAs: 'thin' - }, - p: .1 - }, - totalProb: .07, - metadata: {...defaultMetadata} - }; - - const highestNonKeepSuggestion: CorrectionPredictionTuple= { - correction: { - sample: 'the', - p: .3 - }, - prediction: { - sample: { - transform: { // can be null / "mocked out" - insert: 'e', - deleteLeft: 0 - }, - displayAs: 'the' - }, - p: 1 - }, - totalProb: .3, - metadata: {...defaultMetadata} - }; - - const predictions: CorrectionPredictionTuple[] = [ - keepSuggestion, - highestNonKeepSuggestion, - highestCorrectionSuggestion - ]; - - const totalProb = predictions.reduce((accum, current) => accum + current.totalProb, 0); - assert.isAbove(highestNonKeepSuggestion.totalProb, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); - - const originalPredictions = [].concat(predictions); - assert.doesNotThrow(() => predictionAutoSelect(predictions)); - assert.sameDeepMembers(predictions, originalPredictions); - - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); - assert.isNotOk(autoselected); - }); - // // If we add a setting allowing 'exact', 'sameText', and 'sameKey' tiers to // // all compete equally, rather than having each instantly win over those // // after it, we'd want to add a test such as this. From 1c528dde8fb021fcad00fad46fca3afaea7dd84b Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 31 Aug 2026 21:27:00 +0700 Subject: [PATCH 2/3] change(web): apply MD suggestion from code review Co-authored-by: Marc Durdin --- .../worker-thread/src/main/correction/legacy-quotient-spur.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts index 9f7ddbe5054..b8d69921479 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts @@ -49,7 +49,7 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { this.leftDeleteLength = inputSample.deleteLeft; // Link to the grandparent node if it exists; transposes start construction rooted there. - const grandparentNode = this.parents[0].parents[0] + const grandparentNode = this.parents[0].parents[0]; if(grandparentNode) { this.incomingTransposeRootNodes = [...grandparentNode.previousResults]; this.linkAndQueueFromParent(grandparentNode, this.incomingTransposeRootNodes); From 60f5effe6ed921f91c525941e8ad48ad108b20ac Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 4 Sep 2026 16:50:56 -0500 Subject: [PATCH 3/3] change(web): add transposition unit tests, loosen search correction thresholding --- .../src/main/correction/distance-modeler.ts | 1 + .../main/correction/legacy-quotient-spur.ts | 61 ++++++---- .../worker-thread/src/main/predict-helpers.ts | 9 +- .../early-correction-search-stopping.tests.ts | 14 +-- .../legacy-quotient-spur.tests.ts | 112 +++++++++++++++++- 5 files changed, 162 insertions(+), 35 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index dac322e7670..b85f6b98c07 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -194,6 +194,7 @@ export class SearchNode { // This is unique at each level, though it will reuse a previous ID if no new // one is provided (say, for 'insert' edits). this.spaceId = spaceId ?? priorNode.spaceId; + this.addedEditCost = priorNode.addedEditCost; } else { this.calculation = new ClassicalDistanceCalculation(); this.matchedTraversals = [param1]; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts index 9f7ddbe5054..257c83cc7cc 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts @@ -61,27 +61,7 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { } protected buildEdgesFromResults(priorResults: ReadonlyArray, inputs?: Distribution): SearchNode[] { - const edgeInputs = inputs ?? this.inputs; - - // With a newly-available input, we can extend new input-dependent paths from - // our previously-reached 'extractedResults' nodes. - let outboundNodes = priorResults.map((result) => { - // Hard restriction: no further edits will be supported. This helps keep the search - // more narrowly focused. - const substitutionsOnly = result.editCount == 2; - - let deletionEdges: SearchNode[] = []; - if(!substitutionsOnly) { - deletionEdges = result.buildDeletionEdges(edgeInputs, this.spaceId); - } - const substitutionEdges = result.buildSubstitutionEdges(edgeInputs, this.spaceId); - - // Skip the queue for the first pass; there will ALWAYS be at least one pass, - // and queue-enqueing does come with a cost - avoid unnecessary overhead here. - return substitutionEdges.flatMap(e => e.processSubsetEdge()).concat(deletionEdges); - }).flat(); - - return outboundNodes; + return buildEdgesFromResults(priorResults, inputs ?? this.inputs, this.spaceId); } get currentCost() { @@ -153,11 +133,42 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { protected processPendingRoots(): void { super.processPendingRoots(); - while(this.incomingTransposeRootNodes.length > 0) { - // Build only substitution edges from these. - const transpositionFirstHalves = this.incomingTransposeRootNodes.pop().buildSubstitutionEdges(this.inputs, this.spaceId); - transpositionFirstHalves.forEach((n) => n.addEdit()); + if(this.incomingTransposeRootNodes.length > 0) { + const transpositionFirstHalves = processTransposeRoots(this.incomingTransposeRootNodes, this.inputs, this.spaceId); + + this.incomingTransposeRootNodes.splice(0, this.incomingTransposeRootNodes.length); this.transposeQueue.enqueueAll(transpositionFirstHalves); } } +} + +export function processTransposeRoots(priorResults: TokenResultMapping[], inputs: Distribution, spaceId: number) { + // Build only substitution edges from these. + const transpositionFirstHalves = priorResults + .flatMap((entry) => entry.buildSubstitutionEdges(inputs, spaceId)) + .flatMap(e => e.processSubsetEdge()); + transpositionFirstHalves.forEach((n) => n.addEdit()); + return transpositionFirstHalves; +} + +export function buildEdgesFromResults(priorResults: ReadonlyArray, inputs: Distribution, spaceId: number): SearchNode[] { + // With a newly-available input, we can extend new input-dependent paths from + // our previously-reached 'extractedResults' nodes. + let outboundNodes = priorResults.map((result) => { + // Hard restriction: no further edits will be supported. This helps keep the search + // more narrowly focused. + const substitutionsOnly = result.editCount == 2; + + let deletionEdges: SearchNode[] = []; + if(!substitutionsOnly) { + deletionEdges = result.buildDeletionEdges(inputs, spaceId); + } + const substitutionEdges = result.buildSubstitutionEdges(inputs, spaceId); + + // Skip the queue for the first pass; there will ALWAYS be at least one pass, + // and queue-enqueing does come with a cost - avoid unnecessary overhead here. + return substitutionEdges.flatMap(e => e.processSubsetEdge()).concat(deletionEdges); + }).flat(); + + return outboundNodes; } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 108cb197284..2ba308fdea0 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -21,7 +21,7 @@ import { ContextState, determineContextSlideTransform } from './correction/conte import { ContextTransition, TransitionReversionView } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import { ModelCompositor } from './model-compositor.js'; -import { getBestTokenMatches } from './correction/distance-modeler.js'; +import { EDIT_DISTANCE_COST_SCALE, getBestTokenMatches } from './correction/distance-modeler.js'; import CasingForm = LexicalModelTypes.CasingForm; import Context = LexicalModelTypes.Context; @@ -78,7 +78,12 @@ export const CORRECTION_SEARCH_THRESHOLDS = { * in log-space, the search would stop at a total cost of 1 + this value if * a "full" set of suggestions had already been found. */ - REPLACEMENT_SEARCH_THRESHOLD: 4 as const // e^-4 = 0.0183156388. Allows "80%" of an extra edit. + + // Ensure at least one "edit distance cost unit" so that even heavily + // fat-fingered transpositions have a chance. Note that the level is this + // applied, wordlist weightings have no effect and cannot prevent correction + // thresholding! + REPLACEMENT_SEARCH_THRESHOLD: EDIT_DISTANCE_COST_SCALE * 1.1 } /** diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts index 2afd5f5290a..6958c783a4d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts @@ -38,7 +38,11 @@ describe('correction-search: shouldStopSearchingEarly', () => { }); it('stops checking corrections earlier when enough predictions have been found', () => { - const predictionProbs = [.010, .009, .008, .008, .0075, .0075, .007, .007, .006, .006, .005, .005]; + // Thresholding is performed in log-space. + const baseCost = 1; + const expectedThreshold = CORRECTION_SEARCH_THRESHOLDS.REPLACEMENT_SEARCH_THRESHOLD; + + const predictionProbs = [.010, .009, .008, .008, .0075, .007, .006, .005, .004, .003, .002, Math.exp(- baseCost - expectedThreshold)]; assert.isAtLeast(predictionProbs.length, ModelCompositor.MAX_SUGGESTIONS, "test setup no longer valid"); // The only part for each entry we actually care about here: .totalProb. @@ -49,12 +53,8 @@ describe('correction-search: shouldStopSearchingEarly', () => { } as CorrectionPredictionTupleCore }); - const baseCost = 1; - - // Thresholding is performed in log-space. - const expectedThreshold = CORRECTION_SEARCH_THRESHOLDS.REPLACEMENT_SEARCH_THRESHOLD; - + // The actual assertions. assert.isFalse(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold - 0.01, predictions)); - assert.isTrue(shouldStopSearchingEarly( baseCost, baseCost + expectedThreshold + 0.01, predictions)); + assert.isTrue(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold + 0.01, predictions)); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts index 0989ab9d64e..07e048a46be 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts @@ -9,17 +9,23 @@ import { assert } from 'chai'; +import { LexicalModelTypes } from '@keymanapp/common-types'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { + buildEdgesFromResults, generateSubsetId, LegacyQuotientRoot, LegacyQuotientSpur, - models + models, + processTransposeRoots, + TokenResultMapping } from '@keymanapp/lm-worker/test-index'; import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; import { analyzeQuotientNodeResults } from '../../helpers/analyzeQuotientNodeResults.js'; +import Distribution = LexicalModelTypes.Distribution; +import Transform = LexicalModelTypes.Transform; import TrieModel = models.TrieModel; const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); @@ -322,4 +328,108 @@ describe('LegacyQuotientSpur', () => { assert.isEmpty(analysis.foundWithDuplicates); }); }); + + describe('transposition handling', () => { + const tehDistributions: Distribution[] = [ + [ + { sample: { insert: 't', deleteLeft: 0, id: 1 }, p: .55}, + { sample: { insert: 'r', deleteLeft: 0, id: 1 }, p: .45} + ], [ + { sample: { insert: 'e', deleteLeft: 0, id: 1 }, p: .9}, + { sample: { insert: 's', deleteLeft: 0, id: 1 }, p: .1} + ], [ + { sample: { insert: 'h', deleteLeft: 0, id: 1 }, p: .9}, + { sample: { insert: 'n', deleteLeft: 0, id: 1 }, p: .1} + ] + ]; + + it('corrects to `the` for a targeted, deep search for a `teh` transposition', () => { + const root = new LegacyQuotientRoot(testModel); + + const rootResults: TokenResultMapping[] = []; + while(root.currentCost < Number.POSITIVE_INFINITY) { + const result = root.handleNextNode(); + if(result.type == 'complete') { + rootResults.push(result.mapping); + } + } + + const entry_empty = rootResults.find((entry) => entry.matchString == '') + assert.isOk(entry_empty); + + const firstSpur = new LegacyQuotientSpur(root, tehDistributions[0], tehDistributions[0][0]); + const edgesFromEmpty = buildEdgesFromResults([entry_empty], firstSpur.inputs, firstSpur.spaceId); + const edge_t = edgesFromEmpty.find((entry) => entry.resultKey == 't'); + assert.isOk(edge_t); + + // now, try to do something with entry_t. + const secondSpur = new LegacyQuotientSpur(firstSpur, tehDistributions[1], tehDistributions[1][0]); + const thirdSpur = new LegacyQuotientSpur(secondSpur, tehDistributions[2], tehDistributions[2][0]); + + const entry_t = new TokenResultMapping(firstSpur, edge_t); + const transposeFirstHalves = processTransposeRoots([entry_t], thirdSpur.inputs, thirdSpur.spaceId); + + const edge_th = transposeFirstHalves.find((entry) => entry.resultKey == 'th' && entry.editCount == 1); + assert.isOk(edge_th); + + const entry_th = new TokenResultMapping(thirdSpur, edge_th); + const transposeSecondHalves = buildEdgesFromResults([entry_th], secondSpur.inputs, thirdSpur.spaceId); + const edge_the = transposeSecondHalves.find((entry) => entry.resultKey == 'the' && entry.editCount == 1); + assert.isOk(edge_the); + }); + + it('corrects to `the` for an sequential, broad search for a `teh` transposition', () => { + const root = new LegacyQuotientRoot(testModel); + + const rootResults: TokenResultMapping[] = []; + while(root.currentCost < Number.POSITIVE_INFINITY) { + const result = root.handleNextNode(); + if(result.type == 'complete') { + rootResults.push(result.mapping); + } + } + + const entry_empty = rootResults.find((entry) => entry.matchString == '') + assert.isOk(entry_empty); + + const firstSpur = new LegacyQuotientSpur(root, tehDistributions[0], tehDistributions[0][0]); + const firstResults: TokenResultMapping[] = []; + while(firstSpur.currentCost < Number.POSITIVE_INFINITY) { + const result = firstSpur.handleNextNode(); + if(result.type == 'complete') { + firstResults.push(result.mapping); + } + } + + const entry_t = firstResults.find((entry) => entry.matchString == 't' && entry.editCount == 0); + assert.isOk(entry_t); + + // now, try to do something with entry_t. + const secondSpur = new LegacyQuotientSpur(firstSpur, tehDistributions[1], tehDistributions[1][0]); + const secondResults: TokenResultMapping[] = []; + while(secondSpur.currentCost < Number.POSITIVE_INFINITY) { + const result = secondSpur.handleNextNode(); + if(result.type == 'complete') { + secondResults.push(result.mapping); + } + } + + const thirdSpur = new LegacyQuotientSpur(secondSpur, tehDistributions[2], tehDistributions[2][0]); + const thirdResults: TokenResultMapping[] = []; + while(thirdSpur.currentCost < Number.POSITIVE_INFINITY) { + const result = thirdSpur.handleNextNode(); + if(result.type == 'complete') { + thirdResults.push(result.mapping); + } + } + + const entry_the = thirdResults.find((entry) => entry.matchString == 'the' && entry.editCount == 1); + assert.isOk(entry_the); + + thirdResults.sort((a, b) => a.totalCost - b.totalCost); + const the_index = thirdResults.findIndex((entry) => entry.matchString == 'the' && entry.editCount == 1); + // `teh` should appear fairly early as a viable correction . + assert.isBelow(the_index, 10); + }); + }); }); \ No newline at end of file