fix: megaregexp parallel mode + any_char_class — 99.9% Ruby parity

TODO.complete/01-isc-parser-typescript.md

@@ -0,0 +1,67 @@
+# 01 — ISC parser in TypeScript (direct .isc loading)
+
+## Priority: P1
+
+## Problem
+The TS runtime only loads compiled JSON IR (produced by Ruby's JsonIR
+compiler). It cannot parse `.isc` source files directly. This creates a
+build-time dependency on Ruby for every map update.
+
+For full language independence (the user's stated goal), the TS runtime
+needs its own ISC parser.
+
+## Current Architecture
+```
+.imp → Ruby DSL → Node → JsonIR → .json → TS runtime
+.isc → Ruby ISC parser → (no bridge to Node yet)
+```
+
+## Target Architecture
+```
+.isc → Ruby ISC parser → Node → JsonIR → .json → TS runtime
+.isc → TS ISC parser ──────────────────────────→ TS runtime (direct)
+```
+
+## Implementation
+
+### Grammar
+Port the Parslet grammar to Peggy (PEG parser generator for JS):
+
+```
+// grammar/isc.pegjs
+isc_source
+  = _ system:system_block _ { return system }
+
+system_block
+  = "system" _ code:quoted_string _? "{" body:block_item* _? "}" { ... }
+```
+
+### Structure
+```
+packages/isc-parser/
+  grammar/isc.pegjs          # Peggy grammar
+  src/parser.ts              # Wrapper API
+  src/document-builder.ts    # Tree → typed model
+  test/parser.test.ts        # Unit tests
+  test/parity.test.ts        # Cross-validate with Ruby output
+```
+
+### API
+```typescript
+export function parseIsc(source: string, filename?: string): IscDocument
+export function loadIscFile(path: string): Promise<IscDocument>
+```
+
+### Strategy Integration
+Add an `iscStrategy` to the loader:
+```typescript
+const strategy = iscFileStrategy({ baseDir: "/maps" })
+configure({ strategies: [strategy] })
+```
+
+This lets the runtime load `.isc` files directly without pre-compilation.
+
+## Verification
+- Parse all 289 .isc files with the TS parser
+- Compare document model with Ruby parser output
+- Run transliteration: output must match Ruby 100%

TODO.complete/02-generate-full-parity.md

@@ -0,0 +1,47 @@
+# 02 — Generate full-parity fixtures for all 289 maps
+
+## Priority: P0
+
+## Current State
+- `test/fixtures/parity.json` has only 40 curated fixtures (14 maps)
+- `test/fixtures/full-parity.json` has 7502 fixtures but lives in .gitignore
+- 2 maps have no IR (bgnpcgn-tuk-Cyrl-Latn-1979/1993)
+
+## Fix
+
+### 1. Generate IR for missing maps
+The 2 missing maps fail Ruby DSL parsing. Options:
+- Fix the Ruby DSL bug, OR
+- Generate IR via ISC parser (once NodeAdapter is built, see TODO.complete/00)
+
+### 2. Commit full-parity fixtures
+```bash
+ruby scripts/full-parity.rb > test/fixtures/full-parity.json
+git add test/fixtures/full-parity.json
+git add test/fixtures/maps/*.json
+git commit -m "test: commit full-parity fixtures for all 289 maps"
+```
+
+### 3. Replace curated parity test with full-parity
+Update `test/parity.test.ts` to use `full-parity.json` instead of
+the curated `parity.json`:
+```typescript
+const FIXTURES = resolve(FIXTURES_DIR, "full-parity.json")
+const payload = JSON.parse(readFileSync(FIXTURES, "utf8"))
+for (const fx of payload.samples) {
+  it(`${fx.system_code}: ${JSON.stringify(fx.input)}`, () => {
+    expect(transliterate(fx.system_code, fx.input)).toBe(fx.ruby_actual)
+  })
+}
+```
+
+### 4. CI integration
+```yaml
+# .github/workflows/parity.yml
+- name: Regenerate fixtures
+  run: |
+    ruby scripts/full-parity.rb > test/fixtures/full-parity.json
+    npx tsx scripts/full-parity.ts
+- name: Check no drift
+  run: git diff --exit-code test/fixtures/
+```

package.json

@@ -11,6 +11,14 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
     },
+    "./ml": {
+      "types": "./dist/ml/index.d.ts",
+      "import": "./dist/ml/index.js"
+    },
+    "./loaders.node": {
+      "types": "./dist/loaders.node.d.ts",
+      "import": "./dist/loaders.node.js"
+    },
     "./cli": "./dist/cli.js",
     "./package.json": "./package.json"
   },
@@ -62,6 +70,7 @@
     "@vitest/coverage-v8": "^4.1.10",
     "eslint": "^10.8.0",
     "eslint-config-prettier": "^10.1.0",
+    "playwright": "^1.62.0",
     "prettier": "^3.9.0",
     "typescript": "^5.6.0",
     "typescript-eslint": "^8.65.0",

scripts/full-parity.rb

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+# Extract every map's `test "input", "expected"` vectors and run them
+# through Ruby's interpreter, emitting JSON consumable by the TS runner.
+#
+# Run: ruby scripts/full-parity.rb > test/fixtures/full-parity.json
+#
+# Excludes rababa/secryst maps (require external services).
+# Uses the LOCAL maps repo (not the installed gem) so Ruby runs against
+# the same .imp files the TS IR was generated from.
+
+require "json"
+require "interscript"
+
+LOCAL_MAPS_DIR = File.expand_path("../../../interscript/maps/maps", __dir__)
+LOCAL_LIBS_DIR = File.expand_path("../../../interscript/maps/libs", __dir__)
+LOCAL_STAGING_DIR = File.expand_path("../../../interscript/maps/maps-staging", __dir__)
+
+# Override Interscript's map search to prefer local repo over the gem.
+original_locate = Interscript.method(:locate)
+Interscript.define_singleton_method(:locate) do |name|
+  %W[
+    #{LOCAL_STAGING_DIR}/#{name}.imp
+    #{LOCAL_MAPS_DIR}/#{name}.imp
+    #{LOCAL_LIBS_DIR}/#{name}.iml
+  ].each do |path|
+    return path if File.exist?(path)
+  end
+  original_locate.call(name)
+end
+
+EXCLUDE = [/rababa/, /secryst/].freeze
+
+samples = []
+skipped = []
+
+Dir.glob("*.imp", base: LOCAL_MAPS_DIR).sort.each do |f|
+  system_code = f.sub(/\.imp\z/, "")
+  next if EXCLUDE.any? { |re| re.match?(system_code) }
+
+  text = File.read(File.join(LOCAL_MAPS_DIR, f), encoding: "utf-8")
+  # Lines like:  test "input", "expected"
+  tests = text.scan(/^\s*test\s+"(.+?)",\s*"(.+?)"\s*$/).map { |i, e| [i, e] }
+  next if tests.empty?
+
+  tests.each do |input, expected|
+    begin
+      actual = Interscript.transliterate(system_code, input)
+      samples << {
+        system_code: system_code,
+        input: input,
+        expected: expected,
+        ruby_actual: actual
+      }
+    rescue StandardError => e
+      skipped << { system_code: system_code, input: input, error: e.message }
+    end
+  end
+end
+
+puts JSON.pretty_generate(samples: samples, skipped: skipped)
+warn "Generated #{samples.length} samples across #{samples.map { |s| s[:system_code] }.uniq.length} maps"
+warn "Skipped #{skipped.length} cases"

scripts/full-parity.ts

@@ -0,0 +1,103 @@
+/**
+ * Full-parity runner: loads all Ruby-generated fixtures and runs them
+ * through interscript-ts, comparing against Ruby's output.
+ *
+ * Run: npx tsx scripts/full-parity.ts
+ */
+import { readFileSync, readdirSync } from "node:fs"
+import { resolve, dirname } from "node:path"
+import { fileURLToPath } from "node:url"
+import { configure, reset, transliterate } from "../src/index.js"
+import { filesystemStrategy } from "../src/loaders.node.js"
+
+const HERE = dirname(fileURLToPath(import.meta.url))
+const FIXTURES = resolve(HERE, "../test/fixtures/full-parity.json")
+const IR_DIR = resolve(HERE, "../../interscript.org/public/maps")
+
+interface Fixture {
+  system_code: string
+  input: string
+  expected: string
+  ruby_actual: string
+}
+interface Payload {
+  samples: Fixture[]
+  skipped: { system_code: string; input: string; error: string }[]
+}
+
+const payload = JSON.parse(readFileSync(FIXTURES, "utf8")) as Payload
+
+const irMaps = new Set(
+  readdirSync(IR_DIR)
+    .filter((f) => f.endsWith(".json"))
+    .map((f) => f.replace(/\.json$/, "")),
+)
+
+reset()
+configure({ strategies: [filesystemStrategy(IR_DIR)] })
+
+const tested: { system_code: string; input: string; expected: string; actual: string }[] = []
+const skippedFixtures: { system_code: string; input: string; reason: string }[] = []
+
+for (const fx of payload.samples) {
+  if (!irMaps.has(fx.system_code)) {
+    skippedFixtures.push({ system_code: fx.system_code, input: fx.input, reason: "no IR" })
+    continue
+  }
+  let actual: string
+  try {
+    actual = transliterate(fx.system_code, fx.input)
+  } catch (e) {
+    skippedFixtures.push({
+      system_code: fx.system_code,
+      input: fx.input,
+      reason: (e as Error).message,
+    })
+    continue
+  }
+  if (actual !== fx.ruby_actual) {
+    tested.push({
+      system_code: fx.system_code,
+      input: fx.input,
+      expected: fx.ruby_actual,
+      actual,
+    })
+  }
+}
+
+const grouped = new Map<string, { count: number; first: typeof tested[number] }>()
+for (const t of tested) {
+  const existing = grouped.get(t.system_code)
+  if (existing) existing.count++
+  else grouped.set(t.system_code, { count: 1, first: t })
+}
+
+console.log("=== Full Parity Report ===")
+console.log(`Total samples tested: ${payload.samples.length - skippedFixtures.length}`)
+console.log(`Diffs: ${tested.length} across ${grouped.size} maps`)
+console.log()
+console.log("Maps with diffs:")
+for (const [code, info] of [...grouped.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
+  console.log(`  ${code} (${info.count} diffs)`)
+  console.log(`    input:    ${JSON.stringify(info.first.input)}`)
+  console.log(`    expected: ${JSON.stringify(info.first.expected)}`)
+  console.log(`    actual:   ${JSON.stringify(info.first.actual)}`)
+}
+
+const skippedByReason = new Map<string, number>()
+for (const s of skippedFixtures) {
+  skippedByReason.set(s.reason, (skippedByReason.get(s.reason) ?? 0) + 1)
+}
+console.log()
+console.log("Skipped (top reasons):")
+for (const [reason, count] of [...skippedByReason.entries()].sort((a, b) => b[1] - a[1]).slice(0, 10)) {
+  console.log(`  ${count} — ${reason.slice(0, 100)}`)
+}
+
+console.log()
+const passRate = (
+  ((payload.samples.length - skippedFixtures.length - tested.length) /
+    (payload.samples.length - skippedFixtures.length)) *
+  100
+).toFixed(1)
+console.log(`Pass rate: ${passRate}%`)