test: assert the existing language fixtures

.gitattributes

@@ -1,2 +1,6 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+
+# Keep the fixtures and their snapshots byte for byte, they are compared as is
+examples/languages/** -text
+tests/snapshots/** -text

.github/workflows/test.yml

@@ -0,0 +1,27 @@
+name: Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: read # for checkout
+
+jobs:
+  test:
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Repo
+        uses: actions/checkout@v6
+
+      - name: Setup Node
+        uses: actions/setup-node@v6
+        with:
+          cache: npm
+          node-version: lts/*
+
+      - run: npm clean-install
+      - run: npm test

package.json

@@ -25,7 +25,8 @@
     "example": "examples"
   },
   "scripts": {
-    "build": "bash .github/workflows/build.sh"
+    "build": "bash .github/workflows/build.sh",
+    "test": "node --test tests/*.test.js"
   },
   "repository": {
     "type": "git",

tests/languages.test.js

@@ -0,0 +1,26 @@
+import { test } from 'node:test';
+import { strictEqual } from 'node:assert';
+import { readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { tokenize } from '../src/index.js';
+
+let fixtures = new URL('../examples/languages/', import.meta.url),
+	snapshots = new URL('./snapshots/', import.meta.url);
+
+for (let file of readdirSync(fixtures).filter(file => /^(test|oneline)\./.test(file)))
+	test(file, async () => {
+		let src = readFileSync(new URL(file, fixtures), 'utf8'),
+			snapshot = new URL(`${file}.txt`, snapshots),
+			tokens = [];
+
+		await tokenize(src, file.slice(file.indexOf('.') + 1), (str, token) => tokens.push([token, str]));
+
+		// checked before UPDATE so a refreshed snapshot can never bless dropped code
+		strictEqual(tokens.map(([, str]) => str).join(''), src);
+
+		let got = tokens.filter(([, str]) => str).map(([token, str]) => `${token ?? '.'}\t${JSON.stringify(str)}`).join('\n');
+
+		if (process.env.UPDATE)
+			writeFileSync(snapshot, got);
+		else
+			strictEqual(got, readFileSync(snapshot, 'utf8'));
+	});

tests/snapshots/oneline.bash.txt

@@ -0,0 +1,3 @@
+func	"echo"
+.	" "
+str	"\"hello world\""

tests/snapshots/oneline.http.txt

@@ -0,0 +1,2 @@
+kwd	"GET"
+.	" /path/to/file.js"

tests/snapshots/test.asm.txt

@@ -0,0 +1,83 @@
+func	"global"
+kwd	" main"
+.	"\n\n"
+func	"extern"
+kwd	" printf"
+.	"\n\n"
+func	"section"
+kwd	" .data"
+.	"\n"
+kwd	"\tfmt"
+.	": db "
+str	"'Res: %s'"
+.	","
+num	"0x0a"
+.	", "
+num	"0"
+.	"\n"
+kwd	"\tmsg"
+.	": db "
+str	"'Hello world!'"
+.	","
+num	"0"
+.	"\n\n"
+func	"section"
+kwd	" .text"
+.	"\n"
+kwd	"\tmain"
+.	":\n"
+kwd	"\tmov"
+.	" rsi, msg "
+cmnt	"; ABI is rdi, rsi"
+.	"\n"
+kwd	"\tmov"
+.	" rdi, fmt\n"
+kwd	"\tmov"
+.	" rax, "
+num	"0"
+.	"   "
+cmnt	"; rax is # of non-int args"
+.	"\n\n"
+kwd	"\tcall"
+.	" printf\n\n\t"
+cmnt	"# https://github.com/speed-highlight/core/issues/39"
+.	"\n"
+kwd	"\tvmovddup"
+.	"\t"
+num	"2536"
+.	"("
+oper	"%"
+.	"rbx,"
+oper	"%"
+.	"rax), "
+oper	"%"
+.	"xmm1          "
+cmnt	"# comment"
+.	"\n"
+kwd	"\tvmovaps"
+.	"\t"
+num	"2528"
+.	"("
+oper	"%"
+.	"rbx,"
+oper	"%"
+.	"rax), "
+oper	"%"
+.	"xmm3\n"
+kwd	"\tvfmadd213ps"
+.	"\t"
+oper	"%"
+.	"xmm6, "
+oper	"%"
+.	"xmm5, "
+oper	"%"
+.	"xmm3\n"
+kwd	"\tvpslld"
+.	"\t"
+num	"$23"
+.	", "
+oper	"%"
+.	"xmm4, "
+oper	"%"
+.	"xmm4\n\n"
+kwd	"\tret"

tests/snapshots/test.bash.txt

@@ -0,0 +1,142 @@
+cmnt	"#!/bin/bash"
+.	"\n\n"
+func	"git"
+.	" commit "
+kwd	"&&"
+.	" "
+func	"git"
+.	" push\n"
+func	"git"
+.	" commit "
+kwd	"||"
+.	" "
+func	"echo"
+.	" "
+str	"\"Commit failed\""
+.	"\n\n"
+func	"set"
+kwd	" -euo"
+.	" pipefail\n"
+var	"IFS"
+oper	"="
+.	"$"
+str	"'\\n\\t'"
+.	"\n\n"
+func	"npm"
+.	" "
+kwd	"install"
+.	" speed_highlight_js\n\n"
+var	"TEST"
+oper	"="
+num	"5"
+.	"\n\n"
+var	"HELLO_WORLD"
+oper	"="
+str	"\"John\""
+.	"\n"
+func	"echo"
+.	" "
+str	"\"Hi "
+var	"$NAME"
+str	"\""
+.	"  "
+cmnt	"#=> Hi John"
+.	"\n"
+func	"cat"
+.	" "
+str	"'Hi "
+var	"$NAME"
+str	"'"
+cmnt	"#=> "
+err	"TODO"
+cmnt	" Hi $NAME"
+.	"\n\n"
+func	"printf"
+.	" "
+str	"\"I'm in "
+var	"$(pwd)"
+str	"\""
+.	"\n"
+func	"echo"
+.	" "
+str	"\"I'm in `pwd`\""
+.	" "
+kwd	"|"
+.	" "
+func	"grep"
+.	"\n"
+cmnt	"# Same"
+.	"\n\n"
+kwd	"if"
+.	" [["
+kwd	" -z"
+.	" "
+str	"\""
+var	"$string"
+str	"\""
+.	" ]]"
+kwd	";"
+.	" "
+kwd	"then"
+.	"\n  "
+func	"cd"
+.	" "
+str	"\"String is empty\""
+.	"\n"
+kwd	"elif"
+.	" [["
+kwd	" -n"
+.	" "
+str	"\""
+var	"$string"
+str	"\""
+.	" ]]"
+kwd	";"
+.	" "
+kwd	"then"
+.	"\n  "
+func	"alias"
+.	" "
+str	"\"String is not empty\""
+.	"\n"
+kwd	"fi"
+.	"\n\n"
+func	"minify_file"
+.	" "
+oper	"()"
+.	" "
+oper	"{"
+.	"\n\t"
+var	"file"
+oper	"="
+var	"$1"
+.	"\n\t"
+var	"test"
+oper	"="
+.	"false\n\t"
+var	"out"
+oper	"="
+var	"${ file/${in_dir}"
+.	"/"
+var	"${out_dir}"
+.	" "
+oper	"}"
+.	"\n\t"
+func	"exec_minify_cmd"
+.	" "
+var	"$file"
+.	" "
+var	"$out"
+.	"\n\t"
+func	"test-hello"
+.	" "
+kwd	"&"
+.	"\n\t"
+oper	"./a.out"
+.	" test-hello\n\t"
+func	"cat"
+.	" "
+oper	"/etc/test"
+.	"\n"
+oper	"}"
+.	"\n"

tests/snapshots/test.bf.txt

@@ -0,0 +1,62 @@
+oper	"++"
+.	"       "
+cmnt	"Cell c0 = 2"
+.	"\n"
+kwd	">"
+.	" "
+oper	"+++++"
+.	"  "
+cmnt	"Cell c1 = 5"
+.	"\n\n[        "
+cmnt	"Start your loops with your cell pointer on the loop counter (c1 in our case)"
+.	"\n"
+kwd	"<"
+.	" "
+oper	"+"
+.	"      "
+cmnt	"Add 1 to c0"
+.	"\n"
+kwd	">"
+.	" "
+oper	"-"
+.	"      "
+cmnt	"Subtract 1 from c1"
+.	"\n]        "
+cmnt	"End your loops with the cell pointer on the loop counter"
+.	"\n\n"
+cmnt	"At this point our program has added 5 to 2 leaving 7 in c0 and 0 in c1"
+.	"\n"
+cmnt	"but we cannot output this value to the terminal since it is not ASCII encoded."
+.	"\n\n"
+cmnt	"To display the ASCII character \"7\" we must add 48 to the value 7."
+.	"\n"
+cmnt	"We use a loop to compute 48 = 6 * 8."
+.	"\n\n"
+oper	"++++"
+.	" "
+oper	"++++"
+.	"  "
+cmnt	"c1 = 8 and this will be our loop counter again"
+.	"\n[\n"
+kwd	"<"
+.	" "
+oper	"+++"
+.	" "
+oper	"+++"
+.	"  "
+cmnt	"Add 6 to c0"
+.	"\n"
+kwd	">"
+.	" "
+oper	"-"
+.	"        "
+cmnt	"Subtract 1 from c1"
+.	"\n]\n"
+kwd	"<"
+.	" "
+func	"."
+.	"        "
+cmnt	"Print out c0 which has the value 55 which translates to \"7\"!"
+.	"\n"
+err	"TODO"
+cmnt	" ..."