fix(encoding): decode git's file names as Python does, and write LF (Windows); km 1.0.2

.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "km",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Knowledge management for markdown knowledge bases: answer from it with sources, save notes, decisions and transcripts with validated frontmatter, lint, mount peer brains, contribute to a team brain by PR, and set up a shared team brain with CI (/km init --team).",
   "author": {
     "name": "Roman Passler"

.github/workflows/test.yml

@@ -25,7 +25,8 @@ jobs:
           KM_REPO_URL="file://$GITHUB_WORKSPACE" /tmp/wheel/bin/km init /tmp/wheel-brain --team --name W --desc w --initials WW --folder d=D
       - run: bash tests/gate_smoke.sh
       - run: bash tests/index_smoke.sh
-      - run: bash tests/team_smoke.sh
+      - name: team_smoke.sh, with a Latin-1 locale for its file-name encoding test
+        run: sudo localedef -i en_US -f ISO-8859-1 en_US.ISO-8859-1 && bash tests/team_smoke.sh
 
   # The composite action a brain's CI uses: install km from this checkout, then build and check a
   # team brain with the installed `km` command.
@@ -58,11 +59,33 @@ jobs:
           path: ${{ runner.temp }}/b/dist/served/
           if-no-files-found: error
 
+  # Windows: its default encoding is a code page (cp1252), not UTF-8, and text files are written with
+  # CRLF. A file name with an umlaut must still be found, and km must write the LF the repo holds
+  # (tests/encoding_smoke.sh).
+  windows:
+    runs-on: windows-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.11'
+      - run: python -m pip install --quiet --disable-pip-version-check pyyaml==6.0.3
+      - name: non-ASCII file names and LF writes
+        env:
+          PYTHON: python
+        run: |
+          source tests/lib.sh
+          windows_like "" || { echo "::error::the runner's encoding is UTF-8, this job tests nothing"; exit 1; }
+          with_env "" bash tests/encoding_smoke.sh    # as checked: UTF-8 mode off
+
   # Merging a version bump is the release: once main is green, tag v<__version__> if that tag is new.
   # Brains pin these tags, and `km init` writes the tag of the km that runs it.
   release:
     if: github.event_name == 'push'
-    needs: [smoke, action]
+    needs: [smoke, action, windows]
     runs-on: ubuntu-latest
     permissions:
       contents: write

README.md

@@ -51,9 +51,9 @@ km upgrade                  move a brain's km pins to this version
 ```
 
 Every command takes `--root DIR`; the default is the git work tree around the current directory.
-Install a release by its commit (`git ls-remote https://github.com/pasrom/km refs/tags/v1.0.1` shows
-it), not by the tag: `pip install git+https://github.com/pasrom/km@<commit>  # v1.0.1`, or run it
-without installing: `uvx --from git+https://github.com/pasrom/km@<commit> km validate  # v1.0.1`.
+Install a release by its commit (`git ls-remote https://github.com/pasrom/km refs/tags/v1.0.2` shows
+it), not by the tag: `pip install git+https://github.com/pasrom/km@<commit>  # v1.0.2`, or run it
+without installing: `uvx --from git+https://github.com/pasrom/km@<commit> km validate  # v1.0.2`.
 
 ## What a brain gets
 
@@ -62,13 +62,13 @@ Brains carry no km code. `km init` writes `CONVENTIONS.md`, `CLAUDE.md`, `schema
 generated indexes, a README, and CI that installs km through this repository's GitHub Action:
 
 ```yaml
-- uses: pasrom/km@<commit> # v1.0.1, installs the km CLI at that commit
+- uses: pasrom/km@<commit> # v1.0.2, installs the km CLI at that commit
 - run: km validate
 - run: km gen-index --check
 ```
 
 The pre-commit hook works the same way (`repo: https://github.com/pasrom/km`,
-`rev: <commit>  # frozen: v1.0.1`, `id: km-validate`). Every pin names a release by its commit, so a
+`rev: <commit>  # frozen: v1.0.2`, `id: km-validate`). Every pin names a release by its commit, so a
 tag moved later cannot change what a brain runs; the version rides along as a comment, and
 `km upgrade` checks that the commit is that release's. km's own dependencies are pinned to exact
 versions. `km init` and `km upgrade` look a release's commit up by its tag once, so a release tag here
@@ -91,7 +91,8 @@ tests/                 smoke tests
 ## Development
 
 The smoke tests need Python 3.11+, PyYAML and git: `bash tests/gate_smoke.sh`, `index_smoke.sh`,
-`team_smoke.sh`. CI runs all three plus the GitHub Action on every pull request. A release bumps
+`team_smoke.sh`. CI runs all three plus the GitHub Action on every pull request, and
+`encoding_smoke.sh` on Windows (team_smoke runs it on macOS under non-UTF-8 locales). A release bumps
 `__version__` in `src/km/__init__.py`, `version` in `.claude-plugin/plugin.json` and the version in
 this README together; merging that to main is the release, CI tags `vX.Y.Z` once main is green.
 

src/km/__init__.py

@@ -1,4 +1,4 @@
 """km: knowledge management for markdown knowledge bases."""
 
-__version__ = "1.0.1"
+__version__ = "1.0.2"
 KM_REF = f"v{__version__}"            # this km's release tag; brains pin the commit it points at

src/km/cli.py

@@ -87,6 +87,9 @@ def run_pinned(root: Path, cmd: str, rest: list[str]) -> int | None:
 
 
 def main(argv: list[str] | None = None) -> int:
+    for stream in (sys.stdout, sys.stderr):   # a non-UTF-8 console or pipe (Windows) must not crash on a file name
+        if hasattr(stream, "reconfigure"):
+            stream.reconfigure(errors="backslashreplace")
     root, args = take_root(list(sys.argv[1:] if argv is None else argv))
     if root is not None and not Path(root).is_dir():
         print(f"km: --root {root}: no such directory", file=sys.stderr)

src/km/common.py

@@ -5,14 +5,14 @@
 """
 from __future__ import annotations
 
+import functools
 import re
-import subprocess
 import sys
 from pathlib import Path
 
 import yaml
 
-from km.paths import BASE_SCHEMA, repo_root
+from km.paths import BASE_SCHEMA, git, repo_root
 
 ROOT = repo_root()
 
@@ -80,11 +80,18 @@ def read(rel: str) -> str:
     return (ROOT / rel).read_text(encoding="utf-8-sig", errors="replace")
 
 
-def tracked_md() -> list[str]:
-    res = subprocess.run(["git", "ls-files", "-z", "*.md"], cwd=ROOT, capture_output=True, text=True)
+@functools.cache
+def _ls_md() -> tuple[str, ...]:
+    res = git("ls-files", "-z", "*.md", cwd=ROOT)
     if res.returncode != 0:
-        sys.exit(f"git ls-files failed: {res.stderr.strip()}")
-    return [p for p in res.stdout.split("\x00") if p and (ROOT / p).is_file()]   # a deletion not yet staged is gone
+        sys.exit(f"git ls-files failed (not a git repo?): {res.stderr.strip()}")
+    return tuple(p for p in res.stdout.split("\x00") if p)
+
+
+def tracked_md(existing: bool = True) -> list[str]:
+    """The tracked .md files, listed by git once per run; with existing=False also those deleted but
+    not yet staged."""
+    return [p for p in _ls_md() if not existing or (ROOT / p).is_file()]
 
 
 def is_article(rel: str) -> bool:

src/km/demote.py

@@ -13,6 +13,7 @@
 import sys
 
 from km.common import ROOT, SERVED_STATUS, is_article, read, split_frontmatter, tracked_md
+from km.paths import write_text
 
 TODAY = datetime.date.today()
 
@@ -49,7 +50,7 @@ def main() -> int:
             continue
         stale.append(rel)
         new_fm = re.sub(r"(?m)^approved_(?:by|at):.*\n", "", new_fm)
-        (ROOT / rel).write_text(new_fm + body, encoding="utf-8")
+        write_text(ROOT / rel, new_fm + body)
 
     if stale:
         print(f"km demote: {'demoted' if apply else 'would demote'} {len(stale)} doc(s) past review_by:")

src/km/gen_index.py

@@ -19,6 +19,7 @@
 from pathlib import Path
 
 from km.common import INDEX_SKIP, ROOT, SKIP, folder_of, frontmatter, is_article, read, tracked_md
+from km.paths import write_text
 
 _MARK = re.compile(r"(?m)^## Documents[ \t]*$")
 _NEXT_H2 = re.compile(r"(?m)^## ")
@@ -79,7 +80,7 @@ def main() -> int:
         if new != cur:
             changed.append(idx)
             if not check:
-                (ROOT / idx).write_text(new, encoding="utf-8")
+                write_text(ROOT / idx, new)
 
     if check and changed:
         print("km gen-index: out of date, run `km gen-index`:")

src/km/init.py

@@ -23,7 +23,7 @@
 from pathlib import Path
 
 from km import KM_REF
-from km.paths import TEMPLATES
+from km.paths import TEMPLATES, write_text
 from km.pins import resolve
 
 PERSONAL = {                          # template -> path in the brain (repo-owned after init)
@@ -130,7 +130,7 @@ def main() -> int:
         sys.exit(f"placeholder(s) left after rendering: {', '.join(left)}")
     for dst, text in rendered.items():
         (repo / dst).parent.mkdir(parents=True, exist_ok=True)
-        (repo / dst).write_text(text, encoding="utf-8")
+        write_text(repo / dst, text)
     print(f"km init: {'team' if a.team else 'personal'} brain in {repo}, pinned to km {a.pin}")
     return 0
 

src/km/paths.py

@@ -3,18 +3,34 @@
 
 import os
 import subprocess
+import sys
 from pathlib import Path
 
 PACKAGE = Path(__file__).resolve().parent
 BASE_SCHEMA = PACKAGE / "schema.base.yaml"
 TEMPLATES = PACKAGE / "templates"
 
 
+def git(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
+    """git, with its output decoded the way Python decodes file names, so each name it lists opens
+    the file it names: UTF-8 on Windows and macOS, the locale's encoding on Linux. With the system's
+    code page instead (cp1252 on Windows), a name with an umlaut named no file. Bytes that do not
+    decode (a git message in another encoding, an odd name) are kept, not a crash."""
+    return subprocess.run(["git", *args], cwd=cwd, capture_output=True,
+                          encoding=sys.getfilesystemencoding(), errors="surrogateescape")
+
+
+def write_text(path: Path, text: str) -> None:
+    """Every file km writes: UTF-8 with LF, the bytes the repo holds, also where the default is a code
+    page and CRLF (Windows)."""
+    path.write_text(text, encoding="utf-8", newline="\n")
+
+
 def repo_root() -> Path:
     """The brain km works on: KM_ROOT if set (the CLI sets it from --root), else the git work tree
     containing the current directory, else the current directory."""
     env = os.environ.get("KM_ROOT")
     if env:
         return Path(env).resolve()
-    res = subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True)
+    res = git("rev-parse", "--show-toplevel")
     return Path(res.stdout.strip()).resolve() if res.returncode == 0 else Path.cwd().resolve()

src/km/pins.py

@@ -11,10 +11,11 @@
 
 import os
 import re
-import subprocess
 from dataclasses import dataclass
 from pathlib import Path
 
+from km.paths import git, write_text
+
 KM_REPO = "https://github.com/pasrom/km"
 SHA = re.compile(r"[0-9a-f]{40}")
 VERSION = r"v\d[\w.\-]*"
@@ -113,7 +114,7 @@ def set_pins(root: Path, sha: str, version: str) -> list[str]:
             else:
                 new = PIN.sub(lambda m: m.group(1) + uses_ref(sha, version), text)
             if new != text:
-                p.write_text(new, encoding="utf-8")
+                write_text(p, new)
                 changed.append(str(p.relative_to(root)))
     return changed
 
@@ -132,7 +133,7 @@ def resolve(version: str) -> str | None:
     """The commit a km release tag points at; None when the tag does not exist or the repository
     cannot be reached."""
     tag = f"refs/tags/{version}"
-    out = subprocess.run(["git", "ls-remote", repo_url(), tag, tag + "^{}"], capture_output=True, text=True).stdout
+    out = git("ls-remote", repo_url(), tag, tag + "^{}").stdout
     refs = {name: sha for sha, _, name in (line.partition("\t") for line in out.splitlines())}
     return refs.get(tag + "^{}") or refs.get(tag)   # an annotated tag: the commit it points at
 

src/km/promote.py

@@ -35,6 +35,7 @@
 
 import argparse
 import datetime
+import os
 import re
 import subprocess
 import sys
@@ -43,6 +44,7 @@
 import yaml
 
 from km.common import ROOT, SCHEMA
+from km.paths import write_text
 
 
 def submodule_prefixes() -> set[str]:
@@ -96,7 +98,8 @@ def type_enum() -> set[str] | None:
 def run_gate(target: Path):
     """`km validate` on one file, in a child process: the validator runs its checks at import."""
     return subprocess.run([sys.executable, "-m", "km", "validate", "--root", str(ROOT), str(target)],
-                          cwd=ROOT, capture_output=True, text=True)
+                          cwd=ROOT, capture_output=True, encoding="utf-8", errors="replace",
+                          env={**os.environ, "PYTHONIOENCODING": "utf-8"})   # the child writes UTF-8 too
 
 
 _PROVENANCE = ("status", "approved_by", "approved_at", "supersedes", "superseded_by", "contribution")
@@ -137,7 +140,7 @@ def _stub_source(src: Path, target_rel: str, prefixes: set[str], src_fm: dict, i
     stub["superseded_by"] = target_rel
     tmp = src.parent / (src.stem + ".stub-tmp.md")
     try:
-        tmp.write_text(dump(stub, f"Moved to the served brain. See [{target_rel}]({target_rel})."), encoding="utf-8")
+        write_text(tmp, dump(stub, f"Moved to the served brain. See [{target_rel}]({target_rel})."))
         r = run_gate(tmp)
         if r.returncode != 0:
             tmp.unlink(missing_ok=True)
@@ -364,7 +367,7 @@ def main() -> int:
     content = dump(fm, body)
     target.parent.mkdir(parents=True, exist_ok=True)
     tmp = target.parent / (target.stem + ".promote-tmp.md")
-    tmp.write_text(content, encoding="utf-8")
+    write_text(tmp, content)
     r = run_gate(tmp)
     if r.returncode != 0:
         tmp.unlink(missing_ok=True)

src/km/serve.py

@@ -17,15 +17,14 @@
 from pathlib import Path
 
 from km.common import CUSTOMER_AUDIENCE, ROOT, SERVED_STATUS, is_article, read, split_frontmatter, tracked_md
+from km.paths import write_text
 
 OUT = ROOT / "dist" / "served"
 ALLOWED_AUDIENCE = {"internal", CUSTOMER_AUDIENCE}
 
 
 def write_jsonl(path: Path, records: list[dict]) -> None:
-    with path.open("w", encoding="utf-8") as fh:
-        for r in records:
-            fh.write(json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n")
+    write_text(path, "".join(json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n" for r in records))
 
 
 def main() -> int:
@@ -63,7 +62,7 @@ def main() -> int:
         "counts": {"internal": len(accepted), "customer": len(customer)},
         "skipped_bad_audience": skipped_bad_audience,
     }
-    (OUT / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
+    write_text(OUT / "manifest.json", json.dumps(manifest, indent=2) + "\n")
     print(f"served: internal={len(accepted)} customer={len(customer)} "
           f"(bad-audience skipped {skipped_bad_audience}) -> {OUT.relative_to(ROOT)}")
     return 0

src/km/upgrade.py

@@ -29,7 +29,7 @@
 
 from km import KM_REF
 from km.init import pin_values, template
-from km.paths import repo_root
+from km.paths import repo_root, write_text
 from km.pins import hook_rev, movable_refs, read_pins, resolve, set_pins, version_key, workflow_files
 
 COPIED = {                            # a file km copied in -> text only km's copy of it carries
@@ -92,7 +92,7 @@ def main() -> int:
             continue
         text = p.read_text(encoding="utf-8")
         if rel in rewrites:
-            p.write_text(template(tpl, values), encoding="utf-8")
+            write_text(p, template(tpl, values))
             done.append(f"rewrote {rel} (it was km's own, unchanged) to install km")
             team |= rel.startswith(".github/")
         elif rel == ".pre-commit-config.yaml" and "scripts/validate.py" in text:
@@ -125,12 +125,12 @@ def main() -> int:
     if local.is_file():
         text = local.read_text(encoding="utf-8")
         if (new := MARKER.sub("", text)) != text:
-            local.write_text(new, encoding="utf-8")
+            write_text(local, new)
             done.append("dropped team_brain from schema.local.yaml (no longer used)")
     dependabot = root / ".github/dependabot.yml"
     if team and not dependabot.exists():
         dependabot.parent.mkdir(parents=True, exist_ok=True)
-        dependabot.write_text(template("team/dependabot.yml", {}), encoding="utf-8")
+        write_text(dependabot, template("team/dependabot.yml", {}))
         done.append("added .github/dependabot.yml")
 
     if values and (not current or stray):