ADR-009 stages 9.3-9.6: secret scrubbing, .graphlink archive, asset store, flat edge format

backend/asset_store.py

@@ -0,0 +1,146 @@
+"""ADR-009 stage 9.5: the content-addressed binary asset store.
+
+WHAT THIS REPLACES. Image and chart bytes used to be base64'd inline into
+the chat's `data` blob (see session_save.py's own _serialize_image_node).
+That has two costs the ADR names: autosave rewrites every one of those
+megabytes on every 30-second tick even when no image changed, and an
+export has no way to carry assets as real files. Both dissolve once bytes
+live outside the blob and the blob carries only a reference.
+
+CONTENT-ADDRESSED, so the reference IS the integrity check: a ref is the
+SHA-256 of the bytes it names. Two nodes holding the same image
+deduplicate for free, a re-save of unchanged bytes is a no-op rather than
+a rewrite, and a corrupted file is detectable by rehashing rather than
+being silently served as if it were fine.
+
+WRITES ARE ATOMIC AND IDEMPOTENT. put() writes to a temp name and
+os.replace()s into place, so a crash mid-write can only ever leave a temp
+file, never a truncated file wearing a real ref's name that a later read
+would trust. If the target already exists, put() returns immediately
+without rewriting - by construction the existing file already has exactly
+the content being stored, since its name is that content's hash.
+
+NOTHING IS EVER DELETED HERE. Garbage collection of unreferenced assets is
+deliberately out of scope: an asset is cheap to keep and catastrophic to
+delete while some chat still points at it, and "which chats reference
+which assets" is a whole-database question this module has no business
+answering. A future sweep belongs alongside the export/import story, with
+its own recon.
+
+Two-character shard directories keep any single directory from
+accumulating tens of thousands of entries, which some filesystems handle
+poorly - the same convention git's own object store uses.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Kept deliberately small: this maps the mime types this app actually
+# produces (image nodes and chart PNGs), not a general registry. An
+# unknown type falls back to .bin rather than guessing.
+_EXTENSION_BY_MIME = {
+    "image/png": "png",
+    "image/jpeg": "jpg",
+    "image/gif": "gif",
+    "image/webp": "webp",
+    "image/svg+xml": "svg",
+}
+
+
+def content_ref(data: bytes) -> str:
+    """The SHA-256 hex digest that names these exact bytes."""
+    return hashlib.sha256(data).hexdigest()
+
+
+def extension_for_mime(mime_type: str | None) -> str:
+    return _EXTENSION_BY_MIME.get((mime_type or "").lower(), "bin")
+
+
+def assets_dir_for(db_path: Path) -> Path:
+    """Assets live NEXT TO the database that references them
+    (`db_path.parent / "assets"`), matching backend/db_backup.py's own
+    backups_dir_for convention - so a test passing an isolated
+    `tmp_path / "chats.db"` gets an isolated asset store for free, and the
+    real default lands at ~/.graphlink/assets/."""
+    return db_path.parent / "assets"
+
+
+def store_for(db_path: Path) -> "AssetStore":
+    """The live asset store belonging to a database. The one place the
+    save/load paths call to get a store, so "which directory" is decided
+    here rather than at four separate call sites."""
+    return AssetStore(assets_dir_for(db_path))
+
+
+class AssetStore:
+    """A directory of content-addressed blobs. Construct with the directory
+    itself, not a db path, so it is equally usable for the live store and
+    for an export's staging area."""
+
+    def __init__(self, root: Path):
+        self.root = Path(root)
+
+    def _path_for(self, ref: str) -> Path:
+        return self.root / ref[:2] / ref
+
+    def exists(self, ref: str) -> bool:
+        return self._path_for(ref).is_file()
+
+    def put(self, data: bytes) -> str:
+        """Stores `data` and returns its ref. Idempotent: storing identical
+        bytes twice writes once and returns the same ref both times."""
+        ref = content_ref(data)
+        target = self._path_for(ref)
+        if target.is_file():
+            return ref
+
+        target.parent.mkdir(parents=True, exist_ok=True)
+        tmp = target.with_name(target.name + ".tmp")
+        try:
+            fd = os.open(tmp, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600)
+            with os.fdopen(fd, "wb") as handle:
+                handle.write(data)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(tmp, target)
+        except BaseException:
+            try:
+                tmp.unlink(missing_ok=True)
+            except OSError:
+                pass
+            raise
+        try:
+            os.chmod(target, 0o600)
+        except OSError:
+            logger.warning("could not chmod asset %s to 0600 - continuing", target)
+        return ref
+
+    def get(self, ref: str) -> bytes | None:
+        """The bytes for `ref`, or None if this store has never seen it.
+
+        None rather than an exception because a missing asset must degrade
+        to "this image does not render" rather than "this chat will not
+        load" - a chat's text is worth far more than one of its pictures,
+        and an import from a bundle whose assets were stripped is a
+        legitimate, survivable state."""
+        target = self._path_for(ref)
+        if not target.is_file():
+            return None
+        try:
+            return target.read_bytes()
+        except OSError:
+            logger.warning("could not read asset %s - treating as missing", target)
+            return None
+
+    def verify(self, ref: str) -> bool:
+        """True only if the stored bytes still hash to their own name -
+        the integrity check content addressing makes possible. Used by the
+        import path, which is reading files it did not write."""
+        data = self.get(ref)
+        return data is not None and content_ref(data) == ref

backend/autosave.py

@@ -82,6 +82,7 @@
 from pathlib import Path
 from typing import Any
 
+from backend.asset_store import store_for
 from backend.canvas import SceneDocument
 from backend.chat_library import (
     AUTOSAVE_OWNER,
@@ -121,7 +122,11 @@ async def autosave_tick(
         return
 
     try:
-        chat_data = build_chat_data(canvas_document)
+        # ADR-009 stage 9.5: image bytes go to the content-addressed store
+        # and only a ref is written into the row. This path is the whole
+        # reason that stage exists - without it every 30-second tick
+        # rewrote megabytes of base64 for pictures that had not changed.
+        chat_data = build_chat_data(canvas_document, asset_store=store_for(db_path))
     except Exception:
         logger.exception("autosave: failed to build chat data for session %r", bus.session_id)
         return

backend/chat_library.py

@@ -51,6 +51,7 @@
 from backend.events import SessionBus
 from backend.notifications import NotificationState
 from backend.session_load import restore_chat_into_document
+from backend.asset_store import store_for
 from backend.session_save import build_chat_data
 from graphlink_migrations import run_sqlite_migrations
 
@@ -1178,7 +1179,10 @@ async def load_chat(chat_id: int):
             if canvas_document is None:
                 return
 
-            restore_chat_into_document(canvas_document, row, notes_rows, pins_rows)
+            restore_chat_into_document(
+                canvas_document, row, notes_rows, pins_rows,
+                asset_store=store_for(resolved_path),
+            )
             # R6.5: remember which row this scene now corresponds to, so a
             # later Save updates THIS row instead of always inserting a new
             # one - the backend analog of ChatSessionManager.current_chat_id
@@ -1189,7 +1193,11 @@ async def load_chat(chat_id: int):
             # byte-identical row and bumped updated_at, re-sorting the Chat
             # Library under the user for a session they had only just opened.
             try:
-                fresh = build_chat_data(canvas_document)
+                # Must use the SAME store as autosave below, or the
+                # first tick after a load would see a payload that
+                # differs only in image representation and rewrite a
+                # row that is already correct.
+                fresh = build_chat_data(canvas_document, asset_store=store_for(resolved_path))
                 fresh_notes = fresh.pop("notes_data", [])
                 fresh_pins = fresh.pop("pins_data", [])
                 # ADR-009 stage 9.2: row["updated_at"] is the value that
@@ -1275,7 +1283,7 @@ async def save_chat():
             return
 
         try:
-            chat_data = build_chat_data(canvas_document)
+            chat_data = build_chat_data(canvas_document, asset_store=store_for(resolved_path))
         except Exception as exc:
             if notifications is not None:
                 notifications.show(f"Failed to prepare chat save payload: {exc}", "error")
@@ -1573,7 +1581,7 @@ def flush_dirty_session_before_teardown(
         return
 
     try:
-        chat_data = build_chat_data(canvas_document)
+        chat_data = build_chat_data(canvas_document, asset_store=store_for(db_path))
     except Exception:
         logger.exception("eviction flush: failed to build chat data - the last edit may be lost")
         return

backend/secret_scrub.py

@@ -0,0 +1,135 @@
+"""ADR-009 stage 9.3: the one secret-scrub chokepoint.
+
+Every surface that lets data LEAVE this machine goes through `scrub()`:
+the `.graphlink` export (stage 9.4), and - once they exist - ADR-014/008
+recipe templates and any future share surface. One function, one test
+file, one place to audit.
+
+WHY VALUE-BASED, NOT JUST KEY-BASED. A scrubber that only redacts known
+field NAMES (`openai_api_key`, ...) is one refactor away from useless: the
+day a secret gets copied into a differently-named field, or interpolated
+into an error string stored on a node, or pasted by the user into their own
+chat text, a name-only filter waves it straight through. So this scrubs on
+BOTH axes - the name of the field it sits in, AND the shape of the value
+itself. A value that looks like a credential is redacted no matter what
+key it arrived under, including inside free text.
+
+WHY PATHS TOO. An absolute path is not a credential but it is personal:
+`C:\\Users\\<real name>\\...` carries the operator's account name, and
+deeper segments carry private folder and file names. Paths are redacted
+whole rather than trimmed to a basename, because the basename is exactly
+where the private part usually lives (`quarterly_layoffs.xlsx`).
+
+DELIBERATELY OVER-BROAD. This function will sometimes redact a string that
+was not actually a secret - a base64 blob, a long opaque id, a path the
+user pasted deliberately. That direction is correct: a redacted non-secret
+costs a slightly less useful export, while a leaked real secret costs the
+user their account. Every rule here is chosen to fail toward redaction.
+
+NOT ENCRYPTION, NOT AUTHORIZATION. This removes secrets from data already
+destined to leave. It does not decide WHETHER something may leave; callers
+own that.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+REDACTED = "[redacted]"
+REDACTED_PATH = "[redacted-path]"
+
+# Field names whose VALUE is always a secret, matched case-insensitively.
+# Exact names come from graphlink_settings_store.SettingsManager.SECRET_KEYS
+# (kept in sync by test_secret_scrub.py, which imports that tuple and
+# asserts every member is covered here) - the suffix rules below then
+# generalize to fields that do not exist yet.
+_SECRET_KEY_SUFFIXES = (
+    "api_key",
+    "apikey",
+    "access_token",
+    "refresh_token",
+    "auth_token",
+    "token",
+    "secret",
+    "password",
+    "passphrase",
+    "credential",
+    "credentials",
+    "private_key",
+)
+
+# Value shapes that are a credential regardless of the field they sit in.
+# Each is anchored to a real issuer format rather than "long random-looking
+# string", so ordinary content is not shredded wholesale.
+_SECRET_VALUE_PATTERNS = (
+    re.compile(r"sk-[A-Za-z0-9_\-]{16,}"),           # OpenAI-style
+    re.compile(r"sk-ant-[A-Za-z0-9_\-]{16,}"),       # Anthropic
+    re.compile(r"AIza[A-Za-z0-9_\-]{20,}"),          # Google API key
+    re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"),       # GitHub token family
+    re.compile(r"github_pat_[A-Za-z0-9_]{20,}"),     # GitHub fine-grained PAT
+    re.compile(r"xox[baprs]-[A-Za-z0-9\-]{10,}"),    # Slack
+    re.compile(r"\bBearer\s+[A-Za-z0-9._\-]{16,}"),  # generic bearer header
+    # This codebase's own at-rest wrapper (graphlink_secrets.py). The
+    # ciphertext is only decryptable by this Windows account, but it is
+    # still the secret's stored form and has no business in an export.
+    re.compile(r"dpapi:[A-Za-z0-9+/=]+"),
+)
+
+# Absolute filesystem paths: Windows drive-letter, UNC, and POSIX home-ish
+# roots. Deliberately NOT a bare `/...` match - that would eat ordinary
+# prose containing slashes, and URLs, for no gain.
+_PATH_PATTERNS = (
+    re.compile(r"[A-Za-z]:[\\/][^\s\"'<>|]*"),
+    re.compile(r"\\\\[^\s\"'<>|]+"),
+    re.compile(r"/(?:home|Users|root|var|tmp|opt|mnt|media)/[^\s\"'<>|]*"),
+)
+
+
+def _key_is_secret(key: str) -> bool:
+    lowered = key.lower()
+    return any(lowered.endswith(suffix) for suffix in _SECRET_KEY_SUFFIXES)
+
+
+def scrub_text(text: str) -> str:
+    """Redacts credential-shaped and path-shaped substrings inside one
+    string, leaving the surrounding text intact. Applied to every string
+    reached by scrub(), not just to values under suspicious keys - an error
+    message stored on a node ("failed to read C:\\Users\\ada\\taxes.csv")
+    is exactly the kind of incidental leak a key-name filter misses."""
+    result = text
+    for pattern in _SECRET_VALUE_PATTERNS:
+        result = pattern.sub(REDACTED, result)
+    for pattern in _PATH_PATTERNS:
+        result = pattern.sub(REDACTED_PATH, result)
+    return result
+
+
+def scrub(value: Any) -> Any:
+    """Returns a scrubbed deep copy of `value`. Never mutates its input -
+    callers routinely pass live in-memory state (a SceneDocument's own
+    payload) that must not be damaged by the act of exporting it.
+
+    Containers recurse; strings go through scrub_text; a value under a
+    secret-named key is replaced wholesale rather than pattern-matched,
+    since a credential in a field literally called `api_key` should not
+    have to match a known issuer format to be caught. Non-string scalars
+    (int/float/bool/None) pass through untouched - they cannot carry a
+    credential, and mangling them would corrupt the export's structure."""
+    if isinstance(value, dict):
+        scrubbed: dict[Any, Any] = {}
+        for key, item in value.items():
+            if isinstance(key, str) and _key_is_secret(key):
+                # Preserve "was set" vs "was empty" - useful in a bug
+                # report, and it leaks nothing.
+                scrubbed[key] = REDACTED if item else item
+            else:
+                scrubbed[key] = scrub(item)
+        return scrubbed
+    if isinstance(value, list):
+        return [scrub(item) for item in value]
+    if isinstance(value, tuple):
+        return tuple(scrub(item) for item in value)
+    if isinstance(value, str):
+        return scrub_text(value)
+    return value