ADR-017: Knowledge ingestion and retrieval (local RAG)

api_provider.py

@@ -3903,6 +3903,24 @@ def ollama_supports_tools(model_name: str) -> bool:
     return "tools" in capabilities
 
 
+def ollama_supports_embedding(model_name: str) -> bool:
+    """ADR-017 stage 17.3: same cached show()-backed probe as
+    ollama_supports_tools' own docstring describes, checking for
+    "embedding" in the model's own reported `capabilities` list instead of
+    "tools" - Ollama's model library tags embedding-only models (e.g.
+    nomic-embed-text, mxbai-embed-large) this way. A CHAT model (e.g.
+    llama3) does NOT report "embedding" here, so this is what stops
+    OllamaProvider.capabilities.embedding from being wrongly True just
+    because SOME Ollama model somewhere supports it - ADR-017's own
+    "Provider.embed()" is per configured-model, not per-server. None
+    (server/model/metadata unavailable) is treated as NOT capable, the
+    same conservative default ollama_supports_tools uses."""
+    capabilities = _get_ollama_capabilities(model_name)
+    if capabilities is None:
+        return False
+    return "embedding" in capabilities
+
+
 def get_mode() -> str:
     if USE_API_MODE:
         return "API"

backend/api/intents_knowledge.py

@@ -0,0 +1,138 @@
+"""ADR-017 stage 17.5: the "knowledge" topic's two WS intents.
+
+`search` is read-only (no record_command/publish_scene - same "just return
+a value" shape as backend/api/intents_diagnostics.py's own two intents),
+the frontend-reachable counterpart to backend/tools_knowledge.py's
+ToolRegistry-registered `knowledge.search` (that one is for a future
+ADR-008 model-driven tool call; this one is for the "Knowledge" search
+panel a human drives directly - same backend.knowledge_retrieval.
+hybrid_search() underneath, two different callers).
+
+`setChatIndexIntoKnowledge` DOES mutate the graph (ChatState.
+index_into_knowledge - see backend/domain/node_states.py's own comment),
+so it goes through record_command/publish_scene like every other scene
+setter in this package - but see its own docstring below for why the
+actual knowledge-store write happens BEFORE that call, not inside it.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from backend.api._shared import make_publish_scene
+from backend.domain.graph import SceneDocument, SceneError
+from backend.events import SessionBus
+from backend.knowledge_ingest import IngestError, ingest_text
+from backend.knowledge_retrieval import hybrid_search
+from backend.knowledge_store import DEFAULT_DB_PATH
+from backend.notifications import NotificationState
+
+_DEFAULT_K = 5
+_MAX_K = 25
+
+
+def branch_history_to_text(history: list[dict]) -> str:
+    """Turns chat_branch_history()'s own {"role","content"} list into one
+    plain-text document for branch indexing - content_parts-carrying
+    entries (`content` is a LIST, not a str - see chat_branch_history's
+    own docstring) contribute only their text-type parts, the same
+    flattening posture the node's own `content` field already applies for
+    every other plain-text consumer of chat history."""
+    lines = []
+    for turn in history:
+        content = turn.get("content")
+        if isinstance(content, str):
+            text = content
+        elif isinstance(content, list):
+            text = " ".join(
+                str(part.get("text", "")) for part in content
+                if isinstance(part, dict) and part.get("type") == "text"
+            )
+        else:
+            continue
+        text = text.strip()
+        if text:
+            lines.append(f"{turn.get('role', 'user')}: {text}")
+    return "\n\n".join(lines)
+
+
+def _format_search_results(results: list[dict]) -> list[dict]:
+    return [
+        {
+            "chunkId": r["chunk_id"],
+            "documentId": r["document_id"],
+            "documentTitle": r["document_title"],
+            "sourceUri": r["source_uri"],
+            "text": r["text"],
+            "offsetStart": r["offset_start"],
+            "offsetEnd": r["offset_end"],
+        }
+        for r in results
+    ]
+
+
+def register_knowledge_intents(
+    bus: SessionBus,
+    document: SceneDocument,
+    notifications: NotificationState | None = None,
+) -> None:
+    publish_scene = make_publish_scene(bus)
+
+    async def search(query, k):
+        k = min(max(int(k or _DEFAULT_K), 1), _MAX_K)
+        # Adversarial-review finding: hybrid_search() does real blocking
+        # SQLite I/O (and, when an embedding provider is wired, a network
+        # round-trip) - called inline, it would stall the whole event loop
+        # for every other connection, unlike every other blocking-I/O
+        # intent handler in this codebase (e.g. intents_settings_*.py).
+        results = await asyncio.to_thread(hybrid_search, DEFAULT_DB_PATH, query, k=k)
+        return {"results": _format_search_results(results)}
+
+    async def set_chat_index_into_knowledge(node_id, enabled):
+        """Runs the actual knowledge-store write (chat_branch_history() ->
+        ingest_text()) BEFORE the flag flips - see backend/domain/
+        node_states.py's own comment on ChatState.index_into_knowledge for
+        why: a caller reading `indexIntoKnowledge: true` off the wire must
+        be able to trust the branch really was indexed as of that toggle,
+        never a flag that silently went True while the write underneath it
+        failed. Disabling (`enabled=False`) never de-indexes anything
+        already stored - the flag is a one-shot trigger + a "has this ever
+        been indexed" marker, not a live subscription toggle.
+
+        Validates node_id is a real chat node FIRST, before any knowledge-
+        store write - chat_branch_history() itself does not raise for a
+        bad/non-chat node_id (its own docstring: "should stop the walk
+        quietly rather than raise"), so without this check a non-chat
+        node's empty `content` would silently fail ingestion (IngestError,
+        swallowed below) and this function would return before ever
+        reaching set_chat_index_into_knowledge's own SceneError - the
+        wrong failure surfacing for what is genuinely a bad call, not a
+        transient indexing problem."""
+        node = document.nodes.get(node_id)
+        if node is None or node.kind != "chat":
+            raise SceneError(f"node is not a chat node: {node_id}")
+
+        if enabled:
+            history = document.chat_branch_history(node_id)
+            text = branch_history_to_text(history)
+            try:
+                # Same blocking-I/O concern as search() above - ingest_text()
+                # does real SQLite writes (chunk + embedding-cache rows).
+                await asyncio.to_thread(
+                    ingest_text, text,
+                    source_uri=f"branch:{node_id}", title=f"Branch (node {node_id})",
+                )
+            except IngestError as exc:
+                if notifications is not None:
+                    notifications.show(str(exc), "error")
+                return
+
+        document.record_command(
+            "setChatIndexIntoKnowledge", "user",
+            lambda: document.set_chat_index_into_knowledge(node_id, enabled),
+            node_ids=[node_id],
+        )
+        await publish_scene()
+
+    bus.register_intent("knowledge", "search", search)
+    bus.register_intent("scene", "setChatIndexIntoKnowledge", set_chat_index_into_knowledge)

backend/canvas.py

@@ -260,6 +260,7 @@ def _placeholder_chart_data(chart_type: str) -> dict[str, Any]:
 from backend.api.intents_gitlink import register_gitlink_intents  # noqa: E402
 from backend.api.intents_grid import register_grid_intents  # noqa: E402
 from backend.api.intents_groups import register_groups_intents  # noqa: E402
+from backend.api.intents_knowledge import register_knowledge_intents  # noqa: E402
 from backend.api.intents_model_routing import register_model_routing_intents  # noqa: E402
 from backend.api.intents_nodes import register_node_intents  # noqa: E402
 from backend.api.intents_pins import register_pins_intents  # noqa: E402
@@ -361,6 +362,7 @@ def register_canvas(
     register_code_sandbox_intents(bus, document, notifications, agent_dispatcher)
 
     register_groups_intents(bus, document)
+    register_knowledge_intents(bus, document, notifications)
     register_model_routing_intents(bus, document)
     register_pins_intents(bus, document)
     register_view_intents(bus, document)

backend/db_backup.py

@@ -64,6 +64,15 @@
 KEEP_MOST_RECENT = 10
 
 
+# ADR-017 stage 17.1 note: every function below now accepts an optional
+# `prefix` (default BACKUP_FILENAME_PREFIX, i.e. every existing chats.db
+# call site is byte-identical to before this stage) so a SECOND database
+# file - backend/knowledge_store.py's knowledge.db - can share this module
+# without its backups being misleadingly named "chats-...". backups_dir_for()
+# itself needs no such parameter: it already isolates per db_path
+# (`db_path.parent / "backups"`), so chats.db and knowledge.db - living in
+# different directories - never share a backups/ folder to begin with; only
+# the FILENAME convention inside that folder needed parametrizing.
 def backups_dir_for(db_path: Path) -> Path:
     """Mirrors backend/crash_recovery.py's own base_dir override pattern
     (`_data_dir(base_dir)`  Path.home()/".graphlink" by default,
@@ -83,27 +92,27 @@ def _timestamp_now() -> str:
     return datetime.now(timezone.utc).strftime(_TIMESTAMP_FORMAT)
 
 
-def backup_filename(timestamp: str) -> str:
-    return f"{BACKUP_FILENAME_PREFIX}{timestamp}{BACKUP_FILENAME_SUFFIX}"
+def backup_filename(timestamp: str, *, prefix: str = BACKUP_FILENAME_PREFIX) -> str:
+    return f"{prefix}{timestamp}{BACKUP_FILENAME_SUFFIX}"
 
 
-def _parse_backup_timestamp(path: Path) -> datetime | None:
+def _parse_backup_timestamp(path: Path, *, prefix: str = BACKUP_FILENAME_PREFIX) -> datetime | None:
     """None for anything that isn't one of OUR OWN backup filenames -
     list_backups/prune_backups must never trip over an unrelated file a
     user (or a future feature) happens to drop into the same directory;
     silently ignoring it, not raising, is the safe direction here since
     this is a read-only classification, not a delete decision by itself."""
     name = path.name
-    if not (name.startswith(BACKUP_FILENAME_PREFIX) and name.endswith(BACKUP_FILENAME_SUFFIX)):
+    if not (name.startswith(prefix) and name.endswith(BACKUP_FILENAME_SUFFIX)):
         return None
-    raw = name[len(BACKUP_FILENAME_PREFIX):-len(BACKUP_FILENAME_SUFFIX)]
+    raw = name[len(prefix):-len(BACKUP_FILENAME_SUFFIX)]
     try:
         return datetime.strptime(raw, _TIMESTAMP_FORMAT).replace(tzinfo=timezone.utc)
     except ValueError:
         return None
 
 
-def list_backups(db_path: Path) -> list[Path]:
+def list_backups(db_path: Path, *, prefix: str = BACKUP_FILENAME_PREFIX) -> list[Path]:
     """Every recognized backup for db_path, newest first. Empty (not an
     error) when the backups directory doesn't exist yet - a session that
     has never taken a backup is a normal, common state, not a fault."""
@@ -114,19 +123,19 @@ def list_backups(db_path: Path) -> list[Path]:
     for candidate in backups_dir.iterdir():
         if not candidate.is_file():
             continue
-        timestamp = _parse_backup_timestamp(candidate)
+        timestamp = _parse_backup_timestamp(candidate, prefix=prefix)
         if timestamp is not None:
             entries.append((timestamp, candidate))
     entries.sort(key=lambda pair: pair[0], reverse=True)
     return [path for _, path in entries]
 
 
-def newest_backup(db_path: Path) -> Path | None:
-    backups = list_backups(db_path)
+def newest_backup(db_path: Path, *, prefix: str = BACKUP_FILENAME_PREFIX) -> Path | None:
+    backups = list_backups(db_path, prefix=prefix)
     return backups[0] if backups else None
 
 
-def prune_backups(db_path: Path) -> list[Path]:
+def prune_backups(db_path: Path, *, prefix: str = BACKUP_FILENAME_PREFIX) -> list[Path]:
     """Applies the retention policy described in this module's own
     docstring and deletes whatever doesn't survive it. Returns the paths
     actually deleted (empty if nothing needed pruning) - real signal for
@@ -137,7 +146,7 @@ def prune_backups(db_path: Path) -> list[Path]:
     block every other backup this call would otherwise have pruned, and
     the next prune_backups call (the very next take_backup) will simply
     try it again."""
-    backups = list_backups(db_path)  # newest first
+    backups = list_backups(db_path, prefix=prefix)  # newest first
     recent = backups[:KEEP_MOST_RECENT]
     older = backups[KEEP_MOST_RECENT:]
     keep: set[Path] = set(recent)
@@ -181,7 +190,7 @@ def prune_backups(db_path: Path) -> list[Path]:
     return deleted
 
 
-def take_backup(db_path: Path) -> Path | None:
+def take_backup(db_path: Path, *, prefix: str = BACKUP_FILENAME_PREFIX) -> Path | None:
     """Snapshots db_path into backups_dir_for(db_path) via SQLite's own
     online backup API - see this module's own docstring for why that API,
     not a raw file copy. Returns the new backup's path, or None when
@@ -204,7 +213,7 @@ def take_backup(db_path: Path) -> Path | None:
     backups_dir = backups_dir_for(db_path)
     backups_dir.mkdir(parents=True, exist_ok=True)
     timestamp = _timestamp_now()
-    final_name = backup_filename(timestamp)
+    final_name = backup_filename(timestamp, prefix=prefix)
     final_path = backups_dir / final_name
     tmp_path = backups_dir / f".{final_name}.tmp"
 
@@ -243,11 +252,11 @@ def take_backup(db_path: Path) -> Path | None:
         logger.warning("could not chmod %s to 0600 - continuing", tmp_path)
     os.replace(tmp_path, final_path)
 
-    prune_backups(db_path)
+    prune_backups(db_path, prefix=prefix)
     return final_path
 
 
-def restore_from_newest_backup(db_path: Path) -> Path | None:
+def restore_from_newest_backup(db_path: Path, *, prefix: str = BACKUP_FILENAME_PREFIX) -> Path | None:
     """Copies the newest surviving backup for db_path INTO db_path,
     overwriting whatever (if anything) is currently there. Returns the
     backup path that was restored, or None when no backup exists at all
@@ -277,7 +286,7 @@ def restore_from_newest_backup(db_path: Path) -> Path | None:
     The source backup file itself is never touched (not deleted, not
     moved) - restoring from it must be repeatable, e.g. if this exact
     restore is itself somehow interrupted and retried."""
-    source = newest_backup(db_path)
+    source = newest_backup(db_path, prefix=prefix)
     if source is None:
         return None
 

backend/domain/branches.py

@@ -149,6 +149,29 @@ def clear_model_override(self, node_id: str) -> None:
         node.state.override_provider = ""
         node.state.override_model_id = ""
 
+    def set_chat_index_into_knowledge(self, node_id: str, enabled: bool) -> None:
+        """ADR-017 stage 17.5: sets the branch-indexing opt-in flag - see
+        backend/domain/node_states.py's own comment on ChatState.
+        index_into_knowledge for what this flag means and why it lives on
+        the node it's set on (the caller's job to pass the branch ROOT's
+        node_id, mirroring set_model_override's own "branch root" framing
+        for override fields). PURE - this method only flips the flag; the
+        actual one-time indexing pass over the branch's text is
+        backend/api/intents_knowledge.py's own job (that intent calls
+        chat_branch_history() + backend.knowledge_ingest.ingest_text()
+        BEFORE calling this method, so a document I/O failure never leaves
+        the flag set without the indexing having actually happened) -
+        mirroring this whole file's own "SceneDocument mutates the graph,
+        intents own the side effects" separation (chat_library persistence
+        is owned by intents_chat_library.py, never by graph.py/branches.py
+        directly)."""
+        node = self.nodes.get(node_id)
+        if node is None:
+            raise SceneError(f"unknown node: {node_id}")
+        if node.kind != "chat":
+            raise SceneError(f"node is not a chat node: {node_id}")
+        node.state.index_into_knowledge = bool(enabled)
+
     def resolve_model_for_node(self, node_id: str | None):
         """ADR-018 stage 18.2: the node-override -> branch-override half of
         graphlink_model_catalog.resolve_model_ref's chain - returns

backend/domain/graph.py

@@ -1995,6 +1995,9 @@ def _node_wire(self, n: SceneNode) -> dict[str, Any]:
             # input routing pin).
             "overrideProvider": n.state.override_provider if isinstance(n.state, ChatState) else "",
             "overrideModelId": n.state.override_model_id if isinstance(n.state, ChatState) else "",
+            # ADR-017 stage 17.5: see ChatState's own comment on
+            # index_into_knowledge.
+            "indexIntoKnowledge": n.state.index_into_knowledge if isinstance(n.state, ChatState) else False,
             "isBranchSynthesis": n.state.is_branch_synthesis if isinstance(n.state, ChatState) else False,
             "synthesisInstructions": (
                 n.state.synthesis_instructions if isinstance(n.state, ChatState) else ""

backend/domain/node_states.py

@@ -737,3 +737,22 @@ class ChatState(NodeState):
     # clear_model_override always clears both together.
     override_provider: str = ""
     override_model_id: str = ""
+    # ADR-017 stage 17.5: "a branch can be indexed so later branches can
+    # retrieve from it" (ADR-017 doc's own Decision #2, "Sources"
+    # paragraph) - True after the user has opted THIS node's branch
+    # history (chat_branch_history(this node's id): root down to this
+    # node) into knowledge indexing. Per-node, not cascaded to ancestors/
+    # descendants - the same no-write-time-inheritance posture
+    # branch_status's own comment documents; toggling it on a leaf node
+    # indexes that leaf's own root-to-here history, toggling it on an
+    # earlier node in the same chain indexes a shorter prefix, and neither
+    # write touches the other node's own flag. Setting this to True is
+    # also the TRIGGER for a one-time indexing pass over that history as
+    # of right now (backend/api/intents_knowledge.py's own
+    # set_chat_index_into_knowledge intent runs the ingest BEFORE flipping
+    # this flag) - there is no live "index every new turn automatically"
+    # pipeline yet (that needs ADR-008's tool-use loop machinery to hook a
+    # real per-turn point; this field's job is the opt-in flag + one
+    # honest snapshot, not a promise of continuous re-indexing this
+    # codebase cannot make yet).
+    index_into_knowledge: bool = False