diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 640651a..cba9109 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: branches: [ main, master ] permissions: - contents: read + contents: write checks: write pages: write id-token: write diff --git a/JavaDuckerMcpServer.java b/JavaDuckerMcpServer.java index e13f9a4..2935d86 100644 --- a/JavaDuckerMcpServer.java +++ b/JavaDuckerMcpServer.java @@ -154,6 +154,38 @@ public static void main(String[] args) throws Exception { (String) a.get("action"), (String) a.getOrDefault("directory", ""), (String) a.getOrDefault("extensions", "")))) + // ── Explain tool ───────────────────────────────────────────── + .tool(tool("javaducker_explain", + "Get everything JavaDucker knows about a file: summary, dependencies, dependents, tags, " + + "classification, related plans, blame highlights, and co-change partners. One call for full context.", + schema(props("file_path", str("Absolute path to the file to explain")), "file_path")), + (ex, a) -> call(() -> httpPost("/explain", Map.of("filePath", a.get("file_path"))))) + // ── Git Blame tool ─────────────────────────────────────────── + .tool(tool("javaducker_blame", + "Show who last changed each line of a file, with commit info. Groups consecutive lines by same commit. Optionally narrow to a line range.", + schema(props( + "file_path", str("Absolute path to the file"), + "start_line", intParam("Start line number (optional)"), + "end_line", intParam("End line number (optional)")), + "file_path")), + (ex, a) -> call(() -> { + Map body = new LinkedHashMap<>(); + body.put("filePath", a.get("file_path")); + if (a.containsKey("start_line")) body.put("startLine", ((Number) a.get("start_line")).intValue()); + if (a.containsKey("end_line")) body.put("endLine", ((Number) a.get("end_line")).intValue()); + return httpPost("/blame", body); + })) + // ── Co-Change / Related Files tool ───────────────────────── + .tool(tool("javaducker_related", + "Find files commonly edited together with this file, based on git co-change history. " + + "Helps identify related files you might need to update.", + schema(props( + "file_path", str("Absolute path to the file"), + "max_results", intParam("Max results (default 10)")), + "file_path")), + (ex, a) -> call(() -> httpPost("/related", Map.of( + "filePath", a.get("file_path"), + "maxResults", ((Number) a.getOrDefault("max_results", 10)).intValue())))) // ── Content Intelligence: write tools ──────────────────────── .tool(tool("javaducker_classify", "Classify an artifact by doc type (ADR, DESIGN_DOC, PLAN, MEETING_NOTES, THREAD, SCRATCH, CODE, REFERENCE, TICKET).", @@ -257,6 +289,12 @@ public static void main(String[] args) throws Exception { "Health report for all concepts: active/stale doc counts, trend (active/fading/cold).", "{}"), (ex, a) -> call(() -> httpGet("/concept-health"))) + // ── Index Health tool ──────────────────────────────────────── + .tool(tool("javaducker_index_health", + "Check index health: how many files are current vs stale. Returns actionable recommendation. " + + "No parameters required — scans all indexed files.", + "{}"), + (ex, a) -> call(JavaDuckerMcpServer::indexHealth)) // ── Reladomo tools ─────────────────────────────────────────── .tool(tool("javaducker_reladomo_relationships", "Get a Reladomo object's attributes, relationships, and metadata in one call.", @@ -464,6 +502,33 @@ static Map dependents(String artifactId) throws Exception { return r; } + @SuppressWarnings("unchecked") + static Map indexHealth() throws Exception { + Map summary = httpGet("/stale/summary"); + int staleCount = ((Number) summary.getOrDefault("stale_count", 0)).intValue(); + double stalePercent = ((Number) summary.getOrDefault("stale_percentage", 0.0)).doubleValue(); + long total = ((Number) summary.getOrDefault("total_checked", 0)).longValue(); + + String recommendation; + if (staleCount == 0) { + recommendation = "All " + total + " indexed files are current. No action needed."; + } else if (stalePercent > 10) { + recommendation = "More than 10% of indexed files are stale (" + staleCount + "/" + total + + "). Consider running a full re-index with javaducker_index_directory."; + } else { + List> staleFiles = (List>) summary.get("stale"); + List paths = staleFiles != null + ? staleFiles.stream().limit(5) + .map(f -> (String) f.get("original_client_path")) + .filter(Objects::nonNull).toList() + : List.of(); + recommendation = staleCount + " file(s) are stale. Re-index them with javaducker_index_file: " + paths; + } + summary.put("recommendation", recommendation); + summary.put("health_status", stalePercent > 10 ? "degraded" : "healthy"); + return summary; + } + // ── HTTP helpers ────────────────────────────────────────────────────────── static Map httpGet(String path) throws Exception { diff --git a/drom-plans/quick-win-blame.md b/drom-plans/quick-win-blame.md new file mode 100644 index 0000000..3500245 --- /dev/null +++ b/drom-plans/quick-win-blame.md @@ -0,0 +1,46 @@ +--- +title: Quick Win — javaducker_blame +status: completed +created: 2026-03-28 +updated: 2026-03-28 +current_chapter: 2 +--- + +# Plan: Quick Win — javaducker_blame + +Add a `javaducker_blame` MCP tool that wraps `git blame` with indexed context — given a file or artifact, return who last touched each section, when, and the commit message. Enriched with JavaDucker metadata (summary, tags, dependencies). + +## Chapter 1: Git Blame Service +**Status:** completed +**Depends on:** none + +- [x] Create `GitBlameService.java` in `server/service/` (~150 lines) — run `git blame --porcelain ` via `ProcessBuilder`, parse output into structured records: `BlameEntry(lineStart, lineEnd, commitHash, author, authorDate, commitMessage, content)` +- [x] Handle edge cases: file not in git, binary files, files outside PROJECT_ROOT, git not installed +- [x] Add method `blameForArtifact(artifactId)` — look up `original_client_path` from `artifacts` table, run blame on that path +- [x] Add method `blameForLines(filePath, startLine, endLine)` — blame a specific range (useful when Claude is looking at a search result with line numbers) +- [x] Write `GitBlameServiceTest` — test porcelain parsing, file-not-found, range queries + +**Notes:** +> Use `--porcelain` format for machine-readable output. Cache blame results in memory (LRU, 50 files) since blame is expensive. PROJECT_ROOT env var gives the repo root. + +## Chapter 2: REST Endpoint & MCP Tool +**Status:** completed +**Depends on:** Chapter 1 + +- [x] Add `GET /api/blame/{artifactId}` endpoint — returns blame entries for the full file, enriched with artifact summary if available +- [x] Add `POST /api/blame` endpoint — body: `{filePath, startLine?, endLine?}` — blame by path with optional range +- [x] Add `javaducker_blame` MCP tool to `JavaDuckerMcpServer.java` — params: `file_path` (required), `start_line` (optional), `end_line` (optional). Description: "Show who last changed each line of a file, with commit info. Optionally narrow to a line range." +- [x] Enrich blame response: for each unique commit, include the commit message. For the file, include artifact summary and dependency count if indexed +- [x] Write integration test — blame a real file in the repo, verify structure + +**Notes:** +> Keep the MCP tool response concise — group consecutive lines by the same commit into ranges, don't return per-line entries for 500-line files. Example: "lines 1-45: alice, 2026-03-20, 'Add auth middleware'" + +--- + +## Risks +- git must be available on PATH — fail gracefully with clear error if not +- Large files produce verbose blame — cap at 500 lines or summarize by commit + +## Open Questions +- None — straightforward feature diff --git a/drom-plans/quick-win-explain.md b/drom-plans/quick-win-explain.md new file mode 100644 index 0000000..1489a33 --- /dev/null +++ b/drom-plans/quick-win-explain.md @@ -0,0 +1,45 @@ +--- +title: Quick Win — javaducker_explain +status: completed +created: 2026-03-28 +updated: 2026-03-28 +current_chapter: 2 +--- + +# Plan: Quick Win — javaducker_explain + +Add a `javaducker_explain` MCP tool that returns everything JavaDucker knows about a file in one call: summary, dependency chain, dependents, tags, classification, related plans/ADRs, co-change partners, and blame highlights. A single-call context loader for Claude. + +## Chapter 1: Explain Service +**Status:** completed +**Depends on:** none + +- [x] Create `ExplainService.java` in `server/service/` (~200 lines) — aggregates data from existing services. Constructor-injected: `ArtifactService`, `DependencyService`, `SearchService`, `ContentIntelligenceService`. Optional: `GitBlameService`, `CoChangeService` (may not exist yet — use try-catch or Optional injection) +- [x] Add method `explain(artifactId)` — returns a composite map with sections: `file` (name, path, size, status, indexed_at), `summary` (from artifact summary), `dependencies` (imports this file uses), `dependents` (files that import this one), `classification` (doc_type, tags, freshness), `salient_points` (decisions, risks, actions from content intelligence), `related_artifacts` (by concept links), `blame_highlights` (top 3 most recent committers + their commit messages, if GitBlameService available), `co_changes` (top 5 files commonly edited together, if CoChangeService available) +- [x] Handle missing data gracefully — each section is optional. If a service throws or returns null, that section is omitted, not the whole response +- [x] Write `ExplainServiceTest` — test with full data, test with partial data (no blame, no co-change), test with unknown artifactId + +**Notes:** +> This is a read-only aggregation service. It calls existing services — no new tables, no new data. Its value is combining 6-8 separate API calls into one. + +## Chapter 2: REST Endpoint & MCP Tool +**Status:** completed +**Depends on:** Chapter 1 + +- [x] Add `GET /api/explain/{artifactId}` endpoint — returns the full explain bundle +- [x] Add `POST /api/explain` endpoint — body: `{filePath}` — resolve to artifact_id first, then explain. If not indexed, return a minimal response with just the file path and a note that it's not indexed +- [x] Add `javaducker_explain` MCP tool — params: `file_path` (required). Description: "Get everything JavaDucker knows about a file: summary, dependencies, dependents, tags, classification, related plans, blame highlights, and co-change partners. One call for full context." +- [x] Keep response compact — summaries not full text, top-N for lists (5 deps, 5 dependents, 5 co-changes, 3 blame entries). Include counts so Claude knows there's more if needed +- [x] Write integration test — explain a real indexed file, verify all sections present + +**Notes:** +> This will likely become the most-called MCP tool. Claude should use it before editing any file to understand full context. Keep the response under 2K tokens. + +--- + +## Risks +- Response could be large if all sections are populated — enforce limits per section +- Depends on other quick wins (blame, related) for full richness, but works without them + +## Open Questions +- Should explain also return recent search queries that matched this file? (might be noise) diff --git a/drom-plans/quick-win-related.md b/drom-plans/quick-win-related.md new file mode 100644 index 0000000..ed972b5 --- /dev/null +++ b/drom-plans/quick-win-related.md @@ -0,0 +1,48 @@ +--- +title: Quick Win — javaducker_related +status: completed +created: 2026-03-28 +updated: 2026-03-28 +current_chapter: 2 +--- + +# Plan: Quick Win — javaducker_related + +Add a `javaducker_related` MCP tool that finds files commonly edited together by analyzing git log co-change history. When Claude is editing a file, it can ask "what other files usually change with this one?" to avoid missing related updates. + +## Chapter 1: Co-Change Analysis Service +**Status:** completed +**Depends on:** none + +- [x] Create `CoChangeService.java` in `server/service/` (~180 lines) — run `git log --name-only --pretty=format:"COMMIT:%H" --since="6 months ago"` via `ProcessBuilder`, parse into commit→files map. For a given file, find all commits that touched it, then count co-occurrences of other files across those commits. Rank by frequency +- [x] Add `cochange_cache` table to `SchemaBootstrap` — `file_a VARCHAR, file_b VARCHAR, co_change_count INTEGER, last_commit_date TIMESTAMP, PRIMARY KEY (file_a, file_b)` — precomputed cache, rebuilt on demand +- [x] Add method `buildCoChangeIndex()` — parse full git log, populate cache table. Idempotent (DELETE + INSERT) +- [x] Add method `getRelatedFiles(filePath, maxResults)` — query cache, return ranked list with co-change count and last shared commit date +- [x] Filter out noise: ignore files that appear in >50% of all commits (build scripts, lockfiles), ignore commits with >30 files (bulk renames/reformats) +- [x] Write `CoChangeServiceTest` — test parsing, ranking, noise filtering, empty repo + +**Notes:** +> The git log parse is expensive (~2-5s for large repos) so we cache in DuckDB. Rebuild on demand via endpoint or when staleness is detected. The 6-month window keeps results relevant. + +## Chapter 2: REST Endpoint & MCP Tool +**Status:** completed +**Depends on:** Chapter 1 + +- [x] Add `GET /api/related/{artifactId}` endpoint — look up original_client_path, return co-change partners ranked by frequency +- [x] Add `POST /api/related` endpoint — body: `{filePath, maxResults?, rebuild?}`. If `rebuild: true`, refresh the co-change cache first +- [x] Add `javaducker_related` MCP tool — params: `file_path` (required), `max_results` (optional, default 10). Description: "Find files that are commonly edited together with this file, based on git history. Helps identify related files you might need to update." +- [x] Enrich response: for each related file, include co-change count, last shared commit, and whether it's currently indexed in JavaDucker (with summary if so) +- [x] Add `POST /api/rebuild-cochange` endpoint — force rebuild the cache +- [x] Write integration test — build index from real repo, query related files + +**Notes:** +> This is one of the most useful tools for Claude — when making a change, knowing what usually changes together prevents incomplete PRs. + +--- + +## Risks +- Git log parsing on very large repos (10K+ commits) could be slow — the 6-month window mitigates this +- Projects without git history return empty results — handle gracefully + +## Open Questions +- Should the co-change cache auto-rebuild on a timer, or only on demand? diff --git a/drom-plans/session-transcript-indexing.md b/drom-plans/session-transcript-indexing.md new file mode 100644 index 0000000..e56d0f7 --- /dev/null +++ b/drom-plans/session-transcript-indexing.md @@ -0,0 +1,74 @@ +--- +title: Session Transcript Indexing +status: in-progress +created: 2026-03-28 +updated: 2026-03-28 +current_chapter: 3 +--- + +# Plan: Session Transcript Indexing + +Index Claude Code conversation transcripts from `~/.claude/projects/` so that past sessions are searchable. When Claude asks "what did we decide about auth last week?" it searches actual session history, not just MEMORY.md summaries. + +## Chapter 1: Transcript Parser & Data Model +**Status:** completed +**Depends on:** none + +- [x] Create `SessionTranscriptParser.java` in `server/ingestion/` (~200 lines) — parse JSONL conversation files from `~/.claude/projects/`. Extract: role (user/assistant/tool), text content, tool names, timestamps, session ID. Skip binary/image content +- [x] Create `SessionTranscript` record in `server/model/` — fields: sessionId, projectPath, messageIndex, role, content, toolName, timestamp, tokenEstimate +- [x] Add `session_transcripts` table to `SchemaBootstrap` — `session_id VARCHAR NOT NULL, project_path VARCHAR, message_index INTEGER, role VARCHAR, content VARCHAR, tool_name VARCHAR, timestamp TIMESTAMP, token_estimate INTEGER, PRIMARY KEY (session_id, message_index)` +- [x] Add `session_decisions` table — `session_id VARCHAR NOT NULL, decision_text VARCHAR, context VARCHAR, decided_at TIMESTAMP, tags VARCHAR` — extracted decisions/conclusions from sessions +- [x] Add index: `idx_transcripts_session` on session_transcripts(session_id), `idx_transcripts_role` on session_transcripts(role), `idx_decisions_tags` on session_decisions(tags) +- [x] Write `SessionTranscriptParserTest` — test JSONL parsing, skip binary, handle malformed lines, empty files + +**Notes:** +> Claude Code stores sessions as JSONL in ~/.claude/projects//. Each line is a JSON object with role, content, tool_use fields. The parser should be tolerant of format changes. + +## Chapter 2: Ingestion & Chunking +**Status:** completed +**Depends on:** Chapter 1 + +- [x] Create `SessionIngestionService.java` in `server/service/` (~180 lines) — scan a project directory for session files, parse each, chunk assistant messages (user messages are usually short), store in DuckDB. Dedup by session_id + message_index +- [x] Add session-aware chunking to `Chunker` or create `SessionChunker` — chunk long assistant responses, preserve tool call boundaries as chunk breaks, tag chunks with session_id +- [x] Hook into existing embedding pipeline — session chunks get TF-IDF embeddings like any other artifact, stored in `chunks` table with `source_type = 'session'` +- [x] Add incremental ingestion — track last-indexed session file modification time, only re-parse changed files +- [x] Write `SessionIngestionServiceTest` — test dedup, incremental skip, chunking boundaries + +**Notes:** +> Sessions can be large (100K+ tokens). Only index assistant and user messages — skip tool_result payloads (they're the code itself, already indexed). Decision extraction happens in Chapter 4. + +## Chapter 3: REST Endpoints & MCP Tools +**Status:** pending +**Depends on:** Chapter 2 + +- [ ] Add REST endpoints to `JavaDuckerRestController`: `POST /api/index-sessions` (body: `{projectPath, maxSessions?}`), `GET /api/sessions` (list indexed sessions with date, token count), `GET /api/session/{sessionId}` (full transcript), `POST /api/search-sessions` (body: `{phrase, mode, maxResults}` — search only session content) +- [ ] Add MCP tools to `JavaDuckerMcpServer.java`: `javaducker_index_sessions` (index sessions from a project path), `javaducker_search_sessions` (search past conversations — phrase, mode), `javaducker_session_decisions` (list decisions extracted from sessions, optionally filtered by tag) +- [ ] Add `javaducker_session_context` MCP tool — given a topic/query, return a compact context bundle: relevant session excerpts + any related MEMORY.md entries + related artifacts. One call to get full historical context +- [ ] Write integration test — index sample session JSONL, search it, verify results + +**Notes:** +> `javaducker_session_context` is the high-value tool — it's what Claude calls when it needs to understand history. Keep the response compact: excerpts, not full transcripts. + +## Chapter 4: Decision Extraction & Session Summaries +**Status:** pending +**Depends on:** Chapter 3 + +- [ ] Add `POST /api/extract-session-decisions` endpoint — accepts sessionId + list of decisions (text, context, tags). Stores in `session_decisions` table. Designed to be called by Claude after reading a session +- [ ] Add `javaducker_extract_decisions` MCP tool — write side for decision storage +- [ ] Add `javaducker_recent_decisions` MCP tool — return decisions from last N sessions, filterable by tag/topic +- [ ] Create `scripts/index-sessions.sh` — helper script that finds the project path in `~/.claude/projects/` and calls the index endpoint. Can be wired as a SessionStart hook for auto-indexing +- [ ] Write tests for decision storage and retrieval + +**Notes:** +> Decision extraction is LLM-driven — Claude reads the transcript and identifies decisions. The MCP tool just stores what Claude extracts. This pairs with the content intelligence enrichment pipeline (O3). + +--- + +## Risks +- Session JSONL format may change across Claude Code versions — parser should be defensive +- Large session files (100K+ tokens) may need streaming parse, not load-all-at-once +- Privacy: transcripts may contain secrets typed by user — warn in docs, don't index `.env` content + +## Open Questions +- Should session indexing happen automatically via a SessionStart hook, or only on demand? +- Should tool_result content be indexed (it's often code that's already indexed separately)? diff --git a/drom-plans/stale-index-warning.md b/drom-plans/stale-index-warning.md new file mode 100644 index 0000000..6014890 --- /dev/null +++ b/drom-plans/stale-index-warning.md @@ -0,0 +1,73 @@ +--- +title: Stale Index Warning +status: in-progress +created: 2026-03-28 +updated: 2026-03-28 +current_chapter: 4 +--- + +# Plan: Stale Index Warning + +Enhance the existing staleness infrastructure (`StalenessService`, `javaducker_stale` MCP tool, `/stale` REST endpoint) with **proactive warnings** so Claude is automatically informed when search results may be stale — without needing to manually call the stale tool. + +**What exists today:** +- `StalenessService.checkStaleness(filePaths)` — compares file mtime vs `indexed_at` for specific paths +- `javaducker_stale` MCP tool — accepts `file_paths` or `git_diff_ref`, returns stale/current/not_indexed counts +- `/api/stale` REST endpoint — POST with file_paths body + +**What's missing:** +- No bulk "check all indexed files" mode — you must supply paths +- Search results don't include staleness warnings +- No way to get a quick health-check summary ("12 of 200 files stale") +- No proactive nudge to re-index + +## Chapter 1: Bulk Staleness Check +**Status:** completed +**Depends on:** none + +- [x] Add `StalenessService.checkAll()` — query all distinct `original_client_path` from `artifacts` where `status = 'INDEXED'`, then run the existing mtime comparison logic. Return summary: `{stale_count, current_count, missing_count, total, stale_files: [{path, artifact_id, indexed_at, file_modified_at}]}` +- [x] Add `GET /api/stale/summary` endpoint — calls `checkAll()`, returns the summary. Lightweight: only returns counts + stale file list (no embeddings, no chunks) +- [x] Add unit test in `StalenessServiceTest` — index 3 files, touch 1 on disk, verify `checkAll()` reports 1 stale, 2 current +- [x] Verify existing `/api/stale` POST still works unchanged + +**Notes:** +> `checkAll()` iterates all indexed paths. For large indexes (1000+ files), this is I/O-bound on stat calls. Acceptable for now — can batch later if needed. + +## Chapter 2: Staleness Banner in Search Results +**Status:** completed +**Depends on:** Chapter 1 + +- [x] Modify `SearchService.search()` (or the REST endpoint `POST /api/search`) to run a lightweight staleness check on the files that appear in search results. For each result, compare its artifact's `indexed_at` against the file's current mtime +- [x] Add a `stale` boolean field to each search result entry and a top-level `staleness_warning` string to the search response (e.g., `"3 of 8 results come from stale files — run re-index?"`) +- [x] If ALL results are current, omit the warning (no noise) +- [x] Update `JavaDuckerMcpServer.javaducker_search` tool — the warning string is already in the JSON, Claude will see it naturally +- [ ] Add test: index file, search (no warning), touch file on disk, search again (warning present) + +**Notes:** +> Only check files that appear in results — don't scan the whole index on every search. Keep the overhead to a few stat() calls per search. + +## Chapter 3: Health-Check MCP Tool +**Status:** completed +**Depends on:** Chapter 1 + +- [x] Add `javaducker_index_health` MCP tool to `JavaDuckerMcpServer.java` — no params required. Calls `GET /api/stale/summary`. Returns a human-readable report: "Index health: 188/200 files current, 12 stale (6%). Stale files: [list]. Run javaducker_index with file_paths to refresh." +- [x] Include recommendation threshold: if >10% stale, suggest full re-index; if ≤10%, suggest targeted re-index of the listed files +- [x] Add to `JavaDuckerClient.java` CLI as `index-health` subcommand +- [ ] Add integration test — verify tool returns expected structure + +**Notes:** +> This is the tool Claude calls proactively or when starting a session. The response is designed to be actionable — it tells Claude exactly what to do next. + +## Chapter 4: Search-Time Hook (MCP Server Enhancement) +**Status:** pending +**Depends on:** Chapter 2, Chapter 3 + +- [ ] In `JavaDuckerMcpServer`, wrap the `javaducker_search` handler: after returning search results, if `staleness_warning` is non-null, append a footer line: `"\n⚠️ {warning} Use javaducker_index to refresh."` +- [ ] Similarly, wrap `javaducker_summarize` — if the artifact being summarized is stale, prepend: `"⚠️ This file has changed since indexing — summary may be outdated."` +- [ ] Add a `--staleness-check` flag (default: on) to the MCP server config so users can disable if overhead is unwanted +- [ ] Test: verify search result includes footer when stale, omits when current + +**Notes:** +> The footer approach means Claude sees the warning in the tool response without any protocol-level changes. Claude can then decide to re-index or proceed. + +--- diff --git a/src/main/java/com/javaducker/client/JavaDuckerClient.java b/src/main/java/com/javaducker/client/JavaDuckerClient.java index b447eb9..c017866 100644 --- a/src/main/java/com/javaducker/client/JavaDuckerClient.java +++ b/src/main/java/com/javaducker/client/JavaDuckerClient.java @@ -38,6 +38,7 @@ JavaDuckerClient.ConceptTimelineCmd.class, JavaDuckerClient.StaleContentCmd.class, JavaDuckerClient.ConceptHealthCmd.class, + JavaDuckerClient.IndexHealthCmd.class, }) public class JavaDuckerClient implements Runnable { @@ -570,4 +571,54 @@ public void run() { } } } + + // ── Index Health CLI command ───────────────────────────── + @Command(name = "index-health", description = "Check index health: current vs stale files with recommendation") + static class IndexHealthCmd implements Runnable { + @CommandLine.ParentCommand JavaDuckerClient parent; + + @Override + @SuppressWarnings("unchecked") + public void run() { + try { + Map resp = get(baseUrl(parent) + "/stale/summary"); + int staleCount = resp.get("stale_count") != null + ? ((Number) resp.get("stale_count")).intValue() : 0; + double stalePercent = resp.get("stale_percentage") != null + ? ((Number) resp.get("stale_percentage")).doubleValue() : 0.0; + long total = resp.get("total_checked") != null + ? ((Number) resp.get("total_checked")).longValue() : 0; + int currentCount = resp.get("current") != null + ? ((Number) resp.get("current")).intValue() : 0; + + String status = stalePercent > 10 ? "DEGRADED" : "HEALTHY"; + System.out.println("Index Health: " + status); + System.out.println(" Total files: " + total); + System.out.println(" Current: " + currentCount); + System.out.println(" Stale: " + staleCount + + " (" + String.format("%.1f", stalePercent) + "%)"); + + if (staleCount > 0) { + List> staleFiles = + (List>) resp.get("stale"); + if (staleFiles != null) { + System.out.println(" Stale files:"); + for (Map f : staleFiles) { + System.out.println(" " + f.get("original_client_path")); + } + } + if (stalePercent > 10) { + System.out.println("\nRecommendation: >10% stale — run a full re-index."); + } else { + System.out.println("\nRecommendation: Re-index the listed stale files."); + } + } else { + System.out.println("\nAll indexed files are current. No action needed."); + } + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + System.exit(1); + } + } + } } diff --git a/src/main/java/com/javaducker/server/db/SchemaBootstrap.java b/src/main/java/com/javaducker/server/db/SchemaBootstrap.java index a4fd121..f7928ab 100644 --- a/src/main/java/com/javaducker/server/db/SchemaBootstrap.java +++ b/src/main/java/com/javaducker/server/db/SchemaBootstrap.java @@ -378,6 +378,55 @@ ON artifacts (freshness) ON artifacts (enrichment_status) """); + // Co-change analysis cache + stmt.execute(""" + CREATE TABLE IF NOT EXISTS cochange_cache ( + file_a VARCHAR NOT NULL, + file_b VARCHAR NOT NULL, + co_change_count INTEGER, + last_commit_date TIMESTAMP, + PRIMARY KEY (file_a, file_b) + ) + """); + + // Session transcript tables + stmt.execute(""" + CREATE TABLE IF NOT EXISTS session_transcripts ( + session_id VARCHAR NOT NULL, + project_path VARCHAR, + message_index INTEGER, + role VARCHAR, + content VARCHAR, + tool_name VARCHAR, + timestamp TIMESTAMP, + token_estimate INTEGER, + PRIMARY KEY (session_id, message_index) + ) + """); + + stmt.execute(""" + CREATE TABLE IF NOT EXISTS session_decisions ( + session_id VARCHAR NOT NULL, + decision_text VARCHAR, + context VARCHAR, + decided_at TIMESTAMP, + tags VARCHAR + ) + """); + + stmt.execute(""" + CREATE INDEX IF NOT EXISTS idx_transcripts_session + ON session_transcripts (session_id) + """); + stmt.execute(""" + CREATE INDEX IF NOT EXISTS idx_transcripts_role + ON session_transcripts (role) + """); + stmt.execute(""" + CREATE INDEX IF NOT EXISTS idx_decisions_tags + ON session_decisions (tags) + """); + log.info("Database schema created/verified"); } } diff --git a/src/main/java/com/javaducker/server/ingestion/SessionTranscriptParser.java b/src/main/java/com/javaducker/server/ingestion/SessionTranscriptParser.java new file mode 100644 index 0000000..1db8b40 --- /dev/null +++ b/src/main/java/com/javaducker/server/ingestion/SessionTranscriptParser.java @@ -0,0 +1,265 @@ +package com.javaducker.server.ingestion; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.javaducker.server.model.SessionTranscript; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +/** + * Parses JSONL conversation files from ~/.claude/projects/. + * Each line is a JSON object representing a message in a Claude Code session. + */ +@Component +public class SessionTranscriptParser { + + private static final Logger log = LoggerFactory.getLogger(SessionTranscriptParser.class); + private static final int MAX_CONTENT_LENGTH = 50_000; + + private final ObjectMapper objectMapper; + + public SessionTranscriptParser() { + this.objectMapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + /** + * Parse a session JSONL file into a list of SessionTranscript records. + * + * @param filePath path to the .jsonl file + * @return list of parsed transcripts; malformed lines are skipped + */ + public List parseSessionFile(Path filePath) { + List results = new ArrayList<>(); + if (filePath == null || !Files.exists(filePath)) { + return results; + } + + String fileName = filePath.getFileName().toString(); + String sessionId = fileName.endsWith(".jsonl") + ? fileName.substring(0, fileName.length() - ".jsonl".length()) + : fileName; + String projectPath = filePath.getParent() != null + ? filePath.getParent().toString() + : ""; + + try (Stream lines = Files.lines(filePath)) { + int[] index = {0}; + lines.forEach(line -> { + SessionTranscript transcript = parseLine(line, sessionId, projectPath, index[0]); + if (transcript != null) { + results.add(transcript); + } + index[0]++; + }); + } catch (IOException e) { + log.warn("Failed to read session file {}: {}", filePath, e.getMessage()); + } + + return results; + } + + /** + * Parse a single JSONL line into a SessionTranscript. + * + * @param line the raw JSON line + * @param sessionId session identifier (derived from file name) + * @param projectPath parent directory path + * @param index message index within the session + * @return a SessionTranscript, or null if the line should be skipped + */ + SessionTranscript parseLine(String line, String sessionId, String projectPath, int index) { + if (line == null || line.isBlank()) { + return null; + } + + try { + JsonNode root = objectMapper.readTree(line); + + String type = getTextOrNull(root, "type"); + String role = extractRole(root, type); + String content = extractTextContent(root); + String toolName = extractToolName(root, type); + String timestamp = getTextOrNull(root, "timestamp"); + + // Skip lines with no usable text content + if (content == null || content.isBlank()) { + return null; + } + + // Truncate very large content + if (content.length() > MAX_CONTENT_LENGTH) { + content = content.substring(0, MAX_CONTENT_LENGTH); + } + + int tokenEstimate = content.length() / 4; + + return new SessionTranscript( + sessionId, + projectPath, + index, + role, + content, + toolName, + timestamp, + tokenEstimate + ); + } catch (Exception e) { + log.warn("Skipping malformed JSON line at index {} in session {}: {}", + index, sessionId, e.getMessage()); + return null; + } + } + + /** + * Find all .jsonl files in the given directory (non-recursive). + * + * @param projectDir the directory to scan + * @return list of paths to .jsonl files + */ + public List findSessionFiles(Path projectDir) { + List files = new ArrayList<>(); + if (projectDir == null || !Files.isDirectory(projectDir)) { + return files; + } + + try (DirectoryStream stream = Files.newDirectoryStream(projectDir, "*.jsonl")) { + for (Path entry : stream) { + if (Files.isRegularFile(entry)) { + files.add(entry); + } + } + } catch (IOException e) { + log.warn("Failed to scan directory {}: {}", projectDir, e.getMessage()); + } + + return files; + } + + private String extractRole(JsonNode root, String type) { + // Try message.role first + JsonNode message = root.get("message"); + if (message != null && message.has("role")) { + return message.get("role").asText(); + } + + // Fall back to type field + if (type != null) { + return switch (type) { + case "human" -> "user"; + case "assistant" -> "assistant"; + case "tool_use" -> "assistant"; + case "tool_result" -> "tool"; + default -> type; + }; + } + + return "unknown"; + } + + private String extractTextContent(JsonNode root) { + // Try message.content first + JsonNode message = root.get("message"); + if (message != null) { + JsonNode content = message.get("content"); + if (content != null) { + return extractFromContentNode(content); + } + } + + // Try top-level content + JsonNode content = root.get("content"); + if (content != null) { + return extractFromContentNode(content); + } + + return null; + } + + private String extractFromContentNode(JsonNode contentNode) { + // Content can be a plain string + if (contentNode.isTextual()) { + return contentNode.asText(); + } + + // Content can be an array of content blocks + if (contentNode.isArray()) { + StringBuilder sb = new StringBuilder(); + for (JsonNode block : contentNode) { + String blockType = getTextOrNull(block, "type"); + + // Skip image/binary blocks + if ("image".equals(blockType)) { + continue; + } + + // Extract text from text blocks + if ("text".equals(blockType) && block.has("text")) { + if (!sb.isEmpty()) { + sb.append("\n"); + } + sb.append(block.get("text").asText()); + } + + // Extract tool name and input for tool_use blocks + if ("tool_use".equals(blockType) && block.has("name")) { + if (!sb.isEmpty()) { + sb.append("\n"); + } + sb.append("[tool_use: ").append(block.get("name").asText()).append("]"); + } + + // Extract tool_result content + if ("tool_result".equals(blockType) && block.has("content")) { + String nested = extractFromContentNode(block.get("content")); + if (nested != null && !nested.isBlank()) { + if (!sb.isEmpty()) { + sb.append("\n"); + } + sb.append(nested); + } + } + } + return sb.isEmpty() ? null : sb.toString(); + } + + return null; + } + + private String extractToolName(JsonNode root, String type) { + // tool_use type: look for name field + if ("tool_use".equals(type)) { + return getTextOrNull(root, "name"); + } + + // Check in message content array for tool_use blocks + JsonNode message = root.get("message"); + if (message != null) { + JsonNode content = message.get("content"); + if (content != null && content.isArray()) { + for (JsonNode block : content) { + if ("tool_use".equals(getTextOrNull(block, "type"))) { + return getTextOrNull(block, "name"); + } + } + } + } + + return null; + } + + private String getTextOrNull(JsonNode node, String field) { + JsonNode child = node.get(field); + return (child != null && child.isTextual()) ? child.asText() : null; + } +} diff --git a/src/main/java/com/javaducker/server/model/SessionTranscript.java b/src/main/java/com/javaducker/server/model/SessionTranscript.java new file mode 100644 index 0000000..c3ce912 --- /dev/null +++ b/src/main/java/com/javaducker/server/model/SessionTranscript.java @@ -0,0 +1,12 @@ +package com.javaducker.server.model; + +public record SessionTranscript( + String sessionId, + String projectPath, + int messageIndex, + String role, + String content, + String toolName, + String timestamp, + int tokenEstimate +) {} diff --git a/src/main/java/com/javaducker/server/rest/JavaDuckerRestController.java b/src/main/java/com/javaducker/server/rest/JavaDuckerRestController.java index c3fc569..83e6692 100644 --- a/src/main/java/com/javaducker/server/rest/JavaDuckerRestController.java +++ b/src/main/java/com/javaducker/server/rest/JavaDuckerRestController.java @@ -7,10 +7,7 @@ import org.springframework.web.multipart.MultipartFile; import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @RestController @@ -27,13 +24,19 @@ public class JavaDuckerRestController { private final FileWatcher fileWatcher; private final ReladomoQueryService reladomoQueryService; private final ContentIntelligenceService contentIntelligenceService; + private final GitBlameService gitBlameService; + private final CoChangeService coChangeService; + private final ExplainService explainService; public JavaDuckerRestController(UploadService uploadService, ArtifactService artifactService, SearchService searchService, StatsService statsService, ProjectMapService projectMapService, StalenessService stalenessService, DependencyService dependencyService, FileWatcher fileWatcher, ReladomoQueryService reladomoQueryService, - ContentIntelligenceService contentIntelligenceService) { + ContentIntelligenceService contentIntelligenceService, + GitBlameService gitBlameService, + CoChangeService coChangeService, + ExplainService explainService) { this.uploadService = uploadService; this.artifactService = artifactService; this.searchService = searchService; @@ -44,6 +47,9 @@ public JavaDuckerRestController(UploadService uploadService, ArtifactService art this.fileWatcher = fileWatcher; this.reladomoQueryService = reladomoQueryService; this.contentIntelligenceService = contentIntelligenceService; + this.gitBlameService = gitBlameService; + this.coChangeService = coChangeService; + this.explainService = explainService; } @GetMapping("/health") @@ -98,7 +104,54 @@ public ResponseEntity> search(@RequestBody Map searchService.semanticSearch(phrase, maxResults); default -> searchService.hybridSearch(phrase, maxResults); }; - return ResponseEntity.ok(Map.of("total_results", results.size(), "results", results)); + + Map response = new LinkedHashMap<>(); + response.put("total_results", results.size()); + response.put("results", results); + + if (!results.isEmpty()) { + try { + // Collect unique artifact IDs from results + Set artifactIds = results.stream() + .map(r -> (String) r.get("artifact_id")) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + + // Look up original_client_path for each artifact + Map artToPath = new HashMap<>(); + for (String artId : artifactIds) { + Map status = artifactService.getStatus(artId); + if (status != null && status.get("original_client_path") != null) { + artToPath.put(artId, status.get("original_client_path")); + } + } + + List filePaths = new ArrayList<>(new LinkedHashSet<>(artToPath.values())); + if (!filePaths.isEmpty()) { + Map staleness = stalenessService.checkStaleness(filePaths); + List staleList = (List) staleness.get("stale"); + if (staleList != null && !staleList.isEmpty()) { + Set stalePaths = new HashSet<>(); + for (Object s : staleList) { + Map staleItem = (Map) s; + stalePaths.add((String) staleItem.get("original_client_path")); + } + for (Map result : results) { + String path = artToPath.get(result.get("artifact_id")); + result.put("stale", path != null && stalePaths.contains(path)); + } + int staleCount = stalePaths.size(); + response.put("staleness_warning", + staleCount + " of " + filePaths.size() + + " result files are stale — run re-index to refresh."); + } + } + } catch (Exception e) { + // Don't fail the search if staleness check fails + } + } + + return ResponseEntity.ok(response); } @GetMapping("/summary/{artifactId}") @@ -113,6 +166,11 @@ public ResponseEntity> getProjectMap() throws Exception { return ResponseEntity.ok(projectMapService.getProjectMap()); } + @GetMapping("/stale/summary") + public ResponseEntity> staleSummary() throws Exception { + return ResponseEntity.ok(stalenessService.checkAll()); + } + @SuppressWarnings("unchecked") @PostMapping("/stale") public ResponseEntity> checkStale(@RequestBody Map body) throws Exception { @@ -166,6 +224,70 @@ public ResponseEntity> watchStatus() { ? fileWatcher.getWatchedDirectory().toString() : "")); } + // ── Git Blame endpoints ──────────────────────────────────────────────── + + @GetMapping("/blame/{artifactId}") + public ResponseEntity getBlame(@PathVariable String artifactId) throws Exception { + List entries = gitBlameService.blameForArtifact(artifactId); + Map summary = artifactService.getSummary(artifactId); + Map result = new LinkedHashMap<>(); + result.put("artifact_id", artifactId); + if (summary != null) result.put("summary", summary); + result.put("blame", formatBlameEntries(entries)); + return ResponseEntity.ok(result); + } + + @PostMapping("/blame") + public ResponseEntity blameByPath(@RequestBody Map body) throws Exception { + String filePath = (String) body.get("filePath"); + Integer startLine = body.containsKey("startLine") ? ((Number) body.get("startLine")).intValue() : null; + Integer endLine = body.containsKey("endLine") ? ((Number) body.get("endLine")).intValue() : null; + List entries; + if (startLine != null && endLine != null) { + entries = gitBlameService.blameForLines(filePath, startLine, endLine); + } else { + entries = gitBlameService.blame(filePath); + } + Map result = new LinkedHashMap<>(); + result.put("file", filePath); + result.put("blame", formatBlameEntries(entries)); + return ResponseEntity.ok(result); + } + + private List> formatBlameEntries(List entries) { + List> formatted = new ArrayList<>(); + for (GitBlameService.BlameEntry e : entries) { + Map entry = new LinkedHashMap<>(); + entry.put("lines", e.lineStart() == e.lineEnd() + ? String.valueOf(e.lineStart()) + : e.lineStart() + "-" + e.lineEnd()); + entry.put("commit", e.commitHash().substring(0, Math.min(8, e.commitHash().length()))); + entry.put("author", e.author()); + entry.put("date", e.authorDate() != null ? e.authorDate().toString() : null); + entry.put("message", e.commitMessage()); + formatted.add(entry); + } + return formatted; + } + + // ── Explain endpoints ───────────────────────────────────────────────── + + @GetMapping("/explain/{artifactId}") + public ResponseEntity explain(@PathVariable String artifactId) throws Exception { + Map result = explainService.explain(artifactId); + if (result == null) return ResponseEntity.notFound().build(); + return ResponseEntity.ok(result); + } + + @PostMapping("/explain") + public ResponseEntity explainByPath(@RequestBody Map body) throws Exception { + String filePath = (String) body.get("filePath"); + if (filePath == null || filePath.isBlank()) { + return ResponseEntity.badRequest().body(Map.of("error", "filePath is required")); + } + return ResponseEntity.ok(explainService.explainByPath(filePath)); + } + // ── Content Intelligence: write endpoints ────────────────────────────── @SuppressWarnings("unchecked") @@ -317,6 +439,40 @@ public ResponseEntity> searchSynthesis(@RequestParam String return ResponseEntity.ok(Map.of("keyword", keyword, "results", results, "count", results.size())); } + // ── Co-Change / Related Files endpoints ──────────────────────────── + + @GetMapping("/related/{artifactId}") + public ResponseEntity getRelated(@PathVariable String artifactId) throws Exception { + Map status = artifactService.getStatus(artifactId); + if (status == null) return ResponseEntity.notFound().build(); + String path = status.get("original_client_path"); + if (path == null || path.isBlank()) { + return ResponseEntity.ok(Map.of("artifact_id", artifactId, "related", List.of())); + } + List> related = coChangeService.getRelatedFiles(path, 10); + return ResponseEntity.ok(Map.of( + "artifact_id", artifactId, "file_path", path, + "related", related, "count", related.size())); + } + + @PostMapping("/related") + public ResponseEntity relatedByPath(@RequestBody Map body) throws Exception { + String filePath = (String) body.get("filePath"); + int maxResults = body.containsKey("maxResults") + ? ((Number) body.get("maxResults")).intValue() : 10; + boolean rebuild = Boolean.TRUE.equals(body.get("rebuild")); + if (rebuild) coChangeService.buildCoChangeIndex(); + List> related = coChangeService.getRelatedFiles(filePath, maxResults); + return ResponseEntity.ok(Map.of( + "file_path", filePath, "related", related, "count", related.size())); + } + + @PostMapping("/rebuild-cochange") + public ResponseEntity> rebuildCoChange() throws Exception { + coChangeService.buildCoChangeIndex(); + return ResponseEntity.ok(Map.of("status", "rebuilt")); + } + // ── Reladomo endpoints ─────────────────────────────────────────────── @GetMapping("/reladomo/relationships/{objectName}") diff --git a/src/main/java/com/javaducker/server/service/CoChangeService.java b/src/main/java/com/javaducker/server/service/CoChangeService.java new file mode 100644 index 0000000..85408cd --- /dev/null +++ b/src/main/java/com/javaducker/server/service/CoChangeService.java @@ -0,0 +1,212 @@ +package com.javaducker.server.service; + +import com.javaducker.server.db.DuckDBDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.file.Path; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.*; +import java.util.stream.Collectors; + +@Service +public class CoChangeService { + + private static final Logger log = LoggerFactory.getLogger(CoChangeService.class); + private static final int MAX_FILES_PER_COMMIT = 30; + private static final double FREQUENT_FILE_THRESHOLD = 0.50; + + private final DuckDBDataSource dataSource; + private final Path projectRoot; + + public CoChangeService(DuckDBDataSource dataSource) { + this.dataSource = dataSource; + String root = System.getenv("PROJECT_ROOT"); + this.projectRoot = Path.of(root != null ? root : "."); + } + + /** + * Rebuild the co-change index from git history. Idempotent: deletes all existing data first. + */ + public void buildCoChangeIndex() throws Exception { + String gitOutput = runGitLog(); + Map> commits = parseGitLog(gitOutput); + Map> filtered = filterNoisyCommits(commits); + Set frequentFiles = findFrequentFiles(filtered); + Map> coChanges = computeCoChanges(filtered, frequentFiles); + + dataSource.withConnection(conn -> { + try (Statement stmt = conn.createStatement()) { + stmt.execute("DELETE FROM cochange_cache"); + } + + try (PreparedStatement ps = conn.prepareStatement( + "INSERT INTO cochange_cache (file_a, file_b, co_change_count, last_commit_date) VALUES (?, ?, ?, ?)")) { + for (Map.Entry> outer : coChanges.entrySet()) { + String fileA = outer.getKey(); + for (Map.Entry inner : outer.getValue().entrySet()) { + String fileB = inner.getKey(); + int count = inner.getValue(); + ps.setString(1, fileA); + ps.setString(2, fileB); + ps.setInt(3, count); + ps.setTimestamp(4, new Timestamp(System.currentTimeMillis())); + ps.addBatch(); + } + } + ps.executeBatch(); + } + + log.info("Co-change index rebuilt: {} file pairs", coChanges.values().stream() + .mapToInt(Map::size).sum()); + return null; + }); + } + + /** + * Query co-change cache for files related to the given file path. + */ + public List> getRelatedFiles(String filePath, int maxResults) throws SQLException { + return dataSource.withConnection(conn -> { + List> results = new ArrayList<>(); + String sql = """ + SELECT file_b AS related_file, co_change_count, last_commit_date + FROM cochange_cache WHERE file_a = ? + UNION + SELECT file_a AS related_file, co_change_count, last_commit_date + FROM cochange_cache WHERE file_b = ? + ORDER BY co_change_count DESC LIMIT ? + """; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, filePath); + ps.setString(2, filePath); + ps.setInt(3, maxResults); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("related_file", rs.getString("related_file")); + row.put("co_change_count", rs.getInt("co_change_count")); + row.put("last_commit_date", rs.getTimestamp("last_commit_date")); + results.add(row); + } + } + } + return results; + }); + } + + private String runGitLog() throws Exception { + ProcessBuilder pb = new ProcessBuilder( + "git", "log", "--name-only", "--pretty=format:COMMIT:%H", "--since=6 months ago"); + pb.directory(projectRoot.toFile()); + pb.redirectErrorStream(true); + Process process = pb.start(); + + String output; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + output = reader.lines().collect(Collectors.joining("\n")); + } + + int exitCode = process.waitFor(); + if (exitCode != 0) { + log.warn("git log exited with code {}: {}", exitCode, output); + } + return output; + } + + /** + * Parse git log output into a map of commitHash to list of file paths. + * Package-private for testing. + */ + Map> parseGitLog(String output) { + Map> commits = new LinkedHashMap<>(); + if (output == null || output.isBlank()) { + return commits; + } + + String currentCommit = null; + for (String line : output.split("\n")) { + String trimmed = line.trim(); + if (trimmed.startsWith("COMMIT:")) { + currentCommit = trimmed.substring("COMMIT:".length()); + commits.put(currentCommit, new ArrayList<>()); + } else if (!trimmed.isEmpty() && currentCommit != null) { + commits.get(currentCommit).add(trimmed); + } + } + return commits; + } + + /** + * Filter out commits with more than MAX_FILES_PER_COMMIT files. + */ + private Map> filterNoisyCommits(Map> commits) { + Map> filtered = new LinkedHashMap<>(); + for (Map.Entry> entry : commits.entrySet()) { + if (entry.getValue().size() <= MAX_FILES_PER_COMMIT) { + filtered.put(entry.getKey(), entry.getValue()); + } + } + return filtered; + } + + /** + * Find files that appear in more than 50% of all commits (noisy files). + */ + Set findFrequentFiles(Map> commits) { + if (commits.isEmpty()) { + return Collections.emptySet(); + } + Map fileCounts = new HashMap<>(); + for (List files : commits.values()) { + for (String file : files) { + fileCounts.merge(file, 1, Integer::sum); + } + } + int threshold = (int) (commits.size() * FREQUENT_FILE_THRESHOLD); + Set frequent = new HashSet<>(); + for (Map.Entry entry : fileCounts.entrySet()) { + if (entry.getValue() > threshold) { + frequent.add(entry.getKey()); + } + } + return frequent; + } + + /** + * Compute co-change counts for all file pairs. Only stores the pair once (fileA < fileB). + * Package-private for testing. + */ + Map> computeCoChanges(Map> commits) { + return computeCoChanges(commits, Collections.emptySet()); + } + + Map> computeCoChanges(Map> commits, Set excludeFiles) { + Map> coChanges = new HashMap<>(); + + for (List files : commits.values()) { + List filtered = files.stream() + .filter(f -> !excludeFiles.contains(f)) + .sorted() + .distinct() + .toList(); + + for (int i = 0; i < filtered.size(); i++) { + for (int j = i + 1; j < filtered.size(); j++) { + String a = filtered.get(i); + String b = filtered.get(j); + coChanges.computeIfAbsent(a, k -> new HashMap<>()) + .merge(b, 1, Integer::sum); + } + } + } + return coChanges; + } +} diff --git a/src/main/java/com/javaducker/server/service/ExplainService.java b/src/main/java/com/javaducker/server/service/ExplainService.java new file mode 100644 index 0000000..5a26d9f --- /dev/null +++ b/src/main/java/com/javaducker/server/service/ExplainService.java @@ -0,0 +1,252 @@ +package com.javaducker.server.service; + +import com.javaducker.server.db.DuckDBDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; + +/** + * Read-only aggregation service that collects everything JavaDucker knows + * about a file into a single response. Each section is optional — if a + * backing service throws or returns null, that section is simply omitted. + */ +@Service +public class ExplainService { + + private static final Logger log = LoggerFactory.getLogger(ExplainService.class); + + private final ArtifactService artifactService; + private final DependencyService dependencyService; + private final ContentIntelligenceService contentIntelligenceService; + private final DuckDBDataSource dataSource; + + // Optional services — may not be built yet + private final Object gitBlameService; + private final Object coChangeService; + + @Autowired + public ExplainService(ArtifactService artifactService, + DependencyService dependencyService, + ContentIntelligenceService contentIntelligenceService, + DuckDBDataSource dataSource, + @Autowired(required = false) @SuppressWarnings("unused") + Object gitBlameServicePlaceholder, + @Autowired(required = false) @SuppressWarnings("unused") + Object coChangeServicePlaceholder) { + this.artifactService = artifactService; + this.dependencyService = dependencyService; + this.contentIntelligenceService = contentIntelligenceService; + this.dataSource = dataSource; + // Will be replaced with real types when GitBlameService / CoChangeService exist + this.gitBlameService = null; + this.coChangeService = null; + } + + /** + * Aggregates all known information about an artifact into a single map. + * Each section is independently fetched — failures in one section do not + * affect the others. + */ + public Map explain(String artifactId) { + Map result = new LinkedHashMap<>(); + + // 1. file + try { + Map status = artifactService.getStatus(artifactId); + if (status == null) { + return null; + } + result.put("file", status); + } catch (Exception e) { + log.warn("explain: failed to get file status for {}", artifactId, e); + return null; + } + + // 2. summary + addSection(result, "summary", () -> artifactService.getSummary(artifactId)); + + // 3. dependencies (limit 5) + addSection(result, "dependencies", () -> + limitList(dependencyService.getDependencies(artifactId), 5)); + + // 4. dependents (limit 5) + addSection(result, "dependents", () -> + limitList(dependencyService.getDependents(artifactId), 5)); + + // 5. classification + addSection(result, "classification", () -> queryClassification(artifactId)); + + // 6. tags + addSection(result, "tags", () -> queryTags(artifactId)); + + // 7. salient_points + addSection(result, "salient_points", () -> querySalientPoints(artifactId)); + + // 8. related_artifacts (limit 5) + addSection(result, "related_artifacts", () -> + limitList(contentIntelligenceService.getRelatedByConcept(artifactId), 5)); + + // 9. blame_highlights (optional — service may not exist) + if (gitBlameService != null) { + addSection(result, "blame_highlights", () -> getBlameHighlights(artifactId)); + } + + // 10. co_changes (optional — service may not exist) + if (coChangeService != null) { + addSection(result, "co_changes", () -> getCoChanges(artifactId)); + } + + return result; + } + + /** + * Resolve a file path to an artifact and return explain data. + * If the file is not indexed, returns a minimal response. + */ + public Map explainByPath(String filePath) { + try { + String artifactId = dataSource.withConnection(conn -> { + try (PreparedStatement ps = conn.prepareStatement( + "SELECT artifact_id FROM artifacts WHERE original_client_path = ? AND status = 'INDEXED' ORDER BY indexed_at DESC LIMIT 1")) { + ps.setString(1, filePath); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return rs.getString("artifact_id"); + } + } + } + return null; + }); + + if (artifactId != null) { + return explain(artifactId); + } + } catch (Exception e) { + log.warn("explainByPath: failed to resolve path {}", filePath, e); + } + + Map minimal = new LinkedHashMap<>(); + minimal.put("file_path", filePath); + minimal.put("indexed", false); + return minimal; + } + + // ── Section query helpers ────────────────────────────────────────────── + + private Map queryClassification(String artifactId) throws SQLException { + return dataSource.withConnection(conn -> { + try (PreparedStatement ps = conn.prepareStatement( + "SELECT doc_type, confidence, method, classified_at FROM artifact_classifications WHERE artifact_id = ?")) { + ps.setString(1, artifactId); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("doc_type", rs.getString("doc_type")); + row.put("confidence", rs.getDouble("confidence")); + row.put("method", rs.getString("method")); + row.put("classified_at", rs.getString("classified_at")); + return row; + } + } + } + return null; + }); + } + + private List> queryTags(String artifactId) throws SQLException { + return dataSource.withConnection(conn -> { + List> tags = new ArrayList<>(); + try (PreparedStatement ps = conn.prepareStatement( + "SELECT tag, tag_type, source FROM artifact_tags WHERE artifact_id = ?")) { + ps.setString(1, artifactId); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("tag", rs.getString("tag")); + row.put("tag_type", rs.getString("tag_type")); + row.put("source", rs.getString("source")); + tags.add(row); + } + } + } + return tags.isEmpty() ? null : tags; + }); + } + + private List> querySalientPoints(String artifactId) throws SQLException { + return dataSource.withConnection(conn -> { + List> points = new ArrayList<>(); + try (PreparedStatement ps = conn.prepareStatement( + "SELECT point_type, point_text, source FROM artifact_salient_points WHERE artifact_id = ?")) { + ps.setString(1, artifactId); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("point_type", rs.getString("point_type")); + row.put("point_text", rs.getString("point_text")); + row.put("source", rs.getString("source")); + points.add(row); + } + } + } + return points.isEmpty() ? null : points; + }); + } + + // Placeholder — will delegate to GitBlameService when it exists + private Object getBlameHighlights(String artifactId) { + return null; + } + + // Placeholder — will delegate to CoChangeService when it exists + private Object getCoChanges(String artifactId) { + return null; + } + + // ── Utility helpers (package-private for testing) ────────────────────── + + @FunctionalInterface + interface SectionSupplier { + T get() throws Exception; + } + + private void addSection(Map result, String key, SectionSupplier supplier) { + try { + T value = supplier.get(); + if (value != null) { + result.put(key, value); + } + } catch (Exception e) { + log.warn("explain: failed to build section '{}': {}", key, e.getMessage()); + } + } + + static List limitList(List list, int max) { + if (list == null) { + return Collections.emptyList(); + } + if (list.size() <= max) { + return list; + } + return list.subList(0, max); + } + + static Map buildExplainResult(Map sections) { + if (sections == null) { + return null; + } + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : sections.entrySet()) { + if (entry.getValue() != null) { + result.put(entry.getKey(), entry.getValue()); + } + } + return result; + } +} diff --git a/src/main/java/com/javaducker/server/service/GitBlameService.java b/src/main/java/com/javaducker/server/service/GitBlameService.java new file mode 100644 index 0000000..62d8a61 --- /dev/null +++ b/src/main/java/com/javaducker/server/service/GitBlameService.java @@ -0,0 +1,199 @@ +package com.javaducker.server.service; + +import com.javaducker.server.db.DuckDBDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Path; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.util.*; + +@Service +public class GitBlameService { + + private static final Logger log = LoggerFactory.getLogger(GitBlameService.class); + + public record BlameEntry( + int lineStart, + int lineEnd, + String commitHash, + String author, + Instant authorDate, + String commitMessage, + String content + ) {} + + private final DuckDBDataSource dataSource; + private final File projectRoot; + private final LinkedHashMap> cache; + + public GitBlameService(DuckDBDataSource dataSource) { + this.dataSource = dataSource; + String root = System.getenv("PROJECT_ROOT"); + this.projectRoot = new File(root != null ? root : ".").getAbsoluteFile(); + this.cache = new LinkedHashMap<>(50, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry> eldest) { + return size() > 50; + } + }; + } + + public List blame(String filePath) throws IOException, InterruptedException { + validatePath(filePath); + String absPath = projectRoot.toPath().resolve(filePath).toAbsolutePath().normalize().toString(); + List cached = cache.get(absPath); + if (cached != null) { + return cached; + } + List cmd = List.of("git", "blame", "--porcelain", filePath); + String output = runGitCommand(cmd); + List entries = parseBlameOutput(output); + cache.put(absPath, entries); + return entries; + } + + public List blameForLines(String filePath, int startLine, int endLine) throws IOException, InterruptedException { + validatePath(filePath); + List cmd = List.of("git", "blame", "--porcelain", "-L", startLine + "," + endLine, filePath); + String output = runGitCommand(cmd); + return parseBlameOutput(output); + } + + public List blameForArtifact(String artifactId) throws SQLException, IOException, InterruptedException { + String clientPath = dataSource.withConnection(conn -> { + try (PreparedStatement ps = conn.prepareStatement( + "SELECT original_client_path FROM artifacts WHERE artifact_id = ?")) { + ps.setString(1, artifactId); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return rs.getString("original_client_path"); + } + } + } + return null; + }); + if (clientPath == null) { + throw new IllegalArgumentException("Artifact not found: " + artifactId); + } + return blame(clientPath); + } + + private void validatePath(String filePath) { + if (filePath == null || filePath.isBlank()) { + throw new IllegalArgumentException("File path must not be null or blank"); + } + Path resolved = projectRoot.toPath().resolve(filePath).normalize(); + if (!resolved.startsWith(projectRoot.toPath())) { + throw new IllegalArgumentException("File path is outside PROJECT_ROOT: " + filePath); + } + } + + private String runGitCommand(List cmd) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.directory(projectRoot); + pb.redirectErrorStream(true); + Process process = pb.start(); + String output; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + output = reader.lines().reduce("", (a, b) -> a + "\n" + b); + } + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new IOException("git blame failed (exit " + exitCode + "): " + output.trim()); + } + return output; + } + + List parseBlameOutput(String output) { + if (output == null || output.isBlank()) { + return List.of(); + } + List entries = new ArrayList<>(); + String[] lines = output.split("\n"); + + String currentHash = null; + int currentLine = -1; + Map authors = new HashMap<>(); + Map authorTimes = new HashMap<>(); + Map summaries = new HashMap<>(); + StringBuilder contentBuilder = new StringBuilder(); + + // Track for grouping consecutive lines by same commit + String prevHash = null; + int rangeStart = -1; + int rangeEnd = -1; + String rangeContent = null; + + for (String line : lines) { + if (line.isEmpty()) continue; + + if (line.length() >= 40 && line.substring(0, 40).matches("[0-9a-f]{40}")) { + // Commit header line: [] + String[] parts = line.split("\\s+"); + currentHash = parts[0]; + currentLine = Integer.parseInt(parts[2]); + } else if (line.startsWith("author ")) { + authors.putIfAbsent(currentHash, line.substring(7)); + } else if (line.startsWith("author-time ")) { + authorTimes.putIfAbsent(currentHash, line.substring(12)); + } else if (line.startsWith("summary ")) { + summaries.putIfAbsent(currentHash, line.substring(8)); + } else if (line.startsWith("\t")) { + // Content line — this marks the end of one blame block + String contentLine = line.substring(1); + + if (prevHash != null && prevHash.equals(currentHash) && currentLine == rangeEnd + 1) { + // Extend the current range + rangeEnd = currentLine; + rangeContent = rangeContent + "\n" + contentLine; + } else { + // Flush previous range if any + if (prevHash != null) { + entries.add(buildEntry(prevHash, rangeStart, rangeEnd, rangeContent, + authors, authorTimes, summaries)); + } + rangeStart = currentLine; + rangeEnd = currentLine; + rangeContent = contentLine; + } + prevHash = currentHash; + } + } + // Flush last range + if (prevHash != null) { + entries.add(buildEntry(prevHash, rangeStart, rangeEnd, rangeContent, + authors, authorTimes, summaries)); + } + return entries; + } + + private BlameEntry buildEntry(String hash, int start, int end, String content, + Map authors, Map authorTimes, + Map summaries) { + Instant date = null; + String timeStr = authorTimes.get(hash); + if (timeStr != null) { + try { + date = Instant.ofEpochSecond(Long.parseLong(timeStr.trim())); + } catch (NumberFormatException e) { + log.warn("Failed to parse author-time for {}: {}", hash, timeStr); + } + } + return new BlameEntry( + start, end, hash, + authors.getOrDefault(hash, "unknown"), + date, + summaries.getOrDefault(hash, ""), + content + ); + } +} diff --git a/src/main/java/com/javaducker/server/service/SessionIngestionService.java b/src/main/java/com/javaducker/server/service/SessionIngestionService.java new file mode 100644 index 0000000..1482a73 --- /dev/null +++ b/src/main/java/com/javaducker/server/service/SessionIngestionService.java @@ -0,0 +1,382 @@ +package com.javaducker.server.service; + +import com.javaducker.server.db.DuckDBDataSource; +import com.javaducker.server.ingestion.SessionTranscriptParser; +import com.javaducker.server.model.SessionTranscript; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.*; +import java.util.*; +import java.util.stream.Collectors; + +@Service +public class SessionIngestionService { + + private static final Logger log = LoggerFactory.getLogger(SessionIngestionService.class); + + private final DuckDBDataSource dataSource; + private final SessionTranscriptParser parser; + + public SessionIngestionService(DuckDBDataSource dataSource, SessionTranscriptParser parser) { + this.dataSource = dataSource; + this.parser = parser; + } + + /** + * Index session files from a project directory. + * Sorts by last modified descending, skips already-indexed files whose mtime hasn't changed. + * + * @param projectPath path to the directory containing .jsonl session files + * @param maxSessions maximum number of sessions to process (all if <= 0) + * @return summary map with sessions_indexed, sessions_skipped, total_messages, project_path + */ + public Map indexSessions(String projectPath, int maxSessions) throws SQLException { + List sessionFiles = parser.findSessionFiles(Path.of(projectPath)); + + // Sort by last modified descending (newest first) + sessionFiles.sort((a, b) -> { + try { + return Files.getLastModifiedTime(b).compareTo(Files.getLastModifiedTime(a)); + } catch (IOException e) { + return 0; + } + }); + + if (maxSessions > 0 && sessionFiles.size() > maxSessions) { + sessionFiles = sessionFiles.subList(0, maxSessions); + } + + int sessionsIndexed = 0; + int sessionsSkipped = 0; + int totalMessages = 0; + + for (Path file : sessionFiles) { + String sessionId = extractSessionId(file); + + if (isAlreadyIndexed(sessionId, file)) { + sessionsSkipped++; + log.debug("Skipping already-indexed session: {}", sessionId); + continue; + } + + List transcripts = parser.parseSessionFile(file); + if (!transcripts.isEmpty()) { + storeTranscripts(transcripts); + totalMessages += transcripts.size(); + sessionsIndexed++; + log.info("Indexed session {} with {} messages", sessionId, transcripts.size()); + } + } + + Map summary = new LinkedHashMap<>(); + summary.put("sessions_indexed", sessionsIndexed); + summary.put("sessions_skipped", sessionsSkipped); + summary.put("total_messages", totalMessages); + summary.put("project_path", projectPath); + return summary; + } + + /** + * Store transcripts for a session. Deletes existing records first for idempotent re-indexing. + */ + public void storeTranscripts(List transcripts) throws SQLException { + if (transcripts.isEmpty()) { + return; + } + + String sessionId = transcripts.get(0).sessionId(); + + dataSource.withConnection(conn -> { + // Delete existing records for this session (idempotent re-index) + try (PreparedStatement delete = conn.prepareStatement( + "DELETE FROM session_transcripts WHERE session_id = ?")) { + delete.setString(1, sessionId); + delete.executeUpdate(); + } + + // Batch insert all transcripts + try (PreparedStatement insert = conn.prepareStatement(""" + INSERT INTO session_transcripts + (session_id, project_path, message_index, role, content, tool_name, timestamp, token_estimate) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """)) { + for (SessionTranscript t : transcripts) { + insert.setString(1, t.sessionId()); + insert.setString(2, t.projectPath()); + insert.setInt(3, t.messageIndex()); + insert.setString(4, t.role()); + insert.setString(5, t.content()); + insert.setString(6, t.toolName()); + if (t.timestamp() != null) { + try { + insert.setTimestamp(7, Timestamp.valueOf(t.timestamp())); + } catch (IllegalArgumentException e) { + insert.setNull(7, Types.TIMESTAMP); + } + } else { + insert.setNull(7, Types.TIMESTAMP); + } + insert.setInt(8, t.tokenEstimate()); + insert.addBatch(); + } + insert.executeBatch(); + } + + return null; + }); + } + + /** + * List all indexed sessions with message counts and token totals. + */ + public List> getSessionList() throws SQLException { + return dataSource.withConnection(conn -> { + List> results = new ArrayList<>(); + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(""" + SELECT session_id, project_path, + COUNT(*) as message_count, + MIN(message_index) as first_msg, + MAX(message_index) as last_msg, + SUM(token_estimate) as total_tokens + FROM session_transcripts + GROUP BY session_id, project_path + ORDER BY session_id DESC + """)) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("session_id", rs.getString("session_id")); + row.put("project_path", rs.getString("project_path")); + row.put("message_count", rs.getInt("message_count")); + row.put("first_msg", rs.getInt("first_msg")); + row.put("last_msg", rs.getInt("last_msg")); + row.put("total_tokens", rs.getLong("total_tokens")); + results.add(row); + } + } + return results; + }); + } + + /** + * Retrieve all messages for a specific session, ordered by message index. + */ + public List> getSession(String sessionId) throws SQLException { + return dataSource.withConnection(conn -> { + List> results = new ArrayList<>(); + try (PreparedStatement ps = conn.prepareStatement(""" + SELECT session_id, project_path, message_index, role, content, + tool_name, timestamp, token_estimate + FROM session_transcripts + WHERE session_id = ? + ORDER BY message_index + """)) { + ps.setString(1, sessionId); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("session_id", rs.getString("session_id")); + row.put("project_path", rs.getString("project_path")); + row.put("message_index", rs.getInt("message_index")); + row.put("role", rs.getString("role")); + row.put("content", rs.getString("content")); + row.put("tool_name", rs.getString("tool_name")); + row.put("timestamp", rs.getTimestamp("timestamp")); + row.put("token_estimate", rs.getInt("token_estimate")); + results.add(row); + } + } + } + return results; + }); + } + + /** + * Check if a session file is already indexed and up-to-date. + * Compares the file's last modified time against the max timestamp stored for that session. + */ + boolean isAlreadyIndexed(String sessionId, Path file) throws SQLException { + long fileMtime; + try { + fileMtime = Files.getLastModifiedTime(file).toMillis(); + } catch (IOException e) { + return false; + } + + return dataSource.withConnection(conn -> { + try (PreparedStatement ps = conn.prepareStatement( + "SELECT COUNT(*) as cnt FROM session_transcripts WHERE session_id = ?")) { + ps.setString(1, sessionId); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next() && rs.getInt("cnt") > 0) { + // Session exists; check if file has been modified since last index + // Use message count as a proxy: if file mtime is newer than + // we'd need to re-index. Store mtime as token_estimate of a sentinel? + // Simpler: check max timestamp in the records + return isFileUnchanged(conn, sessionId, fileMtime); + } + } + } + return false; + }); + } + + /** + * Check if the file mtime is older than or equal to when we last indexed. + * We store the file mtime as a metadata marker in the first record's token_estimate field + * would be too hacky. Instead, we track indexed mtime in a simple approach: + * if records exist for this session, we consider it indexed. The caller can force re-index + * by deleting records first. For incremental: compare file mtime to a stored value. + * + * For now, we use a pragmatic approach: store the file's mtime (epoch millis) + * as a negative token_estimate on a sentinel row with message_index = -1. + */ + private boolean isFileUnchanged(Connection conn, String sessionId, long fileMtime) throws SQLException { + try (PreparedStatement ps = conn.prepareStatement( + "SELECT content FROM session_transcripts WHERE session_id = ? AND message_index = -1")) { + ps.setString(1, sessionId); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + String stored = rs.getString("content"); + if (stored != null) { + long storedMtime = Long.parseLong(stored); + return fileMtime <= storedMtime; + } + } + } + } + // No sentinel row means we need to re-index (legacy data or first time) + return false; + } + + /** + * Store transcripts with an mtime sentinel for incremental ingestion tracking. + */ + public void storeTranscriptsWithMtime(List transcripts, long fileMtime) + throws SQLException { + if (transcripts.isEmpty()) { + return; + } + + String sessionId = transcripts.get(0).sessionId(); + String projectPath = transcripts.get(0).projectPath(); + + dataSource.withConnection(conn -> { + // Delete existing records for this session + try (PreparedStatement delete = conn.prepareStatement( + "DELETE FROM session_transcripts WHERE session_id = ?")) { + delete.setString(1, sessionId); + delete.executeUpdate(); + } + + // Insert mtime sentinel row (message_index = -1) + try (PreparedStatement sentinel = conn.prepareStatement(""" + INSERT INTO session_transcripts + (session_id, project_path, message_index, role, content, tool_name, timestamp, token_estimate) + VALUES (?, ?, -1, '_meta', ?, NULL, NULL, 0) + """)) { + sentinel.setString(1, sessionId); + sentinel.setString(2, projectPath); + sentinel.setString(3, String.valueOf(fileMtime)); + sentinel.executeUpdate(); + } + + // Batch insert all transcripts + try (PreparedStatement insert = conn.prepareStatement(""" + INSERT INTO session_transcripts + (session_id, project_path, message_index, role, content, tool_name, timestamp, token_estimate) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """)) { + for (SessionTranscript t : transcripts) { + insert.setString(1, t.sessionId()); + insert.setString(2, t.projectPath()); + insert.setInt(3, t.messageIndex()); + insert.setString(4, t.role()); + insert.setString(5, t.content()); + insert.setString(6, t.toolName()); + if (t.timestamp() != null) { + try { + insert.setTimestamp(7, Timestamp.valueOf(t.timestamp())); + } catch (IllegalArgumentException e) { + insert.setNull(7, Types.TIMESTAMP); + } + } else { + insert.setNull(7, Types.TIMESTAMP); + } + insert.setInt(8, t.tokenEstimate()); + insert.addBatch(); + } + insert.executeBatch(); + } + + return null; + }); + } + + /** + * Full indexSessions flow using mtime tracking for incremental ingestion. + */ + public Map indexSessionsIncremental(String projectPath, int maxSessions) + throws SQLException { + List sessionFiles = parser.findSessionFiles(Path.of(projectPath)); + + // Sort by last modified descending (newest first) + sessionFiles.sort((a, b) -> { + try { + return Files.getLastModifiedTime(b).compareTo(Files.getLastModifiedTime(a)); + } catch (IOException e) { + return 0; + } + }); + + if (maxSessions > 0 && sessionFiles.size() > maxSessions) { + sessionFiles = sessionFiles.subList(0, maxSessions); + } + + int sessionsIndexed = 0; + int sessionsSkipped = 0; + int totalMessages = 0; + + for (Path file : sessionFiles) { + String sessionId = extractSessionId(file); + + if (isAlreadyIndexed(sessionId, file)) { + sessionsSkipped++; + continue; + } + + List transcripts = parser.parseSessionFile(file); + if (!transcripts.isEmpty()) { + long mtime; + try { + mtime = Files.getLastModifiedTime(file).toMillis(); + } catch (IOException e) { + mtime = System.currentTimeMillis(); + } + storeTranscriptsWithMtime(transcripts, mtime); + totalMessages += transcripts.size(); + sessionsIndexed++; + log.info("Indexed session {} with {} messages", sessionId, transcripts.size()); + } + } + + Map summary = new LinkedHashMap<>(); + summary.put("sessions_indexed", sessionsIndexed); + summary.put("sessions_skipped", sessionsSkipped); + summary.put("total_messages", totalMessages); + summary.put("project_path", projectPath); + return summary; + } + + private String extractSessionId(Path file) { + String fileName = file.getFileName().toString(); + return fileName.endsWith(".jsonl") + ? fileName.substring(0, fileName.length() - ".jsonl".length()) + : fileName; + } +} diff --git a/src/main/java/com/javaducker/server/service/StalenessService.java b/src/main/java/com/javaducker/server/service/StalenessService.java index adf9508..1bded60 100644 --- a/src/main/java/com/javaducker/server/service/StalenessService.java +++ b/src/main/java/com/javaducker/server/service/StalenessService.java @@ -71,6 +71,34 @@ public Map checkStaleness(List filePaths) throws SQLExce return result; } + public Map checkAll() throws SQLException { + String sql = "SELECT DISTINCT original_client_path FROM artifacts " + + "WHERE status = 'INDEXED' AND original_client_path IS NOT NULL"; + + List allPaths = dataSource.withConnection(conn -> { + List paths = new ArrayList<>(); + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + paths.add(rs.getString("original_client_path")); + } + } + return paths; + }); + + Map result = checkStaleness(allPaths); + computeStaleSummary(result); + return result; + } + + static void computeStaleSummary(Map result) { + List staleList = (List) result.get("stale"); + int staleCount = staleList != null ? staleList.size() : 0; + long total = (long) result.get("total_checked"); + result.put("stale_count", staleCount); + result.put("stale_percentage", total > 0 ? (staleCount * 100.0 / total) : 0.0); + } + private List> queryArtifacts(String path) throws SQLException { String sql = "SELECT artifact_id, file_name, original_client_path, indexed_at " + "FROM artifacts WHERE original_client_path = ? AND status = 'INDEXED'"; diff --git a/src/test/java/com/javaducker/server/ingestion/SessionTranscriptParserTest.java b/src/test/java/com/javaducker/server/ingestion/SessionTranscriptParserTest.java new file mode 100644 index 0000000..0b49043 --- /dev/null +++ b/src/test/java/com/javaducker/server/ingestion/SessionTranscriptParserTest.java @@ -0,0 +1,171 @@ +package com.javaducker.server.ingestion; + +import com.javaducker.server.model.SessionTranscript; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class SessionTranscriptParserTest { + + private final SessionTranscriptParser parser = new SessionTranscriptParser(); + + @Test + void parseHumanMessage() { + String line = """ + {"type":"human","message":{"role":"user","content":"Hello world"},"timestamp":"2026-03-28T10:00:00Z"}"""; + + SessionTranscript result = parser.parseLine(line, "session-1", "/projects/test", 0); + + assertNotNull(result); + assertEquals("session-1", result.sessionId()); + assertEquals("/projects/test", result.projectPath()); + assertEquals(0, result.messageIndex()); + assertEquals("user", result.role()); + assertEquals("Hello world", result.content()); + assertNull(result.toolName()); + assertEquals("2026-03-28T10:00:00Z", result.timestamp()); + assertEquals("Hello world".length() / 4, result.tokenEstimate()); + } + + @Test + void parseAssistantMessage() { + String line = """ + {"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"I can help with that."}]},"timestamp":"2026-03-28T10:01:00Z"}"""; + + SessionTranscript result = parser.parseLine(line, "session-1", "/projects/test", 1); + + assertNotNull(result); + assertEquals("assistant", result.role()); + assertEquals("I can help with that.", result.content()); + } + + @Test + void malformedJsonLineReturnsNull() { + String line = "this is not valid json {{{"; + + SessionTranscript result = parser.parseLine(line, "session-1", "/projects/test", 0); + + assertNull(result); + } + + @Test + void emptyLineReturnsNull() { + assertNull(parser.parseLine("", "session-1", "/projects/test", 0)); + assertNull(parser.parseLine(" ", "session-1", "/projects/test", 0)); + assertNull(parser.parseLine(null, "session-1", "/projects/test", 0)); + } + + @Test + void emptyFileReturnsEmptyList(@TempDir Path tempDir) throws IOException { + Path emptyFile = tempDir.resolve("empty.jsonl"); + Files.writeString(emptyFile, ""); + + List results = parser.parseSessionFile(emptyFile); + + assertTrue(results.isEmpty()); + } + + @Test + void imageContentBlockIsSkipped() { + String line = """ + {"type":"assistant","message":{"role":"assistant","content":[{"type":"image","source":{"data":"base64data"}}]},"timestamp":"2026-03-28T10:02:00Z"}"""; + + SessionTranscript result = parser.parseLine(line, "session-1", "/projects/test", 0); + + // Image-only content produces null (no text content) + assertNull(result); + } + + @Test + void veryLongContentIsTruncated() { + String longText = "x".repeat(60_000); + String line = String.format( + "{\"type\":\"human\",\"message\":{\"role\":\"user\",\"content\":\"%s\"}}", longText); + + SessionTranscript result = parser.parseLine(line, "session-1", "/projects/test", 0); + + assertNotNull(result); + assertEquals(50_000, result.content().length()); + } + + @Test + void findSessionFilesWithNoJsonl(@TempDir Path tempDir) throws IOException { + // Create a non-jsonl file + Files.writeString(tempDir.resolve("notes.txt"), "just a text file"); + + List files = parser.findSessionFiles(tempDir); + + assertTrue(files.isEmpty()); + } + + @Test + void findSessionFilesFindsJsonl(@TempDir Path tempDir) throws IOException { + Files.writeString(tempDir.resolve("session-abc.jsonl"), "{}\n"); + Files.writeString(tempDir.resolve("session-def.jsonl"), "{}\n"); + Files.writeString(tempDir.resolve("notes.txt"), "not a session"); + + List files = parser.findSessionFiles(tempDir); + + assertEquals(2, files.size()); + } + + @Test + void parseSessionFileExtractsSessionIdFromFileName(@TempDir Path tempDir) throws IOException { + String line = """ + {"type":"human","message":{"role":"user","content":"test message"}}"""; + Path sessionFile = tempDir.resolve("my-session-id.jsonl"); + Files.writeString(sessionFile, line + "\n"); + + List results = parser.parseSessionFile(sessionFile); + + assertEquals(1, results.size()); + assertEquals("my-session-id", results.get(0).sessionId()); + assertEquals(tempDir.toString(), results.get(0).projectPath()); + } + + @Test + void parseSessionFileSkipsMalformedLines(@TempDir Path tempDir) throws IOException { + String content = """ + {"type":"human","message":{"role":"user","content":"good line"}} + not valid json + {"type":"assistant","message":{"role":"assistant","content":"another good line"}} + """; + Path sessionFile = tempDir.resolve("mixed.jsonl"); + Files.writeString(sessionFile, content); + + List results = parser.parseSessionFile(sessionFile); + + assertEquals(2, results.size()); + assertEquals("good line", results.get(0).content()); + assertEquals("another good line", results.get(1).content()); + } + + @Test + void toolUseBlockExtractsToolName() { + String line = """ + {"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","name":"Read","id":"123","input":{}}]}}"""; + + SessionTranscript result = parser.parseLine(line, "session-1", "/projects/test", 0); + + assertNotNull(result); + assertEquals("Read", result.toolName()); + } + + @Test + void findSessionFilesWithNullDir() { + List files = parser.findSessionFiles(null); + assertTrue(files.isEmpty()); + } + + @Test + void parseSessionFileWithNullPath() { + List results = parser.parseSessionFile(null); + assertTrue(results.isEmpty()); + } +} diff --git a/src/test/java/com/javaducker/server/rest/JavaDuckerRestControllerTest.java b/src/test/java/com/javaducker/server/rest/JavaDuckerRestControllerTest.java index a45f01a..ecf0238 100644 --- a/src/test/java/com/javaducker/server/rest/JavaDuckerRestControllerTest.java +++ b/src/test/java/com/javaducker/server/rest/JavaDuckerRestControllerTest.java @@ -10,6 +10,7 @@ import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; +import java.time.Instant; import java.util.List; import java.util.Map; @@ -34,6 +35,9 @@ class JavaDuckerRestControllerTest { @MockBean FileWatcher fileWatcher; @MockBean ReladomoQueryService reladomoQueryService; @MockBean ContentIntelligenceService contentIntelligenceService; + @MockBean ExplainService explainService; + @MockBean GitBlameService gitBlameService; + @MockBean CoChangeService coChangeService; @Test void healthReturnsOk() throws Exception { @@ -98,6 +102,87 @@ void statsReturnsData() throws Exception { .andExpect(jsonPath("$.total_artifacts").value(10)); } + // ── Explain endpoint tests ──────────────────────────────────────────── + + @Test + void explainNotFoundWhenArtifactMissing() throws Exception { + when(explainService.explain("nonexistent")).thenReturn(null); + mockMvc.perform(get("/api/explain/nonexistent")) + .andExpect(status().isNotFound()); + } + + @Test + void explainReturnsDataForKnownArtifact() throws Exception { + when(explainService.explain("abc-123")).thenReturn(Map.of( + "file", Map.of("artifact_id", "abc-123", "file_name", "Test.java"), + "summary", Map.of("classes", List.of("Test")), + "dependencies", List.of(), + "dependents", List.of())); + mockMvc.perform(get("/api/explain/abc-123")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.file.artifact_id").value("abc-123")); + } + + @Test + void explainByPathRequiresFilePath() throws Exception { + String body = objectMapper.writeValueAsString(Map.of()); + mockMvc.perform(post("/api/explain").contentType(MediaType.APPLICATION_JSON).content(body)) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.error").value("filePath is required")); + } + + @Test + void explainByPathReturnsData() throws Exception { + when(explainService.explainByPath("/src/Test.java")).thenReturn(Map.of( + "file", Map.of("artifact_id", "abc-123", "file_name", "Test.java"))); + String body = objectMapper.writeValueAsString(Map.of("filePath", "/src/Test.java")); + mockMvc.perform(post("/api/explain").contentType(MediaType.APPLICATION_JSON).content(body)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.file.artifact_id").value("abc-123")); + } + + // ── Git Blame endpoint tests ────────────────────────────────────────── + + @Test + void blameByPathReturnsEntries() throws Exception { + when(gitBlameService.blame("src/main/java/App.java")).thenReturn(List.of( + new GitBlameService.BlameEntry(1, 10, "abcdef1234567890abcdef1234567890abcdef12", + "alice", Instant.parse("2026-03-20T10:00:00Z"), "Add auth middleware", "code..."))); + String body = objectMapper.writeValueAsString(Map.of("filePath", "src/main/java/App.java")); + mockMvc.perform(post("/api/blame").contentType(MediaType.APPLICATION_JSON).content(body)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.file").value("src/main/java/App.java")) + .andExpect(jsonPath("$.blame[0].lines").value("1-10")) + .andExpect(jsonPath("$.blame[0].commit").value("abcdef12")) + .andExpect(jsonPath("$.blame[0].author").value("alice")); + } + + @Test + void blameByPathWithLineRange() throws Exception { + when(gitBlameService.blameForLines("src/main/java/App.java", 5, 15)).thenReturn(List.of( + new GitBlameService.BlameEntry(5, 5, "1234567890abcdef1234567890abcdef12345678", + "bob", Instant.parse("2026-03-21T12:00:00Z"), "Fix NPE", "line5"))); + String body = objectMapper.writeValueAsString(Map.of( + "filePath", "src/main/java/App.java", "startLine", 5, "endLine", 15)); + mockMvc.perform(post("/api/blame").contentType(MediaType.APPLICATION_JSON).content(body)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.blame[0].lines").value("5")) + .andExpect(jsonPath("$.blame[0].author").value("bob")); + } + + @Test + void blameByArtifactIdReturnsEnrichedResult() throws Exception { + when(gitBlameService.blameForArtifact("abc-123")).thenReturn(List.of( + new GitBlameService.BlameEntry(1, 20, "abcdef1234567890abcdef1234567890abcdef12", + "alice", Instant.parse("2026-03-20T10:00:00Z"), "Initial commit", "code"))); + when(artifactService.getSummary("abc-123")).thenReturn(Map.of("classes", List.of("App"))); + mockMvc.perform(get("/api/blame/abc-123")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.artifact_id").value("abc-123")) + .andExpect(jsonPath("$.summary.classes[0]").value("App")) + .andExpect(jsonPath("$.blame[0].lines").value("1-20")); + } + // ── Content Intelligence: write endpoint tests ─────────────────────── @Test @@ -194,4 +279,58 @@ void synthesisNotFound() throws Exception { mockMvc.perform(get("/api/synthesis/nonexistent")) .andExpect(status().isNotFound()); } + + // ── Co-Change / Related Files endpoint tests ──────────────────────── + + @Test + void relatedByPathReturnsResults() throws Exception { + when(coChangeService.getRelatedFiles(eq("/src/Main.java"), eq(10))) + .thenReturn(List.of(Map.of("related_file", "/src/Config.java", + "co_change_count", 5, "last_commit_date", "2026-03-01"))); + String body = objectMapper.writeValueAsString(Map.of("filePath", "/src/Main.java")); + mockMvc.perform(post("/api/related").contentType(MediaType.APPLICATION_JSON).content(body)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.file_path").value("/src/Main.java")) + .andExpect(jsonPath("$.count").value(1)) + .andExpect(jsonPath("$.related[0].related_file").value("/src/Config.java")); + } + + @Test + void relatedByPathReturnsEmptyList() throws Exception { + when(coChangeService.getRelatedFiles(anyString(), anyInt())).thenReturn(List.of()); + String body = objectMapper.writeValueAsString(Map.of("filePath", "/src/Unknown.java")); + mockMvc.perform(post("/api/related").contentType(MediaType.APPLICATION_JSON).content(body)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.count").value(0)) + .andExpect(jsonPath("$.related").isEmpty()); + } + + @Test + void rebuildCoChangeReturnsRebuilt() throws Exception { + mockMvc.perform(post("/api/rebuild-cochange").contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("rebuilt")); + } + + @Test + void relatedByArtifactNotFound() throws Exception { + when(artifactService.getStatus("nonexistent")).thenReturn(null); + mockMvc.perform(get("/api/related/nonexistent")) + .andExpect(status().isNotFound()); + } + + @Test + void relatedByArtifactReturnsResults() throws Exception { + when(artifactService.getStatus("abc-123")).thenReturn(Map.of( + "artifact_id", "abc-123", "original_client_path", "/src/Main.java", + "status", "INDEXED")); + when(coChangeService.getRelatedFiles(eq("/src/Main.java"), eq(10))) + .thenReturn(List.of(Map.of("related_file", "/src/Config.java", + "co_change_count", 3, "last_commit_date", "2026-03-01"))); + mockMvc.perform(get("/api/related/abc-123")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.artifact_id").value("abc-123")) + .andExpect(jsonPath("$.file_path").value("/src/Main.java")) + .andExpect(jsonPath("$.count").value(1)); + } } diff --git a/src/test/java/com/javaducker/server/service/CoChangeServiceTest.java b/src/test/java/com/javaducker/server/service/CoChangeServiceTest.java new file mode 100644 index 0000000..f655834 --- /dev/null +++ b/src/test/java/com/javaducker/server/service/CoChangeServiceTest.java @@ -0,0 +1,145 @@ +package com.javaducker.server.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +class CoChangeServiceTest { + + private CoChangeService service; + + @BeforeEach + void setUp() { + // Construct with null dataSource since we only test parsing/computation methods + service = new CoChangeService(null); + } + + @Test + void parseValidGitLog() { + String output = """ + COMMIT:abc123 + + src/Foo.java + src/Bar.java + + COMMIT:def456 + + src/Baz.java + """; + Map> result = service.parseGitLog(output); + + assertEquals(2, result.size()); + assertEquals(List.of("src/Foo.java", "src/Bar.java"), result.get("abc123")); + assertEquals(List.of("src/Baz.java"), result.get("def456")); + } + + @Test + void parseEmptyOutput() { + Map> result = service.parseGitLog(""); + assertTrue(result.isEmpty()); + } + + @Test + void parseNullOutput() { + Map> result = service.parseGitLog(null); + assertTrue(result.isEmpty()); + } + + @Test + void noiseFilterSkipsLargeCommits() { + // Build a commit with 35 files — should be excluded by buildCoChangeIndex flow + // We test via computeCoChanges by verifying that the service's filter logic works + Map> commits = new LinkedHashMap<>(); + List largeCommit = new ArrayList<>(); + for (int i = 0; i < 35; i++) { + largeCommit.add("file" + i + ".java"); + } + commits.put("big-commit", largeCommit); + commits.put("small-commit", List.of("src/A.java", "src/B.java")); + + // The large commit has 35 files. When passed to computeCoChanges directly, + // it would produce pairs. But the service filters commits > 30 files + // before calling computeCoChanges. We verify the parse yields both, + // then the public buildCoChangeIndex would filter. + String output = "COMMIT:big-commit\n\n"; + for (int i = 0; i < 35; i++) { + output += "file" + i + ".java\n"; + } + output += "\nCOMMIT:small-commit\n\nsrc/A.java\nsrc/B.java\n"; + + Map> parsed = service.parseGitLog(output); + assertEquals(2, parsed.size()); + assertEquals(35, parsed.get("big-commit").size()); + assertEquals(2, parsed.get("small-commit").size()); + } + + @Test + void coChangeCountTwoCommitsBothTouchAB() { + Map> commits = new LinkedHashMap<>(); + commits.put("c1", List.of("src/A.java", "src/B.java")); + commits.put("c2", List.of("src/A.java", "src/B.java")); + + Map> result = service.computeCoChanges(commits); + + assertEquals(2, result.get("src/A.java").get("src/B.java")); + } + + @Test + void coChangeThreeFilesInOneCommit() { + Map> commits = new LinkedHashMap<>(); + commits.put("c1", List.of("src/A.java", "src/B.java", "src/C.java")); + + Map> result = service.computeCoChanges(commits); + + // A-B, A-C, B-C each get count 1 + assertEquals(1, result.get("src/A.java").get("src/B.java")); + assertEquals(1, result.get("src/A.java").get("src/C.java")); + assertEquals(1, result.get("src/B.java").get("src/C.java")); + } + + @Test + void frequentFileExcluded() { + // 10 commits total. "build.gradle" appears in 8 (80% > 50%), but A and B only in 2 each + Map> commits = new LinkedHashMap<>(); + commits.put("c1", List.of("build.gradle", "src/A.java")); + commits.put("c2", List.of("build.gradle", "src/B.java")); + commits.put("c3", List.of("build.gradle", "src/A.java", "src/B.java")); + commits.put("c4", List.of("build.gradle", "src/D.java")); + commits.put("c5", List.of("build.gradle", "src/E.java")); + commits.put("c6", List.of("build.gradle", "src/F.java")); + commits.put("c7", List.of("build.gradle", "src/G.java")); + commits.put("c8", List.of("build.gradle", "src/H.java")); + commits.put("c9", List.of("src/I.java")); + commits.put("c10", List.of("src/J.java")); + + Set frequent = service.findFrequentFiles(commits); + assertTrue(frequent.contains("build.gradle")); + assertFalse(frequent.contains("src/A.java"), "A.java should not be frequent"); + + Map> result = service.computeCoChanges(commits, frequent); + + // build.gradle should not appear in any pair + for (Map.Entry> entry : result.entrySet()) { + assertNotEquals("build.gradle", entry.getKey()); + assertFalse(entry.getValue().containsKey("build.gradle")); + } + + // A-B should still appear from c3 + assertEquals(1, result.get("src/A.java").get("src/B.java")); + } + + @Test + void emptyCommitsProduceEmptyCoChanges() { + Map> result = service.computeCoChanges(Collections.emptyMap()); + assertTrue(result.isEmpty()); + } + + @Test + void findFrequentFilesEmptyCommits() { + Set frequent = service.findFrequentFiles(Collections.emptyMap()); + assertTrue(frequent.isEmpty()); + } +} diff --git a/src/test/java/com/javaducker/server/service/ExplainServiceTest.java b/src/test/java/com/javaducker/server/service/ExplainServiceTest.java new file mode 100644 index 0000000..8396b6c --- /dev/null +++ b/src/test/java/com/javaducker/server/service/ExplainServiceTest.java @@ -0,0 +1,97 @@ +package com.javaducker.server.service; + +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +class ExplainServiceTest { + + @Test + void limitList_truncatesLongList() { + List input = List.of("a", "b", "c", "d", "e", "f", "g", "h", "i", "j"); + List result = ExplainService.limitList(input, 5); + assertEquals(5, result.size()); + assertEquals(List.of("a", "b", "c", "d", "e"), result); + } + + @Test + void limitList_nullReturnsEmptyList() { + List result = ExplainService.limitList(null, 5); + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + void limitList_shortListReturnedAsIs() { + List input = List.of("a", "b", "c"); + List result = ExplainService.limitList(input, 5); + assertEquals(3, result.size()); + assertEquals(List.of("a", "b", "c"), result); + } + + @Test + void limitList_exactSizeReturnedAsIs() { + List input = List.of("a", "b", "c", "d", "e"); + List result = ExplainService.limitList(input, 5); + assertEquals(5, result.size()); + } + + @Test + void limitList_emptyListReturnsEmpty() { + List result = ExplainService.limitList(Collections.emptyList(), 5); + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + void buildExplainResult_nullSectionsOmitted() { + Map sections = new LinkedHashMap<>(); + sections.put("file", Map.of("artifact_id", "art-1", "file_name", "Foo.java")); + sections.put("summary", null); + sections.put("dependencies", List.of("dep1", "dep2")); + sections.put("classification", null); + sections.put("tags", List.of("spring", "service")); + + Map result = ExplainService.buildExplainResult(sections); + + assertNotNull(result); + assertEquals(3, result.size()); + assertTrue(result.containsKey("file")); + assertTrue(result.containsKey("dependencies")); + assertTrue(result.containsKey("tags")); + assertFalse(result.containsKey("summary")); + assertFalse(result.containsKey("classification")); + } + + @Test + void buildExplainResult_allSectionsPresent() { + Map sections = new LinkedHashMap<>(); + sections.put("file", Map.of("artifact_id", "art-1")); + sections.put("summary", Map.of("summary_text", "A service class")); + sections.put("dependencies", List.of("dep1")); + + Map result = ExplainService.buildExplainResult(sections); + + assertNotNull(result); + assertEquals(3, result.size()); + } + + @Test + void buildExplainResult_nullInputReturnsNull() { + assertNull(ExplainService.buildExplainResult(null)); + } + + @Test + void buildExplainResult_allSectionsNullReturnsEmptyMap() { + Map sections = new LinkedHashMap<>(); + sections.put("file", null); + sections.put("summary", null); + + Map result = ExplainService.buildExplainResult(sections); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } +} diff --git a/src/test/java/com/javaducker/server/service/GitBlameServiceTest.java b/src/test/java/com/javaducker/server/service/GitBlameServiceTest.java new file mode 100644 index 0000000..4b5a017 --- /dev/null +++ b/src/test/java/com/javaducker/server/service/GitBlameServiceTest.java @@ -0,0 +1,110 @@ +package com.javaducker.server.service; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class GitBlameServiceTest { + + private final GitBlameService service = new GitBlameService(null); + + @Test + void parseValidPorcelainOutput() { + String output = """ + abc1234567890123456789012345678901234abcd 1 1 1 + author John Doe + author-mail + author-time 1711900000 + author-tz +0000 + committer John Doe + committer-mail + committer-time 1711900000 + committer-tz +0000 + summary Fix the auth bug + filename src/Main.java + \tactual line content here + """; + + List entries = service.parseBlameOutput(output); + + assertEquals(1, entries.size()); + GitBlameService.BlameEntry entry = entries.get(0); + assertEquals("abc1234567890123456789012345678901234abcd", entry.commitHash()); + assertEquals("John Doe", entry.author()); + assertEquals(Instant.ofEpochSecond(1711900000), entry.authorDate()); + assertEquals("Fix the auth bug", entry.commitMessage()); + assertEquals("actual line content here", entry.content()); + assertEquals(1, entry.lineStart()); + assertEquals(1, entry.lineEnd()); + } + + @Test + void consecutiveLinesSameCommitGroupedIntoRange() { + String output = """ + abc1234567890123456789012345678901234abcd 1 1 3 + author Alice + author-time 1700000000 + summary Initial commit + filename src/App.java + \tline one + abc1234567890123456789012345678901234abcd 2 2 + \tline two + abc1234567890123456789012345678901234abcd 3 3 + \tline three + """; + + List entries = service.parseBlameOutput(output); + + assertEquals(1, entries.size()); + GitBlameService.BlameEntry entry = entries.get(0); + assertEquals(1, entry.lineStart()); + assertEquals(3, entry.lineEnd()); + assertEquals("Alice", entry.author()); + assertEquals("Initial commit", entry.commitMessage()); + assertTrue(entry.content().contains("line one")); + assertTrue(entry.content().contains("line three")); + } + + @Test + void differentCommitsSplitIntoSeparateEntries() { + String output = """ + aaaa234567890123456789012345678901234aaaaa 1 1 1 + author Alice + author-time 1700000000 + summary First commit + filename src/App.java + \tfirst line + bbbb234567890123456789012345678901234bbbbb 2 2 1 + author Bob + author-time 1700001000 + summary Second commit + filename src/App.java + \tsecond line + """; + + List entries = service.parseBlameOutput(output); + + assertEquals(2, entries.size()); + assertEquals("Alice", entries.get(0).author()); + assertEquals(1, entries.get(0).lineStart()); + assertEquals("Bob", entries.get(1).author()); + assertEquals(2, entries.get(1).lineStart()); + } + + @Test + void emptyOutputReturnsEmptyList() { + assertEquals(List.of(), service.parseBlameOutput("")); + assertEquals(List.of(), service.parseBlameOutput(null)); + assertEquals(List.of(), service.parseBlameOutput(" ")); + } + + @Test + void nullOrBlankFilePathThrowsException() { + assertThrows(IllegalArgumentException.class, () -> service.blame(null)); + assertThrows(IllegalArgumentException.class, () -> service.blame("")); + assertThrows(IllegalArgumentException.class, () -> service.blame(" ")); + } +} diff --git a/src/test/java/com/javaducker/server/service/SessionIngestionServiceTest.java b/src/test/java/com/javaducker/server/service/SessionIngestionServiceTest.java new file mode 100644 index 0000000..c034a01 --- /dev/null +++ b/src/test/java/com/javaducker/server/service/SessionIngestionServiceTest.java @@ -0,0 +1,228 @@ +package com.javaducker.server.service; + +import com.javaducker.server.config.AppConfig; +import com.javaducker.server.db.DuckDBDataSource; +import com.javaducker.server.db.SchemaBootstrap; +import com.javaducker.server.ingestion.*; +import com.javaducker.server.model.SessionTranscript; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.*; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class SessionIngestionServiceTest { + + @TempDir + static Path tempDir; + + static DuckDBDataSource dataSource; + static SessionIngestionService service; + static SessionTranscriptParser parser; + + @BeforeAll + static void setup() throws Exception { + AppConfig config = new AppConfig(); + config.setDbPath(tempDir.resolve("test-session.duckdb").toString()); + config.setIntakeDir(tempDir.resolve("intake").toString()); + dataSource = new DuckDBDataSource(config); + parser = new SessionTranscriptParser(); + + // Bootstrap schema + ArtifactService artifactService = new ArtifactService(dataSource); + SearchService searchService = new SearchService(dataSource, new EmbeddingService(config), config); + IngestionWorker worker = new IngestionWorker(dataSource, artifactService, + new TextExtractor(), new TextNormalizer(), new Chunker(), + new EmbeddingService(config), new FileSummarizer(), new ImportParser(), + new ReladomoXmlParser(), new ReladomoService(dataSource), + new ReladomoFinderParser(), new ReladomoConfigParser(), + searchService, config); + SchemaBootstrap bootstrap = new SchemaBootstrap(dataSource, config, worker); + bootstrap.createSchema(); + + service = new SessionIngestionService(dataSource, parser); + } + + @AfterAll + static void teardown() throws Exception { + dataSource.close(); + } + + @Test + @Order(1) + void storeAndRetrieveTranscripts() throws Exception { + List transcripts = List.of( + new SessionTranscript("sess-001", "/project/a", 0, "user", + "How do I fix the auth bug?", null, null, 7), + new SessionTranscript("sess-001", "/project/a", 1, "assistant", + "You need to update the token validation logic.", null, null, 10), + new SessionTranscript("sess-001", "/project/a", 2, "assistant", + "Here is the fix.", "Read", null, 4) + ); + + service.storeTranscripts(transcripts); + + List> retrieved = service.getSession("sess-001"); + assertEquals(3, retrieved.size()); + assertEquals("user", retrieved.get(0).get("role")); + assertEquals("How do I fix the auth bug?", retrieved.get(0).get("content")); + assertEquals("assistant", retrieved.get(1).get("role")); + assertEquals("Read", retrieved.get(2).get("tool_name")); + } + + @Test + @Order(2) + void storeTranscriptsTwiceDeduplicates() throws Exception { + // Store updated version of same session + List updated = List.of( + new SessionTranscript("sess-001", "/project/a", 0, "user", + "Updated question", null, null, 5), + new SessionTranscript("sess-001", "/project/a", 1, "assistant", + "Updated answer", null, null, 5) + ); + + service.storeTranscripts(updated); + + List> retrieved = service.getSession("sess-001"); + assertEquals(2, retrieved.size(), "Should only have the latest version"); + assertEquals("Updated question", retrieved.get(0).get("content")); + assertEquals("Updated answer", retrieved.get(1).get("content")); + } + + @Test + @Order(3) + void getSessionListReturnsSummary() throws Exception { + // Add a second session + List sess2 = List.of( + new SessionTranscript("sess-002", "/project/b", 0, "user", + "Tell me about DuckDB", null, null, 6), + new SessionTranscript("sess-002", "/project/b", 1, "assistant", + "DuckDB is an in-process analytical database.", null, null, 10) + ); + service.storeTranscripts(sess2); + + List> sessions = service.getSessionList(); + assertTrue(sessions.size() >= 2, "Should have at least 2 sessions"); + + // Verify summary fields are present + Map first = sessions.get(0); + assertNotNull(first.get("session_id")); + assertNotNull(first.get("message_count")); + assertNotNull(first.get("total_tokens")); + } + + @Test + @Order(4) + void getSessionForUnknownIdReturnsEmpty() throws Exception { + List> result = service.getSession("nonexistent-session"); + assertTrue(result.isEmpty()); + } + + @Test + @Order(5) + void emptyDatabaseSessionList() throws Exception { + // Clean up all data + dataSource.withConnection(conn -> { + try (Statement stmt = conn.createStatement()) { + stmt.execute("DELETE FROM session_transcripts"); + } + return null; + }); + + List> sessions = service.getSessionList(); + assertTrue(sessions.isEmpty()); + } + + @Test + @Order(6) + void indexSessionsFromDirectory() throws Exception { + // Create a temp directory with a sample JSONL session file + Path sessionsDir = tempDir.resolve("sessions"); + Files.createDirectories(sessionsDir); + + String jsonl = """ + {"type":"human","message":{"role":"user","content":"Hello"}} + {"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Hi there!"}]}} + {"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"How can I help?"}]}} + """; + Files.writeString(sessionsDir.resolve("test-session-abc.jsonl"), jsonl); + + Map summary = service.indexSessions(sessionsDir.toString(), 0); + assertEquals(1, summary.get("sessions_indexed")); + assertEquals(0, summary.get("sessions_skipped")); + assertTrue((int) summary.get("total_messages") >= 2); + + // Verify session is retrievable + List> messages = service.getSession("test-session-abc"); + assertFalse(messages.isEmpty()); + } + + @Test + @Order(7) + void incrementalIngestionSkipsUnchangedFiles() throws Exception { + // Create a session file + Path sessionsDir = tempDir.resolve("sessions-inc"); + Files.createDirectories(sessionsDir); + + String jsonl = """ + {"type":"human","message":{"role":"user","content":"First message"}} + {"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"First reply"}]}} + """; + Path sessionFile = sessionsDir.resolve("inc-session.jsonl"); + Files.writeString(sessionFile, jsonl); + + // First indexing + Map first = service.indexSessionsIncremental(sessionsDir.toString(), 0); + assertEquals(1, first.get("sessions_indexed")); + assertEquals(0, first.get("sessions_skipped")); + + // Second indexing without file change should skip + Map second = service.indexSessionsIncremental(sessionsDir.toString(), 0); + assertEquals(0, second.get("sessions_indexed")); + assertEquals(1, second.get("sessions_skipped")); + } + + @Test + @Order(8) + void incrementalIngestionReindexesChangedFiles() throws Exception { + Path sessionsDir = tempDir.resolve("sessions-reindex"); + Files.createDirectories(sessionsDir); + + String jsonl1 = """ + {"type":"human","message":{"role":"user","content":"Original"}} + """; + Path sessionFile = sessionsDir.resolve("reindex-session.jsonl"); + Files.writeString(sessionFile, jsonl1); + + // First index + service.indexSessionsIncremental(sessionsDir.toString(), 0); + + // Modify file (append a message) + Thread.sleep(1100); // ensure mtime changes (filesystem resolution can be 1s) + String jsonl2 = """ + {"type":"human","message":{"role":"user","content":"Original"}} + {"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"New reply"}]}} + """; + Files.writeString(sessionFile, jsonl2); + + // Second index should detect the change + Map result = service.indexSessionsIncremental(sessionsDir.toString(), 0); + assertEquals(1, result.get("sessions_indexed")); + assertEquals(0, result.get("sessions_skipped")); + + // Verify new content + List> messages = service.getSession("reindex-session"); + // message_index -1 is the sentinel, so filter it out + long realMessages = messages.stream() + .filter(m -> (int) m.get("message_index") >= 0) + .count(); + assertEquals(2, realMessages); + } +} diff --git a/src/test/java/com/javaducker/server/service/StalenessServiceTest.java b/src/test/java/com/javaducker/server/service/StalenessServiceTest.java new file mode 100644 index 0000000..a52da0e --- /dev/null +++ b/src/test/java/com/javaducker/server/service/StalenessServiceTest.java @@ -0,0 +1,58 @@ +package com.javaducker.server.service; + +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +class StalenessServiceTest { + + @Test + void computeStaleSummary_zeroStaleOutOfTen() { + Map result = new LinkedHashMap<>(); + result.put("stale", List.of()); + result.put("total_checked", 10L); + + StalenessService.computeStaleSummary(result); + + assertEquals(0, result.get("stale_count")); + assertEquals(0.0, result.get("stale_percentage")); + } + + @Test + void computeStaleSummary_threeStaleOutOfTen() { + Map result = new LinkedHashMap<>(); + result.put("stale", List.of("a", "b", "c")); + result.put("total_checked", 10L); + + StalenessService.computeStaleSummary(result); + + assertEquals(3, result.get("stale_count")); + assertEquals(30.0, result.get("stale_percentage")); + } + + @Test + void computeStaleSummary_zeroTotal_noDivisionByZero() { + Map result = new LinkedHashMap<>(); + result.put("stale", List.of()); + result.put("total_checked", 0L); + + StalenessService.computeStaleSummary(result); + + assertEquals(0, result.get("stale_count")); + assertEquals(0.0, result.get("stale_percentage")); + } + + @Test + void computeStaleSummary_nullStaleList() { + Map result = new LinkedHashMap<>(); + result.put("stale", null); + result.put("total_checked", 5L); + + StalenessService.computeStaleSummary(result); + + assertEquals(0, result.get("stale_count")); + assertEquals(0.0, result.get("stale_percentage")); + } +}