From df3fac10328a313ac6ec3a67c4a0ff6b06e3f07c Mon Sep 17 00:00:00 2001 From: Administrator Date: Mon, 8 Jun 2026 21:20:31 +0800 Subject: [PATCH] =?UTF-8?q?feat(knowledge):=20=E5=8F=AF=E8=B7=B3=E8=BD=AC?= =?UTF-8?q?=20wikilink=20+=20=E7=9C=9F=E5=AE=9E=E7=9B=B8=E5=85=B3=20note?= =?UTF-8?q?=20+=20=E4=B8=80=E5=AF=B9=E8=AF=9D=E5=A4=9A=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - [[wikilink]] 解析器：note id / vault_path / 标题 / vault 元页 / missing - 前端 MarkdownRenderer 把非数字 [[...]] 渲染成可点链接，元页弹窗查看，缺失置灰 - vault 脚手架补建项目 hub 页，三个结构性链接（index/taxonomy/hub）始终可跳 - capture 时按标题+摘要+区分性标签检索同主题 note，注入真实 [[标题]] 到 Related + 记 related 图谱边 - LLM 提取支持 notes[] 数组，一次对话拆 1~4 页并按标题互链（multi_note_enabled 开关） - 新增接口 POST /resolve、GET /notes/{id}/wikilinks、GET /vault-page - 新增配置 related_notes_enabled / related_notes_limit / multi_note_enabled Co-Authored-By: Claude Opus 4.8 (1M context) --- frontend-web/src/core/knowledge/api.ts | 31 +++ frontend-web/src/core/knowledge/hooks.ts | 20 ++ frontend-web/src/core/knowledge/types.ts | 21 ++ frontend-web/src/pages/KnowledgeBasePage.tsx | 65 +++++- .../strategy-components/MarkdownRenderer.tsx | 63 +++++- offline-backend-20260512/backend/CLAUDE.md | 7 +- .../backend/app/gateway/routers/knowledge.py | 44 +++- .../deerflow/config/knowledge_config.py | 3 + .../harness/deerflow/knowledge/extractor.py | 12 +- .../deerflow/knowledge/llm_extractor.py | 213 +++++++++++++----- .../deerflow/knowledge/markdown_vault.py | 32 +++ .../harness/deerflow/knowledge/repository.py | 42 ++++ .../harness/deerflow/knowledge/service.py | 187 ++++++++++++++- 13 files changed, 668 insertions(+), 72 deletions(-) diff --git a/frontend-web/src/core/knowledge/api.ts b/frontend-web/src/core/knowledge/api.ts index 57824a41..bc4ce250 100644 --- a/frontend-web/src/core/knowledge/api.ts +++ b/frontend-web/src/core/knowledge/api.ts @@ -10,6 +10,8 @@ import type { KnowledgeListResponse, KnowledgeNote, KnowledgeSearchItem, + VaultPage, + WikilinkResolutionMap, } from "./types"; const baseUrl = () => `${getBackendBaseURL()}/api/knowledge`; @@ -125,6 +127,35 @@ export async function ingestThreadsBatch(payload: BatchIngestPayload = {}) { ); } +/** Resolve `[[wikilink]]` targets to note ids / vault pages. */ +export async function resolveWikilinks(targets: string[]) { + const res = await parseResponse<{ resolved: WikilinkResolutionMap }>( + await apiFetch(`${baseUrl()}/resolve`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targets }), + }), + ); + return res.resolved; +} + +/** Resolve every `[[wikilink]]` inside a note's body in one call. */ +export async function resolveNoteWikilinks(noteId: string) { + const res = await parseResponse<{ resolved: WikilinkResolutionMap }>( + await apiFetch( + `${baseUrl()}/notes/${encodeURIComponent(noteId)}/wikilinks`, + ), + ); + return res.resolved; +} + +/** Fetch a raw vault meta/hub Markdown page (no DB note) by its path. */ +export async function fetchVaultPage(path: string) { + return parseResponse( + await apiFetch(`${baseUrl()}/vault-page?path=${encodeURIComponent(path)}`), + ); +} + export async function exportKnowledgeGraph() { return parseResponse( await apiFetch(`${baseUrl()}/export/graph`, { method: "POST" }), diff --git a/frontend-web/src/core/knowledge/hooks.ts b/frontend-web/src/core/knowledge/hooks.ts index 22b27c5e..b3a9b9fb 100644 --- a/frontend-web/src/core/knowledge/hooks.ts +++ b/frontend-web/src/core/knowledge/hooks.ts @@ -9,12 +9,14 @@ import { createKnowledgeNote, deleteKnowledgeNote, exportKnowledgeGraph, + fetchVaultPage, getKnowledgeNote, ingestThread, ingestThreadsBatch, listDuplicates, listKnowledgeNotes, mergeKnowledgeNotes, + resolveNoteWikilinks, searchKnowledge, updateKnowledgeNote, } from "./api"; @@ -40,6 +42,24 @@ export function useKnowledgeNote(noteId: string | undefined) { }); } +export function useNoteWikilinks(noteId: string | undefined) { + return useQuery({ + queryKey: [...ROOT_KEY, "wikilinks", noteId], + queryFn: () => resolveNoteWikilinks(noteId as string), + enabled: !!noteId, + refetchOnWindowFocus: false, + }); +} + +export function useVaultPage(path: string | undefined) { + return useQuery({ + queryKey: [...ROOT_KEY, "vault-page", path], + queryFn: () => fetchVaultPage(path as string), + enabled: !!path, + refetchOnWindowFocus: false, + }); +} + export function useKnowledgeSearch(query: string, enabled: boolean) { return useQuery({ queryKey: [...ROOT_KEY, "search", query], diff --git a/frontend-web/src/core/knowledge/types.ts b/frontend-web/src/core/knowledge/types.ts index 3bdf322c..ec062967 100644 --- a/frontend-web/src/core/knowledge/types.ts +++ b/frontend-web/src/core/knowledge/types.ts @@ -52,6 +52,27 @@ export interface KnowledgeListParams { offset?: number; } +export type WikilinkKind = "note" | "vault" | "missing"; + +export interface WikilinkResolution { + target: string; + normalized?: string; + kind: WikilinkKind; + exists: boolean; + note_id?: string; + vault_path?: string; + title?: string; +} + +/** Map of original `[[target]]` string → its resolution. */ +export type WikilinkResolutionMap = Record; + +export interface VaultPage { + path: string; + title: string; + content_md: string; +} + export interface KnowledgeSearchItem { note_id: string; chunk_id?: string | null; diff --git a/frontend-web/src/pages/KnowledgeBasePage.tsx b/frontend-web/src/pages/KnowledgeBasePage.tsx index 4b7315bd..454ec784 100644 --- a/frontend-web/src/pages/KnowledgeBasePage.tsx +++ b/frontend-web/src/pages/KnowledgeBasePage.tsx @@ -41,10 +41,14 @@ import { useKnowledgeDuplicates, useKnowledgeNote, useKnowledgeNotes, + useNoteWikilinks, + useVaultPage, } from "@/core/knowledge/hooks"; import type { KnowledgeNote } from "@/core/knowledge/types"; import { cn } from "@/lib/utils"; -import MarkdownRenderer from "@/strategy-components/MarkdownRenderer"; +import MarkdownRenderer, { + type WikiLinkTarget, +} from "@/strategy-components/MarkdownRenderer"; const SOURCE_TYPE_LABELS: Record = { thread: "对话", @@ -349,10 +353,56 @@ function DedupDialog({ ); } +function VaultPageDialog({ + path, + onOpenChange, +}: { + path: string | null; + onOpenChange: (open: boolean) => void; +}) { + const { data: page, isLoading } = useVaultPage(path ?? undefined); + return ( + + + + {page?.title || "知识库页面"} + {path} + + +
+ {isLoading ? ( +
加载中...
+ ) : page ? ( + + ) : ( +
页面不存在
+ )} +
+ + + + + + + ); +} + function NoteDetail({ noteId }: { noteId: string }) { const navigate = useNavigate(); const { data: note, isLoading } = useKnowledgeNote(noteId); + const { data: wikiLinks } = useNoteWikilinks(noteId); const { update, remove } = useKnowledgeActions(); + const [vaultPath, setVaultPath] = useState(null); + + const handleWikiLink = (link: WikiLinkTarget) => { + if (link.kind === "note" && link.note_id) { + navigate(`/page/workspace/knowledge/${link.note_id}`); + } else if (link.kind === "vault" && link.vault_path) { + setVaultPath(link.vault_path); + } + }; const [editing, setEditing] = useState(false); const [draftTitle, setDraftTitle] = useState(""); const [draftTags, setDraftTags] = useState(""); @@ -361,6 +411,7 @@ function NoteDetail({ noteId }: { noteId: string }) { useEffect(() => { setEditing(false); + setVaultPath(null); }, [noteId]); useEffect(() => { @@ -495,7 +546,11 @@ function NoteDetail({ noteId }: { noteId: string }) { ))} - + {note.sources && note.sources.length > 0 ? (
@@ -545,6 +600,12 @@ function NoteDetail({ noteId }: { noteId: string }) { )}
+ { + if (!open) setVaultPath(null); + }} + /> ); } diff --git a/frontend-web/src/strategy-components/MarkdownRenderer.tsx b/frontend-web/src/strategy-components/MarkdownRenderer.tsx index 51821967..7f8ae49e 100644 --- a/frontend-web/src/strategy-components/MarkdownRenderer.tsx +++ b/frontend-web/src/strategy-components/MarkdownRenderer.tsx @@ -16,12 +16,33 @@ import tsx from 'react-syntax-highlighter/dist/esm/languages/prism/tsx'; import typescript from 'react-syntax-highlighter/dist/esm/languages/prism/typescript'; import { oneDark } from 'react-syntax-highlighter/dist/esm/styles/prism'; +export interface WikiLinkTarget { + target: string; + kind: 'note' | 'vault' | 'missing'; + exists: boolean; + note_id?: string; + vault_path?: string; + title?: string; +} + +export type WikiLinkMap = Record; + interface MarkdownRendererProps { content: string; className?: string; onCitationClick?: (index: number) => void; + /** Resolved `[[wikilink]]` targets; when provided, non-numeric `[[...]]` render as links. */ + wikiLinks?: WikiLinkMap; + onWikiLinkClick?: (link: WikiLinkTarget) => void; } +const escapeHtmlAttr = (value: string): string => + value + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); + SyntaxHighlighter.registerLanguage('bash', bash); SyntaxHighlighter.registerLanguage('css', css); SyntaxHighlighter.registerLanguage('html', markup); @@ -126,8 +147,35 @@ const CodeBlock: React.FC<{ children: React.ReactNode; className?: string }> = ( ); }; -const MarkdownRenderer: React.FC = ({ content, className = '', onCitationClick }) => { - const contentWithCitationBadge = content.replace( +const MarkdownRenderer: React.FC = ({ + content, + className = '', + onCitationClick, + wikiLinks, + onWikiLinkClick, +}) => { + // 1) Non-numeric [[wikilinks]] → clickable (or greyed-out) links, driven by the + // resolver map. Numeric [[123]] are left untouched here for the citation pass. + let processed = content; + if (wikiLinks) { + processed = processed.replace( + /\[\[([^\]\n|]+)(?:\|([^\]\n]+))?\]\]/g, + (match, rawTarget: string, alias?: string) => { + const target = String(rawTarget).trim(); + if (/^\d+$/.test(target)) return match; // citation badge, handled below + const resolved = wikiLinks[target]; + const label = (alias?.trim() || resolved?.title || target); + const safeLabel = escapeHtmlAttr(label); + const safeTarget = escapeHtmlAttr(target); + if (resolved && resolved.exists) { + return ``; + } + return `${safeLabel}`; + }, + ); + } + // 2) Numeric [[123]] citation badges. + const contentWithCitationBadge = processed.replace( /\[\[(\d+)\]\]/g, '', ); @@ -136,8 +184,17 @@ const MarkdownRenderer: React.FC = ({ content, className
{ - if (!onCitationClick) return; const target = event.target as HTMLElement; + const wikiButton = target.closest('[data-wikilink-target]') as HTMLElement | null; + if (wikiButton && onWikiLinkClick && wikiLinks) { + const key = wikiButton.dataset.wikilinkTarget || ''; + const link = wikiLinks[key]; + if (link) { + onWikiLinkClick(link); + return; + } + } + if (!onCitationClick) return; const button = target.closest('[data-citation-index]') as HTMLElement | null; if (!button) return; const index = Number(button.dataset.citationIndex || '0'); diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index 43b4de47..3d8cb583 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -250,7 +250,7 @@ FastAPI application on port 8001 with health check at `GET /health`. Set `GATEWA | **Scheduled Tasks** (`/api/scheduled-tasks`) | Per-user cron-like agent tasks. `GET/POST /` - list/create owned tasks; `PATCH/DELETE /{id}` - update/delete (owner); `POST /{id}/{pause,resume,trigger}`; `GET /{id}/runs`, `GET /runs/{rid}`, run-file preview/edit/download; `GET /runs/{rid}/messages` - agent messages for the run's underlying agent run (used to inspect a stuck/running run; authorized via `_load_run`); `GET /runs/{rid}/html` - generated HTML document for an `html_page` run; `DELETE /runs/{rid}` - owner-scoped, with an admin fallback so admins can delete any user's run; publish/subscribe under `/{id}/{publish,unpublish,subscribe}` and `GET /published`. **Admin (cross-user):** `GET /admin/all` - every task; `PATCH/DELETE /admin/{id}` - edit/delete any task; `POST /admin/{id}/trigger`; `GET /admin/{id}/runs`. Admin routes require `system_role == "admin"`. | | **HTML Page Favorites** (`/api/html-page-favorites`) | Bookmarked generated HTML pages, owner-scoped. `GET /` - list; `POST /` `{page_id, title?, description?, tags?}` - bookmark a `scheduled_html_pages` row (extracts a reusable style/layout summary via GLM5 at favorite time); a page can only be favorited once per user (`DuplicateFavoriteError` → HTTP 409). `GET/DELETE /{id}` - read/delete (the create-task reference picker deletes favorites here). A favorite can be selected as a style reference when creating a new `html_page` task. | | **Public Scheduled Tasks** (`/api/public/scheduled-tasks`) | **Unauthenticated** mirror for embedding. Adds `GET /runs/{rid}/html` and `GET /{task_id}/latest-html` for the `html_page` preview; the run result keeps `result.html` (only file paths are redacted). | -| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup); `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector→keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe). Backed by the `deerflow.knowledge` module (see below). | +| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup); `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector→keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[…]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). Backed by the `deerflow.knowledge` module (see below). | Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → Gateway. @@ -284,6 +284,11 @@ Self-contained product knowledge base. **All knowledge capability lives in this **Module layout:** `models.py` (ORM: notes/sources/tags/note_tags + embeddings/versions/entities/relations) · `repository.py` (DB only, no LLM) · `schemas.py` (NoteDraft/SourceDraft DTOs) · `skill_templates.py` · `markdown_vault.py` · `manifest.py` · `extractor.py` (rule-based fallback) · `llm_extractor.py` (**default**: distills declarative knowledge per wiki-capture, structured JSON incl. entities/relations; falls back to `extractor.py`) · `sources.py` · `exporter.py` (`graph.json`/`graph.graphml`/`cypher.txt` + self-contained vanilla-JS `graph.html`; entity nodes + relation edges) · `chunking.py` + `embeddings.py` (OpenAI-compatible embedding client, keyword fallback) + `search.py` (keyword/vector/hybrid) · `sensitive.py` (secret/PII redaction) · `dedup.py` (Jaccard + hash dedup) · `auto_ingest.py` (background queue/worker) · `obsidian_adapter.py` · `service.py` (`KnowledgeService` orchestrator — the only thing the router touches). Tests: `tests/test_knowledge.py` (20 cases). Frontend: `frontend-web/src/core/knowledge/*` + `pages/KnowledgeBasePage.tsx` / `KnowledgeGraphPage.tsx` + chat `knowledge-save-trigger.tsx` / `knowledge-rag-toggle.tsx`; spec at `frontend-web/docs/product-knowledge-base-dev.md`. +**Wikilinks & multi-note capture** (`knowledge.{related_notes_enabled,related_notes_limit,multi_note_enabled}`, all default-on): +- **Navigable `[[wikilinks]]`** — `service.resolve_wikilink(s)` maps a `[[target]]` to a knowledge note (by id → `vault_path` → title) or a vault meta/hub page (`read_vault_page`, path-traversal-proof), else `missing`. `markdown_vault.normalize_wikilink_target` canonicalizes targets (drops `|alias`, slashes, `.md`). The frontend `MarkdownRenderer` renders non-numeric `[[...]]` as clickable links from the resolver map (numeric `[[123]]` stay citation badges); note links navigate, vault links open a `VaultPageDialog`, missing links are greyed. The vault scaffold now materializes `projects//synthesis/product-knowledge-base.md` (the hub page every note's `## Related` links to) so the three historically-hardcoded structural links (`[[index]]`, `[[_meta/taxonomy]]`, the hub) always resolve. +- **Real related notes** — on capture, `service._find_related_notes` runs keyword/hybrid search over the draft's title + summary + *distinctive* tags (project/`knowledge-base` defaults excluded) and injects real `[[Title]]` cross-links at the top of `## Related` (plus note→note `related` graph edges), replacing the placeholder-only block. +- **Multi-note split** — `llm_extractor.extract_thread_llm_multi` lets one conversation distill into 1–4 cross-linked notes (`notes:[…]` JSON array; legacy single-object output still accepted). `capture_thread` persists each, sibling-cross-links them by title, and returns `{note, created, notes:[…]}`. Gated by `multi_note_enabled` (false → first note only). Tests: `tests/test_knowledge_wikilinks.py`. + **Phases 2–4 (all config-gated, defaults safe):** - **Phase 2 — vector RAG**: `knowledge.embedding.{enabled,base_url,api_key,model,...}` enables an OpenAI-compatible embedding endpoint; notes are chunked + embedded on write (`knowledge_embeddings`), `POST /search` supports `mode=keyword|vector|hybrid` (vector degrades to keyword when unconfigured). `KnowledgeRagMiddleware` (lead-agent chain) injects a transient `` recall before the model call, gated **per-conversation** by the `knowledge_rag_enabled` runtime flag (frontend chat toggle) falling back to global `knowledge.rag_enabled`. `POST /reindex` rebuilds the index. - **Phase 3 — governance**: `sensitive.py` redacts secrets/PII before persistence (`sensitive_redaction_enabled`); every update snapshots `knowledge_note_versions` (`GET /notes/{id}/versions`); `GET /duplicates` + `POST /merge` for dedup; background `AutoIngestQueue` (wired in `deps.py`, started/stopped in the gateway lifespan) auto-sediments finished conversations **off the chat path** when `auto_ingest_enabled` — enqueued from `KnowledgeRagMiddleware.after_agent`, value-judged by min-chars + `skip_existing`, landing as `draft`. diff --git a/offline-backend-20260512/backend/app/gateway/routers/knowledge.py b/offline-backend-20260512/backend/app/gateway/routers/knowledge.py index 52bc016d..ac2d10b9 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/knowledge.py +++ b/offline-backend-20260512/backend/app/gateway/routers/knowledge.py @@ -95,6 +95,10 @@ class SearchRequest(BaseModel): limit: int = Field(default=8, ge=1, le=50) +class ResolveWikilinksRequest(BaseModel): + targets: list[str] = Field(default_factory=list, description="`[[wikilink]]` targets to resolve to note ids / vault pages.") + + # --------------------------------------------------------------------------- # # Helpers # --------------------------------------------------------------------------- # @@ -198,7 +202,7 @@ async def ingest_thread(thread_id: str, body: IngestThreadRequest, request: Requ ) except EmptyThreadError: raise HTTPException(status_code=400, detail="Thread has no sediment-able content") - return {"note": result["note"], "created": result["created"]} + return {"note": result["note"], "created": result["created"], "notes": result.get("notes", [result["note"]])} @router.post("/ingest/threads/batch") @@ -356,6 +360,44 @@ async def search(body: SearchRequest, request: Request) -> dict[str, Any]: return {"items": items} +# --------------------------------------------------------------------------- # +# Wikilinks ([[...]] resolution + vault meta-page viewer) +# --------------------------------------------------------------------------- # +@router.post("/resolve") +@require_auth +async def resolve_wikilinks(body: ResolveWikilinksRequest, request: Request) -> dict[str, Any]: + """Resolve ``[[wikilink]]`` targets to navigable destinations. + + Each target resolves to a knowledge note (``kind=note``), a vault meta/hub + page (``kind=vault``), or nothing (``kind=missing``). Powers clickable + wikilinks in the knowledge UI. + """ + service = get_knowledge_service(request) + return {"resolved": await service.resolve_wikilinks(body.targets)} + + +@router.get("/notes/{note_id}/wikilinks") +@require_auth +async def resolve_note_wikilinks(note_id: str, request: Request) -> dict[str, Any]: + """Resolve every ``[[wikilink]]`` found in a note's body in one call.""" + service = get_knowledge_service(request) + resolved = await service.resolve_note_wikilinks(note_id) + if resolved is None: + raise HTTPException(status_code=404, detail="Knowledge note not found") + return {"resolved": resolved} + + +@router.get("/vault-page") +@require_auth +async def get_vault_page(request: Request, path: str) -> dict[str, Any]: + """Return a raw vault Markdown meta/hub page (no DB note) by its path.""" + service = get_knowledge_service(request) + page = service.read_vault_page(path) + if page is None: + raise HTTPException(status_code=404, detail="Vault page not found") + return page + + class MergeRequest(BaseModel): primary_id: str duplicate_ids: list[str] = Field(default_factory=list) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py index d815a50d..ee9525c1 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py @@ -62,6 +62,9 @@ class KnowledgeConfig(BaseModel): extract_max_input_chars: int = Field(default=12000, description="Max characters of conversation transcript fed to the extraction LLM.") max_thread_messages: int = Field(default=80, description="Maximum thread messages to scan when capturing.") max_source_items: int = Field(default=20, description="Maximum source items recorded per note.") + related_notes_enabled: bool = Field(default=True, description="On capture, find existing notes on the same topic and link them under ## Related (real cross-references instead of placeholders).") + related_notes_limit: int = Field(default=3, description="Maximum real related notes linked into each captured note.") + multi_note_enabled: bool = Field(default=True, description="Allow the LLM extractor to split one conversation into multiple cross-linked knowledge notes.") rag_enabled: bool = Field(default=False, description="Enable RAG injection before agent runs (phase 2).") rag_limit: int = Field(default=5, description="Maximum knowledge snippets injected per query.") sensitive_redaction_enabled: bool = Field(default=True, description="Redact secrets/PII (tokens, keys, phone, id-card, email) before persisting notes (phase 3).") diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/extractor.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/extractor.py index 82c5d6df..cd044a18 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/extractor.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/extractor.py @@ -105,10 +105,18 @@ def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str: def related_links(project_name: str | None) -> list[str]: - """At least two wikilinks per note (wiki-capture requirement).""" + """Structural wikilinks every note carries (wiki-capture requirement). + + These point at vault meta/hub pages (``index`` / ``_meta/taxonomy`` / the + project hub) that always exist, so they resolve to navigable pages. Content- + specific "related notes" are injected separately by the service layer (which + has DB access) on top of these. + """ + from deerflow.knowledge.markdown_vault import slugify + links = ["[[index]]", "[[_meta/taxonomy]]"] if project_name: - links.append(f"[[projects/{project_name}/synthesis/product-knowledge-base]]") + links.append(f"[[projects/{slugify(project_name)}/synthesis/product-knowledge-base]]") return links diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py index 91665529..e869aebc 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py @@ -40,21 +40,30 @@ _SYSTEM_PROMPT = """\ references（外部来源汇总）、decision（架构/设计决策）、journal（会话纪要）。 - 用中文输出。 +一次对话可能只讲一个主题，也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识： +- 默认只产出 **1 页**；只有当对话明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。 +- 每一页聚焦单一主题，标题与 summary 必须彼此区分，不要把同一结论拆到多页。 +- 不值得沉淀时返回空的 notes 数组。 + 只输出一个 JSON 对象，不要任何额外文字或 Markdown 代码块，结构如下： { "should_ingest": true, "reason": "为什么值得/不值得沉淀", - "title": "简洁的知识标题（不超过 40 字）", - "summary": "1-2 句话概括这页知识", - "category": "synthesis", - "context": "问题背景或触发场景，1-3 句", - "findings": ["结论/要点（陈述式，可含 ^[inferred]）", "..."], - "reasoning": "为什么成立、取舍、不确定性", - "implications": ["后续建议/风险/下一步", "..."], - "tags": ["标签1", "标签2"], - "entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}], - "relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}], - "confidence": 0.0 + "notes": [ + { + "title": "简洁的知识标题（不超过 40 字）", + "summary": "1-2 句话概括这页知识", + "category": "synthesis", + "context": "问题背景或触发场景，1-3 句", + "findings": ["结论/要点（陈述式，可含 ^[inferred]）", "..."], + "reasoning": "为什么成立、取舍、不确定性", + "implications": ["后续建议/风险/下一步", "..."], + "tags": ["标签1", "标签2"], + "entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}], + "relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}], + "confidence": 0.0 + } + ] }""" @@ -128,7 +137,78 @@ def _render_body(data: dict[str, Any], title: str, related: list[str], sources: ) -async def extract_thread_llm( +def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]: + """Normalize the LLM JSON into a list of per-note dicts. + + Accepts the multi-note shape ``{"notes": [...]}`` as well as the legacy + single-note shape (``{"title": ..., "findings": ...}``) for backward + compatibility. Notes without any title/findings are dropped. + """ + raw_notes = data.get("notes") + if isinstance(raw_notes, list): + candidates = [n for n in raw_notes if isinstance(n, dict)] + elif data.get("title") or data.get("findings") or data.get("context"): + candidates = [data] # legacy single-object output + else: + candidates = [] + return [n for n in candidates if str(n.get("title") or "").strip() or n.get("findings")] + + +def _build_note_draft( + note_data: dict[str, Any], + *, + thread_id: str, + thread_title: str | None, + turns: list[dict[str, str]], + title_override: str | None, + tags: list[str] | None, + project_name: str | None, + sources: list[SourceDraft], + source_key_suffix: str = "", +) -> NoteDraft: + """Render one note dict into a :class:`NoteDraft`.""" + category = str(note_data.get("category") or "synthesis").strip().lower() + if category not in _VALID_CATEGORIES: + category = "synthesis" + + final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or extractor._auto_title(thread_title, turns) + summary = str(note_data.get("summary") or "").strip() + related = extractor.related_links(project_name) + body = _render_body(note_data, final_title, related, sources) + + base_tags = list(tags or []) + llm_tags = note_data.get("tags") if isinstance(note_data.get("tags"), list) else [] + for tag in [*llm_tags, project_name or "deerflow", "knowledge-base"]: + tag = str(tag).strip() + if tag and tag not in base_tags: + base_tags.append(tag) + + try: + confidence = float(note_data.get("confidence") or 0.7) + except (TypeError, ValueError): + confidence = 0.7 + + entities = note_data.get("entities") if isinstance(note_data.get("entities"), list) else [] + relations = note_data.get("relations") if isinstance(note_data.get("relations"), list) else [] + + return NoteDraft( + title=final_title, + summary=summary or final_title, + content_md=body, + category=category, + source_type="thread", + source_id=thread_id, + tags=base_tags, + confidence=max(0.0, min(confidence, 1.0)), + related=related, + sources=sources, + entities=[e for e in entities if isinstance(e, dict) and e.get("name")], + relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")], + source_key=f"thread:{thread_id}{source_key_suffix}", + ) + + +async def extract_thread_llm_multi( messages: list[dict[str, Any]], *, thread_id: str, @@ -141,15 +221,19 @@ async def extract_thread_llm( max_messages: int = 80, model_name: str | None = None, max_input_chars: int = 12000, -) -> NoteDraft | None: - """Distill a thread into a :class:`NoteDraft` via the LLM. + allow_multi: bool = True, + max_notes: int = 4, +) -> list[NoteDraft]: + """Distill a thread into one *or several* cross-linkable knowledge notes. - Returns ``None`` when there is nothing to sediment **or** the LLM path - fails — callers should fall back to the rule-based extractor. + The LLM may split a conversation that covers multiple independent topics into + several declarative notes. Returns ``[]`` when there is nothing to sediment + or the LLM path fails — callers should fall back to the rule-based extractor. + When ``allow_multi`` is false only the first note is kept. """ turns = extractor.split_turns(messages, max_messages=max_messages) if not turns: - return None + return [] sources = sources or [] try: @@ -168,47 +252,72 @@ async def extract_thread_llm( data = _parse_json(raw) except Exception: logger.exception("Knowledge LLM extraction failed for thread %s; using rule-based fallback", thread_id) - return None + return [] if not data: - return None + return [] - category = str(data.get("category") or "synthesis").strip().lower() - if category not in _VALID_CATEGORIES: - category = "synthesis" + note_dicts = _notes_from_data(data) + if not note_dicts: + return [] + if not allow_multi: + note_dicts = note_dicts[:1] + else: + note_dicts = note_dicts[: max(1, max_notes)] - final_title = (title or "").strip() or str(data.get("title") or "").strip() or extractor._auto_title(thread_title, turns) - summary = str(data.get("summary") or "").strip() - related = extractor.related_links(project_name) - body = _render_body(data, final_title, related, sources) + drafts: list[NoteDraft] = [] + for i, nd in enumerate(note_dicts): + # The user-supplied title override only applies to a single-note result; + # multi-note splits keep each note's own LLM title. + override = title if (len(note_dicts) == 1) else None + suffix = "" if len(note_dicts) == 1 else f"#{i}" + drafts.append( + _build_note_draft( + nd, + thread_id=thread_id, + thread_title=thread_title, + turns=turns, + title_override=override, + tags=tags, + project_name=project_name, + sources=sources, + source_key_suffix=suffix, + ) + ) + return drafts - base_tags = list(tags or []) - llm_tags = data.get("tags") if isinstance(data.get("tags"), list) else [] - for tag in [*llm_tags, project_name or "deerflow", "knowledge-base"]: - tag = str(tag).strip() - if tag and tag not in base_tags: - base_tags.append(tag) - try: - confidence = float(data.get("confidence") or 0.7) - except (TypeError, ValueError): - confidence = 0.7 +async def extract_thread_llm( + messages: list[dict[str, Any]], + *, + thread_id: str, + thread_title: str | None = None, + mode: str = "summary", + title: str | None = None, + tags: list[str] | None = None, + project_name: str | None = "deerflow", + sources: list[SourceDraft] | None = None, + max_messages: int = 80, + model_name: str | None = None, + max_input_chars: int = 12000, +) -> NoteDraft | None: + """Distill a thread into a single :class:`NoteDraft` (backward-compatible). - entities = data.get("entities") if isinstance(data.get("entities"), list) else [] - relations = data.get("relations") if isinstance(data.get("relations"), list) else [] - - return NoteDraft( - title=final_title, - summary=summary or final_title, - content_md=body, - category=category, - source_type="thread", - source_id=thread_id, - tags=base_tags, - confidence=max(0.0, min(confidence, 1.0)), - related=related, + Thin wrapper over :func:`extract_thread_llm_multi` that returns the first + note, or ``None`` when nothing was produced. + """ + drafts = await extract_thread_llm_multi( + messages, + thread_id=thread_id, + thread_title=thread_title, + mode=mode, + title=title, + tags=tags, + project_name=project_name, sources=sources, - entities=[e for e in entities if isinstance(e, dict) and e.get("name")], - relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")], - source_key=f"thread:{thread_id}", + max_messages=max_messages, + model_name=model_name, + max_input_chars=max_input_chars, + allow_multi=False, ) + return drafts[0] if drafts else None diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/markdown_vault.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/markdown_vault.py index 8bccda2a..2bfde02b 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/markdown_vault.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/markdown_vault.py @@ -52,6 +52,23 @@ def extract_wikilinks(text: str) -> list[str]: return seen +def normalize_wikilink_target(target: str) -> str: + """Canonicalize a ``[[wikilink]]`` target for resolution/comparison. + + Drops any ``|alias`` part, normalizes slashes, strips a leading ``/`` and a + trailing ``.md`` — so ``[[index]]``, ``index.md`` and ``/index`` all map to + the same canonical ``index`` key used by the index/graph/resolver layers. + """ + t = (target or "").strip() + if not t: + return "" + t = t.split("|", 1)[0].strip() + t = t.replace("\\", "/").lstrip("/") + if t.lower().endswith(".md"): + t = t[:-3] + return t + + class VaultPathError(ValueError): """Raised when a requested vault path escapes the vault root.""" @@ -99,6 +116,21 @@ class MarkdownVault: base = self.root / "projects" / slugify(project_name) for d in ("concepts", "references", "synthesis", "journal"): (base / d).mkdir(parents=True, exist_ok=True) + # Project hub page — the structural target every note links to under + # ``## Related`` (``[[projects//synthesis/product-knowledge-base]]``). + # Materialize it so that wikilink is always navigable, not a dangling + # placeholder. + hub = base / "synthesis" / "product-knowledge-base.md" + if not hub.exists(): + hub.write_text( + f"---\ntitle: 产品知识库总览\ncategory: synthesis\n---\n\n" + f"# 产品知识库总览\n\n" + f"本页是 **{project_name}** 项目知识库的入口枢纽（hub）。每条知识都会在 `## Related` 中链接回这里。\n\n" + "- [[index]] — 全部页面索引\n" + "- [[_meta/taxonomy]] — 分类体系\n\n" + "## 主题\n\n_随着知识沉淀，这里会逐步链接到各主题页面。_\n", + encoding="utf-8", + ) # Base vault files. index = self.root / "index.md" if not index.exists(): diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/repository.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/repository.py index a87692e4..b509901d 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/repository.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/repository.py @@ -216,6 +216,48 @@ class KnowledgeRepository: d["sources"] = [self._source_to_dict(r) for r in (await session.execute(stmt)).scalars()] return d + async def find_by_vault_path(self, vault_path: str) -> dict[str, Any] | None: + """Return the first (non-archived) note whose vault_path matches. + + ``vault_path`` is matched both with and without the ``.md`` suffix, since + wikilink targets are stored without it while ``vault_path`` keeps it. + """ + stripped = vault_path[:-3] if vault_path.endswith(".md") else vault_path + candidates = {vault_path, stripped, f"{stripped}.md"} + async with self._sf() as session: + stmt = ( + select(KnowledgeNoteRow) + .where(KnowledgeNoteRow.vault_path.in_(list(candidates))) + .where(KnowledgeNoteRow.status != "archived") + .limit(1) + ) + row = (await session.execute(stmt)).scalars().first() + if row is None: + return None + d = self._note_to_dict(row) + d["tags"] = await self._tags_for(session, row.id) + return d + + async def find_by_title(self, title: str) -> dict[str, Any] | None: + """Return the first (non-archived) note with a case-insensitive title match.""" + name = (title or "").strip() + if not name: + return None + async with self._sf() as session: + stmt = ( + select(KnowledgeNoteRow) + .where(func.lower(KnowledgeNoteRow.title) == name.lower()) + .where(KnowledgeNoteRow.status != "archived") + .order_by(KnowledgeNoteRow.updated_at.desc()) + .limit(1) + ) + row = (await session.execute(stmt)).scalars().first() + if row is None: + return None + d = self._note_to_dict(row) + d["tags"] = await self._tags_for(session, row.id) + return d + async def find_by_source(self, source_type: str, source_id: str) -> dict[str, Any] | None: """Return the first (non-archived) note for a (source_type, source_id).""" async with self._sf() as session: diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py index 1b70e1ea..29542e84 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py @@ -18,7 +18,7 @@ from deerflow.config.runtime_paths import runtime_home from deerflow.knowledge import extractor, llm_extractor from deerflow.knowledge.chunking import chunk_markdown from deerflow.knowledge.embeddings import EmbeddingClient -from deerflow.knowledge.markdown_vault import MarkdownVault +from deerflow.knowledge.markdown_vault import MarkdownVault, VaultPathError, extract_wikilinks, normalize_wikilink_target from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter from deerflow.knowledge.repository import KnowledgeRepository from deerflow.knowledge.schemas import NoteDraft, SourceDraft, normalize_source_type, normalize_status @@ -39,6 +39,24 @@ def _resolve_vault_dir(vault_dir: str) -> Path: return (runtime_home() / vault_dir).resolve() +def _inject_related_notes(body: str, links: list[str]) -> str: + """Insert real related-note ``[[wikilinks]]`` at the top of the Related block. + + Keeps the structural meta-links (index / taxonomy / hub) that the extractor + already rendered, but surfaces the content-specific related notes first. When + there is no ``## Related`` section, one is appended. + """ + if not links: + return body + bullets = "\n".join(f"- {link}" for link in links) + marker = "## Related\n" + idx = body.find(marker) + if idx == -1: + return body.rstrip() + "\n\n## Related\n\n" + bullets + "\n" + insert_at = idx + len(marker) + return body[:insert_at] + "\n" + bullets + "\n" + body[insert_at:] + + class KnowledgeService: """Application-facing knowledge base API.""" @@ -75,9 +93,56 @@ class KnowledgeService: logger.info("Knowledge redaction applied to note '%s': %s", draft.title, ",".join(sorted(set(body_hits)))) return draft + async def _find_related_notes(self, draft: NoteDraft) -> list[dict[str, Any]]: + """Find existing notes on the same topic to cross-link under ## Related. + + Uses the existing keyword/hybrid search over the title (+ summary). The + draft's own note does not exist yet, so we exclude same-title hits to + avoid self-links when re-capturing or deduping. Best-effort: any failure + yields no links rather than blocking the capture. + """ + if not self.config.related_notes_enabled: + return [] + # Build a topical query from the title + summary + the note's *distinctive* + # tags (the per-note project/knowledge-base defaults are dropped, else + # every note would match every other). CJK runs tokenize as a single + # token, so shared distinctive tags are what make cross-topic matches work. + default_tags = {t.lower() for t in (self.project_name or "", "knowledge-base", "deerflow") if t} + distinctive = [t for t in (draft.tags or []) if t and t.lower() not in default_tags] + query = " ".join(p for p in [draft.title or "", draft.summary or "", *distinctive] if p).strip() + if not query: + return [] + limit = max(1, self.config.related_notes_limit) + mode = "hybrid" if self.embedding_client.is_enabled() else "keyword" + try: + items = await search_notes(self.repo, query, mode=mode, limit=limit + 3, embedding_client=self.embedding_client) + except Exception: + logger.exception("Failed to find related notes for '%s'", draft.title) + return [] + own_title = (draft.title or "").strip().lower() + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for it in items: + nid = it.get("note_id") + title = (it.get("title") or "").strip() + if not nid or nid in seen or not title: + continue + if title.lower() == own_title: # the draft itself / a duplicate + continue + seen.add(nid) + out.append({"id": nid, "title": title}) + if len(out) >= limit: + break + return out + async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]: """Create a DB row, write the vault file, attach sources, return note.""" draft = self._redact_draft(draft) + related_notes = await self._find_related_notes(draft) + if related_notes: + real_links = [f"[[{n['title']}]]" for n in related_notes] + draft.content_md = _inject_related_notes(draft.content_md, real_links) + draft.related = list(dict.fromkeys([*real_links, *draft.related])) note = await self.repo.create_note( title=draft.title, summary=draft.summary, @@ -100,6 +165,13 @@ class KnowledgeService: await self.repo.add_sources(note_id, draft.sources, created_by=created_by) await self._reindex_embeddings(note_id, draft.content_md) await self._store_entities(note_id, draft) + # Record note→note "related" edges (after _store_entities, which clears + # this note's relations) so the graph reflects the cross-links too. + for rel_note in related_notes: + try: + await self.repo.add_relation(from_note_id=note_id, to_note_id=rel_note["id"], relation_type="related") + except Exception: + logger.exception("Failed to link related note %s -> %s", note_id, rel_note["id"]) return await self.repo.get_note(note_id) or note async def _store_entities(self, note_id: str, draft: NoteDraft) -> None: @@ -173,12 +245,13 @@ class KnowledgeService: if include_sources and reference_batches: sources = from_reference_batches(reference_batches, thread_id=thread_id, max_items=self.config.max_source_items) - draft: NoteDraft | None = None + drafts: list[NoteDraft] = [] # Prefer LLM distillation (wiki-capture style) so notes are declarative - # knowledge, not raw chat. Falls back to the rule-based extractor when - # the LLM is disabled/unavailable or returns something unusable. + # knowledge, not raw chat. A single conversation may split into several + # cross-linked notes. Falls back to the rule-based extractor (always one + # note) when the LLM is disabled/unavailable or returns nothing usable. if self.config.llm_extract_enabled: - draft = await llm_extractor.extract_thread_llm( + drafts = await llm_extractor.extract_thread_llm_multi( messages, thread_id=thread_id, thread_title=thread_title, @@ -190,9 +263,10 @@ class KnowledgeService: max_messages=self.config.max_thread_messages, model_name=self.config.extract_model_name, max_input_chars=self.config.extract_max_input_chars, + allow_multi=self.config.multi_note_enabled, ) - if draft is None: - draft = extractor.extract_thread( + if not drafts: + single = extractor.extract_thread( messages, thread_id=thread_id, thread_title=thread_title, @@ -203,11 +277,24 @@ class KnowledgeService: sources=sources, max_messages=self.config.max_thread_messages, ) - if draft is None: + drafts = [single] if single else [] + if not drafts: raise EmptyThreadError(f"Thread {thread_id} has no sediment-able content") - draft.status = normalize_status(status) - note = await self._persist_draft(draft, created_by=created_by) - return {"note": note, "created": True} + + normalized = normalize_status(status) + for draft in drafts: + draft.status = normalized + # Cross-link sibling notes from the same conversation by title, so the + # split pages reference each other (resolved once all are persisted). + if len(drafts) > 1: + for i, draft in enumerate(drafts): + siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title] + if siblings: + draft.content_md = _inject_related_notes(draft.content_md, siblings) + draft.related = list(dict.fromkeys([*siblings, *draft.related])) + + notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts] + return {"note": notes[0], "created": True, "notes": notes} async def capture_search( self, @@ -357,6 +444,84 @@ class KnowledgeService: logger.exception("Failed to archive knowledge note %s in vault", note_id) return True + # -- wikilinks -------------------------------------------------------- + async def resolve_wikilink(self, target: str) -> dict[str, Any]: + """Resolve a single ``[[wikilink]]`` target to a navigable destination. + + Returns one of: + - ``{"kind": "note", "note_id", "title", "exists": True}`` — a knowledge note + - ``{"kind": "vault", "vault_path", "title", "exists": True}`` — a vault meta/hub page + - ``{"kind": "missing", "exists": False}`` — nothing matches + + Resolution order: note id → note vault_path → note title → vault file. + """ + norm = normalize_wikilink_target(target) + base = {"target": target, "normalized": norm} + if not norm: + return {**base, "kind": "missing", "exists": False} + + # 1) Direct note id (e.g. "kb_abc..."). + if norm.startswith("kb_"): + note = await self.repo.get_note(norm, with_sources=False) + if note and note.get("status") != "archived": + return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True} + + # 2) Note by vault path (links from index.md / hot.md / structural hub). + note = await self.repo.find_by_vault_path(norm) + if note: + return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True} + + # 3) Note by title (the real related-note cross-links use [[Title]]). + note = await self.repo.find_by_title(norm) or await self.repo.find_by_title(norm.split("/")[-1]) + if note: + return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True} + + # 4) A vault meta/hub page that has no DB note (index, _meta/taxonomy, hub…). + page = self.read_vault_page(norm) + if page is not None: + return {**base, "kind": "vault", "vault_path": page["path"], "title": page["title"], "exists": True} + + return {**base, "kind": "missing", "exists": False} + + async def resolve_wikilinks(self, targets: list[str]) -> dict[str, dict[str, Any]]: + """Resolve many targets at once; returns ``{original_target: resolution}``.""" + out: dict[str, dict[str, Any]] = {} + for target in dict.fromkeys(t for t in targets if t and t.strip()): + out[target] = await self.resolve_wikilink(target) + return out + + async def resolve_note_wikilinks(self, note_id: str) -> dict[str, dict[str, Any]] | None: + """Extract every ``[[wikilink]]`` in a note's body and resolve them. + + Returns ``{target: resolution}`` or ``None`` when the note is missing. + Numeric ``[[123]]`` citation badges are skipped (handled by the renderer). + """ + note = await self.repo.get_note(note_id, with_sources=False) + if note is None: + return None + targets = [t for t in extract_wikilinks(note.get("content_md") or "") if not t.isdigit()] + return await self.resolve_wikilinks(targets) + + def read_vault_page(self, path: str) -> dict[str, Any] | None: + """Read a raw vault Markdown page (meta/hub pages without a DB note). + + Returns ``{"path", "title", "content_md"}`` or ``None`` if the page does + not exist or the path escapes the vault root. + """ + norm = normalize_wikilink_target(path) + if not norm: + return None + rel = f"{norm}.md" + try: + parsed = self.adapter.vault.read_note(rel) + except VaultPathError: + return None + if parsed is None: + return None + fm, body = parsed + title = str(fm.get("title") or "").strip() or norm.split("/")[-1] + return {"path": norm, "title": title, "content_md": body} + # -- search & graph --------------------------------------------------- async def search(self, query: str, *, mode: str = "keyword", limit: int = 8) -> list[dict[str, Any]]: return await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)