feat(knowledge): 可跳转 wikilink + 真实相关 note + 一对话多页

- [[wikilink]] 解析器:note id / vault_path / 标题 / vault 元页 / missing
- 前端 MarkdownRenderer 把非数字 [[...]] 渲染成可点链接,元页弹窗查看,缺失置灰
- vault 脚手架补建项目 hub 页,三个结构性链接(index/taxonomy/hub)始终可跳
- capture 时按标题+摘要+区分性标签检索同主题 note,注入真实 [[标题]] 到 Related + 记 related 图谱边
- LLM 提取支持 notes[] 数组,一次对话拆 1~4 页并按标题互链(multi_note_enabled 开关)
- 新增接口 POST /resolve、GET /notes/{id}/wikilinks、GET /vault-page
- 新增配置 related_notes_enabled / related_notes_limit / multi_note_enabled

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
AdministratorandClaude Opus 4.8 committed 2026-06-08 21:20:31 +08:00
1 parent 92a46003f8
commit df3fac1032
13 files changed
+668 -72

No files matched your search

+31
View File
@@ -10,6 +10,8 @@ import type {
KnowledgeListResponse,
KnowledgeNote,
KnowledgeSearchItem,
VaultPage,
WikilinkResolutionMap,
} from "./types";
const baseUrl = () => `${getBackendBaseURL()}/api/knowledge`;
@@ -125,6 +127,35 @@ export async function ingestThreadsBatch(payload: BatchIngestPayload = {}) {
);
}
/** Resolve `[[wikilink]]` targets to note ids / vault pages. */
export async function resolveWikilinks(targets: string[]) {
const res = await parseResponse<{ resolved: WikilinkResolutionMap }>(
await apiFetch(`${baseUrl()}/resolve`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targets }),
}),
);
return res.resolved;
}
/** Resolve every `[[wikilink]]` inside a note's body in one call. */
export async function resolveNoteWikilinks(noteId: string) {
const res = await parseResponse<{ resolved: WikilinkResolutionMap }>(
await apiFetch(
`${baseUrl()}/notes/${encodeURIComponent(noteId)}/wikilinks`,
),
);
return res.resolved;
}
/** Fetch a raw vault meta/hub Markdown page (no DB note) by its path. */
export async function fetchVaultPage(path: string) {
return parseResponse<VaultPage>(
await apiFetch(`${baseUrl()}/vault-page?path=${encodeURIComponent(path)}`),
);
}
export async function exportKnowledgeGraph() {
return parseResponse<GraphExportResult>(
await apiFetch(`${baseUrl()}/export/graph`, { method: "POST" }),
+20
View File
@@ -9,12 +9,14 @@ import {
createKnowledgeNote,
deleteKnowledgeNote,
exportKnowledgeGraph,
fetchVaultPage,
getKnowledgeNote,
ingestThread,
ingestThreadsBatch,
listDuplicates,
listKnowledgeNotes,
mergeKnowledgeNotes,
resolveNoteWikilinks,
searchKnowledge,
updateKnowledgeNote,
} from "./api";
@@ -40,6 +42,24 @@ export function useKnowledgeNote(noteId: string | undefined) {
});
}
export function useNoteWikilinks(noteId: string | undefined) {
return useQuery({
queryKey: [...ROOT_KEY, "wikilinks", noteId],
queryFn: () => resolveNoteWikilinks(noteId as string),
enabled: !!noteId,
refetchOnWindowFocus: false,
});
}
export function useVaultPage(path: string | undefined) {
return useQuery({
queryKey: [...ROOT_KEY, "vault-page", path],
queryFn: () => fetchVaultPage(path as string),
enabled: !!path,
refetchOnWindowFocus: false,
});
}
export function useKnowledgeSearch(query: string, enabled: boolean) {
return useQuery({
queryKey: [...ROOT_KEY, "search", query],
+21
View File
@@ -52,6 +52,27 @@ export interface KnowledgeListParams {
offset?: number;
}
export type WikilinkKind = "note" | "vault" | "missing";
export interface WikilinkResolution {
target: string;
normalized?: string;
kind: WikilinkKind;
exists: boolean;
note_id?: string;
vault_path?: string;
title?: string;
}
/** Map of original `[[target]]` string → its resolution. */
export type WikilinkResolutionMap = Record<string, WikilinkResolution>;
export interface VaultPage {
path: string;
title: string;
content_md: string;
}
export interface KnowledgeSearchItem {
note_id: string;
chunk_id?: string | null;
+63 -2
View File
@@ -41,10 +41,14 @@ import {
useKnowledgeDuplicates,
useKnowledgeNote,
useKnowledgeNotes,
useNoteWikilinks,
useVaultPage,
} from "@/core/knowledge/hooks";
import type { KnowledgeNote } from "@/core/knowledge/types";
import { cn } from "@/lib/utils";
import MarkdownRenderer from "@/strategy-components/MarkdownRenderer";
import MarkdownRenderer, {
type WikiLinkTarget,
} from "@/strategy-components/MarkdownRenderer";
const SOURCE_TYPE_LABELS: Record<string, string> = {
thread: "对话",
@@ -349,10 +353,56 @@ function DedupDialog({
);
}
function VaultPageDialog({
path,
onOpenChange,
}: {
path: string | null;
onOpenChange: (open: boolean) => void;
}) {
const { data: page, isLoading } = useVaultPage(path ?? undefined);
return (
<Dialog open={!!path} onOpenChange={onOpenChange}>
<DialogContent className="max-w-2xl">
<DialogHeader>
<DialogTitle>{page?.title || "知识库页面"}</DialogTitle>
<DialogDescription>{path}</DialogDescription>
</DialogHeader>
<ScrollArea className="max-h-[60vh]">
<div className="p-1">
{isLoading ? (
<div className="text-muted-foreground p-4 text-sm">加载中...</div>
) : page ? (
<MarkdownRenderer content={page.content_md} />
) : (
<div className="text-muted-foreground p-4 text-sm">页面不存在</div>
)}
</div>
</ScrollArea>
<DialogFooter>
<Button variant="outline" onClick={() => onOpenChange(false)}>
关闭
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
);
}
function NoteDetail({ noteId }: { noteId: string }) {
const navigate = useNavigate();
const { data: note, isLoading } = useKnowledgeNote(noteId);
const { data: wikiLinks } = useNoteWikilinks(noteId);
const { update, remove } = useKnowledgeActions();
const [vaultPath, setVaultPath] = useState<string | null>(null);
const handleWikiLink = (link: WikiLinkTarget) => {
if (link.kind === "note" && link.note_id) {
navigate(`/page/workspace/knowledge/${link.note_id}`);
} else if (link.kind === "vault" && link.vault_path) {
setVaultPath(link.vault_path);
}
};
const [editing, setEditing] = useState(false);
const [draftTitle, setDraftTitle] = useState("");
const [draftTags, setDraftTags] = useState("");
@@ -361,6 +411,7 @@ function NoteDetail({ noteId }: { noteId: string }) {
useEffect(() => {
setEditing(false);
setVaultPath(null);
}, [noteId]);
useEffect(() => {
@@ -495,7 +546,11 @@ function NoteDetail({ noteId }: { noteId: string }) {
</span>
))}
</div>
<MarkdownRenderer content={note.content_md} />
<MarkdownRenderer
content={note.content_md}
wikiLinks={wikiLinks}
onWikiLinkClick={handleWikiLink}
/>
{note.sources && note.sources.length > 0 ? (
<div className="mt-6 border-t pt-4">
@@ -545,6 +600,12 @@ function NoteDetail({ noteId }: { noteId: string }) {
)}
</div>
</ScrollArea>
<VaultPageDialog
path={vaultPath}
onOpenChange={(open) => {
if (!open) setVaultPath(null);
}}
/>
</div>
);
}
@@ -16,12 +16,33 @@ import tsx from 'react-syntax-highlighter/dist/esm/languages/prism/tsx';
import typescript from 'react-syntax-highlighter/dist/esm/languages/prism/typescript';
import { oneDark } from 'react-syntax-highlighter/dist/esm/styles/prism';
export interface WikiLinkTarget {
target: string;
kind: 'note' | 'vault' | 'missing';
exists: boolean;
note_id?: string;
vault_path?: string;
title?: string;
}
export type WikiLinkMap = Record<string, WikiLinkTarget>;
interface MarkdownRendererProps {
content: string;
className?: string;
onCitationClick?: (index: number) => void;
/** Resolved `[[wikilink]]` targets; when provided, non-numeric `[[...]]` render as links. */
wikiLinks?: WikiLinkMap;
onWikiLinkClick?: (link: WikiLinkTarget) => void;
}
const escapeHtmlAttr = (value: string): string =>
value
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;');
SyntaxHighlighter.registerLanguage('bash', bash);
SyntaxHighlighter.registerLanguage('css', css);
SyntaxHighlighter.registerLanguage('html', markup);
@@ -126,8 +147,35 @@ const CodeBlock: React.FC<{ children: React.ReactNode; className?: string }> = (
);
};
const MarkdownRenderer: React.FC<MarkdownRendererProps> = ({ content, className = '', onCitationClick }) => {
const contentWithCitationBadge = content.replace(
const MarkdownRenderer: React.FC<MarkdownRendererProps> = ({
content,
className = '',
onCitationClick,
wikiLinks,
onWikiLinkClick,
}) => {
// 1) Non-numeric [[wikilinks]] → clickable (or greyed-out) links, driven by the
// resolver map. Numeric [[123]] are left untouched here for the citation pass.
let processed = content;
if (wikiLinks) {
processed = processed.replace(
/\[\[([^\]\n|]+)(?:\|([^\]\n]+))?\]\]/g,
(match, rawTarget: string, alias?: string) => {
const target = String(rawTarget).trim();
if (/^\d+$/.test(target)) return match; // citation badge, handled below
const resolved = wikiLinks[target];
const label = (alias?.trim() || resolved?.title || target);
const safeLabel = escapeHtmlAttr(label);
const safeTarget = escapeHtmlAttr(target);
if (resolved && resolved.exists) {
return `<button type="button" data-wikilink-target="${safeTarget}" class="inline items-center font-medium text-blue-600 dark:text-blue-400 hover:underline cursor-pointer align-baseline">${safeLabel}</button>`;
}
return `<span data-wikilink-missing="${safeTarget}" title="页面不存在" class="text-muted-foreground underline decoration-dashed decoration-from-font cursor-not-allowed">${safeLabel}</span>`;
},
);
}
// 2) Numeric [[123]] citation badges.
const contentWithCitationBadge = processed.replace(
/\[\[(\d+)\]\]/g,
'<button type="button" data-citation-index="$1" class="inline-flex items-center justify-center min-w-[1.5rem] h-6 px-2 mx-0.5 rounded-md border text-xs font-semibold align-middle bg-red-50 text-red-600 border-red-200 dark:bg-slate-700 dark:text-slate-100 dark:border-slate-500 cursor-pointer transition-all hover:opacity-85">[$1]</button>',
);
@@ -136,8 +184,17 @@ const MarkdownRenderer: React.FC<MarkdownRendererProps> = ({ content, className
<div
className={`markdown-content ${className}`}
onClick={(event) => {
if (!onCitationClick) return;
const target = event.target as HTMLElement;
const wikiButton = target.closest('[data-wikilink-target]') as HTMLElement | null;
if (wikiButton && onWikiLinkClick && wikiLinks) {
const key = wikiButton.dataset.wikilinkTarget || '';
const link = wikiLinks[key];
if (link) {
onWikiLinkClick(link);
return;
}
}
if (!onCitationClick) return;
const button = target.closest('[data-citation-index]') as HTMLElement | null;
if (!button) return;
const index = Number(button.dataset.citationIndex || '0');
+6 -1
View File
@@ -250,7 +250,7 @@ FastAPI application on port 8001 with health check at `GET /health`. Set `GATEWA
| **Scheduled Tasks** (`/api/scheduled-tasks`) | Per-user cron-like agent tasks. `GET/POST /` - list/create owned tasks; `PATCH/DELETE /{id}` - update/delete (owner); `POST /{id}/{pause,resume,trigger}`; `GET /{id}/runs`, `GET /runs/{rid}`, run-file preview/edit/download; `GET /runs/{rid}/messages` - agent messages for the run's underlying agent run (used to inspect a stuck/running run; authorized via `_load_run`); `GET /runs/{rid}/html` - generated HTML document for an `html_page` run; `DELETE /runs/{rid}` - owner-scoped, with an admin fallback so admins can delete any user's run; publish/subscribe under `/{id}/{publish,unpublish,subscribe}` and `GET /published`. **Admin (cross-user):** `GET /admin/all` - every task; `PATCH/DELETE /admin/{id}` - edit/delete any task; `POST /admin/{id}/trigger`; `GET /admin/{id}/runs`. Admin routes require `system_role == "admin"`. |
| **HTML Page Favorites** (`/api/html-page-favorites`) | Bookmarked generated HTML pages, owner-scoped. `GET /` - list; `POST /` `{page_id, title?, description?, tags?}` - bookmark a `scheduled_html_pages` row (extracts a reusable style/layout summary via GLM5 at favorite time); a page can only be favorited once per user (`DuplicateFavoriteError` → HTTP 409). `GET/DELETE /{id}` - read/delete (the create-task reference picker deletes favorites here). A favorite can be selected as a style reference when creating a new `html_page` task. |
| **Public Scheduled Tasks** (`/api/public/scheduled-tasks`) | **Unauthenticated** mirror for embedding. Adds `GET /runs/{rid}/html` and `GET /{task_id}/latest-html` for the `html_page` preview; the run result keeps `result.html` (only file paths are redacted). |
| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup); `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector→keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe). Backed by the `deerflow.knowledge` module (see below). |
| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup); `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector→keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[…]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). Backed by the `deerflow.knowledge` module (see below). |
Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → Gateway.
@@ -284,6 +284,11 @@ Self-contained product knowledge base. **All knowledge capability lives in this
**Module layout:** `models.py` (ORM: notes/sources/tags/note_tags + embeddings/versions/entities/relations) · `repository.py` (DB only, no LLM) · `schemas.py` (NoteDraft/SourceDraft DTOs) · `skill_templates.py` · `markdown_vault.py` · `manifest.py` · `extractor.py` (rule-based fallback) · `llm_extractor.py` (**default**: distills declarative knowledge per wiki-capture, structured JSON incl. entities/relations; falls back to `extractor.py`) · `sources.py` · `exporter.py` (`graph.json`/`graph.graphml`/`cypher.txt` + self-contained vanilla-JS `graph.html`; entity nodes + relation edges) · `chunking.py` + `embeddings.py` (OpenAI-compatible embedding client, keyword fallback) + `search.py` (keyword/vector/hybrid) · `sensitive.py` (secret/PII redaction) · `dedup.py` (Jaccard + hash dedup) · `auto_ingest.py` (background queue/worker) · `obsidian_adapter.py` · `service.py` (`KnowledgeService` orchestrator — the only thing the router touches). Tests: `tests/test_knowledge.py` (20 cases). Frontend: `frontend-web/src/core/knowledge/*` + `pages/KnowledgeBasePage.tsx` / `KnowledgeGraphPage.tsx` + chat `knowledge-save-trigger.tsx` / `knowledge-rag-toggle.tsx`; spec at `frontend-web/docs/product-knowledge-base-dev.md`.
**Wikilinks & multi-note capture** (`knowledge.{related_notes_enabled,related_notes_limit,multi_note_enabled}`, all default-on):
- **Navigable `[[wikilinks]]`** — `service.resolve_wikilink(s)` maps a `[[target]]` to a knowledge note (by id → `vault_path` → title) or a vault meta/hub page (`read_vault_page`, path-traversal-proof), else `missing`. `markdown_vault.normalize_wikilink_target` canonicalizes targets (drops `|alias`, slashes, `.md`). The frontend `MarkdownRenderer` renders non-numeric `[[...]]` as clickable links from the resolver map (numeric `[[123]]` stay citation badges); note links navigate, vault links open a `VaultPageDialog`, missing links are greyed. The vault scaffold now materializes `projects/<project>/synthesis/product-knowledge-base.md` (the hub page every note's `## Related` links to) so the three historically-hardcoded structural links (`[[index]]`, `[[_meta/taxonomy]]`, the hub) always resolve.
- **Real related notes** — on capture, `service._find_related_notes` runs keyword/hybrid search over the draft's title + summary + *distinctive* tags (project/`knowledge-base` defaults excluded) and injects real `[[Title]]` cross-links at the top of `## Related` (plus note→note `related` graph edges), replacing the placeholder-only block.
- **Multi-note split** — `llm_extractor.extract_thread_llm_multi` lets one conversation distill into 1–4 cross-linked notes (`notes:[…]` JSON array; legacy single-object output still accepted). `capture_thread` persists each, sibling-cross-links them by title, and returns `{note, created, notes:[…]}`. Gated by `multi_note_enabled` (false → first note only). Tests: `tests/test_knowledge_wikilinks.py`.
**Phases 2–4 (all config-gated, defaults safe):**
- **Phase 2 — vector RAG**: `knowledge.embedding.{enabled,base_url,api_key,model,...}` enables an OpenAI-compatible embedding endpoint; notes are chunked + embedded on write (`knowledge_embeddings`), `POST /search` supports `mode=keyword|vector|hybrid` (vector degrades to keyword when unconfigured). `KnowledgeRagMiddleware` (lead-agent chain) injects a transient `<knowledge-context>` recall before the model call, gated **per-conversation** by the `knowledge_rag_enabled` runtime flag (frontend chat toggle) falling back to global `knowledge.rag_enabled`. `POST /reindex` rebuilds the index.
- **Phase 3 — governance**: `sensitive.py` redacts secrets/PII before persistence (`sensitive_redaction_enabled`); every update snapshots `knowledge_note_versions` (`GET /notes/{id}/versions`); `GET /duplicates` + `POST /merge` for dedup; background `AutoIngestQueue` (wired in `deps.py`, started/stopped in the gateway lifespan) auto-sediments finished conversations **off the chat path** when `auto_ingest_enabled` — enqueued from `KnowledgeRagMiddleware.after_agent`, value-judged by min-chars + `skip_existing`, landing as `draft`.
@@ -95,6 +95,10 @@ class SearchRequest(BaseModel):
limit: int = Field(default=8, ge=1, le=50)
class ResolveWikilinksRequest(BaseModel):
targets: list[str] = Field(default_factory=list, description="`[[wikilink]]` targets to resolve to note ids / vault pages.")
# --------------------------------------------------------------------------- #
# Helpers
# --------------------------------------------------------------------------- #
@@ -198,7 +202,7 @@ async def ingest_thread(thread_id: str, body: IngestThreadRequest, request: Requ
)
except EmptyThreadError:
raise HTTPException(status_code=400, detail="Thread has no sediment-able content")
return {"note": result["note"], "created": result["created"]}
return {"note": result["note"], "created": result["created"], "notes": result.get("notes", [result["note"]])}
@router.post("/ingest/threads/batch")
@@ -356,6 +360,44 @@ async def search(body: SearchRequest, request: Request) -> dict[str, Any]:
return {"items": items}
# --------------------------------------------------------------------------- #
# Wikilinks ([[...]] resolution + vault meta-page viewer)
# --------------------------------------------------------------------------- #
@router.post("/resolve")
@require_auth
async def resolve_wikilinks(body: ResolveWikilinksRequest, request: Request) -> dict[str, Any]:
"""Resolve ``[[wikilink]]`` targets to navigable destinations.
Each target resolves to a knowledge note (``kind=note``), a vault meta/hub
page (``kind=vault``), or nothing (``kind=missing``). Powers clickable
wikilinks in the knowledge UI.
"""
service = get_knowledge_service(request)
return {"resolved": await service.resolve_wikilinks(body.targets)}
@router.get("/notes/{note_id}/wikilinks")
@require_auth
async def resolve_note_wikilinks(note_id: str, request: Request) -> dict[str, Any]:
"""Resolve every ``[[wikilink]]`` found in a note's body in one call."""
service = get_knowledge_service(request)
resolved = await service.resolve_note_wikilinks(note_id)
if resolved is None:
raise HTTPException(status_code=404, detail="Knowledge note not found")
return {"resolved": resolved}
@router.get("/vault-page")
@require_auth
async def get_vault_page(request: Request, path: str) -> dict[str, Any]:
"""Return a raw vault Markdown meta/hub page (no DB note) by its path."""
service = get_knowledge_service(request)
page = service.read_vault_page(path)
if page is None:
raise HTTPException(status_code=404, detail="Vault page not found")
return page
class MergeRequest(BaseModel):
primary_id: str
duplicate_ids: list[str] = Field(default_factory=list)
@@ -62,6 +62,9 @@ class KnowledgeConfig(BaseModel):
extract_max_input_chars: int = Field(default=12000, description="Max characters of conversation transcript fed to the extraction LLM.")
max_thread_messages: int = Field(default=80, description="Maximum thread messages to scan when capturing.")
max_source_items: int = Field(default=20, description="Maximum source items recorded per note.")
related_notes_enabled: bool = Field(default=True, description="On capture, find existing notes on the same topic and link them under ## Related (real cross-references instead of placeholders).")
related_notes_limit: int = Field(default=3, description="Maximum real related notes linked into each captured note.")
multi_note_enabled: bool = Field(default=True, description="Allow the LLM extractor to split one conversation into multiple cross-linked knowledge notes.")
rag_enabled: bool = Field(default=False, description="Enable RAG injection before agent runs (phase 2).")
rag_limit: int = Field(default=5, description="Maximum knowledge snippets injected per query.")
sensitive_redaction_enabled: bool = Field(default=True, description="Redact secrets/PII (tokens, keys, phone, id-card, email) before persisting notes (phase 3).")
@@ -105,10 +105,18 @@ def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str:
def related_links(project_name: str | None) -> list[str]:
"""At least two wikilinks per note (wiki-capture requirement)."""
"""Structural wikilinks every note carries (wiki-capture requirement).
These point at vault meta/hub pages (``index`` / ``_meta/taxonomy`` / the
project hub) that always exist, so they resolve to navigable pages. Content-
specific "related notes" are injected separately by the service layer (which
has DB access) on top of these.
"""
from deerflow.knowledge.markdown_vault import slugify
links = ["[[index]]", "[[_meta/taxonomy]]"]
if project_name:
links.append(f"[[projects/{project_name}/synthesis/product-knowledge-base]]")
links.append(f"[[projects/{slugify(project_name)}/synthesis/product-knowledge-base]]")
return links
@@ -40,21 +40,30 @@ _SYSTEM_PROMPT = """\
references(外部来源汇总)、decision(架构/设计决策)、journal(会话纪要)。
- 用中文输出。
一次对话可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
- 默认只产出 **1 页**;只有当对话明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
- 每一页聚焦单一主题,标题与 summary 必须彼此区分,不要把同一结论拆到多页。
- 不值得沉淀时返回空的 notes 数组。
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构如下:
{
"should_ingest": true,
"reason": "为什么值得/不值得沉淀",
"title": "简洁的知识标题(不超过 40 字)",
"summary": "1-2 句话概括这页知识",
"category": "synthesis",
"context": "问题背景或触发场景,1-3 句",
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
"reasoning": "为什么成立、取舍、不确定性",
"implications": ["后续建议/风险/下一步", "..."],
"tags": ["标签1", "标签2"],
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
"confidence": 0.0
"notes": [
{
"title": "简洁的知识标题(不超过 40 字)",
"summary": "1-2 句话概括这页知识",
"category": "synthesis",
"context": "问题背景或触发场景,1-3 句",
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
"reasoning": "为什么成立、取舍、不确定性",
"implications": ["后续建议/风险/下一步", "..."],
"tags": ["标签1", "标签2"],
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
"confidence": 0.0
}
]
}"""
@@ -128,7 +137,78 @@ def _render_body(data: dict[str, Any], title: str, related: list[str], sources:
)
async def extract_thread_llm(
def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]:
"""Normalize the LLM JSON into a list of per-note dicts.
Accepts the multi-note shape ``{"notes": [...]}`` as well as the legacy
single-note shape (``{"title": ..., "findings": ...}``) for backward
compatibility. Notes without any title/findings are dropped.
"""
raw_notes = data.get("notes")
if isinstance(raw_notes, list):
candidates = [n for n in raw_notes if isinstance(n, dict)]
elif data.get("title") or data.get("findings") or data.get("context"):
candidates = [data] # legacy single-object output
else:
candidates = []
return [n for n in candidates if str(n.get("title") or "").strip() or n.get("findings")]
def _build_note_draft(
note_data: dict[str, Any],
*,
thread_id: str,
thread_title: str | None,
turns: list[dict[str, str]],
title_override: str | None,
tags: list[str] | None,
project_name: str | None,
sources: list[SourceDraft],
source_key_suffix: str = "",
) -> NoteDraft:
"""Render one note dict into a :class:`NoteDraft`."""
category = str(note_data.get("category") or "synthesis").strip().lower()
if category not in _VALID_CATEGORIES:
category = "synthesis"
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or extractor._auto_title(thread_title, turns)
summary = str(note_data.get("summary") or "").strip()
related = extractor.related_links(project_name)
body = _render_body(note_data, final_title, related, sources)
base_tags = list(tags or [])
llm_tags = note_data.get("tags") if isinstance(note_data.get("tags"), list) else []
for tag in [*llm_tags, project_name or "deerflow", "knowledge-base"]:
tag = str(tag).strip()
if tag and tag not in base_tags:
base_tags.append(tag)
try:
confidence = float(note_data.get("confidence") or 0.7)
except (TypeError, ValueError):
confidence = 0.7
entities = note_data.get("entities") if isinstance(note_data.get("entities"), list) else []
relations = note_data.get("relations") if isinstance(note_data.get("relations"), list) else []
return NoteDraft(
title=final_title,
summary=summary or final_title,
content_md=body,
category=category,
source_type="thread",
source_id=thread_id,
tags=base_tags,
confidence=max(0.0, min(confidence, 1.0)),
related=related,
sources=sources,
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
source_key=f"thread:{thread_id}{source_key_suffix}",
)
async def extract_thread_llm_multi(
messages: list[dict[str, Any]],
*,
thread_id: str,
@@ -141,15 +221,19 @@ async def extract_thread_llm(
max_messages: int = 80,
model_name: str | None = None,
max_input_chars: int = 12000,
) -> NoteDraft | None:
"""Distill a thread into a :class:`NoteDraft` via the LLM.
allow_multi: bool = True,
max_notes: int = 4,
) -> list[NoteDraft]:
"""Distill a thread into one *or several* cross-linkable knowledge notes.
Returns ``None`` when there is nothing to sediment **or** the LLM path
fails — callers should fall back to the rule-based extractor.
The LLM may split a conversation that covers multiple independent topics into
several declarative notes. Returns ``[]`` when there is nothing to sediment
or the LLM path fails — callers should fall back to the rule-based extractor.
When ``allow_multi`` is false only the first note is kept.
"""
turns = extractor.split_turns(messages, max_messages=max_messages)
if not turns:
return None
return []
sources = sources or []
try:
@@ -168,47 +252,72 @@ async def extract_thread_llm(
data = _parse_json(raw)
except Exception:
logger.exception("Knowledge LLM extraction failed for thread %s; using rule-based fallback", thread_id)
return None
return []
if not data:
return None
return []
category = str(data.get("category") or "synthesis").strip().lower()
if category not in _VALID_CATEGORIES:
category = "synthesis"
note_dicts = _notes_from_data(data)
if not note_dicts:
return []
if not allow_multi:
note_dicts = note_dicts[:1]
else:
note_dicts = note_dicts[: max(1, max_notes)]
final_title = (title or "").strip() or str(data.get("title") or "").strip() or extractor._auto_title(thread_title, turns)
summary = str(data.get("summary") or "").strip()
related = extractor.related_links(project_name)
body = _render_body(data, final_title, related, sources)
drafts: list[NoteDraft] = []
for i, nd in enumerate(note_dicts):
# The user-supplied title override only applies to a single-note result;
# multi-note splits keep each note's own LLM title.
override = title if (len(note_dicts) == 1) else None
suffix = "" if len(note_dicts) == 1 else f"#{i}"
drafts.append(
_build_note_draft(
nd,
thread_id=thread_id,
thread_title=thread_title,
turns=turns,
title_override=override,
tags=tags,
project_name=project_name,
sources=sources,
source_key_suffix=suffix,
)
)
return drafts
base_tags = list(tags or [])
llm_tags = data.get("tags") if isinstance(data.get("tags"), list) else []
for tag in [*llm_tags, project_name or "deerflow", "knowledge-base"]:
tag = str(tag).strip()
if tag and tag not in base_tags:
base_tags.append(tag)
try:
confidence = float(data.get("confidence") or 0.7)
except (TypeError, ValueError):
confidence = 0.7
async def extract_thread_llm(
messages: list[dict[str, Any]],
*,
thread_id: str,
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "deerflow",
sources: list[SourceDraft] | None = None,
max_messages: int = 80,
model_name: str | None = None,
max_input_chars: int = 12000,
) -> NoteDraft | None:
"""Distill a thread into a single :class:`NoteDraft` (backward-compatible).
entities = data.get("entities") if isinstance(data.get("entities"), list) else []
relations = data.get("relations") if isinstance(data.get("relations"), list) else []
return NoteDraft(
title=final_title,
summary=summary or final_title,
content_md=body,
category=category,
source_type="thread",
source_id=thread_id,
tags=base_tags,
confidence=max(0.0, min(confidence, 1.0)),
related=related,
Thin wrapper over :func:`extract_thread_llm_multi` that returns the first
note, or ``None`` when nothing was produced.
"""
drafts = await extract_thread_llm_multi(
messages,
thread_id=thread_id,
thread_title=thread_title,
mode=mode,
title=title,
tags=tags,
project_name=project_name,
sources=sources,
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
source_key=f"thread:{thread_id}",
max_messages=max_messages,
model_name=model_name,
max_input_chars=max_input_chars,
allow_multi=False,
)
return drafts[0] if drafts else None
@@ -52,6 +52,23 @@ def extract_wikilinks(text: str) -> list[str]:
return seen
def normalize_wikilink_target(target: str) -> str:
"""Canonicalize a ``[[wikilink]]`` target for resolution/comparison.
Drops any ``|alias`` part, normalizes slashes, strips a leading ``/`` and a
trailing ``.md`` — so ``[[index]]``, ``index.md`` and ``/index`` all map to
the same canonical ``index`` key used by the index/graph/resolver layers.
"""
t = (target or "").strip()
if not t:
return ""
t = t.split("|", 1)[0].strip()
t = t.replace("\\", "/").lstrip("/")
if t.lower().endswith(".md"):
t = t[:-3]
return t
class VaultPathError(ValueError):
"""Raised when a requested vault path escapes the vault root."""
@@ -99,6 +116,21 @@ class MarkdownVault:
base = self.root / "projects" / slugify(project_name)
for d in ("concepts", "references", "synthesis", "journal"):
(base / d).mkdir(parents=True, exist_ok=True)
# Project hub page — the structural target every note links to under
# ``## Related`` (``[[projects/<project>/synthesis/product-knowledge-base]]``).
# Materialize it so that wikilink is always navigable, not a dangling
# placeholder.
hub = base / "synthesis" / "product-knowledge-base.md"
if not hub.exists():
hub.write_text(
f"---\ntitle: 产品知识库总览\ncategory: synthesis\n---\n\n"
f"# 产品知识库总览\n\n"
f"本页是 **{project_name}** 项目知识库的入口枢纽(hub)。每条知识都会在 `## Related` 中链接回这里。\n\n"
"- [[index]] — 全部页面索引\n"
"- [[_meta/taxonomy]] — 分类体系\n\n"
"## 主题\n\n_随着知识沉淀,这里会逐步链接到各主题页面。_\n",
encoding="utf-8",
)
# Base vault files.
index = self.root / "index.md"
if not index.exists():
@@ -216,6 +216,48 @@ class KnowledgeRepository:
d["sources"] = [self._source_to_dict(r) for r in (await session.execute(stmt)).scalars()]
return d
async def find_by_vault_path(self, vault_path: str) -> dict[str, Any] | None:
"""Return the first (non-archived) note whose vault_path matches.
``vault_path`` is matched both with and without the ``.md`` suffix, since
wikilink targets are stored without it while ``vault_path`` keeps it.
"""
stripped = vault_path[:-3] if vault_path.endswith(".md") else vault_path
candidates = {vault_path, stripped, f"{stripped}.md"}
async with self._sf() as session:
stmt = (
select(KnowledgeNoteRow)
.where(KnowledgeNoteRow.vault_path.in_(list(candidates)))
.where(KnowledgeNoteRow.status != "archived")
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
if row is None:
return None
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, row.id)
return d
async def find_by_title(self, title: str) -> dict[str, Any] | None:
"""Return the first (non-archived) note with a case-insensitive title match."""
name = (title or "").strip()
if not name:
return None
async with self._sf() as session:
stmt = (
select(KnowledgeNoteRow)
.where(func.lower(KnowledgeNoteRow.title) == name.lower())
.where(KnowledgeNoteRow.status != "archived")
.order_by(KnowledgeNoteRow.updated_at.desc())
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
if row is None:
return None
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, row.id)
return d
async def find_by_source(self, source_type: str, source_id: str) -> dict[str, Any] | None:
"""Return the first (non-archived) note for a (source_type, source_id)."""
async with self._sf() as session:
@@ -18,7 +18,7 @@ from deerflow.config.runtime_paths import runtime_home
from deerflow.knowledge import extractor, llm_extractor
from deerflow.knowledge.chunking import chunk_markdown
from deerflow.knowledge.embeddings import EmbeddingClient
from deerflow.knowledge.markdown_vault import MarkdownVault
from deerflow.knowledge.markdown_vault import MarkdownVault, VaultPathError, extract_wikilinks, normalize_wikilink_target
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
from deerflow.knowledge.repository import KnowledgeRepository
from deerflow.knowledge.schemas import NoteDraft, SourceDraft, normalize_source_type, normalize_status
@@ -39,6 +39,24 @@ def _resolve_vault_dir(vault_dir: str) -> Path:
return (runtime_home() / vault_dir).resolve()
def _inject_related_notes(body: str, links: list[str]) -> str:
"""Insert real related-note ``[[wikilinks]]`` at the top of the Related block.
Keeps the structural meta-links (index / taxonomy / hub) that the extractor
already rendered, but surfaces the content-specific related notes first. When
there is no ``## Related`` section, one is appended.
"""
if not links:
return body
bullets = "\n".join(f"- {link}" for link in links)
marker = "## Related\n"
idx = body.find(marker)
if idx == -1:
return body.rstrip() + "\n\n## Related\n\n" + bullets + "\n"
insert_at = idx + len(marker)
return body[:insert_at] + "\n" + bullets + "\n" + body[insert_at:]
class KnowledgeService:
"""Application-facing knowledge base API."""
@@ -75,9 +93,56 @@ class KnowledgeService:
logger.info("Knowledge redaction applied to note '%s': %s", draft.title, ",".join(sorted(set(body_hits))))
return draft
async def _find_related_notes(self, draft: NoteDraft) -> list[dict[str, Any]]:
"""Find existing notes on the same topic to cross-link under ## Related.
Uses the existing keyword/hybrid search over the title (+ summary). The
draft's own note does not exist yet, so we exclude same-title hits to
avoid self-links when re-capturing or deduping. Best-effort: any failure
yields no links rather than blocking the capture.
"""
if not self.config.related_notes_enabled:
return []
# Build a topical query from the title + summary + the note's *distinctive*
# tags (the per-note project/knowledge-base defaults are dropped, else
# every note would match every other). CJK runs tokenize as a single
# token, so shared distinctive tags are what make cross-topic matches work.
default_tags = {t.lower() for t in (self.project_name or "", "knowledge-base", "deerflow") if t}
distinctive = [t for t in (draft.tags or []) if t and t.lower() not in default_tags]
query = " ".join(p for p in [draft.title or "", draft.summary or "", *distinctive] if p).strip()
if not query:
return []
limit = max(1, self.config.related_notes_limit)
mode = "hybrid" if self.embedding_client.is_enabled() else "keyword"
try:
items = await search_notes(self.repo, query, mode=mode, limit=limit + 3, embedding_client=self.embedding_client)
except Exception:
logger.exception("Failed to find related notes for '%s'", draft.title)
return []
own_title = (draft.title or "").strip().lower()
out: list[dict[str, Any]] = []
seen: set[str] = set()
for it in items:
nid = it.get("note_id")
title = (it.get("title") or "").strip()
if not nid or nid in seen or not title:
continue
if title.lower() == own_title: # the draft itself / a duplicate
continue
seen.add(nid)
out.append({"id": nid, "title": title})
if len(out) >= limit:
break
return out
async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]:
"""Create a DB row, write the vault file, attach sources, return note."""
draft = self._redact_draft(draft)
related_notes = await self._find_related_notes(draft)
if related_notes:
real_links = [f"[[{n['title']}]]" for n in related_notes]
draft.content_md = _inject_related_notes(draft.content_md, real_links)
draft.related = list(dict.fromkeys([*real_links, *draft.related]))
note = await self.repo.create_note(
title=draft.title,
summary=draft.summary,
@@ -100,6 +165,13 @@ class KnowledgeService:
await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
await self._reindex_embeddings(note_id, draft.content_md)
await self._store_entities(note_id, draft)
# Record note→note "related" edges (after _store_entities, which clears
# this note's relations) so the graph reflects the cross-links too.
for rel_note in related_notes:
try:
await self.repo.add_relation(from_note_id=note_id, to_note_id=rel_note["id"], relation_type="related")
except Exception:
logger.exception("Failed to link related note %s -> %s", note_id, rel_note["id"])
return await self.repo.get_note(note_id) or note
async def _store_entities(self, note_id: str, draft: NoteDraft) -> None:
@@ -173,12 +245,13 @@ class KnowledgeService:
if include_sources and reference_batches:
sources = from_reference_batches(reference_batches, thread_id=thread_id, max_items=self.config.max_source_items)
draft: NoteDraft | None = None
drafts: list[NoteDraft] = []
# Prefer LLM distillation (wiki-capture style) so notes are declarative
# knowledge, not raw chat. Falls back to the rule-based extractor when
# the LLM is disabled/unavailable or returns something unusable.
# knowledge, not raw chat. A single conversation may split into several
# cross-linked notes. Falls back to the rule-based extractor (always one
# note) when the LLM is disabled/unavailable or returns nothing usable.
if self.config.llm_extract_enabled:
draft = await llm_extractor.extract_thread_llm(
drafts = await llm_extractor.extract_thread_llm_multi(
messages,
thread_id=thread_id,
thread_title=thread_title,
@@ -190,9 +263,10 @@ class KnowledgeService:
max_messages=self.config.max_thread_messages,
model_name=self.config.extract_model_name,
max_input_chars=self.config.extract_max_input_chars,
allow_multi=self.config.multi_note_enabled,
)
if draft is None:
draft = extractor.extract_thread(
if not drafts:
single = extractor.extract_thread(
messages,
thread_id=thread_id,
thread_title=thread_title,
@@ -203,11 +277,24 @@ class KnowledgeService:
sources=sources,
max_messages=self.config.max_thread_messages,
)
if draft is None:
drafts = [single] if single else []
if not drafts:
raise EmptyThreadError(f"Thread {thread_id} has no sediment-able content")
draft.status = normalize_status(status)
note = await self._persist_draft(draft, created_by=created_by)
return {"note": note, "created": True}
normalized = normalize_status(status)
for draft in drafts:
draft.status = normalized
# Cross-link sibling notes from the same conversation by title, so the
# split pages reference each other (resolved once all are persisted).
if len(drafts) > 1:
for i, draft in enumerate(drafts):
siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
if siblings:
draft.content_md = _inject_related_notes(draft.content_md, siblings)
draft.related = list(dict.fromkeys([*siblings, *draft.related]))
notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
return {"note": notes[0], "created": True, "notes": notes}
async def capture_search(
self,
@@ -357,6 +444,84 @@ class KnowledgeService:
logger.exception("Failed to archive knowledge note %s in vault", note_id)
return True
# -- wikilinks --------------------------------------------------------
async def resolve_wikilink(self, target: str) -> dict[str, Any]:
"""Resolve a single ``[[wikilink]]`` target to a navigable destination.
Returns one of:
- ``{"kind": "note", "note_id", "title", "exists": True}`` — a knowledge note
- ``{"kind": "vault", "vault_path", "title", "exists": True}`` — a vault meta/hub page
- ``{"kind": "missing", "exists": False}`` — nothing matches
Resolution order: note id → note vault_path → note title → vault file.
"""
norm = normalize_wikilink_target(target)
base = {"target": target, "normalized": norm}
if not norm:
return {**base, "kind": "missing", "exists": False}
# 1) Direct note id (e.g. "kb_abc...").
if norm.startswith("kb_"):
note = await self.repo.get_note(norm, with_sources=False)
if note and note.get("status") != "archived":
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
# 2) Note by vault path (links from index.md / hot.md / structural hub).
note = await self.repo.find_by_vault_path(norm)
if note:
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
# 3) Note by title (the real related-note cross-links use [[Title]]).
note = await self.repo.find_by_title(norm) or await self.repo.find_by_title(norm.split("/")[-1])
if note:
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
# 4) A vault meta/hub page that has no DB note (index, _meta/taxonomy, hub…).
page = self.read_vault_page(norm)
if page is not None:
return {**base, "kind": "vault", "vault_path": page["path"], "title": page["title"], "exists": True}
return {**base, "kind": "missing", "exists": False}
async def resolve_wikilinks(self, targets: list[str]) -> dict[str, dict[str, Any]]:
"""Resolve many targets at once; returns ``{original_target: resolution}``."""
out: dict[str, dict[str, Any]] = {}
for target in dict.fromkeys(t for t in targets if t and t.strip()):
out[target] = await self.resolve_wikilink(target)
return out
async def resolve_note_wikilinks(self, note_id: str) -> dict[str, dict[str, Any]] | None:
"""Extract every ``[[wikilink]]`` in a note's body and resolve them.
Returns ``{target: resolution}`` or ``None`` when the note is missing.
Numeric ``[[123]]`` citation badges are skipped (handled by the renderer).
"""
note = await self.repo.get_note(note_id, with_sources=False)
if note is None:
return None
targets = [t for t in extract_wikilinks(note.get("content_md") or "") if not t.isdigit()]
return await self.resolve_wikilinks(targets)
def read_vault_page(self, path: str) -> dict[str, Any] | None:
"""Read a raw vault Markdown page (meta/hub pages without a DB note).
Returns ``{"path", "title", "content_md"}`` or ``None`` if the page does
not exist or the path escapes the vault root.
"""
norm = normalize_wikilink_target(path)
if not norm:
return None
rel = f"{norm}.md"
try:
parsed = self.adapter.vault.read_note(rel)
except VaultPathError:
return None
if parsed is None:
return None
fm, body = parsed
title = str(fm.get("title") or "").strip() or norm.split("/")[-1]
return {"path": norm, "title": title, "content_md": body}
# -- search & graph ---------------------------------------------------
async def search(self, query: str, *, mode: str = "keyword", limit: int = 8) -> list[dict[str, Any]]:
return await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)