feat: 产品内知识库(obsidian-wiki 兼容 vault)

后端:新增自包含 deerflow.knowledge 模块(models/repository/schemas/
extractor/sources/markdown_vault/manifest/obsidian_adapter/exporter/
search/service),通过 deps.py 装配为 app.state.knowledge_service,
挂载 /api/knowledge 路由;config.yaml 增加 knowledge 配置段,
AppConfig 注册 KnowledgeConfig,ORM 注册 4 张 knowledge_* 表。

前端:新增知识库页面(KnowledgeBasePage/KnowledgeGraphPage)、
core/knowledge api/hooks/types、聊天侧 knowledge-save-trigger,
侧边栏增加「知识库」入口。

知识库为全局共享,不做 user_id 隔离,写入记录 created_by/updated_by
作为审计。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
AdministratorandClaude Opus 4.8 committed 2026-06-08 17:38:28 +08:00
1 parent 80d2db6315
commit cc99ebba81
28 files changed
+3590 -6

No files matched your search

@@ -0,0 +1,250 @@
"""Extract knowledge notes from threads / searches / tool results.
First-release strategy is rule-based (wiki-capture aligned, §11 of the dev doc):
keep high-value content (user questions, final assistant answers, cited
sources), drop noise (empty/greeting/intermediate reasoning), and write a
*declarative* note rather than a chat transcript. An LLM extractor can later
replace :func:`extract_thread` while keeping the same :class:`NoteDraft` output.
"""
from __future__ import annotations
import re
from typing import Any
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
# Greeting / low-value lines filtered from extracted turns.
_GREETING_RE = re.compile(r"^(你好|您好|hi|hello|hey|thanks?|谢谢|好的|ok|okay)[\s!!。.,,~]*$", re.IGNORECASE)
_WS_RE = re.compile(r"\s+")
def _one_line(text: str, limit: int = 300) -> str:
"""Collapse whitespace and clip ``text`` to a single line."""
collapsed = _WS_RE.sub(" ", text or "").strip()
return collapsed[:limit]
def message_text(msg: dict[str, Any]) -> str:
"""Return the plain-text content of a serialized LangChain message."""
content = msg.get("content")
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
if block.get("type") in (None, "text") and isinstance(block.get("text"), str):
parts.append(block["text"])
return "\n".join(p for p in parts if p).strip()
return ""
def _msg_type(msg: dict[str, Any]) -> str:
return str(msg.get("type") or msg.get("role") or "").lower()
def split_turns(messages: list[dict[str, Any]], *, max_messages: int = 80) -> list[dict[str, str]]:
"""Split messages into ``{user, ai}`` turns keyed by human messages.
For each turn we keep the last substantial AI message (with text body).
"""
msgs = messages[-max_messages:] if max_messages and len(messages) > max_messages else list(messages)
turns: list[dict[str, str]] = []
current: dict[str, str] | None = None
for msg in msgs:
mtype = _msg_type(msg)
text = message_text(msg)
if mtype == "human":
if current and (current.get("user") or current.get("ai")):
turns.append(current)
current = {"user": text, "ai": ""}
elif mtype == "ai" and text:
if current is None:
current = {"user": "", "ai": text}
else:
current["ai"] = text # last substantial AI message wins
if current and (current.get("user") or current.get("ai")):
turns.append(current)
# Drop turns that are pure greetings with no answer.
cleaned = []
for t in turns:
user = (t.get("user") or "").strip()
ai = (t.get("ai") or "").strip()
if not ai and _GREETING_RE.match(user):
continue
if not user and not ai:
continue
cleaned.append({"user": user, "ai": ai})
return cleaned
def _auto_title(thread_title: str | None, turns: list[dict[str, str]]) -> str:
if thread_title and thread_title.strip() and thread_title.strip().lower() != "untitled":
return thread_title.strip()[:200]
for t in turns:
if t.get("user"):
first = t["user"].splitlines()[0].strip()
return (first[:40] + ("…" if len(first) > 40 else "")) or "知识笔记"
return "知识笔记"
def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str:
"""Simple rule-based summary: the first answer, clipped."""
for t in turns:
if t.get("ai"):
text = re.sub(r"\s+", " ", t["ai"]).strip()
return text[:limit] + ("…" if len(text) > limit else "")
for t in turns:
if t.get("user"):
text = re.sub(r"\s+", " ", t["user"]).strip()
return text[:limit] + ("…" if len(text) > limit else "")
return ""
def related_links(project_name: str | None) -> list[str]:
"""At least two wikilinks per note (wiki-capture requirement)."""
links = ["[[index]]", "[[_meta/taxonomy]]"]
if project_name:
links.append(f"[[projects/{project_name}/synthesis/product-knowledge-base]]")
return links
# Backwards-compatible internal alias.
_related_links = related_links
def _build_body(
*,
title: str,
turns: list[dict[str, str]],
mode: str,
project_name: str | None,
sources: list[SourceDraft],
) -> str:
"""Render the wiki-capture 5-section declarative body."""
full = mode == "full"
answer_clip = 4000 if full else 800
context_lines = []
for t in turns:
if t.get("user"):
q = re.sub(r"\s+", " ", t["user"]).strip()
context_lines.append(f"- {q[:300]}")
context = "\n".join(context_lines[:8]) or "- 来自一次对话沉淀。^[inferred]"
finding_lines = []
for i, t in enumerate(turns, 1):
if t.get("ai"):
ans = t["ai"].strip()
ans = ans if full else (ans[:answer_clip] + ("…" if len(ans) > answer_clip else ""))
finding_lines.append(f"### 结论 {i}\n\n{ans}")
finding = "\n\n".join(finding_lines) or "_暂无可提炼的结论。_"
reasoning = "本页由对话自动沉淀而成,结论来自助手最终回答;过程性推理已省略。判断与归纳部分标记为 ^[inferred]。"
implications = "- 后续可在相关问答中参考本页知识。\n- 如内容过期或冲突,请编辑或归档。"
related = "\n".join(f"- {link}" for link in _related_links(project_name))
if sources:
related += "\n\n### Sources\n\n"
for s in sources[:20]:
label = s.title or s.url or "来源"
related += f"- {label}" + (f" — {s.url}" if s.url else "") + "\n"
return (
f"# {title}\n\n"
f"## Context\n\n{context}\n\n"
f"## Finding / Decision\n\n{finding}\n\n"
f"## Reasoning\n\n{reasoning}\n\n"
f"## Implications\n\n{implications}\n\n"
f"## Related\n\n{related}\n"
)
def extract_thread(
messages: list[dict[str, Any]],
*,
thread_id: str,
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "deerflow",
sources: list[SourceDraft] | None = None,
max_messages: int = 80,
) -> NoteDraft | None:
"""Build a :class:`NoteDraft` from thread messages, or ``None`` if empty."""
turns = split_turns(messages, max_messages=max_messages)
if not turns:
return None
final_title = (title or "").strip() or _auto_title(thread_title, turns)
summary = _summary(turns)
sources = sources or []
body = _build_body(title=final_title, turns=turns, mode=mode, project_name=project_name, sources=sources)
base_tags = list(tags or [])
for default in (project_name or "deerflow", "knowledge-base"):
if default and default not in base_tags:
base_tags.append(default)
return NoteDraft(
title=final_title,
summary=summary,
content_md=body,
category="synthesis",
source_type="thread",
source_id=thread_id,
tags=base_tags,
confidence=0.6,
related=_related_links(project_name),
sources=sources,
source_key=f"thread:{thread_id}",
)
def extract_search(
query: str,
sources: list[SourceDraft],
*,
thread_id: str | None = None,
project_name: str | None = "deerflow",
title: str | None = None,
tags: list[str] | None = None,
) -> NoteDraft:
"""Build a reference-category :class:`NoteDraft` from search/tool results."""
final_title = (title or "").strip() or f"检索:{query[:40]}"
body_lines = [f"# {final_title}\n", "## Context\n", f"- 检索关键词:{query}\n", "## Finding / Decision\n"]
for s in sources:
label = s.title or s.url or "来源"
line = f"- **{label}**"
if s.url:
line += f" — {s.url}"
if s.snippet:
line += "\n - " + _one_line(s.snippet)
body_lines.append(line)
body_lines += [
"\n## Reasoning\n",
"本页汇总外部检索来源,供后续问答引用。来源可信度需结合原文判断。^[ambiguous]\n",
"## Implications\n",
"- 引用前请核对原始来源。\n",
"## Related\n",
*[f"- {link}" for link in _related_links(project_name)],
]
base_tags = list(tags or [])
for default in (project_name or "deerflow", "knowledge-base", "reference"):
if default and default not in base_tags:
base_tags.append(default)
return NoteDraft(
title=final_title,
summary=f"检索“{query}”的来源汇总,共 {len(sources)} 条。",
content_md="\n".join(body_lines),
category="references",
source_type="search",
source_id=f"search:{query[:120]}",
tags=base_tags,
confidence=0.4,
related=_related_links(project_name),
sources=sources,
source_key=None,
)