feat: 产品内知识库(obsidian-wiki 兼容 vault)
后端:新增自包含 deerflow.knowledge 模块(models/repository/schemas/ extractor/sources/markdown_vault/manifest/obsidian_adapter/exporter/ search/service),通过 deps.py 装配为 app.state.knowledge_service, 挂载 /api/knowledge 路由;config.yaml 增加 knowledge 配置段, AppConfig 注册 KnowledgeConfig,ORM 注册 4 张 knowledge_* 表。 前端:新增知识库页面(KnowledgeBasePage/KnowledgeGraphPage)、 core/knowledge api/hooks/types、聊天侧 knowledge-save-trigger, 侧边栏增加「知识库」入口。 知识库为全局共享,不做 user_id 隔离,写入记录 created_by/updated_by 作为审计。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
80d2db6315
commit
cc99ebba81
28 files changed
+3590
-6
No files matched your search
@@ -0,0 +1,250 @@
|
||||
"""Extract knowledge notes from threads / searches / tool results.
|
||||
|
||||
First-release strategy is rule-based (wiki-capture aligned, §11 of the dev doc):
|
||||
keep high-value content (user questions, final assistant answers, cited
|
||||
sources), drop noise (empty/greeting/intermediate reasoning), and write a
|
||||
*declarative* note rather than a chat transcript. An LLM extractor can later
|
||||
replace :func:`extract_thread` while keeping the same :class:`NoteDraft` output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
|
||||
|
||||
# Greeting / low-value lines filtered from extracted turns.
|
||||
_GREETING_RE = re.compile(r"^(你好|您好|hi|hello|hey|thanks?|谢谢|好的|ok|okay)[\s!!。.,,~]*$", re.IGNORECASE)
|
||||
_WS_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def _one_line(text: str, limit: int = 300) -> str:
|
||||
"""Collapse whitespace and clip ``text`` to a single line."""
|
||||
collapsed = _WS_RE.sub(" ", text or "").strip()
|
||||
return collapsed[:limit]
|
||||
|
||||
|
||||
def message_text(msg: dict[str, Any]) -> str:
|
||||
"""Return the plain-text content of a serialized LangChain message."""
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
return content.strip()
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict):
|
||||
if block.get("type") in (None, "text") and isinstance(block.get("text"), str):
|
||||
parts.append(block["text"])
|
||||
return "\n".join(p for p in parts if p).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _msg_type(msg: dict[str, Any]) -> str:
|
||||
return str(msg.get("type") or msg.get("role") or "").lower()
|
||||
|
||||
|
||||
def split_turns(messages: list[dict[str, Any]], *, max_messages: int = 80) -> list[dict[str, str]]:
|
||||
"""Split messages into ``{user, ai}`` turns keyed by human messages.
|
||||
|
||||
For each turn we keep the last substantial AI message (with text body).
|
||||
"""
|
||||
msgs = messages[-max_messages:] if max_messages and len(messages) > max_messages else list(messages)
|
||||
turns: list[dict[str, str]] = []
|
||||
current: dict[str, str] | None = None
|
||||
for msg in msgs:
|
||||
mtype = _msg_type(msg)
|
||||
text = message_text(msg)
|
||||
if mtype == "human":
|
||||
if current and (current.get("user") or current.get("ai")):
|
||||
turns.append(current)
|
||||
current = {"user": text, "ai": ""}
|
||||
elif mtype == "ai" and text:
|
||||
if current is None:
|
||||
current = {"user": "", "ai": text}
|
||||
else:
|
||||
current["ai"] = text # last substantial AI message wins
|
||||
if current and (current.get("user") or current.get("ai")):
|
||||
turns.append(current)
|
||||
# Drop turns that are pure greetings with no answer.
|
||||
cleaned = []
|
||||
for t in turns:
|
||||
user = (t.get("user") or "").strip()
|
||||
ai = (t.get("ai") or "").strip()
|
||||
if not ai and _GREETING_RE.match(user):
|
||||
continue
|
||||
if not user and not ai:
|
||||
continue
|
||||
cleaned.append({"user": user, "ai": ai})
|
||||
return cleaned
|
||||
|
||||
|
||||
def _auto_title(thread_title: str | None, turns: list[dict[str, str]]) -> str:
|
||||
if thread_title and thread_title.strip() and thread_title.strip().lower() != "untitled":
|
||||
return thread_title.strip()[:200]
|
||||
for t in turns:
|
||||
if t.get("user"):
|
||||
first = t["user"].splitlines()[0].strip()
|
||||
return (first[:40] + ("…" if len(first) > 40 else "")) or "知识笔记"
|
||||
return "知识笔记"
|
||||
|
||||
|
||||
def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str:
|
||||
"""Simple rule-based summary: the first answer, clipped."""
|
||||
for t in turns:
|
||||
if t.get("ai"):
|
||||
text = re.sub(r"\s+", " ", t["ai"]).strip()
|
||||
return text[:limit] + ("…" if len(text) > limit else "")
|
||||
for t in turns:
|
||||
if t.get("user"):
|
||||
text = re.sub(r"\s+", " ", t["user"]).strip()
|
||||
return text[:limit] + ("…" if len(text) > limit else "")
|
||||
return ""
|
||||
|
||||
|
||||
def related_links(project_name: str | None) -> list[str]:
|
||||
"""At least two wikilinks per note (wiki-capture requirement)."""
|
||||
links = ["[[index]]", "[[_meta/taxonomy]]"]
|
||||
if project_name:
|
||||
links.append(f"[[projects/{project_name}/synthesis/product-knowledge-base]]")
|
||||
return links
|
||||
|
||||
|
||||
# Backwards-compatible internal alias.
|
||||
_related_links = related_links
|
||||
|
||||
|
||||
def _build_body(
|
||||
*,
|
||||
title: str,
|
||||
turns: list[dict[str, str]],
|
||||
mode: str,
|
||||
project_name: str | None,
|
||||
sources: list[SourceDraft],
|
||||
) -> str:
|
||||
"""Render the wiki-capture 5-section declarative body."""
|
||||
full = mode == "full"
|
||||
answer_clip = 4000 if full else 800
|
||||
|
||||
context_lines = []
|
||||
for t in turns:
|
||||
if t.get("user"):
|
||||
q = re.sub(r"\s+", " ", t["user"]).strip()
|
||||
context_lines.append(f"- {q[:300]}")
|
||||
context = "\n".join(context_lines[:8]) or "- 来自一次对话沉淀。^[inferred]"
|
||||
|
||||
finding_lines = []
|
||||
for i, t in enumerate(turns, 1):
|
||||
if t.get("ai"):
|
||||
ans = t["ai"].strip()
|
||||
ans = ans if full else (ans[:answer_clip] + ("…" if len(ans) > answer_clip else ""))
|
||||
finding_lines.append(f"### 结论 {i}\n\n{ans}")
|
||||
finding = "\n\n".join(finding_lines) or "_暂无可提炼的结论。_"
|
||||
|
||||
reasoning = "本页由对话自动沉淀而成,结论来自助手最终回答;过程性推理已省略。判断与归纳部分标记为 ^[inferred]。"
|
||||
|
||||
implications = "- 后续可在相关问答中参考本页知识。\n- 如内容过期或冲突,请编辑或归档。"
|
||||
|
||||
related = "\n".join(f"- {link}" for link in _related_links(project_name))
|
||||
if sources:
|
||||
related += "\n\n### Sources\n\n"
|
||||
for s in sources[:20]:
|
||||
label = s.title or s.url or "来源"
|
||||
related += f"- {label}" + (f" — {s.url}" if s.url else "") + "\n"
|
||||
|
||||
return (
|
||||
f"# {title}\n\n"
|
||||
f"## Context\n\n{context}\n\n"
|
||||
f"## Finding / Decision\n\n{finding}\n\n"
|
||||
f"## Reasoning\n\n{reasoning}\n\n"
|
||||
f"## Implications\n\n{implications}\n\n"
|
||||
f"## Related\n\n{related}\n"
|
||||
)
|
||||
|
||||
|
||||
def extract_thread(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None = None,
|
||||
mode: str = "summary",
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "deerflow",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
max_messages: int = 80,
|
||||
) -> NoteDraft | None:
|
||||
"""Build a :class:`NoteDraft` from thread messages, or ``None`` if empty."""
|
||||
turns = split_turns(messages, max_messages=max_messages)
|
||||
if not turns:
|
||||
return None
|
||||
final_title = (title or "").strip() or _auto_title(thread_title, turns)
|
||||
summary = _summary(turns)
|
||||
sources = sources or []
|
||||
body = _build_body(title=final_title, turns=turns, mode=mode, project_name=project_name, sources=sources)
|
||||
base_tags = list(tags or [])
|
||||
for default in (project_name or "deerflow", "knowledge-base"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
return NoteDraft(
|
||||
title=final_title,
|
||||
summary=summary,
|
||||
content_md=body,
|
||||
category="synthesis",
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
tags=base_tags,
|
||||
confidence=0.6,
|
||||
related=_related_links(project_name),
|
||||
sources=sources,
|
||||
source_key=f"thread:{thread_id}",
|
||||
)
|
||||
|
||||
|
||||
def extract_search(
|
||||
query: str,
|
||||
sources: list[SourceDraft],
|
||||
*,
|
||||
thread_id: str | None = None,
|
||||
project_name: str | None = "deerflow",
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
) -> NoteDraft:
|
||||
"""Build a reference-category :class:`NoteDraft` from search/tool results."""
|
||||
final_title = (title or "").strip() or f"检索:{query[:40]}"
|
||||
body_lines = [f"# {final_title}\n", "## Context\n", f"- 检索关键词:{query}\n", "## Finding / Decision\n"]
|
||||
for s in sources:
|
||||
label = s.title or s.url or "来源"
|
||||
line = f"- **{label}**"
|
||||
if s.url:
|
||||
line += f" — {s.url}"
|
||||
if s.snippet:
|
||||
line += "\n - " + _one_line(s.snippet)
|
||||
body_lines.append(line)
|
||||
body_lines += [
|
||||
"\n## Reasoning\n",
|
||||
"本页汇总外部检索来源,供后续问答引用。来源可信度需结合原文判断。^[ambiguous]\n",
|
||||
"## Implications\n",
|
||||
"- 引用前请核对原始来源。\n",
|
||||
"## Related\n",
|
||||
*[f"- {link}" for link in _related_links(project_name)],
|
||||
]
|
||||
base_tags = list(tags or [])
|
||||
for default in (project_name or "deerflow", "knowledge-base", "reference"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
return NoteDraft(
|
||||
title=final_title,
|
||||
summary=f"检索“{query}”的来源汇总,共 {len(sources)} 条。",
|
||||
content_md="\n".join(body_lines),
|
||||
category="references",
|
||||
source_type="search",
|
||||
source_id=f"search:{query[:120]}",
|
||||
tags=base_tags,
|
||||
confidence=0.4,
|
||||
related=_related_links(project_name),
|
||||
sources=sources,
|
||||
source_key=None,
|
||||
)
|
||||
Reference in new issue
Block a user