feat(knowledge): 支持文件导入沉淀知识,知识图谱可视化随主题切换
- 后端新增 /api/knowledge/ingest/file:上传 md/txt/pdf/word/ppt/excel, 转 Markdown 后走 wiki-capture LLM 蒸馏为陈述式知识笔记,失败回退逐字草稿 - service.capture_file + 文档专用蒸馏提示词,按主题拆分 1~4 页 - 知识图谱导出 HTML 支持 ?theme=light|dark,布局稳定后冻结, 按度数缩放节点,悬停/点击聚焦邻居 - 前端知识库页新增「导入文件」对话框与 ingestKnowledgeFile API - 智能体问答页与简介侧边栏 UI 优化;批量人物蒸馏脚本提取正文逻辑修正 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
17bb3715d1
commit
3845d5090b
14 files changed
+1040
-352
No files matched your search
+146
-13
@@ -84,6 +84,46 @@ references(外部来源汇总)、decision(架构/设计决策)、journal
|
||||
}"""
|
||||
|
||||
|
||||
_DOC_SYSTEM_PROMPT = """\
|
||||
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一篇**文档**提炼成\
|
||||
**陈述式知识(declarative knowledge)**,而不是照抄原文。
|
||||
|
||||
要求:
|
||||
- 只保留高价值内容:核心结论、关键事实、定义、流程、数据、决策与其依据。
|
||||
- 丢弃:目录、页眉页脚、版权声明、重复的样板文字、无意义的排版噪声。
|
||||
- 用自己的话重组为结构化知识,不要逐段复制原文。
|
||||
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
|
||||
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
|
||||
references(外部来源汇总)、decision(架构/设计决策)、journal(纪要)。
|
||||
- 用中文输出。
|
||||
|
||||
一篇文档可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
|
||||
- 默认只产出 **1 页**;只有当文档明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
|
||||
- 每一页聚焦单一主题,标题与 summary 必须彼此区分。
|
||||
- 文档没有可沉淀的知识时返回空的 notes 数组。
|
||||
|
||||
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构与对话提炼一致:
|
||||
{
|
||||
"should_ingest": true,
|
||||
"reason": "为什么值得/不值得沉淀",
|
||||
"notes": [
|
||||
{
|
||||
"title": "简洁的知识标题(不超过 40 字)",
|
||||
"summary": "1-2 句话概括这页知识",
|
||||
"category": "synthesis",
|
||||
"context": "文档背景或适用场景,1-3 句",
|
||||
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
|
||||
"reasoning": "为什么成立、取舍、不确定性",
|
||||
"implications": ["后续建议/风险/下一步", "..."],
|
||||
"tags": ["标签1", "标签2"],
|
||||
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
|
||||
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
|
||||
"confidence": 0.0
|
||||
}
|
||||
]
|
||||
}"""
|
||||
|
||||
|
||||
def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str:
|
||||
parts: list[str] = []
|
||||
for i, t in enumerate(turns, 1):
|
||||
@@ -171,21 +211,27 @@ def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
def _build_note_draft(
|
||||
note_data: dict[str, Any],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None,
|
||||
turns: list[dict[str, str]],
|
||||
source_type: str,
|
||||
source_id: str | None,
|
||||
source_key: str,
|
||||
fallback_title: str,
|
||||
title_override: str | None,
|
||||
tags: list[str] | None,
|
||||
project_name: str | None,
|
||||
sources: list[SourceDraft],
|
||||
source_key_suffix: str = "",
|
||||
) -> NoteDraft:
|
||||
"""Render one note dict into a :class:`NoteDraft`."""
|
||||
"""Render one note dict into a :class:`NoteDraft`.
|
||||
|
||||
Source-agnostic: the caller supplies ``source_type`` / ``source_id`` /
|
||||
``source_key`` and a ``fallback_title`` (used when neither the user override
|
||||
nor the LLM produced a usable title), so both thread and document capture
|
||||
share one renderer.
|
||||
"""
|
||||
category = str(note_data.get("category") or "synthesis").strip().lower()
|
||||
if category not in _VALID_CATEGORIES:
|
||||
category = "synthesis"
|
||||
|
||||
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or extractor._auto_title(thread_title, turns)
|
||||
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or fallback_title
|
||||
summary = str(note_data.get("summary") or "").strip()
|
||||
related = extractor.related_links(project_name)
|
||||
body = _render_body(note_data, final_title, related, sources)
|
||||
@@ -210,15 +256,15 @@ def _build_note_draft(
|
||||
summary=summary or final_title,
|
||||
content_md=body,
|
||||
category=category,
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
source_type=source_type,
|
||||
source_id=source_id,
|
||||
tags=base_tags,
|
||||
confidence=max(0.0, min(confidence, 1.0)),
|
||||
related=related,
|
||||
sources=sources,
|
||||
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
|
||||
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
|
||||
source_key=f"thread:{thread_id}{source_key_suffix}",
|
||||
source_key=source_key,
|
||||
)
|
||||
|
||||
|
||||
@@ -293,6 +339,7 @@ async def extract_thread_llm_multi(
|
||||
else:
|
||||
note_dicts = note_dicts[: max(1, max_notes)]
|
||||
|
||||
fallback_title = extractor._auto_title(thread_title, turns)
|
||||
drafts: list[NoteDraft] = []
|
||||
for i, nd in enumerate(note_dicts):
|
||||
# The user-supplied title override only applies to a single-note result;
|
||||
@@ -302,14 +349,100 @@ async def extract_thread_llm_multi(
|
||||
drafts.append(
|
||||
_build_note_draft(
|
||||
nd,
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
turns=turns,
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
source_key=f"thread:{thread_id}{suffix}",
|
||||
fallback_title=fallback_title,
|
||||
title_override=override,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
|
||||
async def extract_document_llm_multi(
|
||||
text: str,
|
||||
*,
|
||||
doc_id: str,
|
||||
doc_title: str | None = None,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
model_name: str | None = None,
|
||||
max_input_chars: int = 12000,
|
||||
allow_multi: bool = True,
|
||||
max_notes: int = 4,
|
||||
) -> list[NoteDraft]:
|
||||
"""Distill an imported **document** into one *or several* knowledge notes.
|
||||
|
||||
Mirrors :func:`extract_thread_llm_multi` but takes raw document text (already
|
||||
converted to Markdown/plain text) instead of a conversation. Returns ``[]``
|
||||
when the LLM path fails (callers fall back to a verbatim note); raises
|
||||
:class:`LLMRejectedIngest` when the model deliberately judges the document
|
||||
worthless. Resulting notes carry ``source_type="file"``.
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
sources = sources or []
|
||||
|
||||
try:
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
model = create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
body = text if len(text) <= max_input_chars else text[:max_input_chars] + "\n…(已截断)"
|
||||
header = f"文档标题:{doc_title}\n\n" if doc_title else ""
|
||||
prompt = [
|
||||
SystemMessage(content=_DOC_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE),
|
||||
HumanMessage(content=f"以下是一篇文档,请按规范提炼为知识 JSON:\n\n{header}{body}"),
|
||||
]
|
||||
response = await model.ainvoke(prompt)
|
||||
raw = response.content if isinstance(response.content, str) else str(response.content)
|
||||
data = _parse_json(raw)
|
||||
except Exception:
|
||||
logger.exception("[knowledge] document %s — LLM extraction call failed; using rule-based fallback", doc_id)
|
||||
return []
|
||||
|
||||
if not data:
|
||||
logger.warning("[knowledge] document %s — LLM returned no parsable JSON; using rule-based fallback", doc_id)
|
||||
return []
|
||||
|
||||
note_dicts = _notes_from_data(data)
|
||||
if not note_dicts:
|
||||
reason = str(data.get("reason") or "")[:200]
|
||||
if _is_falsey(data.get("should_ingest")):
|
||||
logger.warning("[knowledge] document %s SKIPPED — model judged not worth ingesting: %s", doc_id, reason)
|
||||
raise LLMRejectedIngest(reason or "no knowledge value")
|
||||
logger.warning(
|
||||
"[knowledge] document %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
|
||||
doc_id,
|
||||
data.get("should_ingest"),
|
||||
reason,
|
||||
)
|
||||
return []
|
||||
note_dicts = note_dicts[:1] if not allow_multi else note_dicts[: max(1, max_notes)]
|
||||
|
||||
fallback_title = (doc_title or title or "文档知识").strip() or "文档知识"
|
||||
drafts: list[NoteDraft] = []
|
||||
for i, nd in enumerate(note_dicts):
|
||||
override = title if (len(note_dicts) == 1) else None
|
||||
suffix = "" if len(note_dicts) == 1 else f"#{i}"
|
||||
drafts.append(
|
||||
_build_note_draft(
|
||||
nd,
|
||||
source_type="file",
|
||||
source_id=doc_id,
|
||||
source_key=f"file:{doc_id}{suffix}",
|
||||
fallback_title=fallback_title,
|
||||
title_override=override,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
source_key_suffix=suffix,
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
Reference in new issue
Block a user