Files
deerflow-code/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py
T
AdministratorandClaude Opus 4.8 81e4fd92ce feat(knowledge): 阶段三 知识治理 + 阶段四 图谱增强
阶段三(治理):
- sensitive.py:写入前脱敏 token/密钥/手机号/身份证/邮箱(sensitive_redaction_enabled)
- 版本快照:更新前写 knowledge_note_versions;GET /notes/{id}/versions
- dedup.py:标题/摘要 Jaccard + 内容哈希查重;GET /duplicates、POST /merge(来源并入主条目、重复项归档)
- auto_ingest.py:独立后台队列/线程沉淀,不阻塞问答;中间件 after_agent 在 auto_ingest_enabled 时入队;价值判断(最小字符)+ skip_existing 去重;自动沉淀落 draft 待审

阶段四(图谱):
- LLM 抽取 entities/relations;service 存 knowledge_entities/knowledge_relations 并连边
- exporter.build_graph 支持实体节点 + note→entity/entity→entity 边;graph.html/json 增强
- GET /api/knowledge/graph 返回产品图谱数据

前端:知识库页「查重合并」对话框 + 相关 api/hooks
配置:sensitive_redaction_enabled / dedup_similarity_threshold / entity_extraction_enabled
测试:tests/test_knowledge.py 20 passed

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-08 18:31:33 +08:00

215 lines
8.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""LLM-based knowledge extraction (wiki-capture style).
Turns a conversation into a *declarative* knowledge note instead of dumping the
raw chat. Follows the obsidian-wiki ``wiki-capture`` skill: classify the note,
keep only high-value findings/decisions, drop greetings / intermediate
reasoning / verbatim transcripts, mark inferred content, and emit structured
JSON that we render into the Context / Finding / Reasoning / Implications /
Related body.
Falls back to the rule-based :func:`deerflow.knowledge.extractor.extract_thread`
when the LLM is unavailable or returns something unusable, so capture never
hard-fails.
"""
from __future__ import annotations
import json
import logging
import re
from typing import Any
from deerflow.knowledge import extractor
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
from deerflow.knowledge.skill_templates import WIKI_CAPTURE_GUIDANCE
logger = logging.getLogger(__name__)
_VALID_CATEGORIES = {"synthesis", "concepts", "references", "decision", "journal"}
_SYSTEM_PROMPT = """\
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一次对话提炼成\
**陈述式知识(declarative knowledge)**,而不是聊天流水账。
要求:
- 只保留高价值内容:明确结论、方案、决策、关键事实、被引用的来源。
- 丢弃:寒暄、客套、中间推理过程、重复内容、无意义的原文堆砌。
- 不要写“用户问了什么、AI 回答了什么”,要写“结论是什么、为什么成立”。
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
references(外部来源汇总)、decision(架构/设计决策)、journal(会话纪要)。
- 用中文输出。
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构如下:
{
"should_ingest": true,
"reason": "为什么值得/不值得沉淀",
"title": "简洁的知识标题(不超过 40 字)",
"summary": "1-2 句话概括这页知识",
"category": "synthesis",
"context": "问题背景或触发场景,1-3 句",
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
"reasoning": "为什么成立、取舍、不确定性",
"implications": ["后续建议/风险/下一步", "..."],
"tags": ["标签1", "标签2"],
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
"confidence": 0.0
}"""
def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str:
parts: list[str] = []
for i, t in enumerate(turns, 1):
if t.get("user"):
parts.append(f"[用户 {i}] {t['user']}")
if t.get("ai"):
parts.append(f"[助手 {i}] {t['ai']}")
if sources:
parts.append("\n[来源]")
for s in sources[:20]:
label = s.title or s.url or s.tool_name or "来源"
parts.append(f"- {label}" + (f" ({s.url})" if s.url else ""))
text = "\n\n".join(parts)
if len(text) > max_chars:
text = text[:max_chars] + "\n…(已截断)"
return text
def _parse_json(raw: str) -> dict[str, Any] | None:
if not raw:
return None
text = raw.strip()
# Strip ```json fences if present.
fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL)
if fence:
text = fence.group(1).strip()
# Otherwise grab the outermost { ... }.
if not text.startswith("{"):
start = text.find("{")
end = text.rfind("}")
if start >= 0 and end > start:
text = text[start : end + 1]
try:
data = json.loads(text)
return data if isinstance(data, dict) else None
except json.JSONDecodeError:
logger.warning("Knowledge LLM returned non-JSON output; falling back to rule-based")
return None
def _render_body(data: dict[str, Any], title: str, related: list[str], sources: list[SourceDraft]) -> str:
def _bullets(items: Any) -> str:
if isinstance(items, str):
items = [items]
if not isinstance(items, list):
return ""
return "\n".join(f"- {str(x).strip()}" for x in items if str(x).strip())
context = str(data.get("context") or "").strip() or "- 来自一次对话沉淀。^[inferred]"
findings = _bullets(data.get("findings")) or "_暂无可提炼的结论。_"
reasoning = str(data.get("reasoning") or "").strip() or "(未提供推理说明)"
implications = _bullets(data.get("implications")) or "- 如内容过期或冲突,请编辑或归档。"
related_block = "\n".join(f"- {link}" for link in related)
if sources:
related_block += "\n\n### Sources\n\n"
for s in sources[:20]:
label = s.title or s.url or "来源"
related_block += f"- {label}" + (f" — {s.url}" if s.url else "") + "\n"
return (
f"# {title}\n\n"
f"## Context\n\n{context}\n\n"
f"## Finding / Decision\n\n{findings}\n\n"
f"## Reasoning\n\n{reasoning}\n\n"
f"## Implications\n\n{implications}\n\n"
f"## Related\n\n{related_block}\n"
)
async def extract_thread_llm(
messages: list[dict[str, Any]],
*,
thread_id: str,
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "deerflow",
sources: list[SourceDraft] | None = None,
max_messages: int = 80,
model_name: str | None = None,
max_input_chars: int = 12000,
) -> NoteDraft | None:
"""Distill a thread into a :class:`NoteDraft` via the LLM.
Returns ``None`` when there is nothing to sediment **or** the LLM path
fails — callers should fall back to the rule-based extractor.
"""
turns = extractor.split_turns(messages, max_messages=max_messages)
if not turns:
return None
sources = sources or []
try:
from langchain_core.messages import HumanMessage, SystemMessage
from deerflow.models import create_chat_model
model = create_chat_model(name=model_name or None, thinking_enabled=False)
transcript = _build_transcript(turns, sources, max_chars=max_input_chars)
prompt = [
SystemMessage(content=_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE),
HumanMessage(content=f"以下是一次对话,请按规范提炼为知识 JSON:\n\n{transcript}"),
]
response = await model.ainvoke(prompt)
raw = response.content if isinstance(response.content, str) else str(response.content)
data = _parse_json(raw)
except Exception:
logger.exception("Knowledge LLM extraction failed for thread %s; using rule-based fallback", thread_id)
return None
if not data:
return None
category = str(data.get("category") or "synthesis").strip().lower()
if category not in _VALID_CATEGORIES:
category = "synthesis"
final_title = (title or "").strip() or str(data.get("title") or "").strip() or extractor._auto_title(thread_title, turns)
summary = str(data.get("summary") or "").strip()
related = extractor.related_links(project_name)
body = _render_body(data, final_title, related, sources)
base_tags = list(tags or [])
llm_tags = data.get("tags") if isinstance(data.get("tags"), list) else []
for tag in [*llm_tags, project_name or "deerflow", "knowledge-base"]:
tag = str(tag).strip()
if tag and tag not in base_tags:
base_tags.append(tag)
try:
confidence = float(data.get("confidence") or 0.7)
except (TypeError, ValueError):
confidence = 0.7
entities = data.get("entities") if isinstance(data.get("entities"), list) else []
relations = data.get("relations") if isinstance(data.get("relations"), list) else []
return NoteDraft(
title=final_title,
summary=summary or final_title,
content_md=body,
category=category,
source_type="thread",
source_id=thread_id,
tags=base_tags,
confidence=max(0.0, min(confidence, 1.0)),
related=related,
sources=sources,
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
source_key=f"thread:{thread_id}",
)