diff --git a/offline-backend-20260512/backend/config.yaml b/offline-backend-20260512/backend/config.yaml index b72df22f..1904f4df 100644 --- a/offline-backend-20260512/backend/config.yaml +++ b/offline-backend-20260512/backend/config.yaml @@ -354,6 +354,11 @@ knowledge: project_name: deerflow auto_ingest_enabled: false auto_ingest_min_chars: 300 + # 用大模型按 wiki-capture 规范提炼陈述式知识(而非堆砌对话原文)。 + # 关闭则回退到规则抽取;extract_model_name 留空时用 models[0]。 + llm_extract_enabled: true + extract_model_name: "" + extract_max_input_chars: 12000 max_thread_messages: 80 max_source_items: 20 rag_enabled: false diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py index 914f293a..18900103 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py @@ -46,6 +46,9 @@ class KnowledgeConfig(BaseModel): project_name: str = Field(default="deerflow", description="Project namespace for projects// pages.") auto_ingest_enabled: bool = Field(default=False, description="Auto-sediment conversations in the background (phase 3).") auto_ingest_min_chars: int = Field(default=300, description="Minimum characters before auto-ingest considers a turn.") + llm_extract_enabled: bool = Field(default=True, description="Distill notes with an LLM (wiki-capture style) instead of the rule-based fallback.") + extract_model_name: str | None = Field(default=None, description="Model (from models[]) used to distill knowledge. Falls back to models[0] when unset.") + extract_max_input_chars: int = Field(default=12000, description="Max characters of conversation transcript fed to the extraction LLM.") max_thread_messages: int = Field(default=80, description="Maximum thread messages to scan when capturing.") max_source_items: int = Field(default=20, description="Maximum source items recorded per note.") rag_enabled: bool = Field(default=False, description="Enable RAG injection before agent runs (phase 2).") diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py new file mode 100644 index 00000000..bad26148 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py @@ -0,0 +1,207 @@ +"""LLM-based knowledge extraction (wiki-capture style). + +Turns a conversation into a *declarative* knowledge note instead of dumping the +raw chat. Follows the obsidian-wiki ``wiki-capture`` skill: classify the note, +keep only high-value findings/decisions, drop greetings / intermediate +reasoning / verbatim transcripts, mark inferred content, and emit structured +JSON that we render into the Context / Finding / Reasoning / Implications / +Related body. + +Falls back to the rule-based :func:`deerflow.knowledge.extractor.extract_thread` +when the LLM is unavailable or returns something unusable, so capture never +hard-fails. +""" + +from __future__ import annotations + +import json +import logging +import re +from typing import Any + +from deerflow.knowledge import extractor +from deerflow.knowledge.schemas import NoteDraft, SourceDraft +from deerflow.knowledge.skill_templates import WIKI_CAPTURE_GUIDANCE + +logger = logging.getLogger(__name__) + +_VALID_CATEGORIES = {"synthesis", "concepts", "references", "decision", "journal"} + +_SYSTEM_PROMPT = """\ +你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一次对话提炼成\ +**陈述式知识(declarative knowledge)**,而不是聊天流水账。 + +要求: +- 只保留高价值内容:明确结论、方案、决策、关键事实、被引用的来源。 +- 丢弃:寒暄、客套、中间推理过程、重复内容、无意义的原文堆砌。 +- 不要写“用户问了什么、AI 回答了什么”,要写“结论是什么、为什么成立”。 +- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。 +- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\ +references(外部来源汇总)、decision(架构/设计决策)、journal(会话纪要)。 +- 用中文输出。 + +只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构如下: +{ + "should_ingest": true, + "reason": "为什么值得/不值得沉淀", + "title": "简洁的知识标题(不超过 40 字)", + "summary": "1-2 句话概括这页知识", + "category": "synthesis", + "context": "问题背景或触发场景,1-3 句", + "findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."], + "reasoning": "为什么成立、取舍、不确定性", + "implications": ["后续建议/风险/下一步", "..."], + "tags": ["标签1", "标签2"], + "confidence": 0.0 +}""" + + +def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str: + parts: list[str] = [] + for i, t in enumerate(turns, 1): + if t.get("user"): + parts.append(f"[用户 {i}] {t['user']}") + if t.get("ai"): + parts.append(f"[助手 {i}] {t['ai']}") + if sources: + parts.append("\n[来源]") + for s in sources[:20]: + label = s.title or s.url or s.tool_name or "来源" + parts.append(f"- {label}" + (f" ({s.url})" if s.url else "")) + text = "\n\n".join(parts) + if len(text) > max_chars: + text = text[:max_chars] + "\n…(已截断)" + return text + + +def _parse_json(raw: str) -> dict[str, Any] | None: + if not raw: + return None + text = raw.strip() + # Strip ```json fences if present. + fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL) + if fence: + text = fence.group(1).strip() + # Otherwise grab the outermost { ... }. + if not text.startswith("{"): + start = text.find("{") + end = text.rfind("}") + if start >= 0 and end > start: + text = text[start : end + 1] + try: + data = json.loads(text) + return data if isinstance(data, dict) else None + except json.JSONDecodeError: + logger.warning("Knowledge LLM returned non-JSON output; falling back to rule-based") + return None + + +def _render_body(data: dict[str, Any], title: str, related: list[str], sources: list[SourceDraft]) -> str: + def _bullets(items: Any) -> str: + if isinstance(items, str): + items = [items] + if not isinstance(items, list): + return "" + return "\n".join(f"- {str(x).strip()}" for x in items if str(x).strip()) + + context = str(data.get("context") or "").strip() or "- 来自一次对话沉淀。^[inferred]" + findings = _bullets(data.get("findings")) or "_暂无可提炼的结论。_" + reasoning = str(data.get("reasoning") or "").strip() or "(未提供推理说明)" + implications = _bullets(data.get("implications")) or "- 如内容过期或冲突,请编辑或归档。" + + related_block = "\n".join(f"- {link}" for link in related) + if sources: + related_block += "\n\n### Sources\n\n" + for s in sources[:20]: + label = s.title or s.url or "来源" + related_block += f"- {label}" + (f" — {s.url}" if s.url else "") + "\n" + + return ( + f"# {title}\n\n" + f"## Context\n\n{context}\n\n" + f"## Finding / Decision\n\n{findings}\n\n" + f"## Reasoning\n\n{reasoning}\n\n" + f"## Implications\n\n{implications}\n\n" + f"## Related\n\n{related_block}\n" + ) + + +async def extract_thread_llm( + messages: list[dict[str, Any]], + *, + thread_id: str, + thread_title: str | None = None, + mode: str = "summary", + title: str | None = None, + tags: list[str] | None = None, + project_name: str | None = "deerflow", + sources: list[SourceDraft] | None = None, + max_messages: int = 80, + model_name: str | None = None, + max_input_chars: int = 12000, +) -> NoteDraft | None: + """Distill a thread into a :class:`NoteDraft` via the LLM. + + Returns ``None`` when there is nothing to sediment **or** the LLM path + fails — callers should fall back to the rule-based extractor. + """ + turns = extractor.split_turns(messages, max_messages=max_messages) + if not turns: + return None + sources = sources or [] + + try: + from langchain_core.messages import HumanMessage, SystemMessage + + from deerflow.models import create_chat_model + + model = create_chat_model(name=model_name or None, thinking_enabled=False) + transcript = _build_transcript(turns, sources, max_chars=max_input_chars) + prompt = [ + SystemMessage(content=_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE), + HumanMessage(content=f"以下是一次对话,请按规范提炼为知识 JSON:\n\n{transcript}"), + ] + response = await model.ainvoke(prompt) + raw = response.content if isinstance(response.content, str) else str(response.content) + data = _parse_json(raw) + except Exception: + logger.exception("Knowledge LLM extraction failed for thread %s; using rule-based fallback", thread_id) + return None + + if not data: + return None + + category = str(data.get("category") or "synthesis").strip().lower() + if category not in _VALID_CATEGORIES: + category = "synthesis" + + final_title = (title or "").strip() or str(data.get("title") or "").strip() or extractor._auto_title(thread_title, turns) + summary = str(data.get("summary") or "").strip() + related = extractor.related_links(project_name) + body = _render_body(data, final_title, related, sources) + + base_tags = list(tags or []) + llm_tags = data.get("tags") if isinstance(data.get("tags"), list) else [] + for tag in [*llm_tags, project_name or "deerflow", "knowledge-base"]: + tag = str(tag).strip() + if tag and tag not in base_tags: + base_tags.append(tag) + + try: + confidence = float(data.get("confidence") or 0.7) + except (TypeError, ValueError): + confidence = 0.7 + + return NoteDraft( + title=final_title, + summary=summary or final_title, + content_md=body, + category=category, + source_type="thread", + source_id=thread_id, + tags=base_tags, + confidence=max(0.0, min(confidence, 1.0)), + related=related, + sources=sources, + source_key=f"thread:{thread_id}", + ) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py index 8ce20b93..de258ab8 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py @@ -15,7 +15,7 @@ from typing import Any from deerflow.config.knowledge_config import KnowledgeConfig from deerflow.config.runtime_paths import runtime_home -from deerflow.knowledge import extractor +from deerflow.knowledge import extractor, llm_extractor from deerflow.knowledge.markdown_vault import MarkdownVault from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter from deerflow.knowledge.repository import KnowledgeRepository @@ -113,17 +113,36 @@ class KnowledgeService: if include_sources and reference_batches: sources = from_reference_batches(reference_batches, thread_id=thread_id, max_items=self.config.max_source_items) - draft = extractor.extract_thread( - messages, - thread_id=thread_id, - thread_title=thread_title, - mode=mode, - title=title, - tags=tags, - project_name=self.project_name, - sources=sources, - max_messages=self.config.max_thread_messages, - ) + draft: NoteDraft | None = None + # Prefer LLM distillation (wiki-capture style) so notes are declarative + # knowledge, not raw chat. Falls back to the rule-based extractor when + # the LLM is disabled/unavailable or returns something unusable. + if self.config.llm_extract_enabled: + draft = await llm_extractor.extract_thread_llm( + messages, + thread_id=thread_id, + thread_title=thread_title, + mode=mode, + title=title, + tags=tags, + project_name=self.project_name, + sources=sources, + max_messages=self.config.max_thread_messages, + model_name=self.config.extract_model_name, + max_input_chars=self.config.extract_max_input_chars, + ) + if draft is None: + draft = extractor.extract_thread( + messages, + thread_id=thread_id, + thread_title=thread_title, + mode=mode, + title=title, + tags=tags, + project_name=self.project_name, + sources=sources, + max_messages=self.config.max_thread_messages, + ) if draft is None: raise EmptyThreadError(f"Thread {thread_id} has no sediment-able content") draft.status = normalize_status(status)