This commit is contained in:
吉鑫升 committed 2026-06-08 18:02:14 +08:00
commit fb692a0555
30 files changed
+3618 -6

No files matched your search

@@ -9,13 +9,14 @@ from dotenv import load_dotenv
from pydantic import BaseModel, ConfigDict, Field
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.log_system_config import LogSystemConfig
from deerflow.config.login_config import AuthLoginConfig
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
@@ -133,8 +134,9 @@ class AppConfig(BaseModel):
gateway: GatewayConfig = Field(default_factory=GatewayConfig, description="Gateway FastAPI listening port (used by intra-process loopback)")
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration")
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
knowledge: KnowledgeConfig = Field(default_factory=KnowledgeConfig, description="Product knowledge base (obsidian-wiki compatible vault) configuration")
@classmethod
def resolve_config_path(cls, config_path: str | None = None) -> Path:
@@ -0,0 +1,55 @@
"""Configuration for the product knowledge base (obsidian-wiki compatible)."""
from pydantic import BaseModel, Field
class KnowledgeExportConfig(BaseModel):
"""Knowledge graph export switches (``wiki-export`` compatible output)."""
enabled: bool = Field(default=True, description="Master switch for graph export.")
graph_html_enabled: bool = Field(default=True, description="Generate wiki-export/graph.html.")
graph_json_enabled: bool = Field(default=True, description="Generate wiki-export/graph.json.")
class KnowledgeQueryConfig(BaseModel):
"""Layered query strategy flags (``wiki-query`` style)."""
use_hot_md: bool = Field(default=True, description="Consult hot.md when searching.")
use_index_md: bool = Field(default=True, description="Consult index.md when searching.")
use_frontmatter_scan: bool = Field(default=True, description="Scan frontmatter when searching.")
class KnowledgeEmbeddingConfig(BaseModel):
"""Vector embedding config (phase 2 — disabled in the first release)."""
enabled: bool = Field(default=False, description="Enable vector embeddings.")
provider: str = Field(default="local", description="Embedding provider.")
model: str = Field(default="", description="Embedding model name.")
class KnowledgeConfig(BaseModel):
"""Product knowledge base configuration.
The first release ships a global, shared, obsidian-wiki compatible vault.
Knowledge metadata is mirrored into the database for fast listing/search,
while ``content_md`` is written to a Markdown vault that stays compatible
with ``ar9av/obsidian-wiki`` (index.md / log.md / hot.md / .manifest.json,
frontmatter, wikilinks, wiki-export graph artifacts).
"""
enabled: bool = Field(default=True, description="Enable the knowledge base feature.")
provider: str = Field(default="obsidian_wiki", description="Knowledge organization provider.")
vault_dir: str = Field(
default="data/knowledge/obsidian-vault",
description="Vault directory (relative to DEER_FLOW_HOME, or absolute).",
)
project_name: str = Field(default="deerflow", description="Project namespace for projects/<name>/ pages.")
auto_ingest_enabled: bool = Field(default=False, description="Auto-sediment conversations in the background (phase 3).")
auto_ingest_min_chars: int = Field(default=300, description="Minimum characters before auto-ingest considers a turn.")
max_thread_messages: int = Field(default=80, description="Maximum thread messages to scan when capturing.")
max_source_items: int = Field(default=20, description="Maximum source items recorded per note.")
rag_enabled: bool = Field(default=False, description="Enable RAG injection before agent runs (phase 2).")
rag_limit: int = Field(default=5, description="Maximum knowledge snippets injected per query.")
export: KnowledgeExportConfig = Field(default_factory=KnowledgeExportConfig, description="Graph export configuration.")
query: KnowledgeQueryConfig = Field(default_factory=KnowledgeQueryConfig, description="Layered query configuration.")
embedding: KnowledgeEmbeddingConfig = Field(default_factory=KnowledgeEmbeddingConfig, description="Embedding configuration.")
@@ -0,0 +1,41 @@
"""Product knowledge base module (obsidian-wiki compatible).
Self-contained: knowledge ingestion, storage (DB mirror + Markdown vault),
search and graph export all live here. The Gateway only depends on
:class:`KnowledgeService` via :func:`make_knowledge_service`.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
from deerflow.knowledge.repository import KnowledgeRepository
from deerflow.knowledge.service import EmptyThreadError, KnowledgeService
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.config.knowledge_config import KnowledgeConfig
__all__ = [
"EmptyThreadError",
"KnowledgeRepository",
"KnowledgeService",
"ObsidianWikiAdapter",
"make_knowledge_service",
]
def make_knowledge_service(
session_factory: async_sessionmaker[AsyncSession] | None,
config: KnowledgeConfig,
) -> KnowledgeService | None:
"""Build the knowledge service.
Returns ``None`` when persistence is memory-only (no session factory) or
the feature is disabled — the router then responds 503.
"""
if session_factory is None or not config.enabled:
return None
return KnowledgeService(KnowledgeRepository(session_factory), config)
@@ -0,0 +1,190 @@
"""obsidian-wiki ``wiki-export`` compatible graph generation.
Builds a knowledge graph from the notes (nodes) plus their wikilinks and shared
tags (edges) and writes the four native artifacts into ``<vault>/wiki-export/``:
``graph.json``, ``graph.graphml``, ``cypher.txt`` and a self-contained
``graph.html`` (no external CDN, safe to iframe offline).
"""
from __future__ import annotations
import html
import json
from pathlib import Path
from typing import Any
from deerflow.knowledge.markdown_vault import extract_wikilinks
def _vault_key(vault_path: str | None) -> str | None:
if not vault_path:
return None
key = vault_path.replace("\\", "/")
if key.endswith(".md"):
key = key[:-3]
return key
def build_graph(notes: list[dict[str, Any]]) -> dict[str, Any]:
"""Return ``{nodes, edges}`` from note dicts (id, title, tags, vault_path, content_md)."""
nodes = []
by_key: dict[str, str] = {}
for n in notes:
nodes.append({"id": n["id"], "type": "note", "label": n.get("title") or n["id"], "category": n.get("source_type")})
key = _vault_key(n.get("vault_path"))
if key:
by_key[key] = n["id"]
edges: list[dict[str, Any]] = []
edge_seen: set[tuple[str, str, str]] = set()
def add_edge(a: str, b: str, label: str, weight: float) -> None:
if a == b:
return
key = (min(a, b), max(a, b), label)
if key in edge_seen:
return
edge_seen.add(key)
edges.append({"id": f"rel_{len(edges) + 1}", "source": a, "target": b, "label": label, "weight": weight})
# Wikilink edges.
for n in notes:
for target in extract_wikilinks(n.get("content_md") or ""):
tid = by_key.get(target.replace("\\", "/").removesuffix(".md"))
if tid:
add_edge(n["id"], tid, "links", 2)
# Shared-tag edges.
tag_to_notes: dict[str, list[str]] = {}
for n in notes:
for tag in n.get("tags") or []:
tag_to_notes.setdefault(tag.lower(), []).append(n["id"])
for tag, ids in tag_to_notes.items():
if len(ids) < 2 or len(ids) > 50: # skip ubiquitous tags (e.g. knowledge-base)
continue
for i in range(len(ids)):
for j in range(i + 1, len(ids)):
add_edge(ids[i], ids[j], "related", 1)
return {"nodes": nodes, "edges": edges}
def to_graphml(graph: dict[str, Any]) -> str:
lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<graphml xmlns="http://graphml.graphdrawing.org/xmlns">',
' <key id="label" for="node" attr.name="label" attr.type="string"/>',
' <key id="elabel" for="edge" attr.name="label" attr.type="string"/>',
' <graph edgedefault="undirected">',
]
for n in graph["nodes"]:
lines.append(f' <node id="{html.escape(str(n["id"]))}"><data key="label">{html.escape(str(n["label"]))}</data></node>')
for e in graph["edges"]:
lines.append(
f' <edge id="{html.escape(str(e["id"]))}" source="{html.escape(str(e["source"]))}" '
f'target="{html.escape(str(e["target"]))}"><data key="elabel">{html.escape(str(e["label"]))}</data></edge>'
)
lines += [" </graph>", "</graphml>"]
return "\n".join(lines)
def to_cypher(graph: dict[str, Any]) -> str:
lines = []
for n in graph["nodes"]:
label = str(n["label"]).replace("'", "\\'")
lines.append(f"CREATE (:Note {{id:'{n['id']}', title:'{label}'}});")
for e in graph["edges"]:
rel = str(e["label"]).upper()
lines.append(f"MATCH (a:Note {{id:'{e['source']}'}}),(b:Note {{id:'{e['target']}'}}) CREATE (a)-[:{rel} {{weight:{e['weight']}}}]->(b);")
return "\n".join(lines)
def to_html(graph: dict[str, Any]) -> str:
"""Self-contained force-directed graph (canvas + vanilla JS, no CDN)."""
data_json = json.dumps(graph, ensure_ascii=False)
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
def export_all(
vault_root: Path | str,
notes: list[dict[str, Any]],
*,
write_html: bool = True,
write_json: bool = True,
) -> dict[str, Any]:
"""Generate graph artifacts under ``<vault>/wiki-export/`` and return stats."""
out_dir = Path(vault_root).resolve() / "wiki-export"
out_dir.mkdir(parents=True, exist_ok=True)
graph = build_graph(notes)
written: list[str] = []
if write_json:
(out_dir / "graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8")
written.append("graph.json")
(out_dir / "graph.graphml").write_text(to_graphml(graph), encoding="utf-8")
written.append("graph.graphml")
(out_dir / "cypher.txt").write_text(to_cypher(graph), encoding="utf-8")
written.append("cypher.txt")
if write_html:
(out_dir / "graph.html").write_text(to_html(graph), encoding="utf-8")
written.append("graph.html")
return {
"files": written,
"stats": {"nodes": len(graph["nodes"]), "edges": len(graph["edges"])},
"graph": graph,
}
_HTML_TEMPLATE = """<!DOCTYPE html>
<html lang="zh">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>Knowledge Graph</title>
<style>
html,body{margin:0;height:100%;background:#0b1020;color:#e7ecff;font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
#hud{position:fixed;top:10px;left:12px;font-size:12px;opacity:.8;z-index:2}
#tip{position:fixed;padding:4px 8px;background:#111a36;border:1px solid #2a3a66;border-radius:4px;font-size:12px;pointer-events:none;display:none;z-index:3}
canvas{display:block}
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.6}
</style>
</head>
<body>
<div id="hud"></div>
<div id="tip"></div>
<canvas id="c"></canvas>
<script>
const GRAPH = __GRAPH_DATA__;
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
let W=0,H=0;
function resize(){W=canvas.width=innerWidth;H=canvas.height=innerHeight;}
addEventListener('resize',resize);resize();
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id=\"empty\">暂无知识图谱数据</div>');}
const idx={};GRAPH.nodes.forEach((n,i)=>{idx[n.id]=n;n.x=W/2+Math.cos(i)*200*Math.random()+(Math.random()-.5)*200;n.y=H/2+Math.sin(i)*200*Math.random()+(Math.random()-.5)*200;n.vx=0;n.vy=0;});
const edges=GRAPH.edges.map(e=>({s:idx[e.source],t:idx[e.target],w:e.w||e.weight||1})).filter(e=>e.s&&e.t);
let cam={x:0,y:0,k:1},drag=null,pan=null;
function step(){
for(const n of GRAPH.nodes){for(const m of GRAPH.nodes){if(n===m)continue;let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=1200/d2;n.vx+=dx*f*0.0006;n.vy+=dy*f*0.0006;}}
for(const e of edges){let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.sqrt(dx*dx+dy*dy)||1,f=(d-110)*0.01;e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;}
for(const n of GRAPH.nodes){n.vx+=(W/2-n.x)*0.0008;n.vy+=(H/2-n.y)*0.0008;n.x+=n.vx*=0.85;n.y+=n.vy*=0.85;}
}
function draw(){
ctx.setTransform(cam.k,0,0,cam.k,cam.x,cam.y);ctx.clearRect(-cam.x/cam.k,-cam.y/cam.k,W/cam.k,H/cam.k);
ctx.strokeStyle='rgba(120,150,255,.25)';
for(const e of edges){ctx.lineWidth=e.w*0.6;ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();}
for(const n of GRAPH.nodes){const r=6;ctx.fillStyle='#5b8cff';ctx.beginPath();ctx.arc(n.x,n.y,r,0,7);ctx.fill();
ctx.fillStyle='rgba(231,236,255,.85)';ctx.font='11px system-ui';ctx.fillText((n.label||'').slice(0,18),n.x+9,n.y+3);}
}
function loop(){step();draw();requestAnimationFrame(loop);}loop();
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
canvas.addEventListener('mousedown',ev=>{const p=toWorld(ev.clientX,ev.clientY);let hit=null;
for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hit=n;break;}}if(hit)drag=hit;else pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};});
addEventListener('mousemove',ev=>{const p=toWorld(ev.clientX,ev.clientY);if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;}else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
let hov=null;for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hov=n;break;}}
if(hov){tip.style.display='block';tip.style.left=(ev.clientX+12)+'px';tip.style.top=(ev.clientY+12)+'px';tip.textContent=hov.label;}else{tip.style.display='none';}});
addEventListener('mouseup',()=>{drag=null;pan=null;});
canvas.addEventListener('wheel',ev=>{ev.preventDefault();const s=ev.deltaY<0?1.1:0.9;cam.k*=s;cam.x=ev.clientX-(ev.clientX-cam.x)*s;cam.y=ev.clientY-(ev.clientY-cam.y)*s;},{passive:false});
</script>
</body>
</html>"""
@@ -0,0 +1,250 @@
"""Extract knowledge notes from threads / searches / tool results.
First-release strategy is rule-based (wiki-capture aligned, §11 of the dev doc):
keep high-value content (user questions, final assistant answers, cited
sources), drop noise (empty/greeting/intermediate reasoning), and write a
*declarative* note rather than a chat transcript. An LLM extractor can later
replace :func:`extract_thread` while keeping the same :class:`NoteDraft` output.
"""
from __future__ import annotations
import re
from typing import Any
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
# Greeting / low-value lines filtered from extracted turns.
_GREETING_RE = re.compile(r"^(你好|您好|hi|hello|hey|thanks?|谢谢|好的|ok|okay)[\s!!。.,,~]*$", re.IGNORECASE)
_WS_RE = re.compile(r"\s+")
def _one_line(text: str, limit: int = 300) -> str:
"""Collapse whitespace and clip ``text`` to a single line."""
collapsed = _WS_RE.sub(" ", text or "").strip()
return collapsed[:limit]
def message_text(msg: dict[str, Any]) -> str:
"""Return the plain-text content of a serialized LangChain message."""
content = msg.get("content")
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
if block.get("type") in (None, "text") and isinstance(block.get("text"), str):
parts.append(block["text"])
return "\n".join(p for p in parts if p).strip()
return ""
def _msg_type(msg: dict[str, Any]) -> str:
return str(msg.get("type") or msg.get("role") or "").lower()
def split_turns(messages: list[dict[str, Any]], *, max_messages: int = 80) -> list[dict[str, str]]:
"""Split messages into ``{user, ai}`` turns keyed by human messages.
For each turn we keep the last substantial AI message (with text body).
"""
msgs = messages[-max_messages:] if max_messages and len(messages) > max_messages else list(messages)
turns: list[dict[str, str]] = []
current: dict[str, str] | None = None
for msg in msgs:
mtype = _msg_type(msg)
text = message_text(msg)
if mtype == "human":
if current and (current.get("user") or current.get("ai")):
turns.append(current)
current = {"user": text, "ai": ""}
elif mtype == "ai" and text:
if current is None:
current = {"user": "", "ai": text}
else:
current["ai"] = text # last substantial AI message wins
if current and (current.get("user") or current.get("ai")):
turns.append(current)
# Drop turns that are pure greetings with no answer.
cleaned = []
for t in turns:
user = (t.get("user") or "").strip()
ai = (t.get("ai") or "").strip()
if not ai and _GREETING_RE.match(user):
continue
if not user and not ai:
continue
cleaned.append({"user": user, "ai": ai})
return cleaned
def _auto_title(thread_title: str | None, turns: list[dict[str, str]]) -> str:
if thread_title and thread_title.strip() and thread_title.strip().lower() != "untitled":
return thread_title.strip()[:200]
for t in turns:
if t.get("user"):
first = t["user"].splitlines()[0].strip()
return (first[:40] + ("…" if len(first) > 40 else "")) or "知识笔记"
return "知识笔记"
def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str:
"""Simple rule-based summary: the first answer, clipped."""
for t in turns:
if t.get("ai"):
text = re.sub(r"\s+", " ", t["ai"]).strip()
return text[:limit] + ("…" if len(text) > limit else "")
for t in turns:
if t.get("user"):
text = re.sub(r"\s+", " ", t["user"]).strip()
return text[:limit] + ("…" if len(text) > limit else "")
return ""
def related_links(project_name: str | None) -> list[str]:
"""At least two wikilinks per note (wiki-capture requirement)."""
links = ["[[index]]", "[[_meta/taxonomy]]"]
if project_name:
links.append(f"[[projects/{project_name}/synthesis/product-knowledge-base]]")
return links
# Backwards-compatible internal alias.
_related_links = related_links
def _build_body(
*,
title: str,
turns: list[dict[str, str]],
mode: str,
project_name: str | None,
sources: list[SourceDraft],
) -> str:
"""Render the wiki-capture 5-section declarative body."""
full = mode == "full"
answer_clip = 4000 if full else 800
context_lines = []
for t in turns:
if t.get("user"):
q = re.sub(r"\s+", " ", t["user"]).strip()
context_lines.append(f"- {q[:300]}")
context = "\n".join(context_lines[:8]) or "- 来自一次对话沉淀。^[inferred]"
finding_lines = []
for i, t in enumerate(turns, 1):
if t.get("ai"):
ans = t["ai"].strip()
ans = ans if full else (ans[:answer_clip] + ("…" if len(ans) > answer_clip else ""))
finding_lines.append(f"### 结论 {i}\n\n{ans}")
finding = "\n\n".join(finding_lines) or "_暂无可提炼的结论。_"
reasoning = "本页由对话自动沉淀而成,结论来自助手最终回答;过程性推理已省略。判断与归纳部分标记为 ^[inferred]。"
implications = "- 后续可在相关问答中参考本页知识。\n- 如内容过期或冲突,请编辑或归档。"
related = "\n".join(f"- {link}" for link in _related_links(project_name))
if sources:
related += "\n\n### Sources\n\n"
for s in sources[:20]:
label = s.title or s.url or "来源"
related += f"- {label}" + (f" — {s.url}" if s.url else "") + "\n"
return (
f"# {title}\n\n"
f"## Context\n\n{context}\n\n"
f"## Finding / Decision\n\n{finding}\n\n"
f"## Reasoning\n\n{reasoning}\n\n"
f"## Implications\n\n{implications}\n\n"
f"## Related\n\n{related}\n"
)
def extract_thread(
messages: list[dict[str, Any]],
*,
thread_id: str,
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "deerflow",
sources: list[SourceDraft] | None = None,
max_messages: int = 80,
) -> NoteDraft | None:
"""Build a :class:`NoteDraft` from thread messages, or ``None`` if empty."""
turns = split_turns(messages, max_messages=max_messages)
if not turns:
return None
final_title = (title or "").strip() or _auto_title(thread_title, turns)
summary = _summary(turns)
sources = sources or []
body = _build_body(title=final_title, turns=turns, mode=mode, project_name=project_name, sources=sources)
base_tags = list(tags or [])
for default in (project_name or "deerflow", "knowledge-base"):
if default and default not in base_tags:
base_tags.append(default)
return NoteDraft(
title=final_title,
summary=summary,
content_md=body,
category="synthesis",
source_type="thread",
source_id=thread_id,
tags=base_tags,
confidence=0.6,
related=_related_links(project_name),
sources=sources,
source_key=f"thread:{thread_id}",
)
def extract_search(
query: str,
sources: list[SourceDraft],
*,
thread_id: str | None = None,
project_name: str | None = "deerflow",
title: str | None = None,
tags: list[str] | None = None,
) -> NoteDraft:
"""Build a reference-category :class:`NoteDraft` from search/tool results."""
final_title = (title or "").strip() or f"检索:{query[:40]}"
body_lines = [f"# {final_title}\n", "## Context\n", f"- 检索关键词:{query}\n", "## Finding / Decision\n"]
for s in sources:
label = s.title or s.url or "来源"
line = f"- **{label}**"
if s.url:
line += f" — {s.url}"
if s.snippet:
line += "\n - " + _one_line(s.snippet)
body_lines.append(line)
body_lines += [
"\n## Reasoning\n",
"本页汇总外部检索来源,供后续问答引用。来源可信度需结合原文判断。^[ambiguous]\n",
"## Implications\n",
"- 引用前请核对原始来源。\n",
"## Related\n",
*[f"- {link}" for link in _related_links(project_name)],
]
base_tags = list(tags or [])
for default in (project_name or "deerflow", "knowledge-base", "reference"):
if default and default not in base_tags:
base_tags.append(default)
return NoteDraft(
title=final_title,
summary=f"检索“{query}”的来源汇总,共 {len(sources)} 条。",
content_md="\n".join(body_lines),
category="references",
source_type="search",
source_id=f"search:{query[:120]}",
tags=base_tags,
confidence=0.4,
related=_related_links(project_name),
sources=sources,
source_key=None,
)
@@ -0,0 +1,74 @@
""".manifest.json read/write for the vault.
The manifest records, per page, the source key, content hash, vault path and
update time — so batch ingest can skip already-sedimented threads (delta/dedup)
exactly like obsidian-wiki's native ``.manifest.json``.
"""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from typing import Any
def content_hash(text: str) -> str:
"""Stable sha256 hex digest of ``text`` (used for dedup)."""
return hashlib.sha256((text or "").encode("utf-8")).hexdigest()
class Manifest:
"""JSON manifest stored at ``<vault>/.manifest.json``."""
def __init__(self, root: Path | str) -> None:
self.path = Path(root).resolve() / ".manifest.json"
def load(self) -> dict[str, Any]:
if not self.path.exists():
return {"version": 1, "updated": None, "pages": {}, "sources": {}}
try:
data = json.loads(self.path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return {"version": 1, "updated": None, "pages": {}, "sources": {}}
data.setdefault("pages", {})
data.setdefault("sources", {})
return data
def save(self, data: dict[str, Any]) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
def record_page(
self,
*,
note_id: str,
vault_path: str,
source_key: str | None,
hash_value: str,
updated: str,
) -> None:
data = self.load()
data["pages"][note_id] = {
"vault_path": vault_path,
"source": source_key,
"hash": hash_value,
"updated": updated,
}
if source_key:
data["sources"][source_key] = {"note_id": note_id, "hash": hash_value, "updated": updated}
data["updated"] = updated
self.save(data)
def remove_page(self, note_id: str) -> None:
data = self.load()
page = data["pages"].pop(note_id, None)
if page and page.get("source"):
data["sources"].pop(page["source"], None)
self.save(data)
def has_source(self, source_key: str) -> bool:
return source_key in self.load().get("sources", {})
def get_source(self, source_key: str) -> dict[str, Any] | None:
return self.load().get("sources", {}).get(source_key)
@@ -0,0 +1,196 @@
"""obsidian-wiki compatible Markdown vault writer/reader.
Responsibilities (dev doc §10 ``markdown_vault.py``):
- Generate obsidian-wiki compatible frontmatter.
- Produce safe, path-traversal-proof file names/paths.
- Write/update/read Markdown note files.
- Parse frontmatter + extract ``[[wikilinks]]``.
The vault layout follows §8 of the dev doc and stays compatible with
``ar9av/obsidian-wiki`` (concepts/ references/ synthesis/ journal/ projects/…,
plus index.md / log.md / hot.md / .manifest.json / _meta/taxonomy.md /
wiki-export/).
"""
from __future__ import annotations
import re
import unicodedata
from datetime import datetime
from pathlib import Path
from typing import Any
import yaml
_WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")
_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?(.*)$", re.DOTALL)
_UNSAFE_CHARS_RE = re.compile(r"[^\w一-鿿-]+")
# Top-level vault directories that must exist (obsidian-wiki native layout).
_VAULT_DIRS = ("concepts", "entities", "skills", "references", "synthesis", "journal", "projects", "_meta", "wiki-export")
def slugify(title: str, *, max_len: int = 60) -> str:
"""Return a filesystem-safe slug (ASCII + CJK), never empty."""
text = unicodedata.normalize("NFKC", (title or "").strip()).lower()
text = text.replace(" ", "-")
text = _UNSAFE_CHARS_RE.sub("-", text).strip("-")
text = re.sub(r"-{2,}", "-", text)
if len(text) > max_len:
text = text[:max_len].rstrip("-")
return text or "note"
def extract_wikilinks(text: str) -> list[str]:
"""Return the targets of every ``[[wikilink]]`` in ``text`` (de-duplicated)."""
seen: list[str] = []
for match in _WIKILINK_RE.finditer(text or ""):
target = match.group(1).strip()
if target and target not in seen:
seen.append(target)
return seen
class VaultPathError(ValueError):
"""Raised when a requested vault path escapes the vault root."""
class MarkdownVault:
"""Reads/writes Markdown files inside a single vault root directory."""
def __init__(self, root: Path | str) -> None:
self.root = Path(root).resolve()
# -- path safety ------------------------------------------------------
def resolve(self, rel_path: str) -> Path:
"""Resolve ``rel_path`` under the vault root, blocking traversal."""
rel = str(rel_path).replace("\\", "/").lstrip("/")
candidate = (self.root / rel).resolve()
try:
candidate.relative_to(self.root)
except ValueError as exc:
raise VaultPathError(f"Path escapes vault root: {rel_path!r}") from exc
return candidate
def note_relpath(self, *, category: str, project_name: str | None, slug: str, note_id: str) -> str:
"""Build a vault-relative path for a note.
Project knowledge goes under ``projects/<project>/<category>/`` per
wiki-capture; a short id suffix guarantees uniqueness.
"""
from deerflow.knowledge.skill_templates import category_to_dir
cat_dir = category_to_dir(category)
safe_slug = slugify(slug)
suffix = note_id.split("_")[-1][:8]
filename = f"{safe_slug}-{suffix}.md"
if project_name:
return f"projects/{slugify(project_name)}/{cat_dir}/{filename}"
return f"{cat_dir}/{filename}"
# -- scaffold ---------------------------------------------------------
def ensure_scaffold(self, *, project_name: str | None = None) -> None:
"""Create the vault directory tree + base files if missing."""
self.root.mkdir(parents=True, exist_ok=True)
for d in _VAULT_DIRS:
(self.root / d).mkdir(parents=True, exist_ok=True)
if project_name:
base = self.root / "projects" / slugify(project_name)
for d in ("concepts", "references", "synthesis", "journal"):
(base / d).mkdir(parents=True, exist_ok=True)
# Base vault files.
index = self.root / "index.md"
if not index.exists():
index.write_text("# Knowledge Index\n\n_页面索引。每次写入新页面后自动更新。_\n\n", encoding="utf-8")
log = self.root / "log.md"
if not log.exists():
log.write_text("# Activity Log\n\n_CAPTURE / INGEST / QUERY / EXPORT 记录。_\n\n", encoding="utf-8")
hot = self.root / "hot.md"
if not hot.exists():
hot.write_text("# Hot\n\n_近期活动摘要。_\n\n", encoding="utf-8")
taxonomy = self.root / "_meta" / "taxonomy.md"
if not taxonomy.exists():
taxonomy.write_text(
"# Taxonomy\n\n- synthesis — 多步骤分析、方案、结论\n- concepts — 概念、框架、模型\n"
"- references — 外部来源、搜索结果、文章资料\n- decision — 架构或设计决策\n- journal — 完整会话摘要\n",
encoding="utf-8",
)
# -- frontmatter ------------------------------------------------------
@staticmethod
def build_frontmatter(
*,
note_id: str,
title: str,
category: str,
tags: list[str],
sources: list[str],
summary: str,
created: str,
updated: str,
base_confidence: float,
lifecycle: str,
provenance: dict[str, float] | None = None,
relationships: list[dict[str, str]] | None = None,
) -> dict[str, Any]:
"""Build an obsidian-wiki / wiki-capture compatible frontmatter dict."""
created_date = (created or "")[:10]
return {
"id": note_id,
"title": title,
"category": category,
"tags": list(tags or []),
"sources": list(sources or []),
"created": created,
"updated": updated,
"summary": summary or "",
"provenance": provenance or {"extracted": 0.7, "inferred": 0.2, "ambiguous": 0.1},
"base_confidence": round(float(base_confidence or 0.0), 2),
"lifecycle": lifecycle,
"lifecycle_changed": created_date,
"relationships": relationships or [],
}
# -- IO ---------------------------------------------------------------
def write_note(self, rel_path: str, frontmatter: dict[str, Any], body: str) -> Path:
"""Write ``frontmatter`` + ``body`` to ``rel_path`` (creates dirs)."""
path = self.resolve(rel_path)
path.parent.mkdir(parents=True, exist_ok=True)
fm = yaml.safe_dump(frontmatter, allow_unicode=True, sort_keys=False, default_flow_style=False)
path.write_text(f"---\n{fm}---\n\n{body.strip()}\n", encoding="utf-8")
return path
def read_note(self, rel_path: str) -> tuple[dict[str, Any], str] | None:
"""Return ``(frontmatter, body)`` for a note, or ``None`` if missing."""
path = self.resolve(rel_path)
if not path.exists():
return None
return self.parse(path.read_text(encoding="utf-8"))
@staticmethod
def parse(text: str) -> tuple[dict[str, Any], str]:
"""Split raw Markdown into ``(frontmatter dict, body)``."""
match = _FRONTMATTER_RE.match(text or "")
if not match:
return {}, text or ""
try:
fm = yaml.safe_load(match.group(1)) or {}
except yaml.YAMLError:
fm = {}
if not isinstance(fm, dict):
fm = {}
return fm, match.group(2) or ""
def delete_note(self, rel_path: str) -> None:
"""Remove a note file if present (used on hard delete; archive keeps it)."""
path = self.resolve(rel_path)
if path.exists():
path.unlink()
def now_iso_with_tz(dt: datetime) -> str:
"""ISO-8601 string with timezone offset (frontmatter ``created``/``updated``)."""
if dt.tzinfo is None:
return dt.isoformat()
return dt.isoformat(timespec="seconds")
@@ -0,0 +1,79 @@
"""ORM models for the product knowledge base.
Tables mirror the obsidian-wiki vault into the database so the product can
list, filter, search and audit notes quickly without re-parsing Markdown. The
vault Markdown remains the source-of-truth for ``content_md`` / frontmatter /
graph; these rows are an index + audit layer.
The knowledge base is **global / shared** — there is no ``user_id`` isolation.
Writes record ``created_by`` / ``updated_by`` for auditing only.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, ForeignKey, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class KnowledgeNoteRow(Base):
"""Knowledge note main table (mirrors ``knowledge_notes``)."""
__tablename__ = "knowledge_notes"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
title: Mapped[str] = mapped_column(String(512))
summary: Mapped[str | None] = mapped_column(Text)
content_md: Mapped[str] = mapped_column(PortableLongText())
vault_path: Mapped[str | None] = mapped_column(String(1024), index=True)
source_type: Mapped[str] = mapped_column(String(64), index=True)
source_id: Mapped[str | None] = mapped_column(String(256), index=True)
status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
confidence: Mapped[float] = mapped_column(Float, default=0.0)
created_by: Mapped[str | None] = mapped_column(String(128), index=True)
updated_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), index=True)
class KnowledgeSourceRow(Base):
"""Knowledge source table (mirrors ``knowledge_sources``)."""
__tablename__ = "knowledge_sources"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
source_type: Mapped[str] = mapped_column(String(64))
thread_id: Mapped[str | None] = mapped_column(String(128), index=True)
message_id: Mapped[str | None] = mapped_column(String(128))
tool_name: Mapped[str | None] = mapped_column(String(128))
title: Mapped[str | None] = mapped_column(String(512))
url: Mapped[str | None] = mapped_column(Text)
snippet: Mapped[str | None] = mapped_column(Text)
raw_json: Mapped[str | None] = mapped_column(PortableLongText())
created_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeTagRow(Base):
"""Tag table (mirrors ``knowledge_tags``)."""
__tablename__ = "knowledge_tags"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(128), unique=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeNoteTagRow(Base):
"""Note ↔ tag join table (mirrors ``knowledge_note_tags``)."""
__tablename__ = "knowledge_note_tags"
__table_args__ = (UniqueConstraint("note_id", "tag_id", name="uq_knowledge_note_tags"),)
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), primary_key=True)
tag_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_tags.id", ondelete="CASCADE"), primary_key=True)
@@ -0,0 +1,240 @@
"""ObsidianWikiAdapter — the obsidian-wiki vault facade.
Owns the Markdown vault: writes notes with wiki-capture frontmatter/body, keeps
``index.md`` / ``log.md`` / ``hot.md`` / ``.manifest.json`` in sync after every
write, and produces wiki-export graph artifacts. Output stays compatible with
``ar9av/obsidian-wiki`` without ever invoking its CLI.
This class is storage-only (no DB, no LLM). The :class:`KnowledgeService`
combines it with the repository + extractor.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
from deerflow.knowledge import exporter
from deerflow.knowledge.manifest import Manifest, content_hash
from deerflow.knowledge.markdown_vault import MarkdownVault
from deerflow.knowledge.schemas import NoteDraft
_INDEX_PAGES_HEADER = "## Pages"
_HOT_HEADER = "## Recent"
_HOT_MAX = 20
class ObsidianWikiAdapter:
"""Read/write an obsidian-wiki compatible vault."""
def __init__(self, vault_root: Path | str, *, project_name: str = "deerflow", export_html: bool = True, export_json: bool = True) -> None:
self.vault = MarkdownVault(vault_root)
self.root = self.vault.root
self.manifest = Manifest(self.root)
self.project_name = project_name
self.export_html = export_html
self.export_json = export_json
def ensure_ready(self) -> None:
self.vault.ensure_scaffold(project_name=self.project_name)
# -- writes -----------------------------------------------------------
def write_note(
self,
note_id: str,
draft: NoteDraft,
*,
created: str,
updated: str,
status: str,
) -> str:
"""Write the note file, sync vault metadata, return the vault path."""
self.ensure_ready()
rel = self.vault.note_relpath(category=draft.category, project_name=self.project_name, slug=draft.title, note_id=note_id)
sources_fm = self._frontmatter_sources(draft)
relationships = [{"target": link, "type": "related"} for link in (draft.related or [])[:8]]
frontmatter = self.vault.build_frontmatter(
note_id=note_id,
title=draft.title,
category=draft.category,
tags=draft.tags,
sources=sources_fm,
summary=draft.summary,
created=created,
updated=updated,
base_confidence=draft.confidence,
lifecycle=status,
relationships=relationships,
)
self.vault.write_note(rel, frontmatter, draft.content_md)
self.manifest.record_page(
note_id=note_id,
vault_path=rel,
source_key=draft.source_key,
hash_value=content_hash(draft.content_md),
updated=updated,
)
self._update_index(note_id, rel, draft.title, draft.summary)
self._append_log("CAPTURE" if draft.source_type in ("thread", "search", "tool") else "INGEST", note_id, draft.title, updated)
self._update_hot(draft.title, rel, updated)
return rel
def update_note_file(
self,
note_id: str,
vault_path: str | None,
*,
title: str,
content_md: str,
tags: list[str],
status: str,
summary: str,
category: str,
created: str,
updated: str,
) -> str:
"""Rewrite a note file in place (frontmatter refreshed), sync metadata."""
self.ensure_ready()
rel = vault_path
if not rel:
rel = self.vault.note_relpath(category=category, project_name=self.project_name, slug=title, note_id=note_id)
# Preserve original `sources`/`provenance`/`category` frontmatter when present.
existing = self.vault.read_note(rel)
sources_fm = existing[0].get("sources", []) if existing else []
provenance = existing[0].get("provenance") if existing else None
relationships = existing[0].get("relationships") if existing else []
effective_category = (existing[0].get("category") if existing else None) or category
frontmatter = self.vault.build_frontmatter(
note_id=note_id,
title=title,
category=effective_category,
tags=tags,
sources=sources_fm,
summary=summary,
created=created,
updated=updated,
base_confidence=float((existing[0].get("base_confidence") if existing else 0.0) or 0.0),
lifecycle=status,
provenance=provenance,
relationships=relationships,
)
self.vault.write_note(rel, frontmatter, content_md)
self.manifest.record_page(
note_id=note_id,
vault_path=rel,
source_key=self.manifest.load().get("pages", {}).get(note_id, {}).get("source"),
hash_value=content_hash(content_md),
updated=updated,
)
self._update_index(note_id, rel, title, summary)
self._append_log("UPDATE", note_id, title, updated)
self._update_hot(title, rel, updated)
return rel
def archive_note(self, note_id: str, vault_path: str | None, *, updated: str) -> None:
"""Mark a note archived in its frontmatter + log (soft delete)."""
if vault_path:
parsed = self.vault.read_note(vault_path)
if parsed:
fm, body = parsed
fm["lifecycle"] = "archived"
fm["updated"] = updated
self.vault.write_note(vault_path, fm, body)
self._remove_from_index(note_id)
self._append_log("ARCHIVE", note_id, "", updated)
def export_graph(self, notes: list[dict[str, Any]], *, updated: str) -> dict[str, Any]:
self.ensure_ready()
result = exporter.export_all(self.root, notes, write_html=self.export_html, write_json=self.export_json)
self._append_log("EXPORT", "graph", f"nodes={result['stats']['nodes']} edges={result['stats']['edges']}", updated)
return result
def export_file_path(self, filename: str) -> Path | None:
allowed = {"graph.json", "graph.graphml", "cypher.txt", "graph.html"}
if filename not in allowed:
return None
path = (self.root / "wiki-export" / filename).resolve()
try:
path.relative_to(self.root)
except ValueError:
return None
return path if path.exists() else None
# -- frontmatter source serialization ---------------------------------
@staticmethod
def _frontmatter_sources(draft: NoteDraft) -> list[str]:
out: list[str] = []
if draft.source_type == "thread" and draft.source_id:
out.append(f"thread:{draft.source_id}")
for s in draft.sources[:20]:
if s.url:
out.append(f"url:{s.url}")
elif s.thread_id:
out.append(f"thread:{s.thread_id}")
elif s.tool_name:
out.append(f"tool:{s.tool_name}")
# De-dup preserving order.
seen: set[str] = set()
deduped = []
for item in out:
if item not in seen:
seen.add(item)
deduped.append(item)
return deduped
# -- index.md ---------------------------------------------------------
def _index_path(self) -> Path:
return self.root / "index.md"
def _update_index(self, note_id: str, rel: str, title: str, summary: str | None) -> None:
path = self._index_path()
text = path.read_text(encoding="utf-8") if path.exists() else "# Knowledge Index\n\n"
link = rel[:-3] if rel.endswith(".md") else rel
summary_txt = (summary or "").strip().replace("\n", " ")
if len(summary_txt) > 120:
summary_txt = summary_txt[:120] + "…"
line = f"- [[{link}|{title}]]" + (f" — {summary_txt}" if summary_txt else "") + f" <!-- id:{note_id} -->"
lines = text.splitlines()
# Drop any existing line for this note id.
lines = [ln for ln in lines if f"<!-- id:{note_id} -->" not in ln]
if _INDEX_PAGES_HEADER not in text:
lines.append("")
lines.append(_INDEX_PAGES_HEADER)
lines.append("")
# Insert after the Pages header.
try:
insert_at = lines.index(_INDEX_PAGES_HEADER) + 1
except ValueError:
insert_at = len(lines)
lines.insert(insert_at, line)
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
def _remove_from_index(self, note_id: str) -> None:
path = self._index_path()
if not path.exists():
return
lines = [ln for ln in path.read_text(encoding="utf-8").splitlines() if f"<!-- id:{note_id} -->" not in ln]
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
# -- log.md -----------------------------------------------------------
def _append_log(self, action: str, note_id: str, label: str, updated: str) -> None:
path = self.root / "log.md"
text = path.read_text(encoding="utf-8") if path.exists() else "# Activity Log\n\n"
entry = f"- {updated} {action} {note_id}" + (f" — {label}" if label else "")
path.write_text(text.rstrip() + "\n" + entry + "\n", encoding="utf-8")
# -- hot.md -----------------------------------------------------------
def _update_hot(self, title: str, rel: str, updated: str) -> None:
path = self.root / "hot.md"
link = rel[:-3] if rel.endswith(".md") else rel
new_line = f"- {updated} [[{link}|{title}]]"
existing_lines: list[str] = []
if path.exists():
for ln in path.read_text(encoding="utf-8").splitlines():
if ln.startswith("- ") and "[[" in ln:
existing_lines.append(ln)
existing_lines = [ln for ln in existing_lines if f"[[{link}" not in ln]
existing_lines.insert(0, new_line)
existing_lines = existing_lines[:_HOT_MAX]
body = "# Hot\n\n_近期活动摘要。_\n\n" + _HOT_HEADER + "\n\n" + "\n".join(existing_lines) + "\n"
path.write_text(body, encoding="utf-8")
@@ -0,0 +1,315 @@
"""Database read/write for the knowledge base.
This layer never calls an LLM and never touches the Markdown vault — it only
persists/queries the relational mirror (notes, sources, tags). The vault and
extraction live in :mod:`deerflow.knowledge.markdown_vault` and
:mod:`deerflow.knowledge.extractor`; orchestration lives in the service.
"""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.knowledge.models import (
KnowledgeNoteRow,
KnowledgeNoteTagRow,
KnowledgeSourceRow,
KnowledgeTagRow,
)
from deerflow.knowledge.schemas import SourceDraft
def _new_id(prefix: str) -> str:
return f"{prefix}_{uuid.uuid4().hex[:24]}"
def _iso(value: Any) -> Any:
if isinstance(value, datetime):
return value.isoformat()
return value
class KnowledgeRepository:
"""SQLAlchemy-backed repository for knowledge notes/sources/tags."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# -- serialization ----------------------------------------------------
@staticmethod
def _note_to_dict(row: KnowledgeNoteRow) -> dict[str, Any]:
d = row.to_dict()
d["created_at"] = _iso(d.get("created_at"))
d["updated_at"] = _iso(d.get("updated_at"))
return d
@staticmethod
def _source_to_dict(row: KnowledgeSourceRow) -> dict[str, Any]:
d = row.to_dict()
d["created_at"] = _iso(d.get("created_at"))
return d
# -- tags -------------------------------------------------------------
async def _get_or_create_tags(self, session: AsyncSession, names: list[str]) -> list[KnowledgeTagRow]:
cleaned = []
seen: set[str] = set()
for raw in names or []:
name = (raw or "").strip()
if not name or name.lower() in seen:
continue
seen.add(name.lower())
cleaned.append(name)
if not cleaned:
return []
existing = {
r.name.lower(): r
for r in (await session.execute(select(KnowledgeTagRow).where(func.lower(KnowledgeTagRow.name).in_([n.lower() for n in cleaned])))).scalars()
}
result: list[KnowledgeTagRow] = []
for name in cleaned:
row = existing.get(name.lower())
if row is None:
row = KnowledgeTagRow(id=_new_id("kbtag"), name=name, created_at=datetime.now(UTC))
session.add(row)
await session.flush()
existing[name.lower()] = row
result.append(row)
return result
async def _tags_for(self, session: AsyncSession, note_id: str) -> list[str]:
stmt = (
select(KnowledgeTagRow.name)
.join(KnowledgeNoteTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
.where(KnowledgeNoteTagRow.note_id == note_id)
.order_by(KnowledgeTagRow.name)
)
return list((await session.execute(stmt)).scalars())
async def _tags_for_many(self, session: AsyncSession, note_ids: list[str]) -> dict[str, list[str]]:
if not note_ids:
return {}
stmt = (
select(KnowledgeNoteTagRow.note_id, KnowledgeTagRow.name)
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
.where(KnowledgeNoteTagRow.note_id.in_(note_ids))
.order_by(KnowledgeTagRow.name)
)
out: dict[str, list[str]] = {nid: [] for nid in note_ids}
for note_id, name in (await session.execute(stmt)).all():
out.setdefault(note_id, []).append(name)
return out
async def _replace_note_tags(self, session: AsyncSession, note_id: str, names: list[str]) -> None:
await session.execute(delete(KnowledgeNoteTagRow).where(KnowledgeNoteTagRow.note_id == note_id))
tags = await self._get_or_create_tags(session, names)
for tag in tags:
session.add(KnowledgeNoteTagRow(note_id=note_id, tag_id=tag.id))
# -- notes ------------------------------------------------------------
async def create_note(
self,
*,
title: str,
summary: str | None,
content_md: str,
source_type: str,
source_id: str | None,
status: str,
confidence: float = 0.0,
vault_path: str | None = None,
tags: list[str] | None = None,
created_by: str | None = None,
) -> dict[str, Any]:
note_id = _new_id("kb")
now = datetime.now(UTC)
row = KnowledgeNoteRow(
id=note_id,
title=title[:512],
summary=summary,
content_md=content_md,
vault_path=vault_path,
source_type=source_type,
source_id=source_id,
status=status,
confidence=confidence,
created_by=created_by,
updated_by=created_by,
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.flush()
await self._replace_note_tags(session, note_id, tags or [])
await session.commit()
await session.refresh(row)
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, note_id)
return d
async def set_vault_path(self, note_id: str, vault_path: str) -> None:
async with self._sf() as session:
row = await session.get(KnowledgeNoteRow, note_id)
if row is None:
return
row.vault_path = vault_path
await session.commit()
async def update_note(
self,
note_id: str,
*,
title: str | None = None,
summary: str | None = None,
content_md: str | None = None,
status: str | None = None,
tags: list[str] | None = None,
vault_path: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(KnowledgeNoteRow, note_id)
if row is None:
return None
if title is not None:
row.title = title[:512]
if summary is not None:
row.summary = summary
if content_md is not None:
row.content_md = content_md
if status is not None:
row.status = status
if vault_path is not None:
row.vault_path = vault_path
if updated_by is not None:
row.updated_by = updated_by
row.updated_at = datetime.now(UTC)
if tags is not None:
await self._replace_note_tags(session, note_id, tags)
await session.commit()
await session.refresh(row)
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, note_id)
return d
async def set_status(self, note_id: str, status: str, *, updated_by: str | None = None) -> dict[str, Any] | None:
return await self.update_note(note_id, status=status, updated_by=updated_by)
async def get_note(self, note_id: str, *, with_sources: bool = True) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(KnowledgeNoteRow, note_id)
if row is None:
return None
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, note_id)
if with_sources:
stmt = select(KnowledgeSourceRow).where(KnowledgeSourceRow.note_id == note_id).order_by(KnowledgeSourceRow.created_at)
d["sources"] = [self._source_to_dict(r) for r in (await session.execute(stmt)).scalars()]
return d
async def find_by_source(self, source_type: str, source_id: str) -> dict[str, Any] | None:
"""Return the first (non-archived) note for a (source_type, source_id)."""
async with self._sf() as session:
stmt = (
select(KnowledgeNoteRow)
.where(KnowledgeNoteRow.source_type == source_type, KnowledgeNoteRow.source_id == source_id)
.where(KnowledgeNoteRow.status != "archived")
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
return self._note_to_dict(row) if row else None
async def list_notes(
self,
*,
keyword: str | None = None,
tag: str | None = None,
source_type: str | None = None,
status: str | None = None,
include_archived: bool = False,
limit: int = 20,
offset: int = 0,
) -> tuple[list[dict[str, Any]], int]:
async with self._sf() as session:
stmt = select(KnowledgeNoteRow)
count_stmt = select(func.count()).select_from(KnowledgeNoteRow)
conditions = []
if status:
conditions.append(KnowledgeNoteRow.status == status)
elif not include_archived:
conditions.append(KnowledgeNoteRow.status != "archived")
if source_type:
conditions.append(KnowledgeNoteRow.source_type == source_type)
if keyword:
like = f"%{keyword}%"
conditions.append(or_(KnowledgeNoteRow.title.like(like), KnowledgeNoteRow.summary.like(like), KnowledgeNoteRow.content_md.like(like)))
if tag:
tag_subq = (
select(KnowledgeNoteTagRow.note_id)
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
.where(func.lower(KnowledgeTagRow.name) == tag.lower())
)
conditions.append(KnowledgeNoteRow.id.in_(tag_subq))
for cond in conditions:
stmt = stmt.where(cond)
count_stmt = count_stmt.where(cond)
total = (await session.execute(count_stmt)).scalar_one()
stmt = stmt.order_by(KnowledgeNoteRow.updated_at.desc()).limit(limit).offset(offset)
rows = list((await session.execute(stmt)).scalars())
ids = [r.id for r in rows]
tags_map = await self._tags_for_many(session, ids)
items = []
for r in rows:
d = self._note_to_dict(r)
d["tags"] = tags_map.get(r.id, [])
items.append(d)
return items, int(total)
async def list_all(self, *, include_archived: bool = False) -> list[dict[str, Any]]:
"""Return every note (light fields + tags) for graph export."""
async with self._sf() as session:
stmt = select(KnowledgeNoteRow)
if not include_archived:
stmt = stmt.where(KnowledgeNoteRow.status != "archived")
rows = list((await session.execute(stmt)).scalars())
tags_map = await self._tags_for_many(session, [r.id for r in rows])
out = []
for r in rows:
d = self._note_to_dict(r)
d["tags"] = tags_map.get(r.id, [])
out.append(d)
return out
# -- sources ----------------------------------------------------------
async def add_sources(self, note_id: str, sources: list[SourceDraft], *, created_by: str | None = None) -> None:
if not sources:
return
import json
async with self._sf() as session:
for src in sources:
session.add(
KnowledgeSourceRow(
id=_new_id("kbsrc"),
note_id=note_id,
source_type=src.source_type,
thread_id=src.thread_id,
message_id=src.message_id,
tool_name=src.tool_name,
title=(src.title or "")[:512] or None,
url=src.url,
snippet=src.snippet,
raw_json=json.dumps(src.raw, ensure_ascii=False) if src.raw else None,
created_by=created_by,
created_at=datetime.now(UTC),
)
)
await session.commit()
@@ -0,0 +1,71 @@
"""Internal domain models for the knowledge module.
These are plain dataclasses passed between the extractor, the vault writer and
the repository. The Gateway router defines its own Pydantic request/response
models on top of the dicts the service returns.
"""
from __future__ import annotations
from dataclasses import dataclass, field
# Allowed note source types (see product-knowledge-base-dev.md §7.1).
SOURCE_TYPES = ("thread", "search", "tool", "file", "manual", "report")
# Allowed lifecycle states.
STATUSES = ("draft", "approved", "archived")
# obsidian-wiki wiki-capture categories.
CATEGORIES = ("synthesis", "concepts", "references", "decision", "journal")
@dataclass
class SourceDraft:
"""A normalized knowledge source (search result, tool call, thread, …)."""
source_type: str
thread_id: str | None = None
message_id: str | None = None
tool_name: str | None = None
title: str | None = None
url: str | None = None
snippet: str | None = None
raw: dict | None = None
@dataclass
class NoteDraft:
"""A knowledge note ready to be persisted + written to the vault.
Produced by the extractor (from a thread / search / tool result) or built
directly for a manual note. ``content_md`` here is the *body* — the vault
writer prepends obsidian-wiki frontmatter.
"""
title: str
summary: str
content_md: str
category: str = "synthesis"
source_type: str = "manual"
source_id: str | None = None
status: str = "approved"
confidence: float = 0.0
tags: list[str] = field(default_factory=list)
key_points: list[str] = field(default_factory=list)
actions: list[str] = field(default_factory=list)
related: list[str] = field(default_factory=list)
sources: list[SourceDraft] = field(default_factory=list)
# Stable identity hint used for manifest dedup (e.g. ``thread:thread_id``).
source_key: str | None = None
def normalize_status(status: str | None) -> str:
"""Coerce an arbitrary status string to a valid lifecycle state."""
if status in STATUSES:
return status
return "approved"
def normalize_source_type(source_type: str | None) -> str:
"""Coerce an arbitrary source-type string to a valid value."""
if source_type in SOURCE_TYPES:
return source_type
return "manual"
@@ -0,0 +1,90 @@
"""Keyword / frontmatter / Markdown-content search (first release).
Vector and hybrid modes are reserved for phase 2 (see the dev doc §9.9 / §13);
for now ``vector`` and ``hybrid`` transparently fall back to keyword search so
the API contract is stable. Scoring weights title > tags > summary > body.
"""
from __future__ import annotations
import re
from typing import Any
from deerflow.knowledge.repository import KnowledgeRepository
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
def _tokens(text: str) -> list[str]:
return [t.lower() for t in _TOKEN_RE.findall(text or "")]
def _snippet(content: str, terms: list[str], *, width: int = 160) -> str:
body = re.sub(r"\s+", " ", content or "").strip()
low = body.lower()
for term in terms:
idx = low.find(term)
if idx >= 0:
start = max(0, idx - width // 3)
end = min(len(body), start + width)
prefix = "…" if start > 0 else ""
suffix = "…" if end < len(body) else ""
return f"{prefix}{body[start:end]}{suffix}"
return body[:width] + ("…" if len(body) > width else "")
async def search_notes(
repo: KnowledgeRepository,
query: str,
*,
mode: str = "keyword",
limit: int = 8,
candidate_cap: int = 500,
) -> list[dict[str, Any]]:
"""Score notes against ``query`` and return the top ``limit`` matches."""
terms = _tokens(query)
if not terms:
return []
# Pull a candidate window (non-archived). Phase 2 will replace this with
# an index; for the first release in-Python scoring is sufficient.
candidates, _total = await repo.list_notes(limit=candidate_cap, offset=0)
scored: list[tuple[float, dict[str, Any]]] = []
for note in candidates:
title = (note.get("title") or "").lower()
summary = (note.get("summary") or "").lower()
content = (note.get("content_md") or "").lower()
tags = " ".join(note.get("tags") or []).lower()
score = 0.0
for term in terms:
if term in title:
score += 3.0
if term in tags:
score += 2.0
if term in summary:
score += 2.0
score += min(content.count(term), 5) * 0.5
if score <= 0:
continue
scored.append((score, note))
scored.sort(key=lambda x: x[0], reverse=True)
max_score = scored[0][0] if scored else 1.0
results = []
for score, note in scored[:limit]:
results.append(
{
"note_id": note["id"],
"chunk_id": None,
"title": note.get("title"),
"summary": note.get("summary"),
"snippet": _snippet(note.get("content_md") or note.get("summary") or "", terms),
"score": round(score / max_score, 4),
"tags": note.get("tags") or [],
"source_type": note.get("source_type"),
"updated_at": note.get("updated_at"),
"sources": [],
}
)
return results
@@ -0,0 +1,282 @@
"""KnowledgeService — high-level orchestration for the knowledge base.
Ties together the repository (DB mirror), the :class:`ObsidianWikiAdapter`
(Markdown vault + index/log/hot/manifest + graph export) and the extractor
(thread/search → :class:`NoteDraft`). The Gateway router only talks to this
service; everything obsidian-wiki specific stays inside the ``knowledge``
module.
"""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.runtime_paths import runtime_home
from deerflow.knowledge import extractor
from deerflow.knowledge.markdown_vault import MarkdownVault
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
from deerflow.knowledge.repository import KnowledgeRepository
from deerflow.knowledge.schemas import NoteDraft, normalize_source_type, normalize_status
from deerflow.knowledge.search import search_notes
from deerflow.knowledge.sources import from_reference_batches, from_search_results
logger = logging.getLogger(__name__)
class EmptyThreadError(ValueError):
"""Raised when a thread has no sediment-able content."""
def _resolve_vault_dir(vault_dir: str) -> Path:
path = Path(vault_dir)
if path.is_absolute():
return path.resolve()
return (runtime_home() / vault_dir).resolve()
class KnowledgeService:
"""Application-facing knowledge base API."""
def __init__(self, repo: KnowledgeRepository, config: KnowledgeConfig) -> None:
self.repo = repo
self.config = config
self.project_name = config.project_name
vault_root = _resolve_vault_dir(config.vault_dir)
self.adapter = ObsidianWikiAdapter(
vault_root,
project_name=config.project_name,
export_html=config.export.graph_html_enabled,
export_json=config.export.graph_json_enabled,
)
try:
self.adapter.ensure_ready()
except Exception: # pragma: no cover - vault init best-effort
logger.exception("Failed to initialize knowledge vault at %s", vault_root)
# -- internal helpers -------------------------------------------------
async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]:
"""Create a DB row, write the vault file, attach sources, return note."""
note = await self.repo.create_note(
title=draft.title,
summary=draft.summary,
content_md=draft.content_md,
source_type=draft.source_type,
source_id=draft.source_id,
status=draft.status,
confidence=draft.confidence,
tags=draft.tags,
created_by=created_by,
)
note_id = note["id"]
try:
rel = self.adapter.write_note(note_id, draft, created=note["created_at"], updated=note["updated_at"], status=draft.status)
await self.repo.set_vault_path(note_id, rel)
note["vault_path"] = rel
except Exception:
logger.exception("Failed to write knowledge note %s to vault", note_id)
if draft.sources:
await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
return await self.repo.get_note(note_id) or note
# -- capture ----------------------------------------------------------
async def capture_thread(
self,
*,
thread_id: str,
messages: list[dict[str, Any]],
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
status: str = "approved",
tags: list[str] | None = None,
include_sources: bool = True,
reference_batches: list[dict[str, Any]] | None = None,
created_by: str | None = None,
skip_existing: bool = False,
) -> dict[str, Any]:
"""Sediment a thread into a knowledge note.
Returns ``{"note": <note>, "created": bool}``. Raises
:class:`EmptyThreadError` when there is nothing worth keeping.
"""
if skip_existing:
existing = await self.repo.find_by_source("thread", thread_id)
if existing is None and self.adapter.manifest.has_source(f"thread:{thread_id}"):
existing = await self.repo.find_by_source("thread", thread_id)
if existing:
return {"note": existing, "created": False}
sources = []
if include_sources and reference_batches:
sources = from_reference_batches(reference_batches, thread_id=thread_id, max_items=self.config.max_source_items)
draft = extractor.extract_thread(
messages,
thread_id=thread_id,
thread_title=thread_title,
mode=mode,
title=title,
tags=tags,
project_name=self.project_name,
sources=sources,
max_messages=self.config.max_thread_messages,
)
if draft is None:
raise EmptyThreadError(f"Thread {thread_id} has no sediment-able content")
draft.status = normalize_status(status)
note = await self._persist_draft(draft, created_by=created_by)
return {"note": note, "created": True}
async def capture_search(
self,
*,
query: str,
results: list[dict[str, Any]] | None,
thread_id: str | None = None,
message_id: str | None = None,
title: str | None = None,
tags: list[str] | None = None,
status: str = "approved",
created_by: str | None = None,
) -> dict[str, Any]:
"""Sediment search/tool results into a reference note."""
sources = from_search_results(query, results, thread_id=thread_id, message_id=message_id, max_items=self.config.max_source_items)
draft = extractor.extract_search(query, sources, thread_id=thread_id, project_name=self.project_name, title=title, tags=tags)
draft.status = normalize_status(status)
return await self._persist_draft(draft, created_by=created_by)
async def create_manual_note(
self,
*,
title: str,
content_md: str,
tags: list[str] | None = None,
status: str = "approved",
summary: str | None = None,
created_by: str | None = None,
) -> dict[str, Any]:
"""Create a note directly from product-edited Markdown."""
# Strip any frontmatter the user may have pasted; vault writer adds its own.
_, body = MarkdownVault.parse(content_md)
body = body or content_md
base_tags = list(tags or [])
for default in (self.project_name, "knowledge-base"):
if default and default not in base_tags:
base_tags.append(default)
draft = NoteDraft(
title=title.strip() or "知识笔记",
summary=(summary or body.strip().splitlines()[0] if body.strip() else "")[:280],
content_md=body,
category="synthesis",
source_type="manual",
source_id=None,
status=normalize_status(status),
tags=base_tags,
confidence=0.5,
related=extractor.related_links(self.project_name),
)
return await self._persist_draft(draft, created_by=created_by)
# -- CRUD -------------------------------------------------------------
async def list_notes(
self,
*,
keyword: str | None = None,
tag: str | None = None,
source_type: str | None = None,
status: str | None = None,
limit: int = 20,
offset: int = 0,
) -> dict[str, Any]:
items, total = await self.repo.list_notes(
keyword=keyword,
tag=tag,
source_type=normalize_source_type(source_type) if source_type else None,
status=status,
limit=limit,
offset=offset,
)
return {"items": items, "total": total}
async def get_note(self, note_id: str) -> dict[str, Any] | None:
return await self.repo.get_note(note_id)
async def update_note(
self,
note_id: str,
*,
title: str | None = None,
content_md: str | None = None,
tags: list[str] | None = None,
status: str | None = None,
summary: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any] | None:
current = await self.repo.get_note(note_id, with_sources=False)
if current is None:
return None
body = content_md
if body is not None:
_, parsed = MarkdownVault.parse(body)
body = parsed or body
note = await self.repo.update_note(
note_id,
title=title,
summary=summary,
content_md=body,
status=normalize_status(status) if status else None,
tags=tags,
updated_by=updated_by,
)
if note is None:
return None
try:
rel = self.adapter.update_note_file(
note_id,
current.get("vault_path"),
title=note["title"],
content_md=note["content_md"],
tags=note.get("tags") or [],
status=note["status"],
summary=note.get("summary") or "",
category="synthesis",
created=note["created_at"],
updated=note["updated_at"],
)
if rel and rel != current.get("vault_path"):
await self.repo.set_vault_path(note_id, rel)
except Exception:
logger.exception("Failed to update knowledge note %s in vault", note_id)
return await self.repo.get_note(note_id)
async def delete_note(self, note_id: str, *, updated_by: str | None = None, hard: bool = False) -> bool:
"""Soft delete (archive) by default; hard delete removes the vault file."""
current = await self.repo.get_note(note_id, with_sources=False)
if current is None:
return False
note = await self.repo.set_status(note_id, "archived", updated_by=updated_by)
try:
self.adapter.archive_note(note_id, current.get("vault_path"), updated=(note or current)["updated_at"])
if hard and current.get("vault_path"):
self.adapter.vault.delete_note(current["vault_path"])
self.adapter.manifest.remove_page(note_id)
except Exception:
logger.exception("Failed to archive knowledge note %s in vault", note_id)
return True
# -- search & graph ---------------------------------------------------
async def search(self, query: str, *, mode: str = "keyword", limit: int = 8) -> list[dict[str, Any]]:
return await search_notes(self.repo, query, mode=mode, limit=limit)
async def export_graph(self) -> dict[str, Any]:
from deerflow.utils.time import now_iso
notes = await self.repo.list_all()
result = self.adapter.export_graph(notes, updated=now_iso())
return {"files": result["files"], "stats": result["stats"]}
def export_file_path(self, filename: str) -> Path | None:
return self.adapter.export_file_path(filename)
@@ -0,0 +1,51 @@
"""Built-in obsidian-wiki skill templates.
Mirrors the key intent of ``ar9av/obsidian-wiki`` skills so the backend can
write vault content that stays compatible with the original tooling without
shelling out to its CLI:
- ``wiki-capture`` — how a conversation becomes a *declarative* knowledge note
(frontmatter schema + Context/Finding/Reasoning/Implications/Related body).
- ``wiki-query`` — layered retrieval (hot.md → index.md → frontmatter → body).
- ``wiki-export`` — graph artifacts (graph.json / graph.html / graphml / cypher).
These are intentionally lightweight: the structures here are what
:mod:`deerflow.knowledge.markdown_vault` and :mod:`deerflow.knowledge.extractor`
produce, and what the first-release LLM prompt (future enhancement) should
target.
"""
from __future__ import annotations
# Category → vault sub-directory (wiki-capture page taxonomy, §8 of the dev doc).
CATEGORY_DIRS = {
"synthesis": "synthesis",
"concepts": "concepts",
"concept": "concepts",
"references": "references",
"reference": "references",
"source": "references",
"decision": "synthesis",
"journal": "journal",
"session": "journal",
}
# Body section headings (wiki-capture). Declarative knowledge, never a chat log.
BODY_SECTIONS = ("Context", "Finding / Decision", "Reasoning", "Implications", "Related")
# Inline provenance markers reused from wiki-capture.
INFERRED_MARKER = "^[inferred]"
AMBIGUOUS_MARKER = "^[ambiguous]"
WIKI_CAPTURE_GUIDANCE = """\
Capture *declarative* knowledge, not a chat transcript. Classify the note as one
of: synthesis, concept, source, decision, session. Prefer at least two
[[wikilinks]]. Mark inferred content with ^[inferred] and uncertain/conflicting
content with ^[ambiguous]. Project-specific knowledge belongs under
projects/<project-name>/...
"""
def category_to_dir(category: str | None) -> str:
"""Map a wiki-capture category to its vault sub-directory."""
return CATEGORY_DIRS.get((category or "").strip().lower(), "synthesis")
@@ -0,0 +1,82 @@
"""Normalize knowledge sources from threads / searches / tool results.
Source items come from a few shapes: backend reference-batches (built by
``deerflow.runtime.references``), raw search-result payloads, and tool-call
messages. They are all normalized into :class:`SourceDraft` so the repository
and frontend see one consistent structure.
"""
from __future__ import annotations
from typing import Any
from deerflow.knowledge.schemas import SourceDraft
def _clip(value: Any, limit: int = 2000) -> str | None:
if value is None:
return None
text = str(value).strip()
if not text:
return None
return text[:limit]
def from_reference_batches(batches: list[dict[str, Any]] | None, *, thread_id: str | None = None, max_items: int = 20) -> list[SourceDraft]:
"""Convert backend reference-batches into :class:`SourceDraft`."""
out: list[SourceDraft] = []
seen: set[str] = set()
for batch in batches or []:
msg_id = batch.get("assistant_message_id") or batch.get("id")
for src in batch.get("sources", []) or []:
if not isinstance(src, dict):
continue
url = _clip(src.get("url"), 1024)
title = _clip(src.get("title"), 512) or _clip(src.get("source"), 512) or "来源"
dedup_key = url or f"{title}:{_clip(src.get('snippet'), 80)}"
if dedup_key in seen:
continue
seen.add(dedup_key)
out.append(
SourceDraft(
source_type="search" if url else "tool",
thread_id=thread_id,
message_id=_clip(msg_id, 128),
tool_name=_clip(src.get("skillName"), 128),
title=title,
url=url,
snippet=_clip(src.get("snippet")),
raw=None,
)
)
if len(out) >= max_items:
return out
return out
def from_search_results(
query: str,
results: list[dict[str, Any]] | None,
*,
thread_id: str | None = None,
message_id: str | None = None,
max_items: int = 20,
) -> list[SourceDraft]:
"""Convert raw search results (title/url/snippet) into :class:`SourceDraft`."""
out: list[SourceDraft] = []
for item in (results or [])[:max_items]:
if not isinstance(item, dict):
continue
out.append(
SourceDraft(
source_type="search",
thread_id=thread_id,
message_id=message_id,
tool_name=_clip(item.get("tool_name") or item.get("tool"), 128),
title=_clip(item.get("title"), 512) or query[:200],
url=_clip(item.get("url"), 1024),
snippet=_clip(item.get("snippet") or item.get("content")),
raw={k: item.get(k) for k in ("score",) if item.get(k) is not None} or None,
)
)
return out
@@ -24,6 +24,12 @@ from deerflow.persistence.agents.model import (
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.knowledge.models import (
KnowledgeNoteRow,
KnowledgeNoteTagRow,
KnowledgeSourceRow,
KnowledgeTagRow,
)
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.notifications.model import NotificationRow
@@ -60,6 +66,10 @@ __all__ = [
"AgentRow",
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"KnowledgeNoteRow",
"KnowledgeNoteTagRow",
"KnowledgeSourceRow",
"KnowledgeTagRow",
"LlmCallMetricRow",
"NotificationRow",
"RecommendedQuestionRow",