初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,41 @@
"""Product knowledge base module (obsidian-wiki compatible).
Self-contained: knowledge ingestion, storage (DB mirror + Markdown vault),
search and graph export all live here. The Gateway only depends on
:class:`KnowledgeService` via :func:`make_knowledge_service`.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
from deerflow.knowledge.repository import KnowledgeRepository
from deerflow.knowledge.service import EmptyThreadError, KnowledgeService
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.config.knowledge_config import KnowledgeConfig
__all__ = [
"EmptyThreadError",
"KnowledgeRepository",
"KnowledgeService",
"ObsidianWikiAdapter",
"make_knowledge_service",
]
def make_knowledge_service(
session_factory: async_sessionmaker[AsyncSession] | None,
config: KnowledgeConfig,
) -> KnowledgeService | None:
"""Build the knowledge service.
Returns ``None`` when persistence is memory-only (no session factory) or
the feature is disabled — the router then responds 503.
"""
if session_factory is None or not config.enabled:
return None
return KnowledgeService(KnowledgeRepository(session_factory), config)
@@ -0,0 +1,125 @@
"""Background auto-sediment queue (phase 3).
A single asyncio worker drains a bounded queue of finished conversations and
sediments the worthwhile ones into the knowledge base — entirely off the chat
request path, so it never adds latency to a reply. Enqueue happens from
``KnowledgeRagMiddleware.after_agent`` when ``knowledge.auto_ingest_enabled``.
Value judgment is intentionally cheap (min assistant chars + dedup via
``skip_existing``); the LLM still distills the kept conversations via the normal
``capture_thread`` path.
"""
from __future__ import annotations
import asyncio
import contextlib
import logging
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
logger = logging.getLogger(__name__)
_QUEUE: AutoIngestQueue | None = None
@dataclass
class _QueueUser:
"""Minimal CurrentUser stand-in (only ``.id`` is needed for context)."""
id: str
@dataclass
class AutoIngestItem:
thread_id: str
user_id: str
messages: list[dict[str, Any]]
thread_title: str | None = None
reference_batches: list[dict[str, Any]] = field(default_factory=list)
class AutoIngestQueue:
"""Bounded background queue + single worker for auto-sedimentation."""
def __init__(self, service_provider: Callable[[], Any], *, min_chars: int = 300, maxsize: int = 256) -> None:
self._provider = service_provider
self._min_chars = min_chars
self._queue: asyncio.Queue[AutoIngestItem] = asyncio.Queue(maxsize=maxsize)
self._task: asyncio.Task | None = None
def start(self) -> None:
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._worker(), name="knowledge-auto-ingest")
logger.info("Knowledge auto-ingest worker started")
async def stop(self) -> None:
if self._task is not None:
self._task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._task
self._task = None
def enqueue(self, item: AutoIngestItem) -> None:
try:
self._queue.put_nowait(item)
except asyncio.QueueFull:
logger.warning("Knowledge auto-ingest queue full; dropping thread %s", item.thread_id)
def _worth_ingesting(self, item: AutoIngestItem) -> bool:
chars = 0
for m in item.messages:
if str(m.get("type") or m.get("role") or "").lower() == "ai":
content = m.get("content")
chars += len(content) if isinstance(content, str) else len(str(content or ""))
return chars >= self._min_chars
async def _process(self, item: AutoIngestItem) -> None:
service = self._provider()
if service is None:
return
if not self._worth_ingesting(item):
logger.debug("Auto-ingest skipped (below min chars) thread=%s", item.thread_id)
return
from deerflow.runtime.user_context import reset_current_user, set_current_user
token = set_current_user(_QueueUser(id=item.user_id))
try:
result = await service.capture_thread(
thread_id=item.thread_id,
messages=item.messages,
thread_title=item.thread_title,
status="draft", # auto-sedimented notes land as drafts for review
include_sources=True,
reference_batches=item.reference_batches,
created_by=item.user_id,
skip_existing=True,
)
if result.get("created"):
logger.info("Auto-ingested thread %s -> note %s", item.thread_id, result["note"]["id"])
except Exception:
logger.exception("Auto-ingest failed for thread %s", item.thread_id)
finally:
reset_current_user(token)
async def _worker(self) -> None:
while True:
item = await self._queue.get()
try:
await self._process(item)
except asyncio.CancelledError:
raise
except Exception:
logger.exception("Auto-ingest worker error")
finally:
self._queue.task_done()
def set_auto_ingest_queue(queue: AutoIngestQueue | None) -> None:
global _QUEUE
_QUEUE = queue
def get_auto_ingest_queue() -> AutoIngestQueue | None:
return _QUEUE
@@ -0,0 +1,60 @@
"""Markdown chunking for embedding/index (phase 2).
Splits a note body into overlapping chunks along Markdown structure (headings →
paragraphs) so each embedded chunk is semantically coherent and bounded in
size. Frontmatter is assumed already stripped by the caller.
"""
from __future__ import annotations
import re
_HEADING_RE = re.compile(r"^#{1,6}\s+", re.MULTILINE)
def _split_paragraphs(text: str) -> list[str]:
return [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
def chunk_markdown(content: str, *, max_chars: int = 800, overlap: int = 120) -> list[str]:
"""Return a list of text chunks for ``content``.
Sections are cut at Markdown headings; long sections are further split into
~``max_chars`` windows with ``overlap`` characters of carry-over.
"""
text = (content or "").strip()
if not text:
return []
# Split into heading-delimited sections, keeping the heading with its body.
sections: list[str] = []
last = 0
for m in _HEADING_RE.finditer(text):
if m.start() > last:
sections.append(text[last : m.start()].strip())
last = m.start()
sections.append(text[last:].strip())
sections = [s for s in sections if s]
chunks: list[str] = []
for section in sections:
if len(section) <= max_chars:
chunks.append(section)
continue
# Pack paragraphs into windows, then hard-split anything still too long.
buf = ""
for para in _split_paragraphs(section):
if len(buf) + len(para) + 2 <= max_chars:
buf = f"{buf}\n\n{para}".strip()
else:
if buf:
chunks.append(buf)
if len(para) <= max_chars:
buf = para
else:
for i in range(0, len(para), max_chars - overlap):
chunks.append(para[i : i + max_chars])
buf = ""
if buf:
chunks.append(buf)
return [c for c in chunks if c.strip()]
@@ -0,0 +1,75 @@
"""Duplicate detection for knowledge notes (phase 3).
First-release heuristic: token Jaccard over ``title + summary + tags`` plus an
exact ``content_md`` hash match. Cheap, deterministic and embedding-free so it
works on offline deployments. Returns *suggestions*; merging is an explicit,
user-driven action.
"""
from __future__ import annotations
import re
from typing import Any
from deerflow.knowledge.manifest import content_hash
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
def _tokens(*texts: str) -> set[str]:
out: set[str] = set()
for t in texts:
out.update(m.lower() for m in _TOKEN_RE.findall(t or ""))
return out
def _jaccard(a: set[str], b: set[str]) -> float:
if not a or not b:
return 0.0
inter = len(a & b)
union = len(a | b)
return inter / union if union else 0.0
def find_duplicate_groups(notes: list[dict[str, Any]], *, threshold: float = 0.82, max_notes: int = 800) -> list[dict[str, Any]]:
"""Return groups ``{primary, duplicates:[{id,title,score,reason}]}``.
``primary`` is the most recently updated note in each cluster.
"""
notes = notes[:max_notes]
feats: list[tuple[dict[str, Any], set[str], str]] = []
for n in notes:
toks = _tokens(n.get("title") or "", n.get("summary") or "", " ".join(n.get("tags") or []))
feats.append((n, toks, content_hash(n.get("content_md") or "")))
used: set[str] = set()
groups: list[dict[str, Any]] = []
for i in range(len(feats)):
ni, ti, hi = feats[i]
if ni["id"] in used:
continue
dups: list[dict[str, Any]] = []
for j in range(i + 1, len(feats)):
nj, tj, hj = feats[j]
if nj["id"] in used:
continue
if hi and hi == hj:
score, reason = 1.0, "identical-content"
else:
score = _jaccard(ti, tj)
reason = "similar-title-summary"
if score >= threshold:
dups.append({"id": nj["id"], "title": nj.get("title"), "score": round(score, 4), "reason": reason})
used.add(nj["id"])
if dups:
used.add(ni["id"])
# Primary = newest by updated_at.
cluster = [ni, *[next(f[0] for f in feats if f[0]["id"] == d["id"]) for d in dups]]
primary = max(cluster, key=lambda x: x.get("updated_at") or "")
others = [
{"id": c["id"], "title": c.get("title"), "score": next((d["score"] for d in dups if d["id"] == c["id"]), 1.0)}
for c in cluster
if c["id"] != primary["id"]
]
groups.append({"primary": {"id": primary["id"], "title": primary.get("title")}, "duplicates": others})
return groups
@@ -0,0 +1,79 @@
"""Embedding generation for vector RAG (phase 2).
Talks to an OpenAI-compatible ``/embeddings`` endpoint configured under
``knowledge.embedding``. Everything here is **best-effort**: if embeddings are
disabled, unconfigured, or the endpoint errors, callers get an empty result and
the search layer falls back to keyword scoring — so vector RAG never breaks a
deployment that has no embedding model.
"""
from __future__ import annotations
import logging
import math
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deerflow.config.knowledge_config import KnowledgeEmbeddingConfig
logger = logging.getLogger(__name__)
def cosine_similarity(a: list[float], b: list[float]) -> float:
"""Cosine similarity of two equal-length vectors (0 when degenerate)."""
if not a or not b or len(a) != len(b):
return 0.0
dot = sum(x * y for x, y in zip(a, b, strict=False))
na = math.sqrt(sum(x * x for x in a))
nb = math.sqrt(sum(y * y for y in b))
if na == 0 or nb == 0:
return 0.0
return dot / (na * nb)
class EmbeddingClient:
"""Thin async client over an OpenAI-compatible embeddings endpoint."""
def __init__(self, config: KnowledgeEmbeddingConfig) -> None:
self.config = config
def is_enabled(self) -> bool:
return bool(self.config.enabled and self.config.base_url and self.config.model)
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
"""Return one vector per input text; ``[]`` on any failure/disabled."""
if not self.is_enabled() or not texts:
return []
try:
import httpx
except Exception: # pragma: no cover - httpx is a core dep
logger.warning("httpx unavailable; knowledge embeddings disabled")
return []
url = self.config.base_url.rstrip("/") + "/embeddings"
headers = {"Content-Type": "application/json"}
if self.config.api_key:
headers["Authorization"] = f"Bearer {self.config.api_key}"
vectors: list[list[float]] = []
batch = max(1, self.config.batch_size)
try:
async with httpx.AsyncClient(timeout=self.config.timeout_seconds) as client:
for start in range(0, len(texts), batch):
chunk = texts[start : start + batch]
payload: dict = {"model": self.config.model, "input": chunk}
if self.config.dimensions:
payload["dimensions"] = self.config.dimensions
resp = await client.post(url, headers=headers, json=payload)
resp.raise_for_status()
data = resp.json()
items = sorted(data.get("data", []), key=lambda d: d.get("index", 0))
vectors.extend([item.get("embedding", []) for item in items])
except Exception:
logger.exception("Knowledge embedding request failed; falling back to keyword search")
return []
return vectors
async def embed_query(self, text: str) -> list[float]:
vecs = await self.embed_texts([text])
return vecs[0] if vecs else []
@@ -0,0 +1,463 @@
"""obsidian-wiki ``wiki-export`` compatible graph generation.
Builds a knowledge graph from the notes (nodes) plus their wikilinks and shared
tags (edges) and writes the four native artifacts into ``<vault>/wiki-export/``:
``graph.json``, ``graph.graphml``, ``cypher.txt`` and a self-contained
``graph.html`` (no external CDN, safe to iframe offline).
"""
from __future__ import annotations
import html
import json
from pathlib import Path
from typing import Any
from deerflow.knowledge.markdown_vault import extract_wikilinks
def _vault_key(vault_path: str | None) -> str | None:
if not vault_path:
return None
key = vault_path.replace("\\", "/")
if key.endswith(".md"):
key = key[:-3]
return key
def normalize_blacklist(labels: list[str] | set[str] | None) -> set[str]:
"""Normalize blacklist labels for matching (trimmed, lower-cased, non-empty)."""
out: set[str] = set()
for raw in labels or []:
label = (raw or "").strip().lower()
if label:
out.add(label)
return out
def build_graph(
notes: list[dict[str, Any]],
entities: list[dict[str, Any]] | None = None,
relations: list[dict[str, Any]] | None = None,
*,
blacklist: set[str] | None = None,
) -> dict[str, Any]:
"""Return ``{nodes, edges}`` from notes (+ optional entities/relations).
Note nodes come from ``notes`` (id, title, tags, vault_path, content_md);
when ``entities``/``relations`` are supplied (phase 4), entity nodes and
note→entity / entity→entity edges are added on top of the wikilink and
shared-tag edges.
``blacklist`` is a set of *normalized* labels (see :func:`normalize_blacklist`);
notes whose title and entities whose name match are dropped, together with
every edge touching them (edges only join surviving nodes).
"""
blocked = blacklist or set()
def _is_blocked(label: str | None) -> bool:
return bool(blocked) and (label or "").strip().lower() in blocked
# Drop blacklisted notes up front so the wikilink / shared-tag edge loops
# below never reference them either.
notes = [n for n in notes if not _is_blocked(n.get("title"))]
nodes = []
by_key: dict[str, str] = {}
note_ids: set[str] = set()
for n in notes:
nodes.append({"id": n["id"], "type": "note", "label": n.get("title") or n["id"], "category": n.get("source_type")})
note_ids.add(n["id"])
key = _vault_key(n.get("vault_path"))
if key:
by_key[key] = n["id"]
# Only surface entities that are still mentioned by a LIVE note. Entities whose
# only referencing notes were deleted/archived would otherwise linger forever
# as orphan nodes ("graph doesn't follow my deletions").
mentioned: set[str] = set()
for rel in relations or []:
if rel.get("from_note_id") in note_ids and rel.get("to_entity_id"):
mentioned.add(rel["to_entity_id"])
entity_ids: set[str] = set()
for ent in entities or []:
if ent["id"] not in mentioned:
continue
if _is_blocked(ent.get("name")):
continue
nodes.append({"id": ent["id"], "type": "entity", "label": ent.get("name") or ent["id"], "category": ent.get("entity_type")})
entity_ids.add(ent["id"])
edges: list[dict[str, Any]] = []
edge_seen: set[tuple[str, str, str]] = set()
def add_edge(a: str, b: str, label: str, weight: float) -> None:
if a == b:
return
key = (min(a, b), max(a, b), label)
if key in edge_seen:
return
edge_seen.add(key)
edges.append({"id": f"rel_{len(edges) + 1}", "source": a, "target": b, "label": label, "weight": weight})
# Wikilink edges.
for n in notes:
for target in extract_wikilinks(n.get("content_md") or ""):
tid = by_key.get(target.replace("\\", "/").removesuffix(".md"))
if tid:
add_edge(n["id"], tid, "links", 2)
# Shared-tag edges.
tag_to_notes: dict[str, list[str]] = {}
for n in notes:
for tag in n.get("tags") or []:
tag_to_notes.setdefault(tag.lower(), []).append(n["id"])
for tag, ids in tag_to_notes.items():
if len(ids) < 2 or len(ids) > 50: # skip ubiquitous tags (e.g. knowledge-base)
continue
for i in range(len(ids)):
for j in range(i + 1, len(ids)):
add_edge(ids[i], ids[j], "related", 1)
# Entity / relation edges (phase 4) — only when both endpoints exist as nodes.
valid = note_ids | entity_ids
for rel in relations or []:
src = rel.get("from_note_id") or rel.get("from_entity_id")
tgt = rel.get("to_note_id") or rel.get("to_entity_id")
if src in valid and tgt in valid:
add_edge(src, tgt, rel.get("relation_type") or "related", float(rel.get("weight") or 1))
return {"nodes": nodes, "edges": edges}
def to_graphml(graph: dict[str, Any]) -> str:
lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<graphml xmlns="http://graphml.graphdrawing.org/xmlns">',
' <key id="label" for="node" attr.name="label" attr.type="string"/>',
' <key id="elabel" for="edge" attr.name="label" attr.type="string"/>',
' <graph edgedefault="undirected">',
]
for n in graph["nodes"]:
lines.append(f' <node id="{html.escape(str(n["id"]))}"><data key="label">{html.escape(str(n["label"]))}</data></node>')
for e in graph["edges"]:
lines.append(
f' <edge id="{html.escape(str(e["id"]))}" source="{html.escape(str(e["source"]))}" '
f'target="{html.escape(str(e["target"]))}"><data key="elabel">{html.escape(str(e["label"]))}</data></edge>'
)
lines += [" </graph>", "</graphml>"]
return "\n".join(lines)
def to_cypher(graph: dict[str, Any]) -> str:
lines = []
for n in graph["nodes"]:
label = str(n["label"]).replace("'", "\\'")
lines.append(f"CREATE (:Note {{id:'{n['id']}', title:'{label}'}});")
for e in graph["edges"]:
rel = str(e["label"]).upper()
lines.append(f"MATCH (a:Note {{id:'{e['source']}'}}),(b:Note {{id:'{e['target']}'}}) CREATE (a)-[:{rel} {{weight:{e['weight']}}}]->(b);")
return "\n".join(lines)
def to_html(graph: dict[str, Any]) -> str:
"""Self-contained, theme-aware force-directed graph (canvas + vanilla JS, no CDN).
The page reads ``?theme=light|dark`` from its own URL so the embedding iframe
can stay in sync with the app's day/night mode (an iframe cannot inherit the
parent's CSS). The layout settles and then **freezes** so a dense graph stops
drifting and stays clickable; node size scales with degree; and hovering or
clicking a node focuses it + its direct neighbours while dimming everything
else, which cuts through heavy node/edge clutter.
"""
data_json = json.dumps(graph, ensure_ascii=False)
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
def export_all(
vault_root: Path | str,
notes: list[dict[str, Any]],
*,
entities: list[dict[str, Any]] | None = None,
relations: list[dict[str, Any]] | None = None,
write_html: bool = True,
write_json: bool = True,
blacklist: set[str] | None = None,
) -> dict[str, Any]:
"""Generate graph artifacts under ``<vault>/wiki-export/`` and return stats."""
out_dir = Path(vault_root).resolve() / "wiki-export"
out_dir.mkdir(parents=True, exist_ok=True)
graph = build_graph(notes, entities, relations, blacklist=blacklist)
written: list[str] = []
if write_json:
(out_dir / "graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8")
written.append("graph.json")
(out_dir / "graph.graphml").write_text(to_graphml(graph), encoding="utf-8")
written.append("graph.graphml")
(out_dir / "cypher.txt").write_text(to_cypher(graph), encoding="utf-8")
written.append("cypher.txt")
if write_html:
(out_dir / "graph.html").write_text(to_html(graph), encoding="utf-8")
written.append("graph.html")
return {
"files": written,
"stats": {"nodes": len(graph["nodes"]), "edges": len(graph["edges"])},
"graph": graph,
}
_HTML_TEMPLATE = """<!DOCTYPE html>
<html lang="zh">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>Knowledge Graph</title>
<style>
:root{
--bg:#0b1020; --fg:#e7ecff; --sub:rgba(231,236,255,.78);
--panel:rgba(17,26,54,.92); --panel-border:#2a3a66;
}
html.light{
--bg:#f6f8fc; --fg:#1f2433; --sub:rgba(31,36,51,.72);
--panel:rgba(255,255,255,.94); --panel-border:#d7deec;
}
html,body{margin:0;height:100%;background:var(--bg);color:var(--fg);font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
canvas{display:block;cursor:grab}
canvas.grabbing{cursor:grabbing}
.card{position:fixed;background:var(--panel);border:1px solid var(--panel-border);border-radius:8px;font-size:12px;z-index:3;backdrop-filter:blur(6px)}
#hud{top:10px;left:12px;padding:6px 10px;color:var(--sub);z-index:2}
#legend{top:10px;right:12px;padding:8px 10px;display:flex;flex-direction:column;gap:6px}
#legend .row{display:flex;align-items:center;gap:6px;color:var(--sub)}
#legend .dot{width:10px;height:10px;border-radius:50%}
#tip{padding:4px 8px;pointer-events:none;display:none}
#panel{left:12px;bottom:12px;max-width:300px;padding:12px 14px;display:none}
#panel h3{margin:0 0 6px;font-size:14px;line-height:1.3;word-break:break-word}
#panel .meta{color:var(--sub);margin-bottom:8px}
#panel .nb{display:flex;flex-wrap:wrap;gap:5px;max-height:120px;overflow:auto}
#panel .nb span{padding:2px 7px;border:1px solid var(--panel-border);border-radius:999px;line-height:1.4;max-width:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
#panel .close{position:absolute;top:8px;right:10px;cursor:pointer;color:var(--sub);font-size:14px}
#reset{bottom:12px;right:12px;padding:7px 12px;cursor:pointer;color:var(--fg);user-select:none}
#reset:hover{filter:brightness(1.1)}
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.55}
</style>
</head>
<body>
<div id="hud" class="card"></div>
<div id="legend" class="card">
<div class="row"><span class="dot" id="lg-note"></span>知识笔记</div>
<div class="row"><span class="dot" id="lg-entity"></span>实体</div>
</div>
<div id="tip" class="card"></div>
<div id="panel" class="card"><span class="close" id="panel-close">×</span><h3></h3><div class="meta"></div><div class="nb"></div></div>
<div id="reset" class="card">重置视图</div>
<canvas id="c"></canvas>
<script>
const GRAPH = __GRAPH_DATA__;
// Theme is supplied by the embedding iframe (it can't read the parent's CSS).
const THEME = (new URLSearchParams(location.search).get('theme') === 'light') ? 'light' : 'dark';
const PAL = {
dark: {note:'#5b8cff', entity:'#ffb454', label:'rgba(231,236,255,.92)', edge:[150,178,255], halo:'rgba(11,16,32,.55)'},
light: {note:'#2f6df6', entity:'#d97706', label:'rgba(31,36,51,.92)', edge:[63,96,168], halo:'rgba(255,255,255,.7)'},
}[THEME];
document.documentElement.classList.toggle('light', THEME === 'light');
document.getElementById('lg-note').style.background = PAL.note;
document.getElementById('lg-entity').style.background = PAL.entity;
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
const panel = document.getElementById('panel');
let W=0,H=0,DPR=Math.min(window.devicePixelRatio||1,2);
function resize(){W=innerWidth;H=innerHeight;canvas.width=W*DPR;canvas.height=H*DPR;canvas.style.width=W+'px';canvas.style.height=H+'px';}
addEventListener('resize',()=>{resize();bump();});resize();
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id="empty">暂无知识图谱数据</div>');}
// --- index, adjacency, degree, node sizing -------------------------------
const idx={}, adj={};
GRAPH.nodes.forEach(n=>{idx[n.id]=n;adj[n.id]=new Set();});
const edges=[];
for(const e of GRAPH.edges){
const s=idx[e.source], t=idx[e.target];
if(!s||!t||s===t) continue;
edges.push({s,t,w:e.weight||e.w||1,label:e.label||''});
adj[e.source].add(e.target); adj[e.target].add(e.source);
}
const N=GRAPH.nodes.length;
GRAPH.nodes.forEach((n,i)=>{
n.deg=adj[n.id].size;
n.r=4.5+Math.min(11,Math.sqrt(n.deg)*2.4); // hubs are visibly bigger → easier to hit
const a=i*2.399963, rad=18*Math.sqrt(i); // phyllotaxis spread = no initial overlap
n.x=Math.cos(a)*rad; n.y=Math.sin(a)*rad; n.vx=0; n.vy=0;
});
// "Hub" nodes keep their label even when zoomed out; the rest declutter.
const degs=GRAPH.nodes.map(n=>n.deg).sort((a,b)=>b-a);
const HUB=Math.max(3,(degs[Math.floor(N*0.08)]||3));
// --- force layout that cools down and FREEZES (frozen graph = clickable) --
const REP=9000, LINK=92, CENTER=0.0016;
let alpha=1, drag=null;
function physics(){
for(let i=0;i<N;i++){
const n=GRAPH.nodes[i];
for(let j=i+1;j<N;j++){
const m=GRAPH.nodes[j];
let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=REP/d2;
let fx=dx*f,fy=dy*f; n.vx+=fx;n.vy+=fy;m.vx-=fx;m.vy-=fy;
}
}
for(const e of edges){
let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.hypot(dx,dy)||1,f=(d-LINK)*0.02;
e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;
}
for(const n of GRAPH.nodes){
if(n===drag){n.vx=n.vy=0;continue;}
n.vx-=n.x*CENTER; n.vy-=n.y*CENTER;
n.x+=(n.vx*=0.82)*alpha; n.y+=(n.vy*=0.82)*alpha;
}
alpha*=0.985;
}
// --- camera / fit-to-view -------------------------------------------------
let cam={x:0,y:0,k:1};
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
function fitView(){
if(!N){return;}
let a=1e9,b=1e9,c=-1e9,d=-1e9;
for(const n of GRAPH.nodes){a=Math.min(a,n.x);b=Math.min(b,n.y);c=Math.max(c,n.x);d=Math.max(d,n.y);}
const gw=Math.max(c-a,1),gh=Math.max(d-b,1),pad=90;
cam.k=Math.max(0.12,Math.min(2.2,Math.min((W-pad)/gw,(H-pad)/gh)));
cam.x=W/2-((a+c)/2)*cam.k; cam.y=H/2-((b+d)/2)*cam.k;
}
let fitted=false;
function bump(){alpha=Math.max(alpha,0.35);} // re-warm the sim on interaction
// --- focus / hover state --------------------------------------------------
let hoverNode=null, focusNode=null;
function active(){return focusNode||hoverNode;}
function neighbors(n){return n?adj[n.id]:null;}
function draw(){
ctx.setTransform(DPR,0,0,DPR,0,0);
ctx.clearRect(0,0,W,H);
ctx.translate(cam.x,cam.y); ctx.scale(cam.k,cam.k);
const act=active(), nb=neighbors(act), zoomed=cam.k>1.35;
const [er,eg,eb]=PAL.edge;
// edges: clearly visible by default; when a node is focused, only its edges
// stay bright and the rest fade back (but never to invisible).
for(const e of edges){
const lit = act && (e.s===act||e.t===act);
const op = act ? (lit?0.98:0.12) : Math.min(0.85,0.55+e.w*0.06);
// Keep a stable on-screen thickness regardless of zoom (>=1px), so lines
// never thin out to a hairline when zoomed in or on dense graphs.
const screenW = lit ? 2.6 : Math.min(2.4,1.4+e.w*0.45);
ctx.strokeStyle='rgba('+er+','+eg+','+eb+','+op+')';
ctx.lineWidth=Math.max(1,screenW)/cam.k;
ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();
}
// nodes
for(const n of GRAPH.nodes){
const dim = act && n!==act && !(nb&&nb.has(n.id));
ctx.globalAlpha = dim?0.22:1;
ctx.fillStyle = n.type==='entity'?PAL.entity:PAL.note;
ctx.beginPath();ctx.arc(n.x,n.y,n.r,0,6.2832);ctx.fill();
if(n===act){ctx.lineWidth=2/cam.k;ctx.strokeStyle=PAL.label;ctx.stroke();}
ctx.globalAlpha=1;
}
// labels: only for focus/neighbors, hubs, or when zoomed in — kills clutter.
ctx.textBaseline='middle';
for(const n of GRAPH.nodes){
const show = act ? (n===act||(nb&&nb.has(n.id))) : (zoomed||n.deg>=HUB);
if(!show) continue;
const txt=(n.label||'').slice(0,20);
if(!txt) continue;
ctx.font=(n===act?'bold ':'')+(11/cam.k)+'px system-ui';
const tx=n.x+n.r+4/cam.k, ty=n.y;
ctx.lineWidth=3/cam.k; ctx.strokeStyle=PAL.halo;
ctx.strokeText(txt,tx,ty);
ctx.fillStyle=PAL.label; ctx.fillText(txt,tx,ty);
}
}
function loop(){
if(alpha>0.02 || drag){physics();} else if(!fitted){fitView();fitted=true;}
draw();
requestAnimationFrame(loop);
}
// Pre-settle a few iterations so the first paint is already roughly laid out.
for(let i=0;i<60;i++)physics();
fitView();
loop();
// --- picking & interaction ------------------------------------------------
function pick(p){
let best=null,bd=1e9;
for(const n of GRAPH.nodes){
const dx=n.x-p.x,dy=n.y-p.y,d=dx*dx+dy*dy, hit=(n.r+8/cam.k);
if(d<hit*hit && d<bd){bd=d;best=n;}
}
return best;
}
let pan=null, downXY=null, moved=false;
canvas.addEventListener('mousedown',ev=>{
const p=toWorld(ev.clientX,ev.clientY), hit=pick(p);
downXY={x:ev.clientX,y:ev.clientY,hit}; moved=false;
if(hit){drag=hit;} else {pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};}
canvas.classList.add('grabbing');
});
addEventListener('mousemove',ev=>{
const p=toWorld(ev.clientX,ev.clientY);
if(downXY && (Math.abs(ev.clientX-downXY.x)>3||Math.abs(ev.clientY-downXY.y)>3)) moved=true;
if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;bump();}
else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
hoverNode=drag||pan?hoverNode:pick(p);
if(hoverNode && !drag && !pan){
tip.style.display='block';
tip.style.left=Math.min(ev.clientX+14,W-220)+'px';
tip.style.top=(ev.clientY+14)+'px';
tip.textContent=hoverNode.label||'';
} else { tip.style.display='none'; }
});
addEventListener('mouseup',ev=>{
if(downXY && !moved){ // a click, not a drag → toggle focus / detail panel
focusNode = (downXY.hit && downXY.hit!==focusNode) ? downXY.hit : null;
renderPanel();
}
drag=null;pan=null;downXY=null;canvas.classList.remove('grabbing');
});
canvas.addEventListener('wheel',ev=>{
ev.preventDefault();
const s=ev.deltaY<0?1.12:0.89;
const nk=Math.max(0.08,Math.min(6,cam.k*s)), r=nk/cam.k;
cam.x=ev.clientX-(ev.clientX-cam.x)*r; cam.y=ev.clientY-(ev.clientY-cam.y)*r; cam.k=nk;
},{passive:false});
// --- detail panel ---------------------------------------------------------
const TYPE_LABEL={note:'知识笔记',entity:'实体'};
function renderPanel(){
if(!focusNode){panel.style.display='none';return;}
const n=focusNode;
panel.querySelector('h3').textContent=n.label||n.id;
const cat=n.category?' · '+n.category:'';
panel.querySelector('.meta').textContent=(TYPE_LABEL[n.type]||n.type||'节点')+cat+' · '+n.deg+' 条关系';
const nb=panel.querySelector('.nb'); nb.innerHTML='';
let count=0;
for(const id of adj[n.id]){
if(count++>=24)break;
const m=idx[id]; if(!m)continue;
const s=document.createElement('span'); s.textContent=m.label||id;
s.title=m.label||id;
s.style.cursor='pointer';
s.onclick=()=>{focusNode=m;renderPanel();};
nb.appendChild(s);
}
if(!adj[n.id].size){const s=document.createElement('span');s.textContent='(无关联)';s.style.opacity=.6;nb.appendChild(s);}
panel.style.display='block';
}
document.getElementById('panel-close').onclick=()=>{focusNode=null;renderPanel();};
document.getElementById('reset').onclick=()=>{focusNode=null;renderPanel();fitView();};
</script>
</body>
</html>"""
@@ -0,0 +1,266 @@
"""Extract knowledge notes from threads / searches / tool results.
First-release strategy is rule-based (wiki-capture aligned, §11 of the dev doc):
keep high-value content (user questions, final assistant answers, cited
sources), drop noise (empty/greeting/intermediate reasoning), and write a
*declarative* note rather than a chat transcript. An LLM extractor can later
replace :func:`extract_thread` while keeping the same :class:`NoteDraft` output.
"""
from __future__ import annotations
import re
from typing import Any
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
# Greeting / low-value lines filtered from extracted turns.
_GREETING_RE = re.compile(r"^(你好|您好|hi|hello|hey|thanks?|谢谢|好的|ok|okay)[\s!!。.,,~]*$", re.IGNORECASE)
_WS_RE = re.compile(r"\s+")
def _one_line(text: str, limit: int = 300) -> str:
"""Collapse whitespace and clip ``text`` to a single line."""
collapsed = _WS_RE.sub(" ", text or "").strip()
return collapsed[:limit]
def message_text(msg: dict[str, Any]) -> str:
"""Return the plain-text content of a serialized LangChain message."""
content = msg.get("content")
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
if block.get("type") in (None, "text") and isinstance(block.get("text"), str):
parts.append(block["text"])
return "\n".join(p for p in parts if p).strip()
return ""
def _msg_type(msg: dict[str, Any]) -> str:
return str(msg.get("type") or msg.get("role") or "").lower()
def split_turns(messages: list[dict[str, Any]], *, max_messages: int = 80) -> list[dict[str, str]]:
"""Split messages into ``{user, ai}`` turns keyed by human messages.
For each turn we keep the last substantial AI message (with text body).
"""
msgs = messages[-max_messages:] if max_messages and len(messages) > max_messages else list(messages)
turns: list[dict[str, str]] = []
current: dict[str, str] | None = None
for msg in msgs:
mtype = _msg_type(msg)
text = message_text(msg)
if mtype == "human":
if current and (current.get("user") or current.get("ai")):
turns.append(current)
current = {"user": text, "ai": ""}
elif mtype == "ai" and text:
if current is None:
current = {"user": "", "ai": text}
else:
current["ai"] = text # last substantial AI message wins
if current and (current.get("user") or current.get("ai")):
turns.append(current)
# Drop turns that are pure greetings with no answer.
cleaned = []
for t in turns:
user = (t.get("user") or "").strip()
ai = (t.get("ai") or "").strip()
if not ai and _GREETING_RE.match(user):
continue
if not user and not ai:
continue
cleaned.append({"user": user, "ai": ai})
return cleaned
def _auto_title(thread_title: str | None, turns: list[dict[str, str]]) -> str:
if thread_title and thread_title.strip() and thread_title.strip().lower() != "untitled":
return thread_title.strip()[:200]
for t in turns:
if t.get("user"):
first = t["user"].splitlines()[0].strip()
return (first[:40] + ("…" if len(first) > 40 else "")) or "知识笔记"
return "知识笔记"
def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str:
"""Simple rule-based summary: the first answer, clipped."""
for t in turns:
if t.get("ai"):
text = re.sub(r"\s+", " ", t["ai"]).strip()
return text[:limit] + ("…" if len(text) > limit else "")
for t in turns:
if t.get("user"):
text = re.sub(r"\s+", " ", t["user"]).strip()
return text[:limit] + ("…" if len(text) > limit else "")
return ""
def related_links(project_name: str | None) -> list[str]: # noqa: ARG001
"""Structural wikilinks every note used to carry.
Product decision: notes no longer auto-link the vault meta/hub pages
(``index`` / ``_meta/taxonomy`` / the project hub) — they were noise in every
note's ``## Related``. Only real, content-specific related notes (injected by
the service layer, which has DB access) are linked now. Kept as a function
(returns empty) so existing callers and the wiki-capture contract stay intact.
"""
return []
# Backwards-compatible internal alias.
_related_links = related_links
def _related_section_md(related_lines: list[str], source_lines: list[str] | None = None) -> str: # noqa: ARG001
"""Render the ``## Related`` section, or ``""`` when there is nothing to show.
Sources are intentionally **not** rendered into the body anymore. The note
detail page already shows every source once via the structured ``来源`` block
(``note.sources``); emitting a ``### Sources`` sub-section here duplicated that
list (and broke the frontend's clean ``## Related`` extraction). The
``source_lines`` argument is kept for backward compatibility and ignored.
"""
if not related_lines:
return ""
inner = "\n".join(related_lines)
return f"## Related\n\n{inner}\n"
def _build_body(
*,
title: str,
turns: list[dict[str, str]],
mode: str,
project_name: str | None,
sources: list[SourceDraft],
) -> str:
"""Render the wiki-capture 5-section declarative body."""
full = mode == "full"
answer_clip = 4000 if full else 800
context_lines = []
for t in turns:
if t.get("user"):
q = re.sub(r"\s+", " ", t["user"]).strip()
context_lines.append(f"- {q[:300]}")
context = "\n".join(context_lines[:8]) or "- 来自一次对话沉淀。^[inferred]"
finding_lines = []
for i, t in enumerate(turns, 1):
if t.get("ai"):
ans = t["ai"].strip()
ans = ans if full else (ans[:answer_clip] + ("…" if len(ans) > answer_clip else ""))
finding_lines.append(f"### 结论 {i}\n\n{ans}")
finding = "\n\n".join(finding_lines) or "_暂无可提炼的结论。_"
reasoning = "本页由对话自动沉淀而成,结论来自助手最终回答;过程性推理已省略。判断与归纳部分标记为 ^[inferred]。"
implications = "- 后续可在相关问答中参考本页知识。\n- 如内容过期或冲突,请编辑或归档。"
related_lines = [f"- {link}" for link in _related_links(project_name)]
source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
related_section = _related_section_md(related_lines, source_lines)
return (
f"# {title}\n\n"
f"## Context\n\n{context}\n\n"
f"## Finding / Decision\n\n{finding}\n\n"
f"## Reasoning\n\n{reasoning}\n\n"
f"## Implications\n\n{implications}\n\n"
f"{related_section}"
)
def extract_thread(
messages: list[dict[str, Any]],
*,
thread_id: str,
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "zncm",
sources: list[SourceDraft] | None = None,
max_messages: int = 80,
) -> NoteDraft | None:
"""Build a :class:`NoteDraft` from thread messages, or ``None`` if empty."""
turns = split_turns(messages, max_messages=max_messages)
if not turns:
return None
final_title = (title or "").strip() or _auto_title(thread_title, turns)
summary = _summary(turns)
sources = sources or []
body = _build_body(title=final_title, turns=turns, mode=mode, project_name=project_name, sources=sources)
base_tags = list(tags or [])
for default in (project_name or "zncm", "knowledge-base"):
if default and default not in base_tags:
base_tags.append(default)
return NoteDraft(
title=final_title,
summary=summary,
content_md=body,
category="synthesis",
source_type="thread",
source_id=thread_id,
tags=base_tags,
confidence=0.6,
related=_related_links(project_name),
sources=sources,
source_key=f"thread:{thread_id}",
)
def extract_search(
query: str,
sources: list[SourceDraft],
*,
thread_id: str | None = None,
project_name: str | None = "zncm",
title: str | None = None,
tags: list[str] | None = None,
) -> NoteDraft:
"""Build a reference-category :class:`NoteDraft` from search/tool results."""
final_title = (title or "").strip() or f"检索:{query[:40]}"
body_lines = [f"# {final_title}\n", "## Context\n", f"- 检索关键词:{query}\n", "## Finding / Decision\n"]
for s in sources:
label = s.title or s.url or "来源"
line = f"- **{label}**"
if s.url:
line += f" — {s.url}"
if s.snippet:
line += "\n - " + _one_line(s.snippet)
body_lines.append(line)
body_lines += [
"\n## Reasoning\n",
"本页汇总外部检索来源,供后续问答引用。来源可信度需结合原文判断。^[ambiguous]\n",
"## Implications\n",
"- 引用前请核对原始来源。\n",
"## Related\n",
*[f"- {link}" for link in _related_links(project_name)],
]
base_tags = list(tags or [])
for default in (project_name or "zncm", "knowledge-base", "reference"):
if default and default not in base_tags:
base_tags.append(default)
return NoteDraft(
title=final_title,
summary=f"检索“{query}”的来源汇总,共 {len(sources)} 条。",
content_md="\n".join(body_lines),
category="references",
source_type="search",
source_id=f"search:{query[:120]}",
tags=base_tags,
confidence=0.4,
related=_related_links(project_name),
sources=sources,
source_key=None,
)
@@ -0,0 +1,493 @@
"""LLM-based knowledge extraction (wiki-capture style).
Turns a conversation into a *declarative* knowledge note instead of dumping the
raw chat. Follows the obsidian-wiki ``wiki-capture`` skill: classify the note,
keep only high-value findings/decisions, drop greetings / intermediate
reasoning / verbatim transcripts, mark inferred content, and emit structured
JSON that we render into the Context / Finding / Reasoning / Implications /
Related body.
Falls back to the rule-based :func:`deerflow.knowledge.extractor.extract_thread`
when the LLM is unavailable or returns something unusable, so capture never
hard-fails.
"""
from __future__ import annotations
import json
import logging
import re
from typing import Any
from deerflow.knowledge import extractor
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
from deerflow.knowledge.skill_templates import WIKI_CAPTURE_GUIDANCE
logger = logging.getLogger(__name__)
_VALID_CATEGORIES = {"synthesis", "concepts", "references", "decision", "journal"}
class LLMRejectedIngest(Exception):
"""The LLM judged the conversation as having no knowledge value.
Raised (instead of returning ``[]``) when the model explicitly sets
``should_ingest: false`` with no notes — so callers can *skip* ingestion
entirely rather than falling back to the rule-based verbatim dump.
"""
def _is_falsey(value: Any) -> bool:
if value is False:
return True
if isinstance(value, str):
return value.strip().lower() in ("false", "0", "no")
return False
_SYSTEM_PROMPT = """\
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一次对话提炼成\
**陈述式知识(declarative knowledge)**,而不是聊天流水账。
要求:
- 只保留高价值内容:明确结论、方案、决策、关键事实、被引用的来源。
- 丢弃:寒暄、客套、中间推理过程、重复内容、无意义的原文堆砌。
- 不要写“用户问了什么、AI 回答了什么”,要写“结论是什么、为什么成立”。
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
references(外部来源汇总)、decision(架构/设计决策)、journal(会话纪要)。
- 用中文输出。
一次对话可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
- 默认只产出 **1 页**;只有当对话明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
- 每一页聚焦单一主题,标题与 summary 必须彼此区分,不要把同一结论拆到多页。
- 不值得沉淀时返回空的 notes 数组。
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构如下:
{
"should_ingest": true,
"reason": "为什么值得/不值得沉淀",
"notes": [
{
"title": "简洁的知识标题(不超过 40 字)",
"summary": "1-2 句话概括这页知识",
"category": "synthesis",
"context": "问题背景或触发场景,1-3 句",
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
"reasoning": "为什么成立、取舍、不确定性",
"implications": ["后续建议/风险/下一步", "..."],
"tags": ["标签1", "标签2"],
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
"confidence": 0.0
}
]
}"""
_DOC_SYSTEM_PROMPT = """\
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一篇**文档**提炼成\
**陈述式知识(declarative knowledge)**,而不是照抄原文。
要求:
- 只保留高价值内容:核心结论、关键事实、定义、流程、数据、决策与其依据。
- 丢弃:目录、页眉页脚、版权声明、重复的样板文字、无意义的排版噪声。
- 用自己的话重组为结构化知识,不要逐段复制原文。
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
references(外部来源汇总)、decision(架构/设计决策)、journal(纪要)。
- 用中文输出。
一篇文档可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
- 默认只产出 **1 页**;只有当文档明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
- 每一页聚焦单一主题,标题与 summary 必须彼此区分。
- 文档没有可沉淀的知识时返回空的 notes 数组。
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构与对话提炼一致:
{
"should_ingest": true,
"reason": "为什么值得/不值得沉淀",
"notes": [
{
"title": "简洁的知识标题(不超过 40 字)",
"summary": "1-2 句话概括这页知识",
"category": "synthesis",
"context": "文档背景或适用场景,1-3 句",
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
"reasoning": "为什么成立、取舍、不确定性",
"implications": ["后续建议/风险/下一步", "..."],
"tags": ["标签1", "标签2"],
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
"confidence": 0.0
}
]
}"""
def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str:
parts: list[str] = []
for i, t in enumerate(turns, 1):
if t.get("user"):
parts.append(f"[用户 {i}] {t['user']}")
if t.get("ai"):
parts.append(f"[助手 {i}] {t['ai']}")
if sources:
parts.append("\n[来源]")
for s in sources[:20]:
label = s.title or s.url or s.tool_name or "来源"
parts.append(f"- {label}" + (f" ({s.url})" if s.url else ""))
text = "\n\n".join(parts)
if len(text) > max_chars:
text = text[:max_chars] + "\n…(已截断)"
return text
def _parse_json(raw: str) -> dict[str, Any] | None:
if not raw:
return None
text = raw.strip()
# Strip ```json fences if present.
fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL)
if fence:
text = fence.group(1).strip()
# Otherwise grab the outermost { ... }.
if not text.startswith("{"):
start = text.find("{")
end = text.rfind("}")
if start >= 0 and end > start:
text = text[start : end + 1]
try:
data = json.loads(text)
return data if isinstance(data, dict) else None
except json.JSONDecodeError:
logger.warning("Knowledge LLM returned non-JSON output; falling back to rule-based")
return None
def _render_body(data: dict[str, Any], title: str, related: list[str], sources: list[SourceDraft]) -> str:
def _bullets(items: Any) -> str:
if isinstance(items, str):
items = [items]
if not isinstance(items, list):
return ""
return "\n".join(f"- {str(x).strip()}" for x in items if str(x).strip())
context = str(data.get("context") or "").strip() or "- 来自一次对话沉淀。^[inferred]"
findings = _bullets(data.get("findings")) or "_暂无可提炼的结论。_"
reasoning = str(data.get("reasoning") or "").strip() or "(未提供推理说明)"
implications = _bullets(data.get("implications")) or "- 如内容过期或冲突,请编辑或归档。"
related_lines = [f"- {link}" for link in related]
source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
related_section = extractor._related_section_md(related_lines, source_lines)
return (
f"# {title}\n\n"
f"## Context\n\n{context}\n\n"
f"## Finding / Decision\n\n{findings}\n\n"
f"## Reasoning\n\n{reasoning}\n\n"
f"## Implications\n\n{implications}\n\n"
f"{related_section}"
)
def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]:
"""Normalize the LLM JSON into a list of per-note dicts.
Accepts the multi-note shape ``{"notes": [...]}`` as well as the legacy
single-note shape (``{"title": ..., "findings": ...}``) for backward
compatibility. Notes without any title/findings are dropped.
"""
raw_notes = data.get("notes")
if isinstance(raw_notes, list):
candidates = [n for n in raw_notes if isinstance(n, dict)]
elif data.get("title") or data.get("findings") or data.get("context"):
candidates = [data] # legacy single-object output
else:
candidates = []
return [n for n in candidates if str(n.get("title") or "").strip() or n.get("findings")]
def _build_note_draft(
note_data: dict[str, Any],
*,
source_type: str,
source_id: str | None,
source_key: str,
fallback_title: str,
title_override: str | None,
tags: list[str] | None,
project_name: str | None,
sources: list[SourceDraft],
) -> NoteDraft:
"""Render one note dict into a :class:`NoteDraft`.
Source-agnostic: the caller supplies ``source_type`` / ``source_id`` /
``source_key`` and a ``fallback_title`` (used when neither the user override
nor the LLM produced a usable title), so both thread and document capture
share one renderer.
"""
category = str(note_data.get("category") or "synthesis").strip().lower()
if category not in _VALID_CATEGORIES:
category = "synthesis"
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or fallback_title
summary = str(note_data.get("summary") or "").strip()
related = extractor.related_links(project_name)
body = _render_body(note_data, final_title, related, sources)
base_tags = list(tags or [])
llm_tags = note_data.get("tags") if isinstance(note_data.get("tags"), list) else []
for tag in [*llm_tags, project_name or "zncm", "knowledge-base"]:
tag = str(tag).strip()
if tag and tag not in base_tags:
base_tags.append(tag)
try:
confidence = float(note_data.get("confidence") or 0.7)
except (TypeError, ValueError):
confidence = 0.7
entities = note_data.get("entities") if isinstance(note_data.get("entities"), list) else []
relations = note_data.get("relations") if isinstance(note_data.get("relations"), list) else []
return NoteDraft(
title=final_title,
summary=summary or final_title,
content_md=body,
category=category,
source_type=source_type,
source_id=source_id,
tags=base_tags,
confidence=max(0.0, min(confidence, 1.0)),
related=related,
sources=sources,
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
source_key=source_key,
)
async def extract_thread_llm_multi(
messages: list[dict[str, Any]],
*,
thread_id: str,
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "zncm",
sources: list[SourceDraft] | None = None,
max_messages: int = 80,
model_name: str | None = None,
max_input_chars: int = 12000,
allow_multi: bool = True,
max_notes: int = 4,
system_prompt: str | None = None,
) -> list[NoteDraft]:
"""Distill a thread into one *or several* cross-linkable knowledge notes.
The LLM may split a conversation that covers multiple independent topics into
several declarative notes. Returns ``[]`` when there is nothing to sediment
or the LLM path fails — callers should fall back to the rule-based extractor.
When ``allow_multi`` is false only the first note is kept. ``system_prompt``
overrides the built-in distillation prompt with an operator-edited template
(used as-is, since the template is expected to be self-contained).
"""
turns = extractor.split_turns(messages, max_messages=max_messages)
if not turns:
return []
sources = sources or []
try:
from langchain_core.messages import HumanMessage, SystemMessage
from deerflow.models import create_chat_model
model = create_chat_model(name=model_name or None, thinking_enabled=False)
transcript = _build_transcript(turns, sources, max_chars=max_input_chars)
system_text = (system_prompt or "").strip() or (_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE)
prompt = [
SystemMessage(content=system_text),
HumanMessage(content=f"以下是一次对话,请按规范提炼为知识 JSON:\n\n{transcript}"),
]
response = await model.ainvoke(prompt)
raw = response.content if isinstance(response.content, str) else str(response.content)
data = _parse_json(raw)
except Exception:
logger.exception("[knowledge] thread %s — LLM extraction call failed; using rule-based fallback", thread_id)
return []
if not data:
logger.warning("[knowledge] thread %s — LLM returned no parsable JSON; using rule-based fallback", thread_id)
return []
note_dicts = _notes_from_data(data)
if not note_dicts:
reason = str(data.get("reason") or "")[:200]
if _is_falsey(data.get("should_ingest")):
# The model deliberately judged this conversation worthless (greeting,
# meta chatter, no findings). Skip ingestion entirely — do NOT fall
# back to the rule-based verbatim dump.
logger.warning("[knowledge] thread %s SKIPPED — model judged not worth ingesting: %s", thread_id, reason)
raise LLMRejectedIngest(reason or "no knowledge value")
logger.warning(
"[knowledge] thread %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
thread_id,
data.get("should_ingest"),
reason,
)
return []
if not allow_multi:
note_dicts = note_dicts[:1]
else:
note_dicts = note_dicts[: max(1, max_notes)]
fallback_title = extractor._auto_title(thread_title, turns)
drafts: list[NoteDraft] = []
for i, nd in enumerate(note_dicts):
# The user-supplied title override only applies to a single-note result;
# multi-note splits keep each note's own LLM title.
override = title if (len(note_dicts) == 1) else None
suffix = "" if len(note_dicts) == 1 else f"#{i}"
drafts.append(
_build_note_draft(
nd,
source_type="thread",
source_id=thread_id,
source_key=f"thread:{thread_id}{suffix}",
fallback_title=fallback_title,
title_override=override,
tags=tags,
project_name=project_name,
sources=sources,
)
)
return drafts
async def extract_document_llm_multi(
text: str,
*,
doc_id: str,
doc_title: str | None = None,
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "zncm",
sources: list[SourceDraft] | None = None,
model_name: str | None = None,
max_input_chars: int = 12000,
allow_multi: bool = True,
max_notes: int = 4,
system_prompt: str | None = None,
) -> list[NoteDraft]:
"""Distill an imported **document** into one *or several* knowledge notes.
Mirrors :func:`extract_thread_llm_multi` but takes raw document text (already
converted to Markdown/plain text) instead of a conversation. Returns ``[]``
when the LLM path fails (callers fall back to a verbatim note); raises
:class:`LLMRejectedIngest` when the model deliberately judges the document
worthless. Resulting notes carry ``source_type="file"``.
"""
text = (text or "").strip()
if not text:
return []
sources = sources or []
try:
from langchain_core.messages import HumanMessage, SystemMessage
from deerflow.models import create_chat_model
model = create_chat_model(name=model_name or None, thinking_enabled=False)
body = text if len(text) <= max_input_chars else text[:max_input_chars] + "\n…(已截断)"
header = f"文档标题:{doc_title}\n\n" if doc_title else ""
system_text = (system_prompt or "").strip() or (_DOC_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE)
prompt = [
SystemMessage(content=system_text),
HumanMessage(content=f"以下是一篇文档,请按规范提炼为知识 JSON:\n\n{header}{body}"),
]
response = await model.ainvoke(prompt)
raw = response.content if isinstance(response.content, str) else str(response.content)
data = _parse_json(raw)
except Exception:
logger.exception("[knowledge] document %s — LLM extraction call failed; using rule-based fallback", doc_id)
return []
if not data:
logger.warning("[knowledge] document %s — LLM returned no parsable JSON; using rule-based fallback", doc_id)
return []
note_dicts = _notes_from_data(data)
if not note_dicts:
reason = str(data.get("reason") or "")[:200]
if _is_falsey(data.get("should_ingest")):
logger.warning("[knowledge] document %s SKIPPED — model judged not worth ingesting: %s", doc_id, reason)
raise LLMRejectedIngest(reason or "no knowledge value")
logger.warning(
"[knowledge] document %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
doc_id,
data.get("should_ingest"),
reason,
)
return []
note_dicts = note_dicts[:1] if not allow_multi else note_dicts[: max(1, max_notes)]
fallback_title = (doc_title or title or "文档知识").strip() or "文档知识"
drafts: list[NoteDraft] = []
for i, nd in enumerate(note_dicts):
override = title if (len(note_dicts) == 1) else None
suffix = "" if len(note_dicts) == 1 else f"#{i}"
drafts.append(
_build_note_draft(
nd,
source_type="file",
source_id=doc_id,
source_key=f"file:{doc_id}{suffix}",
fallback_title=fallback_title,
title_override=override,
tags=tags,
project_name=project_name,
sources=sources,
)
)
return drafts
async def extract_thread_llm(
messages: list[dict[str, Any]],
*,
thread_id: str,
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "zncm",
sources: list[SourceDraft] | None = None,
max_messages: int = 80,
model_name: str | None = None,
max_input_chars: int = 12000,
) -> NoteDraft | None:
"""Distill a thread into a single :class:`NoteDraft` (backward-compatible).
Thin wrapper over :func:`extract_thread_llm_multi` that returns the first
note, or ``None`` when nothing was produced.
"""
try:
drafts = await extract_thread_llm_multi(
messages,
thread_id=thread_id,
thread_title=thread_title,
mode=mode,
title=title,
tags=tags,
project_name=project_name,
sources=sources,
max_messages=max_messages,
model_name=model_name,
max_input_chars=max_input_chars,
allow_multi=False,
)
except LLMRejectedIngest:
return None
return drafts[0] if drafts else None
@@ -0,0 +1,74 @@
""".manifest.json read/write for the vault.
The manifest records, per page, the source key, content hash, vault path and
update time — so batch ingest can skip already-sedimented threads (delta/dedup)
exactly like obsidian-wiki's native ``.manifest.json``.
"""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from typing import Any
def content_hash(text: str) -> str:
"""Stable sha256 hex digest of ``text`` (used for dedup)."""
return hashlib.sha256((text or "").encode("utf-8")).hexdigest()
class Manifest:
"""JSON manifest stored at ``<vault>/.manifest.json``."""
def __init__(self, root: Path | str) -> None:
self.path = Path(root).resolve() / ".manifest.json"
def load(self) -> dict[str, Any]:
if not self.path.exists():
return {"version": 1, "updated": None, "pages": {}, "sources": {}}
try:
data = json.loads(self.path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return {"version": 1, "updated": None, "pages": {}, "sources": {}}
data.setdefault("pages", {})
data.setdefault("sources", {})
return data
def save(self, data: dict[str, Any]) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
def record_page(
self,
*,
note_id: str,
vault_path: str,
source_key: str | None,
hash_value: str,
updated: str,
) -> None:
data = self.load()
data["pages"][note_id] = {
"vault_path": vault_path,
"source": source_key,
"hash": hash_value,
"updated": updated,
}
if source_key:
data["sources"][source_key] = {"note_id": note_id, "hash": hash_value, "updated": updated}
data["updated"] = updated
self.save(data)
def remove_page(self, note_id: str) -> None:
data = self.load()
page = data["pages"].pop(note_id, None)
if page and page.get("source"):
data["sources"].pop(page["source"], None)
self.save(data)
def has_source(self, source_key: str) -> bool:
return source_key in self.load().get("sources", {})
def get_source(self, source_key: str) -> dict[str, Any] | None:
return self.load().get("sources", {}).get(source_key)
@@ -0,0 +1,213 @@
"""obsidian-wiki compatible Markdown vault writer/reader.
Responsibilities (dev doc §10 ``markdown_vault.py``):
- Generate obsidian-wiki compatible frontmatter.
- Produce safe, path-traversal-proof file names/paths.
- Write/update/read Markdown note files.
- Parse frontmatter + extract ``[[wikilinks]]``.
The vault layout follows §8 of the dev doc and stays compatible with
``ar9av/obsidian-wiki`` (concepts/ references/ synthesis/ journal/ projects/…,
plus index.md / log.md / hot.md / .manifest.json / _meta/taxonomy.md /
wiki-export/).
"""
from __future__ import annotations
import re
import unicodedata
from datetime import datetime
from pathlib import Path
from typing import Any
import yaml
_WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")
_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?(.*)$", re.DOTALL)
_UNSAFE_CHARS_RE = re.compile(r"[^\w一-鿿-]+")
# Top-level vault directories that must exist (obsidian-wiki native layout).
_VAULT_DIRS = ("concepts", "entities", "skills", "references", "synthesis", "journal", "projects", "_meta", "wiki-export")
def slugify(title: str, *, max_len: int = 60) -> str:
"""Return a filesystem-safe slug (ASCII + CJK), never empty."""
text = unicodedata.normalize("NFKC", (title or "").strip()).lower()
text = text.replace(" ", "-")
text = _UNSAFE_CHARS_RE.sub("-", text).strip("-")
text = re.sub(r"-{2,}", "-", text)
if len(text) > max_len:
text = text[:max_len].rstrip("-")
return text or "note"
def extract_wikilinks(text: str) -> list[str]:
"""Return the targets of every ``[[wikilink]]`` in ``text`` (de-duplicated)."""
seen: list[str] = []
for match in _WIKILINK_RE.finditer(text or ""):
target = match.group(1).strip()
if target and target not in seen:
seen.append(target)
return seen
def normalize_wikilink_target(target: str) -> str:
"""Canonicalize a ``[[wikilink]]`` target for resolution/comparison.
Drops any ``|alias`` part, normalizes slashes, strips a leading ``/`` and a
trailing ``.md`` — so ``[[index]]``, ``index.md`` and ``/index`` all map to
the same canonical ``index`` key used by the index/graph/resolver layers.
"""
t = (target or "").strip()
if not t:
return ""
t = t.split("|", 1)[0].strip()
t = t.replace("\\", "/").lstrip("/")
if t.lower().endswith(".md"):
t = t[:-3]
return t
class VaultPathError(ValueError):
"""Raised when a requested vault path escapes the vault root."""
class MarkdownVault:
"""Reads/writes Markdown files inside a single vault root directory."""
def __init__(self, root: Path | str) -> None:
self.root = Path(root).resolve()
# -- path safety ------------------------------------------------------
def resolve(self, rel_path: str) -> Path:
"""Resolve ``rel_path`` under the vault root, blocking traversal."""
rel = str(rel_path).replace("\\", "/").lstrip("/")
candidate = (self.root / rel).resolve()
try:
candidate.relative_to(self.root)
except ValueError as exc:
raise VaultPathError(f"Path escapes vault root: {rel_path!r}") from exc
return candidate
def note_relpath(self, *, category: str, project_name: str | None, slug: str, note_id: str) -> str:
"""Build a vault-relative path for a note.
Project knowledge goes under ``projects/<project>/<category>/`` per
wiki-capture; a short id suffix guarantees uniqueness.
"""
from deerflow.knowledge.skill_templates import category_to_dir
cat_dir = category_to_dir(category)
safe_slug = slugify(slug)
suffix = note_id.split("_")[-1][:8]
filename = f"{safe_slug}-{suffix}.md"
if project_name:
return f"projects/{slugify(project_name)}/{cat_dir}/{filename}"
return f"{cat_dir}/{filename}"
# -- scaffold ---------------------------------------------------------
def ensure_scaffold(self, *, project_name: str | None = None) -> None:
"""Create the vault directory tree + base files if missing."""
self.root.mkdir(parents=True, exist_ok=True)
for d in _VAULT_DIRS:
(self.root / d).mkdir(parents=True, exist_ok=True)
if project_name:
base = self.root / "projects" / slugify(project_name)
for d in ("concepts", "references", "synthesis", "journal"):
(base / d).mkdir(parents=True, exist_ok=True)
# Base vault files.
index = self.root / "index.md"
if not index.exists():
index.write_text("# Knowledge Index\n\n_页面索引。每次写入新页面后自动更新。_\n\n", encoding="utf-8")
log = self.root / "log.md"
if not log.exists():
log.write_text("# Activity Log\n\n_CAPTURE / INGEST / QUERY / EXPORT 记录。_\n\n", encoding="utf-8")
hot = self.root / "hot.md"
if not hot.exists():
hot.write_text("# Hot\n\n_近期活动摘要。_\n\n", encoding="utf-8")
taxonomy = self.root / "_meta" / "taxonomy.md"
if not taxonomy.exists():
taxonomy.write_text(
"# Taxonomy\n\n- synthesis — 多步骤分析、方案、结论\n- concepts — 概念、框架、模型\n"
"- references — 外部来源、搜索结果、文章资料\n- decision — 架构或设计决策\n- journal — 完整会话摘要\n",
encoding="utf-8",
)
# -- frontmatter ------------------------------------------------------
@staticmethod
def build_frontmatter(
*,
note_id: str,
title: str,
category: str,
tags: list[str],
sources: list[str],
summary: str,
created: str,
updated: str,
base_confidence: float,
lifecycle: str,
provenance: dict[str, float] | None = None,
relationships: list[dict[str, str]] | None = None,
) -> dict[str, Any]:
"""Build an obsidian-wiki / wiki-capture compatible frontmatter dict."""
created_date = (created or "")[:10]
return {
"id": note_id,
"title": title,
"category": category,
"tags": list(tags or []),
"sources": list(sources or []),
"created": created,
"updated": updated,
"summary": summary or "",
"provenance": provenance or {"extracted": 0.7, "inferred": 0.2, "ambiguous": 0.1},
"base_confidence": round(float(base_confidence or 0.0), 2),
"lifecycle": lifecycle,
"lifecycle_changed": created_date,
"relationships": relationships or [],
}
# -- IO ---------------------------------------------------------------
def write_note(self, rel_path: str, frontmatter: dict[str, Any], body: str) -> Path:
"""Write ``frontmatter`` + ``body`` to ``rel_path`` (creates dirs)."""
path = self.resolve(rel_path)
path.parent.mkdir(parents=True, exist_ok=True)
fm = yaml.safe_dump(frontmatter, allow_unicode=True, sort_keys=False, default_flow_style=False)
path.write_text(f"---\n{fm}---\n\n{body.strip()}\n", encoding="utf-8")
return path
def read_note(self, rel_path: str) -> tuple[dict[str, Any], str] | None:
"""Return ``(frontmatter, body)`` for a note, or ``None`` if missing."""
path = self.resolve(rel_path)
if not path.exists():
return None
return self.parse(path.read_text(encoding="utf-8"))
@staticmethod
def parse(text: str) -> tuple[dict[str, Any], str]:
"""Split raw Markdown into ``(frontmatter dict, body)``."""
match = _FRONTMATTER_RE.match(text or "")
if not match:
return {}, text or ""
try:
fm = yaml.safe_load(match.group(1)) or {}
except yaml.YAMLError:
fm = {}
if not isinstance(fm, dict):
fm = {}
return fm, match.group(2) or ""
def delete_note(self, rel_path: str) -> None:
"""Remove a note file if present (used on hard delete; archive keeps it)."""
path = self.resolve(rel_path)
if path.exists():
path.unlink()
def now_iso_with_tz(dt: datetime) -> str:
"""ISO-8601 string with timezone offset (frontmatter ``created``/``updated``)."""
if dt.tzinfo is None:
return dt.isoformat()
return dt.isoformat(timespec="seconds")
@@ -0,0 +1,223 @@
"""ORM models for the product knowledge base.
Tables mirror the obsidian-wiki vault into the database so the product can
list, filter, search and audit notes quickly without re-parsing Markdown. The
vault Markdown remains the source-of-truth for ``content_md`` / frontmatter /
graph; these rows are an index + audit layer.
The knowledge base is **global / shared** — there is no ``user_id`` isolation.
Writes record ``created_by`` / ``updated_by`` for auditing only.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
# MySQL InnoDB with the legacy 767-byte index-prefix limit (5.6, or 5.7 with
# COMPACT row format) allows at most 191 utf8mb4 chars per index key. Columns
# longer than that use an explicit prefix index (``mysql_length`` — ignored by
# SQLite/Postgres) instead of ``index=True``, otherwise ``create_all`` fails
# with error 1071 ("Specified key was too long").
class KnowledgeNoteRow(Base):
"""Knowledge note main table (mirrors ``knowledge_notes``)."""
__tablename__ = "knowledge_notes"
__table_args__ = (
Index("ix_knowledge_notes_vault_path", "vault_path", mysql_length=191),
Index("ix_knowledge_notes_source_id", "source_id", mysql_length=191),
Index("ix_knowledge_notes_folder", "folder", mysql_length=191),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
title: Mapped[str] = mapped_column(String(512))
summary: Mapped[str | None] = mapped_column(Text)
content_md: Mapped[str] = mapped_column(PortableLongText())
vault_path: Mapped[str | None] = mapped_column(String(1024))
# User-assigned directory (slash-joined path, e.g. "投研/行业"). Independent
# of the wiki-capture ``category`` and of ``vault_path``; drives the product
# directory tree. NULL = unfiled (shown under a fallback group).
folder: Mapped[str | None] = mapped_column(String(512))
source_type: Mapped[str] = mapped_column(String(64), index=True)
source_id: Mapped[str | None] = mapped_column(String(256))
status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
confidence: Mapped[float] = mapped_column(Float, default=0.0)
created_by: Mapped[str | None] = mapped_column(String(128), index=True)
updated_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), index=True)
class KnowledgeSourceRow(Base):
"""Knowledge source table (mirrors ``knowledge_sources``)."""
__tablename__ = "knowledge_sources"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
source_type: Mapped[str] = mapped_column(String(64))
thread_id: Mapped[str | None] = mapped_column(String(128), index=True)
message_id: Mapped[str | None] = mapped_column(String(128))
tool_name: Mapped[str | None] = mapped_column(String(128))
title: Mapped[str | None] = mapped_column(String(512))
url: Mapped[str | None] = mapped_column(Text)
snippet: Mapped[str | None] = mapped_column(Text)
raw_json: Mapped[str | None] = mapped_column(PortableLongText())
created_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeTagRow(Base):
"""Tag table (mirrors ``knowledge_tags``)."""
__tablename__ = "knowledge_tags"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(128), unique=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeNoteTagRow(Base):
"""Note ↔ tag join table (mirrors ``knowledge_note_tags``)."""
__tablename__ = "knowledge_note_tags"
__table_args__ = (UniqueConstraint("note_id", "tag_id", name="uq_knowledge_note_tags"),)
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), primary_key=True)
tag_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_tags.id", ondelete="CASCADE"), primary_key=True)
class KnowledgeEmbeddingRow(Base):
"""Per-chunk embedding index (phase 2, mirrors ``knowledge_embeddings``).
The vector is stored as a JSON float array; first-release similarity is
computed in Python (no external vector DB). ``vector_id`` is reserved for a
future external store mapping.
"""
__tablename__ = "knowledge_embeddings"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
chunk_index: Mapped[int] = mapped_column(Integer, default=0)
content: Mapped[str] = mapped_column(Text)
vector: Mapped[list] = mapped_column(PortableJSON(), default=list)
vector_id: Mapped[str | None] = mapped_column(String(256))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeNoteVersionRow(Base):
"""Immutable snapshot of a note before each update (phase 3 versioning)."""
__tablename__ = "knowledge_note_versions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
title: Mapped[str] = mapped_column(String(512))
summary: Mapped[str | None] = mapped_column(Text)
content_md: Mapped[str] = mapped_column(PortableLongText())
status: Mapped[str] = mapped_column(String(32))
tags_json: Mapped[list] = mapped_column(PortableJSON(), default=list)
edited_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
class KnowledgeEntityRow(Base):
"""Knowledge entity (phase 4, mirrors ``knowledge_entities``)."""
__tablename__ = "knowledge_entities"
__table_args__ = (
Index("ix_knowledge_entities_name", "name", mysql_length=191),
# Prefix-unique: (120 + 64) * 4 = 736 bytes <= 767. Entity names are
# realistically far shorter than 120 chars, so the prefix is lossless.
Index("uq_knowledge_entities_name_type", "name", "entity_type", unique=True, mysql_length={"name": 120}),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256))
entity_type: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeGraphBlacklistRow(Base):
"""Graph blacklist: node labels (entity names / note titles) hidden from the graph.
Matching is case-insensitive on the trimmed label. Global/shared like the
rest of the knowledge base; ``created_by`` is audit-only.
"""
__tablename__ = "knowledge_graph_blacklist"
__table_args__ = (Index("ix_knowledge_graph_blacklist_label", "label", unique=True, mysql_length=191),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
label: Mapped[str] = mapped_column(String(512))
created_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeRelationRow(Base):
"""Relation between notes/entities (phase 4, mirrors ``knowledge_relations``)."""
__tablename__ = "knowledge_relations"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
from_note_id: Mapped[str | None] = mapped_column(String(64), index=True)
to_note_id: Mapped[str | None] = mapped_column(String(64), index=True)
from_entity_id: Mapped[str | None] = mapped_column(String(64), index=True)
to_entity_id: Mapped[str | None] = mapped_column(String(64), index=True)
relation_type: Mapped[str] = mapped_column(String(128))
weight: Mapped[float] = mapped_column(Float, default=1.0)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeFolderRow(Base):
"""User-managed knowledge directory (mirrors ``knowledge_folders``).
A folder is identified by its full slash-joined ``path`` (e.g. ``投研/行业``);
nesting is derived by splitting on ``/`` on the client. Folders exist
independently of notes so an operator can create an empty directory and file
notes into it later (``knowledge_notes.folder`` stores the chosen path).
Global/shared like the rest of the KB; ``created_by`` is audit-only.
"""
__tablename__ = "knowledge_folders"
__table_args__ = (Index("uq_knowledge_folders_path", "path", unique=True, mysql_length=191),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
path: Mapped[str] = mapped_column(String(512))
name: Mapped[str] = mapped_column(String(256))
sort_order: Mapped[int] = mapped_column(Integer, default=0)
created_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeExtractTemplateRow(Base):
"""Editable knowledge-extraction prompt template (mirrors ``knowledge_extract_templates``).
Lets an operator open up, edit and add multiple distillation prompts instead
of the hard-coded constants. ``scope`` is ``thread`` / ``document`` / ``both``;
at capture time the service resolves the chosen template (or the default for
that scope) and feeds its ``system_prompt`` to the extractor. Global/shared.
"""
__tablename__ = "knowledge_extract_templates"
__table_args__ = (Index("uq_knowledge_extract_templates_name", "name", unique=True, mysql_length=191),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256))
description: Mapped[str | None] = mapped_column(Text)
scope: Mapped[str] = mapped_column(String(16), default="both", index=True)
system_prompt: Mapped[str] = mapped_column(PortableLongText())
is_default: Mapped[bool] = mapped_column(default=False, index=True)
enabled: Mapped[bool] = mapped_column(default=True)
created_by: Mapped[str | None] = mapped_column(String(128))
updated_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,256 @@
"""ObsidianWikiAdapter — the obsidian-wiki vault facade.
Owns the Markdown vault: writes notes with wiki-capture frontmatter/body, keeps
``index.md`` / ``log.md`` / ``hot.md`` / ``.manifest.json`` in sync after every
write, and produces wiki-export graph artifacts. Output stays compatible with
``ar9av/obsidian-wiki`` without ever invoking its CLI.
This class is storage-only (no DB, no LLM). The :class:`KnowledgeService`
combines it with the repository + extractor.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
from deerflow.knowledge import exporter
from deerflow.knowledge.manifest import Manifest, content_hash
from deerflow.knowledge.markdown_vault import MarkdownVault
from deerflow.knowledge.schemas import NoteDraft
_INDEX_PAGES_HEADER = "## Pages"
_HOT_HEADER = "## Recent"
_HOT_MAX = 20
class ObsidianWikiAdapter:
"""Read/write an obsidian-wiki compatible vault."""
def __init__(self, vault_root: Path | str, *, project_name: str = "zncm", export_html: bool = True, export_json: bool = True) -> None:
self.vault = MarkdownVault(vault_root)
self.root = self.vault.root
self.manifest = Manifest(self.root)
self.project_name = project_name
self.export_html = export_html
self.export_json = export_json
def ensure_ready(self) -> None:
self.vault.ensure_scaffold(project_name=self.project_name)
# -- writes -----------------------------------------------------------
def write_note(
self,
note_id: str,
draft: NoteDraft,
*,
created: str,
updated: str,
status: str,
) -> str:
"""Write the note file, sync vault metadata, return the vault path."""
self.ensure_ready()
rel = self.vault.note_relpath(category=draft.category, project_name=self.project_name, slug=draft.title, note_id=note_id)
sources_fm = self._frontmatter_sources(draft)
relationships = [{"target": link, "type": "related"} for link in (draft.related or [])[:8]]
frontmatter = self.vault.build_frontmatter(
note_id=note_id,
title=draft.title,
category=draft.category,
tags=draft.tags,
sources=sources_fm,
summary=draft.summary,
created=created,
updated=updated,
base_confidence=draft.confidence,
lifecycle=status,
relationships=relationships,
)
self.vault.write_note(rel, frontmatter, draft.content_md)
self.manifest.record_page(
note_id=note_id,
vault_path=rel,
source_key=draft.source_key,
hash_value=content_hash(draft.content_md),
updated=updated,
)
self._update_index(note_id, rel, draft.title, draft.summary)
self._append_log("CAPTURE" if draft.source_type in ("thread", "search", "tool") else "INGEST", note_id, draft.title, updated)
self._update_hot(draft.title, rel, updated)
return rel
def update_note_file(
self,
note_id: str,
vault_path: str | None,
*,
title: str,
content_md: str,
tags: list[str],
status: str,
summary: str,
category: str,
created: str,
updated: str,
) -> str:
"""Rewrite a note file in place (frontmatter refreshed), sync metadata."""
self.ensure_ready()
rel = vault_path
if not rel:
rel = self.vault.note_relpath(category=category, project_name=self.project_name, slug=title, note_id=note_id)
# Preserve original `sources`/`provenance`/`category` frontmatter when present.
existing = self.vault.read_note(rel)
sources_fm = existing[0].get("sources", []) if existing else []
provenance = existing[0].get("provenance") if existing else None
relationships = existing[0].get("relationships") if existing else []
effective_category = (existing[0].get("category") if existing else None) or category
frontmatter = self.vault.build_frontmatter(
note_id=note_id,
title=title,
category=effective_category,
tags=tags,
sources=sources_fm,
summary=summary,
created=created,
updated=updated,
base_confidence=float((existing[0].get("base_confidence") if existing else 0.0) or 0.0),
lifecycle=status,
provenance=provenance,
relationships=relationships,
)
self.vault.write_note(rel, frontmatter, content_md)
self.manifest.record_page(
note_id=note_id,
vault_path=rel,
source_key=self.manifest.load().get("pages", {}).get(note_id, {}).get("source"),
hash_value=content_hash(content_md),
updated=updated,
)
self._update_index(note_id, rel, title, summary)
self._append_log("UPDATE", note_id, title, updated)
self._update_hot(title, rel, updated)
return rel
def archive_note(self, note_id: str, vault_path: str | None, *, updated: str) -> None:
"""Mark a note archived in its frontmatter + log (soft delete)."""
if vault_path:
parsed = self.vault.read_note(vault_path)
if parsed:
fm, body = parsed
fm["lifecycle"] = "archived"
fm["updated"] = updated
self.vault.write_note(vault_path, fm, body)
self._remove_from_index(note_id)
self._append_log("ARCHIVE", note_id, "", updated)
def export_graph(
self,
notes: list[dict[str, Any]],
*,
entities: list[dict[str, Any]] | None = None,
relations: list[dict[str, Any]] | None = None,
updated: str,
blacklist: set[str] | None = None,
) -> dict[str, Any]:
self.ensure_ready()
result = exporter.export_all(
self.root,
notes,
entities=entities,
relations=relations,
write_html=self.export_html,
write_json=self.export_json,
blacklist=blacklist,
)
self._append_log("EXPORT", "graph", f"nodes={result['stats']['nodes']} edges={result['stats']['edges']}", updated)
return result
def export_file_path(self, filename: str) -> Path | None:
allowed = {"graph.json", "graph.graphml", "cypher.txt", "graph.html"}
if filename not in allowed:
return None
path = (self.root / "wiki-export" / filename).resolve()
try:
path.relative_to(self.root)
except ValueError:
return None
return path if path.exists() else None
# -- frontmatter source serialization ---------------------------------
@staticmethod
def _frontmatter_sources(draft: NoteDraft) -> list[str]:
out: list[str] = []
if draft.source_type == "thread" and draft.source_id:
out.append(f"thread:{draft.source_id}")
for s in draft.sources[:20]:
if s.url:
out.append(f"url:{s.url}")
elif s.thread_id:
out.append(f"thread:{s.thread_id}")
elif s.tool_name:
out.append(f"tool:{s.tool_name}")
# De-dup preserving order.
seen: set[str] = set()
deduped = []
for item in out:
if item not in seen:
seen.add(item)
deduped.append(item)
return deduped
# -- index.md ---------------------------------------------------------
def _index_path(self) -> Path:
return self.root / "index.md"
def _update_index(self, note_id: str, rel: str, title: str, summary: str | None) -> None:
path = self._index_path()
text = path.read_text(encoding="utf-8") if path.exists() else "# Knowledge Index\n\n"
link = rel[:-3] if rel.endswith(".md") else rel
summary_txt = (summary or "").strip().replace("\n", " ")
if len(summary_txt) > 120:
summary_txt = summary_txt[:120] + "…"
line = f"- [[{link}|{title}]]" + (f" — {summary_txt}" if summary_txt else "") + f" <!-- id:{note_id} -->"
lines = text.splitlines()
# Drop any existing line for this note id.
lines = [ln for ln in lines if f"<!-- id:{note_id} -->" not in ln]
if _INDEX_PAGES_HEADER not in text:
lines.append("")
lines.append(_INDEX_PAGES_HEADER)
lines.append("")
# Insert after the Pages header.
try:
insert_at = lines.index(_INDEX_PAGES_HEADER) + 1
except ValueError:
insert_at = len(lines)
lines.insert(insert_at, line)
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
def _remove_from_index(self, note_id: str) -> None:
path = self._index_path()
if not path.exists():
return
lines = [ln for ln in path.read_text(encoding="utf-8").splitlines() if f"<!-- id:{note_id} -->" not in ln]
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
# -- log.md -----------------------------------------------------------
def _append_log(self, action: str, note_id: str, label: str, updated: str) -> None:
path = self.root / "log.md"
text = path.read_text(encoding="utf-8") if path.exists() else "# Activity Log\n\n"
entry = f"- {updated} {action} {note_id}" + (f" — {label}" if label else "")
path.write_text(text.rstrip() + "\n" + entry + "\n", encoding="utf-8")
# -- hot.md -----------------------------------------------------------
def _update_hot(self, title: str, rel: str, updated: str) -> None:
path = self.root / "hot.md"
link = rel[:-3] if rel.endswith(".md") else rel
new_line = f"- {updated} [[{link}|{title}]]"
existing_lines: list[str] = []
if path.exists():
for ln in path.read_text(encoding="utf-8").splitlines():
if ln.startswith("- ") and "[[" in ln:
existing_lines.append(ln)
existing_lines = [ln for ln in existing_lines if f"[[{link}" not in ln]
existing_lines.insert(0, new_line)
existing_lines = existing_lines[:_HOT_MAX]
body = "# Hot\n\n_近期活动摘要。_\n\n" + _HOT_HEADER + "\n\n" + "\n".join(existing_lines) + "\n"
path.write_text(body, encoding="utf-8")
@@ -0,0 +1,781 @@
"""Database read/write for the knowledge base.
This layer never calls an LLM and never touches the Markdown vault — it only
persists/queries the relational mirror (notes, sources, tags). The vault and
extraction live in :mod:`deerflow.knowledge.markdown_vault` and
:mod:`deerflow.knowledge.extractor`; orchestration lives in the service.
"""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.knowledge.models import (
KnowledgeEmbeddingRow,
KnowledgeEntityRow,
KnowledgeExtractTemplateRow,
KnowledgeFolderRow,
KnowledgeGraphBlacklistRow,
KnowledgeNoteRow,
KnowledgeNoteTagRow,
KnowledgeNoteVersionRow,
KnowledgeRelationRow,
KnowledgeSourceRow,
KnowledgeTagRow,
)
from deerflow.knowledge.schemas import SourceDraft
def _new_id(prefix: str) -> str:
return f"{prefix}_{uuid.uuid4().hex[:24]}"
def _iso(value: Any) -> Any:
if isinstance(value, datetime):
return value.isoformat()
return value
class KnowledgeRepository:
"""SQLAlchemy-backed repository for knowledge notes/sources/tags."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# -- serialization ----------------------------------------------------
@staticmethod
def _note_to_dict(row: KnowledgeNoteRow) -> dict[str, Any]:
d = row.to_dict()
d["created_at"] = _iso(d.get("created_at"))
d["updated_at"] = _iso(d.get("updated_at"))
return d
@staticmethod
def _source_to_dict(row: KnowledgeSourceRow) -> dict[str, Any]:
d = row.to_dict()
d["created_at"] = _iso(d.get("created_at"))
return d
# -- tags -------------------------------------------------------------
async def _get_or_create_tags(self, session: AsyncSession, names: list[str]) -> list[KnowledgeTagRow]:
cleaned = []
seen: set[str] = set()
for raw in names or []:
name = (raw or "").strip()
if not name or name.lower() in seen:
continue
seen.add(name.lower())
cleaned.append(name)
if not cleaned:
return []
existing = {
r.name.lower(): r
for r in (await session.execute(select(KnowledgeTagRow).where(func.lower(KnowledgeTagRow.name).in_([n.lower() for n in cleaned])))).scalars()
}
result: list[KnowledgeTagRow] = []
for name in cleaned:
row = existing.get(name.lower())
if row is None:
row = KnowledgeTagRow(id=_new_id("kbtag"), name=name, created_at=datetime.now(UTC))
session.add(row)
await session.flush()
existing[name.lower()] = row
result.append(row)
return result
async def _tags_for(self, session: AsyncSession, note_id: str) -> list[str]:
stmt = (
select(KnowledgeTagRow.name)
.join(KnowledgeNoteTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
.where(KnowledgeNoteTagRow.note_id == note_id)
.order_by(KnowledgeTagRow.name)
)
return list((await session.execute(stmt)).scalars())
async def _tags_for_many(self, session: AsyncSession, note_ids: list[str]) -> dict[str, list[str]]:
if not note_ids:
return {}
stmt = (
select(KnowledgeNoteTagRow.note_id, KnowledgeTagRow.name)
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
.where(KnowledgeNoteTagRow.note_id.in_(note_ids))
.order_by(KnowledgeTagRow.name)
)
out: dict[str, list[str]] = {nid: [] for nid in note_ids}
for note_id, name in (await session.execute(stmt)).all():
out.setdefault(note_id, []).append(name)
return out
async def _replace_note_tags(self, session: AsyncSession, note_id: str, names: list[str]) -> None:
await session.execute(delete(KnowledgeNoteTagRow).where(KnowledgeNoteTagRow.note_id == note_id))
tags = await self._get_or_create_tags(session, names)
for tag in tags:
session.add(KnowledgeNoteTagRow(note_id=note_id, tag_id=tag.id))
# -- notes ------------------------------------------------------------
async def create_note(
self,
*,
title: str,
summary: str | None,
content_md: str,
source_type: str,
source_id: str | None,
status: str,
confidence: float = 0.0,
vault_path: str | None = None,
folder: str | None = None,
tags: list[str] | None = None,
created_by: str | None = None,
) -> dict[str, Any]:
note_id = _new_id("kb")
now = datetime.now(UTC)
row = KnowledgeNoteRow(
id=note_id,
title=title[:512],
summary=summary,
content_md=content_md,
vault_path=vault_path,
folder=(folder or None),
source_type=source_type,
source_id=source_id,
status=status,
confidence=confidence,
created_by=created_by,
updated_by=created_by,
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.flush()
await self._replace_note_tags(session, note_id, tags or [])
await session.commit()
await session.refresh(row)
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, note_id)
return d
async def set_vault_path(self, note_id: str, vault_path: str) -> None:
async with self._sf() as session:
row = await session.get(KnowledgeNoteRow, note_id)
if row is None:
return
row.vault_path = vault_path
await session.commit()
async def update_note(
self,
note_id: str,
*,
title: str | None = None,
summary: str | None = None,
content_md: str | None = None,
status: str | None = None,
tags: list[str] | None = None,
vault_path: str | None = None,
folder: str | None = None,
set_folder: bool = False,
updated_by: str | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(KnowledgeNoteRow, note_id)
if row is None:
return None
if title is not None:
row.title = title[:512]
if summary is not None:
row.summary = summary
if content_md is not None:
row.content_md = content_md
if status is not None:
row.status = status
if vault_path is not None:
row.vault_path = vault_path
# ``set_folder`` lets the caller assign *or clear* (folder=None) the
# directory; a bare ``folder=None`` without the flag leaves it as-is.
if set_folder:
row.folder = (folder or None)
if updated_by is not None:
row.updated_by = updated_by
row.updated_at = datetime.now(UTC)
if tags is not None:
await self._replace_note_tags(session, note_id, tags)
await session.commit()
await session.refresh(row)
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, note_id)
return d
async def set_status(self, note_id: str, status: str, *, updated_by: str | None = None) -> dict[str, Any] | None:
return await self.update_note(note_id, status=status, updated_by=updated_by)
async def get_note(self, note_id: str, *, with_sources: bool = True) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(KnowledgeNoteRow, note_id)
if row is None:
return None
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, note_id)
if with_sources:
stmt = select(KnowledgeSourceRow).where(KnowledgeSourceRow.note_id == note_id).order_by(KnowledgeSourceRow.created_at)
d["sources"] = [self._source_to_dict(r) for r in (await session.execute(stmt)).scalars()]
return d
async def find_by_vault_path(self, vault_path: str) -> dict[str, Any] | None:
"""Return the first (non-archived) note whose vault_path matches.
``vault_path`` is matched both with and without the ``.md`` suffix, since
wikilink targets are stored without it while ``vault_path`` keeps it.
"""
stripped = vault_path[:-3] if vault_path.endswith(".md") else vault_path
candidates = {vault_path, stripped, f"{stripped}.md"}
async with self._sf() as session:
stmt = (
select(KnowledgeNoteRow)
.where(KnowledgeNoteRow.vault_path.in_(list(candidates)))
.where(KnowledgeNoteRow.status != "archived")
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
if row is None:
return None
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, row.id)
return d
async def find_by_title(self, title: str) -> dict[str, Any] | None:
"""Return the first (non-archived) note with a case-insensitive title match."""
name = (title or "").strip()
if not name:
return None
async with self._sf() as session:
stmt = (
select(KnowledgeNoteRow)
.where(func.lower(KnowledgeNoteRow.title) == name.lower())
.where(KnowledgeNoteRow.status != "archived")
.order_by(KnowledgeNoteRow.updated_at.desc())
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
if row is None:
return None
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, row.id)
return d
async def find_by_source(self, source_type: str, source_id: str) -> dict[str, Any] | None:
"""Return the first (non-archived) note for a (source_type, source_id)."""
async with self._sf() as session:
stmt = (
select(KnowledgeNoteRow)
.where(KnowledgeNoteRow.source_type == source_type, KnowledgeNoteRow.source_id == source_id)
.where(KnowledgeNoteRow.status != "archived")
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
return self._note_to_dict(row) if row else None
async def list_notes(
self,
*,
keyword: str | None = None,
tag: str | None = None,
entity: str | None = None,
source_type: str | None = None,
status: str | None = None,
folder: str | None = None,
include_archived: bool = False,
limit: int = 20,
offset: int = 0,
) -> tuple[list[dict[str, Any]], int]:
async with self._sf() as session:
stmt = select(KnowledgeNoteRow)
count_stmt = select(func.count()).select_from(KnowledgeNoteRow)
conditions = []
if status:
conditions.append(KnowledgeNoteRow.status == status)
elif not include_archived:
conditions.append(KnowledgeNoteRow.status != "archived")
if source_type:
conditions.append(KnowledgeNoteRow.source_type == source_type)
if folder is not None:
# Exact folder, plus any nested sub-folders (``folder/…``).
if folder == "":
conditions.append(KnowledgeNoteRow.folder.is_(None))
else:
conditions.append(or_(KnowledgeNoteRow.folder == folder, KnowledgeNoteRow.folder.like(f"{folder}/%")))
if keyword:
like = f"%{keyword}%"
conditions.append(or_(KnowledgeNoteRow.title.like(like), KnowledgeNoteRow.summary.like(like), KnowledgeNoteRow.content_md.like(like)))
if tag:
tag_subq = (
select(KnowledgeNoteTagRow.note_id)
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
.where(func.lower(KnowledgeTagRow.name) == tag.lower())
)
conditions.append(KnowledgeNoteRow.id.in_(tag_subq))
if entity:
# Exact entity match via the note→entity relation graph — precise
# and index-backed, unlike a content LIKE scan on large vaults.
entity_subq = (
select(KnowledgeRelationRow.from_note_id)
.join(KnowledgeEntityRow, KnowledgeRelationRow.to_entity_id == KnowledgeEntityRow.id)
.where(func.lower(KnowledgeEntityRow.name) == entity.lower())
.where(KnowledgeRelationRow.from_note_id.is_not(None))
)
conditions.append(KnowledgeNoteRow.id.in_(entity_subq))
for cond in conditions:
stmt = stmt.where(cond)
count_stmt = count_stmt.where(cond)
total = (await session.execute(count_stmt)).scalar_one()
stmt = stmt.order_by(KnowledgeNoteRow.updated_at.desc()).limit(limit).offset(offset)
rows = list((await session.execute(stmt)).scalars())
ids = [r.id for r in rows]
tags_map = await self._tags_for_many(session, ids)
items = []
for r in rows:
d = self._note_to_dict(r)
d["tags"] = tags_map.get(r.id, [])
items.append(d)
return items, int(total)
async def list_all(self, *, include_archived: bool = False) -> list[dict[str, Any]]:
"""Return every note (light fields + tags) for graph export."""
async with self._sf() as session:
stmt = select(KnowledgeNoteRow)
if not include_archived:
stmt = stmt.where(KnowledgeNoteRow.status != "archived")
rows = list((await session.execute(stmt)).scalars())
tags_map = await self._tags_for_many(session, [r.id for r in rows])
out = []
for r in rows:
d = self._note_to_dict(r)
d["tags"] = tags_map.get(r.id, [])
out.append(d)
return out
# -- embeddings (phase 2) --------------------------------------------
async def replace_embeddings(self, note_id: str, chunks: list[tuple[int, str, list[float]]]) -> None:
"""Replace all embedding rows for a note with ``(index, content, vector)``."""
async with self._sf() as session:
await session.execute(delete(KnowledgeEmbeddingRow).where(KnowledgeEmbeddingRow.note_id == note_id))
for idx, content, vector in chunks:
session.add(
KnowledgeEmbeddingRow(
id=_new_id("kbemb"),
note_id=note_id,
chunk_index=idx,
content=content,
vector=list(vector),
created_at=datetime.now(UTC),
)
)
await session.commit()
async def delete_embeddings(self, note_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(KnowledgeEmbeddingRow).where(KnowledgeEmbeddingRow.note_id == note_id))
await session.commit()
async def all_embeddings(self) -> list[dict[str, Any]]:
"""Return every embedding chunk joined to its (non-archived) note."""
async with self._sf() as session:
stmt = (
select(KnowledgeEmbeddingRow, KnowledgeNoteRow.title, KnowledgeNoteRow.status)
.join(KnowledgeNoteRow, KnowledgeNoteRow.id == KnowledgeEmbeddingRow.note_id)
.where(KnowledgeNoteRow.status != "archived")
)
out = []
for emb, title, _status in (await session.execute(stmt)).all():
out.append({"note_id": emb.note_id, "chunk_index": emb.chunk_index, "content": emb.content, "vector": emb.vector or [], "title": title})
return out
async def has_embeddings(self) -> bool:
async with self._sf() as session:
row = (await session.execute(select(KnowledgeEmbeddingRow.id).limit(1))).first()
return row is not None
# -- versions (phase 3) ----------------------------------------------
async def add_version(self, note: dict[str, Any], *, edited_by: str | None) -> None:
async with self._sf() as session:
session.add(
KnowledgeNoteVersionRow(
id=_new_id("kbver"),
note_id=note["id"],
title=note.get("title") or "",
summary=note.get("summary"),
content_md=note.get("content_md") or "",
status=note.get("status") or "approved",
tags_json=note.get("tags") or [],
edited_by=edited_by,
created_at=datetime.now(UTC),
)
)
await session.commit()
async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = select(KnowledgeNoteVersionRow).where(KnowledgeNoteVersionRow.note_id == note_id).order_by(KnowledgeNoteVersionRow.created_at.desc())
out = []
for r in (await session.execute(stmt)).scalars():
d = r.to_dict()
d["tags"] = d.pop("tags_json", [])
d["created_at"] = _iso(d.get("created_at"))
out.append(d)
return out
# -- entities & relations (phase 4) ----------------------------------
async def get_or_create_entity(self, name: str, entity_type: str | None) -> dict[str, Any]:
name = (name or "").strip()
async with self._sf() as session:
stmt = select(KnowledgeEntityRow).where(func.lower(KnowledgeEntityRow.name) == name.lower())
row = (await session.execute(stmt)).scalars().first()
if row is None:
row = KnowledgeEntityRow(id=_new_id("kbent"), name=name, entity_type=entity_type, created_at=datetime.now(UTC))
session.add(row)
await session.commit()
await session.refresh(row)
return {"id": row.id, "name": row.name, "entity_type": row.entity_type}
async def add_relation(
self,
*,
from_note_id: str | None = None,
to_note_id: str | None = None,
from_entity_id: str | None = None,
to_entity_id: str | None = None,
relation_type: str,
weight: float = 1.0,
) -> None:
async with self._sf() as session:
session.add(
KnowledgeRelationRow(
id=_new_id("kbrel"),
from_note_id=from_note_id,
to_note_id=to_note_id,
from_entity_id=from_entity_id,
to_entity_id=to_entity_id,
relation_type=relation_type,
weight=weight,
created_at=datetime.now(UTC),
)
)
await session.commit()
async def clear_note_relations(self, note_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(KnowledgeRelationRow).where(KnowledgeRelationRow.from_note_id == note_id))
await session.commit()
async def list_note_entities(self, note_id: str) -> list[dict[str, Any]]:
"""Return the entities mentioned by a note (for note-detail display)."""
async with self._sf() as session:
stmt = (
select(KnowledgeEntityRow.name, KnowledgeEntityRow.entity_type)
.join(KnowledgeRelationRow, KnowledgeRelationRow.to_entity_id == KnowledgeEntityRow.id)
.where(KnowledgeRelationRow.from_note_id == note_id)
.order_by(KnowledgeEntityRow.name)
)
seen: set[str] = set()
out: list[dict[str, Any]] = []
for name, etype in (await session.execute(stmt)).all():
key = (name or "").lower()
if not name or key in seen:
continue
seen.add(key)
out.append({"name": name, "entity_type": etype})
return out
async def prune_orphan_entities(self) -> int:
"""Delete entity rows no longer referenced by any relation. Returns count."""
async with self._sf() as session:
referenced: set[str] = set()
for col in (KnowledgeRelationRow.from_entity_id, KnowledgeRelationRow.to_entity_id):
for (eid,) in (await session.execute(select(col).where(col.isnot(None)))).all():
referenced.add(eid)
all_ids = list((await session.execute(select(KnowledgeEntityRow.id))).scalars())
orphans = [eid for eid in all_ids if eid not in referenced]
if orphans:
await session.execute(delete(KnowledgeEntityRow).where(KnowledgeEntityRow.id.in_(orphans)))
await session.commit()
return len(orphans)
async def list_entities(self) -> list[dict[str, Any]]:
async with self._sf() as session:
rows = (await session.execute(select(KnowledgeEntityRow))).scalars()
return [{"id": r.id, "name": r.name, "entity_type": r.entity_type} for r in rows]
# -- graph blacklist ---------------------------------------------------
async def list_graph_blacklist(self) -> list[dict[str, Any]]:
"""Return every blacklisted graph node label, newest first."""
async with self._sf() as session:
stmt = select(KnowledgeGraphBlacklistRow).order_by(KnowledgeGraphBlacklistRow.created_at.desc())
rows = (await session.execute(stmt)).scalars()
return [{"id": r.id, "label": r.label, "created_by": r.created_by, "created_at": _iso(r.created_at)} for r in rows]
async def add_graph_blacklist(self, labels: list[str], *, created_by: str | None = None) -> int:
"""Add labels to the graph blacklist (trimmed, case-insensitive dedup). Returns the count actually added."""
cleaned: list[str] = []
seen: set[str] = set()
for raw in labels or []:
label = (raw or "").strip()
if not label or label.lower() in seen:
continue
seen.add(label.lower())
cleaned.append(label)
if not cleaned:
return 0
async with self._sf() as session:
existing = {
value
for (value,) in (
await session.execute(select(func.lower(KnowledgeGraphBlacklistRow.label)).where(func.lower(KnowledgeGraphBlacklistRow.label).in_([label.lower() for label in cleaned])))
).all()
}
added = 0
for label in cleaned:
if label.lower() in existing:
continue
session.add(KnowledgeGraphBlacklistRow(id=_new_id("kbblk"), label=label, created_by=created_by, created_at=datetime.now(UTC)))
added += 1
if added:
await session.commit()
return added
async def remove_graph_blacklist(self, entry_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(delete(KnowledgeGraphBlacklistRow).where(KnowledgeGraphBlacklistRow.id == entry_id))
await session.commit()
return bool(result.rowcount)
async def list_relations(self) -> list[dict[str, Any]]:
async with self._sf() as session:
rows = (await session.execute(select(KnowledgeRelationRow))).scalars()
return [
{
"id": r.id,
"from_note_id": r.from_note_id,
"to_note_id": r.to_note_id,
"from_entity_id": r.from_entity_id,
"to_entity_id": r.to_entity_id,
"relation_type": r.relation_type,
"weight": r.weight,
}
for r in rows
]
# -- sources ----------------------------------------------------------
async def add_sources(self, note_id: str, sources: list[SourceDraft], *, created_by: str | None = None) -> None:
if not sources:
return
import json
async with self._sf() as session:
for src in sources:
session.add(
KnowledgeSourceRow(
id=_new_id("kbsrc"),
note_id=note_id,
source_type=src.source_type,
thread_id=src.thread_id,
message_id=src.message_id,
tool_name=src.tool_name,
title=(src.title or "")[:512] or None,
url=src.url,
snippet=src.snippet,
raw_json=json.dumps(src.raw, ensure_ascii=False) if src.raw else None,
created_by=created_by,
created_at=datetime.now(UTC),
)
)
await session.commit()
# -- folders ----------------------------------------------------------
async def list_folders(self) -> list[dict[str, Any]]:
"""Return every managed directory, ordered by sort_order then path."""
async with self._sf() as session:
stmt = select(KnowledgeFolderRow).order_by(KnowledgeFolderRow.sort_order, KnowledgeFolderRow.path)
return [{"id": r.id, "path": r.path, "name": r.name, "sort_order": r.sort_order} for r in (await session.execute(stmt)).scalars()]
async def create_folder(self, path: str, *, created_by: str | None = None) -> dict[str, Any]:
"""Create a directory by its full slash-joined path (idempotent on path).
Parent directories are auto-created so ``投研/行业/寿险`` materializes all
three levels. Returns the leaf folder row.
"""
segments = [s.strip() for s in (path or "").split("/") if s.strip()]
if not segments:
raise ValueError("folder path is empty")
async with self._sf() as session:
acc: list[str] = []
leaf: KnowledgeFolderRow | None = None
for i, seg in enumerate(segments):
acc.append(seg)
full = "/".join(acc)
row = await self._folder_by_path(session, full)
if row is None:
row = KnowledgeFolderRow(id=_new_id("kbdir"), path=full, name=seg, sort_order=i, created_by=created_by, created_at=datetime.now(UTC))
session.add(row)
await session.flush()
leaf = row
# Capture before commit: the async session expires attributes on
# commit, and re-reading them outside an awaited load would error.
result = {"id": leaf.id, "path": leaf.path, "name": leaf.name, "sort_order": leaf.sort_order}
await session.commit()
return result
@staticmethod
async def _folder_by_path(session: AsyncSession, path: str) -> KnowledgeFolderRow | None:
return (await session.execute(select(KnowledgeFolderRow).where(KnowledgeFolderRow.path == path))).scalars().first()
async def rename_folder(self, folder_id: str, new_path: str) -> dict[str, Any] | None:
"""Rename/move a folder and re-point every descendant folder + note.
Both the folder rows under ``old/…`` and notes whose ``folder`` is ``old``
or ``old/…`` are rewritten to the new prefix, so the move is atomic.
"""
new_path = "/".join(s.strip() for s in (new_path or "").split("/") if s.strip())
if not new_path:
raise ValueError("folder path is empty")
async with self._sf() as session:
row = await session.get(KnowledgeFolderRow, folder_id)
if row is None:
return None
old = row.path
if new_path == old:
return {"id": row.id, "path": row.path, "name": row.name, "sort_order": row.sort_order}
for f in (await session.execute(select(KnowledgeFolderRow).where(or_(KnowledgeFolderRow.path == old, KnowledgeFolderRow.path.like(f"{old}/%"))))).scalars():
f.path = new_path + f.path[len(old):]
f.name = f.path.split("/")[-1]
for n in (await session.execute(select(KnowledgeNoteRow).where(or_(KnowledgeNoteRow.folder == old, KnowledgeNoteRow.folder.like(f"{old}/%"))))).scalars():
n.folder = new_path + (n.folder[len(old):] if n.folder else "")
await session.commit()
refreshed = await session.get(KnowledgeFolderRow, folder_id)
return {"id": refreshed.id, "path": refreshed.path, "name": refreshed.name, "sort_order": refreshed.sort_order}
async def delete_folder(self, folder_id: str, *, reassign_to: str | None = None) -> bool:
"""Delete a folder (and its sub-folders); notes inside move to ``reassign_to`` (or unfiled)."""
async with self._sf() as session:
row = await session.get(KnowledgeFolderRow, folder_id)
if row is None:
return False
old = row.path
target = (reassign_to or "").strip() or None
for n in (await session.execute(select(KnowledgeNoteRow).where(or_(KnowledgeNoteRow.folder == old, KnowledgeNoteRow.folder.like(f"{old}/%"))))).scalars():
n.folder = target
await session.execute(delete(KnowledgeFolderRow).where(or_(KnowledgeFolderRow.path == old, KnowledgeFolderRow.path.like(f"{old}/%"))))
await session.commit()
return True
# -- extraction templates ---------------------------------------------
@staticmethod
def _template_to_dict(row: KnowledgeExtractTemplateRow) -> dict[str, Any]:
d = row.to_dict()
d["created_at"] = _iso(d.get("created_at"))
d["updated_at"] = _iso(d.get("updated_at"))
return d
async def list_templates(self, *, scope: str | None = None, enabled_only: bool = False) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = select(KnowledgeExtractTemplateRow)
if scope:
stmt = stmt.where(or_(KnowledgeExtractTemplateRow.scope == scope, KnowledgeExtractTemplateRow.scope == "both"))
if enabled_only:
stmt = stmt.where(KnowledgeExtractTemplateRow.enabled.is_(True))
stmt = stmt.order_by(KnowledgeExtractTemplateRow.is_default.desc(), KnowledgeExtractTemplateRow.created_at)
return [self._template_to_dict(r) for r in (await session.execute(stmt)).scalars()]
async def get_template(self, template_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(KnowledgeExtractTemplateRow, template_id)
return self._template_to_dict(row) if row else None
async def get_default_template(self, scope: str) -> dict[str, Any] | None:
"""Return the default enabled template applicable to ``scope`` (thread/document)."""
async with self._sf() as session:
stmt = (
select(KnowledgeExtractTemplateRow)
.where(KnowledgeExtractTemplateRow.enabled.is_(True))
.where(KnowledgeExtractTemplateRow.is_default.is_(True))
.where(or_(KnowledgeExtractTemplateRow.scope == scope, KnowledgeExtractTemplateRow.scope == "both"))
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
return self._template_to_dict(row) if row else None
async def template_name_taken(self, name: str, *, exclude_id: str | None = None) -> bool:
async with self._sf() as session:
stmt = select(KnowledgeExtractTemplateRow.id).where(func.lower(KnowledgeExtractTemplateRow.name) == (name or "").strip().lower())
if exclude_id:
stmt = stmt.where(KnowledgeExtractTemplateRow.id != exclude_id)
return (await session.execute(stmt)).first() is not None
async def create_template(self, *, name: str, system_prompt: str, scope: str = "both", description: str | None = None, is_default: bool = False, enabled: bool = True, created_by: str | None = None) -> dict[str, Any]:
async with self._sf() as session:
if is_default:
await self._clear_default(session, scope)
row = KnowledgeExtractTemplateRow(
id=_new_id("kbtpl"),
name=name.strip()[:256],
description=description,
scope=scope,
system_prompt=system_prompt,
is_default=is_default,
enabled=enabled,
created_by=created_by,
updated_by=created_by,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
)
session.add(row)
await session.commit()
await session.refresh(row)
return self._template_to_dict(row)
async def update_template(self, template_id: str, *, name: str | None = None, system_prompt: str | None = None, scope: str | None = None, description: str | None = None, is_default: bool | None = None, enabled: bool | None = None, updated_by: str | None = None) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(KnowledgeExtractTemplateRow, template_id)
if row is None:
return None
if name is not None:
row.name = name.strip()[:256]
if system_prompt is not None:
row.system_prompt = system_prompt
if scope is not None:
row.scope = scope
if description is not None:
row.description = description
if enabled is not None:
row.enabled = enabled
if is_default:
await self._clear_default(session, scope or row.scope)
row.is_default = True
elif is_default is False:
row.is_default = False
row.updated_by = updated_by
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._template_to_dict(row)
async def delete_template(self, template_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(delete(KnowledgeExtractTemplateRow).where(KnowledgeExtractTemplateRow.id == template_id))
await session.commit()
return bool(result.rowcount)
@staticmethod
async def _clear_default(session: AsyncSession, scope: str) -> None:
"""Unset is_default on any template that would collide with a new default for ``scope``."""
scopes = {scope, "both"} if scope != "both" else {"thread", "document", "both"}
for row in (await session.execute(select(KnowledgeExtractTemplateRow).where(KnowledgeExtractTemplateRow.is_default.is_(True)).where(KnowledgeExtractTemplateRow.scope.in_(scopes)))).scalars():
row.is_default = False
@@ -0,0 +1,77 @@
"""Internal domain models for the knowledge module.
These are plain dataclasses passed between the extractor, the vault writer and
the repository. The Gateway router defines its own Pydantic request/response
models on top of the dicts the service returns.
"""
from __future__ import annotations
from dataclasses import dataclass, field
# Allowed note source types (see product-knowledge-base-dev.md §7.1).
SOURCE_TYPES = ("thread", "search", "tool", "file", "manual", "report")
# Allowed lifecycle states.
STATUSES = ("draft", "approved", "archived")
# obsidian-wiki wiki-capture categories.
CATEGORIES = ("synthesis", "concepts", "references", "decision", "journal")
@dataclass
class SourceDraft:
"""A normalized knowledge source (search result, tool call, thread, …)."""
source_type: str
thread_id: str | None = None
message_id: str | None = None
tool_name: str | None = None
title: str | None = None
url: str | None = None
snippet: str | None = None
raw: dict | None = None
@dataclass
class NoteDraft:
"""A knowledge note ready to be persisted + written to the vault.
Produced by the extractor (from a thread / search / tool result) or built
directly for a manual note. ``content_md`` here is the *body* — the vault
writer prepends obsidian-wiki frontmatter.
"""
title: str
summary: str
content_md: str
category: str = "synthesis"
source_type: str = "manual"
source_id: str | None = None
status: str = "approved"
confidence: float = 0.0
# User-assigned directory (slash-joined path); None = unfiled.
folder: str | None = None
tags: list[str] = field(default_factory=list)
key_points: list[str] = field(default_factory=list)
actions: list[str] = field(default_factory=list)
related: list[str] = field(default_factory=list)
sources: list[SourceDraft] = field(default_factory=list)
# Phase 4: extracted graph signal. entities=[{name,type}],
# relations=[{from,to,type}] (names refer to entities or this note's title).
entities: list[dict] = field(default_factory=list)
relations: list[dict] = field(default_factory=list)
# Stable identity hint used for manifest dedup (e.g. ``thread:thread_id``).
source_key: str | None = None
def normalize_status(status: str | None) -> str:
"""Coerce an arbitrary status string to a valid lifecycle state."""
if status in STATUSES:
return status
return "approved"
def normalize_source_type(source_type: str | None) -> str:
"""Coerce an arbitrary source-type string to a valid value."""
if source_type in SOURCE_TYPES:
return source_type
return "manual"
@@ -0,0 +1,160 @@
"""Knowledge search: keyword / vector / hybrid (phase 1 + 2).
- ``keyword`` — title>tags>summary>body scoring over the DB mirror (always works).
- ``vector`` — cosine over per-chunk embeddings (requires a configured
embedding endpoint; otherwise transparently falls back to ``keyword``).
- ``hybrid`` — min-max-normalized blend of keyword + vector.
"""
from __future__ import annotations
import re
from typing import TYPE_CHECKING, Any
from deerflow.knowledge.embeddings import cosine_similarity
from deerflow.knowledge.repository import KnowledgeRepository
if TYPE_CHECKING:
from deerflow.knowledge.embeddings import EmbeddingClient
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
def _tokens(text: str) -> list[str]:
return [t.lower() for t in _TOKEN_RE.findall(text or "")]
def _snippet(content: str, terms: list[str], *, width: int = 160) -> str:
body = re.sub(r"\s+", " ", content or "").strip()
low = body.lower()
for term in terms:
idx = low.find(term)
if idx >= 0:
start = max(0, idx - width // 3)
end = min(len(body), start + width)
prefix = "…" if start > 0 else ""
suffix = "…" if end < len(body) else ""
return f"{prefix}{body[start:end]}{suffix}"
return body[:width] + ("…" if len(body) > width else "")
async def _keyword_scores(repo: KnowledgeRepository, query: str, *, candidate_cap: int) -> dict[str, tuple[float, dict[str, Any]]]:
terms = _tokens(query)
if not terms:
return {}
candidates, _total = await repo.list_notes(limit=candidate_cap, offset=0)
scored: dict[str, tuple[float, dict[str, Any]]] = {}
for note in candidates:
title = (note.get("title") or "").lower()
summary = (note.get("summary") or "").lower()
content = (note.get("content_md") or "").lower()
tags = " ".join(note.get("tags") or []).lower()
score = 0.0
for term in terms:
if term in title:
score += 3.0
if term in tags:
score += 2.0
if term in summary:
score += 2.0
score += min(content.count(term), 5) * 0.5
if score > 0:
scored[note["id"]] = (score, note)
return scored
async def _vector_scores(
repo: KnowledgeRepository,
query: str,
embedding_client: EmbeddingClient,
) -> dict[str, tuple[float, dict[str, Any]]]:
qvec = await embedding_client.embed_query(query)
if not qvec:
return {}
embeddings = await repo.all_embeddings()
best: dict[str, tuple[float, str, str]] = {} # note_id -> (score, title, snippet)
for emb in embeddings:
sim = cosine_similarity(qvec, emb.get("vector") or [])
if sim <= 0:
continue
nid = emb["note_id"]
if nid not in best or sim > best[nid][0]:
best[nid] = (sim, emb.get("title") or "", emb.get("content") or "")
return {nid: (score, {"id": nid, "title": title, "_chunk": chunk}) for nid, (score, title, chunk) in best.items()}
def _normalize(scores: dict[str, float]) -> dict[str, float]:
if not scores:
return {}
mx = max(scores.values())
if mx <= 0:
return {k: 0.0 for k in scores}
return {k: v / mx for k, v in scores.items()}
async def search_notes(
repo: KnowledgeRepository,
query: str,
*,
mode: str = "keyword",
limit: int = 8,
candidate_cap: int = 500,
embedding_client: EmbeddingClient | None = None,
) -> list[dict[str, Any]]:
"""Score notes against ``query`` and return the top ``limit`` matches."""
terms = _tokens(query)
if not terms:
return []
want_vector = mode in ("vector", "hybrid") and embedding_client is not None and embedding_client.is_enabled()
keyword = await _keyword_scores(repo, query, candidate_cap=candidate_cap) if mode != "vector" or not want_vector else {}
vector = await _vector_scores(repo, query, embedding_client) if want_vector else {}
# Pure vector (when available); else keyword; hybrid blends both.
if mode == "vector" and want_vector:
combined = {nid: (sc, meta) for nid, (sc, meta) in vector.items()}
ranked = sorted(combined.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
return await _materialize(repo, ranked, terms)
if mode == "hybrid" and want_vector:
kw_norm = _normalize({k: v[0] for k, v in keyword.items()})
vec_norm = _normalize({k: v[0] for k, v in vector.items()})
ids = set(kw_norm) | set(vec_norm)
blended: dict[str, tuple[float, dict[str, Any]]] = {}
for nid in ids:
score = 0.5 * kw_norm.get(nid, 0.0) + 0.5 * vec_norm.get(nid, 0.0)
meta = keyword.get(nid, (0, None))[1] or vector.get(nid, (0, {}))[1]
blended[nid] = (score, meta)
ranked = sorted(blended.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
return await _materialize(repo, ranked, terms)
# keyword (default, and fallback when vector unavailable)
ranked = sorted(keyword.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
if not ranked and vector: # mode=vector requested but keyword empty: use vector
ranked = sorted(vector.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
return await _materialize(repo, ranked, terms)
async def _materialize(repo: KnowledgeRepository, ranked: list[tuple[str, tuple[float, dict[str, Any]]]], terms: list[str]) -> list[dict[str, Any]]:
max_score = ranked[0][1][0] if ranked else 1.0
results: list[dict[str, Any]] = []
for nid, (score, meta) in ranked:
note = meta if (meta and meta.get("content_md")) else await repo.get_note(nid, with_sources=False)
if note is None:
continue
snippet_src = note.get("content_md") or meta.get("_chunk") or note.get("summary") or ""
results.append(
{
"note_id": nid,
"chunk_id": None,
"title": note.get("title") or meta.get("title"),
"summary": note.get("summary"),
"snippet": _snippet(snippet_src, terms),
"score": round(score / max_score, 4) if max_score else 0.0,
"tags": note.get("tags") or [],
"source_type": note.get("source_type"),
"updated_at": note.get("updated_at"),
"sources": [],
}
)
return results
@@ -0,0 +1,54 @@
"""Sensitive-information redaction for knowledge content (phase 3).
Best-effort masking of secrets / PII before a note is persisted: API keys,
bearer tokens, generic ``key=secret`` assignments, Chinese mobile numbers,
ID-card numbers and email addresses. Conservative by design — it favours not
mangling legitimate prose over catching every possible secret.
"""
from __future__ import annotations
import re
_MASK = "[REDACTED]"
# Order matters: more specific patterns first.
_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
("openai_key", re.compile(r"\bsk-[A-Za-z0-9]{20,}\b")),
("aws_key", re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b")),
("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]{16,}")),
# key/secret/password/token assignment: keep the field name, mask the value.
("assignment", re.compile(r"(?i)\b(api[_-]?key|secret|password|passwd|token|access[_-]?token)\b\s*[:=]\s*[\"']?[^\s\"']{6,}[\"']?")),
("cn_id_card", re.compile(r"\b\d{17}[\dXx]\b")),
("cn_mobile", re.compile(r"(?<!\d)1[3-9]\d{9}(?!\d)")),
("email", re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")),
]
def redact(text: str | None) -> tuple[str, list[str]]:
"""Return ``(redacted_text, hit_kinds)``.
``hit_kinds`` lists which detectors fired (for logging/audit). When ``text``
is falsy it is returned unchanged with no hits.
"""
if not text:
return text or "", []
hits: list[str] = []
out = text
for kind, pattern in _PATTERNS:
def _sub(m: re.Match[str], _kind: str = kind) -> str:
# For assignments keep the field name + delimiter, mask only the value.
if _kind == "assignment":
head = re.split(r"[:=]", m.group(0), maxsplit=1)[0]
sep = ":" if ":" in m.group(0) else "="
return f"{head}{sep} {_MASK}"
if _kind == "bearer":
return f"Bearer {_MASK}"
return _MASK
new_out, n = pattern.subn(_sub, out)
if n:
hits.append(kind)
out = new_out
return out, hits
@@ -0,0 +1,936 @@
"""KnowledgeService — high-level orchestration for the knowledge base.
Ties together the repository (DB mirror), the :class:`ObsidianWikiAdapter`
(Markdown vault + index/log/hot/manifest + graph export) and the extractor
(thread/search → :class:`NoteDraft`). The Gateway router only talks to this
service; everything obsidian-wiki specific stays inside the ``knowledge``
module.
"""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.runtime_paths import runtime_home
from deerflow.knowledge import extractor, llm_extractor
from deerflow.knowledge.chunking import chunk_markdown
from deerflow.knowledge.embeddings import EmbeddingClient
from deerflow.knowledge.markdown_vault import MarkdownVault, VaultPathError, extract_wikilinks, normalize_wikilink_target
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
from deerflow.knowledge.repository import KnowledgeRepository
from deerflow.knowledge.schemas import NoteDraft, SourceDraft, normalize_source_type, normalize_status
from deerflow.knowledge.search import search_notes
from deerflow.knowledge.sources import from_reference_batches, from_search_results
logger = logging.getLogger(__name__)
class EmptyThreadError(ValueError):
"""Raised when a thread has no sediment-able content."""
def _resolve_vault_dir(vault_dir: str) -> Path:
path = Path(vault_dir)
if path.is_absolute():
return path.resolve()
return (runtime_home() / vault_dir).resolve()
def _inject_related_notes(body: str, links: list[str]) -> str:
"""Insert real related-note ``[[wikilinks]]`` into the Related block.
Notes carry only content-specific related notes now (the structural meta-links
were dropped), so this is the sole writer of ``## Related`` links. When the
body has no ``## Related`` section yet, one is appended.
"""
if not links:
return body
bullets = "\n".join(f"- {link}" for link in links)
marker = "## Related\n"
idx = body.find(marker)
if idx == -1:
return body.rstrip() + "\n\n## Related\n\n" + bullets + "\n"
insert_at = idx + len(marker)
return body[:insert_at] + "\n" + bullets + "\n" + body[insert_at:]
class KnowledgeService:
"""Application-facing knowledge base API."""
def __init__(self, repo: KnowledgeRepository, config: KnowledgeConfig) -> None:
self.repo = repo
self.config = config
self.project_name = config.project_name
vault_root = _resolve_vault_dir(config.vault_dir)
self.adapter = ObsidianWikiAdapter(
vault_root,
project_name=config.project_name,
export_html=config.export.graph_html_enabled,
export_json=config.export.graph_json_enabled,
)
self.embedding_client = EmbeddingClient(config.embedding)
try:
self.adapter.ensure_ready()
except Exception: # pragma: no cover - vault init best-effort
logger.exception("Failed to initialize knowledge vault at %s", vault_root)
# -- internal helpers -------------------------------------------------
def _redact_draft(self, draft: NoteDraft) -> NoteDraft:
"""Mask secrets/PII in a draft before persistence (phase 3)."""
if not self.config.sensitive_redaction_enabled:
return draft
from deerflow.knowledge.sensitive import redact
draft.content_md, body_hits = redact(draft.content_md)
draft.summary, _ = redact(draft.summary)
for src in draft.sources:
if src.snippet:
src.snippet, _ = redact(src.snippet)
if body_hits:
logger.info("Knowledge redaction applied to note '%s': %s", draft.title, ",".join(sorted(set(body_hits))))
return draft
async def _find_related_notes(self, draft: NoteDraft) -> list[dict[str, Any]]:
"""Find existing notes to cross-link under ## Related — by **shared tags**.
Matching is strictly tag-based: a candidate must share at least one of the
draft's *distinctive* tags (the per-note project / ``knowledge-base``
defaults are dropped, else everything matches everything). Free-text
title/summary matching was removed because CJK tokenization made generic
words (助手 / 知识图谱 / 页面) link unrelated notes. No distinctive tags →
no related notes (better empty than noisy). Best-effort.
"""
if not self.config.related_notes_enabled:
return []
default_tags = {t.lower() for t in (self.project_name or "", "knowledge-base", "zncm") if t}
distinctive = [t for t in (draft.tags or []) if t and t.lower() not in default_tags]
if not distinctive:
return []
own_title = (draft.title or "").strip().lower()
# Rank candidates by how many distinctive tags they share with the draft.
shared: dict[str, dict[str, Any]] = {}
for tag in distinctive:
try:
items, _total = await self.repo.list_notes(tag=tag, limit=50, offset=0)
except Exception:
logger.exception("Related-notes tag lookup failed for tag '%s'", tag)
continue
for it in items:
nid = it.get("id")
title = (it.get("title") or "").strip()
if not nid or not title or title.lower() == own_title:
continue
entry = shared.setdefault(nid, {"id": nid, "title": title, "count": 0})
entry["count"] += 1
ranked = sorted(shared.values(), key=lambda e: e["count"], reverse=True)
limit = max(1, self.config.related_notes_limit)
return [{"id": e["id"], "title": e["title"]} for e in ranked[:limit]]
async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]:
"""Create a DB row, write the vault file, attach sources, return note."""
draft = self._redact_draft(draft)
related_notes = await self._find_related_notes(draft)
if related_notes:
real_links = [f"[[{n['title']}]]" for n in related_notes]
draft.content_md = _inject_related_notes(draft.content_md, real_links)
draft.related = list(dict.fromkeys([*real_links, *draft.related]))
note = await self.repo.create_note(
title=draft.title,
summary=draft.summary,
content_md=draft.content_md,
source_type=draft.source_type,
source_id=draft.source_id,
status=draft.status,
confidence=draft.confidence,
folder=draft.folder,
tags=draft.tags,
created_by=created_by,
)
note_id = note["id"]
try:
rel = self.adapter.write_note(note_id, draft, created=note["created_at"], updated=note["updated_at"], status=draft.status)
await self.repo.set_vault_path(note_id, rel)
note["vault_path"] = rel
except Exception:
logger.exception("Failed to write knowledge note %s to vault", note_id)
if draft.sources:
await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
await self._reindex_embeddings(note_id, draft.content_md)
await self._store_entities(note_id, draft)
# Record note→note "related" edges (after _store_entities, which clears
# this note's relations) so the graph reflects the cross-links too.
for rel_note in related_notes:
try:
await self.repo.add_relation(from_note_id=note_id, to_note_id=rel_note["id"], relation_type="related")
except Exception:
logger.exception("Failed to link related note %s -> %s", note_id, rel_note["id"])
return await self.repo.get_note(note_id) or note
async def _store_entities(self, note_id: str, draft: NoteDraft) -> None:
"""Persist extracted entities/relations and link them to the note (phase 4)."""
if not self.config.entity_extraction_enabled or not draft.entities:
return
try:
await self.repo.clear_note_relations(note_id)
name_to_id: dict[str, str] = {}
for ent in draft.entities:
name = str(ent.get("name") or "").strip()
if not name:
continue
row = await self.repo.get_or_create_entity(name, ent.get("type"))
name_to_id[name.lower()] = row["id"]
# note -> entity "mentions"
await self.repo.add_relation(from_note_id=note_id, to_entity_id=row["id"], relation_type="mentions")
for rel in draft.relations:
a = name_to_id.get(str(rel.get("from") or "").strip().lower())
b = name_to_id.get(str(rel.get("to") or "").strip().lower())
if a and b and a != b:
await self.repo.add_relation(from_entity_id=a, to_entity_id=b, relation_type=str(rel.get("type") or "relates_to"))
except Exception:
logger.exception("Failed to store entities/relations for note %s", note_id)
async def _reindex_embeddings(self, note_id: str, content_md: str) -> None:
"""Best-effort: (re)build the per-chunk embedding index for a note."""
if not self.embedding_client.is_enabled():
return
try:
chunks = chunk_markdown(content_md)
if not chunks:
await self.repo.delete_embeddings(note_id)
return
vectors = await self.embedding_client.embed_texts(chunks)
if not vectors or len(vectors) != len(chunks):
return
await self.repo.replace_embeddings(note_id, [(i, c, v) for i, (c, v) in enumerate(zip(chunks, vectors, strict=False))])
except Exception:
logger.exception("Failed to (re)index embeddings for note %s", note_id)
# -- capture ----------------------------------------------------------
async def capture_thread(
self,
*,
thread_id: str,
messages: list[dict[str, Any]],
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
status: str = "approved",
tags: list[str] | None = None,
include_sources: bool = True,
reference_batches: list[dict[str, Any]] | None = None,
created_by: str | None = None,
skip_existing: bool = False,
model_name: str | None = None,
template_id: str | None = None,
folder: str | None = None,
) -> dict[str, Any]:
"""Sediment a thread into a knowledge note.
``model_name`` optionally overrides the configured extraction model for
this one capture (falls back to ``config.extract_model_name``).
``template_id`` selects an operator-edited extraction prompt template
(falls back to the ``thread`` default, then the built-in prompt).
``folder`` files the resulting note(s) under a user directory.
Returns ``{"note": <note>, "created": bool}``. Raises
:class:`EmptyThreadError` when there is nothing worth keeping.
"""
extract_prompt = await self._resolve_extract_prompt(template_id, "thread")
# Always-on entry log: proves a capture was triggered for this thread,
# regardless of how it ends (success / skip / fallback). Lets you tell
# "capture never ran" apart from "logs not visible" via one grep.
logger.info(
"[knowledge] capturing thread %s (msgs=%d, llm_extract=%s, skip_existing=%s)",
thread_id,
len(messages or []),
self.config.llm_extract_enabled,
skip_existing,
)
if skip_existing:
existing = await self.repo.find_by_source("thread", thread_id)
if existing is None and self.adapter.manifest.has_source(f"thread:{thread_id}"):
existing = await self.repo.find_by_source("thread", thread_id)
if existing:
logger.info("[knowledge] thread %s already captured (note %s); skipping", thread_id, existing.get("id"))
return {"note": existing, "created": False}
sources = []
if include_sources and reference_batches:
sources = from_reference_batches(reference_batches, thread_id=thread_id, max_items=self.config.max_source_items)
drafts: list[NoteDraft] = []
# Prefer LLM distillation (wiki-capture style) so notes are declarative
# knowledge, not raw chat. A single conversation may split into several
# cross-linked notes. Falls back to the rule-based extractor (always one
# note) when the LLM is disabled/unavailable or returns nothing usable.
llm_attempted = self.config.llm_extract_enabled
llm_fallback = False
if llm_attempted:
try:
drafts = await llm_extractor.extract_thread_llm_multi(
messages,
thread_id=thread_id,
thread_title=thread_title,
mode=mode,
title=title,
tags=tags,
project_name=self.project_name,
sources=sources,
max_messages=self.config.max_thread_messages,
model_name=model_name or self.config.extract_model_name,
max_input_chars=self.config.extract_max_input_chars,
allow_multi=self.config.multi_note_enabled,
system_prompt=extract_prompt,
)
except llm_extractor.LLMRejectedIngest as exc:
# The model judged this conversation worthless (greeting / meta
# chatter). Skip ingestion entirely — do NOT dump the raw chat.
raise EmptyThreadError(f"Thread {thread_id} judged not worth ingesting: {exc}") from exc
if not drafts:
# The LLM path was supposed to run but produced nothing usable
# (API error / non-JSON). We will fall back to the rule-based
# extractor, which yields a low-quality verbatim dump with no
# entities/tags — flag it so it lands as a draft.
llm_fallback = True
if not drafts:
single = extractor.extract_thread(
messages,
thread_id=thread_id,
thread_title=thread_title,
mode=mode,
title=title,
tags=tags,
project_name=self.project_name,
sources=sources,
max_messages=self.config.max_thread_messages,
)
drafts = [single] if single else []
if not drafts:
logger.info("[knowledge] thread %s has no sediment-able content; skipping", thread_id)
raise EmptyThreadError(f"Thread {thread_id} has no sediment-able content")
# When the LLM was enabled but silently fell back to rule-based extraction,
# the note is a verbatim dump (no distillation / entities / tags). Land it
# as a *draft* for review instead of silently approving it, and log loudly
# so the extraction failure is diagnosable (see also the WARNING/EXCEPTION
# logged inside llm_extractor.extract_thread_llm_multi).
effective_status = status
if llm_fallback:
logger.warning(
"[knowledge] thread %s — capture fell back to rule-based extraction "
"(LLM distillation produced nothing usable); landing as DRAFT. "
"Check extraction model '%s' / earlier [knowledge] logs for the cause.",
thread_id,
self.config.extract_model_name or "models[0]",
)
if normalize_status(status) == "approved":
effective_status = "draft"
normalized = normalize_status(effective_status)
for draft in drafts:
draft.status = normalized
if folder:
draft.folder = folder
# Cross-link sibling notes from the same conversation by title, so the
# split pages reference each other (resolved once all are persisted).
if len(drafts) > 1:
for i, draft in enumerate(drafts):
siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
if siblings:
draft.content_md = _inject_related_notes(draft.content_md, siblings)
draft.related = list(dict.fromkeys([*siblings, *draft.related]))
notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
logger.info(
"[knowledge] thread %s captured → %d note(s) [%s], status=%s, llm_fallback=%s",
thread_id,
len(notes),
", ".join(n.get("title", "") for n in notes),
normalized,
llm_fallback,
)
return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback}
async def capture_search(
self,
*,
query: str,
results: list[dict[str, Any]] | None,
thread_id: str | None = None,
message_id: str | None = None,
title: str | None = None,
tags: list[str] | None = None,
status: str = "approved",
created_by: str | None = None,
) -> dict[str, Any]:
"""Sediment search/tool results into a reference note."""
sources = from_search_results(query, results, thread_id=thread_id, message_id=message_id, max_items=self.config.max_source_items)
draft = extractor.extract_search(query, sources, thread_id=thread_id, project_name=self.project_name, title=title, tags=tags)
draft.status = normalize_status(status)
return await self._persist_draft(draft, created_by=created_by)
async def create_manual_note(
self,
*,
title: str,
content_md: str,
tags: list[str] | None = None,
status: str = "approved",
summary: str | None = None,
folder: str | None = None,
created_by: str | None = None,
) -> dict[str, Any]:
"""Create a note directly from product-edited Markdown."""
# Strip any frontmatter the user may have pasted; vault writer adds its own.
_, body = MarkdownVault.parse(content_md)
body = body or content_md
base_tags = list(tags or [])
for default in (self.project_name, "knowledge-base"):
if default and default not in base_tags:
base_tags.append(default)
draft = NoteDraft(
title=title.strip() or "知识笔记",
summary=(summary or body.strip().splitlines()[0] if body.strip() else "")[:280],
content_md=body,
category="synthesis",
source_type="manual",
source_id=None,
status=normalize_status(status),
folder=(folder or None),
tags=base_tags,
confidence=0.5,
related=extractor.related_links(self.project_name),
)
return await self._persist_draft(draft, created_by=created_by)
def _verbatim_file_draft(self, body: str, filename: str, *, title: str | None, tags: list[str] | None, doc_id: str) -> NoteDraft:
"""Build a verbatim fallback note from a file when LLM distillation fails.
Lands as a ``draft`` (like the thread fallback) since it is the raw
document, not distilled knowledge.
"""
base_tags = list(tags or [])
for default in (self.project_name, "knowledge-base"):
if default and default not in base_tags:
base_tags.append(default)
name = (title or Path(filename).stem or "文档知识").strip() or "文档知识"
summary = next((ln.strip() for ln in body.splitlines() if ln.strip()), name)[:280]
return NoteDraft(
title=name,
summary=summary,
content_md=body,
category="references",
source_type="file",
source_id=doc_id,
status="draft",
tags=base_tags,
confidence=0.4,
related=extractor.related_links(self.project_name),
source_key=f"file:{doc_id}",
)
async def capture_file(
self,
*,
content_md: str,
filename: str,
title: str | None = None,
tags: list[str] | None = None,
status: str = "approved",
created_by: str | None = None,
model_name: str | None = None,
skip_existing: bool = False,
template_id: str | None = None,
folder: str | None = None,
) -> dict[str, Any]:
"""Import a file (already converted to Markdown/text) into knowledge notes.
Runs the same wiki-capture LLM distillation as thread capture, so an
uploaded document becomes one or more declarative wiki notes in the vault.
Falls back to a verbatim ``draft`` note when the LLM is disabled or returns
nothing usable. Returns ``{"note", "created", "notes", "llm_fallback"}``;
raises :class:`EmptyThreadError` when there is nothing to keep.
"""
import hashlib
text = (content_md or "").strip()
if not text:
raise EmptyThreadError(f"File {filename} is empty")
# Strip any frontmatter the document carries; the vault writer adds its own.
_, parsed = MarkdownVault.parse(text)
body = (parsed or text).strip()
if not body:
raise EmptyThreadError(f"File {filename} has no body content")
# Stable id from name + size so re-importing the same file can be skipped.
doc_id = hashlib.sha1(f"{filename}:{len(body)}".encode("utf-8")).hexdigest()[:16]
logger.info(
"[knowledge] importing file '%s' (chars=%d, doc_id=%s, llm_extract=%s, skip_existing=%s)",
filename,
len(body),
doc_id,
self.config.llm_extract_enabled,
skip_existing,
)
if skip_existing:
existing = await self.repo.find_by_source("file", doc_id)
if existing:
logger.info("[knowledge] file '%s' already imported (note %s); skipping", filename, existing.get("id"))
return {"note": existing, "created": False, "notes": [existing], "llm_fallback": False}
extract_prompt = await self._resolve_extract_prompt(template_id, "document")
drafts: list[NoteDraft] = []
llm_fallback = False
if self.config.llm_extract_enabled:
try:
drafts = await llm_extractor.extract_document_llm_multi(
body,
doc_id=doc_id,
doc_title=title or Path(filename).stem,
title=title,
tags=tags,
project_name=self.project_name,
model_name=model_name or self.config.extract_model_name,
max_input_chars=self.config.extract_max_input_chars,
allow_multi=self.config.multi_note_enabled,
system_prompt=extract_prompt,
)
except llm_extractor.LLMRejectedIngest as exc:
raise EmptyThreadError(f"File {filename} judged not worth ingesting: {exc}") from exc
if not drafts:
llm_fallback = True
if not drafts:
drafts = [self._verbatim_file_draft(body, filename, title=title, tags=tags, doc_id=doc_id)]
effective_status = status
if llm_fallback and normalize_status(status) == "approved":
effective_status = "draft"
normalized = normalize_status(effective_status)
for draft in drafts:
draft.status = normalized
if folder:
draft.folder = folder
# Cross-link sibling notes split from the same document (resolved by title
# once all are persisted), mirroring multi-note thread capture.
if len(drafts) > 1:
for i, draft in enumerate(drafts):
siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
if siblings:
draft.content_md = _inject_related_notes(draft.content_md, siblings)
draft.related = list(dict.fromkeys([*siblings, *draft.related]))
notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
logger.info(
"[knowledge] file '%s' imported → %d note(s) [%s], status=%s, llm_fallback=%s",
filename,
len(notes),
", ".join(n.get("title", "") for n in notes),
normalized,
llm_fallback,
)
return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback}
# -- CRUD -------------------------------------------------------------
async def list_notes(
self,
*,
keyword: str | None = None,
tag: str | None = None,
entity: str | None = None,
source_type: str | None = None,
status: str | None = None,
folder: str | None = None,
limit: int = 20,
offset: int = 0,
) -> dict[str, Any]:
items, total = await self.repo.list_notes(
keyword=keyword,
tag=tag,
entity=entity,
source_type=normalize_source_type(source_type) if source_type else None,
status=status,
folder=folder,
limit=limit,
offset=offset,
)
return {"items": items, "total": total}
async def get_note(self, note_id: str) -> dict[str, Any] | None:
note = await self.repo.get_note(note_id)
if note is not None:
note["entities"] = await self.repo.list_note_entities(note_id)
return note
async def update_note(
self,
note_id: str,
*,
title: str | None = None,
content_md: str | None = None,
tags: list[str] | None = None,
status: str | None = None,
summary: str | None = None,
folder: str | None = None,
set_folder: bool = False,
updated_by: str | None = None,
) -> dict[str, Any] | None:
current = await self.repo.get_note(note_id, with_sources=False)
if current is None:
return None
# Snapshot the pre-edit state for version history (phase 3).
try:
await self.repo.add_version(current, edited_by=updated_by)
except Exception:
logger.exception("Failed to snapshot version for note %s", note_id)
body = content_md
if body is not None:
_, parsed = MarkdownVault.parse(body)
body = parsed or body
note = await self.repo.update_note(
note_id,
title=title,
summary=summary,
content_md=body,
status=normalize_status(status) if status else None,
tags=tags,
folder=folder,
set_folder=set_folder,
updated_by=updated_by,
)
if note is None:
return None
try:
rel = self.adapter.update_note_file(
note_id,
current.get("vault_path"),
title=note["title"],
content_md=note["content_md"],
tags=note.get("tags") or [],
status=note["status"],
summary=note.get("summary") or "",
category="synthesis",
created=note["created_at"],
updated=note["updated_at"],
)
if rel and rel != current.get("vault_path"):
await self.repo.set_vault_path(note_id, rel)
except Exception:
logger.exception("Failed to update knowledge note %s in vault", note_id)
if note.get("content_md") is not None:
await self._reindex_embeddings(note_id, note["content_md"])
return await self.repo.get_note(note_id)
async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
return await self.repo.list_versions(note_id)
# -- folders ----------------------------------------------------------
async def list_folders(self) -> list[dict[str, Any]]:
return await self.repo.list_folders()
async def create_folder(self, path: str, *, created_by: str | None = None) -> dict[str, Any]:
return await self.repo.create_folder(path, created_by=created_by)
async def rename_folder(self, folder_id: str, new_path: str) -> dict[str, Any] | None:
return await self.repo.rename_folder(folder_id, new_path)
async def delete_folder(self, folder_id: str, *, reassign_to: str | None = None) -> bool:
return await self.repo.delete_folder(folder_id, reassign_to=reassign_to)
# -- extraction templates ---------------------------------------------
async def _resolve_extract_prompt(self, template_id: str | None, scope: str) -> str | None:
"""Return the system prompt for a capture, or ``None`` for the built-in default.
Resolution: explicit ``template_id`` → the scope's default template →
``None`` (the extractor then uses its hard-coded prompt). Best-effort: any
lookup failure degrades to the built-in prompt so capture never breaks.
"""
try:
if template_id:
tpl = await self.repo.get_template(template_id)
if tpl and tpl.get("enabled") and (tpl.get("system_prompt") or "").strip():
return tpl["system_prompt"]
default = await self.repo.get_default_template(scope)
if default and (default.get("system_prompt") or "").strip():
return default["system_prompt"]
except Exception:
logger.exception("Failed to resolve knowledge extraction template (id=%s, scope=%s)", template_id, scope)
return None
async def list_extract_templates(self, *, scope: str | None = None, enabled_only: bool = False) -> list[dict[str, Any]]:
return await self.repo.list_templates(scope=scope, enabled_only=enabled_only)
async def get_extract_template(self, template_id: str) -> dict[str, Any] | None:
return await self.repo.get_template(template_id)
async def create_extract_template(self, **kwargs: Any) -> dict[str, Any]:
return await self.repo.create_template(**kwargs)
async def update_extract_template(self, template_id: str, **kwargs: Any) -> dict[str, Any] | None:
return await self.repo.update_template(template_id, **kwargs)
async def delete_extract_template(self, template_id: str) -> bool:
return await self.repo.delete_template(template_id)
async def template_name_taken(self, name: str, *, exclude_id: str | None = None) -> bool:
return await self.repo.template_name_taken(name, exclude_id=exclude_id)
async def seed_default_templates(self, *, created_by: str | None = None) -> int:
"""Seed the two built-in distillation prompts as editable templates (idempotent).
Runs once on startup so an operator immediately sees — and can edit — the
prompts that drive 对话 / 文档 extraction. Existing templates with the same
name are left untouched. Returns the number of templates created.
"""
from deerflow.knowledge import llm_extractor
guidance = llm_extractor.WIKI_CAPTURE_GUIDANCE
seeds = [
("默认 · 对话提炼", "thread", llm_extractor._SYSTEM_PROMPT + "\n\n" + guidance, "对话沉淀的默认提炼提示词(wiki-capture 规范)。"),
("默认 · 文档提炼", "document", llm_extractor._DOC_SYSTEM_PROMPT + "\n\n" + guidance, "文档导入的默认提炼提示词(wiki-capture 规范)。"),
]
created = 0
for name, scope, prompt, desc in seeds:
try:
if await self.repo.template_name_taken(name):
continue
await self.repo.create_template(name=name, system_prompt=prompt, scope=scope, description=desc, is_default=True, enabled=True, created_by=created_by)
created += 1
except Exception:
logger.exception("Failed to seed knowledge extraction template '%s'", name)
if created:
logger.info("[knowledge] seeded %d default extraction template(s)", created)
return created
async def delete_note(self, note_id: str, *, updated_by: str | None = None, hard: bool = False) -> bool:
"""Soft delete (archive) by default; hard delete removes the vault file."""
current = await self.repo.get_note(note_id, with_sources=False)
if current is None:
return False
note = await self.repo.set_status(note_id, "archived", updated_by=updated_by)
try:
self.adapter.archive_note(note_id, current.get("vault_path"), updated=(note or current)["updated_at"])
if hard and current.get("vault_path"):
self.adapter.vault.delete_note(current["vault_path"])
self.adapter.manifest.remove_page(note_id)
await self.repo.delete_embeddings(note_id)
except Exception:
logger.exception("Failed to archive knowledge note %s in vault", note_id)
# Drop this note's entity mentions + any now-orphaned entities so the graph
# follows the deletion (no lingering nodes from removed notes).
try:
await self.repo.clear_note_relations(note_id)
pruned = await self.repo.prune_orphan_entities()
if pruned:
logger.info("[knowledge] deleted note %s → pruned %d orphan entities", note_id, pruned)
except Exception:
logger.exception("Failed to prune relations/entities for deleted note %s", note_id)
return True
# -- wikilinks --------------------------------------------------------
async def resolve_wikilink(self, target: str) -> dict[str, Any]:
"""Resolve a single ``[[wikilink]]`` target to a navigable destination.
Returns one of:
- ``{"kind": "note", "note_id", "title", "exists": True}`` — a knowledge note
- ``{"kind": "vault", "vault_path", "title", "exists": True}`` — a vault meta/hub page
- ``{"kind": "missing", "exists": False}`` — nothing matches
Resolution order: note id → note vault_path → note title → vault file.
"""
norm = normalize_wikilink_target(target)
base = {"target": target, "normalized": norm}
if not norm:
return {**base, "kind": "missing", "exists": False}
# 1) Direct note id (e.g. "kb_abc...").
if norm.startswith("kb_"):
note = await self.repo.get_note(norm, with_sources=False)
if note and note.get("status") != "archived":
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
# 2) Note by vault path (links from index.md / hot.md / structural hub).
note = await self.repo.find_by_vault_path(norm)
if note:
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
# 3) Note by title (the real related-note cross-links use [[Title]]).
note = await self.repo.find_by_title(norm) or await self.repo.find_by_title(norm.split("/")[-1])
if note:
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
# 4) A vault meta/hub page that has no DB note (index, _meta/taxonomy, hub…).
page = self.read_vault_page(norm)
if page is not None:
return {**base, "kind": "vault", "vault_path": page["path"], "title": page["title"], "exists": True}
return {**base, "kind": "missing", "exists": False}
async def resolve_wikilinks(self, targets: list[str]) -> dict[str, dict[str, Any]]:
"""Resolve many targets at once; returns ``{original_target: resolution}``."""
out: dict[str, dict[str, Any]] = {}
for target in dict.fromkeys(t for t in targets if t and t.strip()):
out[target] = await self.resolve_wikilink(target)
return out
async def resolve_note_wikilinks(self, note_id: str) -> dict[str, dict[str, Any]] | None:
"""Extract every ``[[wikilink]]`` in a note's body and resolve them.
Returns ``{target: resolution}`` or ``None`` when the note is missing.
Numeric ``[[123]]`` citation badges are skipped (handled by the renderer).
"""
note = await self.repo.get_note(note_id, with_sources=False)
if note is None:
return None
targets = [t for t in extract_wikilinks(note.get("content_md") or "") if not t.isdigit()]
return await self.resolve_wikilinks(targets)
def read_vault_page(self, path: str) -> dict[str, Any] | None:
"""Read a raw vault Markdown page (meta/hub pages without a DB note).
Returns ``{"path", "title", "content_md"}`` or ``None`` if the page does
not exist or the path escapes the vault root.
"""
norm = normalize_wikilink_target(path)
if not norm:
return None
rel = f"{norm}.md"
try:
parsed = self.adapter.vault.read_note(rel)
except VaultPathError:
return None
if parsed is None:
return None
fm, body = parsed
title = str(fm.get("title") or "").strip() or norm.split("/")[-1]
return {"path": norm, "title": title, "content_md": body}
# -- search & graph ---------------------------------------------------
async def search(self, query: str, *, mode: str = "keyword", limit: int = 8) -> list[dict[str, Any]]:
return await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
async def build_rag_context(self, query: str, *, limit: int = 5) -> dict[str, Any]:
"""Retrieve top knowledge for ``query`` and format an injectable block.
Returns ``{"text": <system context or "">, "items": [...]}``. Uses hybrid
search when embeddings are available, else keyword (wiki-query style).
"""
mode = "hybrid" if self.embedding_client.is_enabled() else "keyword"
items = await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
if not items:
return {"text": "", "items": []}
lines = [
"以下是产品知识库中与本问题相关的资料。回答时优先参考;若使用其中信息,请在回答中标注来源(note_id)。",
"",
]
for i, it in enumerate(items, 1):
lines.append(f"[知识 {i}] {it.get('title') or ''}")
if it.get("snippet"):
lines.append(it["snippet"])
lines.append(f"来源:{it['note_id']}")
lines.append("")
return {"text": "\n".join(lines).strip(), "items": items}
# -- dedup & merge (phase 3) -----------------------------------------
async def find_duplicates(self) -> list[dict[str, Any]]:
from deerflow.knowledge.dedup import find_duplicate_groups
notes = await self.repo.list_all()
return find_duplicate_groups(notes, threshold=self.config.dedup_similarity_threshold)
async def merge_notes(self, primary_id: str, duplicate_ids: list[str], *, updated_by: str | None = None) -> dict[str, Any] | None:
"""Merge ``duplicate_ids`` into ``primary_id``.
Sources from each duplicate are re-attached to the primary, a wikilink
+ "Merged from" note is appended to the primary body, the duplicates are
archived (soft delete), and a ``merged_into`` relation is recorded.
"""
primary = await self.repo.get_note(primary_id)
if primary is None:
return None
merged_titles: list[str] = []
for dup_id in duplicate_ids:
if dup_id == primary_id:
continue
dup = await self.repo.get_note(dup_id)
if dup is None:
continue
# Move sources.
for src in dup.get("sources") or []:
await self.repo.add_sources(
primary_id,
[
SourceDraft(
source_type=src.get("source_type") or "tool",
thread_id=src.get("thread_id"),
message_id=src.get("message_id"),
tool_name=src.get("tool_name"),
title=src.get("title"),
url=src.get("url"),
snippet=src.get("snippet"),
)
],
created_by=updated_by,
)
await self.repo.add_relation(from_note_id=dup_id, to_note_id=primary_id, relation_type="merged_into")
await self.delete_note(dup_id, updated_by=updated_by)
merged_titles.append(dup.get("title") or dup_id)
if merged_titles:
appended = primary["content_md"].rstrip() + "\n\n## Merged\n\n" + "\n".join(f"- {t}" for t in merged_titles) + "\n"
await self.update_note(primary_id, content_md=appended, updated_by=updated_by)
return await self.repo.get_note(primary_id)
async def reindex_all(self) -> dict[str, int]:
"""Rebuild embeddings for every non-archived note (admin/maintenance)."""
notes = await self.repo.list_all()
count = 0
for note in notes:
await self._reindex_embeddings(note["id"], note.get("content_md") or "")
count += 1
return {"reindexed": count, "embedding_enabled": int(self.embedding_client.is_enabled())}
async def export_graph(self) -> dict[str, Any]:
from deerflow.utils.time import now_iso
notes = await self.repo.list_all()
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
blacklist = await self._graph_blacklist_set()
result = self.adapter.export_graph(notes, entities=entities, relations=relations, updated=now_iso(), blacklist=blacklist)
return {"files": result["files"], "stats": result["stats"]}
async def get_graph(self) -> dict[str, Any]:
"""Return product graph data ``{nodes, edges}`` (notes + entities)."""
from deerflow.knowledge import exporter
notes = await self.repo.list_all()
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
blacklist = await self._graph_blacklist_set()
return exporter.build_graph(notes, entities, relations, blacklist=blacklist)
# -- graph blacklist ---------------------------------------------------
async def _graph_blacklist_set(self) -> set[str]:
from deerflow.knowledge import exporter
items = await self.repo.list_graph_blacklist()
return exporter.normalize_blacklist([item["label"] for item in items])
async def list_graph_blacklist(self) -> list[dict[str, Any]]:
"""Return blacklist entries (node labels hidden from the graph)."""
return await self.repo.list_graph_blacklist()
async def add_graph_blacklist(self, labels: list[str], *, created_by: str | None = None) -> int:
"""Blacklist node labels (entity names / note titles). Returns the count added."""
return await self.repo.add_graph_blacklist(labels, created_by=created_by)
async def remove_graph_blacklist(self, entry_id: str) -> bool:
return await self.repo.remove_graph_blacklist(entry_id)
def export_file_path(self, filename: str) -> Path | None:
return self.adapter.export_file_path(filename)
@@ -0,0 +1,51 @@
"""Built-in obsidian-wiki skill templates.
Mirrors the key intent of ``ar9av/obsidian-wiki`` skills so the backend can
write vault content that stays compatible with the original tooling without
shelling out to its CLI:
- ``wiki-capture`` — how a conversation becomes a *declarative* knowledge note
(frontmatter schema + Context/Finding/Reasoning/Implications/Related body).
- ``wiki-query`` — layered retrieval (hot.md → index.md → frontmatter → body).
- ``wiki-export`` — graph artifacts (graph.json / graph.html / graphml / cypher).
These are intentionally lightweight: the structures here are what
:mod:`deerflow.knowledge.markdown_vault` and :mod:`deerflow.knowledge.extractor`
produce, and what the first-release LLM prompt (future enhancement) should
target.
"""
from __future__ import annotations
# Category → vault sub-directory (wiki-capture page taxonomy, §8 of the dev doc).
CATEGORY_DIRS = {
"synthesis": "synthesis",
"concepts": "concepts",
"concept": "concepts",
"references": "references",
"reference": "references",
"source": "references",
"decision": "synthesis",
"journal": "journal",
"session": "journal",
}
# Body section headings (wiki-capture). Declarative knowledge, never a chat log.
BODY_SECTIONS = ("Context", "Finding / Decision", "Reasoning", "Implications", "Related")
# Inline provenance markers reused from wiki-capture.
INFERRED_MARKER = "^[inferred]"
AMBIGUOUS_MARKER = "^[ambiguous]"
WIKI_CAPTURE_GUIDANCE = """\
Capture *declarative* knowledge, not a chat transcript. Classify the note as one
of: synthesis, concept, source, decision, session. Prefer at least two
[[wikilinks]]. Mark inferred content with ^[inferred] and uncertain/conflicting
content with ^[ambiguous]. Project-specific knowledge belongs under
projects/<project-name>/...
"""
def category_to_dir(category: str | None) -> str:
"""Map a wiki-capture category to its vault sub-directory."""
return CATEGORY_DIRS.get((category or "").strip().lower(), "synthesis")
@@ -0,0 +1,82 @@
"""Normalize knowledge sources from threads / searches / tool results.
Source items come from a few shapes: backend reference-batches (built by
``deerflow.runtime.references``), raw search-result payloads, and tool-call
messages. They are all normalized into :class:`SourceDraft` so the repository
and frontend see one consistent structure.
"""
from __future__ import annotations
from typing import Any
from deerflow.knowledge.schemas import SourceDraft
def _clip(value: Any, limit: int = 2000) -> str | None:
if value is None:
return None
text = str(value).strip()
if not text:
return None
return text[:limit]
def from_reference_batches(batches: list[dict[str, Any]] | None, *, thread_id: str | None = None, max_items: int = 20) -> list[SourceDraft]:
"""Convert backend reference-batches into :class:`SourceDraft`."""
out: list[SourceDraft] = []
seen: set[str] = set()
for batch in batches or []:
msg_id = batch.get("assistant_message_id") or batch.get("id")
for src in batch.get("sources", []) or []:
if not isinstance(src, dict):
continue
url = _clip(src.get("url"), 1024)
title = _clip(src.get("title"), 512) or _clip(src.get("source"), 512) or "来源"
dedup_key = url or f"{title}:{_clip(src.get('snippet'), 80)}"
if dedup_key in seen:
continue
seen.add(dedup_key)
out.append(
SourceDraft(
source_type="search" if url else "tool",
thread_id=thread_id,
message_id=_clip(msg_id, 128),
tool_name=_clip(src.get("skillName"), 128),
title=title,
url=url,
snippet=_clip(src.get("snippet")),
raw=None,
)
)
if len(out) >= max_items:
return out
return out
def from_search_results(
query: str,
results: list[dict[str, Any]] | None,
*,
thread_id: str | None = None,
message_id: str | None = None,
max_items: int = 20,
) -> list[SourceDraft]:
"""Convert raw search results (title/url/snippet) into :class:`SourceDraft`."""
out: list[SourceDraft] = []
for item in (results or [])[:max_items]:
if not isinstance(item, dict):
continue
out.append(
SourceDraft(
source_type="search",
thread_id=thread_id,
message_id=message_id,
tool_name=_clip(item.get("tool_name") or item.get("tool"), 128),
title=_clip(item.get("title"), 512) or query[:200],
url=_clip(item.get("url"), 1024),
snippet=_clip(item.get("snippet") or item.get("content")),
raw={k: item.get(k) for k in ("score",) if item.get(k) is not None} or None,
)
)
return out