Files
deerflow-code/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/exporter.py
T
AdministratorandClaude Opus 4.8 b70ed38d51 feat(knowledge): 图谱跟随删除 + 笔记页展示实体 + 标签美化
图谱不随删除变化的根因:实体/关系删除时从不清理,build_graph 又把所有实体
都当节点,导致已删笔记的实体(六福珠宝/COMEX/CM助手…)永久残留。

- build_graph 只保留被「活笔记」提及的实体,孤儿实体节点不再出现(render 层兜底,
  历史脏数据无需迁移即生效)
- delete_note 删除时 clear_note_relations + prune_orphan_entities,DB 同步收敛
- get_note 返回 entities(repo.list_note_entities),笔记详情页展示实体
- 前端:实体按类型彩色 chip 展示(人物/机构/概念/产品/技术/地区…),标签改为 pill 样式
- 测试:get_note 返回实体 + 删除后图谱与实体表同步收敛

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-08 23:39:25 +08:00

228 lines
10 KiB
Python

"""obsidian-wiki ``wiki-export`` compatible graph generation.
Builds a knowledge graph from the notes (nodes) plus their wikilinks and shared
tags (edges) and writes the four native artifacts into ``<vault>/wiki-export/``:
``graph.json``, ``graph.graphml``, ``cypher.txt`` and a self-contained
``graph.html`` (no external CDN, safe to iframe offline).
"""
from __future__ import annotations
import html
import json
from pathlib import Path
from typing import Any
from deerflow.knowledge.markdown_vault import extract_wikilinks
def _vault_key(vault_path: str | None) -> str | None:
if not vault_path:
return None
key = vault_path.replace("\\", "/")
if key.endswith(".md"):
key = key[:-3]
return key
def build_graph(
notes: list[dict[str, Any]],
entities: list[dict[str, Any]] | None = None,
relations: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""Return ``{nodes, edges}`` from notes (+ optional entities/relations).
Note nodes come from ``notes`` (id, title, tags, vault_path, content_md);
when ``entities``/``relations`` are supplied (phase 4), entity nodes and
note→entity / entity→entity edges are added on top of the wikilink and
shared-tag edges.
"""
nodes = []
by_key: dict[str, str] = {}
note_ids: set[str] = set()
for n in notes:
nodes.append({"id": n["id"], "type": "note", "label": n.get("title") or n["id"], "category": n.get("source_type")})
note_ids.add(n["id"])
key = _vault_key(n.get("vault_path"))
if key:
by_key[key] = n["id"]
# Only surface entities that are still mentioned by a LIVE note. Entities whose
# only referencing notes were deleted/archived would otherwise linger forever
# as orphan nodes ("graph doesn't follow my deletions").
mentioned: set[str] = set()
for rel in relations or []:
if rel.get("from_note_id") in note_ids and rel.get("to_entity_id"):
mentioned.add(rel["to_entity_id"])
entity_ids: set[str] = set()
for ent in entities or []:
if ent["id"] not in mentioned:
continue
nodes.append({"id": ent["id"], "type": "entity", "label": ent.get("name") or ent["id"], "category": ent.get("entity_type")})
entity_ids.add(ent["id"])
edges: list[dict[str, Any]] = []
edge_seen: set[tuple[str, str, str]] = set()
def add_edge(a: str, b: str, label: str, weight: float) -> None:
if a == b:
return
key = (min(a, b), max(a, b), label)
if key in edge_seen:
return
edge_seen.add(key)
edges.append({"id": f"rel_{len(edges) + 1}", "source": a, "target": b, "label": label, "weight": weight})
# Wikilink edges.
for n in notes:
for target in extract_wikilinks(n.get("content_md") or ""):
tid = by_key.get(target.replace("\\", "/").removesuffix(".md"))
if tid:
add_edge(n["id"], tid, "links", 2)
# Shared-tag edges.
tag_to_notes: dict[str, list[str]] = {}
for n in notes:
for tag in n.get("tags") or []:
tag_to_notes.setdefault(tag.lower(), []).append(n["id"])
for tag, ids in tag_to_notes.items():
if len(ids) < 2 or len(ids) > 50: # skip ubiquitous tags (e.g. knowledge-base)
continue
for i in range(len(ids)):
for j in range(i + 1, len(ids)):
add_edge(ids[i], ids[j], "related", 1)
# Entity / relation edges (phase 4) — only when both endpoints exist as nodes.
valid = note_ids | entity_ids
for rel in relations or []:
src = rel.get("from_note_id") or rel.get("from_entity_id")
tgt = rel.get("to_note_id") or rel.get("to_entity_id")
if src in valid and tgt in valid:
add_edge(src, tgt, rel.get("relation_type") or "related", float(rel.get("weight") or 1))
return {"nodes": nodes, "edges": edges}
def to_graphml(graph: dict[str, Any]) -> str:
lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<graphml xmlns="http://graphml.graphdrawing.org/xmlns">',
' <key id="label" for="node" attr.name="label" attr.type="string"/>',
' <key id="elabel" for="edge" attr.name="label" attr.type="string"/>',
' <graph edgedefault="undirected">',
]
for n in graph["nodes"]:
lines.append(f' <node id="{html.escape(str(n["id"]))}"><data key="label">{html.escape(str(n["label"]))}</data></node>')
for e in graph["edges"]:
lines.append(
f' <edge id="{html.escape(str(e["id"]))}" source="{html.escape(str(e["source"]))}" '
f'target="{html.escape(str(e["target"]))}"><data key="elabel">{html.escape(str(e["label"]))}</data></edge>'
)
lines += [" </graph>", "</graphml>"]
return "\n".join(lines)
def to_cypher(graph: dict[str, Any]) -> str:
lines = []
for n in graph["nodes"]:
label = str(n["label"]).replace("'", "\\'")
lines.append(f"CREATE (:Note {{id:'{n['id']}', title:'{label}'}});")
for e in graph["edges"]:
rel = str(e["label"]).upper()
lines.append(f"MATCH (a:Note {{id:'{e['source']}'}}),(b:Note {{id:'{e['target']}'}}) CREATE (a)-[:{rel} {{weight:{e['weight']}}}]->(b);")
return "\n".join(lines)
def to_html(graph: dict[str, Any]) -> str:
"""Self-contained force-directed graph (canvas + vanilla JS, no CDN)."""
data_json = json.dumps(graph, ensure_ascii=False)
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
def export_all(
vault_root: Path | str,
notes: list[dict[str, Any]],
*,
entities: list[dict[str, Any]] | None = None,
relations: list[dict[str, Any]] | None = None,
write_html: bool = True,
write_json: bool = True,
) -> dict[str, Any]:
"""Generate graph artifacts under ``<vault>/wiki-export/`` and return stats."""
out_dir = Path(vault_root).resolve() / "wiki-export"
out_dir.mkdir(parents=True, exist_ok=True)
graph = build_graph(notes, entities, relations)
written: list[str] = []
if write_json:
(out_dir / "graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8")
written.append("graph.json")
(out_dir / "graph.graphml").write_text(to_graphml(graph), encoding="utf-8")
written.append("graph.graphml")
(out_dir / "cypher.txt").write_text(to_cypher(graph), encoding="utf-8")
written.append("cypher.txt")
if write_html:
(out_dir / "graph.html").write_text(to_html(graph), encoding="utf-8")
written.append("graph.html")
return {
"files": written,
"stats": {"nodes": len(graph["nodes"]), "edges": len(graph["edges"])},
"graph": graph,
}
_HTML_TEMPLATE = """<!DOCTYPE html>
<html lang="zh">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>Knowledge Graph</title>
<style>
html,body{margin:0;height:100%;background:#0b1020;color:#e7ecff;font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
#hud{position:fixed;top:10px;left:12px;font-size:12px;opacity:.8;z-index:2}
#tip{position:fixed;padding:4px 8px;background:#111a36;border:1px solid #2a3a66;border-radius:4px;font-size:12px;pointer-events:none;display:none;z-index:3}
canvas{display:block}
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.6}
</style>
</head>
<body>
<div id="hud"></div>
<div id="tip"></div>
<canvas id="c"></canvas>
<script>
const GRAPH = __GRAPH_DATA__;
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
let W=0,H=0;
function resize(){W=canvas.width=innerWidth;H=canvas.height=innerHeight;}
addEventListener('resize',resize);resize();
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id=\"empty\">暂无知识图谱数据</div>');}
const idx={};GRAPH.nodes.forEach((n,i)=>{idx[n.id]=n;n.x=W/2+Math.cos(i)*200*Math.random()+(Math.random()-.5)*200;n.y=H/2+Math.sin(i)*200*Math.random()+(Math.random()-.5)*200;n.vx=0;n.vy=0;});
const edges=GRAPH.edges.map(e=>({s:idx[e.source],t:idx[e.target],w:e.w||e.weight||1})).filter(e=>e.s&&e.t);
let cam={x:0,y:0,k:1},drag=null,pan=null;
function step(){
for(const n of GRAPH.nodes){for(const m of GRAPH.nodes){if(n===m)continue;let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=1200/d2;n.vx+=dx*f*0.0006;n.vy+=dy*f*0.0006;}}
for(const e of edges){let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.sqrt(dx*dx+dy*dy)||1,f=(d-110)*0.01;e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;}
for(const n of GRAPH.nodes){n.vx+=(W/2-n.x)*0.0008;n.vy+=(H/2-n.y)*0.0008;n.x+=n.vx*=0.85;n.y+=n.vy*=0.85;}
}
function draw(){
ctx.setTransform(cam.k,0,0,cam.k,cam.x,cam.y);ctx.clearRect(-cam.x/cam.k,-cam.y/cam.k,W/cam.k,H/cam.k);
ctx.strokeStyle='rgba(120,150,255,.25)';
for(const e of edges){ctx.lineWidth=e.w*0.6;ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();}
for(const n of GRAPH.nodes){const r=6;ctx.fillStyle='#5b8cff';ctx.beginPath();ctx.arc(n.x,n.y,r,0,7);ctx.fill();
ctx.fillStyle='rgba(231,236,255,.85)';ctx.font='11px system-ui';ctx.fillText((n.label||'').slice(0,18),n.x+9,n.y+3);}
}
function loop(){step();draw();requestAnimationFrame(loop);}loop();
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
canvas.addEventListener('mousedown',ev=>{const p=toWorld(ev.clientX,ev.clientY);let hit=null;
for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hit=n;break;}}if(hit)drag=hit;else pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};});
addEventListener('mousemove',ev=>{const p=toWorld(ev.clientX,ev.clientY);if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;}else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
let hov=null;for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hov=n;break;}}
if(hov){tip.style.display='block';tip.style.left=(ev.clientX+12)+'px';tip.style.top=(ev.clientY+12)+'px';tip.textContent=hov.label;}else{tip.style.display='none';}});
addEventListener('mouseup',()=>{drag=null;pan=null;});
canvas.addEventListener('wheel',ev=>{ev.preventDefault();const s=ev.deltaY<0?1.1:0.9;cam.k*=s;cam.x=ev.clientX-(ev.clientX-cam.x)*s;cam.y=ev.clientY-(ev.clientY-cam.y)*s;},{passive:false});
</script>
</body>
</html>"""