feat: 产品内知识库(obsidian-wiki 兼容 vault)
后端:新增自包含 deerflow.knowledge 模块(models/repository/schemas/ extractor/sources/markdown_vault/manifest/obsidian_adapter/exporter/ search/service),通过 deps.py 装配为 app.state.knowledge_service, 挂载 /api/knowledge 路由;config.yaml 增加 knowledge 配置段, AppConfig 注册 KnowledgeConfig,ORM 注册 4 张 knowledge_* 表。 前端:新增知识库页面(KnowledgeBasePage/KnowledgeGraphPage)、 core/knowledge api/hooks/types、聊天侧 knowledge-save-trigger, 侧边栏增加「知识库」入口。 知识库为全局共享,不做 user_id 隔离,写入记录 created_by/updated_by 作为审计。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
80d2db6315
commit
cc99ebba81
28 files changed
+3590
-6
No files matched your search
@@ -0,0 +1,190 @@
|
||||
"""obsidian-wiki ``wiki-export`` compatible graph generation.
|
||||
|
||||
Builds a knowledge graph from the notes (nodes) plus their wikilinks and shared
|
||||
tags (edges) and writes the four native artifacts into ``<vault>/wiki-export/``:
|
||||
``graph.json``, ``graph.graphml``, ``cypher.txt`` and a self-contained
|
||||
``graph.html`` (no external CDN, safe to iframe offline).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.markdown_vault import extract_wikilinks
|
||||
|
||||
|
||||
def _vault_key(vault_path: str | None) -> str | None:
|
||||
if not vault_path:
|
||||
return None
|
||||
key = vault_path.replace("\\", "/")
|
||||
if key.endswith(".md"):
|
||||
key = key[:-3]
|
||||
return key
|
||||
|
||||
|
||||
def build_graph(notes: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Return ``{nodes, edges}`` from note dicts (id, title, tags, vault_path, content_md)."""
|
||||
nodes = []
|
||||
by_key: dict[str, str] = {}
|
||||
for n in notes:
|
||||
nodes.append({"id": n["id"], "type": "note", "label": n.get("title") or n["id"], "category": n.get("source_type")})
|
||||
key = _vault_key(n.get("vault_path"))
|
||||
if key:
|
||||
by_key[key] = n["id"]
|
||||
|
||||
edges: list[dict[str, Any]] = []
|
||||
edge_seen: set[tuple[str, str, str]] = set()
|
||||
|
||||
def add_edge(a: str, b: str, label: str, weight: float) -> None:
|
||||
if a == b:
|
||||
return
|
||||
key = (min(a, b), max(a, b), label)
|
||||
if key in edge_seen:
|
||||
return
|
||||
edge_seen.add(key)
|
||||
edges.append({"id": f"rel_{len(edges) + 1}", "source": a, "target": b, "label": label, "weight": weight})
|
||||
|
||||
# Wikilink edges.
|
||||
for n in notes:
|
||||
for target in extract_wikilinks(n.get("content_md") or ""):
|
||||
tid = by_key.get(target.replace("\\", "/").removesuffix(".md"))
|
||||
if tid:
|
||||
add_edge(n["id"], tid, "links", 2)
|
||||
|
||||
# Shared-tag edges.
|
||||
tag_to_notes: dict[str, list[str]] = {}
|
||||
for n in notes:
|
||||
for tag in n.get("tags") or []:
|
||||
tag_to_notes.setdefault(tag.lower(), []).append(n["id"])
|
||||
for tag, ids in tag_to_notes.items():
|
||||
if len(ids) < 2 or len(ids) > 50: # skip ubiquitous tags (e.g. knowledge-base)
|
||||
continue
|
||||
for i in range(len(ids)):
|
||||
for j in range(i + 1, len(ids)):
|
||||
add_edge(ids[i], ids[j], "related", 1)
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
|
||||
def to_graphml(graph: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<graphml xmlns="http://graphml.graphdrawing.org/xmlns">',
|
||||
' <key id="label" for="node" attr.name="label" attr.type="string"/>',
|
||||
' <key id="elabel" for="edge" attr.name="label" attr.type="string"/>',
|
||||
' <graph edgedefault="undirected">',
|
||||
]
|
||||
for n in graph["nodes"]:
|
||||
lines.append(f' <node id="{html.escape(str(n["id"]))}"><data key="label">{html.escape(str(n["label"]))}</data></node>')
|
||||
for e in graph["edges"]:
|
||||
lines.append(
|
||||
f' <edge id="{html.escape(str(e["id"]))}" source="{html.escape(str(e["source"]))}" '
|
||||
f'target="{html.escape(str(e["target"]))}"><data key="elabel">{html.escape(str(e["label"]))}</data></edge>'
|
||||
)
|
||||
lines += [" </graph>", "</graphml>"]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_cypher(graph: dict[str, Any]) -> str:
|
||||
lines = []
|
||||
for n in graph["nodes"]:
|
||||
label = str(n["label"]).replace("'", "\\'")
|
||||
lines.append(f"CREATE (:Note {{id:'{n['id']}', title:'{label}'}});")
|
||||
for e in graph["edges"]:
|
||||
rel = str(e["label"]).upper()
|
||||
lines.append(f"MATCH (a:Note {{id:'{e['source']}'}}),(b:Note {{id:'{e['target']}'}}) CREATE (a)-[:{rel} {{weight:{e['weight']}}}]->(b);")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_html(graph: dict[str, Any]) -> str:
|
||||
"""Self-contained force-directed graph (canvas + vanilla JS, no CDN)."""
|
||||
data_json = json.dumps(graph, ensure_ascii=False)
|
||||
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
|
||||
|
||||
|
||||
def export_all(
|
||||
vault_root: Path | str,
|
||||
notes: list[dict[str, Any]],
|
||||
*,
|
||||
write_html: bool = True,
|
||||
write_json: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate graph artifacts under ``<vault>/wiki-export/`` and return stats."""
|
||||
out_dir = Path(vault_root).resolve() / "wiki-export"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
graph = build_graph(notes)
|
||||
written: list[str] = []
|
||||
if write_json:
|
||||
(out_dir / "graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
written.append("graph.json")
|
||||
(out_dir / "graph.graphml").write_text(to_graphml(graph), encoding="utf-8")
|
||||
written.append("graph.graphml")
|
||||
(out_dir / "cypher.txt").write_text(to_cypher(graph), encoding="utf-8")
|
||||
written.append("cypher.txt")
|
||||
if write_html:
|
||||
(out_dir / "graph.html").write_text(to_html(graph), encoding="utf-8")
|
||||
written.append("graph.html")
|
||||
return {
|
||||
"files": written,
|
||||
"stats": {"nodes": len(graph["nodes"]), "edges": len(graph["edges"])},
|
||||
"graph": graph,
|
||||
}
|
||||
|
||||
|
||||
_HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="zh">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>Knowledge Graph</title>
|
||||
<style>
|
||||
html,body{margin:0;height:100%;background:#0b1020;color:#e7ecff;font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
|
||||
#hud{position:fixed;top:10px;left:12px;font-size:12px;opacity:.8;z-index:2}
|
||||
#tip{position:fixed;padding:4px 8px;background:#111a36;border:1px solid #2a3a66;border-radius:4px;font-size:12px;pointer-events:none;display:none;z-index:3}
|
||||
canvas{display:block}
|
||||
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.6}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="hud"></div>
|
||||
<div id="tip"></div>
|
||||
<canvas id="c"></canvas>
|
||||
<script>
|
||||
const GRAPH = __GRAPH_DATA__;
|
||||
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
|
||||
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
|
||||
let W=0,H=0;
|
||||
function resize(){W=canvas.width=innerWidth;H=canvas.height=innerHeight;}
|
||||
addEventListener('resize',resize);resize();
|
||||
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
|
||||
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id=\"empty\">暂无知识图谱数据</div>');}
|
||||
const idx={};GRAPH.nodes.forEach((n,i)=>{idx[n.id]=n;n.x=W/2+Math.cos(i)*200*Math.random()+(Math.random()-.5)*200;n.y=H/2+Math.sin(i)*200*Math.random()+(Math.random()-.5)*200;n.vx=0;n.vy=0;});
|
||||
const edges=GRAPH.edges.map(e=>({s:idx[e.source],t:idx[e.target],w:e.w||e.weight||1})).filter(e=>e.s&&e.t);
|
||||
let cam={x:0,y:0,k:1},drag=null,pan=null;
|
||||
function step(){
|
||||
for(const n of GRAPH.nodes){for(const m of GRAPH.nodes){if(n===m)continue;let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=1200/d2;n.vx+=dx*f*0.0006;n.vy+=dy*f*0.0006;}}
|
||||
for(const e of edges){let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.sqrt(dx*dx+dy*dy)||1,f=(d-110)*0.01;e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;}
|
||||
for(const n of GRAPH.nodes){n.vx+=(W/2-n.x)*0.0008;n.vy+=(H/2-n.y)*0.0008;n.x+=n.vx*=0.85;n.y+=n.vy*=0.85;}
|
||||
}
|
||||
function draw(){
|
||||
ctx.setTransform(cam.k,0,0,cam.k,cam.x,cam.y);ctx.clearRect(-cam.x/cam.k,-cam.y/cam.k,W/cam.k,H/cam.k);
|
||||
ctx.strokeStyle='rgba(120,150,255,.25)';
|
||||
for(const e of edges){ctx.lineWidth=e.w*0.6;ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();}
|
||||
for(const n of GRAPH.nodes){const r=6;ctx.fillStyle='#5b8cff';ctx.beginPath();ctx.arc(n.x,n.y,r,0,7);ctx.fill();
|
||||
ctx.fillStyle='rgba(231,236,255,.85)';ctx.font='11px system-ui';ctx.fillText((n.label||'').slice(0,18),n.x+9,n.y+3);}
|
||||
}
|
||||
function loop(){step();draw();requestAnimationFrame(loop);}loop();
|
||||
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
|
||||
canvas.addEventListener('mousedown',ev=>{const p=toWorld(ev.clientX,ev.clientY);let hit=null;
|
||||
for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hit=n;break;}}if(hit)drag=hit;else pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};});
|
||||
addEventListener('mousemove',ev=>{const p=toWorld(ev.clientX,ev.clientY);if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;}else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
|
||||
let hov=null;for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hov=n;break;}}
|
||||
if(hov){tip.style.display='block';tip.style.left=(ev.clientX+12)+'px';tip.style.top=(ev.clientY+12)+'px';tip.textContent=hov.label;}else{tip.style.display='none';}});
|
||||
addEventListener('mouseup',()=>{drag=null;pan=null;});
|
||||
canvas.addEventListener('wheel',ev=>{ev.preventDefault();const s=ev.deltaY<0?1.1:0.9;cam.k*=s;cam.x=ev.clientX-(ev.clientX-cam.x)*s;cam.y=ev.clientY-(ev.clientY-cam.y)*s;},{passive:false});
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
Reference in new issue
Block a user