初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
@@ -0,0 +1,41 @@
|
||||
"""Product knowledge base module (obsidian-wiki compatible).
|
||||
|
||||
Self-contained: knowledge ingestion, storage (DB mirror + Markdown vault),
|
||||
search and graph export all live here. The Gateway only depends on
|
||||
:class:`KnowledgeService` via :func:`make_knowledge_service`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
|
||||
from deerflow.knowledge.repository import KnowledgeRepository
|
||||
from deerflow.knowledge.service import EmptyThreadError, KnowledgeService
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.config.knowledge_config import KnowledgeConfig
|
||||
|
||||
__all__ = [
|
||||
"EmptyThreadError",
|
||||
"KnowledgeRepository",
|
||||
"KnowledgeService",
|
||||
"ObsidianWikiAdapter",
|
||||
"make_knowledge_service",
|
||||
]
|
||||
|
||||
|
||||
def make_knowledge_service(
|
||||
session_factory: async_sessionmaker[AsyncSession] | None,
|
||||
config: KnowledgeConfig,
|
||||
) -> KnowledgeService | None:
|
||||
"""Build the knowledge service.
|
||||
|
||||
Returns ``None`` when persistence is memory-only (no session factory) or
|
||||
the feature is disabled — the router then responds 503.
|
||||
"""
|
||||
if session_factory is None or not config.enabled:
|
||||
return None
|
||||
return KnowledgeService(KnowledgeRepository(session_factory), config)
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Background auto-sediment queue (phase 3).
|
||||
|
||||
A single asyncio worker drains a bounded queue of finished conversations and
|
||||
sediments the worthwhile ones into the knowledge base — entirely off the chat
|
||||
request path, so it never adds latency to a reply. Enqueue happens from
|
||||
``KnowledgeRagMiddleware.after_agent`` when ``knowledge.auto_ingest_enabled``.
|
||||
|
||||
Value judgment is intentionally cheap (min assistant chars + dedup via
|
||||
``skip_existing``); the LLM still distills the kept conversations via the normal
|
||||
``capture_thread`` path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_QUEUE: AutoIngestQueue | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _QueueUser:
|
||||
"""Minimal CurrentUser stand-in (only ``.id`` is needed for context)."""
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class AutoIngestItem:
|
||||
thread_id: str
|
||||
user_id: str
|
||||
messages: list[dict[str, Any]]
|
||||
thread_title: str | None = None
|
||||
reference_batches: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
class AutoIngestQueue:
|
||||
"""Bounded background queue + single worker for auto-sedimentation."""
|
||||
|
||||
def __init__(self, service_provider: Callable[[], Any], *, min_chars: int = 300, maxsize: int = 256) -> None:
|
||||
self._provider = service_provider
|
||||
self._min_chars = min_chars
|
||||
self._queue: asyncio.Queue[AutoIngestItem] = asyncio.Queue(maxsize=maxsize)
|
||||
self._task: asyncio.Task | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._worker(), name="knowledge-auto-ingest")
|
||||
logger.info("Knowledge auto-ingest worker started")
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await self._task
|
||||
self._task = None
|
||||
|
||||
def enqueue(self, item: AutoIngestItem) -> None:
|
||||
try:
|
||||
self._queue.put_nowait(item)
|
||||
except asyncio.QueueFull:
|
||||
logger.warning("Knowledge auto-ingest queue full; dropping thread %s", item.thread_id)
|
||||
|
||||
def _worth_ingesting(self, item: AutoIngestItem) -> bool:
|
||||
chars = 0
|
||||
for m in item.messages:
|
||||
if str(m.get("type") or m.get("role") or "").lower() == "ai":
|
||||
content = m.get("content")
|
||||
chars += len(content) if isinstance(content, str) else len(str(content or ""))
|
||||
return chars >= self._min_chars
|
||||
|
||||
async def _process(self, item: AutoIngestItem) -> None:
|
||||
service = self._provider()
|
||||
if service is None:
|
||||
return
|
||||
if not self._worth_ingesting(item):
|
||||
logger.debug("Auto-ingest skipped (below min chars) thread=%s", item.thread_id)
|
||||
return
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
|
||||
token = set_current_user(_QueueUser(id=item.user_id))
|
||||
try:
|
||||
result = await service.capture_thread(
|
||||
thread_id=item.thread_id,
|
||||
messages=item.messages,
|
||||
thread_title=item.thread_title,
|
||||
status="draft", # auto-sedimented notes land as drafts for review
|
||||
include_sources=True,
|
||||
reference_batches=item.reference_batches,
|
||||
created_by=item.user_id,
|
||||
skip_existing=True,
|
||||
)
|
||||
if result.get("created"):
|
||||
logger.info("Auto-ingested thread %s -> note %s", item.thread_id, result["note"]["id"])
|
||||
except Exception:
|
||||
logger.exception("Auto-ingest failed for thread %s", item.thread_id)
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
|
||||
async def _worker(self) -> None:
|
||||
while True:
|
||||
item = await self._queue.get()
|
||||
try:
|
||||
await self._process(item)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Auto-ingest worker error")
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
|
||||
|
||||
def set_auto_ingest_queue(queue: AutoIngestQueue | None) -> None:
|
||||
global _QUEUE
|
||||
_QUEUE = queue
|
||||
|
||||
|
||||
def get_auto_ingest_queue() -> AutoIngestQueue | None:
|
||||
return _QUEUE
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Markdown chunking for embedding/index (phase 2).
|
||||
|
||||
Splits a note body into overlapping chunks along Markdown structure (headings →
|
||||
paragraphs) so each embedded chunk is semantically coherent and bounded in
|
||||
size. Frontmatter is assumed already stripped by the caller.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_HEADING_RE = re.compile(r"^#{1,6}\s+", re.MULTILINE)
|
||||
|
||||
|
||||
def _split_paragraphs(text: str) -> list[str]:
|
||||
return [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
|
||||
|
||||
|
||||
def chunk_markdown(content: str, *, max_chars: int = 800, overlap: int = 120) -> list[str]:
|
||||
"""Return a list of text chunks for ``content``.
|
||||
|
||||
Sections are cut at Markdown headings; long sections are further split into
|
||||
~``max_chars`` windows with ``overlap`` characters of carry-over.
|
||||
"""
|
||||
text = (content or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# Split into heading-delimited sections, keeping the heading with its body.
|
||||
sections: list[str] = []
|
||||
last = 0
|
||||
for m in _HEADING_RE.finditer(text):
|
||||
if m.start() > last:
|
||||
sections.append(text[last : m.start()].strip())
|
||||
last = m.start()
|
||||
sections.append(text[last:].strip())
|
||||
sections = [s for s in sections if s]
|
||||
|
||||
chunks: list[str] = []
|
||||
for section in sections:
|
||||
if len(section) <= max_chars:
|
||||
chunks.append(section)
|
||||
continue
|
||||
# Pack paragraphs into windows, then hard-split anything still too long.
|
||||
buf = ""
|
||||
for para in _split_paragraphs(section):
|
||||
if len(buf) + len(para) + 2 <= max_chars:
|
||||
buf = f"{buf}\n\n{para}".strip()
|
||||
else:
|
||||
if buf:
|
||||
chunks.append(buf)
|
||||
if len(para) <= max_chars:
|
||||
buf = para
|
||||
else:
|
||||
for i in range(0, len(para), max_chars - overlap):
|
||||
chunks.append(para[i : i + max_chars])
|
||||
buf = ""
|
||||
if buf:
|
||||
chunks.append(buf)
|
||||
return [c for c in chunks if c.strip()]
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Duplicate detection for knowledge notes (phase 3).
|
||||
|
||||
First-release heuristic: token Jaccard over ``title + summary + tags`` plus an
|
||||
exact ``content_md`` hash match. Cheap, deterministic and embedding-free so it
|
||||
works on offline deployments. Returns *suggestions*; merging is an explicit,
|
||||
user-driven action.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.manifest import content_hash
|
||||
|
||||
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
|
||||
|
||||
|
||||
def _tokens(*texts: str) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for t in texts:
|
||||
out.update(m.lower() for m in _TOKEN_RE.findall(t or ""))
|
||||
return out
|
||||
|
||||
|
||||
def _jaccard(a: set[str], b: set[str]) -> float:
|
||||
if not a or not b:
|
||||
return 0.0
|
||||
inter = len(a & b)
|
||||
union = len(a | b)
|
||||
return inter / union if union else 0.0
|
||||
|
||||
|
||||
def find_duplicate_groups(notes: list[dict[str, Any]], *, threshold: float = 0.82, max_notes: int = 800) -> list[dict[str, Any]]:
|
||||
"""Return groups ``{primary, duplicates:[{id,title,score,reason}]}``.
|
||||
|
||||
``primary`` is the most recently updated note in each cluster.
|
||||
"""
|
||||
notes = notes[:max_notes]
|
||||
feats: list[tuple[dict[str, Any], set[str], str]] = []
|
||||
for n in notes:
|
||||
toks = _tokens(n.get("title") or "", n.get("summary") or "", " ".join(n.get("tags") or []))
|
||||
feats.append((n, toks, content_hash(n.get("content_md") or "")))
|
||||
|
||||
used: set[str] = set()
|
||||
groups: list[dict[str, Any]] = []
|
||||
for i in range(len(feats)):
|
||||
ni, ti, hi = feats[i]
|
||||
if ni["id"] in used:
|
||||
continue
|
||||
dups: list[dict[str, Any]] = []
|
||||
for j in range(i + 1, len(feats)):
|
||||
nj, tj, hj = feats[j]
|
||||
if nj["id"] in used:
|
||||
continue
|
||||
if hi and hi == hj:
|
||||
score, reason = 1.0, "identical-content"
|
||||
else:
|
||||
score = _jaccard(ti, tj)
|
||||
reason = "similar-title-summary"
|
||||
if score >= threshold:
|
||||
dups.append({"id": nj["id"], "title": nj.get("title"), "score": round(score, 4), "reason": reason})
|
||||
used.add(nj["id"])
|
||||
if dups:
|
||||
used.add(ni["id"])
|
||||
# Primary = newest by updated_at.
|
||||
cluster = [ni, *[next(f[0] for f in feats if f[0]["id"] == d["id"]) for d in dups]]
|
||||
primary = max(cluster, key=lambda x: x.get("updated_at") or "")
|
||||
others = [
|
||||
{"id": c["id"], "title": c.get("title"), "score": next((d["score"] for d in dups if d["id"] == c["id"]), 1.0)}
|
||||
for c in cluster
|
||||
if c["id"] != primary["id"]
|
||||
]
|
||||
groups.append({"primary": {"id": primary["id"], "title": primary.get("title")}, "duplicates": others})
|
||||
return groups
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Embedding generation for vector RAG (phase 2).
|
||||
|
||||
Talks to an OpenAI-compatible ``/embeddings`` endpoint configured under
|
||||
``knowledge.embedding``. Everything here is **best-effort**: if embeddings are
|
||||
disabled, unconfigured, or the endpoint errors, callers get an empty result and
|
||||
the search layer falls back to keyword scoring — so vector RAG never breaks a
|
||||
deployment that has no embedding model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.config.knowledge_config import KnowledgeEmbeddingConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def cosine_similarity(a: list[float], b: list[float]) -> float:
|
||||
"""Cosine similarity of two equal-length vectors (0 when degenerate)."""
|
||||
if not a or not b or len(a) != len(b):
|
||||
return 0.0
|
||||
dot = sum(x * y for x, y in zip(a, b, strict=False))
|
||||
na = math.sqrt(sum(x * x for x in a))
|
||||
nb = math.sqrt(sum(y * y for y in b))
|
||||
if na == 0 or nb == 0:
|
||||
return 0.0
|
||||
return dot / (na * nb)
|
||||
|
||||
|
||||
class EmbeddingClient:
|
||||
"""Thin async client over an OpenAI-compatible embeddings endpoint."""
|
||||
|
||||
def __init__(self, config: KnowledgeEmbeddingConfig) -> None:
|
||||
self.config = config
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
return bool(self.config.enabled and self.config.base_url and self.config.model)
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Return one vector per input text; ``[]`` on any failure/disabled."""
|
||||
if not self.is_enabled() or not texts:
|
||||
return []
|
||||
try:
|
||||
import httpx
|
||||
except Exception: # pragma: no cover - httpx is a core dep
|
||||
logger.warning("httpx unavailable; knowledge embeddings disabled")
|
||||
return []
|
||||
|
||||
url = self.config.base_url.rstrip("/") + "/embeddings"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if self.config.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.config.api_key}"
|
||||
|
||||
vectors: list[list[float]] = []
|
||||
batch = max(1, self.config.batch_size)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.config.timeout_seconds) as client:
|
||||
for start in range(0, len(texts), batch):
|
||||
chunk = texts[start : start + batch]
|
||||
payload: dict = {"model": self.config.model, "input": chunk}
|
||||
if self.config.dimensions:
|
||||
payload["dimensions"] = self.config.dimensions
|
||||
resp = await client.post(url, headers=headers, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
items = sorted(data.get("data", []), key=lambda d: d.get("index", 0))
|
||||
vectors.extend([item.get("embedding", []) for item in items])
|
||||
except Exception:
|
||||
logger.exception("Knowledge embedding request failed; falling back to keyword search")
|
||||
return []
|
||||
return vectors
|
||||
|
||||
async def embed_query(self, text: str) -> list[float]:
|
||||
vecs = await self.embed_texts([text])
|
||||
return vecs[0] if vecs else []
|
||||
@@ -0,0 +1,463 @@
|
||||
"""obsidian-wiki ``wiki-export`` compatible graph generation.
|
||||
|
||||
Builds a knowledge graph from the notes (nodes) plus their wikilinks and shared
|
||||
tags (edges) and writes the four native artifacts into ``<vault>/wiki-export/``:
|
||||
``graph.json``, ``graph.graphml``, ``cypher.txt`` and a self-contained
|
||||
``graph.html`` (no external CDN, safe to iframe offline).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.markdown_vault import extract_wikilinks
|
||||
|
||||
|
||||
def _vault_key(vault_path: str | None) -> str | None:
|
||||
if not vault_path:
|
||||
return None
|
||||
key = vault_path.replace("\\", "/")
|
||||
if key.endswith(".md"):
|
||||
key = key[:-3]
|
||||
return key
|
||||
|
||||
|
||||
def normalize_blacklist(labels: list[str] | set[str] | None) -> set[str]:
|
||||
"""Normalize blacklist labels for matching (trimmed, lower-cased, non-empty)."""
|
||||
out: set[str] = set()
|
||||
for raw in labels or []:
|
||||
label = (raw or "").strip().lower()
|
||||
if label:
|
||||
out.add(label)
|
||||
return out
|
||||
|
||||
|
||||
def build_graph(
|
||||
notes: list[dict[str, Any]],
|
||||
entities: list[dict[str, Any]] | None = None,
|
||||
relations: list[dict[str, Any]] | None = None,
|
||||
*,
|
||||
blacklist: set[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return ``{nodes, edges}`` from notes (+ optional entities/relations).
|
||||
|
||||
Note nodes come from ``notes`` (id, title, tags, vault_path, content_md);
|
||||
when ``entities``/``relations`` are supplied (phase 4), entity nodes and
|
||||
note→entity / entity→entity edges are added on top of the wikilink and
|
||||
shared-tag edges.
|
||||
|
||||
``blacklist`` is a set of *normalized* labels (see :func:`normalize_blacklist`);
|
||||
notes whose title and entities whose name match are dropped, together with
|
||||
every edge touching them (edges only join surviving nodes).
|
||||
"""
|
||||
blocked = blacklist or set()
|
||||
|
||||
def _is_blocked(label: str | None) -> bool:
|
||||
return bool(blocked) and (label or "").strip().lower() in blocked
|
||||
|
||||
# Drop blacklisted notes up front so the wikilink / shared-tag edge loops
|
||||
# below never reference them either.
|
||||
notes = [n for n in notes if not _is_blocked(n.get("title"))]
|
||||
|
||||
nodes = []
|
||||
by_key: dict[str, str] = {}
|
||||
note_ids: set[str] = set()
|
||||
for n in notes:
|
||||
nodes.append({"id": n["id"], "type": "note", "label": n.get("title") or n["id"], "category": n.get("source_type")})
|
||||
note_ids.add(n["id"])
|
||||
key = _vault_key(n.get("vault_path"))
|
||||
if key:
|
||||
by_key[key] = n["id"]
|
||||
|
||||
# Only surface entities that are still mentioned by a LIVE note. Entities whose
|
||||
# only referencing notes were deleted/archived would otherwise linger forever
|
||||
# as orphan nodes ("graph doesn't follow my deletions").
|
||||
mentioned: set[str] = set()
|
||||
for rel in relations or []:
|
||||
if rel.get("from_note_id") in note_ids and rel.get("to_entity_id"):
|
||||
mentioned.add(rel["to_entity_id"])
|
||||
|
||||
entity_ids: set[str] = set()
|
||||
for ent in entities or []:
|
||||
if ent["id"] not in mentioned:
|
||||
continue
|
||||
if _is_blocked(ent.get("name")):
|
||||
continue
|
||||
nodes.append({"id": ent["id"], "type": "entity", "label": ent.get("name") or ent["id"], "category": ent.get("entity_type")})
|
||||
entity_ids.add(ent["id"])
|
||||
|
||||
edges: list[dict[str, Any]] = []
|
||||
edge_seen: set[tuple[str, str, str]] = set()
|
||||
|
||||
def add_edge(a: str, b: str, label: str, weight: float) -> None:
|
||||
if a == b:
|
||||
return
|
||||
key = (min(a, b), max(a, b), label)
|
||||
if key in edge_seen:
|
||||
return
|
||||
edge_seen.add(key)
|
||||
edges.append({"id": f"rel_{len(edges) + 1}", "source": a, "target": b, "label": label, "weight": weight})
|
||||
|
||||
# Wikilink edges.
|
||||
for n in notes:
|
||||
for target in extract_wikilinks(n.get("content_md") or ""):
|
||||
tid = by_key.get(target.replace("\\", "/").removesuffix(".md"))
|
||||
if tid:
|
||||
add_edge(n["id"], tid, "links", 2)
|
||||
|
||||
# Shared-tag edges.
|
||||
tag_to_notes: dict[str, list[str]] = {}
|
||||
for n in notes:
|
||||
for tag in n.get("tags") or []:
|
||||
tag_to_notes.setdefault(tag.lower(), []).append(n["id"])
|
||||
for tag, ids in tag_to_notes.items():
|
||||
if len(ids) < 2 or len(ids) > 50: # skip ubiquitous tags (e.g. knowledge-base)
|
||||
continue
|
||||
for i in range(len(ids)):
|
||||
for j in range(i + 1, len(ids)):
|
||||
add_edge(ids[i], ids[j], "related", 1)
|
||||
|
||||
# Entity / relation edges (phase 4) — only when both endpoints exist as nodes.
|
||||
valid = note_ids | entity_ids
|
||||
for rel in relations or []:
|
||||
src = rel.get("from_note_id") or rel.get("from_entity_id")
|
||||
tgt = rel.get("to_note_id") or rel.get("to_entity_id")
|
||||
if src in valid and tgt in valid:
|
||||
add_edge(src, tgt, rel.get("relation_type") or "related", float(rel.get("weight") or 1))
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
|
||||
def to_graphml(graph: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<graphml xmlns="http://graphml.graphdrawing.org/xmlns">',
|
||||
' <key id="label" for="node" attr.name="label" attr.type="string"/>',
|
||||
' <key id="elabel" for="edge" attr.name="label" attr.type="string"/>',
|
||||
' <graph edgedefault="undirected">',
|
||||
]
|
||||
for n in graph["nodes"]:
|
||||
lines.append(f' <node id="{html.escape(str(n["id"]))}"><data key="label">{html.escape(str(n["label"]))}</data></node>')
|
||||
for e in graph["edges"]:
|
||||
lines.append(
|
||||
f' <edge id="{html.escape(str(e["id"]))}" source="{html.escape(str(e["source"]))}" '
|
||||
f'target="{html.escape(str(e["target"]))}"><data key="elabel">{html.escape(str(e["label"]))}</data></edge>'
|
||||
)
|
||||
lines += [" </graph>", "</graphml>"]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_cypher(graph: dict[str, Any]) -> str:
|
||||
lines = []
|
||||
for n in graph["nodes"]:
|
||||
label = str(n["label"]).replace("'", "\\'")
|
||||
lines.append(f"CREATE (:Note {{id:'{n['id']}', title:'{label}'}});")
|
||||
for e in graph["edges"]:
|
||||
rel = str(e["label"]).upper()
|
||||
lines.append(f"MATCH (a:Note {{id:'{e['source']}'}}),(b:Note {{id:'{e['target']}'}}) CREATE (a)-[:{rel} {{weight:{e['weight']}}}]->(b);")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_html(graph: dict[str, Any]) -> str:
|
||||
"""Self-contained, theme-aware force-directed graph (canvas + vanilla JS, no CDN).
|
||||
|
||||
The page reads ``?theme=light|dark`` from its own URL so the embedding iframe
|
||||
can stay in sync with the app's day/night mode (an iframe cannot inherit the
|
||||
parent's CSS). The layout settles and then **freezes** so a dense graph stops
|
||||
drifting and stays clickable; node size scales with degree; and hovering or
|
||||
clicking a node focuses it + its direct neighbours while dimming everything
|
||||
else, which cuts through heavy node/edge clutter.
|
||||
"""
|
||||
data_json = json.dumps(graph, ensure_ascii=False)
|
||||
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
|
||||
|
||||
|
||||
def export_all(
|
||||
vault_root: Path | str,
|
||||
notes: list[dict[str, Any]],
|
||||
*,
|
||||
entities: list[dict[str, Any]] | None = None,
|
||||
relations: list[dict[str, Any]] | None = None,
|
||||
write_html: bool = True,
|
||||
write_json: bool = True,
|
||||
blacklist: set[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate graph artifacts under ``<vault>/wiki-export/`` and return stats."""
|
||||
out_dir = Path(vault_root).resolve() / "wiki-export"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
graph = build_graph(notes, entities, relations, blacklist=blacklist)
|
||||
written: list[str] = []
|
||||
if write_json:
|
||||
(out_dir / "graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
written.append("graph.json")
|
||||
(out_dir / "graph.graphml").write_text(to_graphml(graph), encoding="utf-8")
|
||||
written.append("graph.graphml")
|
||||
(out_dir / "cypher.txt").write_text(to_cypher(graph), encoding="utf-8")
|
||||
written.append("cypher.txt")
|
||||
if write_html:
|
||||
(out_dir / "graph.html").write_text(to_html(graph), encoding="utf-8")
|
||||
written.append("graph.html")
|
||||
return {
|
||||
"files": written,
|
||||
"stats": {"nodes": len(graph["nodes"]), "edges": len(graph["edges"])},
|
||||
"graph": graph,
|
||||
}
|
||||
|
||||
|
||||
_HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="zh">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>Knowledge Graph</title>
|
||||
<style>
|
||||
:root{
|
||||
--bg:#0b1020; --fg:#e7ecff; --sub:rgba(231,236,255,.78);
|
||||
--panel:rgba(17,26,54,.92); --panel-border:#2a3a66;
|
||||
}
|
||||
html.light{
|
||||
--bg:#f6f8fc; --fg:#1f2433; --sub:rgba(31,36,51,.72);
|
||||
--panel:rgba(255,255,255,.94); --panel-border:#d7deec;
|
||||
}
|
||||
html,body{margin:0;height:100%;background:var(--bg);color:var(--fg);font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
|
||||
canvas{display:block;cursor:grab}
|
||||
canvas.grabbing{cursor:grabbing}
|
||||
.card{position:fixed;background:var(--panel);border:1px solid var(--panel-border);border-radius:8px;font-size:12px;z-index:3;backdrop-filter:blur(6px)}
|
||||
#hud{top:10px;left:12px;padding:6px 10px;color:var(--sub);z-index:2}
|
||||
#legend{top:10px;right:12px;padding:8px 10px;display:flex;flex-direction:column;gap:6px}
|
||||
#legend .row{display:flex;align-items:center;gap:6px;color:var(--sub)}
|
||||
#legend .dot{width:10px;height:10px;border-radius:50%}
|
||||
#tip{padding:4px 8px;pointer-events:none;display:none}
|
||||
#panel{left:12px;bottom:12px;max-width:300px;padding:12px 14px;display:none}
|
||||
#panel h3{margin:0 0 6px;font-size:14px;line-height:1.3;word-break:break-word}
|
||||
#panel .meta{color:var(--sub);margin-bottom:8px}
|
||||
#panel .nb{display:flex;flex-wrap:wrap;gap:5px;max-height:120px;overflow:auto}
|
||||
#panel .nb span{padding:2px 7px;border:1px solid var(--panel-border);border-radius:999px;line-height:1.4;max-width:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
#panel .close{position:absolute;top:8px;right:10px;cursor:pointer;color:var(--sub);font-size:14px}
|
||||
#reset{bottom:12px;right:12px;padding:7px 12px;cursor:pointer;color:var(--fg);user-select:none}
|
||||
#reset:hover{filter:brightness(1.1)}
|
||||
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.55}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="hud" class="card"></div>
|
||||
<div id="legend" class="card">
|
||||
<div class="row"><span class="dot" id="lg-note"></span>知识笔记</div>
|
||||
<div class="row"><span class="dot" id="lg-entity"></span>实体</div>
|
||||
</div>
|
||||
<div id="tip" class="card"></div>
|
||||
<div id="panel" class="card"><span class="close" id="panel-close">×</span><h3></h3><div class="meta"></div><div class="nb"></div></div>
|
||||
<div id="reset" class="card">重置视图</div>
|
||||
<canvas id="c"></canvas>
|
||||
<script>
|
||||
const GRAPH = __GRAPH_DATA__;
|
||||
// Theme is supplied by the embedding iframe (it can't read the parent's CSS).
|
||||
const THEME = (new URLSearchParams(location.search).get('theme') === 'light') ? 'light' : 'dark';
|
||||
const PAL = {
|
||||
dark: {note:'#5b8cff', entity:'#ffb454', label:'rgba(231,236,255,.92)', edge:[150,178,255], halo:'rgba(11,16,32,.55)'},
|
||||
light: {note:'#2f6df6', entity:'#d97706', label:'rgba(31,36,51,.92)', edge:[63,96,168], halo:'rgba(255,255,255,.7)'},
|
||||
}[THEME];
|
||||
document.documentElement.classList.toggle('light', THEME === 'light');
|
||||
document.getElementById('lg-note').style.background = PAL.note;
|
||||
document.getElementById('lg-entity').style.background = PAL.entity;
|
||||
|
||||
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
|
||||
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
|
||||
const panel = document.getElementById('panel');
|
||||
let W=0,H=0,DPR=Math.min(window.devicePixelRatio||1,2);
|
||||
function resize(){W=innerWidth;H=innerHeight;canvas.width=W*DPR;canvas.height=H*DPR;canvas.style.width=W+'px';canvas.style.height=H+'px';}
|
||||
addEventListener('resize',()=>{resize();bump();});resize();
|
||||
|
||||
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
|
||||
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id="empty">暂无知识图谱数据</div>');}
|
||||
|
||||
// --- index, adjacency, degree, node sizing -------------------------------
|
||||
const idx={}, adj={};
|
||||
GRAPH.nodes.forEach(n=>{idx[n.id]=n;adj[n.id]=new Set();});
|
||||
const edges=[];
|
||||
for(const e of GRAPH.edges){
|
||||
const s=idx[e.source], t=idx[e.target];
|
||||
if(!s||!t||s===t) continue;
|
||||
edges.push({s,t,w:e.weight||e.w||1,label:e.label||''});
|
||||
adj[e.source].add(e.target); adj[e.target].add(e.source);
|
||||
}
|
||||
const N=GRAPH.nodes.length;
|
||||
GRAPH.nodes.forEach((n,i)=>{
|
||||
n.deg=adj[n.id].size;
|
||||
n.r=4.5+Math.min(11,Math.sqrt(n.deg)*2.4); // hubs are visibly bigger → easier to hit
|
||||
const a=i*2.399963, rad=18*Math.sqrt(i); // phyllotaxis spread = no initial overlap
|
||||
n.x=Math.cos(a)*rad; n.y=Math.sin(a)*rad; n.vx=0; n.vy=0;
|
||||
});
|
||||
// "Hub" nodes keep their label even when zoomed out; the rest declutter.
|
||||
const degs=GRAPH.nodes.map(n=>n.deg).sort((a,b)=>b-a);
|
||||
const HUB=Math.max(3,(degs[Math.floor(N*0.08)]||3));
|
||||
|
||||
// --- force layout that cools down and FREEZES (frozen graph = clickable) --
|
||||
const REP=9000, LINK=92, CENTER=0.0016;
|
||||
let alpha=1, drag=null;
|
||||
function physics(){
|
||||
for(let i=0;i<N;i++){
|
||||
const n=GRAPH.nodes[i];
|
||||
for(let j=i+1;j<N;j++){
|
||||
const m=GRAPH.nodes[j];
|
||||
let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=REP/d2;
|
||||
let fx=dx*f,fy=dy*f; n.vx+=fx;n.vy+=fy;m.vx-=fx;m.vy-=fy;
|
||||
}
|
||||
}
|
||||
for(const e of edges){
|
||||
let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.hypot(dx,dy)||1,f=(d-LINK)*0.02;
|
||||
e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;
|
||||
}
|
||||
for(const n of GRAPH.nodes){
|
||||
if(n===drag){n.vx=n.vy=0;continue;}
|
||||
n.vx-=n.x*CENTER; n.vy-=n.y*CENTER;
|
||||
n.x+=(n.vx*=0.82)*alpha; n.y+=(n.vy*=0.82)*alpha;
|
||||
}
|
||||
alpha*=0.985;
|
||||
}
|
||||
|
||||
// --- camera / fit-to-view -------------------------------------------------
|
||||
let cam={x:0,y:0,k:1};
|
||||
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
|
||||
function fitView(){
|
||||
if(!N){return;}
|
||||
let a=1e9,b=1e9,c=-1e9,d=-1e9;
|
||||
for(const n of GRAPH.nodes){a=Math.min(a,n.x);b=Math.min(b,n.y);c=Math.max(c,n.x);d=Math.max(d,n.y);}
|
||||
const gw=Math.max(c-a,1),gh=Math.max(d-b,1),pad=90;
|
||||
cam.k=Math.max(0.12,Math.min(2.2,Math.min((W-pad)/gw,(H-pad)/gh)));
|
||||
cam.x=W/2-((a+c)/2)*cam.k; cam.y=H/2-((b+d)/2)*cam.k;
|
||||
}
|
||||
let fitted=false;
|
||||
function bump(){alpha=Math.max(alpha,0.35);} // re-warm the sim on interaction
|
||||
|
||||
// --- focus / hover state --------------------------------------------------
|
||||
let hoverNode=null, focusNode=null;
|
||||
function active(){return focusNode||hoverNode;}
|
||||
function neighbors(n){return n?adj[n.id]:null;}
|
||||
|
||||
function draw(){
|
||||
ctx.setTransform(DPR,0,0,DPR,0,0);
|
||||
ctx.clearRect(0,0,W,H);
|
||||
ctx.translate(cam.x,cam.y); ctx.scale(cam.k,cam.k);
|
||||
const act=active(), nb=neighbors(act), zoomed=cam.k>1.35;
|
||||
const [er,eg,eb]=PAL.edge;
|
||||
|
||||
// edges: clearly visible by default; when a node is focused, only its edges
|
||||
// stay bright and the rest fade back (but never to invisible).
|
||||
for(const e of edges){
|
||||
const lit = act && (e.s===act||e.t===act);
|
||||
const op = act ? (lit?0.98:0.12) : Math.min(0.85,0.55+e.w*0.06);
|
||||
// Keep a stable on-screen thickness regardless of zoom (>=1px), so lines
|
||||
// never thin out to a hairline when zoomed in or on dense graphs.
|
||||
const screenW = lit ? 2.6 : Math.min(2.4,1.4+e.w*0.45);
|
||||
ctx.strokeStyle='rgba('+er+','+eg+','+eb+','+op+')';
|
||||
ctx.lineWidth=Math.max(1,screenW)/cam.k;
|
||||
ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();
|
||||
}
|
||||
// nodes
|
||||
for(const n of GRAPH.nodes){
|
||||
const dim = act && n!==act && !(nb&&nb.has(n.id));
|
||||
ctx.globalAlpha = dim?0.22:1;
|
||||
ctx.fillStyle = n.type==='entity'?PAL.entity:PAL.note;
|
||||
ctx.beginPath();ctx.arc(n.x,n.y,n.r,0,6.2832);ctx.fill();
|
||||
if(n===act){ctx.lineWidth=2/cam.k;ctx.strokeStyle=PAL.label;ctx.stroke();}
|
||||
ctx.globalAlpha=1;
|
||||
}
|
||||
// labels: only for focus/neighbors, hubs, or when zoomed in — kills clutter.
|
||||
ctx.textBaseline='middle';
|
||||
for(const n of GRAPH.nodes){
|
||||
const show = act ? (n===act||(nb&&nb.has(n.id))) : (zoomed||n.deg>=HUB);
|
||||
if(!show) continue;
|
||||
const txt=(n.label||'').slice(0,20);
|
||||
if(!txt) continue;
|
||||
ctx.font=(n===act?'bold ':'')+(11/cam.k)+'px system-ui';
|
||||
const tx=n.x+n.r+4/cam.k, ty=n.y;
|
||||
ctx.lineWidth=3/cam.k; ctx.strokeStyle=PAL.halo;
|
||||
ctx.strokeText(txt,tx,ty);
|
||||
ctx.fillStyle=PAL.label; ctx.fillText(txt,tx,ty);
|
||||
}
|
||||
}
|
||||
|
||||
function loop(){
|
||||
if(alpha>0.02 || drag){physics();} else if(!fitted){fitView();fitted=true;}
|
||||
draw();
|
||||
requestAnimationFrame(loop);
|
||||
}
|
||||
// Pre-settle a few iterations so the first paint is already roughly laid out.
|
||||
for(let i=0;i<60;i++)physics();
|
||||
fitView();
|
||||
loop();
|
||||
|
||||
// --- picking & interaction ------------------------------------------------
|
||||
function pick(p){
|
||||
let best=null,bd=1e9;
|
||||
for(const n of GRAPH.nodes){
|
||||
const dx=n.x-p.x,dy=n.y-p.y,d=dx*dx+dy*dy, hit=(n.r+8/cam.k);
|
||||
if(d<hit*hit && d<bd){bd=d;best=n;}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
let pan=null, downXY=null, moved=false;
|
||||
canvas.addEventListener('mousedown',ev=>{
|
||||
const p=toWorld(ev.clientX,ev.clientY), hit=pick(p);
|
||||
downXY={x:ev.clientX,y:ev.clientY,hit}; moved=false;
|
||||
if(hit){drag=hit;} else {pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};}
|
||||
canvas.classList.add('grabbing');
|
||||
});
|
||||
addEventListener('mousemove',ev=>{
|
||||
const p=toWorld(ev.clientX,ev.clientY);
|
||||
if(downXY && (Math.abs(ev.clientX-downXY.x)>3||Math.abs(ev.clientY-downXY.y)>3)) moved=true;
|
||||
if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;bump();}
|
||||
else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
|
||||
hoverNode=drag||pan?hoverNode:pick(p);
|
||||
if(hoverNode && !drag && !pan){
|
||||
tip.style.display='block';
|
||||
tip.style.left=Math.min(ev.clientX+14,W-220)+'px';
|
||||
tip.style.top=(ev.clientY+14)+'px';
|
||||
tip.textContent=hoverNode.label||'';
|
||||
} else { tip.style.display='none'; }
|
||||
});
|
||||
addEventListener('mouseup',ev=>{
|
||||
if(downXY && !moved){ // a click, not a drag → toggle focus / detail panel
|
||||
focusNode = (downXY.hit && downXY.hit!==focusNode) ? downXY.hit : null;
|
||||
renderPanel();
|
||||
}
|
||||
drag=null;pan=null;downXY=null;canvas.classList.remove('grabbing');
|
||||
});
|
||||
canvas.addEventListener('wheel',ev=>{
|
||||
ev.preventDefault();
|
||||
const s=ev.deltaY<0?1.12:0.89;
|
||||
const nk=Math.max(0.08,Math.min(6,cam.k*s)), r=nk/cam.k;
|
||||
cam.x=ev.clientX-(ev.clientX-cam.x)*r; cam.y=ev.clientY-(ev.clientY-cam.y)*r; cam.k=nk;
|
||||
},{passive:false});
|
||||
|
||||
// --- detail panel ---------------------------------------------------------
|
||||
const TYPE_LABEL={note:'知识笔记',entity:'实体'};
|
||||
function renderPanel(){
|
||||
if(!focusNode){panel.style.display='none';return;}
|
||||
const n=focusNode;
|
||||
panel.querySelector('h3').textContent=n.label||n.id;
|
||||
const cat=n.category?' · '+n.category:'';
|
||||
panel.querySelector('.meta').textContent=(TYPE_LABEL[n.type]||n.type||'节点')+cat+' · '+n.deg+' 条关系';
|
||||
const nb=panel.querySelector('.nb'); nb.innerHTML='';
|
||||
let count=0;
|
||||
for(const id of adj[n.id]){
|
||||
if(count++>=24)break;
|
||||
const m=idx[id]; if(!m)continue;
|
||||
const s=document.createElement('span'); s.textContent=m.label||id;
|
||||
s.title=m.label||id;
|
||||
s.style.cursor='pointer';
|
||||
s.onclick=()=>{focusNode=m;renderPanel();};
|
||||
nb.appendChild(s);
|
||||
}
|
||||
if(!adj[n.id].size){const s=document.createElement('span');s.textContent='(无关联)';s.style.opacity=.6;nb.appendChild(s);}
|
||||
panel.style.display='block';
|
||||
}
|
||||
document.getElementById('panel-close').onclick=()=>{focusNode=null;renderPanel();};
|
||||
document.getElementById('reset').onclick=()=>{focusNode=null;renderPanel();fitView();};
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Extract knowledge notes from threads / searches / tool results.
|
||||
|
||||
First-release strategy is rule-based (wiki-capture aligned, §11 of the dev doc):
|
||||
keep high-value content (user questions, final assistant answers, cited
|
||||
sources), drop noise (empty/greeting/intermediate reasoning), and write a
|
||||
*declarative* note rather than a chat transcript. An LLM extractor can later
|
||||
replace :func:`extract_thread` while keeping the same :class:`NoteDraft` output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
|
||||
|
||||
# Greeting / low-value lines filtered from extracted turns.
|
||||
_GREETING_RE = re.compile(r"^(你好|您好|hi|hello|hey|thanks?|谢谢|好的|ok|okay)[\s!!。.,,~]*$", re.IGNORECASE)
|
||||
_WS_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def _one_line(text: str, limit: int = 300) -> str:
|
||||
"""Collapse whitespace and clip ``text`` to a single line."""
|
||||
collapsed = _WS_RE.sub(" ", text or "").strip()
|
||||
return collapsed[:limit]
|
||||
|
||||
|
||||
def message_text(msg: dict[str, Any]) -> str:
|
||||
"""Return the plain-text content of a serialized LangChain message."""
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
return content.strip()
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict):
|
||||
if block.get("type") in (None, "text") and isinstance(block.get("text"), str):
|
||||
parts.append(block["text"])
|
||||
return "\n".join(p for p in parts if p).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _msg_type(msg: dict[str, Any]) -> str:
|
||||
return str(msg.get("type") or msg.get("role") or "").lower()
|
||||
|
||||
|
||||
def split_turns(messages: list[dict[str, Any]], *, max_messages: int = 80) -> list[dict[str, str]]:
|
||||
"""Split messages into ``{user, ai}`` turns keyed by human messages.
|
||||
|
||||
For each turn we keep the last substantial AI message (with text body).
|
||||
"""
|
||||
msgs = messages[-max_messages:] if max_messages and len(messages) > max_messages else list(messages)
|
||||
turns: list[dict[str, str]] = []
|
||||
current: dict[str, str] | None = None
|
||||
for msg in msgs:
|
||||
mtype = _msg_type(msg)
|
||||
text = message_text(msg)
|
||||
if mtype == "human":
|
||||
if current and (current.get("user") or current.get("ai")):
|
||||
turns.append(current)
|
||||
current = {"user": text, "ai": ""}
|
||||
elif mtype == "ai" and text:
|
||||
if current is None:
|
||||
current = {"user": "", "ai": text}
|
||||
else:
|
||||
current["ai"] = text # last substantial AI message wins
|
||||
if current and (current.get("user") or current.get("ai")):
|
||||
turns.append(current)
|
||||
# Drop turns that are pure greetings with no answer.
|
||||
cleaned = []
|
||||
for t in turns:
|
||||
user = (t.get("user") or "").strip()
|
||||
ai = (t.get("ai") or "").strip()
|
||||
if not ai and _GREETING_RE.match(user):
|
||||
continue
|
||||
if not user and not ai:
|
||||
continue
|
||||
cleaned.append({"user": user, "ai": ai})
|
||||
return cleaned
|
||||
|
||||
|
||||
def _auto_title(thread_title: str | None, turns: list[dict[str, str]]) -> str:
|
||||
if thread_title and thread_title.strip() and thread_title.strip().lower() != "untitled":
|
||||
return thread_title.strip()[:200]
|
||||
for t in turns:
|
||||
if t.get("user"):
|
||||
first = t["user"].splitlines()[0].strip()
|
||||
return (first[:40] + ("…" if len(first) > 40 else "")) or "知识笔记"
|
||||
return "知识笔记"
|
||||
|
||||
|
||||
def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str:
|
||||
"""Simple rule-based summary: the first answer, clipped."""
|
||||
for t in turns:
|
||||
if t.get("ai"):
|
||||
text = re.sub(r"\s+", " ", t["ai"]).strip()
|
||||
return text[:limit] + ("…" if len(text) > limit else "")
|
||||
for t in turns:
|
||||
if t.get("user"):
|
||||
text = re.sub(r"\s+", " ", t["user"]).strip()
|
||||
return text[:limit] + ("…" if len(text) > limit else "")
|
||||
return ""
|
||||
|
||||
|
||||
def related_links(project_name: str | None) -> list[str]: # noqa: ARG001
|
||||
"""Structural wikilinks every note used to carry.
|
||||
|
||||
Product decision: notes no longer auto-link the vault meta/hub pages
|
||||
(``index`` / ``_meta/taxonomy`` / the project hub) — they were noise in every
|
||||
note's ``## Related``. Only real, content-specific related notes (injected by
|
||||
the service layer, which has DB access) are linked now. Kept as a function
|
||||
(returns empty) so existing callers and the wiki-capture contract stay intact.
|
||||
"""
|
||||
return []
|
||||
|
||||
|
||||
# Backwards-compatible internal alias.
|
||||
_related_links = related_links
|
||||
|
||||
|
||||
def _related_section_md(related_lines: list[str], source_lines: list[str] | None = None) -> str: # noqa: ARG001
|
||||
"""Render the ``## Related`` section, or ``""`` when there is nothing to show.
|
||||
|
||||
Sources are intentionally **not** rendered into the body anymore. The note
|
||||
detail page already shows every source once via the structured ``来源`` block
|
||||
(``note.sources``); emitting a ``### Sources`` sub-section here duplicated that
|
||||
list (and broke the frontend's clean ``## Related`` extraction). The
|
||||
``source_lines`` argument is kept for backward compatibility and ignored.
|
||||
"""
|
||||
if not related_lines:
|
||||
return ""
|
||||
inner = "\n".join(related_lines)
|
||||
return f"## Related\n\n{inner}\n"
|
||||
|
||||
|
||||
def _build_body(
|
||||
*,
|
||||
title: str,
|
||||
turns: list[dict[str, str]],
|
||||
mode: str,
|
||||
project_name: str | None,
|
||||
sources: list[SourceDraft],
|
||||
) -> str:
|
||||
"""Render the wiki-capture 5-section declarative body."""
|
||||
full = mode == "full"
|
||||
answer_clip = 4000 if full else 800
|
||||
|
||||
context_lines = []
|
||||
for t in turns:
|
||||
if t.get("user"):
|
||||
q = re.sub(r"\s+", " ", t["user"]).strip()
|
||||
context_lines.append(f"- {q[:300]}")
|
||||
context = "\n".join(context_lines[:8]) or "- 来自一次对话沉淀。^[inferred]"
|
||||
|
||||
finding_lines = []
|
||||
for i, t in enumerate(turns, 1):
|
||||
if t.get("ai"):
|
||||
ans = t["ai"].strip()
|
||||
ans = ans if full else (ans[:answer_clip] + ("…" if len(ans) > answer_clip else ""))
|
||||
finding_lines.append(f"### 结论 {i}\n\n{ans}")
|
||||
finding = "\n\n".join(finding_lines) or "_暂无可提炼的结论。_"
|
||||
|
||||
reasoning = "本页由对话自动沉淀而成,结论来自助手最终回答;过程性推理已省略。判断与归纳部分标记为 ^[inferred]。"
|
||||
|
||||
implications = "- 后续可在相关问答中参考本页知识。\n- 如内容过期或冲突,请编辑或归档。"
|
||||
|
||||
related_lines = [f"- {link}" for link in _related_links(project_name)]
|
||||
source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
|
||||
related_section = _related_section_md(related_lines, source_lines)
|
||||
|
||||
return (
|
||||
f"# {title}\n\n"
|
||||
f"## Context\n\n{context}\n\n"
|
||||
f"## Finding / Decision\n\n{finding}\n\n"
|
||||
f"## Reasoning\n\n{reasoning}\n\n"
|
||||
f"## Implications\n\n{implications}\n\n"
|
||||
f"{related_section}"
|
||||
)
|
||||
|
||||
|
||||
def extract_thread(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None = None,
|
||||
mode: str = "summary",
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
max_messages: int = 80,
|
||||
) -> NoteDraft | None:
|
||||
"""Build a :class:`NoteDraft` from thread messages, or ``None`` if empty."""
|
||||
turns = split_turns(messages, max_messages=max_messages)
|
||||
if not turns:
|
||||
return None
|
||||
final_title = (title or "").strip() or _auto_title(thread_title, turns)
|
||||
summary = _summary(turns)
|
||||
sources = sources or []
|
||||
body = _build_body(title=final_title, turns=turns, mode=mode, project_name=project_name, sources=sources)
|
||||
base_tags = list(tags or [])
|
||||
for default in (project_name or "zncm", "knowledge-base"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
return NoteDraft(
|
||||
title=final_title,
|
||||
summary=summary,
|
||||
content_md=body,
|
||||
category="synthesis",
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
tags=base_tags,
|
||||
confidence=0.6,
|
||||
related=_related_links(project_name),
|
||||
sources=sources,
|
||||
source_key=f"thread:{thread_id}",
|
||||
)
|
||||
|
||||
|
||||
def extract_search(
|
||||
query: str,
|
||||
sources: list[SourceDraft],
|
||||
*,
|
||||
thread_id: str | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
) -> NoteDraft:
|
||||
"""Build a reference-category :class:`NoteDraft` from search/tool results."""
|
||||
final_title = (title or "").strip() or f"检索:{query[:40]}"
|
||||
body_lines = [f"# {final_title}\n", "## Context\n", f"- 检索关键词:{query}\n", "## Finding / Decision\n"]
|
||||
for s in sources:
|
||||
label = s.title or s.url or "来源"
|
||||
line = f"- **{label}**"
|
||||
if s.url:
|
||||
line += f" — {s.url}"
|
||||
if s.snippet:
|
||||
line += "\n - " + _one_line(s.snippet)
|
||||
body_lines.append(line)
|
||||
body_lines += [
|
||||
"\n## Reasoning\n",
|
||||
"本页汇总外部检索来源,供后续问答引用。来源可信度需结合原文判断。^[ambiguous]\n",
|
||||
"## Implications\n",
|
||||
"- 引用前请核对原始来源。\n",
|
||||
"## Related\n",
|
||||
*[f"- {link}" for link in _related_links(project_name)],
|
||||
]
|
||||
base_tags = list(tags or [])
|
||||
for default in (project_name or "zncm", "knowledge-base", "reference"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
return NoteDraft(
|
||||
title=final_title,
|
||||
summary=f"检索“{query}”的来源汇总,共 {len(sources)} 条。",
|
||||
content_md="\n".join(body_lines),
|
||||
category="references",
|
||||
source_type="search",
|
||||
source_id=f"search:{query[:120]}",
|
||||
tags=base_tags,
|
||||
confidence=0.4,
|
||||
related=_related_links(project_name),
|
||||
sources=sources,
|
||||
source_key=None,
|
||||
)
|
||||
+493
@@ -0,0 +1,493 @@
|
||||
"""LLM-based knowledge extraction (wiki-capture style).
|
||||
|
||||
Turns a conversation into a *declarative* knowledge note instead of dumping the
|
||||
raw chat. Follows the obsidian-wiki ``wiki-capture`` skill: classify the note,
|
||||
keep only high-value findings/decisions, drop greetings / intermediate
|
||||
reasoning / verbatim transcripts, mark inferred content, and emit structured
|
||||
JSON that we render into the Context / Finding / Reasoning / Implications /
|
||||
Related body.
|
||||
|
||||
Falls back to the rule-based :func:`deerflow.knowledge.extractor.extract_thread`
|
||||
when the LLM is unavailable or returns something unusable, so capture never
|
||||
hard-fails.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge import extractor
|
||||
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
|
||||
from deerflow.knowledge.skill_templates import WIKI_CAPTURE_GUIDANCE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_VALID_CATEGORIES = {"synthesis", "concepts", "references", "decision", "journal"}
|
||||
|
||||
|
||||
class LLMRejectedIngest(Exception):
|
||||
"""The LLM judged the conversation as having no knowledge value.
|
||||
|
||||
Raised (instead of returning ``[]``) when the model explicitly sets
|
||||
``should_ingest: false`` with no notes — so callers can *skip* ingestion
|
||||
entirely rather than falling back to the rule-based verbatim dump.
|
||||
"""
|
||||
|
||||
|
||||
def _is_falsey(value: Any) -> bool:
|
||||
if value is False:
|
||||
return True
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in ("false", "0", "no")
|
||||
return False
|
||||
|
||||
_SYSTEM_PROMPT = """\
|
||||
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一次对话提炼成\
|
||||
**陈述式知识(declarative knowledge)**,而不是聊天流水账。
|
||||
|
||||
要求:
|
||||
- 只保留高价值内容:明确结论、方案、决策、关键事实、被引用的来源。
|
||||
- 丢弃:寒暄、客套、中间推理过程、重复内容、无意义的原文堆砌。
|
||||
- 不要写“用户问了什么、AI 回答了什么”,要写“结论是什么、为什么成立”。
|
||||
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
|
||||
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
|
||||
references(外部来源汇总)、decision(架构/设计决策)、journal(会话纪要)。
|
||||
- 用中文输出。
|
||||
|
||||
一次对话可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
|
||||
- 默认只产出 **1 页**;只有当对话明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
|
||||
- 每一页聚焦单一主题,标题与 summary 必须彼此区分,不要把同一结论拆到多页。
|
||||
- 不值得沉淀时返回空的 notes 数组。
|
||||
|
||||
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构如下:
|
||||
{
|
||||
"should_ingest": true,
|
||||
"reason": "为什么值得/不值得沉淀",
|
||||
"notes": [
|
||||
{
|
||||
"title": "简洁的知识标题(不超过 40 字)",
|
||||
"summary": "1-2 句话概括这页知识",
|
||||
"category": "synthesis",
|
||||
"context": "问题背景或触发场景,1-3 句",
|
||||
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
|
||||
"reasoning": "为什么成立、取舍、不确定性",
|
||||
"implications": ["后续建议/风险/下一步", "..."],
|
||||
"tags": ["标签1", "标签2"],
|
||||
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
|
||||
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
|
||||
"confidence": 0.0
|
||||
}
|
||||
]
|
||||
}"""
|
||||
|
||||
|
||||
_DOC_SYSTEM_PROMPT = """\
|
||||
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一篇**文档**提炼成\
|
||||
**陈述式知识(declarative knowledge)**,而不是照抄原文。
|
||||
|
||||
要求:
|
||||
- 只保留高价值内容:核心结论、关键事实、定义、流程、数据、决策与其依据。
|
||||
- 丢弃:目录、页眉页脚、版权声明、重复的样板文字、无意义的排版噪声。
|
||||
- 用自己的话重组为结构化知识,不要逐段复制原文。
|
||||
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
|
||||
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
|
||||
references(外部来源汇总)、decision(架构/设计决策)、journal(纪要)。
|
||||
- 用中文输出。
|
||||
|
||||
一篇文档可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
|
||||
- 默认只产出 **1 页**;只有当文档明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
|
||||
- 每一页聚焦单一主题,标题与 summary 必须彼此区分。
|
||||
- 文档没有可沉淀的知识时返回空的 notes 数组。
|
||||
|
||||
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构与对话提炼一致:
|
||||
{
|
||||
"should_ingest": true,
|
||||
"reason": "为什么值得/不值得沉淀",
|
||||
"notes": [
|
||||
{
|
||||
"title": "简洁的知识标题(不超过 40 字)",
|
||||
"summary": "1-2 句话概括这页知识",
|
||||
"category": "synthesis",
|
||||
"context": "文档背景或适用场景,1-3 句",
|
||||
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
|
||||
"reasoning": "为什么成立、取舍、不确定性",
|
||||
"implications": ["后续建议/风险/下一步", "..."],
|
||||
"tags": ["标签1", "标签2"],
|
||||
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
|
||||
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
|
||||
"confidence": 0.0
|
||||
}
|
||||
]
|
||||
}"""
|
||||
|
||||
|
||||
def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str:
|
||||
parts: list[str] = []
|
||||
for i, t in enumerate(turns, 1):
|
||||
if t.get("user"):
|
||||
parts.append(f"[用户 {i}] {t['user']}")
|
||||
if t.get("ai"):
|
||||
parts.append(f"[助手 {i}] {t['ai']}")
|
||||
if sources:
|
||||
parts.append("\n[来源]")
|
||||
for s in sources[:20]:
|
||||
label = s.title or s.url or s.tool_name or "来源"
|
||||
parts.append(f"- {label}" + (f" ({s.url})" if s.url else ""))
|
||||
text = "\n\n".join(parts)
|
||||
if len(text) > max_chars:
|
||||
text = text[:max_chars] + "\n…(已截断)"
|
||||
return text
|
||||
|
||||
|
||||
def _parse_json(raw: str) -> dict[str, Any] | None:
|
||||
if not raw:
|
||||
return None
|
||||
text = raw.strip()
|
||||
# Strip ```json fences if present.
|
||||
fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL)
|
||||
if fence:
|
||||
text = fence.group(1).strip()
|
||||
# Otherwise grab the outermost { ... }.
|
||||
if not text.startswith("{"):
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start >= 0 and end > start:
|
||||
text = text[start : end + 1]
|
||||
try:
|
||||
data = json.loads(text)
|
||||
return data if isinstance(data, dict) else None
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Knowledge LLM returned non-JSON output; falling back to rule-based")
|
||||
return None
|
||||
|
||||
|
||||
def _render_body(data: dict[str, Any], title: str, related: list[str], sources: list[SourceDraft]) -> str:
|
||||
def _bullets(items: Any) -> str:
|
||||
if isinstance(items, str):
|
||||
items = [items]
|
||||
if not isinstance(items, list):
|
||||
return ""
|
||||
return "\n".join(f"- {str(x).strip()}" for x in items if str(x).strip())
|
||||
|
||||
context = str(data.get("context") or "").strip() or "- 来自一次对话沉淀。^[inferred]"
|
||||
findings = _bullets(data.get("findings")) or "_暂无可提炼的结论。_"
|
||||
reasoning = str(data.get("reasoning") or "").strip() or "(未提供推理说明)"
|
||||
implications = _bullets(data.get("implications")) or "- 如内容过期或冲突,请编辑或归档。"
|
||||
|
||||
related_lines = [f"- {link}" for link in related]
|
||||
source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
|
||||
related_section = extractor._related_section_md(related_lines, source_lines)
|
||||
|
||||
return (
|
||||
f"# {title}\n\n"
|
||||
f"## Context\n\n{context}\n\n"
|
||||
f"## Finding / Decision\n\n{findings}\n\n"
|
||||
f"## Reasoning\n\n{reasoning}\n\n"
|
||||
f"## Implications\n\n{implications}\n\n"
|
||||
f"{related_section}"
|
||||
)
|
||||
|
||||
|
||||
def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""Normalize the LLM JSON into a list of per-note dicts.
|
||||
|
||||
Accepts the multi-note shape ``{"notes": [...]}`` as well as the legacy
|
||||
single-note shape (``{"title": ..., "findings": ...}``) for backward
|
||||
compatibility. Notes without any title/findings are dropped.
|
||||
"""
|
||||
raw_notes = data.get("notes")
|
||||
if isinstance(raw_notes, list):
|
||||
candidates = [n for n in raw_notes if isinstance(n, dict)]
|
||||
elif data.get("title") or data.get("findings") or data.get("context"):
|
||||
candidates = [data] # legacy single-object output
|
||||
else:
|
||||
candidates = []
|
||||
return [n for n in candidates if str(n.get("title") or "").strip() or n.get("findings")]
|
||||
|
||||
|
||||
def _build_note_draft(
|
||||
note_data: dict[str, Any],
|
||||
*,
|
||||
source_type: str,
|
||||
source_id: str | None,
|
||||
source_key: str,
|
||||
fallback_title: str,
|
||||
title_override: str | None,
|
||||
tags: list[str] | None,
|
||||
project_name: str | None,
|
||||
sources: list[SourceDraft],
|
||||
) -> NoteDraft:
|
||||
"""Render one note dict into a :class:`NoteDraft`.
|
||||
|
||||
Source-agnostic: the caller supplies ``source_type`` / ``source_id`` /
|
||||
``source_key`` and a ``fallback_title`` (used when neither the user override
|
||||
nor the LLM produced a usable title), so both thread and document capture
|
||||
share one renderer.
|
||||
"""
|
||||
category = str(note_data.get("category") or "synthesis").strip().lower()
|
||||
if category not in _VALID_CATEGORIES:
|
||||
category = "synthesis"
|
||||
|
||||
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or fallback_title
|
||||
summary = str(note_data.get("summary") or "").strip()
|
||||
related = extractor.related_links(project_name)
|
||||
body = _render_body(note_data, final_title, related, sources)
|
||||
|
||||
base_tags = list(tags or [])
|
||||
llm_tags = note_data.get("tags") if isinstance(note_data.get("tags"), list) else []
|
||||
for tag in [*llm_tags, project_name or "zncm", "knowledge-base"]:
|
||||
tag = str(tag).strip()
|
||||
if tag and tag not in base_tags:
|
||||
base_tags.append(tag)
|
||||
|
||||
try:
|
||||
confidence = float(note_data.get("confidence") or 0.7)
|
||||
except (TypeError, ValueError):
|
||||
confidence = 0.7
|
||||
|
||||
entities = note_data.get("entities") if isinstance(note_data.get("entities"), list) else []
|
||||
relations = note_data.get("relations") if isinstance(note_data.get("relations"), list) else []
|
||||
|
||||
return NoteDraft(
|
||||
title=final_title,
|
||||
summary=summary or final_title,
|
||||
content_md=body,
|
||||
category=category,
|
||||
source_type=source_type,
|
||||
source_id=source_id,
|
||||
tags=base_tags,
|
||||
confidence=max(0.0, min(confidence, 1.0)),
|
||||
related=related,
|
||||
sources=sources,
|
||||
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
|
||||
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
|
||||
source_key=source_key,
|
||||
)
|
||||
|
||||
|
||||
async def extract_thread_llm_multi(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None = None,
|
||||
mode: str = "summary",
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
max_messages: int = 80,
|
||||
model_name: str | None = None,
|
||||
max_input_chars: int = 12000,
|
||||
allow_multi: bool = True,
|
||||
max_notes: int = 4,
|
||||
system_prompt: str | None = None,
|
||||
) -> list[NoteDraft]:
|
||||
"""Distill a thread into one *or several* cross-linkable knowledge notes.
|
||||
|
||||
The LLM may split a conversation that covers multiple independent topics into
|
||||
several declarative notes. Returns ``[]`` when there is nothing to sediment
|
||||
or the LLM path fails — callers should fall back to the rule-based extractor.
|
||||
When ``allow_multi`` is false only the first note is kept. ``system_prompt``
|
||||
overrides the built-in distillation prompt with an operator-edited template
|
||||
(used as-is, since the template is expected to be self-contained).
|
||||
"""
|
||||
turns = extractor.split_turns(messages, max_messages=max_messages)
|
||||
if not turns:
|
||||
return []
|
||||
sources = sources or []
|
||||
|
||||
try:
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
model = create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
transcript = _build_transcript(turns, sources, max_chars=max_input_chars)
|
||||
system_text = (system_prompt or "").strip() or (_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE)
|
||||
prompt = [
|
||||
SystemMessage(content=system_text),
|
||||
HumanMessage(content=f"以下是一次对话,请按规范提炼为知识 JSON:\n\n{transcript}"),
|
||||
]
|
||||
response = await model.ainvoke(prompt)
|
||||
raw = response.content if isinstance(response.content, str) else str(response.content)
|
||||
data = _parse_json(raw)
|
||||
except Exception:
|
||||
logger.exception("[knowledge] thread %s — LLM extraction call failed; using rule-based fallback", thread_id)
|
||||
return []
|
||||
|
||||
if not data:
|
||||
logger.warning("[knowledge] thread %s — LLM returned no parsable JSON; using rule-based fallback", thread_id)
|
||||
return []
|
||||
|
||||
note_dicts = _notes_from_data(data)
|
||||
if not note_dicts:
|
||||
reason = str(data.get("reason") or "")[:200]
|
||||
if _is_falsey(data.get("should_ingest")):
|
||||
# The model deliberately judged this conversation worthless (greeting,
|
||||
# meta chatter, no findings). Skip ingestion entirely — do NOT fall
|
||||
# back to the rule-based verbatim dump.
|
||||
logger.warning("[knowledge] thread %s SKIPPED — model judged not worth ingesting: %s", thread_id, reason)
|
||||
raise LLMRejectedIngest(reason or "no knowledge value")
|
||||
logger.warning(
|
||||
"[knowledge] thread %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
|
||||
thread_id,
|
||||
data.get("should_ingest"),
|
||||
reason,
|
||||
)
|
||||
return []
|
||||
if not allow_multi:
|
||||
note_dicts = note_dicts[:1]
|
||||
else:
|
||||
note_dicts = note_dicts[: max(1, max_notes)]
|
||||
|
||||
fallback_title = extractor._auto_title(thread_title, turns)
|
||||
drafts: list[NoteDraft] = []
|
||||
for i, nd in enumerate(note_dicts):
|
||||
# The user-supplied title override only applies to a single-note result;
|
||||
# multi-note splits keep each note's own LLM title.
|
||||
override = title if (len(note_dicts) == 1) else None
|
||||
suffix = "" if len(note_dicts) == 1 else f"#{i}"
|
||||
drafts.append(
|
||||
_build_note_draft(
|
||||
nd,
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
source_key=f"thread:{thread_id}{suffix}",
|
||||
fallback_title=fallback_title,
|
||||
title_override=override,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
|
||||
async def extract_document_llm_multi(
|
||||
text: str,
|
||||
*,
|
||||
doc_id: str,
|
||||
doc_title: str | None = None,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
model_name: str | None = None,
|
||||
max_input_chars: int = 12000,
|
||||
allow_multi: bool = True,
|
||||
max_notes: int = 4,
|
||||
system_prompt: str | None = None,
|
||||
) -> list[NoteDraft]:
|
||||
"""Distill an imported **document** into one *or several* knowledge notes.
|
||||
|
||||
Mirrors :func:`extract_thread_llm_multi` but takes raw document text (already
|
||||
converted to Markdown/plain text) instead of a conversation. Returns ``[]``
|
||||
when the LLM path fails (callers fall back to a verbatim note); raises
|
||||
:class:`LLMRejectedIngest` when the model deliberately judges the document
|
||||
worthless. Resulting notes carry ``source_type="file"``.
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
sources = sources or []
|
||||
|
||||
try:
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
model = create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
body = text if len(text) <= max_input_chars else text[:max_input_chars] + "\n…(已截断)"
|
||||
header = f"文档标题:{doc_title}\n\n" if doc_title else ""
|
||||
system_text = (system_prompt or "").strip() or (_DOC_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE)
|
||||
prompt = [
|
||||
SystemMessage(content=system_text),
|
||||
HumanMessage(content=f"以下是一篇文档,请按规范提炼为知识 JSON:\n\n{header}{body}"),
|
||||
]
|
||||
response = await model.ainvoke(prompt)
|
||||
raw = response.content if isinstance(response.content, str) else str(response.content)
|
||||
data = _parse_json(raw)
|
||||
except Exception:
|
||||
logger.exception("[knowledge] document %s — LLM extraction call failed; using rule-based fallback", doc_id)
|
||||
return []
|
||||
|
||||
if not data:
|
||||
logger.warning("[knowledge] document %s — LLM returned no parsable JSON; using rule-based fallback", doc_id)
|
||||
return []
|
||||
|
||||
note_dicts = _notes_from_data(data)
|
||||
if not note_dicts:
|
||||
reason = str(data.get("reason") or "")[:200]
|
||||
if _is_falsey(data.get("should_ingest")):
|
||||
logger.warning("[knowledge] document %s SKIPPED — model judged not worth ingesting: %s", doc_id, reason)
|
||||
raise LLMRejectedIngest(reason or "no knowledge value")
|
||||
logger.warning(
|
||||
"[knowledge] document %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
|
||||
doc_id,
|
||||
data.get("should_ingest"),
|
||||
reason,
|
||||
)
|
||||
return []
|
||||
note_dicts = note_dicts[:1] if not allow_multi else note_dicts[: max(1, max_notes)]
|
||||
|
||||
fallback_title = (doc_title or title or "文档知识").strip() or "文档知识"
|
||||
drafts: list[NoteDraft] = []
|
||||
for i, nd in enumerate(note_dicts):
|
||||
override = title if (len(note_dicts) == 1) else None
|
||||
suffix = "" if len(note_dicts) == 1 else f"#{i}"
|
||||
drafts.append(
|
||||
_build_note_draft(
|
||||
nd,
|
||||
source_type="file",
|
||||
source_id=doc_id,
|
||||
source_key=f"file:{doc_id}{suffix}",
|
||||
fallback_title=fallback_title,
|
||||
title_override=override,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
|
||||
async def extract_thread_llm(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None = None,
|
||||
mode: str = "summary",
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
max_messages: int = 80,
|
||||
model_name: str | None = None,
|
||||
max_input_chars: int = 12000,
|
||||
) -> NoteDraft | None:
|
||||
"""Distill a thread into a single :class:`NoteDraft` (backward-compatible).
|
||||
|
||||
Thin wrapper over :func:`extract_thread_llm_multi` that returns the first
|
||||
note, or ``None`` when nothing was produced.
|
||||
"""
|
||||
try:
|
||||
drafts = await extract_thread_llm_multi(
|
||||
messages,
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
mode=mode,
|
||||
title=title,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
max_messages=max_messages,
|
||||
model_name=model_name,
|
||||
max_input_chars=max_input_chars,
|
||||
allow_multi=False,
|
||||
)
|
||||
except LLMRejectedIngest:
|
||||
return None
|
||||
return drafts[0] if drafts else None
|
||||
@@ -0,0 +1,74 @@
|
||||
""".manifest.json read/write for the vault.
|
||||
|
||||
The manifest records, per page, the source key, content hash, vault path and
|
||||
update time — so batch ingest can skip already-sedimented threads (delta/dedup)
|
||||
exactly like obsidian-wiki's native ``.manifest.json``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def content_hash(text: str) -> str:
|
||||
"""Stable sha256 hex digest of ``text`` (used for dedup)."""
|
||||
return hashlib.sha256((text or "").encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class Manifest:
|
||||
"""JSON manifest stored at ``<vault>/.manifest.json``."""
|
||||
|
||||
def __init__(self, root: Path | str) -> None:
|
||||
self.path = Path(root).resolve() / ".manifest.json"
|
||||
|
||||
def load(self) -> dict[str, Any]:
|
||||
if not self.path.exists():
|
||||
return {"version": 1, "updated": None, "pages": {}, "sources": {}}
|
||||
try:
|
||||
data = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return {"version": 1, "updated": None, "pages": {}, "sources": {}}
|
||||
data.setdefault("pages", {})
|
||||
data.setdefault("sources", {})
|
||||
return data
|
||||
|
||||
def save(self, data: dict[str, Any]) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
def record_page(
|
||||
self,
|
||||
*,
|
||||
note_id: str,
|
||||
vault_path: str,
|
||||
source_key: str | None,
|
||||
hash_value: str,
|
||||
updated: str,
|
||||
) -> None:
|
||||
data = self.load()
|
||||
data["pages"][note_id] = {
|
||||
"vault_path": vault_path,
|
||||
"source": source_key,
|
||||
"hash": hash_value,
|
||||
"updated": updated,
|
||||
}
|
||||
if source_key:
|
||||
data["sources"][source_key] = {"note_id": note_id, "hash": hash_value, "updated": updated}
|
||||
data["updated"] = updated
|
||||
self.save(data)
|
||||
|
||||
def remove_page(self, note_id: str) -> None:
|
||||
data = self.load()
|
||||
page = data["pages"].pop(note_id, None)
|
||||
if page and page.get("source"):
|
||||
data["sources"].pop(page["source"], None)
|
||||
self.save(data)
|
||||
|
||||
def has_source(self, source_key: str) -> bool:
|
||||
return source_key in self.load().get("sources", {})
|
||||
|
||||
def get_source(self, source_key: str) -> dict[str, Any] | None:
|
||||
return self.load().get("sources", {}).get(source_key)
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
"""obsidian-wiki compatible Markdown vault writer/reader.
|
||||
|
||||
Responsibilities (dev doc §10 ``markdown_vault.py``):
|
||||
|
||||
- Generate obsidian-wiki compatible frontmatter.
|
||||
- Produce safe, path-traversal-proof file names/paths.
|
||||
- Write/update/read Markdown note files.
|
||||
- Parse frontmatter + extract ``[[wikilinks]]``.
|
||||
|
||||
The vault layout follows §8 of the dev doc and stays compatible with
|
||||
``ar9av/obsidian-wiki`` (concepts/ references/ synthesis/ journal/ projects/…,
|
||||
plus index.md / log.md / hot.md / .manifest.json / _meta/taxonomy.md /
|
||||
wiki-export/).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
_WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")
|
||||
_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?(.*)$", re.DOTALL)
|
||||
_UNSAFE_CHARS_RE = re.compile(r"[^\w一-鿿-]+")
|
||||
|
||||
# Top-level vault directories that must exist (obsidian-wiki native layout).
|
||||
_VAULT_DIRS = ("concepts", "entities", "skills", "references", "synthesis", "journal", "projects", "_meta", "wiki-export")
|
||||
|
||||
|
||||
def slugify(title: str, *, max_len: int = 60) -> str:
|
||||
"""Return a filesystem-safe slug (ASCII + CJK), never empty."""
|
||||
text = unicodedata.normalize("NFKC", (title or "").strip()).lower()
|
||||
text = text.replace(" ", "-")
|
||||
text = _UNSAFE_CHARS_RE.sub("-", text).strip("-")
|
||||
text = re.sub(r"-{2,}", "-", text)
|
||||
if len(text) > max_len:
|
||||
text = text[:max_len].rstrip("-")
|
||||
return text or "note"
|
||||
|
||||
|
||||
def extract_wikilinks(text: str) -> list[str]:
|
||||
"""Return the targets of every ``[[wikilink]]`` in ``text`` (de-duplicated)."""
|
||||
seen: list[str] = []
|
||||
for match in _WIKILINK_RE.finditer(text or ""):
|
||||
target = match.group(1).strip()
|
||||
if target and target not in seen:
|
||||
seen.append(target)
|
||||
return seen
|
||||
|
||||
|
||||
def normalize_wikilink_target(target: str) -> str:
|
||||
"""Canonicalize a ``[[wikilink]]`` target for resolution/comparison.
|
||||
|
||||
Drops any ``|alias`` part, normalizes slashes, strips a leading ``/`` and a
|
||||
trailing ``.md`` — so ``[[index]]``, ``index.md`` and ``/index`` all map to
|
||||
the same canonical ``index`` key used by the index/graph/resolver layers.
|
||||
"""
|
||||
t = (target or "").strip()
|
||||
if not t:
|
||||
return ""
|
||||
t = t.split("|", 1)[0].strip()
|
||||
t = t.replace("\\", "/").lstrip("/")
|
||||
if t.lower().endswith(".md"):
|
||||
t = t[:-3]
|
||||
return t
|
||||
|
||||
|
||||
class VaultPathError(ValueError):
|
||||
"""Raised when a requested vault path escapes the vault root."""
|
||||
|
||||
|
||||
class MarkdownVault:
|
||||
"""Reads/writes Markdown files inside a single vault root directory."""
|
||||
|
||||
def __init__(self, root: Path | str) -> None:
|
||||
self.root = Path(root).resolve()
|
||||
|
||||
# -- path safety ------------------------------------------------------
|
||||
def resolve(self, rel_path: str) -> Path:
|
||||
"""Resolve ``rel_path`` under the vault root, blocking traversal."""
|
||||
rel = str(rel_path).replace("\\", "/").lstrip("/")
|
||||
candidate = (self.root / rel).resolve()
|
||||
try:
|
||||
candidate.relative_to(self.root)
|
||||
except ValueError as exc:
|
||||
raise VaultPathError(f"Path escapes vault root: {rel_path!r}") from exc
|
||||
return candidate
|
||||
|
||||
def note_relpath(self, *, category: str, project_name: str | None, slug: str, note_id: str) -> str:
|
||||
"""Build a vault-relative path for a note.
|
||||
|
||||
Project knowledge goes under ``projects/<project>/<category>/`` per
|
||||
wiki-capture; a short id suffix guarantees uniqueness.
|
||||
"""
|
||||
from deerflow.knowledge.skill_templates import category_to_dir
|
||||
|
||||
cat_dir = category_to_dir(category)
|
||||
safe_slug = slugify(slug)
|
||||
suffix = note_id.split("_")[-1][:8]
|
||||
filename = f"{safe_slug}-{suffix}.md"
|
||||
if project_name:
|
||||
return f"projects/{slugify(project_name)}/{cat_dir}/{filename}"
|
||||
return f"{cat_dir}/{filename}"
|
||||
|
||||
# -- scaffold ---------------------------------------------------------
|
||||
def ensure_scaffold(self, *, project_name: str | None = None) -> None:
|
||||
"""Create the vault directory tree + base files if missing."""
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
for d in _VAULT_DIRS:
|
||||
(self.root / d).mkdir(parents=True, exist_ok=True)
|
||||
if project_name:
|
||||
base = self.root / "projects" / slugify(project_name)
|
||||
for d in ("concepts", "references", "synthesis", "journal"):
|
||||
(base / d).mkdir(parents=True, exist_ok=True)
|
||||
# Base vault files.
|
||||
index = self.root / "index.md"
|
||||
if not index.exists():
|
||||
index.write_text("# Knowledge Index\n\n_页面索引。每次写入新页面后自动更新。_\n\n", encoding="utf-8")
|
||||
log = self.root / "log.md"
|
||||
if not log.exists():
|
||||
log.write_text("# Activity Log\n\n_CAPTURE / INGEST / QUERY / EXPORT 记录。_\n\n", encoding="utf-8")
|
||||
hot = self.root / "hot.md"
|
||||
if not hot.exists():
|
||||
hot.write_text("# Hot\n\n_近期活动摘要。_\n\n", encoding="utf-8")
|
||||
taxonomy = self.root / "_meta" / "taxonomy.md"
|
||||
if not taxonomy.exists():
|
||||
taxonomy.write_text(
|
||||
"# Taxonomy\n\n- synthesis — 多步骤分析、方案、结论\n- concepts — 概念、框架、模型\n"
|
||||
"- references — 外部来源、搜索结果、文章资料\n- decision — 架构或设计决策\n- journal — 完整会话摘要\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# -- frontmatter ------------------------------------------------------
|
||||
@staticmethod
|
||||
def build_frontmatter(
|
||||
*,
|
||||
note_id: str,
|
||||
title: str,
|
||||
category: str,
|
||||
tags: list[str],
|
||||
sources: list[str],
|
||||
summary: str,
|
||||
created: str,
|
||||
updated: str,
|
||||
base_confidence: float,
|
||||
lifecycle: str,
|
||||
provenance: dict[str, float] | None = None,
|
||||
relationships: list[dict[str, str]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build an obsidian-wiki / wiki-capture compatible frontmatter dict."""
|
||||
created_date = (created or "")[:10]
|
||||
return {
|
||||
"id": note_id,
|
||||
"title": title,
|
||||
"category": category,
|
||||
"tags": list(tags or []),
|
||||
"sources": list(sources or []),
|
||||
"created": created,
|
||||
"updated": updated,
|
||||
"summary": summary or "",
|
||||
"provenance": provenance or {"extracted": 0.7, "inferred": 0.2, "ambiguous": 0.1},
|
||||
"base_confidence": round(float(base_confidence or 0.0), 2),
|
||||
"lifecycle": lifecycle,
|
||||
"lifecycle_changed": created_date,
|
||||
"relationships": relationships or [],
|
||||
}
|
||||
|
||||
# -- IO ---------------------------------------------------------------
|
||||
def write_note(self, rel_path: str, frontmatter: dict[str, Any], body: str) -> Path:
|
||||
"""Write ``frontmatter`` + ``body`` to ``rel_path`` (creates dirs)."""
|
||||
path = self.resolve(rel_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fm = yaml.safe_dump(frontmatter, allow_unicode=True, sort_keys=False, default_flow_style=False)
|
||||
path.write_text(f"---\n{fm}---\n\n{body.strip()}\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
def read_note(self, rel_path: str) -> tuple[dict[str, Any], str] | None:
|
||||
"""Return ``(frontmatter, body)`` for a note, or ``None`` if missing."""
|
||||
path = self.resolve(rel_path)
|
||||
if not path.exists():
|
||||
return None
|
||||
return self.parse(path.read_text(encoding="utf-8"))
|
||||
|
||||
@staticmethod
|
||||
def parse(text: str) -> tuple[dict[str, Any], str]:
|
||||
"""Split raw Markdown into ``(frontmatter dict, body)``."""
|
||||
match = _FRONTMATTER_RE.match(text or "")
|
||||
if not match:
|
||||
return {}, text or ""
|
||||
try:
|
||||
fm = yaml.safe_load(match.group(1)) or {}
|
||||
except yaml.YAMLError:
|
||||
fm = {}
|
||||
if not isinstance(fm, dict):
|
||||
fm = {}
|
||||
return fm, match.group(2) or ""
|
||||
|
||||
def delete_note(self, rel_path: str) -> None:
|
||||
"""Remove a note file if present (used on hard delete; archive keeps it)."""
|
||||
path = self.resolve(rel_path)
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
|
||||
|
||||
def now_iso_with_tz(dt: datetime) -> str:
|
||||
"""ISO-8601 string with timezone offset (frontmatter ``created``/``updated``)."""
|
||||
if dt.tzinfo is None:
|
||||
return dt.isoformat()
|
||||
return dt.isoformat(timespec="seconds")
|
||||
@@ -0,0 +1,223 @@
|
||||
"""ORM models for the product knowledge base.
|
||||
|
||||
Tables mirror the obsidian-wiki vault into the database so the product can
|
||||
list, filter, search and audit notes quickly without re-parsing Markdown. The
|
||||
vault Markdown remains the source-of-truth for ``content_md`` / frontmatter /
|
||||
graph; these rows are an index + audit layer.
|
||||
|
||||
The knowledge base is **global / shared** — there is no ``user_id`` isolation.
|
||||
Writes record ``created_by`` / ``updated_by`` for auditing only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
# MySQL InnoDB with the legacy 767-byte index-prefix limit (5.6, or 5.7 with
|
||||
# COMPACT row format) allows at most 191 utf8mb4 chars per index key. Columns
|
||||
# longer than that use an explicit prefix index (``mysql_length`` — ignored by
|
||||
# SQLite/Postgres) instead of ``index=True``, otherwise ``create_all`` fails
|
||||
# with error 1071 ("Specified key was too long").
|
||||
|
||||
|
||||
class KnowledgeNoteRow(Base):
|
||||
"""Knowledge note main table (mirrors ``knowledge_notes``)."""
|
||||
|
||||
__tablename__ = "knowledge_notes"
|
||||
__table_args__ = (
|
||||
Index("ix_knowledge_notes_vault_path", "vault_path", mysql_length=191),
|
||||
Index("ix_knowledge_notes_source_id", "source_id", mysql_length=191),
|
||||
Index("ix_knowledge_notes_folder", "folder", mysql_length=191),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
title: Mapped[str] = mapped_column(String(512))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
content_md: Mapped[str] = mapped_column(PortableLongText())
|
||||
vault_path: Mapped[str | None] = mapped_column(String(1024))
|
||||
# User-assigned directory (slash-joined path, e.g. "投研/行业"). Independent
|
||||
# of the wiki-capture ``category`` and of ``vault_path``; drives the product
|
||||
# directory tree. NULL = unfiled (shown under a fallback group).
|
||||
folder: Mapped[str | None] = mapped_column(String(512))
|
||||
source_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
source_id: Mapped[str | None] = mapped_column(String(256))
|
||||
status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=0.0)
|
||||
created_by: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
|
||||
class KnowledgeSourceRow(Base):
|
||||
"""Knowledge source table (mirrors ``knowledge_sources``)."""
|
||||
|
||||
__tablename__ = "knowledge_sources"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
|
||||
source_type: Mapped[str] = mapped_column(String(64))
|
||||
thread_id: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
message_id: Mapped[str | None] = mapped_column(String(128))
|
||||
tool_name: Mapped[str | None] = mapped_column(String(128))
|
||||
title: Mapped[str | None] = mapped_column(String(512))
|
||||
url: Mapped[str | None] = mapped_column(Text)
|
||||
snippet: Mapped[str | None] = mapped_column(Text)
|
||||
raw_json: Mapped[str | None] = mapped_column(PortableLongText())
|
||||
created_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeTagRow(Base):
|
||||
"""Tag table (mirrors ``knowledge_tags``)."""
|
||||
|
||||
__tablename__ = "knowledge_tags"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(128), unique=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeNoteTagRow(Base):
|
||||
"""Note ↔ tag join table (mirrors ``knowledge_note_tags``)."""
|
||||
|
||||
__tablename__ = "knowledge_note_tags"
|
||||
__table_args__ = (UniqueConstraint("note_id", "tag_id", name="uq_knowledge_note_tags"),)
|
||||
|
||||
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), primary_key=True)
|
||||
tag_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_tags.id", ondelete="CASCADE"), primary_key=True)
|
||||
|
||||
|
||||
class KnowledgeEmbeddingRow(Base):
|
||||
"""Per-chunk embedding index (phase 2, mirrors ``knowledge_embeddings``).
|
||||
|
||||
The vector is stored as a JSON float array; first-release similarity is
|
||||
computed in Python (no external vector DB). ``vector_id`` is reserved for a
|
||||
future external store mapping.
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_embeddings"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
|
||||
chunk_index: Mapped[int] = mapped_column(Integer, default=0)
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
vector: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
vector_id: Mapped[str | None] = mapped_column(String(256))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeNoteVersionRow(Base):
|
||||
"""Immutable snapshot of a note before each update (phase 3 versioning)."""
|
||||
|
||||
__tablename__ = "knowledge_note_versions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
|
||||
title: Mapped[str] = mapped_column(String(512))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
content_md: Mapped[str] = mapped_column(PortableLongText())
|
||||
status: Mapped[str] = mapped_column(String(32))
|
||||
tags_json: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
edited_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
|
||||
class KnowledgeEntityRow(Base):
|
||||
"""Knowledge entity (phase 4, mirrors ``knowledge_entities``)."""
|
||||
|
||||
__tablename__ = "knowledge_entities"
|
||||
__table_args__ = (
|
||||
Index("ix_knowledge_entities_name", "name", mysql_length=191),
|
||||
# Prefix-unique: (120 + 64) * 4 = 736 bytes <= 767. Entity names are
|
||||
# realistically far shorter than 120 chars, so the prefix is lossless.
|
||||
Index("uq_knowledge_entities_name_type", "name", "entity_type", unique=True, mysql_length={"name": 120}),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
entity_type: Mapped[str | None] = mapped_column(String(64))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeGraphBlacklistRow(Base):
|
||||
"""Graph blacklist: node labels (entity names / note titles) hidden from the graph.
|
||||
|
||||
Matching is case-insensitive on the trimmed label. Global/shared like the
|
||||
rest of the knowledge base; ``created_by`` is audit-only.
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_graph_blacklist"
|
||||
__table_args__ = (Index("ix_knowledge_graph_blacklist_label", "label", unique=True, mysql_length=191),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
label: Mapped[str] = mapped_column(String(512))
|
||||
created_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeRelationRow(Base):
|
||||
"""Relation between notes/entities (phase 4, mirrors ``knowledge_relations``)."""
|
||||
|
||||
__tablename__ = "knowledge_relations"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
from_note_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
to_note_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
from_entity_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
to_entity_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
relation_type: Mapped[str] = mapped_column(String(128))
|
||||
weight: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeFolderRow(Base):
|
||||
"""User-managed knowledge directory (mirrors ``knowledge_folders``).
|
||||
|
||||
A folder is identified by its full slash-joined ``path`` (e.g. ``投研/行业``);
|
||||
nesting is derived by splitting on ``/`` on the client. Folders exist
|
||||
independently of notes so an operator can create an empty directory and file
|
||||
notes into it later (``knowledge_notes.folder`` stores the chosen path).
|
||||
Global/shared like the rest of the KB; ``created_by`` is audit-only.
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_folders"
|
||||
__table_args__ = (Index("uq_knowledge_folders_path", "path", unique=True, mysql_length=191),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
path: Mapped[str] = mapped_column(String(512))
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0)
|
||||
created_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeExtractTemplateRow(Base):
|
||||
"""Editable knowledge-extraction prompt template (mirrors ``knowledge_extract_templates``).
|
||||
|
||||
Lets an operator open up, edit and add multiple distillation prompts instead
|
||||
of the hard-coded constants. ``scope`` is ``thread`` / ``document`` / ``both``;
|
||||
at capture time the service resolves the chosen template (or the default for
|
||||
that scope) and feeds its ``system_prompt`` to the extractor. Global/shared.
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_extract_templates"
|
||||
__table_args__ = (Index("uq_knowledge_extract_templates_name", "name", unique=True, mysql_length=191),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
scope: Mapped[str] = mapped_column(String(16), default="both", index=True)
|
||||
system_prompt: Mapped[str] = mapped_column(PortableLongText())
|
||||
is_default: Mapped[bool] = mapped_column(default=False, index=True)
|
||||
enabled: Mapped[bool] = mapped_column(default=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(128))
|
||||
updated_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+256
@@ -0,0 +1,256 @@
|
||||
"""ObsidianWikiAdapter — the obsidian-wiki vault facade.
|
||||
|
||||
Owns the Markdown vault: writes notes with wiki-capture frontmatter/body, keeps
|
||||
``index.md`` / ``log.md`` / ``hot.md`` / ``.manifest.json`` in sync after every
|
||||
write, and produces wiki-export graph artifacts. Output stays compatible with
|
||||
``ar9av/obsidian-wiki`` without ever invoking its CLI.
|
||||
|
||||
This class is storage-only (no DB, no LLM). The :class:`KnowledgeService`
|
||||
combines it with the repository + extractor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge import exporter
|
||||
from deerflow.knowledge.manifest import Manifest, content_hash
|
||||
from deerflow.knowledge.markdown_vault import MarkdownVault
|
||||
from deerflow.knowledge.schemas import NoteDraft
|
||||
|
||||
_INDEX_PAGES_HEADER = "## Pages"
|
||||
_HOT_HEADER = "## Recent"
|
||||
_HOT_MAX = 20
|
||||
|
||||
|
||||
class ObsidianWikiAdapter:
|
||||
"""Read/write an obsidian-wiki compatible vault."""
|
||||
|
||||
def __init__(self, vault_root: Path | str, *, project_name: str = "zncm", export_html: bool = True, export_json: bool = True) -> None:
|
||||
self.vault = MarkdownVault(vault_root)
|
||||
self.root = self.vault.root
|
||||
self.manifest = Manifest(self.root)
|
||||
self.project_name = project_name
|
||||
self.export_html = export_html
|
||||
self.export_json = export_json
|
||||
|
||||
def ensure_ready(self) -> None:
|
||||
self.vault.ensure_scaffold(project_name=self.project_name)
|
||||
|
||||
# -- writes -----------------------------------------------------------
|
||||
def write_note(
|
||||
self,
|
||||
note_id: str,
|
||||
draft: NoteDraft,
|
||||
*,
|
||||
created: str,
|
||||
updated: str,
|
||||
status: str,
|
||||
) -> str:
|
||||
"""Write the note file, sync vault metadata, return the vault path."""
|
||||
self.ensure_ready()
|
||||
rel = self.vault.note_relpath(category=draft.category, project_name=self.project_name, slug=draft.title, note_id=note_id)
|
||||
sources_fm = self._frontmatter_sources(draft)
|
||||
relationships = [{"target": link, "type": "related"} for link in (draft.related or [])[:8]]
|
||||
frontmatter = self.vault.build_frontmatter(
|
||||
note_id=note_id,
|
||||
title=draft.title,
|
||||
category=draft.category,
|
||||
tags=draft.tags,
|
||||
sources=sources_fm,
|
||||
summary=draft.summary,
|
||||
created=created,
|
||||
updated=updated,
|
||||
base_confidence=draft.confidence,
|
||||
lifecycle=status,
|
||||
relationships=relationships,
|
||||
)
|
||||
self.vault.write_note(rel, frontmatter, draft.content_md)
|
||||
self.manifest.record_page(
|
||||
note_id=note_id,
|
||||
vault_path=rel,
|
||||
source_key=draft.source_key,
|
||||
hash_value=content_hash(draft.content_md),
|
||||
updated=updated,
|
||||
)
|
||||
self._update_index(note_id, rel, draft.title, draft.summary)
|
||||
self._append_log("CAPTURE" if draft.source_type in ("thread", "search", "tool") else "INGEST", note_id, draft.title, updated)
|
||||
self._update_hot(draft.title, rel, updated)
|
||||
return rel
|
||||
|
||||
def update_note_file(
|
||||
self,
|
||||
note_id: str,
|
||||
vault_path: str | None,
|
||||
*,
|
||||
title: str,
|
||||
content_md: str,
|
||||
tags: list[str],
|
||||
status: str,
|
||||
summary: str,
|
||||
category: str,
|
||||
created: str,
|
||||
updated: str,
|
||||
) -> str:
|
||||
"""Rewrite a note file in place (frontmatter refreshed), sync metadata."""
|
||||
self.ensure_ready()
|
||||
rel = vault_path
|
||||
if not rel:
|
||||
rel = self.vault.note_relpath(category=category, project_name=self.project_name, slug=title, note_id=note_id)
|
||||
# Preserve original `sources`/`provenance`/`category` frontmatter when present.
|
||||
existing = self.vault.read_note(rel)
|
||||
sources_fm = existing[0].get("sources", []) if existing else []
|
||||
provenance = existing[0].get("provenance") if existing else None
|
||||
relationships = existing[0].get("relationships") if existing else []
|
||||
effective_category = (existing[0].get("category") if existing else None) or category
|
||||
frontmatter = self.vault.build_frontmatter(
|
||||
note_id=note_id,
|
||||
title=title,
|
||||
category=effective_category,
|
||||
tags=tags,
|
||||
sources=sources_fm,
|
||||
summary=summary,
|
||||
created=created,
|
||||
updated=updated,
|
||||
base_confidence=float((existing[0].get("base_confidence") if existing else 0.0) or 0.0),
|
||||
lifecycle=status,
|
||||
provenance=provenance,
|
||||
relationships=relationships,
|
||||
)
|
||||
self.vault.write_note(rel, frontmatter, content_md)
|
||||
self.manifest.record_page(
|
||||
note_id=note_id,
|
||||
vault_path=rel,
|
||||
source_key=self.manifest.load().get("pages", {}).get(note_id, {}).get("source"),
|
||||
hash_value=content_hash(content_md),
|
||||
updated=updated,
|
||||
)
|
||||
self._update_index(note_id, rel, title, summary)
|
||||
self._append_log("UPDATE", note_id, title, updated)
|
||||
self._update_hot(title, rel, updated)
|
||||
return rel
|
||||
|
||||
def archive_note(self, note_id: str, vault_path: str | None, *, updated: str) -> None:
|
||||
"""Mark a note archived in its frontmatter + log (soft delete)."""
|
||||
if vault_path:
|
||||
parsed = self.vault.read_note(vault_path)
|
||||
if parsed:
|
||||
fm, body = parsed
|
||||
fm["lifecycle"] = "archived"
|
||||
fm["updated"] = updated
|
||||
self.vault.write_note(vault_path, fm, body)
|
||||
self._remove_from_index(note_id)
|
||||
self._append_log("ARCHIVE", note_id, "", updated)
|
||||
|
||||
def export_graph(
|
||||
self,
|
||||
notes: list[dict[str, Any]],
|
||||
*,
|
||||
entities: list[dict[str, Any]] | None = None,
|
||||
relations: list[dict[str, Any]] | None = None,
|
||||
updated: str,
|
||||
blacklist: set[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
self.ensure_ready()
|
||||
result = exporter.export_all(
|
||||
self.root,
|
||||
notes,
|
||||
entities=entities,
|
||||
relations=relations,
|
||||
write_html=self.export_html,
|
||||
write_json=self.export_json,
|
||||
blacklist=blacklist,
|
||||
)
|
||||
self._append_log("EXPORT", "graph", f"nodes={result['stats']['nodes']} edges={result['stats']['edges']}", updated)
|
||||
return result
|
||||
|
||||
def export_file_path(self, filename: str) -> Path | None:
|
||||
allowed = {"graph.json", "graph.graphml", "cypher.txt", "graph.html"}
|
||||
if filename not in allowed:
|
||||
return None
|
||||
path = (self.root / "wiki-export" / filename).resolve()
|
||||
try:
|
||||
path.relative_to(self.root)
|
||||
except ValueError:
|
||||
return None
|
||||
return path if path.exists() else None
|
||||
|
||||
# -- frontmatter source serialization ---------------------------------
|
||||
@staticmethod
|
||||
def _frontmatter_sources(draft: NoteDraft) -> list[str]:
|
||||
out: list[str] = []
|
||||
if draft.source_type == "thread" and draft.source_id:
|
||||
out.append(f"thread:{draft.source_id}")
|
||||
for s in draft.sources[:20]:
|
||||
if s.url:
|
||||
out.append(f"url:{s.url}")
|
||||
elif s.thread_id:
|
||||
out.append(f"thread:{s.thread_id}")
|
||||
elif s.tool_name:
|
||||
out.append(f"tool:{s.tool_name}")
|
||||
# De-dup preserving order.
|
||||
seen: set[str] = set()
|
||||
deduped = []
|
||||
for item in out:
|
||||
if item not in seen:
|
||||
seen.add(item)
|
||||
deduped.append(item)
|
||||
return deduped
|
||||
|
||||
# -- index.md ---------------------------------------------------------
|
||||
def _index_path(self) -> Path:
|
||||
return self.root / "index.md"
|
||||
|
||||
def _update_index(self, note_id: str, rel: str, title: str, summary: str | None) -> None:
|
||||
path = self._index_path()
|
||||
text = path.read_text(encoding="utf-8") if path.exists() else "# Knowledge Index\n\n"
|
||||
link = rel[:-3] if rel.endswith(".md") else rel
|
||||
summary_txt = (summary or "").strip().replace("\n", " ")
|
||||
if len(summary_txt) > 120:
|
||||
summary_txt = summary_txt[:120] + "…"
|
||||
line = f"- [[{link}|{title}]]" + (f" — {summary_txt}" if summary_txt else "") + f" <!-- id:{note_id} -->"
|
||||
lines = text.splitlines()
|
||||
# Drop any existing line for this note id.
|
||||
lines = [ln for ln in lines if f"<!-- id:{note_id} -->" not in ln]
|
||||
if _INDEX_PAGES_HEADER not in text:
|
||||
lines.append("")
|
||||
lines.append(_INDEX_PAGES_HEADER)
|
||||
lines.append("")
|
||||
# Insert after the Pages header.
|
||||
try:
|
||||
insert_at = lines.index(_INDEX_PAGES_HEADER) + 1
|
||||
except ValueError:
|
||||
insert_at = len(lines)
|
||||
lines.insert(insert_at, line)
|
||||
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
|
||||
|
||||
def _remove_from_index(self, note_id: str) -> None:
|
||||
path = self._index_path()
|
||||
if not path.exists():
|
||||
return
|
||||
lines = [ln for ln in path.read_text(encoding="utf-8").splitlines() if f"<!-- id:{note_id} -->" not in ln]
|
||||
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
|
||||
|
||||
# -- log.md -----------------------------------------------------------
|
||||
def _append_log(self, action: str, note_id: str, label: str, updated: str) -> None:
|
||||
path = self.root / "log.md"
|
||||
text = path.read_text(encoding="utf-8") if path.exists() else "# Activity Log\n\n"
|
||||
entry = f"- {updated} {action} {note_id}" + (f" — {label}" if label else "")
|
||||
path.write_text(text.rstrip() + "\n" + entry + "\n", encoding="utf-8")
|
||||
|
||||
# -- hot.md -----------------------------------------------------------
|
||||
def _update_hot(self, title: str, rel: str, updated: str) -> None:
|
||||
path = self.root / "hot.md"
|
||||
link = rel[:-3] if rel.endswith(".md") else rel
|
||||
new_line = f"- {updated} [[{link}|{title}]]"
|
||||
existing_lines: list[str] = []
|
||||
if path.exists():
|
||||
for ln in path.read_text(encoding="utf-8").splitlines():
|
||||
if ln.startswith("- ") and "[[" in ln:
|
||||
existing_lines.append(ln)
|
||||
existing_lines = [ln for ln in existing_lines if f"[[{link}" not in ln]
|
||||
existing_lines.insert(0, new_line)
|
||||
existing_lines = existing_lines[:_HOT_MAX]
|
||||
body = "# Hot\n\n_近期活动摘要。_\n\n" + _HOT_HEADER + "\n\n" + "\n".join(existing_lines) + "\n"
|
||||
path.write_text(body, encoding="utf-8")
|
||||
@@ -0,0 +1,781 @@
|
||||
"""Database read/write for the knowledge base.
|
||||
|
||||
This layer never calls an LLM and never touches the Markdown vault — it only
|
||||
persists/queries the relational mirror (notes, sources, tags). The vault and
|
||||
extraction live in :mod:`deerflow.knowledge.markdown_vault` and
|
||||
:mod:`deerflow.knowledge.extractor`; orchestration lives in the service.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.knowledge.models import (
|
||||
KnowledgeEmbeddingRow,
|
||||
KnowledgeEntityRow,
|
||||
KnowledgeExtractTemplateRow,
|
||||
KnowledgeFolderRow,
|
||||
KnowledgeGraphBlacklistRow,
|
||||
KnowledgeNoteRow,
|
||||
KnowledgeNoteTagRow,
|
||||
KnowledgeNoteVersionRow,
|
||||
KnowledgeRelationRow,
|
||||
KnowledgeSourceRow,
|
||||
KnowledgeTagRow,
|
||||
)
|
||||
from deerflow.knowledge.schemas import SourceDraft
|
||||
|
||||
|
||||
def _new_id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid.uuid4().hex[:24]}"
|
||||
|
||||
|
||||
def _iso(value: Any) -> Any:
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
return value
|
||||
|
||||
|
||||
class KnowledgeRepository:
|
||||
"""SQLAlchemy-backed repository for knowledge notes/sources/tags."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# -- serialization ----------------------------------------------------
|
||||
@staticmethod
|
||||
def _note_to_dict(row: KnowledgeNoteRow) -> dict[str, Any]:
|
||||
d = row.to_dict()
|
||||
d["created_at"] = _iso(d.get("created_at"))
|
||||
d["updated_at"] = _iso(d.get("updated_at"))
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def _source_to_dict(row: KnowledgeSourceRow) -> dict[str, Any]:
|
||||
d = row.to_dict()
|
||||
d["created_at"] = _iso(d.get("created_at"))
|
||||
return d
|
||||
|
||||
# -- tags -------------------------------------------------------------
|
||||
async def _get_or_create_tags(self, session: AsyncSession, names: list[str]) -> list[KnowledgeTagRow]:
|
||||
cleaned = []
|
||||
seen: set[str] = set()
|
||||
for raw in names or []:
|
||||
name = (raw or "").strip()
|
||||
if not name or name.lower() in seen:
|
||||
continue
|
||||
seen.add(name.lower())
|
||||
cleaned.append(name)
|
||||
if not cleaned:
|
||||
return []
|
||||
existing = {
|
||||
r.name.lower(): r
|
||||
for r in (await session.execute(select(KnowledgeTagRow).where(func.lower(KnowledgeTagRow.name).in_([n.lower() for n in cleaned])))).scalars()
|
||||
}
|
||||
result: list[KnowledgeTagRow] = []
|
||||
for name in cleaned:
|
||||
row = existing.get(name.lower())
|
||||
if row is None:
|
||||
row = KnowledgeTagRow(id=_new_id("kbtag"), name=name, created_at=datetime.now(UTC))
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
existing[name.lower()] = row
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
async def _tags_for(self, session: AsyncSession, note_id: str) -> list[str]:
|
||||
stmt = (
|
||||
select(KnowledgeTagRow.name)
|
||||
.join(KnowledgeNoteTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
|
||||
.where(KnowledgeNoteTagRow.note_id == note_id)
|
||||
.order_by(KnowledgeTagRow.name)
|
||||
)
|
||||
return list((await session.execute(stmt)).scalars())
|
||||
|
||||
async def _tags_for_many(self, session: AsyncSession, note_ids: list[str]) -> dict[str, list[str]]:
|
||||
if not note_ids:
|
||||
return {}
|
||||
stmt = (
|
||||
select(KnowledgeNoteTagRow.note_id, KnowledgeTagRow.name)
|
||||
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
|
||||
.where(KnowledgeNoteTagRow.note_id.in_(note_ids))
|
||||
.order_by(KnowledgeTagRow.name)
|
||||
)
|
||||
out: dict[str, list[str]] = {nid: [] for nid in note_ids}
|
||||
for note_id, name in (await session.execute(stmt)).all():
|
||||
out.setdefault(note_id, []).append(name)
|
||||
return out
|
||||
|
||||
async def _replace_note_tags(self, session: AsyncSession, note_id: str, names: list[str]) -> None:
|
||||
await session.execute(delete(KnowledgeNoteTagRow).where(KnowledgeNoteTagRow.note_id == note_id))
|
||||
tags = await self._get_or_create_tags(session, names)
|
||||
for tag in tags:
|
||||
session.add(KnowledgeNoteTagRow(note_id=note_id, tag_id=tag.id))
|
||||
|
||||
# -- notes ------------------------------------------------------------
|
||||
async def create_note(
|
||||
self,
|
||||
*,
|
||||
title: str,
|
||||
summary: str | None,
|
||||
content_md: str,
|
||||
source_type: str,
|
||||
source_id: str | None,
|
||||
status: str,
|
||||
confidence: float = 0.0,
|
||||
vault_path: str | None = None,
|
||||
folder: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
created_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
note_id = _new_id("kb")
|
||||
now = datetime.now(UTC)
|
||||
row = KnowledgeNoteRow(
|
||||
id=note_id,
|
||||
title=title[:512],
|
||||
summary=summary,
|
||||
content_md=content_md,
|
||||
vault_path=vault_path,
|
||||
folder=(folder or None),
|
||||
source_type=source_type,
|
||||
source_id=source_id,
|
||||
status=status,
|
||||
confidence=confidence,
|
||||
created_by=created_by,
|
||||
updated_by=created_by,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
await self._replace_note_tags(session, note_id, tags or [])
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
d = self._note_to_dict(row)
|
||||
d["tags"] = await self._tags_for(session, note_id)
|
||||
return d
|
||||
|
||||
async def set_vault_path(self, note_id: str, vault_path: str) -> None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeNoteRow, note_id)
|
||||
if row is None:
|
||||
return
|
||||
row.vault_path = vault_path
|
||||
await session.commit()
|
||||
|
||||
async def update_note(
|
||||
self,
|
||||
note_id: str,
|
||||
*,
|
||||
title: str | None = None,
|
||||
summary: str | None = None,
|
||||
content_md: str | None = None,
|
||||
status: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
vault_path: str | None = None,
|
||||
folder: str | None = None,
|
||||
set_folder: bool = False,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeNoteRow, note_id)
|
||||
if row is None:
|
||||
return None
|
||||
if title is not None:
|
||||
row.title = title[:512]
|
||||
if summary is not None:
|
||||
row.summary = summary
|
||||
if content_md is not None:
|
||||
row.content_md = content_md
|
||||
if status is not None:
|
||||
row.status = status
|
||||
if vault_path is not None:
|
||||
row.vault_path = vault_path
|
||||
# ``set_folder`` lets the caller assign *or clear* (folder=None) the
|
||||
# directory; a bare ``folder=None`` without the flag leaves it as-is.
|
||||
if set_folder:
|
||||
row.folder = (folder or None)
|
||||
if updated_by is not None:
|
||||
row.updated_by = updated_by
|
||||
row.updated_at = datetime.now(UTC)
|
||||
if tags is not None:
|
||||
await self._replace_note_tags(session, note_id, tags)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
d = self._note_to_dict(row)
|
||||
d["tags"] = await self._tags_for(session, note_id)
|
||||
return d
|
||||
|
||||
async def set_status(self, note_id: str, status: str, *, updated_by: str | None = None) -> dict[str, Any] | None:
|
||||
return await self.update_note(note_id, status=status, updated_by=updated_by)
|
||||
|
||||
async def get_note(self, note_id: str, *, with_sources: bool = True) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeNoteRow, note_id)
|
||||
if row is None:
|
||||
return None
|
||||
d = self._note_to_dict(row)
|
||||
d["tags"] = await self._tags_for(session, note_id)
|
||||
if with_sources:
|
||||
stmt = select(KnowledgeSourceRow).where(KnowledgeSourceRow.note_id == note_id).order_by(KnowledgeSourceRow.created_at)
|
||||
d["sources"] = [self._source_to_dict(r) for r in (await session.execute(stmt)).scalars()]
|
||||
return d
|
||||
|
||||
async def find_by_vault_path(self, vault_path: str) -> dict[str, Any] | None:
|
||||
"""Return the first (non-archived) note whose vault_path matches.
|
||||
|
||||
``vault_path`` is matched both with and without the ``.md`` suffix, since
|
||||
wikilink targets are stored without it while ``vault_path`` keeps it.
|
||||
"""
|
||||
stripped = vault_path[:-3] if vault_path.endswith(".md") else vault_path
|
||||
candidates = {vault_path, stripped, f"{stripped}.md"}
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeNoteRow)
|
||||
.where(KnowledgeNoteRow.vault_path.in_(list(candidates)))
|
||||
.where(KnowledgeNoteRow.status != "archived")
|
||||
.limit(1)
|
||||
)
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
if row is None:
|
||||
return None
|
||||
d = self._note_to_dict(row)
|
||||
d["tags"] = await self._tags_for(session, row.id)
|
||||
return d
|
||||
|
||||
async def find_by_title(self, title: str) -> dict[str, Any] | None:
|
||||
"""Return the first (non-archived) note with a case-insensitive title match."""
|
||||
name = (title or "").strip()
|
||||
if not name:
|
||||
return None
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeNoteRow)
|
||||
.where(func.lower(KnowledgeNoteRow.title) == name.lower())
|
||||
.where(KnowledgeNoteRow.status != "archived")
|
||||
.order_by(KnowledgeNoteRow.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
if row is None:
|
||||
return None
|
||||
d = self._note_to_dict(row)
|
||||
d["tags"] = await self._tags_for(session, row.id)
|
||||
return d
|
||||
|
||||
async def find_by_source(self, source_type: str, source_id: str) -> dict[str, Any] | None:
|
||||
"""Return the first (non-archived) note for a (source_type, source_id)."""
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeNoteRow)
|
||||
.where(KnowledgeNoteRow.source_type == source_type, KnowledgeNoteRow.source_id == source_id)
|
||||
.where(KnowledgeNoteRow.status != "archived")
|
||||
.limit(1)
|
||||
)
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
return self._note_to_dict(row) if row else None
|
||||
|
||||
async def list_notes(
|
||||
self,
|
||||
*,
|
||||
keyword: str | None = None,
|
||||
tag: str | None = None,
|
||||
entity: str | None = None,
|
||||
source_type: str | None = None,
|
||||
status: str | None = None,
|
||||
folder: str | None = None,
|
||||
include_archived: bool = False,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeNoteRow)
|
||||
count_stmt = select(func.count()).select_from(KnowledgeNoteRow)
|
||||
|
||||
conditions = []
|
||||
if status:
|
||||
conditions.append(KnowledgeNoteRow.status == status)
|
||||
elif not include_archived:
|
||||
conditions.append(KnowledgeNoteRow.status != "archived")
|
||||
if source_type:
|
||||
conditions.append(KnowledgeNoteRow.source_type == source_type)
|
||||
if folder is not None:
|
||||
# Exact folder, plus any nested sub-folders (``folder/…``).
|
||||
if folder == "":
|
||||
conditions.append(KnowledgeNoteRow.folder.is_(None))
|
||||
else:
|
||||
conditions.append(or_(KnowledgeNoteRow.folder == folder, KnowledgeNoteRow.folder.like(f"{folder}/%")))
|
||||
if keyword:
|
||||
like = f"%{keyword}%"
|
||||
conditions.append(or_(KnowledgeNoteRow.title.like(like), KnowledgeNoteRow.summary.like(like), KnowledgeNoteRow.content_md.like(like)))
|
||||
if tag:
|
||||
tag_subq = (
|
||||
select(KnowledgeNoteTagRow.note_id)
|
||||
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
|
||||
.where(func.lower(KnowledgeTagRow.name) == tag.lower())
|
||||
)
|
||||
conditions.append(KnowledgeNoteRow.id.in_(tag_subq))
|
||||
if entity:
|
||||
# Exact entity match via the note→entity relation graph — precise
|
||||
# and index-backed, unlike a content LIKE scan on large vaults.
|
||||
entity_subq = (
|
||||
select(KnowledgeRelationRow.from_note_id)
|
||||
.join(KnowledgeEntityRow, KnowledgeRelationRow.to_entity_id == KnowledgeEntityRow.id)
|
||||
.where(func.lower(KnowledgeEntityRow.name) == entity.lower())
|
||||
.where(KnowledgeRelationRow.from_note_id.is_not(None))
|
||||
)
|
||||
conditions.append(KnowledgeNoteRow.id.in_(entity_subq))
|
||||
|
||||
for cond in conditions:
|
||||
stmt = stmt.where(cond)
|
||||
count_stmt = count_stmt.where(cond)
|
||||
|
||||
total = (await session.execute(count_stmt)).scalar_one()
|
||||
stmt = stmt.order_by(KnowledgeNoteRow.updated_at.desc()).limit(limit).offset(offset)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
ids = [r.id for r in rows]
|
||||
tags_map = await self._tags_for_many(session, ids)
|
||||
items = []
|
||||
for r in rows:
|
||||
d = self._note_to_dict(r)
|
||||
d["tags"] = tags_map.get(r.id, [])
|
||||
items.append(d)
|
||||
return items, int(total)
|
||||
|
||||
async def list_all(self, *, include_archived: bool = False) -> list[dict[str, Any]]:
|
||||
"""Return every note (light fields + tags) for graph export."""
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeNoteRow)
|
||||
if not include_archived:
|
||||
stmt = stmt.where(KnowledgeNoteRow.status != "archived")
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
tags_map = await self._tags_for_many(session, [r.id for r in rows])
|
||||
out = []
|
||||
for r in rows:
|
||||
d = self._note_to_dict(r)
|
||||
d["tags"] = tags_map.get(r.id, [])
|
||||
out.append(d)
|
||||
return out
|
||||
|
||||
# -- embeddings (phase 2) --------------------------------------------
|
||||
async def replace_embeddings(self, note_id: str, chunks: list[tuple[int, str, list[float]]]) -> None:
|
||||
"""Replace all embedding rows for a note with ``(index, content, vector)``."""
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(KnowledgeEmbeddingRow).where(KnowledgeEmbeddingRow.note_id == note_id))
|
||||
for idx, content, vector in chunks:
|
||||
session.add(
|
||||
KnowledgeEmbeddingRow(
|
||||
id=_new_id("kbemb"),
|
||||
note_id=note_id,
|
||||
chunk_index=idx,
|
||||
content=content,
|
||||
vector=list(vector),
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def delete_embeddings(self, note_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(KnowledgeEmbeddingRow).where(KnowledgeEmbeddingRow.note_id == note_id))
|
||||
await session.commit()
|
||||
|
||||
async def all_embeddings(self) -> list[dict[str, Any]]:
|
||||
"""Return every embedding chunk joined to its (non-archived) note."""
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeEmbeddingRow, KnowledgeNoteRow.title, KnowledgeNoteRow.status)
|
||||
.join(KnowledgeNoteRow, KnowledgeNoteRow.id == KnowledgeEmbeddingRow.note_id)
|
||||
.where(KnowledgeNoteRow.status != "archived")
|
||||
)
|
||||
out = []
|
||||
for emb, title, _status in (await session.execute(stmt)).all():
|
||||
out.append({"note_id": emb.note_id, "chunk_index": emb.chunk_index, "content": emb.content, "vector": emb.vector or [], "title": title})
|
||||
return out
|
||||
|
||||
async def has_embeddings(self) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(KnowledgeEmbeddingRow.id).limit(1))).first()
|
||||
return row is not None
|
||||
|
||||
# -- versions (phase 3) ----------------------------------------------
|
||||
async def add_version(self, note: dict[str, Any], *, edited_by: str | None) -> None:
|
||||
async with self._sf() as session:
|
||||
session.add(
|
||||
KnowledgeNoteVersionRow(
|
||||
id=_new_id("kbver"),
|
||||
note_id=note["id"],
|
||||
title=note.get("title") or "",
|
||||
summary=note.get("summary"),
|
||||
content_md=note.get("content_md") or "",
|
||||
status=note.get("status") or "approved",
|
||||
tags_json=note.get("tags") or [],
|
||||
edited_by=edited_by,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeNoteVersionRow).where(KnowledgeNoteVersionRow.note_id == note_id).order_by(KnowledgeNoteVersionRow.created_at.desc())
|
||||
out = []
|
||||
for r in (await session.execute(stmt)).scalars():
|
||||
d = r.to_dict()
|
||||
d["tags"] = d.pop("tags_json", [])
|
||||
d["created_at"] = _iso(d.get("created_at"))
|
||||
out.append(d)
|
||||
return out
|
||||
|
||||
# -- entities & relations (phase 4) ----------------------------------
|
||||
async def get_or_create_entity(self, name: str, entity_type: str | None) -> dict[str, Any]:
|
||||
name = (name or "").strip()
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeEntityRow).where(func.lower(KnowledgeEntityRow.name) == name.lower())
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
if row is None:
|
||||
row = KnowledgeEntityRow(id=_new_id("kbent"), name=name, entity_type=entity_type, created_at=datetime.now(UTC))
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return {"id": row.id, "name": row.name, "entity_type": row.entity_type}
|
||||
|
||||
async def add_relation(
|
||||
self,
|
||||
*,
|
||||
from_note_id: str | None = None,
|
||||
to_note_id: str | None = None,
|
||||
from_entity_id: str | None = None,
|
||||
to_entity_id: str | None = None,
|
||||
relation_type: str,
|
||||
weight: float = 1.0,
|
||||
) -> None:
|
||||
async with self._sf() as session:
|
||||
session.add(
|
||||
KnowledgeRelationRow(
|
||||
id=_new_id("kbrel"),
|
||||
from_note_id=from_note_id,
|
||||
to_note_id=to_note_id,
|
||||
from_entity_id=from_entity_id,
|
||||
to_entity_id=to_entity_id,
|
||||
relation_type=relation_type,
|
||||
weight=weight,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def clear_note_relations(self, note_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(KnowledgeRelationRow).where(KnowledgeRelationRow.from_note_id == note_id))
|
||||
await session.commit()
|
||||
|
||||
async def list_note_entities(self, note_id: str) -> list[dict[str, Any]]:
|
||||
"""Return the entities mentioned by a note (for note-detail display)."""
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeEntityRow.name, KnowledgeEntityRow.entity_type)
|
||||
.join(KnowledgeRelationRow, KnowledgeRelationRow.to_entity_id == KnowledgeEntityRow.id)
|
||||
.where(KnowledgeRelationRow.from_note_id == note_id)
|
||||
.order_by(KnowledgeEntityRow.name)
|
||||
)
|
||||
seen: set[str] = set()
|
||||
out: list[dict[str, Any]] = []
|
||||
for name, etype in (await session.execute(stmt)).all():
|
||||
key = (name or "").lower()
|
||||
if not name or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append({"name": name, "entity_type": etype})
|
||||
return out
|
||||
|
||||
async def prune_orphan_entities(self) -> int:
|
||||
"""Delete entity rows no longer referenced by any relation. Returns count."""
|
||||
async with self._sf() as session:
|
||||
referenced: set[str] = set()
|
||||
for col in (KnowledgeRelationRow.from_entity_id, KnowledgeRelationRow.to_entity_id):
|
||||
for (eid,) in (await session.execute(select(col).where(col.isnot(None)))).all():
|
||||
referenced.add(eid)
|
||||
all_ids = list((await session.execute(select(KnowledgeEntityRow.id))).scalars())
|
||||
orphans = [eid for eid in all_ids if eid not in referenced]
|
||||
if orphans:
|
||||
await session.execute(delete(KnowledgeEntityRow).where(KnowledgeEntityRow.id.in_(orphans)))
|
||||
await session.commit()
|
||||
return len(orphans)
|
||||
|
||||
async def list_entities(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(select(KnowledgeEntityRow))).scalars()
|
||||
return [{"id": r.id, "name": r.name, "entity_type": r.entity_type} for r in rows]
|
||||
|
||||
# -- graph blacklist ---------------------------------------------------
|
||||
async def list_graph_blacklist(self) -> list[dict[str, Any]]:
|
||||
"""Return every blacklisted graph node label, newest first."""
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeGraphBlacklistRow).order_by(KnowledgeGraphBlacklistRow.created_at.desc())
|
||||
rows = (await session.execute(stmt)).scalars()
|
||||
return [{"id": r.id, "label": r.label, "created_by": r.created_by, "created_at": _iso(r.created_at)} for r in rows]
|
||||
|
||||
async def add_graph_blacklist(self, labels: list[str], *, created_by: str | None = None) -> int:
|
||||
"""Add labels to the graph blacklist (trimmed, case-insensitive dedup). Returns the count actually added."""
|
||||
cleaned: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in labels or []:
|
||||
label = (raw or "").strip()
|
||||
if not label or label.lower() in seen:
|
||||
continue
|
||||
seen.add(label.lower())
|
||||
cleaned.append(label)
|
||||
if not cleaned:
|
||||
return 0
|
||||
async with self._sf() as session:
|
||||
existing = {
|
||||
value
|
||||
for (value,) in (
|
||||
await session.execute(select(func.lower(KnowledgeGraphBlacklistRow.label)).where(func.lower(KnowledgeGraphBlacklistRow.label).in_([label.lower() for label in cleaned])))
|
||||
).all()
|
||||
}
|
||||
added = 0
|
||||
for label in cleaned:
|
||||
if label.lower() in existing:
|
||||
continue
|
||||
session.add(KnowledgeGraphBlacklistRow(id=_new_id("kbblk"), label=label, created_by=created_by, created_at=datetime.now(UTC)))
|
||||
added += 1
|
||||
if added:
|
||||
await session.commit()
|
||||
return added
|
||||
|
||||
async def remove_graph_blacklist(self, entry_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(KnowledgeGraphBlacklistRow).where(KnowledgeGraphBlacklistRow.id == entry_id))
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def list_relations(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(select(KnowledgeRelationRow))).scalars()
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"from_note_id": r.from_note_id,
|
||||
"to_note_id": r.to_note_id,
|
||||
"from_entity_id": r.from_entity_id,
|
||||
"to_entity_id": r.to_entity_id,
|
||||
"relation_type": r.relation_type,
|
||||
"weight": r.weight,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
# -- sources ----------------------------------------------------------
|
||||
async def add_sources(self, note_id: str, sources: list[SourceDraft], *, created_by: str | None = None) -> None:
|
||||
if not sources:
|
||||
return
|
||||
import json
|
||||
|
||||
async with self._sf() as session:
|
||||
for src in sources:
|
||||
session.add(
|
||||
KnowledgeSourceRow(
|
||||
id=_new_id("kbsrc"),
|
||||
note_id=note_id,
|
||||
source_type=src.source_type,
|
||||
thread_id=src.thread_id,
|
||||
message_id=src.message_id,
|
||||
tool_name=src.tool_name,
|
||||
title=(src.title or "")[:512] or None,
|
||||
url=src.url,
|
||||
snippet=src.snippet,
|
||||
raw_json=json.dumps(src.raw, ensure_ascii=False) if src.raw else None,
|
||||
created_by=created_by,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
# -- folders ----------------------------------------------------------
|
||||
async def list_folders(self) -> list[dict[str, Any]]:
|
||||
"""Return every managed directory, ordered by sort_order then path."""
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeFolderRow).order_by(KnowledgeFolderRow.sort_order, KnowledgeFolderRow.path)
|
||||
return [{"id": r.id, "path": r.path, "name": r.name, "sort_order": r.sort_order} for r in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def create_folder(self, path: str, *, created_by: str | None = None) -> dict[str, Any]:
|
||||
"""Create a directory by its full slash-joined path (idempotent on path).
|
||||
|
||||
Parent directories are auto-created so ``投研/行业/寿险`` materializes all
|
||||
three levels. Returns the leaf folder row.
|
||||
"""
|
||||
segments = [s.strip() for s in (path or "").split("/") if s.strip()]
|
||||
if not segments:
|
||||
raise ValueError("folder path is empty")
|
||||
async with self._sf() as session:
|
||||
acc: list[str] = []
|
||||
leaf: KnowledgeFolderRow | None = None
|
||||
for i, seg in enumerate(segments):
|
||||
acc.append(seg)
|
||||
full = "/".join(acc)
|
||||
row = await self._folder_by_path(session, full)
|
||||
if row is None:
|
||||
row = KnowledgeFolderRow(id=_new_id("kbdir"), path=full, name=seg, sort_order=i, created_by=created_by, created_at=datetime.now(UTC))
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
leaf = row
|
||||
# Capture before commit: the async session expires attributes on
|
||||
# commit, and re-reading them outside an awaited load would error.
|
||||
result = {"id": leaf.id, "path": leaf.path, "name": leaf.name, "sort_order": leaf.sort_order}
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
async def _folder_by_path(session: AsyncSession, path: str) -> KnowledgeFolderRow | None:
|
||||
return (await session.execute(select(KnowledgeFolderRow).where(KnowledgeFolderRow.path == path))).scalars().first()
|
||||
|
||||
async def rename_folder(self, folder_id: str, new_path: str) -> dict[str, Any] | None:
|
||||
"""Rename/move a folder and re-point every descendant folder + note.
|
||||
|
||||
Both the folder rows under ``old/…`` and notes whose ``folder`` is ``old``
|
||||
or ``old/…`` are rewritten to the new prefix, so the move is atomic.
|
||||
"""
|
||||
new_path = "/".join(s.strip() for s in (new_path or "").split("/") if s.strip())
|
||||
if not new_path:
|
||||
raise ValueError("folder path is empty")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeFolderRow, folder_id)
|
||||
if row is None:
|
||||
return None
|
||||
old = row.path
|
||||
if new_path == old:
|
||||
return {"id": row.id, "path": row.path, "name": row.name, "sort_order": row.sort_order}
|
||||
for f in (await session.execute(select(KnowledgeFolderRow).where(or_(KnowledgeFolderRow.path == old, KnowledgeFolderRow.path.like(f"{old}/%"))))).scalars():
|
||||
f.path = new_path + f.path[len(old):]
|
||||
f.name = f.path.split("/")[-1]
|
||||
for n in (await session.execute(select(KnowledgeNoteRow).where(or_(KnowledgeNoteRow.folder == old, KnowledgeNoteRow.folder.like(f"{old}/%"))))).scalars():
|
||||
n.folder = new_path + (n.folder[len(old):] if n.folder else "")
|
||||
await session.commit()
|
||||
refreshed = await session.get(KnowledgeFolderRow, folder_id)
|
||||
return {"id": refreshed.id, "path": refreshed.path, "name": refreshed.name, "sort_order": refreshed.sort_order}
|
||||
|
||||
async def delete_folder(self, folder_id: str, *, reassign_to: str | None = None) -> bool:
|
||||
"""Delete a folder (and its sub-folders); notes inside move to ``reassign_to`` (or unfiled)."""
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeFolderRow, folder_id)
|
||||
if row is None:
|
||||
return False
|
||||
old = row.path
|
||||
target = (reassign_to or "").strip() or None
|
||||
for n in (await session.execute(select(KnowledgeNoteRow).where(or_(KnowledgeNoteRow.folder == old, KnowledgeNoteRow.folder.like(f"{old}/%"))))).scalars():
|
||||
n.folder = target
|
||||
await session.execute(delete(KnowledgeFolderRow).where(or_(KnowledgeFolderRow.path == old, KnowledgeFolderRow.path.like(f"{old}/%"))))
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
# -- extraction templates ---------------------------------------------
|
||||
@staticmethod
|
||||
def _template_to_dict(row: KnowledgeExtractTemplateRow) -> dict[str, Any]:
|
||||
d = row.to_dict()
|
||||
d["created_at"] = _iso(d.get("created_at"))
|
||||
d["updated_at"] = _iso(d.get("updated_at"))
|
||||
return d
|
||||
|
||||
async def list_templates(self, *, scope: str | None = None, enabled_only: bool = False) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeExtractTemplateRow)
|
||||
if scope:
|
||||
stmt = stmt.where(or_(KnowledgeExtractTemplateRow.scope == scope, KnowledgeExtractTemplateRow.scope == "both"))
|
||||
if enabled_only:
|
||||
stmt = stmt.where(KnowledgeExtractTemplateRow.enabled.is_(True))
|
||||
stmt = stmt.order_by(KnowledgeExtractTemplateRow.is_default.desc(), KnowledgeExtractTemplateRow.created_at)
|
||||
return [self._template_to_dict(r) for r in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_template(self, template_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeExtractTemplateRow, template_id)
|
||||
return self._template_to_dict(row) if row else None
|
||||
|
||||
async def get_default_template(self, scope: str) -> dict[str, Any] | None:
|
||||
"""Return the default enabled template applicable to ``scope`` (thread/document)."""
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeExtractTemplateRow)
|
||||
.where(KnowledgeExtractTemplateRow.enabled.is_(True))
|
||||
.where(KnowledgeExtractTemplateRow.is_default.is_(True))
|
||||
.where(or_(KnowledgeExtractTemplateRow.scope == scope, KnowledgeExtractTemplateRow.scope == "both"))
|
||||
.limit(1)
|
||||
)
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
return self._template_to_dict(row) if row else None
|
||||
|
||||
async def template_name_taken(self, name: str, *, exclude_id: str | None = None) -> bool:
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeExtractTemplateRow.id).where(func.lower(KnowledgeExtractTemplateRow.name) == (name or "").strip().lower())
|
||||
if exclude_id:
|
||||
stmt = stmt.where(KnowledgeExtractTemplateRow.id != exclude_id)
|
||||
return (await session.execute(stmt)).first() is not None
|
||||
|
||||
async def create_template(self, *, name: str, system_prompt: str, scope: str = "both", description: str | None = None, is_default: bool = False, enabled: bool = True, created_by: str | None = None) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
if is_default:
|
||||
await self._clear_default(session, scope)
|
||||
row = KnowledgeExtractTemplateRow(
|
||||
id=_new_id("kbtpl"),
|
||||
name=name.strip()[:256],
|
||||
description=description,
|
||||
scope=scope,
|
||||
system_prompt=system_prompt,
|
||||
is_default=is_default,
|
||||
enabled=enabled,
|
||||
created_by=created_by,
|
||||
updated_by=created_by,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._template_to_dict(row)
|
||||
|
||||
async def update_template(self, template_id: str, *, name: str | None = None, system_prompt: str | None = None, scope: str | None = None, description: str | None = None, is_default: bool | None = None, enabled: bool | None = None, updated_by: str | None = None) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeExtractTemplateRow, template_id)
|
||||
if row is None:
|
||||
return None
|
||||
if name is not None:
|
||||
row.name = name.strip()[:256]
|
||||
if system_prompt is not None:
|
||||
row.system_prompt = system_prompt
|
||||
if scope is not None:
|
||||
row.scope = scope
|
||||
if description is not None:
|
||||
row.description = description
|
||||
if enabled is not None:
|
||||
row.enabled = enabled
|
||||
if is_default:
|
||||
await self._clear_default(session, scope or row.scope)
|
||||
row.is_default = True
|
||||
elif is_default is False:
|
||||
row.is_default = False
|
||||
row.updated_by = updated_by
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._template_to_dict(row)
|
||||
|
||||
async def delete_template(self, template_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(KnowledgeExtractTemplateRow).where(KnowledgeExtractTemplateRow.id == template_id))
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
@staticmethod
|
||||
async def _clear_default(session: AsyncSession, scope: str) -> None:
|
||||
"""Unset is_default on any template that would collide with a new default for ``scope``."""
|
||||
scopes = {scope, "both"} if scope != "both" else {"thread", "document", "both"}
|
||||
for row in (await session.execute(select(KnowledgeExtractTemplateRow).where(KnowledgeExtractTemplateRow.is_default.is_(True)).where(KnowledgeExtractTemplateRow.scope.in_(scopes)))).scalars():
|
||||
row.is_default = False
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Internal domain models for the knowledge module.
|
||||
|
||||
These are plain dataclasses passed between the extractor, the vault writer and
|
||||
the repository. The Gateway router defines its own Pydantic request/response
|
||||
models on top of the dicts the service returns.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
# Allowed note source types (see product-knowledge-base-dev.md §7.1).
|
||||
SOURCE_TYPES = ("thread", "search", "tool", "file", "manual", "report")
|
||||
# Allowed lifecycle states.
|
||||
STATUSES = ("draft", "approved", "archived")
|
||||
# obsidian-wiki wiki-capture categories.
|
||||
CATEGORIES = ("synthesis", "concepts", "references", "decision", "journal")
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceDraft:
|
||||
"""A normalized knowledge source (search result, tool call, thread, …)."""
|
||||
|
||||
source_type: str
|
||||
thread_id: str | None = None
|
||||
message_id: str | None = None
|
||||
tool_name: str | None = None
|
||||
title: str | None = None
|
||||
url: str | None = None
|
||||
snippet: str | None = None
|
||||
raw: dict | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class NoteDraft:
|
||||
"""A knowledge note ready to be persisted + written to the vault.
|
||||
|
||||
Produced by the extractor (from a thread / search / tool result) or built
|
||||
directly for a manual note. ``content_md`` here is the *body* — the vault
|
||||
writer prepends obsidian-wiki frontmatter.
|
||||
"""
|
||||
|
||||
title: str
|
||||
summary: str
|
||||
content_md: str
|
||||
category: str = "synthesis"
|
||||
source_type: str = "manual"
|
||||
source_id: str | None = None
|
||||
status: str = "approved"
|
||||
confidence: float = 0.0
|
||||
# User-assigned directory (slash-joined path); None = unfiled.
|
||||
folder: str | None = None
|
||||
tags: list[str] = field(default_factory=list)
|
||||
key_points: list[str] = field(default_factory=list)
|
||||
actions: list[str] = field(default_factory=list)
|
||||
related: list[str] = field(default_factory=list)
|
||||
sources: list[SourceDraft] = field(default_factory=list)
|
||||
# Phase 4: extracted graph signal. entities=[{name,type}],
|
||||
# relations=[{from,to,type}] (names refer to entities or this note's title).
|
||||
entities: list[dict] = field(default_factory=list)
|
||||
relations: list[dict] = field(default_factory=list)
|
||||
# Stable identity hint used for manifest dedup (e.g. ``thread:thread_id``).
|
||||
source_key: str | None = None
|
||||
|
||||
|
||||
def normalize_status(status: str | None) -> str:
|
||||
"""Coerce an arbitrary status string to a valid lifecycle state."""
|
||||
if status in STATUSES:
|
||||
return status
|
||||
return "approved"
|
||||
|
||||
|
||||
def normalize_source_type(source_type: str | None) -> str:
|
||||
"""Coerce an arbitrary source-type string to a valid value."""
|
||||
if source_type in SOURCE_TYPES:
|
||||
return source_type
|
||||
return "manual"
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Knowledge search: keyword / vector / hybrid (phase 1 + 2).
|
||||
|
||||
- ``keyword`` — title>tags>summary>body scoring over the DB mirror (always works).
|
||||
- ``vector`` — cosine over per-chunk embeddings (requires a configured
|
||||
embedding endpoint; otherwise transparently falls back to ``keyword``).
|
||||
- ``hybrid`` — min-max-normalized blend of keyword + vector.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from deerflow.knowledge.embeddings import cosine_similarity
|
||||
from deerflow.knowledge.repository import KnowledgeRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.knowledge.embeddings import EmbeddingClient
|
||||
|
||||
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
|
||||
|
||||
|
||||
def _tokens(text: str) -> list[str]:
|
||||
return [t.lower() for t in _TOKEN_RE.findall(text or "")]
|
||||
|
||||
|
||||
def _snippet(content: str, terms: list[str], *, width: int = 160) -> str:
|
||||
body = re.sub(r"\s+", " ", content or "").strip()
|
||||
low = body.lower()
|
||||
for term in terms:
|
||||
idx = low.find(term)
|
||||
if idx >= 0:
|
||||
start = max(0, idx - width // 3)
|
||||
end = min(len(body), start + width)
|
||||
prefix = "…" if start > 0 else ""
|
||||
suffix = "…" if end < len(body) else ""
|
||||
return f"{prefix}{body[start:end]}{suffix}"
|
||||
return body[:width] + ("…" if len(body) > width else "")
|
||||
|
||||
|
||||
async def _keyword_scores(repo: KnowledgeRepository, query: str, *, candidate_cap: int) -> dict[str, tuple[float, dict[str, Any]]]:
|
||||
terms = _tokens(query)
|
||||
if not terms:
|
||||
return {}
|
||||
candidates, _total = await repo.list_notes(limit=candidate_cap, offset=0)
|
||||
scored: dict[str, tuple[float, dict[str, Any]]] = {}
|
||||
for note in candidates:
|
||||
title = (note.get("title") or "").lower()
|
||||
summary = (note.get("summary") or "").lower()
|
||||
content = (note.get("content_md") or "").lower()
|
||||
tags = " ".join(note.get("tags") or []).lower()
|
||||
score = 0.0
|
||||
for term in terms:
|
||||
if term in title:
|
||||
score += 3.0
|
||||
if term in tags:
|
||||
score += 2.0
|
||||
if term in summary:
|
||||
score += 2.0
|
||||
score += min(content.count(term), 5) * 0.5
|
||||
if score > 0:
|
||||
scored[note["id"]] = (score, note)
|
||||
return scored
|
||||
|
||||
|
||||
async def _vector_scores(
|
||||
repo: KnowledgeRepository,
|
||||
query: str,
|
||||
embedding_client: EmbeddingClient,
|
||||
) -> dict[str, tuple[float, dict[str, Any]]]:
|
||||
qvec = await embedding_client.embed_query(query)
|
||||
if not qvec:
|
||||
return {}
|
||||
embeddings = await repo.all_embeddings()
|
||||
best: dict[str, tuple[float, str, str]] = {} # note_id -> (score, title, snippet)
|
||||
for emb in embeddings:
|
||||
sim = cosine_similarity(qvec, emb.get("vector") or [])
|
||||
if sim <= 0:
|
||||
continue
|
||||
nid = emb["note_id"]
|
||||
if nid not in best or sim > best[nid][0]:
|
||||
best[nid] = (sim, emb.get("title") or "", emb.get("content") or "")
|
||||
return {nid: (score, {"id": nid, "title": title, "_chunk": chunk}) for nid, (score, title, chunk) in best.items()}
|
||||
|
||||
|
||||
def _normalize(scores: dict[str, float]) -> dict[str, float]:
|
||||
if not scores:
|
||||
return {}
|
||||
mx = max(scores.values())
|
||||
if mx <= 0:
|
||||
return {k: 0.0 for k in scores}
|
||||
return {k: v / mx for k, v in scores.items()}
|
||||
|
||||
|
||||
async def search_notes(
|
||||
repo: KnowledgeRepository,
|
||||
query: str,
|
||||
*,
|
||||
mode: str = "keyword",
|
||||
limit: int = 8,
|
||||
candidate_cap: int = 500,
|
||||
embedding_client: EmbeddingClient | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Score notes against ``query`` and return the top ``limit`` matches."""
|
||||
terms = _tokens(query)
|
||||
if not terms:
|
||||
return []
|
||||
|
||||
want_vector = mode in ("vector", "hybrid") and embedding_client is not None and embedding_client.is_enabled()
|
||||
keyword = await _keyword_scores(repo, query, candidate_cap=candidate_cap) if mode != "vector" or not want_vector else {}
|
||||
vector = await _vector_scores(repo, query, embedding_client) if want_vector else {}
|
||||
|
||||
# Pure vector (when available); else keyword; hybrid blends both.
|
||||
if mode == "vector" and want_vector:
|
||||
combined = {nid: (sc, meta) for nid, (sc, meta) in vector.items()}
|
||||
ranked = sorted(combined.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
|
||||
return await _materialize(repo, ranked, terms)
|
||||
|
||||
if mode == "hybrid" and want_vector:
|
||||
kw_norm = _normalize({k: v[0] for k, v in keyword.items()})
|
||||
vec_norm = _normalize({k: v[0] for k, v in vector.items()})
|
||||
ids = set(kw_norm) | set(vec_norm)
|
||||
blended: dict[str, tuple[float, dict[str, Any]]] = {}
|
||||
for nid in ids:
|
||||
score = 0.5 * kw_norm.get(nid, 0.0) + 0.5 * vec_norm.get(nid, 0.0)
|
||||
meta = keyword.get(nid, (0, None))[1] or vector.get(nid, (0, {}))[1]
|
||||
blended[nid] = (score, meta)
|
||||
ranked = sorted(blended.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
|
||||
return await _materialize(repo, ranked, terms)
|
||||
|
||||
# keyword (default, and fallback when vector unavailable)
|
||||
ranked = sorted(keyword.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
|
||||
if not ranked and vector: # mode=vector requested but keyword empty: use vector
|
||||
ranked = sorted(vector.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
|
||||
return await _materialize(repo, ranked, terms)
|
||||
|
||||
|
||||
async def _materialize(repo: KnowledgeRepository, ranked: list[tuple[str, tuple[float, dict[str, Any]]]], terms: list[str]) -> list[dict[str, Any]]:
|
||||
max_score = ranked[0][1][0] if ranked else 1.0
|
||||
results: list[dict[str, Any]] = []
|
||||
for nid, (score, meta) in ranked:
|
||||
note = meta if (meta and meta.get("content_md")) else await repo.get_note(nid, with_sources=False)
|
||||
if note is None:
|
||||
continue
|
||||
snippet_src = note.get("content_md") or meta.get("_chunk") or note.get("summary") or ""
|
||||
results.append(
|
||||
{
|
||||
"note_id": nid,
|
||||
"chunk_id": None,
|
||||
"title": note.get("title") or meta.get("title"),
|
||||
"summary": note.get("summary"),
|
||||
"snippet": _snippet(snippet_src, terms),
|
||||
"score": round(score / max_score, 4) if max_score else 0.0,
|
||||
"tags": note.get("tags") or [],
|
||||
"source_type": note.get("source_type"),
|
||||
"updated_at": note.get("updated_at"),
|
||||
"sources": [],
|
||||
}
|
||||
)
|
||||
return results
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Sensitive-information redaction for knowledge content (phase 3).
|
||||
|
||||
Best-effort masking of secrets / PII before a note is persisted: API keys,
|
||||
bearer tokens, generic ``key=secret`` assignments, Chinese mobile numbers,
|
||||
ID-card numbers and email addresses. Conservative by design — it favours not
|
||||
mangling legitimate prose over catching every possible secret.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_MASK = "[REDACTED]"
|
||||
|
||||
# Order matters: more specific patterns first.
|
||||
_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
("openai_key", re.compile(r"\bsk-[A-Za-z0-9]{20,}\b")),
|
||||
("aws_key", re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
|
||||
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b")),
|
||||
("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]{16,}")),
|
||||
# key/secret/password/token assignment: keep the field name, mask the value.
|
||||
("assignment", re.compile(r"(?i)\b(api[_-]?key|secret|password|passwd|token|access[_-]?token)\b\s*[:=]\s*[\"']?[^\s\"']{6,}[\"']?")),
|
||||
("cn_id_card", re.compile(r"\b\d{17}[\dXx]\b")),
|
||||
("cn_mobile", re.compile(r"(?<!\d)1[3-9]\d{9}(?!\d)")),
|
||||
("email", re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")),
|
||||
]
|
||||
|
||||
|
||||
def redact(text: str | None) -> tuple[str, list[str]]:
|
||||
"""Return ``(redacted_text, hit_kinds)``.
|
||||
|
||||
``hit_kinds`` lists which detectors fired (for logging/audit). When ``text``
|
||||
is falsy it is returned unchanged with no hits.
|
||||
"""
|
||||
if not text:
|
||||
return text or "", []
|
||||
hits: list[str] = []
|
||||
out = text
|
||||
for kind, pattern in _PATTERNS:
|
||||
def _sub(m: re.Match[str], _kind: str = kind) -> str:
|
||||
# For assignments keep the field name + delimiter, mask only the value.
|
||||
if _kind == "assignment":
|
||||
head = re.split(r"[:=]", m.group(0), maxsplit=1)[0]
|
||||
sep = ":" if ":" in m.group(0) else "="
|
||||
return f"{head}{sep} {_MASK}"
|
||||
if _kind == "bearer":
|
||||
return f"Bearer {_MASK}"
|
||||
return _MASK
|
||||
|
||||
new_out, n = pattern.subn(_sub, out)
|
||||
if n:
|
||||
hits.append(kind)
|
||||
out = new_out
|
||||
return out, hits
|
||||
@@ -0,0 +1,936 @@
|
||||
"""KnowledgeService — high-level orchestration for the knowledge base.
|
||||
|
||||
Ties together the repository (DB mirror), the :class:`ObsidianWikiAdapter`
|
||||
(Markdown vault + index/log/hot/manifest + graph export) and the extractor
|
||||
(thread/search → :class:`NoteDraft`). The Gateway router only talks to this
|
||||
service; everything obsidian-wiki specific stays inside the ``knowledge``
|
||||
module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.knowledge_config import KnowledgeConfig
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.knowledge import extractor, llm_extractor
|
||||
from deerflow.knowledge.chunking import chunk_markdown
|
||||
from deerflow.knowledge.embeddings import EmbeddingClient
|
||||
from deerflow.knowledge.markdown_vault import MarkdownVault, VaultPathError, extract_wikilinks, normalize_wikilink_target
|
||||
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
|
||||
from deerflow.knowledge.repository import KnowledgeRepository
|
||||
from deerflow.knowledge.schemas import NoteDraft, SourceDraft, normalize_source_type, normalize_status
|
||||
from deerflow.knowledge.search import search_notes
|
||||
from deerflow.knowledge.sources import from_reference_batches, from_search_results
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EmptyThreadError(ValueError):
|
||||
"""Raised when a thread has no sediment-able content."""
|
||||
|
||||
|
||||
def _resolve_vault_dir(vault_dir: str) -> Path:
|
||||
path = Path(vault_dir)
|
||||
if path.is_absolute():
|
||||
return path.resolve()
|
||||
return (runtime_home() / vault_dir).resolve()
|
||||
|
||||
|
||||
def _inject_related_notes(body: str, links: list[str]) -> str:
|
||||
"""Insert real related-note ``[[wikilinks]]`` into the Related block.
|
||||
|
||||
Notes carry only content-specific related notes now (the structural meta-links
|
||||
were dropped), so this is the sole writer of ``## Related`` links. When the
|
||||
body has no ``## Related`` section yet, one is appended.
|
||||
"""
|
||||
if not links:
|
||||
return body
|
||||
bullets = "\n".join(f"- {link}" for link in links)
|
||||
marker = "## Related\n"
|
||||
idx = body.find(marker)
|
||||
if idx == -1:
|
||||
return body.rstrip() + "\n\n## Related\n\n" + bullets + "\n"
|
||||
insert_at = idx + len(marker)
|
||||
return body[:insert_at] + "\n" + bullets + "\n" + body[insert_at:]
|
||||
|
||||
|
||||
class KnowledgeService:
|
||||
"""Application-facing knowledge base API."""
|
||||
|
||||
def __init__(self, repo: KnowledgeRepository, config: KnowledgeConfig) -> None:
|
||||
self.repo = repo
|
||||
self.config = config
|
||||
self.project_name = config.project_name
|
||||
vault_root = _resolve_vault_dir(config.vault_dir)
|
||||
self.adapter = ObsidianWikiAdapter(
|
||||
vault_root,
|
||||
project_name=config.project_name,
|
||||
export_html=config.export.graph_html_enabled,
|
||||
export_json=config.export.graph_json_enabled,
|
||||
)
|
||||
self.embedding_client = EmbeddingClient(config.embedding)
|
||||
try:
|
||||
self.adapter.ensure_ready()
|
||||
except Exception: # pragma: no cover - vault init best-effort
|
||||
logger.exception("Failed to initialize knowledge vault at %s", vault_root)
|
||||
|
||||
# -- internal helpers -------------------------------------------------
|
||||
def _redact_draft(self, draft: NoteDraft) -> NoteDraft:
|
||||
"""Mask secrets/PII in a draft before persistence (phase 3)."""
|
||||
if not self.config.sensitive_redaction_enabled:
|
||||
return draft
|
||||
from deerflow.knowledge.sensitive import redact
|
||||
|
||||
draft.content_md, body_hits = redact(draft.content_md)
|
||||
draft.summary, _ = redact(draft.summary)
|
||||
for src in draft.sources:
|
||||
if src.snippet:
|
||||
src.snippet, _ = redact(src.snippet)
|
||||
if body_hits:
|
||||
logger.info("Knowledge redaction applied to note '%s': %s", draft.title, ",".join(sorted(set(body_hits))))
|
||||
return draft
|
||||
|
||||
async def _find_related_notes(self, draft: NoteDraft) -> list[dict[str, Any]]:
|
||||
"""Find existing notes to cross-link under ## Related — by **shared tags**.
|
||||
|
||||
Matching is strictly tag-based: a candidate must share at least one of the
|
||||
draft's *distinctive* tags (the per-note project / ``knowledge-base``
|
||||
defaults are dropped, else everything matches everything). Free-text
|
||||
title/summary matching was removed because CJK tokenization made generic
|
||||
words (助手 / 知识图谱 / 页面) link unrelated notes. No distinctive tags →
|
||||
no related notes (better empty than noisy). Best-effort.
|
||||
"""
|
||||
if not self.config.related_notes_enabled:
|
||||
return []
|
||||
default_tags = {t.lower() for t in (self.project_name or "", "knowledge-base", "zncm") if t}
|
||||
distinctive = [t for t in (draft.tags or []) if t and t.lower() not in default_tags]
|
||||
if not distinctive:
|
||||
return []
|
||||
own_title = (draft.title or "").strip().lower()
|
||||
# Rank candidates by how many distinctive tags they share with the draft.
|
||||
shared: dict[str, dict[str, Any]] = {}
|
||||
for tag in distinctive:
|
||||
try:
|
||||
items, _total = await self.repo.list_notes(tag=tag, limit=50, offset=0)
|
||||
except Exception:
|
||||
logger.exception("Related-notes tag lookup failed for tag '%s'", tag)
|
||||
continue
|
||||
for it in items:
|
||||
nid = it.get("id")
|
||||
title = (it.get("title") or "").strip()
|
||||
if not nid or not title or title.lower() == own_title:
|
||||
continue
|
||||
entry = shared.setdefault(nid, {"id": nid, "title": title, "count": 0})
|
||||
entry["count"] += 1
|
||||
ranked = sorted(shared.values(), key=lambda e: e["count"], reverse=True)
|
||||
limit = max(1, self.config.related_notes_limit)
|
||||
return [{"id": e["id"], "title": e["title"]} for e in ranked[:limit]]
|
||||
|
||||
async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]:
|
||||
"""Create a DB row, write the vault file, attach sources, return note."""
|
||||
draft = self._redact_draft(draft)
|
||||
related_notes = await self._find_related_notes(draft)
|
||||
if related_notes:
|
||||
real_links = [f"[[{n['title']}]]" for n in related_notes]
|
||||
draft.content_md = _inject_related_notes(draft.content_md, real_links)
|
||||
draft.related = list(dict.fromkeys([*real_links, *draft.related]))
|
||||
note = await self.repo.create_note(
|
||||
title=draft.title,
|
||||
summary=draft.summary,
|
||||
content_md=draft.content_md,
|
||||
source_type=draft.source_type,
|
||||
source_id=draft.source_id,
|
||||
status=draft.status,
|
||||
confidence=draft.confidence,
|
||||
folder=draft.folder,
|
||||
tags=draft.tags,
|
||||
created_by=created_by,
|
||||
)
|
||||
note_id = note["id"]
|
||||
try:
|
||||
rel = self.adapter.write_note(note_id, draft, created=note["created_at"], updated=note["updated_at"], status=draft.status)
|
||||
await self.repo.set_vault_path(note_id, rel)
|
||||
note["vault_path"] = rel
|
||||
except Exception:
|
||||
logger.exception("Failed to write knowledge note %s to vault", note_id)
|
||||
if draft.sources:
|
||||
await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
|
||||
await self._reindex_embeddings(note_id, draft.content_md)
|
||||
await self._store_entities(note_id, draft)
|
||||
# Record note→note "related" edges (after _store_entities, which clears
|
||||
# this note's relations) so the graph reflects the cross-links too.
|
||||
for rel_note in related_notes:
|
||||
try:
|
||||
await self.repo.add_relation(from_note_id=note_id, to_note_id=rel_note["id"], relation_type="related")
|
||||
except Exception:
|
||||
logger.exception("Failed to link related note %s -> %s", note_id, rel_note["id"])
|
||||
return await self.repo.get_note(note_id) or note
|
||||
|
||||
async def _store_entities(self, note_id: str, draft: NoteDraft) -> None:
|
||||
"""Persist extracted entities/relations and link them to the note (phase 4)."""
|
||||
if not self.config.entity_extraction_enabled or not draft.entities:
|
||||
return
|
||||
try:
|
||||
await self.repo.clear_note_relations(note_id)
|
||||
name_to_id: dict[str, str] = {}
|
||||
for ent in draft.entities:
|
||||
name = str(ent.get("name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
row = await self.repo.get_or_create_entity(name, ent.get("type"))
|
||||
name_to_id[name.lower()] = row["id"]
|
||||
# note -> entity "mentions"
|
||||
await self.repo.add_relation(from_note_id=note_id, to_entity_id=row["id"], relation_type="mentions")
|
||||
for rel in draft.relations:
|
||||
a = name_to_id.get(str(rel.get("from") or "").strip().lower())
|
||||
b = name_to_id.get(str(rel.get("to") or "").strip().lower())
|
||||
if a and b and a != b:
|
||||
await self.repo.add_relation(from_entity_id=a, to_entity_id=b, relation_type=str(rel.get("type") or "relates_to"))
|
||||
except Exception:
|
||||
logger.exception("Failed to store entities/relations for note %s", note_id)
|
||||
|
||||
async def _reindex_embeddings(self, note_id: str, content_md: str) -> None:
|
||||
"""Best-effort: (re)build the per-chunk embedding index for a note."""
|
||||
if not self.embedding_client.is_enabled():
|
||||
return
|
||||
try:
|
||||
chunks = chunk_markdown(content_md)
|
||||
if not chunks:
|
||||
await self.repo.delete_embeddings(note_id)
|
||||
return
|
||||
vectors = await self.embedding_client.embed_texts(chunks)
|
||||
if not vectors or len(vectors) != len(chunks):
|
||||
return
|
||||
await self.repo.replace_embeddings(note_id, [(i, c, v) for i, (c, v) in enumerate(zip(chunks, vectors, strict=False))])
|
||||
except Exception:
|
||||
logger.exception("Failed to (re)index embeddings for note %s", note_id)
|
||||
|
||||
# -- capture ----------------------------------------------------------
|
||||
async def capture_thread(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
messages: list[dict[str, Any]],
|
||||
thread_title: str | None = None,
|
||||
mode: str = "summary",
|
||||
title: str | None = None,
|
||||
status: str = "approved",
|
||||
tags: list[str] | None = None,
|
||||
include_sources: bool = True,
|
||||
reference_batches: list[dict[str, Any]] | None = None,
|
||||
created_by: str | None = None,
|
||||
skip_existing: bool = False,
|
||||
model_name: str | None = None,
|
||||
template_id: str | None = None,
|
||||
folder: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Sediment a thread into a knowledge note.
|
||||
|
||||
``model_name`` optionally overrides the configured extraction model for
|
||||
this one capture (falls back to ``config.extract_model_name``).
|
||||
``template_id`` selects an operator-edited extraction prompt template
|
||||
(falls back to the ``thread`` default, then the built-in prompt).
|
||||
``folder`` files the resulting note(s) under a user directory.
|
||||
|
||||
Returns ``{"note": <note>, "created": bool}``. Raises
|
||||
:class:`EmptyThreadError` when there is nothing worth keeping.
|
||||
"""
|
||||
extract_prompt = await self._resolve_extract_prompt(template_id, "thread")
|
||||
# Always-on entry log: proves a capture was triggered for this thread,
|
||||
# regardless of how it ends (success / skip / fallback). Lets you tell
|
||||
# "capture never ran" apart from "logs not visible" via one grep.
|
||||
logger.info(
|
||||
"[knowledge] capturing thread %s (msgs=%d, llm_extract=%s, skip_existing=%s)",
|
||||
thread_id,
|
||||
len(messages or []),
|
||||
self.config.llm_extract_enabled,
|
||||
skip_existing,
|
||||
)
|
||||
if skip_existing:
|
||||
existing = await self.repo.find_by_source("thread", thread_id)
|
||||
if existing is None and self.adapter.manifest.has_source(f"thread:{thread_id}"):
|
||||
existing = await self.repo.find_by_source("thread", thread_id)
|
||||
if existing:
|
||||
logger.info("[knowledge] thread %s already captured (note %s); skipping", thread_id, existing.get("id"))
|
||||
return {"note": existing, "created": False}
|
||||
|
||||
sources = []
|
||||
if include_sources and reference_batches:
|
||||
sources = from_reference_batches(reference_batches, thread_id=thread_id, max_items=self.config.max_source_items)
|
||||
|
||||
drafts: list[NoteDraft] = []
|
||||
# Prefer LLM distillation (wiki-capture style) so notes are declarative
|
||||
# knowledge, not raw chat. A single conversation may split into several
|
||||
# cross-linked notes. Falls back to the rule-based extractor (always one
|
||||
# note) when the LLM is disabled/unavailable or returns nothing usable.
|
||||
llm_attempted = self.config.llm_extract_enabled
|
||||
llm_fallback = False
|
||||
if llm_attempted:
|
||||
try:
|
||||
drafts = await llm_extractor.extract_thread_llm_multi(
|
||||
messages,
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
mode=mode,
|
||||
title=title,
|
||||
tags=tags,
|
||||
project_name=self.project_name,
|
||||
sources=sources,
|
||||
max_messages=self.config.max_thread_messages,
|
||||
model_name=model_name or self.config.extract_model_name,
|
||||
max_input_chars=self.config.extract_max_input_chars,
|
||||
allow_multi=self.config.multi_note_enabled,
|
||||
system_prompt=extract_prompt,
|
||||
)
|
||||
except llm_extractor.LLMRejectedIngest as exc:
|
||||
# The model judged this conversation worthless (greeting / meta
|
||||
# chatter). Skip ingestion entirely — do NOT dump the raw chat.
|
||||
raise EmptyThreadError(f"Thread {thread_id} judged not worth ingesting: {exc}") from exc
|
||||
if not drafts:
|
||||
# The LLM path was supposed to run but produced nothing usable
|
||||
# (API error / non-JSON). We will fall back to the rule-based
|
||||
# extractor, which yields a low-quality verbatim dump with no
|
||||
# entities/tags — flag it so it lands as a draft.
|
||||
llm_fallback = True
|
||||
if not drafts:
|
||||
single = extractor.extract_thread(
|
||||
messages,
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
mode=mode,
|
||||
title=title,
|
||||
tags=tags,
|
||||
project_name=self.project_name,
|
||||
sources=sources,
|
||||
max_messages=self.config.max_thread_messages,
|
||||
)
|
||||
drafts = [single] if single else []
|
||||
if not drafts:
|
||||
logger.info("[knowledge] thread %s has no sediment-able content; skipping", thread_id)
|
||||
raise EmptyThreadError(f"Thread {thread_id} has no sediment-able content")
|
||||
|
||||
# When the LLM was enabled but silently fell back to rule-based extraction,
|
||||
# the note is a verbatim dump (no distillation / entities / tags). Land it
|
||||
# as a *draft* for review instead of silently approving it, and log loudly
|
||||
# so the extraction failure is diagnosable (see also the WARNING/EXCEPTION
|
||||
# logged inside llm_extractor.extract_thread_llm_multi).
|
||||
effective_status = status
|
||||
if llm_fallback:
|
||||
logger.warning(
|
||||
"[knowledge] thread %s — capture fell back to rule-based extraction "
|
||||
"(LLM distillation produced nothing usable); landing as DRAFT. "
|
||||
"Check extraction model '%s' / earlier [knowledge] logs for the cause.",
|
||||
thread_id,
|
||||
self.config.extract_model_name or "models[0]",
|
||||
)
|
||||
if normalize_status(status) == "approved":
|
||||
effective_status = "draft"
|
||||
|
||||
normalized = normalize_status(effective_status)
|
||||
for draft in drafts:
|
||||
draft.status = normalized
|
||||
if folder:
|
||||
draft.folder = folder
|
||||
# Cross-link sibling notes from the same conversation by title, so the
|
||||
# split pages reference each other (resolved once all are persisted).
|
||||
if len(drafts) > 1:
|
||||
for i, draft in enumerate(drafts):
|
||||
siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
|
||||
if siblings:
|
||||
draft.content_md = _inject_related_notes(draft.content_md, siblings)
|
||||
draft.related = list(dict.fromkeys([*siblings, *draft.related]))
|
||||
|
||||
notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
|
||||
logger.info(
|
||||
"[knowledge] thread %s captured → %d note(s) [%s], status=%s, llm_fallback=%s",
|
||||
thread_id,
|
||||
len(notes),
|
||||
", ".join(n.get("title", "") for n in notes),
|
||||
normalized,
|
||||
llm_fallback,
|
||||
)
|
||||
return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback}
|
||||
|
||||
async def capture_search(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
results: list[dict[str, Any]] | None,
|
||||
thread_id: str | None = None,
|
||||
message_id: str | None = None,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
status: str = "approved",
|
||||
created_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Sediment search/tool results into a reference note."""
|
||||
sources = from_search_results(query, results, thread_id=thread_id, message_id=message_id, max_items=self.config.max_source_items)
|
||||
draft = extractor.extract_search(query, sources, thread_id=thread_id, project_name=self.project_name, title=title, tags=tags)
|
||||
draft.status = normalize_status(status)
|
||||
return await self._persist_draft(draft, created_by=created_by)
|
||||
|
||||
async def create_manual_note(
|
||||
self,
|
||||
*,
|
||||
title: str,
|
||||
content_md: str,
|
||||
tags: list[str] | None = None,
|
||||
status: str = "approved",
|
||||
summary: str | None = None,
|
||||
folder: str | None = None,
|
||||
created_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a note directly from product-edited Markdown."""
|
||||
# Strip any frontmatter the user may have pasted; vault writer adds its own.
|
||||
_, body = MarkdownVault.parse(content_md)
|
||||
body = body or content_md
|
||||
base_tags = list(tags or [])
|
||||
for default in (self.project_name, "knowledge-base"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
draft = NoteDraft(
|
||||
title=title.strip() or "知识笔记",
|
||||
summary=(summary or body.strip().splitlines()[0] if body.strip() else "")[:280],
|
||||
content_md=body,
|
||||
category="synthesis",
|
||||
source_type="manual",
|
||||
source_id=None,
|
||||
status=normalize_status(status),
|
||||
folder=(folder or None),
|
||||
tags=base_tags,
|
||||
confidence=0.5,
|
||||
related=extractor.related_links(self.project_name),
|
||||
)
|
||||
return await self._persist_draft(draft, created_by=created_by)
|
||||
|
||||
def _verbatim_file_draft(self, body: str, filename: str, *, title: str | None, tags: list[str] | None, doc_id: str) -> NoteDraft:
|
||||
"""Build a verbatim fallback note from a file when LLM distillation fails.
|
||||
|
||||
Lands as a ``draft`` (like the thread fallback) since it is the raw
|
||||
document, not distilled knowledge.
|
||||
"""
|
||||
base_tags = list(tags or [])
|
||||
for default in (self.project_name, "knowledge-base"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
name = (title or Path(filename).stem or "文档知识").strip() or "文档知识"
|
||||
summary = next((ln.strip() for ln in body.splitlines() if ln.strip()), name)[:280]
|
||||
return NoteDraft(
|
||||
title=name,
|
||||
summary=summary,
|
||||
content_md=body,
|
||||
category="references",
|
||||
source_type="file",
|
||||
source_id=doc_id,
|
||||
status="draft",
|
||||
tags=base_tags,
|
||||
confidence=0.4,
|
||||
related=extractor.related_links(self.project_name),
|
||||
source_key=f"file:{doc_id}",
|
||||
)
|
||||
|
||||
async def capture_file(
|
||||
self,
|
||||
*,
|
||||
content_md: str,
|
||||
filename: str,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
status: str = "approved",
|
||||
created_by: str | None = None,
|
||||
model_name: str | None = None,
|
||||
skip_existing: bool = False,
|
||||
template_id: str | None = None,
|
||||
folder: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Import a file (already converted to Markdown/text) into knowledge notes.
|
||||
|
||||
Runs the same wiki-capture LLM distillation as thread capture, so an
|
||||
uploaded document becomes one or more declarative wiki notes in the vault.
|
||||
Falls back to a verbatim ``draft`` note when the LLM is disabled or returns
|
||||
nothing usable. Returns ``{"note", "created", "notes", "llm_fallback"}``;
|
||||
raises :class:`EmptyThreadError` when there is nothing to keep.
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
text = (content_md or "").strip()
|
||||
if not text:
|
||||
raise EmptyThreadError(f"File {filename} is empty")
|
||||
# Strip any frontmatter the document carries; the vault writer adds its own.
|
||||
_, parsed = MarkdownVault.parse(text)
|
||||
body = (parsed or text).strip()
|
||||
if not body:
|
||||
raise EmptyThreadError(f"File {filename} has no body content")
|
||||
# Stable id from name + size so re-importing the same file can be skipped.
|
||||
doc_id = hashlib.sha1(f"{filename}:{len(body)}".encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
logger.info(
|
||||
"[knowledge] importing file '%s' (chars=%d, doc_id=%s, llm_extract=%s, skip_existing=%s)",
|
||||
filename,
|
||||
len(body),
|
||||
doc_id,
|
||||
self.config.llm_extract_enabled,
|
||||
skip_existing,
|
||||
)
|
||||
|
||||
if skip_existing:
|
||||
existing = await self.repo.find_by_source("file", doc_id)
|
||||
if existing:
|
||||
logger.info("[knowledge] file '%s' already imported (note %s); skipping", filename, existing.get("id"))
|
||||
return {"note": existing, "created": False, "notes": [existing], "llm_fallback": False}
|
||||
|
||||
extract_prompt = await self._resolve_extract_prompt(template_id, "document")
|
||||
drafts: list[NoteDraft] = []
|
||||
llm_fallback = False
|
||||
if self.config.llm_extract_enabled:
|
||||
try:
|
||||
drafts = await llm_extractor.extract_document_llm_multi(
|
||||
body,
|
||||
doc_id=doc_id,
|
||||
doc_title=title or Path(filename).stem,
|
||||
title=title,
|
||||
tags=tags,
|
||||
project_name=self.project_name,
|
||||
model_name=model_name or self.config.extract_model_name,
|
||||
max_input_chars=self.config.extract_max_input_chars,
|
||||
allow_multi=self.config.multi_note_enabled,
|
||||
system_prompt=extract_prompt,
|
||||
)
|
||||
except llm_extractor.LLMRejectedIngest as exc:
|
||||
raise EmptyThreadError(f"File {filename} judged not worth ingesting: {exc}") from exc
|
||||
if not drafts:
|
||||
llm_fallback = True
|
||||
if not drafts:
|
||||
drafts = [self._verbatim_file_draft(body, filename, title=title, tags=tags, doc_id=doc_id)]
|
||||
|
||||
effective_status = status
|
||||
if llm_fallback and normalize_status(status) == "approved":
|
||||
effective_status = "draft"
|
||||
normalized = normalize_status(effective_status)
|
||||
for draft in drafts:
|
||||
draft.status = normalized
|
||||
if folder:
|
||||
draft.folder = folder
|
||||
|
||||
# Cross-link sibling notes split from the same document (resolved by title
|
||||
# once all are persisted), mirroring multi-note thread capture.
|
||||
if len(drafts) > 1:
|
||||
for i, draft in enumerate(drafts):
|
||||
siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
|
||||
if siblings:
|
||||
draft.content_md = _inject_related_notes(draft.content_md, siblings)
|
||||
draft.related = list(dict.fromkeys([*siblings, *draft.related]))
|
||||
|
||||
notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
|
||||
logger.info(
|
||||
"[knowledge] file '%s' imported → %d note(s) [%s], status=%s, llm_fallback=%s",
|
||||
filename,
|
||||
len(notes),
|
||||
", ".join(n.get("title", "") for n in notes),
|
||||
normalized,
|
||||
llm_fallback,
|
||||
)
|
||||
return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback}
|
||||
|
||||
# -- CRUD -------------------------------------------------------------
|
||||
async def list_notes(
|
||||
self,
|
||||
*,
|
||||
keyword: str | None = None,
|
||||
tag: str | None = None,
|
||||
entity: str | None = None,
|
||||
source_type: str | None = None,
|
||||
status: str | None = None,
|
||||
folder: str | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
items, total = await self.repo.list_notes(
|
||||
keyword=keyword,
|
||||
tag=tag,
|
||||
entity=entity,
|
||||
source_type=normalize_source_type(source_type) if source_type else None,
|
||||
status=status,
|
||||
folder=folder,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
return {"items": items, "total": total}
|
||||
|
||||
async def get_note(self, note_id: str) -> dict[str, Any] | None:
|
||||
note = await self.repo.get_note(note_id)
|
||||
if note is not None:
|
||||
note["entities"] = await self.repo.list_note_entities(note_id)
|
||||
return note
|
||||
|
||||
async def update_note(
|
||||
self,
|
||||
note_id: str,
|
||||
*,
|
||||
title: str | None = None,
|
||||
content_md: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
status: str | None = None,
|
||||
summary: str | None = None,
|
||||
folder: str | None = None,
|
||||
set_folder: bool = False,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
current = await self.repo.get_note(note_id, with_sources=False)
|
||||
if current is None:
|
||||
return None
|
||||
# Snapshot the pre-edit state for version history (phase 3).
|
||||
try:
|
||||
await self.repo.add_version(current, edited_by=updated_by)
|
||||
except Exception:
|
||||
logger.exception("Failed to snapshot version for note %s", note_id)
|
||||
body = content_md
|
||||
if body is not None:
|
||||
_, parsed = MarkdownVault.parse(body)
|
||||
body = parsed or body
|
||||
note = await self.repo.update_note(
|
||||
note_id,
|
||||
title=title,
|
||||
summary=summary,
|
||||
content_md=body,
|
||||
status=normalize_status(status) if status else None,
|
||||
tags=tags,
|
||||
folder=folder,
|
||||
set_folder=set_folder,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
if note is None:
|
||||
return None
|
||||
try:
|
||||
rel = self.adapter.update_note_file(
|
||||
note_id,
|
||||
current.get("vault_path"),
|
||||
title=note["title"],
|
||||
content_md=note["content_md"],
|
||||
tags=note.get("tags") or [],
|
||||
status=note["status"],
|
||||
summary=note.get("summary") or "",
|
||||
category="synthesis",
|
||||
created=note["created_at"],
|
||||
updated=note["updated_at"],
|
||||
)
|
||||
if rel and rel != current.get("vault_path"):
|
||||
await self.repo.set_vault_path(note_id, rel)
|
||||
except Exception:
|
||||
logger.exception("Failed to update knowledge note %s in vault", note_id)
|
||||
if note.get("content_md") is not None:
|
||||
await self._reindex_embeddings(note_id, note["content_md"])
|
||||
return await self.repo.get_note(note_id)
|
||||
|
||||
async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
|
||||
return await self.repo.list_versions(note_id)
|
||||
|
||||
# -- folders ----------------------------------------------------------
|
||||
async def list_folders(self) -> list[dict[str, Any]]:
|
||||
return await self.repo.list_folders()
|
||||
|
||||
async def create_folder(self, path: str, *, created_by: str | None = None) -> dict[str, Any]:
|
||||
return await self.repo.create_folder(path, created_by=created_by)
|
||||
|
||||
async def rename_folder(self, folder_id: str, new_path: str) -> dict[str, Any] | None:
|
||||
return await self.repo.rename_folder(folder_id, new_path)
|
||||
|
||||
async def delete_folder(self, folder_id: str, *, reassign_to: str | None = None) -> bool:
|
||||
return await self.repo.delete_folder(folder_id, reassign_to=reassign_to)
|
||||
|
||||
# -- extraction templates ---------------------------------------------
|
||||
async def _resolve_extract_prompt(self, template_id: str | None, scope: str) -> str | None:
|
||||
"""Return the system prompt for a capture, or ``None`` for the built-in default.
|
||||
|
||||
Resolution: explicit ``template_id`` → the scope's default template →
|
||||
``None`` (the extractor then uses its hard-coded prompt). Best-effort: any
|
||||
lookup failure degrades to the built-in prompt so capture never breaks.
|
||||
"""
|
||||
try:
|
||||
if template_id:
|
||||
tpl = await self.repo.get_template(template_id)
|
||||
if tpl and tpl.get("enabled") and (tpl.get("system_prompt") or "").strip():
|
||||
return tpl["system_prompt"]
|
||||
default = await self.repo.get_default_template(scope)
|
||||
if default and (default.get("system_prompt") or "").strip():
|
||||
return default["system_prompt"]
|
||||
except Exception:
|
||||
logger.exception("Failed to resolve knowledge extraction template (id=%s, scope=%s)", template_id, scope)
|
||||
return None
|
||||
|
||||
async def list_extract_templates(self, *, scope: str | None = None, enabled_only: bool = False) -> list[dict[str, Any]]:
|
||||
return await self.repo.list_templates(scope=scope, enabled_only=enabled_only)
|
||||
|
||||
async def get_extract_template(self, template_id: str) -> dict[str, Any] | None:
|
||||
return await self.repo.get_template(template_id)
|
||||
|
||||
async def create_extract_template(self, **kwargs: Any) -> dict[str, Any]:
|
||||
return await self.repo.create_template(**kwargs)
|
||||
|
||||
async def update_extract_template(self, template_id: str, **kwargs: Any) -> dict[str, Any] | None:
|
||||
return await self.repo.update_template(template_id, **kwargs)
|
||||
|
||||
async def delete_extract_template(self, template_id: str) -> bool:
|
||||
return await self.repo.delete_template(template_id)
|
||||
|
||||
async def template_name_taken(self, name: str, *, exclude_id: str | None = None) -> bool:
|
||||
return await self.repo.template_name_taken(name, exclude_id=exclude_id)
|
||||
|
||||
async def seed_default_templates(self, *, created_by: str | None = None) -> int:
|
||||
"""Seed the two built-in distillation prompts as editable templates (idempotent).
|
||||
|
||||
Runs once on startup so an operator immediately sees — and can edit — the
|
||||
prompts that drive 对话 / 文档 extraction. Existing templates with the same
|
||||
name are left untouched. Returns the number of templates created.
|
||||
"""
|
||||
from deerflow.knowledge import llm_extractor
|
||||
|
||||
guidance = llm_extractor.WIKI_CAPTURE_GUIDANCE
|
||||
seeds = [
|
||||
("默认 · 对话提炼", "thread", llm_extractor._SYSTEM_PROMPT + "\n\n" + guidance, "对话沉淀的默认提炼提示词(wiki-capture 规范)。"),
|
||||
("默认 · 文档提炼", "document", llm_extractor._DOC_SYSTEM_PROMPT + "\n\n" + guidance, "文档导入的默认提炼提示词(wiki-capture 规范)。"),
|
||||
]
|
||||
created = 0
|
||||
for name, scope, prompt, desc in seeds:
|
||||
try:
|
||||
if await self.repo.template_name_taken(name):
|
||||
continue
|
||||
await self.repo.create_template(name=name, system_prompt=prompt, scope=scope, description=desc, is_default=True, enabled=True, created_by=created_by)
|
||||
created += 1
|
||||
except Exception:
|
||||
logger.exception("Failed to seed knowledge extraction template '%s'", name)
|
||||
if created:
|
||||
logger.info("[knowledge] seeded %d default extraction template(s)", created)
|
||||
return created
|
||||
|
||||
async def delete_note(self, note_id: str, *, updated_by: str | None = None, hard: bool = False) -> bool:
|
||||
"""Soft delete (archive) by default; hard delete removes the vault file."""
|
||||
current = await self.repo.get_note(note_id, with_sources=False)
|
||||
if current is None:
|
||||
return False
|
||||
note = await self.repo.set_status(note_id, "archived", updated_by=updated_by)
|
||||
try:
|
||||
self.adapter.archive_note(note_id, current.get("vault_path"), updated=(note or current)["updated_at"])
|
||||
if hard and current.get("vault_path"):
|
||||
self.adapter.vault.delete_note(current["vault_path"])
|
||||
self.adapter.manifest.remove_page(note_id)
|
||||
await self.repo.delete_embeddings(note_id)
|
||||
except Exception:
|
||||
logger.exception("Failed to archive knowledge note %s in vault", note_id)
|
||||
# Drop this note's entity mentions + any now-orphaned entities so the graph
|
||||
# follows the deletion (no lingering nodes from removed notes).
|
||||
try:
|
||||
await self.repo.clear_note_relations(note_id)
|
||||
pruned = await self.repo.prune_orphan_entities()
|
||||
if pruned:
|
||||
logger.info("[knowledge] deleted note %s → pruned %d orphan entities", note_id, pruned)
|
||||
except Exception:
|
||||
logger.exception("Failed to prune relations/entities for deleted note %s", note_id)
|
||||
return True
|
||||
|
||||
# -- wikilinks --------------------------------------------------------
|
||||
async def resolve_wikilink(self, target: str) -> dict[str, Any]:
|
||||
"""Resolve a single ``[[wikilink]]`` target to a navigable destination.
|
||||
|
||||
Returns one of:
|
||||
- ``{"kind": "note", "note_id", "title", "exists": True}`` — a knowledge note
|
||||
- ``{"kind": "vault", "vault_path", "title", "exists": True}`` — a vault meta/hub page
|
||||
- ``{"kind": "missing", "exists": False}`` — nothing matches
|
||||
|
||||
Resolution order: note id → note vault_path → note title → vault file.
|
||||
"""
|
||||
norm = normalize_wikilink_target(target)
|
||||
base = {"target": target, "normalized": norm}
|
||||
if not norm:
|
||||
return {**base, "kind": "missing", "exists": False}
|
||||
|
||||
# 1) Direct note id (e.g. "kb_abc...").
|
||||
if norm.startswith("kb_"):
|
||||
note = await self.repo.get_note(norm, with_sources=False)
|
||||
if note and note.get("status") != "archived":
|
||||
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
|
||||
|
||||
# 2) Note by vault path (links from index.md / hot.md / structural hub).
|
||||
note = await self.repo.find_by_vault_path(norm)
|
||||
if note:
|
||||
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
|
||||
|
||||
# 3) Note by title (the real related-note cross-links use [[Title]]).
|
||||
note = await self.repo.find_by_title(norm) or await self.repo.find_by_title(norm.split("/")[-1])
|
||||
if note:
|
||||
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
|
||||
|
||||
# 4) A vault meta/hub page that has no DB note (index, _meta/taxonomy, hub…).
|
||||
page = self.read_vault_page(norm)
|
||||
if page is not None:
|
||||
return {**base, "kind": "vault", "vault_path": page["path"], "title": page["title"], "exists": True}
|
||||
|
||||
return {**base, "kind": "missing", "exists": False}
|
||||
|
||||
async def resolve_wikilinks(self, targets: list[str]) -> dict[str, dict[str, Any]]:
|
||||
"""Resolve many targets at once; returns ``{original_target: resolution}``."""
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for target in dict.fromkeys(t for t in targets if t and t.strip()):
|
||||
out[target] = await self.resolve_wikilink(target)
|
||||
return out
|
||||
|
||||
async def resolve_note_wikilinks(self, note_id: str) -> dict[str, dict[str, Any]] | None:
|
||||
"""Extract every ``[[wikilink]]`` in a note's body and resolve them.
|
||||
|
||||
Returns ``{target: resolution}`` or ``None`` when the note is missing.
|
||||
Numeric ``[[123]]`` citation badges are skipped (handled by the renderer).
|
||||
"""
|
||||
note = await self.repo.get_note(note_id, with_sources=False)
|
||||
if note is None:
|
||||
return None
|
||||
targets = [t for t in extract_wikilinks(note.get("content_md") or "") if not t.isdigit()]
|
||||
return await self.resolve_wikilinks(targets)
|
||||
|
||||
def read_vault_page(self, path: str) -> dict[str, Any] | None:
|
||||
"""Read a raw vault Markdown page (meta/hub pages without a DB note).
|
||||
|
||||
Returns ``{"path", "title", "content_md"}`` or ``None`` if the page does
|
||||
not exist or the path escapes the vault root.
|
||||
"""
|
||||
norm = normalize_wikilink_target(path)
|
||||
if not norm:
|
||||
return None
|
||||
rel = f"{norm}.md"
|
||||
try:
|
||||
parsed = self.adapter.vault.read_note(rel)
|
||||
except VaultPathError:
|
||||
return None
|
||||
if parsed is None:
|
||||
return None
|
||||
fm, body = parsed
|
||||
title = str(fm.get("title") or "").strip() or norm.split("/")[-1]
|
||||
return {"path": norm, "title": title, "content_md": body}
|
||||
|
||||
# -- search & graph ---------------------------------------------------
|
||||
async def search(self, query: str, *, mode: str = "keyword", limit: int = 8) -> list[dict[str, Any]]:
|
||||
return await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
|
||||
|
||||
async def build_rag_context(self, query: str, *, limit: int = 5) -> dict[str, Any]:
|
||||
"""Retrieve top knowledge for ``query`` and format an injectable block.
|
||||
|
||||
Returns ``{"text": <system context or "">, "items": [...]}``. Uses hybrid
|
||||
search when embeddings are available, else keyword (wiki-query style).
|
||||
"""
|
||||
mode = "hybrid" if self.embedding_client.is_enabled() else "keyword"
|
||||
items = await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
|
||||
if not items:
|
||||
return {"text": "", "items": []}
|
||||
lines = [
|
||||
"以下是产品知识库中与本问题相关的资料。回答时优先参考;若使用其中信息,请在回答中标注来源(note_id)。",
|
||||
"",
|
||||
]
|
||||
for i, it in enumerate(items, 1):
|
||||
lines.append(f"[知识 {i}] {it.get('title') or ''}")
|
||||
if it.get("snippet"):
|
||||
lines.append(it["snippet"])
|
||||
lines.append(f"来源:{it['note_id']}")
|
||||
lines.append("")
|
||||
return {"text": "\n".join(lines).strip(), "items": items}
|
||||
|
||||
# -- dedup & merge (phase 3) -----------------------------------------
|
||||
async def find_duplicates(self) -> list[dict[str, Any]]:
|
||||
from deerflow.knowledge.dedup import find_duplicate_groups
|
||||
|
||||
notes = await self.repo.list_all()
|
||||
return find_duplicate_groups(notes, threshold=self.config.dedup_similarity_threshold)
|
||||
|
||||
async def merge_notes(self, primary_id: str, duplicate_ids: list[str], *, updated_by: str | None = None) -> dict[str, Any] | None:
|
||||
"""Merge ``duplicate_ids`` into ``primary_id``.
|
||||
|
||||
Sources from each duplicate are re-attached to the primary, a wikilink
|
||||
+ "Merged from" note is appended to the primary body, the duplicates are
|
||||
archived (soft delete), and a ``merged_into`` relation is recorded.
|
||||
"""
|
||||
primary = await self.repo.get_note(primary_id)
|
||||
if primary is None:
|
||||
return None
|
||||
merged_titles: list[str] = []
|
||||
for dup_id in duplicate_ids:
|
||||
if dup_id == primary_id:
|
||||
continue
|
||||
dup = await self.repo.get_note(dup_id)
|
||||
if dup is None:
|
||||
continue
|
||||
# Move sources.
|
||||
for src in dup.get("sources") or []:
|
||||
await self.repo.add_sources(
|
||||
primary_id,
|
||||
[
|
||||
SourceDraft(
|
||||
source_type=src.get("source_type") or "tool",
|
||||
thread_id=src.get("thread_id"),
|
||||
message_id=src.get("message_id"),
|
||||
tool_name=src.get("tool_name"),
|
||||
title=src.get("title"),
|
||||
url=src.get("url"),
|
||||
snippet=src.get("snippet"),
|
||||
)
|
||||
],
|
||||
created_by=updated_by,
|
||||
)
|
||||
await self.repo.add_relation(from_note_id=dup_id, to_note_id=primary_id, relation_type="merged_into")
|
||||
await self.delete_note(dup_id, updated_by=updated_by)
|
||||
merged_titles.append(dup.get("title") or dup_id)
|
||||
|
||||
if merged_titles:
|
||||
appended = primary["content_md"].rstrip() + "\n\n## Merged\n\n" + "\n".join(f"- {t}" for t in merged_titles) + "\n"
|
||||
await self.update_note(primary_id, content_md=appended, updated_by=updated_by)
|
||||
return await self.repo.get_note(primary_id)
|
||||
|
||||
async def reindex_all(self) -> dict[str, int]:
|
||||
"""Rebuild embeddings for every non-archived note (admin/maintenance)."""
|
||||
notes = await self.repo.list_all()
|
||||
count = 0
|
||||
for note in notes:
|
||||
await self._reindex_embeddings(note["id"], note.get("content_md") or "")
|
||||
count += 1
|
||||
return {"reindexed": count, "embedding_enabled": int(self.embedding_client.is_enabled())}
|
||||
|
||||
async def export_graph(self) -> dict[str, Any]:
|
||||
from deerflow.utils.time import now_iso
|
||||
|
||||
notes = await self.repo.list_all()
|
||||
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
|
||||
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
|
||||
blacklist = await self._graph_blacklist_set()
|
||||
result = self.adapter.export_graph(notes, entities=entities, relations=relations, updated=now_iso(), blacklist=blacklist)
|
||||
return {"files": result["files"], "stats": result["stats"]}
|
||||
|
||||
async def get_graph(self) -> dict[str, Any]:
|
||||
"""Return product graph data ``{nodes, edges}`` (notes + entities)."""
|
||||
from deerflow.knowledge import exporter
|
||||
|
||||
notes = await self.repo.list_all()
|
||||
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
|
||||
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
|
||||
blacklist = await self._graph_blacklist_set()
|
||||
return exporter.build_graph(notes, entities, relations, blacklist=blacklist)
|
||||
|
||||
# -- graph blacklist ---------------------------------------------------
|
||||
async def _graph_blacklist_set(self) -> set[str]:
|
||||
from deerflow.knowledge import exporter
|
||||
|
||||
items = await self.repo.list_graph_blacklist()
|
||||
return exporter.normalize_blacklist([item["label"] for item in items])
|
||||
|
||||
async def list_graph_blacklist(self) -> list[dict[str, Any]]:
|
||||
"""Return blacklist entries (node labels hidden from the graph)."""
|
||||
return await self.repo.list_graph_blacklist()
|
||||
|
||||
async def add_graph_blacklist(self, labels: list[str], *, created_by: str | None = None) -> int:
|
||||
"""Blacklist node labels (entity names / note titles). Returns the count added."""
|
||||
return await self.repo.add_graph_blacklist(labels, created_by=created_by)
|
||||
|
||||
async def remove_graph_blacklist(self, entry_id: str) -> bool:
|
||||
return await self.repo.remove_graph_blacklist(entry_id)
|
||||
|
||||
def export_file_path(self, filename: str) -> Path | None:
|
||||
return self.adapter.export_file_path(filename)
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
"""Built-in obsidian-wiki skill templates.
|
||||
|
||||
Mirrors the key intent of ``ar9av/obsidian-wiki`` skills so the backend can
|
||||
write vault content that stays compatible with the original tooling without
|
||||
shelling out to its CLI:
|
||||
|
||||
- ``wiki-capture`` — how a conversation becomes a *declarative* knowledge note
|
||||
(frontmatter schema + Context/Finding/Reasoning/Implications/Related body).
|
||||
- ``wiki-query`` — layered retrieval (hot.md → index.md → frontmatter → body).
|
||||
- ``wiki-export`` — graph artifacts (graph.json / graph.html / graphml / cypher).
|
||||
|
||||
These are intentionally lightweight: the structures here are what
|
||||
:mod:`deerflow.knowledge.markdown_vault` and :mod:`deerflow.knowledge.extractor`
|
||||
produce, and what the first-release LLM prompt (future enhancement) should
|
||||
target.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Category → vault sub-directory (wiki-capture page taxonomy, §8 of the dev doc).
|
||||
CATEGORY_DIRS = {
|
||||
"synthesis": "synthesis",
|
||||
"concepts": "concepts",
|
||||
"concept": "concepts",
|
||||
"references": "references",
|
||||
"reference": "references",
|
||||
"source": "references",
|
||||
"decision": "synthesis",
|
||||
"journal": "journal",
|
||||
"session": "journal",
|
||||
}
|
||||
|
||||
# Body section headings (wiki-capture). Declarative knowledge, never a chat log.
|
||||
BODY_SECTIONS = ("Context", "Finding / Decision", "Reasoning", "Implications", "Related")
|
||||
|
||||
# Inline provenance markers reused from wiki-capture.
|
||||
INFERRED_MARKER = "^[inferred]"
|
||||
AMBIGUOUS_MARKER = "^[ambiguous]"
|
||||
|
||||
WIKI_CAPTURE_GUIDANCE = """\
|
||||
Capture *declarative* knowledge, not a chat transcript. Classify the note as one
|
||||
of: synthesis, concept, source, decision, session. Prefer at least two
|
||||
[[wikilinks]]. Mark inferred content with ^[inferred] and uncertain/conflicting
|
||||
content with ^[ambiguous]. Project-specific knowledge belongs under
|
||||
projects/<project-name>/...
|
||||
"""
|
||||
|
||||
|
||||
def category_to_dir(category: str | None) -> str:
|
||||
"""Map a wiki-capture category to its vault sub-directory."""
|
||||
return CATEGORY_DIRS.get((category or "").strip().lower(), "synthesis")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Normalize knowledge sources from threads / searches / tool results.
|
||||
|
||||
Source items come from a few shapes: backend reference-batches (built by
|
||||
``deerflow.runtime.references``), raw search-result payloads, and tool-call
|
||||
messages. They are all normalized into :class:`SourceDraft` so the repository
|
||||
and frontend see one consistent structure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.schemas import SourceDraft
|
||||
|
||||
|
||||
def _clip(value: Any, limit: int = 2000) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
return text[:limit]
|
||||
|
||||
|
||||
def from_reference_batches(batches: list[dict[str, Any]] | None, *, thread_id: str | None = None, max_items: int = 20) -> list[SourceDraft]:
|
||||
"""Convert backend reference-batches into :class:`SourceDraft`."""
|
||||
out: list[SourceDraft] = []
|
||||
seen: set[str] = set()
|
||||
for batch in batches or []:
|
||||
msg_id = batch.get("assistant_message_id") or batch.get("id")
|
||||
for src in batch.get("sources", []) or []:
|
||||
if not isinstance(src, dict):
|
||||
continue
|
||||
url = _clip(src.get("url"), 1024)
|
||||
title = _clip(src.get("title"), 512) or _clip(src.get("source"), 512) or "来源"
|
||||
dedup_key = url or f"{title}:{_clip(src.get('snippet'), 80)}"
|
||||
if dedup_key in seen:
|
||||
continue
|
||||
seen.add(dedup_key)
|
||||
out.append(
|
||||
SourceDraft(
|
||||
source_type="search" if url else "tool",
|
||||
thread_id=thread_id,
|
||||
message_id=_clip(msg_id, 128),
|
||||
tool_name=_clip(src.get("skillName"), 128),
|
||||
title=title,
|
||||
url=url,
|
||||
snippet=_clip(src.get("snippet")),
|
||||
raw=None,
|
||||
)
|
||||
)
|
||||
if len(out) >= max_items:
|
||||
return out
|
||||
return out
|
||||
|
||||
|
||||
def from_search_results(
|
||||
query: str,
|
||||
results: list[dict[str, Any]] | None,
|
||||
*,
|
||||
thread_id: str | None = None,
|
||||
message_id: str | None = None,
|
||||
max_items: int = 20,
|
||||
) -> list[SourceDraft]:
|
||||
"""Convert raw search results (title/url/snippet) into :class:`SourceDraft`."""
|
||||
out: list[SourceDraft] = []
|
||||
for item in (results or [])[:max_items]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
out.append(
|
||||
SourceDraft(
|
||||
source_type="search",
|
||||
thread_id=thread_id,
|
||||
message_id=message_id,
|
||||
tool_name=_clip(item.get("tool_name") or item.get("tool"), 128),
|
||||
title=_clip(item.get("title"), 512) or query[:200],
|
||||
url=_clip(item.get("url"), 1024),
|
||||
snippet=_clip(item.get("snippet") or item.get("content")),
|
||||
raw={k: item.get(k) for k in ("score",) if item.get(k) is not None} or None,
|
||||
)
|
||||
)
|
||||
return out
|
||||
Reference in new issue
Block a user