AdministratorandClaude Opus 4.8
81e4fd92ce
阶段三（治理）:
- sensitive.py：写入前脱敏 token/密钥/手机号/身份证/邮箱（sensitive_redaction_enabled）
- 版本快照：更新前写 knowledge_note_versions；GET /notes/{id}/versions
- dedup.py：标题/摘要 Jaccard + 内容哈希查重；GET /duplicates、POST /merge（来源并入主条目、重复项归档）
- auto_ingest.py：独立后台队列/线程沉淀，不阻塞问答；中间件 after_agent 在 auto_ingest_enabled 时入队；价值判断（最小字符）+ skip_existing 去重；自动沉淀落 draft 待审
阶段四（图谱）:
- LLM 抽取 entities/relations；service 存 knowledge_entities/knowledge_relations 并连边
- exporter.build_graph 支持实体节点 + note→entity/entity→entity 边；graph.html/json 增强
- GET /api/knowledge/graph 返回产品图谱数据
前端：知识库页「查重合并」对话框 + 相关 api/hooks
配置：sensitive_redaction_enabled / dedup_similarity_threshold / entity_extraction_enabled
测试：tests/test_knowledge.py 20 passed
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
76 lines
2.7 KiB
Python
76 lines
2.7 KiB
Python
"""Duplicate detection for knowledge notes (phase 3).
|
|
|
|
First-release heuristic: token Jaccard over ``title + summary + tags`` plus an
|
|
exact ``content_md`` hash match. Cheap, deterministic and embedding-free so it
|
|
works on offline deployments. Returns *suggestions*; merging is an explicit,
|
|
user-driven action.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
from deerflow.knowledge.manifest import content_hash
|
|
|
|
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
|
|
|
|
|
|
def _tokens(*texts: str) -> set[str]:
|
|
out: set[str] = set()
|
|
for t in texts:
|
|
out.update(m.lower() for m in _TOKEN_RE.findall(t or ""))
|
|
return out
|
|
|
|
|
|
def _jaccard(a: set[str], b: set[str]) -> float:
|
|
if not a or not b:
|
|
return 0.0
|
|
inter = len(a & b)
|
|
union = len(a | b)
|
|
return inter / union if union else 0.0
|
|
|
|
|
|
def find_duplicate_groups(notes: list[dict[str, Any]], *, threshold: float = 0.82, max_notes: int = 800) -> list[dict[str, Any]]:
|
|
"""Return groups ``{primary, duplicates:[{id,title,score,reason}]}``.
|
|
|
|
``primary`` is the most recently updated note in each cluster.
|
|
"""
|
|
notes = notes[:max_notes]
|
|
feats: list[tuple[dict[str, Any], set[str], str]] = []
|
|
for n in notes:
|
|
toks = _tokens(n.get("title") or "", n.get("summary") or "", " ".join(n.get("tags") or []))
|
|
feats.append((n, toks, content_hash(n.get("content_md") or "")))
|
|
|
|
used: set[str] = set()
|
|
groups: list[dict[str, Any]] = []
|
|
for i in range(len(feats)):
|
|
ni, ti, hi = feats[i]
|
|
if ni["id"] in used:
|
|
continue
|
|
dups: list[dict[str, Any]] = []
|
|
for j in range(i + 1, len(feats)):
|
|
nj, tj, hj = feats[j]
|
|
if nj["id"] in used:
|
|
continue
|
|
if hi and hi == hj:
|
|
score, reason = 1.0, "identical-content"
|
|
else:
|
|
score = _jaccard(ti, tj)
|
|
reason = "similar-title-summary"
|
|
if score >= threshold:
|
|
dups.append({"id": nj["id"], "title": nj.get("title"), "score": round(score, 4), "reason": reason})
|
|
used.add(nj["id"])
|
|
if dups:
|
|
used.add(ni["id"])
|
|
# Primary = newest by updated_at.
|
|
cluster = [ni, *[next(f[0] for f in feats if f[0]["id"] == d["id"]) for d in dups]]
|
|
primary = max(cluster, key=lambda x: x.get("updated_at") or "")
|
|
others = [
|
|
{"id": c["id"], "title": c.get("title"), "score": next((d["score"] for d in dups if d["id"] == c["id"]), 1.0)}
|
|
for c in cluster
|
|
if c["id"] != primary["id"]
|
|
]
|
|
groups.append({"primary": {"id": primary["id"], "title": primary.get("title")}, "duplicates": others})
|
|
return groups
|