初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
@@ -0,0 +1,75 @@
|
||||
"""Duplicate detection for knowledge notes (phase 3).
|
||||
|
||||
First-release heuristic: token Jaccard over ``title + summary + tags`` plus an
|
||||
exact ``content_md`` hash match. Cheap, deterministic and embedding-free so it
|
||||
works on offline deployments. Returns *suggestions*; merging is an explicit,
|
||||
user-driven action.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.manifest import content_hash
|
||||
|
||||
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
|
||||
|
||||
|
||||
def _tokens(*texts: str) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for t in texts:
|
||||
out.update(m.lower() for m in _TOKEN_RE.findall(t or ""))
|
||||
return out
|
||||
|
||||
|
||||
def _jaccard(a: set[str], b: set[str]) -> float:
|
||||
if not a or not b:
|
||||
return 0.0
|
||||
inter = len(a & b)
|
||||
union = len(a | b)
|
||||
return inter / union if union else 0.0
|
||||
|
||||
|
||||
def find_duplicate_groups(notes: list[dict[str, Any]], *, threshold: float = 0.82, max_notes: int = 800) -> list[dict[str, Any]]:
|
||||
"""Return groups ``{primary, duplicates:[{id,title,score,reason}]}``.
|
||||
|
||||
``primary`` is the most recently updated note in each cluster.
|
||||
"""
|
||||
notes = notes[:max_notes]
|
||||
feats: list[tuple[dict[str, Any], set[str], str]] = []
|
||||
for n in notes:
|
||||
toks = _tokens(n.get("title") or "", n.get("summary") or "", " ".join(n.get("tags") or []))
|
||||
feats.append((n, toks, content_hash(n.get("content_md") or "")))
|
||||
|
||||
used: set[str] = set()
|
||||
groups: list[dict[str, Any]] = []
|
||||
for i in range(len(feats)):
|
||||
ni, ti, hi = feats[i]
|
||||
if ni["id"] in used:
|
||||
continue
|
||||
dups: list[dict[str, Any]] = []
|
||||
for j in range(i + 1, len(feats)):
|
||||
nj, tj, hj = feats[j]
|
||||
if nj["id"] in used:
|
||||
continue
|
||||
if hi and hi == hj:
|
||||
score, reason = 1.0, "identical-content"
|
||||
else:
|
||||
score = _jaccard(ti, tj)
|
||||
reason = "similar-title-summary"
|
||||
if score >= threshold:
|
||||
dups.append({"id": nj["id"], "title": nj.get("title"), "score": round(score, 4), "reason": reason})
|
||||
used.add(nj["id"])
|
||||
if dups:
|
||||
used.add(ni["id"])
|
||||
# Primary = newest by updated_at.
|
||||
cluster = [ni, *[next(f[0] for f in feats if f[0]["id"] == d["id"]) for d in dups]]
|
||||
primary = max(cluster, key=lambda x: x.get("updated_at") or "")
|
||||
others = [
|
||||
{"id": c["id"], "title": c.get("title"), "score": next((d["score"] for d in dups if d["id"] == c["id"]), 1.0)}
|
||||
for c in cluster
|
||||
if c["id"] != primary["id"]
|
||||
]
|
||||
groups.append({"primary": {"id": primary["id"], "title": primary.get("title")}, "duplicates": others})
|
||||
return groups
|
||||
Reference in new issue
Block a user