AdministratorandClaude Opus 4.8
81e4fd92ce
阶段三（治理）:
- sensitive.py：写入前脱敏 token/密钥/手机号/身份证/邮箱（sensitive_redaction_enabled）
- 版本快照：更新前写 knowledge_note_versions；GET /notes/{id}/versions
- dedup.py：标题/摘要 Jaccard + 内容哈希查重；GET /duplicates、POST /merge（来源并入主条目、重复项归档）
- auto_ingest.py：独立后台队列/线程沉淀，不阻塞问答；中间件 after_agent 在 auto_ingest_enabled 时入队；价值判断（最小字符）+ skip_existing 去重；自动沉淀落 draft 待审
阶段四（图谱）:
- LLM 抽取 entities/relations；service 存 knowledge_entities/knowledge_relations 并连边
- exporter.build_graph 支持实体节点 + note→entity/entity→entity 边；graph.html/json 增强
- GET /api/knowledge/graph 返回产品图谱数据
前端：知识库页「查重合并」对话框 + 相关 api/hooks
配置：sensitive_redaction_enabled / dedup_similarity_threshold / entity_extraction_enabled
测试：tests/test_knowledge.py 20 passed
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
55 lines
2.1 KiB
Python
55 lines
2.1 KiB
Python
"""Sensitive-information redaction for knowledge content (phase 3).
|
|
|
|
Best-effort masking of secrets / PII before a note is persisted: API keys,
|
|
bearer tokens, generic ``key=secret`` assignments, Chinese mobile numbers,
|
|
ID-card numbers and email addresses. Conservative by design — it favours not
|
|
mangling legitimate prose over catching every possible secret.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
_MASK = "[REDACTED]"
|
|
|
|
# Order matters: more specific patterns first.
|
|
_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|
("openai_key", re.compile(r"\bsk-[A-Za-z0-9]{20,}\b")),
|
|
("aws_key", re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
|
|
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b")),
|
|
("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]{16,}")),
|
|
# key/secret/password/token assignment: keep the field name, mask the value.
|
|
("assignment", re.compile(r"(?i)\b(api[_-]?key|secret|password|passwd|token|access[_-]?token)\b\s*[:=]\s*[\"']?[^\s\"']{6,}[\"']?")),
|
|
("cn_id_card", re.compile(r"\b\d{17}[\dXx]\b")),
|
|
("cn_mobile", re.compile(r"(?<!\d)1[3-9]\d{9}(?!\d)")),
|
|
("email", re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")),
|
|
]
|
|
|
|
|
|
def redact(text: str | None) -> tuple[str, list[str]]:
|
|
"""Return ``(redacted_text, hit_kinds)``.
|
|
|
|
``hit_kinds`` lists which detectors fired (for logging/audit). When ``text``
|
|
is falsy it is returned unchanged with no hits.
|
|
"""
|
|
if not text:
|
|
return text or "", []
|
|
hits: list[str] = []
|
|
out = text
|
|
for kind, pattern in _PATTERNS:
|
|
def _sub(m: re.Match[str], _kind: str = kind) -> str:
|
|
# For assignments keep the field name + delimiter, mask only the value.
|
|
if _kind == "assignment":
|
|
head = re.split(r"[:=]", m.group(0), maxsplit=1)[0]
|
|
sep = ":" if ":" in m.group(0) else "="
|
|
return f"{head}{sep} {_MASK}"
|
|
if _kind == "bearer":
|
|
return f"Bearer {_MASK}"
|
|
return _MASK
|
|
|
|
new_out, n = pattern.subn(_sub, out)
|
|
if n:
|
|
hits.append(kind)
|
|
out = new_out
|
|
return out, hits
|