初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
@@ -0,0 +1,54 @@
|
||||
"""Sensitive-information redaction for knowledge content (phase 3).
|
||||
|
||||
Best-effort masking of secrets / PII before a note is persisted: API keys,
|
||||
bearer tokens, generic ``key=secret`` assignments, Chinese mobile numbers,
|
||||
ID-card numbers and email addresses. Conservative by design — it favours not
|
||||
mangling legitimate prose over catching every possible secret.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_MASK = "[REDACTED]"
|
||||
|
||||
# Order matters: more specific patterns first.
|
||||
_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
("openai_key", re.compile(r"\bsk-[A-Za-z0-9]{20,}\b")),
|
||||
("aws_key", re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
|
||||
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b")),
|
||||
("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]{16,}")),
|
||||
# key/secret/password/token assignment: keep the field name, mask the value.
|
||||
("assignment", re.compile(r"(?i)\b(api[_-]?key|secret|password|passwd|token|access[_-]?token)\b\s*[:=]\s*[\"']?[^\s\"']{6,}[\"']?")),
|
||||
("cn_id_card", re.compile(r"\b\d{17}[\dXx]\b")),
|
||||
("cn_mobile", re.compile(r"(?<!\d)1[3-9]\d{9}(?!\d)")),
|
||||
("email", re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")),
|
||||
]
|
||||
|
||||
|
||||
def redact(text: str | None) -> tuple[str, list[str]]:
|
||||
"""Return ``(redacted_text, hit_kinds)``.
|
||||
|
||||
``hit_kinds`` lists which detectors fired (for logging/audit). When ``text``
|
||||
is falsy it is returned unchanged with no hits.
|
||||
"""
|
||||
if not text:
|
||||
return text or "", []
|
||||
hits: list[str] = []
|
||||
out = text
|
||||
for kind, pattern in _PATTERNS:
|
||||
def _sub(m: re.Match[str], _kind: str = kind) -> str:
|
||||
# For assignments keep the field name + delimiter, mask only the value.
|
||||
if _kind == "assignment":
|
||||
head = re.split(r"[:=]", m.group(0), maxsplit=1)[0]
|
||||
sep = ":" if ":" in m.group(0) else "="
|
||||
return f"{head}{sep} {_MASK}"
|
||||
if _kind == "bearer":
|
||||
return f"Bearer {_MASK}"
|
||||
return _MASK
|
||||
|
||||
new_out, n = pattern.subn(_sub, out)
|
||||
if n:
|
||||
hits.append(kind)
|
||||
out = new_out
|
||||
return out, hits
|
||||
Reference in new issue
Block a user