feat: add skill and assistant knowledge bases

This commit is contained in:
Administrator committed 2026-09-02 00:49:13 +08:00
1 parent dd0ee282df
commit fbd8dec48f
45 files changed
+5262 -21

No files matched your search

@@ -0,0 +1,10 @@
"""Skill knowledge distillation primitives.
The package is deliberately independent from ``app.*``. Gateway composition
injects persistence and remote adapters at runtime.
"""
from deerflow.skill_knowledge.extractor import build_skill_ir
from deerflow.skill_knowledge.scanner import ScanResult, enrich_converted_documents, scan_skill
__all__ = ["ScanResult", "build_skill_ir", "enrich_converted_documents", "scan_skill"]
@@ -0,0 +1,152 @@
"""IR construction and validation.
V1 intentionally starts with deterministic, evidence-backed extraction. The
schema is compatible with a future model-backed extractor, while never letting
unvalidated model output flow directly into persistence or graph writers.
"""
from __future__ import annotations
import hashlib
import re
from typing import Any
from deerflow.skill_knowledge.scanner import ScanResult
EXTRACTOR_VERSION = "skill-knowledge-v1"
PROMPT_HASH = f"sha256:{hashlib.sha256(b'skill-knowledge-v1-evidence-only').hexdigest()}"
PREDICATE_PATTERNS = {
"depends_on": re.compile(r"(?:依赖|需要|requires?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"uses": re.compile(r"(?:调用|使用|uses?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"produces": re.compile(r"(?:生成|输出|produces?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"consumes": re.compile(r"(?:读取|输入|consumes?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
}
def _evidence(path: str, text: str, start: int, end: int) -> dict[str, Any]:
line_start = text.count("\n", 0, start) + 1
line_end = text.count("\n", 0, end) + 1
quote = " ".join(text[start:end].split())[:240]
return {"file_path": path, "line_start": line_start, "line_end": line_end, "quote": quote}
def build_skill_ir(scan: ScanResult, *, display_name: str | None = None, description: str = "") -> dict[str, Any]:
entities: list[dict[str, Any]] = [
{
"local_id": "ent_skill",
"type": "skill",
"name": display_name or scan.skill_name,
"aliases": [scan.skill_name] if display_name and display_name != scan.skill_name else [],
"description": description,
"attributes": {},
"evidence": [],
"confidence": 1.0,
}
]
relations: list[dict[str, Any]] = []
by_key: dict[tuple[str, str], str] = {("skill", (display_name or scan.skill_name).casefold()): "ent_skill"}
for file in scan.files:
if not file.text:
continue
for predicate, pattern in PREDICATE_PATTERNS.items():
for match in pattern.finditer(file.text):
name = match.group(1).strip(" `\"'“”()[]")
if not name or len(name) > 80:
continue
key = ("concept", name.casefold())
entity_id = by_key.get(key)
if entity_id is None:
entity_id = f"ent_{len(entities):04d}"
by_key[key] = entity_id
entities.append(
{
"local_id": entity_id,
"type": "concept",
"name": name,
"aliases": [],
"description": "从技能业务说明中抽取的知识对象",
"attributes": {},
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
"confidence": 0.82,
}
)
relations.append(
{
"local_id": f"rel_{len(relations):04d}",
"source_local_id": "ent_skill",
"predicate": predicate,
"target_local_id": entity_id,
"description": " ".join(match.group(0).split())[:240],
"relation_source": "explicit_text",
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
"confidence": 0.82,
}
)
for ref in file.references:
name = ref["target_path"]
key = ("file", name.casefold())
entity_id = by_key.get(key)
if entity_id is None:
entity_id = f"ent_{len(entities):04d}"
by_key[key] = entity_id
entities.append(
{
"local_id": entity_id,
"type": "file",
"name": name,
"aliases": [],
"description": "技能文档引用的资料文件",
"attributes": {},
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
"confidence": 1.0,
}
)
relations.append(
{
"local_id": f"rel_{len(relations):04d}",
"source_local_id": "ent_skill",
"predicate": "references",
"target_local_id": entity_id,
"description": f"{file.path} 引用 {name}",
"relation_source": "deterministic_structure",
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
"confidence": 1.0,
}
)
title = display_name or scan.skill_name
summaries = [f"- **{item.title}**(`{item.path}`)" for item in scan.files[:100]]
content = "\n".join(
[
f"# {title}",
"",
description or "由技能知识文件安全归纳生成。",
"",
"## 知识资料",
"",
*(summaries or ["- 暂无可归纳知识文件"]),
"",
f"> 源摘要:`{scan.source_digest}`;代码与脚本文件未读取。",
]
)
return {
"extractor_version": EXTRACTOR_VERSION,
"prompt_hash": PROMPT_HASH,
"model_name": "deterministic-v1",
"skill": {
"name": scan.skill_name,
"display_name": title,
"description": description,
"source_root": scan.source_root,
"source_digest": scan.source_digest,
},
"files": scan.manifest,
"entities": entities,
"relations": relations,
"wiki": {
"skill_page": {"slug": f"skills/{scan.skill_name}/overview", "title": title, "content": content},
"pages": [],
},
"safety": {"blocked_files": scan.blocked_files},
}
@@ -0,0 +1,115 @@
"""Guarded graph-write plans.
No caller can submit Cypher. The writer accepts only normalized node/edge
records and executes fixed parameterized templates when explicitly enabled.
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from typing import Any, Protocol
class GraphTransaction(Protocol):
async def run(self, query: str, parameters: dict[str, Any]) -> Any: ...
@dataclass(frozen=True, slots=True)
class GraphWritePlan:
scope: str
nodes: tuple[dict[str, Any], ...]
edges: tuple[dict[str, Any], ...]
NODE_QUERY = """
MERGE (n:DeerFlowKnowledge {scope: $scope, entity_key: $entity_key})
SET n.display_name = $display_name, n.entity_type = $entity_type,
n.binding_id = $binding_id, n.snapshot_id = $snapshot_id, n.job_id = $job_id,
n.confidence = $confidence, n.updated_at = datetime()
""".strip()
EDGE_QUERY = """
MATCH (a:DeerFlowKnowledge {scope: $scope, entity_key: $source_key})
MATCH (b:DeerFlowKnowledge {scope: $scope, entity_key: $target_key})
MERGE (a)-[r:DEERFLOW_RELATION {edge_key: $edge_key}]->(b)
SET r.predicate = $predicate, r.binding_id = $binding_id,
r.snapshot_id = $snapshot_id, r.job_id = $job_id, r.confidence = $confidence,
r.evidence = $evidence, r.updated_at = datetime()
""".strip()
def _key(*parts: str) -> str:
return hashlib.sha256("\x1f".join(parts).encode()).hexdigest()
def plan_from_ir(
ir: dict[str, Any],
*,
target_type: str,
target_id: str,
binding_id: str,
snapshot_id: str,
job_id: str,
) -> GraphWritePlan:
"""Build a stable write plan; callers cannot supply executable Cypher."""
scope = f"{target_type}:{target_id}"
local_keys: dict[str, str] = {}
nodes: list[dict[str, Any]] = []
for entity in ir.get("entities") or []:
entity_type = str(entity.get("type") or "concept")
normalized_name = str(entity.get("normalized_name") or entity.get("name") or "").strip().casefold()
if not normalized_name:
continue
entity_key = _key(entity_type, normalized_name)
local_keys[str(entity.get("local_id") or normalized_name)] = entity_key
nodes.append(
{
"entity_key": entity_key,
"display_name": str(entity.get("name") or normalized_name),
"entity_type": entity_type,
"binding_id": binding_id,
"snapshot_id": snapshot_id,
"job_id": job_id,
"confidence": float(entity.get("confidence") or 1.0),
}
)
edges: list[dict[str, Any]] = []
for relation in ir.get("relations") or []:
source_key = local_keys.get(str(relation.get("source_local_id") or ""))
target_key = local_keys.get(str(relation.get("target_local_id") or ""))
evidence = list(relation.get("evidence") or [])
confidence = float(relation.get("confidence") or 0)
predicate = str(relation.get("predicate") or "related_to")
if not source_key or not target_key or confidence < 0.75 or not evidence:
continue
edges.append(
{
"edge_key": _key(source_key, predicate, target_key, binding_id, snapshot_id),
"source_key": source_key,
"target_key": target_key,
"predicate": predicate,
"binding_id": binding_id,
"snapshot_id": snapshot_id,
"job_id": job_id,
"confidence": confidence,
"evidence": evidence,
}
)
return GraphWritePlan(scope=scope, nodes=tuple(nodes), edges=tuple(edges))
def parameterized_statements(plan: GraphWritePlan) -> list[dict[str, Any]]:
"""Return Neo4j transactional HTTP statements using only fixed templates."""
statements = [{"statement": NODE_QUERY, "parameters": {"scope": plan.scope, **node}} for node in plan.nodes]
statements.extend({"statement": EDGE_QUERY, "parameters": {"scope": plan.scope, **edge}} for edge in plan.edges)
return statements
async def execute_plan(tx: GraphTransaction, plan: GraphWritePlan, *, enabled: bool) -> dict[str, int]:
if not enabled:
return {"nodes": 0, "edges": 0}
for node in plan.nodes:
await tx.run(NODE_QUERY, {"scope": plan.scope, **node})
for edge in plan.edges:
await tx.run(EDGE_QUERY, {"scope": plan.scope, **edge})
return {"nodes": len(plan.nodes), "edges": len(plan.edges)}
@@ -0,0 +1,31 @@
"""Portable entity/relation normalization used by both knowledge products."""
from __future__ import annotations
import re
import unicodedata
def normalize_name(value: str) -> str:
text = unicodedata.normalize("NFKC", value or "").strip().casefold()
text = text.strip("`'\"“”‘’《》〈〉[](){}")
text = re.sub(r"\s+", " ", text)
return text
def normalize_predicate(value: str) -> str:
normalized = normalize_name(value).replace(" ", "_")
aliases = {
"依赖": "depends_on",
"需要": "depends_on",
"requires": "depends_on",
"调用": "uses",
"使用": "uses",
"uses": "uses",
"生成": "produces",
"输出": "produces",
"读取": "consumes",
"输入": "consumes",
"引用": "references",
}
return aliases.get(normalized, normalized or "related_to")
@@ -0,0 +1,351 @@
"""Safe, deterministic scanner for the knowledge-bearing part of a skill."""
from __future__ import annotations
import csv
import hashlib
import json
import re
import zipfile
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
CODE_SUFFIXES = {
".py",
".pyc",
".pyo",
".ts",
".tsx",
".js",
".jsx",
".mjs",
".cjs",
".sh",
".bash",
".zsh",
".fish",
".bat",
".cmd",
".ps1",
".psm1",
".java",
".kt",
".kts",
".go",
".rs",
".c",
".cc",
".cpp",
".h",
".hpp",
".cs",
".php",
".rb",
".pl",
".lua",
".swift",
".scala",
".sql",
".vue",
".svelte",
}
TEXT_SUFFIXES = {".md", ".mdx", ".txt", ".json", ".yaml", ".yml", ".toml", ".ini", ".csv", ".tsv"}
BINARY_KNOWLEDGE_SUFFIXES = {".xlsx", ".xls", ".pdf", ".docx", ".pptx", ".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".zip"}
EXCLUDED_DIRS = {".git", "node_modules", "__pycache__", ".pytest_cache", "dist", "build", ".mypy_cache", ".ruff_cache", ".address-edits"}
EXCLUDED_FILENAMES = {
"package-lock.json",
"pnpm-lock.yaml",
"yarn.lock",
"bun.lockb",
"poetry.lock",
"uv.lock",
"pipfile.lock",
"cargo.lock",
"composer.lock",
}
SENSITIVE_NAME_RE = re.compile(
r"(^|[._-])(\.env|id_rsa|id_dsa|id_ecdsa|id_ed25519|private[_-]?key|credentials?|secrets?)([._-]|$)",
re.IGNORECASE,
)
SECRET_CONTENT_RE = re.compile(r"(?im)^\s*(?:api[_-]?key|access[_-]?key|secret(?:[_-]?key)?|password|token|private[_-]?key)\s*[:=]\s*[^\s]{8,}\s*$")
PEM_PRIVATE_RE = re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")
FENCE_RE = re.compile(r"(?ms)^\s*(```|~~~).*?^\s*\1\s*$")
MARKDOWN_LINK_RE = re.compile(r"\[[^\]]+\]\(([^)#?]+)(?:#[^)]*)?\)")
MAX_TEXT_BYTES = 2 * 1024 * 1024
MAX_ARCHIVE_FILES = 1_000
MAX_ARCHIVE_ENTRY_BYTES = 20 * 1024 * 1024
MAX_ARCHIVE_TOTAL_BYTES = 200 * 1024 * 1024
@dataclass(slots=True)
class ScannedFile:
path: str
kind: str
sha256: str
size: int
text: str = ""
title: str = ""
references: list[dict[str, Any]] = field(default_factory=list)
safety: dict[str, Any] = field(default_factory=lambda: {"status": "allowed", "warnings": []})
def manifest(self) -> dict[str, Any]:
return {
"path": self.path,
"kind": self.kind,
"sha256": self.sha256,
"size": self.size,
"text_digest": f"sha256:{hashlib.sha256(self.text.encode('utf-8')).hexdigest()}" if self.text else None,
"title": self.title,
"references": self.references,
"safety": self.safety,
}
@dataclass(slots=True)
class ScanResult:
skill_name: str
source_root: str
source_digest: str
files: list[ScannedFile]
blocked_files: list[dict[str, str]]
skipped_code_count: int
skipped_other_count: int
@property
def manifest(self) -> list[dict[str, Any]]:
return [item.manifest() for item in self.files] + [{"path": item["path"], "kind": "blocked", "safety": {"status": "blocked", "warnings": [item["reason"]]}} for item in self.blocked_files]
@property
def counts(self) -> dict[str, int]:
return {
"knowledge_files": len(self.files),
"blocked_files": len(self.blocked_files),
"skipped_code_files": self.skipped_code_count,
"skipped_other_files": self.skipped_other_count,
}
def _safe_relative(path: Path, root: Path) -> str:
resolved = path.resolve()
resolved_root = root.resolve()
try:
relative = resolved.relative_to(resolved_root)
except ValueError as exc:
raise ValueError("Skill file escapes its source root") from exc
return relative.as_posix()
def _strip_markdown_code(text: str) -> str:
return FENCE_RE.sub("\n[示例代码已按安全策略省略]\n", text)
def _read_text(path: Path, suffix: str) -> str:
raw = path.read_bytes()
if len(raw) > MAX_TEXT_BYTES:
raise ValueError("text file exceeds 2 MiB parsing limit")
text = raw.decode("utf-8-sig", errors="replace")
if suffix in {".md", ".mdx"}:
return _strip_markdown_code(text)
if suffix in {".csv", ".tsv"}:
delimiter = "\t" if suffix == ".tsv" else ","
rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
return "\n".join(" | ".join(cell.strip() for cell in row) for row in rows[:500])
if suffix == ".json":
try:
return json.dumps(json.loads(text), ensure_ascii=False, indent=2)
except json.JSONDecodeError:
return text
return text
def _title(text: str, fallback: str) -> str:
match = re.search(r"(?m)^#\s+(.+?)\s*$", text)
return match.group(1).strip() if match else fallback
def _kind(suffix: str) -> str:
return {
".md": "markdown",
".mdx": "markdown",
".txt": "text",
".csv": "table",
".tsv": "table",
".xlsx": "spreadsheet",
".xls": "spreadsheet",
".pdf": "pdf",
".docx": "office",
".pptx": "office",
".png": "image",
".jpg": "image",
".jpeg": "image",
".webp": "image",
".gif": "image",
".bmp": "image",
".zip": "archive",
}.get(suffix, "structured_data")
def _zip_members(path: Path, archive_relative: str) -> tuple[list[ScannedFile], list[dict[str, str]], int, int]:
files: list[ScannedFile] = []
blocked: list[dict[str, str]] = []
skipped_code = 0
skipped_other = 0
with zipfile.ZipFile(path) as archive:
members = archive.infolist()
if len(members) > MAX_ARCHIVE_FILES:
return [], [{"path": archive_relative, "reason": "压缩包文件数超过安全上限"}], 0, 0
total_size = sum(member.file_size for member in members)
if total_size > MAX_ARCHIVE_TOTAL_BYTES:
return [], [{"path": archive_relative, "reason": "压缩包展开体积超过安全上限"}], 0, 0
for member in members:
member_path = Path(member.filename.replace("\\", "/"))
virtual_path = f"{archive_relative}!/{member_path.as_posix()}"
unix_mode = member.external_attr >> 16
if member.is_dir():
continue
if member_path.is_absolute() or ".." in member_path.parts or (unix_mode & 0o170000) == 0o120000:
blocked.append({"path": virtual_path, "reason": "压缩包包含路径穿越或符号链接"})
continue
suffix = member_path.suffix.lower()
lower_name = member_path.name.lower()
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
skipped_code += 1
continue
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
blocked.append({"path": virtual_path, "reason": "疑似密钥或敏感配置文件"})
continue
if suffix not in TEXT_SUFFIXES or member.file_size > MAX_ARCHIVE_ENTRY_BYTES:
skipped_other += 1
continue
raw = archive.read(member)
text = raw.decode("utf-8-sig", errors="replace")
if suffix in {".md", ".mdx"}:
text = _strip_markdown_code(text)
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
blocked.append({"path": virtual_path, "reason": "内容命中高置信敏感信息规则"})
continue
files.append(
ScannedFile(
path=virtual_path,
kind=_kind(suffix),
sha256=hashlib.sha256(raw).hexdigest(),
size=len(raw),
text=text,
title=_title(text, member_path.stem),
)
)
return files, blocked, skipped_code, skipped_other
def scan_skill(skill_name: str, skill_root: Path) -> ScanResult:
"""Scan one already-resolved skill directory without following symlinks."""
root = skill_root.resolve()
if not root.is_dir():
raise FileNotFoundError(f"Skill '{skill_name}' source directory does not exist")
files: list[ScannedFile] = []
blocked: list[dict[str, str]] = []
skipped_code = 0
skipped_other = 0
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix().lower()):
if path.is_symlink() or not path.is_file():
continue
relative = _safe_relative(path, root)
if any(part.lower() in EXCLUDED_DIRS for part in Path(relative).parts[:-1]):
continue
lower_name = path.name.lower()
suffix = path.suffix.lower()
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
skipped_code += 1
continue
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
blocked.append({"path": relative, "reason": "疑似密钥或敏感配置文件"})
continue
if suffix not in TEXT_SUFFIXES | BINARY_KNOWLEDGE_SUFFIXES:
skipped_other += 1
continue
# Never upload an archive wholesale: it may contain excluded code or a
# blocked secret. Only independently validated knowledge members enter
# the manifest and digest.
if suffix == ".zip":
try:
nested, nested_blocked, nested_code, nested_other = _zip_members(path, relative)
files.extend(nested)
blocked.extend(nested_blocked)
skipped_code += nested_code
skipped_other += nested_other
except (OSError, zipfile.BadZipFile):
blocked.append({"path": relative, "reason": "压缩包损坏或格式不受支持"})
continue
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
text = ""
warnings: list[str] = []
if suffix in TEXT_SUFFIXES:
try:
text = _read_text(path, suffix)
except ValueError as exc:
warnings.append(str(exc))
if text and (SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text)):
blocked.append({"path": relative, "reason": "内容命中高置信敏感信息规则"})
continue
references = []
if suffix in {".md", ".mdx"}:
for target in MARKDOWN_LINK_RE.findall(text):
normalized = target.replace("\\", "/").lstrip("./")
if normalized:
references.append({"target_path": normalized, "evidence": "markdown_link", "confidence": 1.0})
files.append(
ScannedFile(
path=relative,
kind=_kind(suffix),
sha256=digest,
size=len(raw),
text=text,
title=_title(text, path.stem),
references=references,
safety={"status": "allowed", "warnings": warnings},
)
)
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in files).encode("utf-8")
source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
return ScanResult(
skill_name=skill_name,
source_root=str(root),
source_digest=source_digest,
files=files,
blocked_files=blocked,
skipped_code_count=skipped_code,
skipped_other_count=skipped_other,
)
async def enrich_converted_documents(scan: ScanResult) -> ScanResult:
"""Extract Office/PDF/spreadsheet text without modifying the skill tree."""
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, extract_file_to_markdown
allowed: list[ScannedFile] = []
root = Path(scan.source_root)
for item in scan.files:
if Path(item.path).suffix.lower() in CONVERTIBLE_EXTENSIONS:
text = await extract_file_to_markdown(root / item.path)
if text:
text = _strip_markdown_code(text)
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
scan.blocked_files.append({"path": item.path, "reason": "转换文本命中高置信敏感信息规则"})
continue
item.text = text
item.title = _title(text, item.title)
else:
item.safety["warnings"].append("文档转换不可用,仅保留原始附件")
allowed.append(item)
scan.files = allowed
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in scan.files).encode("utf-8")
scan.source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
return scan