feat: add skill and assistant knowledge bases
This commit is contained in:
1 parent
dd0ee282df
commit
fbd8dec48f
45 files changed
+5262
-21
No files matched your search
+10
@@ -0,0 +1,10 @@
|
||||
"""Skill knowledge distillation primitives.
|
||||
|
||||
The package is deliberately independent from ``app.*``. Gateway composition
|
||||
injects persistence and remote adapters at runtime.
|
||||
"""
|
||||
|
||||
from deerflow.skill_knowledge.extractor import build_skill_ir
|
||||
from deerflow.skill_knowledge.scanner import ScanResult, enrich_converted_documents, scan_skill
|
||||
|
||||
__all__ = ["ScanResult", "build_skill_ir", "enrich_converted_documents", "scan_skill"]
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
"""IR construction and validation.
|
||||
|
||||
V1 intentionally starts with deterministic, evidence-backed extraction. The
|
||||
schema is compatible with a future model-backed extractor, while never letting
|
||||
unvalidated model output flow directly into persistence or graph writers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.skill_knowledge.scanner import ScanResult
|
||||
|
||||
EXTRACTOR_VERSION = "skill-knowledge-v1"
|
||||
PROMPT_HASH = f"sha256:{hashlib.sha256(b'skill-knowledge-v1-evidence-only').hexdigest()}"
|
||||
PREDICATE_PATTERNS = {
|
||||
"depends_on": re.compile(r"(?:依赖|需要|requires?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
|
||||
"uses": re.compile(r"(?:调用|使用|uses?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
|
||||
"produces": re.compile(r"(?:生成|输出|produces?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
|
||||
"consumes": re.compile(r"(?:读取|输入|consumes?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
|
||||
}
|
||||
|
||||
|
||||
def _evidence(path: str, text: str, start: int, end: int) -> dict[str, Any]:
|
||||
line_start = text.count("\n", 0, start) + 1
|
||||
line_end = text.count("\n", 0, end) + 1
|
||||
quote = " ".join(text[start:end].split())[:240]
|
||||
return {"file_path": path, "line_start": line_start, "line_end": line_end, "quote": quote}
|
||||
|
||||
|
||||
def build_skill_ir(scan: ScanResult, *, display_name: str | None = None, description: str = "") -> dict[str, Any]:
|
||||
entities: list[dict[str, Any]] = [
|
||||
{
|
||||
"local_id": "ent_skill",
|
||||
"type": "skill",
|
||||
"name": display_name or scan.skill_name,
|
||||
"aliases": [scan.skill_name] if display_name and display_name != scan.skill_name else [],
|
||||
"description": description,
|
||||
"attributes": {},
|
||||
"evidence": [],
|
||||
"confidence": 1.0,
|
||||
}
|
||||
]
|
||||
relations: list[dict[str, Any]] = []
|
||||
by_key: dict[tuple[str, str], str] = {("skill", (display_name or scan.skill_name).casefold()): "ent_skill"}
|
||||
|
||||
for file in scan.files:
|
||||
if not file.text:
|
||||
continue
|
||||
for predicate, pattern in PREDICATE_PATTERNS.items():
|
||||
for match in pattern.finditer(file.text):
|
||||
name = match.group(1).strip(" `\"'“”()[]")
|
||||
if not name or len(name) > 80:
|
||||
continue
|
||||
key = ("concept", name.casefold())
|
||||
entity_id = by_key.get(key)
|
||||
if entity_id is None:
|
||||
entity_id = f"ent_{len(entities):04d}"
|
||||
by_key[key] = entity_id
|
||||
entities.append(
|
||||
{
|
||||
"local_id": entity_id,
|
||||
"type": "concept",
|
||||
"name": name,
|
||||
"aliases": [],
|
||||
"description": "从技能业务说明中抽取的知识对象",
|
||||
"attributes": {},
|
||||
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
|
||||
"confidence": 0.82,
|
||||
}
|
||||
)
|
||||
relations.append(
|
||||
{
|
||||
"local_id": f"rel_{len(relations):04d}",
|
||||
"source_local_id": "ent_skill",
|
||||
"predicate": predicate,
|
||||
"target_local_id": entity_id,
|
||||
"description": " ".join(match.group(0).split())[:240],
|
||||
"relation_source": "explicit_text",
|
||||
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
|
||||
"confidence": 0.82,
|
||||
}
|
||||
)
|
||||
for ref in file.references:
|
||||
name = ref["target_path"]
|
||||
key = ("file", name.casefold())
|
||||
entity_id = by_key.get(key)
|
||||
if entity_id is None:
|
||||
entity_id = f"ent_{len(entities):04d}"
|
||||
by_key[key] = entity_id
|
||||
entities.append(
|
||||
{
|
||||
"local_id": entity_id,
|
||||
"type": "file",
|
||||
"name": name,
|
||||
"aliases": [],
|
||||
"description": "技能文档引用的资料文件",
|
||||
"attributes": {},
|
||||
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
|
||||
"confidence": 1.0,
|
||||
}
|
||||
)
|
||||
relations.append(
|
||||
{
|
||||
"local_id": f"rel_{len(relations):04d}",
|
||||
"source_local_id": "ent_skill",
|
||||
"predicate": "references",
|
||||
"target_local_id": entity_id,
|
||||
"description": f"{file.path} 引用 {name}",
|
||||
"relation_source": "deterministic_structure",
|
||||
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
|
||||
"confidence": 1.0,
|
||||
}
|
||||
)
|
||||
|
||||
title = display_name or scan.skill_name
|
||||
summaries = [f"- **{item.title}**(`{item.path}`)" for item in scan.files[:100]]
|
||||
content = "\n".join(
|
||||
[
|
||||
f"# {title}",
|
||||
"",
|
||||
description or "由技能知识文件安全归纳生成。",
|
||||
"",
|
||||
"## 知识资料",
|
||||
"",
|
||||
*(summaries or ["- 暂无可归纳知识文件"]),
|
||||
"",
|
||||
f"> 源摘要:`{scan.source_digest}`;代码与脚本文件未读取。",
|
||||
]
|
||||
)
|
||||
return {
|
||||
"extractor_version": EXTRACTOR_VERSION,
|
||||
"prompt_hash": PROMPT_HASH,
|
||||
"model_name": "deterministic-v1",
|
||||
"skill": {
|
||||
"name": scan.skill_name,
|
||||
"display_name": title,
|
||||
"description": description,
|
||||
"source_root": scan.source_root,
|
||||
"source_digest": scan.source_digest,
|
||||
},
|
||||
"files": scan.manifest,
|
||||
"entities": entities,
|
||||
"relations": relations,
|
||||
"wiki": {
|
||||
"skill_page": {"slug": f"skills/{scan.skill_name}/overview", "title": title, "content": content},
|
||||
"pages": [],
|
||||
},
|
||||
"safety": {"blocked_files": scan.blocked_files},
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
"""Guarded graph-write plans.
|
||||
|
||||
No caller can submit Cypher. The writer accepts only normalized node/edge
|
||||
records and executes fixed parameterized templates when explicitly enabled.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
class GraphTransaction(Protocol):
|
||||
async def run(self, query: str, parameters: dict[str, Any]) -> Any: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GraphWritePlan:
|
||||
scope: str
|
||||
nodes: tuple[dict[str, Any], ...]
|
||||
edges: tuple[dict[str, Any], ...]
|
||||
|
||||
|
||||
NODE_QUERY = """
|
||||
MERGE (n:DeerFlowKnowledge {scope: $scope, entity_key: $entity_key})
|
||||
SET n.display_name = $display_name, n.entity_type = $entity_type,
|
||||
n.binding_id = $binding_id, n.snapshot_id = $snapshot_id, n.job_id = $job_id,
|
||||
n.confidence = $confidence, n.updated_at = datetime()
|
||||
""".strip()
|
||||
EDGE_QUERY = """
|
||||
MATCH (a:DeerFlowKnowledge {scope: $scope, entity_key: $source_key})
|
||||
MATCH (b:DeerFlowKnowledge {scope: $scope, entity_key: $target_key})
|
||||
MERGE (a)-[r:DEERFLOW_RELATION {edge_key: $edge_key}]->(b)
|
||||
SET r.predicate = $predicate, r.binding_id = $binding_id,
|
||||
r.snapshot_id = $snapshot_id, r.job_id = $job_id, r.confidence = $confidence,
|
||||
r.evidence = $evidence, r.updated_at = datetime()
|
||||
""".strip()
|
||||
|
||||
|
||||
def _key(*parts: str) -> str:
|
||||
return hashlib.sha256("\x1f".join(parts).encode()).hexdigest()
|
||||
|
||||
|
||||
def plan_from_ir(
|
||||
ir: dict[str, Any],
|
||||
*,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
binding_id: str,
|
||||
snapshot_id: str,
|
||||
job_id: str,
|
||||
) -> GraphWritePlan:
|
||||
"""Build a stable write plan; callers cannot supply executable Cypher."""
|
||||
scope = f"{target_type}:{target_id}"
|
||||
local_keys: dict[str, str] = {}
|
||||
nodes: list[dict[str, Any]] = []
|
||||
for entity in ir.get("entities") or []:
|
||||
entity_type = str(entity.get("type") or "concept")
|
||||
normalized_name = str(entity.get("normalized_name") or entity.get("name") or "").strip().casefold()
|
||||
if not normalized_name:
|
||||
continue
|
||||
entity_key = _key(entity_type, normalized_name)
|
||||
local_keys[str(entity.get("local_id") or normalized_name)] = entity_key
|
||||
nodes.append(
|
||||
{
|
||||
"entity_key": entity_key,
|
||||
"display_name": str(entity.get("name") or normalized_name),
|
||||
"entity_type": entity_type,
|
||||
"binding_id": binding_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"job_id": job_id,
|
||||
"confidence": float(entity.get("confidence") or 1.0),
|
||||
}
|
||||
)
|
||||
edges: list[dict[str, Any]] = []
|
||||
for relation in ir.get("relations") or []:
|
||||
source_key = local_keys.get(str(relation.get("source_local_id") or ""))
|
||||
target_key = local_keys.get(str(relation.get("target_local_id") or ""))
|
||||
evidence = list(relation.get("evidence") or [])
|
||||
confidence = float(relation.get("confidence") or 0)
|
||||
predicate = str(relation.get("predicate") or "related_to")
|
||||
if not source_key or not target_key or confidence < 0.75 or not evidence:
|
||||
continue
|
||||
edges.append(
|
||||
{
|
||||
"edge_key": _key(source_key, predicate, target_key, binding_id, snapshot_id),
|
||||
"source_key": source_key,
|
||||
"target_key": target_key,
|
||||
"predicate": predicate,
|
||||
"binding_id": binding_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"job_id": job_id,
|
||||
"confidence": confidence,
|
||||
"evidence": evidence,
|
||||
}
|
||||
)
|
||||
return GraphWritePlan(scope=scope, nodes=tuple(nodes), edges=tuple(edges))
|
||||
|
||||
|
||||
def parameterized_statements(plan: GraphWritePlan) -> list[dict[str, Any]]:
|
||||
"""Return Neo4j transactional HTTP statements using only fixed templates."""
|
||||
statements = [{"statement": NODE_QUERY, "parameters": {"scope": plan.scope, **node}} for node in plan.nodes]
|
||||
statements.extend({"statement": EDGE_QUERY, "parameters": {"scope": plan.scope, **edge}} for edge in plan.edges)
|
||||
return statements
|
||||
|
||||
|
||||
async def execute_plan(tx: GraphTransaction, plan: GraphWritePlan, *, enabled: bool) -> dict[str, int]:
|
||||
if not enabled:
|
||||
return {"nodes": 0, "edges": 0}
|
||||
for node in plan.nodes:
|
||||
await tx.run(NODE_QUERY, {"scope": plan.scope, **node})
|
||||
for edge in plan.edges:
|
||||
await tx.run(EDGE_QUERY, {"scope": plan.scope, **edge})
|
||||
return {"nodes": len(plan.nodes), "edges": len(plan.edges)}
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Portable entity/relation normalization used by both knowledge products."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
|
||||
def normalize_name(value: str) -> str:
|
||||
text = unicodedata.normalize("NFKC", value or "").strip().casefold()
|
||||
text = text.strip("`'\"“”‘’《》〈〉[](){}")
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
return text
|
||||
|
||||
|
||||
def normalize_predicate(value: str) -> str:
|
||||
normalized = normalize_name(value).replace(" ", "_")
|
||||
aliases = {
|
||||
"依赖": "depends_on",
|
||||
"需要": "depends_on",
|
||||
"requires": "depends_on",
|
||||
"调用": "uses",
|
||||
"使用": "uses",
|
||||
"uses": "uses",
|
||||
"生成": "produces",
|
||||
"输出": "produces",
|
||||
"读取": "consumes",
|
||||
"输入": "consumes",
|
||||
"引用": "references",
|
||||
}
|
||||
return aliases.get(normalized, normalized or "related_to")
|
||||
+351
@@ -0,0 +1,351 @@
|
||||
"""Safe, deterministic scanner for the knowledge-bearing part of a skill."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
CODE_SUFFIXES = {
|
||||
".py",
|
||||
".pyc",
|
||||
".pyo",
|
||||
".ts",
|
||||
".tsx",
|
||||
".js",
|
||||
".jsx",
|
||||
".mjs",
|
||||
".cjs",
|
||||
".sh",
|
||||
".bash",
|
||||
".zsh",
|
||||
".fish",
|
||||
".bat",
|
||||
".cmd",
|
||||
".ps1",
|
||||
".psm1",
|
||||
".java",
|
||||
".kt",
|
||||
".kts",
|
||||
".go",
|
||||
".rs",
|
||||
".c",
|
||||
".cc",
|
||||
".cpp",
|
||||
".h",
|
||||
".hpp",
|
||||
".cs",
|
||||
".php",
|
||||
".rb",
|
||||
".pl",
|
||||
".lua",
|
||||
".swift",
|
||||
".scala",
|
||||
".sql",
|
||||
".vue",
|
||||
".svelte",
|
||||
}
|
||||
TEXT_SUFFIXES = {".md", ".mdx", ".txt", ".json", ".yaml", ".yml", ".toml", ".ini", ".csv", ".tsv"}
|
||||
BINARY_KNOWLEDGE_SUFFIXES = {".xlsx", ".xls", ".pdf", ".docx", ".pptx", ".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".zip"}
|
||||
EXCLUDED_DIRS = {".git", "node_modules", "__pycache__", ".pytest_cache", "dist", "build", ".mypy_cache", ".ruff_cache", ".address-edits"}
|
||||
EXCLUDED_FILENAMES = {
|
||||
"package-lock.json",
|
||||
"pnpm-lock.yaml",
|
||||
"yarn.lock",
|
||||
"bun.lockb",
|
||||
"poetry.lock",
|
||||
"uv.lock",
|
||||
"pipfile.lock",
|
||||
"cargo.lock",
|
||||
"composer.lock",
|
||||
}
|
||||
SENSITIVE_NAME_RE = re.compile(
|
||||
r"(^|[._-])(\.env|id_rsa|id_dsa|id_ecdsa|id_ed25519|private[_-]?key|credentials?|secrets?)([._-]|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
SECRET_CONTENT_RE = re.compile(r"(?im)^\s*(?:api[_-]?key|access[_-]?key|secret(?:[_-]?key)?|password|token|private[_-]?key)\s*[:=]\s*[^\s]{8,}\s*$")
|
||||
PEM_PRIVATE_RE = re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")
|
||||
FENCE_RE = re.compile(r"(?ms)^\s*(```|~~~).*?^\s*\1\s*$")
|
||||
MARKDOWN_LINK_RE = re.compile(r"\[[^\]]+\]\(([^)#?]+)(?:#[^)]*)?\)")
|
||||
MAX_TEXT_BYTES = 2 * 1024 * 1024
|
||||
MAX_ARCHIVE_FILES = 1_000
|
||||
MAX_ARCHIVE_ENTRY_BYTES = 20 * 1024 * 1024
|
||||
MAX_ARCHIVE_TOTAL_BYTES = 200 * 1024 * 1024
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ScannedFile:
|
||||
path: str
|
||||
kind: str
|
||||
sha256: str
|
||||
size: int
|
||||
text: str = ""
|
||||
title: str = ""
|
||||
references: list[dict[str, Any]] = field(default_factory=list)
|
||||
safety: dict[str, Any] = field(default_factory=lambda: {"status": "allowed", "warnings": []})
|
||||
|
||||
def manifest(self) -> dict[str, Any]:
|
||||
return {
|
||||
"path": self.path,
|
||||
"kind": self.kind,
|
||||
"sha256": self.sha256,
|
||||
"size": self.size,
|
||||
"text_digest": f"sha256:{hashlib.sha256(self.text.encode('utf-8')).hexdigest()}" if self.text else None,
|
||||
"title": self.title,
|
||||
"references": self.references,
|
||||
"safety": self.safety,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ScanResult:
|
||||
skill_name: str
|
||||
source_root: str
|
||||
source_digest: str
|
||||
files: list[ScannedFile]
|
||||
blocked_files: list[dict[str, str]]
|
||||
skipped_code_count: int
|
||||
skipped_other_count: int
|
||||
|
||||
@property
|
||||
def manifest(self) -> list[dict[str, Any]]:
|
||||
return [item.manifest() for item in self.files] + [{"path": item["path"], "kind": "blocked", "safety": {"status": "blocked", "warnings": [item["reason"]]}} for item in self.blocked_files]
|
||||
|
||||
@property
|
||||
def counts(self) -> dict[str, int]:
|
||||
return {
|
||||
"knowledge_files": len(self.files),
|
||||
"blocked_files": len(self.blocked_files),
|
||||
"skipped_code_files": self.skipped_code_count,
|
||||
"skipped_other_files": self.skipped_other_count,
|
||||
}
|
||||
|
||||
|
||||
def _safe_relative(path: Path, root: Path) -> str:
|
||||
resolved = path.resolve()
|
||||
resolved_root = root.resolve()
|
||||
try:
|
||||
relative = resolved.relative_to(resolved_root)
|
||||
except ValueError as exc:
|
||||
raise ValueError("Skill file escapes its source root") from exc
|
||||
return relative.as_posix()
|
||||
|
||||
|
||||
def _strip_markdown_code(text: str) -> str:
|
||||
return FENCE_RE.sub("\n[示例代码已按安全策略省略]\n", text)
|
||||
|
||||
|
||||
def _read_text(path: Path, suffix: str) -> str:
|
||||
raw = path.read_bytes()
|
||||
if len(raw) > MAX_TEXT_BYTES:
|
||||
raise ValueError("text file exceeds 2 MiB parsing limit")
|
||||
text = raw.decode("utf-8-sig", errors="replace")
|
||||
if suffix in {".md", ".mdx"}:
|
||||
return _strip_markdown_code(text)
|
||||
if suffix in {".csv", ".tsv"}:
|
||||
delimiter = "\t" if suffix == ".tsv" else ","
|
||||
rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
|
||||
return "\n".join(" | ".join(cell.strip() for cell in row) for row in rows[:500])
|
||||
if suffix == ".json":
|
||||
try:
|
||||
return json.dumps(json.loads(text), ensure_ascii=False, indent=2)
|
||||
except json.JSONDecodeError:
|
||||
return text
|
||||
return text
|
||||
|
||||
|
||||
def _title(text: str, fallback: str) -> str:
|
||||
match = re.search(r"(?m)^#\s+(.+?)\s*$", text)
|
||||
return match.group(1).strip() if match else fallback
|
||||
|
||||
|
||||
def _kind(suffix: str) -> str:
|
||||
return {
|
||||
".md": "markdown",
|
||||
".mdx": "markdown",
|
||||
".txt": "text",
|
||||
".csv": "table",
|
||||
".tsv": "table",
|
||||
".xlsx": "spreadsheet",
|
||||
".xls": "spreadsheet",
|
||||
".pdf": "pdf",
|
||||
".docx": "office",
|
||||
".pptx": "office",
|
||||
".png": "image",
|
||||
".jpg": "image",
|
||||
".jpeg": "image",
|
||||
".webp": "image",
|
||||
".gif": "image",
|
||||
".bmp": "image",
|
||||
".zip": "archive",
|
||||
}.get(suffix, "structured_data")
|
||||
|
||||
|
||||
def _zip_members(path: Path, archive_relative: str) -> tuple[list[ScannedFile], list[dict[str, str]], int, int]:
|
||||
files: list[ScannedFile] = []
|
||||
blocked: list[dict[str, str]] = []
|
||||
skipped_code = 0
|
||||
skipped_other = 0
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
members = archive.infolist()
|
||||
if len(members) > MAX_ARCHIVE_FILES:
|
||||
return [], [{"path": archive_relative, "reason": "压缩包文件数超过安全上限"}], 0, 0
|
||||
total_size = sum(member.file_size for member in members)
|
||||
if total_size > MAX_ARCHIVE_TOTAL_BYTES:
|
||||
return [], [{"path": archive_relative, "reason": "压缩包展开体积超过安全上限"}], 0, 0
|
||||
for member in members:
|
||||
member_path = Path(member.filename.replace("\\", "/"))
|
||||
virtual_path = f"{archive_relative}!/{member_path.as_posix()}"
|
||||
unix_mode = member.external_attr >> 16
|
||||
if member.is_dir():
|
||||
continue
|
||||
if member_path.is_absolute() or ".." in member_path.parts or (unix_mode & 0o170000) == 0o120000:
|
||||
blocked.append({"path": virtual_path, "reason": "压缩包包含路径穿越或符号链接"})
|
||||
continue
|
||||
suffix = member_path.suffix.lower()
|
||||
lower_name = member_path.name.lower()
|
||||
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
|
||||
skipped_code += 1
|
||||
continue
|
||||
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
|
||||
blocked.append({"path": virtual_path, "reason": "疑似密钥或敏感配置文件"})
|
||||
continue
|
||||
if suffix not in TEXT_SUFFIXES or member.file_size > MAX_ARCHIVE_ENTRY_BYTES:
|
||||
skipped_other += 1
|
||||
continue
|
||||
raw = archive.read(member)
|
||||
text = raw.decode("utf-8-sig", errors="replace")
|
||||
if suffix in {".md", ".mdx"}:
|
||||
text = _strip_markdown_code(text)
|
||||
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
|
||||
blocked.append({"path": virtual_path, "reason": "内容命中高置信敏感信息规则"})
|
||||
continue
|
||||
files.append(
|
||||
ScannedFile(
|
||||
path=virtual_path,
|
||||
kind=_kind(suffix),
|
||||
sha256=hashlib.sha256(raw).hexdigest(),
|
||||
size=len(raw),
|
||||
text=text,
|
||||
title=_title(text, member_path.stem),
|
||||
)
|
||||
)
|
||||
return files, blocked, skipped_code, skipped_other
|
||||
|
||||
|
||||
def scan_skill(skill_name: str, skill_root: Path) -> ScanResult:
|
||||
"""Scan one already-resolved skill directory without following symlinks."""
|
||||
|
||||
root = skill_root.resolve()
|
||||
if not root.is_dir():
|
||||
raise FileNotFoundError(f"Skill '{skill_name}' source directory does not exist")
|
||||
files: list[ScannedFile] = []
|
||||
blocked: list[dict[str, str]] = []
|
||||
skipped_code = 0
|
||||
skipped_other = 0
|
||||
|
||||
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix().lower()):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
continue
|
||||
relative = _safe_relative(path, root)
|
||||
if any(part.lower() in EXCLUDED_DIRS for part in Path(relative).parts[:-1]):
|
||||
continue
|
||||
lower_name = path.name.lower()
|
||||
suffix = path.suffix.lower()
|
||||
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
|
||||
skipped_code += 1
|
||||
continue
|
||||
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
|
||||
blocked.append({"path": relative, "reason": "疑似密钥或敏感配置文件"})
|
||||
continue
|
||||
if suffix not in TEXT_SUFFIXES | BINARY_KNOWLEDGE_SUFFIXES:
|
||||
skipped_other += 1
|
||||
continue
|
||||
|
||||
# Never upload an archive wholesale: it may contain excluded code or a
|
||||
# blocked secret. Only independently validated knowledge members enter
|
||||
# the manifest and digest.
|
||||
if suffix == ".zip":
|
||||
try:
|
||||
nested, nested_blocked, nested_code, nested_other = _zip_members(path, relative)
|
||||
files.extend(nested)
|
||||
blocked.extend(nested_blocked)
|
||||
skipped_code += nested_code
|
||||
skipped_other += nested_other
|
||||
except (OSError, zipfile.BadZipFile):
|
||||
blocked.append({"path": relative, "reason": "压缩包损坏或格式不受支持"})
|
||||
continue
|
||||
|
||||
raw = path.read_bytes()
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
text = ""
|
||||
warnings: list[str] = []
|
||||
if suffix in TEXT_SUFFIXES:
|
||||
try:
|
||||
text = _read_text(path, suffix)
|
||||
except ValueError as exc:
|
||||
warnings.append(str(exc))
|
||||
if text and (SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text)):
|
||||
blocked.append({"path": relative, "reason": "内容命中高置信敏感信息规则"})
|
||||
continue
|
||||
references = []
|
||||
if suffix in {".md", ".mdx"}:
|
||||
for target in MARKDOWN_LINK_RE.findall(text):
|
||||
normalized = target.replace("\\", "/").lstrip("./")
|
||||
if normalized:
|
||||
references.append({"target_path": normalized, "evidence": "markdown_link", "confidence": 1.0})
|
||||
files.append(
|
||||
ScannedFile(
|
||||
path=relative,
|
||||
kind=_kind(suffix),
|
||||
sha256=digest,
|
||||
size=len(raw),
|
||||
text=text,
|
||||
title=_title(text, path.stem),
|
||||
references=references,
|
||||
safety={"status": "allowed", "warnings": warnings},
|
||||
)
|
||||
)
|
||||
|
||||
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in files).encode("utf-8")
|
||||
source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
|
||||
return ScanResult(
|
||||
skill_name=skill_name,
|
||||
source_root=str(root),
|
||||
source_digest=source_digest,
|
||||
files=files,
|
||||
blocked_files=blocked,
|
||||
skipped_code_count=skipped_code,
|
||||
skipped_other_count=skipped_other,
|
||||
)
|
||||
|
||||
|
||||
async def enrich_converted_documents(scan: ScanResult) -> ScanResult:
|
||||
"""Extract Office/PDF/spreadsheet text without modifying the skill tree."""
|
||||
|
||||
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, extract_file_to_markdown
|
||||
|
||||
allowed: list[ScannedFile] = []
|
||||
root = Path(scan.source_root)
|
||||
for item in scan.files:
|
||||
if Path(item.path).suffix.lower() in CONVERTIBLE_EXTENSIONS:
|
||||
text = await extract_file_to_markdown(root / item.path)
|
||||
if text:
|
||||
text = _strip_markdown_code(text)
|
||||
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
|
||||
scan.blocked_files.append({"path": item.path, "reason": "转换文本命中高置信敏感信息规则"})
|
||||
continue
|
||||
item.text = text
|
||||
item.title = _title(text, item.title)
|
||||
else:
|
||||
item.safety["warnings"].append("文档转换不可用,仅保留原始附件")
|
||||
allowed.append(item)
|
||||
scan.files = allowed
|
||||
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in scan.files).encode("utf-8")
|
||||
scan.source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
|
||||
return scan
|
||||
Reference in new issue
Block a user