初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,5 @@
"""Assistant knowledge package helpers."""
from deerflow.assistant_knowledge.importer import export_package_from_skill_ir
__all__ = ["export_package_from_skill_ir"]
@@ -0,0 +1,79 @@
"""Disk-backed, replayable package records. Memory is bounded by one JSONL record.
The staging index is local SQLite, never a remote provider's database. It also
allows restoration to fetch only the vectors belonging to the current page.
"""
from __future__ import annotations
import json
import sqlite3
import tempfile
from collections.abc import Iterator
from contextlib import closing
from pathlib import Path
from typing import Any
class Records:
def __init__(self, path: Path, kind: str, slug: str | None = None):
self.path, self.kind, self.slug = path, kind, slug
def _where(self):
return ("kind=?", (self.kind,)) if self.slug is None else ("kind=? AND slug=?", (self.kind, self.slug))
def __iter__(self) -> Iterator[dict[str, Any]]:
where, args = self._where()
with closing(sqlite3.connect(self.path)) as db:
for (payload,) in db.execute(f"SELECT payload FROM records WHERE {where} ORDER BY seq", args):
yield json.loads(payload)
def __len__(self) -> int:
where, args = self._where()
with closing(sqlite3.connect(self.path)) as db:
return db.execute(f"SELECT count(*) FROM records WHERE {where}", args).fetchone()[0]
class PackageArchive(dict):
def __init__(self, path: Path):
super().__init__()
temporary = tempfile.NamedTemporaryFile(prefix="wiki-package-", suffix=".sqlite", dir=path.parent, delete=False)
self.path = Path(temporary.name)
temporary.close()
try:
with closing(sqlite3.connect(self.path)) as db, path.open(encoding="utf-8-sig") as handle:
db.execute("CREATE TABLE records(seq INTEGER PRIMARY KEY, kind TEXT, slug TEXT, payload TEXT)")
for line_no, line in enumerate(handle, 1):
if not line.strip():
continue
record = json.loads(line)
kind, payload = record.get("record_type"), record.get("payload")
if not isinstance(payload, dict):
raise ValueError(f"导出包第 {line_no} 行格式错误")
if kind == "source":
self["source"] = {**self.get("source", {}), **payload}
elif kind == "graph":
for key, subkind in (("nodes", "graph_node"), ("edges", "graph_edge")):
db.executemany("INSERT INTO records(kind, payload) VALUES (?, ?)", [(subkind, json.dumps(v, ensure_ascii=False)) for v in payload.get(key) or []])
elif kind in {"wiki_page", "document", "chunk", "vector", "entity", "relation"}:
slug = str(payload.get("wiki_slug") or payload.get("page_slug") or payload.get("slug") or "").strip("/")
db.execute("INSERT INTO records(kind, slug, payload) VALUES (?, ?, ?)", (kind, slug, json.dumps(payload, ensure_ascii=False)))
else:
raise ValueError(f"不支持的导出记录类型:{kind}")
db.execute("CREATE INDEX by_kind_slug ON records(kind, slug)")
db.commit()
source = self.get("source", {})
if source.get("package_version") != 1 or (source.get("provider") or source.get("type")) not in {"weknora", "assistant"}:
raise ValueError("仅接受 WeKnora / 助手知识库生成的兼容 Wiki 向量包(v1)")
for key, kind in (("wiki_pages", "wiki_page"), ("documents", "document"), ("chunks", "chunk"), ("vectors", "vector"), ("entities", "entity"), ("relations", "relation")):
self[key] = Records(self.path, kind)
self["graph"] = {"nodes": Records(self.path, "graph_node"), "edges": Records(self.path, "graph_edge")}
except BaseException:
self.close()
raise
def vectors_for(self, slug: str) -> Records:
return Records(self.path, "vector", slug)
def close(self):
self.path.unlink(missing_ok=True)
@@ -0,0 +1,22 @@
"""Normalize heterogeneous sources into the assistant import-package shape."""
from __future__ import annotations
from typing import Any
def export_package_from_skill_ir(ir: dict[str, Any]) -> dict[str, Any]:
skill = ir.get("skill") or {}
return {
"source": {
"type": "skill",
"skill_name": skill.get("name"),
"source_digest": skill.get("source_digest"),
},
"wiki_pages": [ir.get("wiki", {}).get("skill_page")],
"documents": ir.get("files") or [],
"chunks": [],
"entities": ir.get("entities") or [],
"relations": ir.get("relations") or [],
"graph": {"nodes": ir.get("entities") or [], "edges": ir.get("relations") or []},
}
@@ -0,0 +1,109 @@
"""File-backed assistant knowledge import/export packages.
The assistant knowledge base is fed by WeKnora-exported data. Keep that
boundary explicit: a WeKnora export is first materialized as a durable JSONL
package, then imported asynchronously from that package. JSONL keeps large
exports append-friendly and avoids treating the HTTP request body as the job
state.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import Any
PACKAGE_VERSION = 1
PACKAGE_RECORD_TYPES = {
"source",
"wiki_page",
"document",
"chunk",
"vector",
"entity",
"relation",
"graph",
}
def package_path(root: Path, job_id: str) -> Path:
safe = "".join(ch for ch in str(job_id) if ch.isalnum() or ch in {"-", "_"})
return root / f"{safe}.jsonl"
def write_jsonl_record(handle: Any, record_type: str, payload: Mapping[str, Any]) -> None:
handle.write(json.dumps({"record_type": record_type, "payload": dict(payload)}, ensure_ascii=False))
handle.write("\n")
def write_package(path: Path, records: Iterable[tuple[str, Mapping[str, Any]]]) -> dict[str, int]:
path.parent.mkdir(parents=True, exist_ok=True)
counts: dict[str, int] = {key: 0 for key in PACKAGE_RECORD_TYPES}
tmp_path = path.with_suffix(path.suffix + ".tmp")
with tmp_path.open("w", encoding="utf-8", newline="\n") as handle:
write_jsonl_record(
handle,
"source",
{
"package_version": PACKAGE_VERSION,
},
)
counts["source"] += 1
for record_type, payload in records:
if record_type not in PACKAGE_RECORD_TYPES:
continue
write_jsonl_record(handle, record_type, payload)
counts[record_type] = counts.get(record_type, 0) + 1
tmp_path.replace(path)
return counts
def read_package(path: Path) -> dict[str, Any]:
package: dict[str, Any] = {
"source": {},
"wiki_pages": [],
"documents": [],
"chunks": [],
"vectors": [],
"entities": [],
"relations": [],
"graph": {"nodes": [], "edges": []},
}
with path.open("r", encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line:
continue
record = json.loads(line)
if not isinstance(record, dict):
continue
record_type = str(record.get("record_type") or "")
payload = record.get("payload")
if not isinstance(payload, dict):
continue
if record_type == "source":
package["source"] = {**package.get("source", {}), **payload}
elif record_type == "wiki_page":
package["wiki_pages"].append(payload)
elif record_type == "document":
package["documents"].append(payload)
elif record_type == "chunk":
package["chunks"].append(payload)
elif record_type == "vector":
package["vectors"].append(payload)
elif record_type == "entity":
package["entities"].append(payload)
elif record_type == "relation":
package["relations"].append(payload)
elif record_type == "graph":
nodes = payload.get("nodes")
edges = payload.get("edges")
if isinstance(nodes, list):
package["graph"]["nodes"].extend(item for item in nodes if isinstance(item, dict))
if isinstance(edges, list):
package["graph"]["edges"].extend(item for item in edges if isinstance(item, dict))
for key, value in payload.items():
if key not in {"nodes", "edges"}:
package["graph"][key] = value
return package