初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
+5
@@ -0,0 +1,5 @@
|
||||
"""Assistant knowledge package helpers."""
|
||||
|
||||
from deerflow.assistant_knowledge.importer import export_package_from_skill_ir
|
||||
|
||||
__all__ = ["export_package_from_skill_ir"]
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
"""Disk-backed, replayable package records. Memory is bounded by one JSONL record.
|
||||
|
||||
The staging index is local SQLite, never a remote provider's database. It also
|
||||
allows restoration to fetch only the vectors belonging to the current page.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from collections.abc import Iterator
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
class Records:
|
||||
def __init__(self, path: Path, kind: str, slug: str | None = None):
|
||||
self.path, self.kind, self.slug = path, kind, slug
|
||||
|
||||
def _where(self):
|
||||
return ("kind=?", (self.kind,)) if self.slug is None else ("kind=? AND slug=?", (self.kind, self.slug))
|
||||
|
||||
def __iter__(self) -> Iterator[dict[str, Any]]:
|
||||
where, args = self._where()
|
||||
with closing(sqlite3.connect(self.path)) as db:
|
||||
for (payload,) in db.execute(f"SELECT payload FROM records WHERE {where} ORDER BY seq", args):
|
||||
yield json.loads(payload)
|
||||
|
||||
def __len__(self) -> int:
|
||||
where, args = self._where()
|
||||
with closing(sqlite3.connect(self.path)) as db:
|
||||
return db.execute(f"SELECT count(*) FROM records WHERE {where}", args).fetchone()[0]
|
||||
|
||||
|
||||
class PackageArchive(dict):
|
||||
def __init__(self, path: Path):
|
||||
super().__init__()
|
||||
temporary = tempfile.NamedTemporaryFile(prefix="wiki-package-", suffix=".sqlite", dir=path.parent, delete=False)
|
||||
self.path = Path(temporary.name)
|
||||
temporary.close()
|
||||
try:
|
||||
with closing(sqlite3.connect(self.path)) as db, path.open(encoding="utf-8-sig") as handle:
|
||||
db.execute("CREATE TABLE records(seq INTEGER PRIMARY KEY, kind TEXT, slug TEXT, payload TEXT)")
|
||||
for line_no, line in enumerate(handle, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
record = json.loads(line)
|
||||
kind, payload = record.get("record_type"), record.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"导出包第 {line_no} 行格式错误")
|
||||
if kind == "source":
|
||||
self["source"] = {**self.get("source", {}), **payload}
|
||||
elif kind == "graph":
|
||||
for key, subkind in (("nodes", "graph_node"), ("edges", "graph_edge")):
|
||||
db.executemany("INSERT INTO records(kind, payload) VALUES (?, ?)", [(subkind, json.dumps(v, ensure_ascii=False)) for v in payload.get(key) or []])
|
||||
elif kind in {"wiki_page", "document", "chunk", "vector", "entity", "relation"}:
|
||||
slug = str(payload.get("wiki_slug") or payload.get("page_slug") or payload.get("slug") or "").strip("/")
|
||||
db.execute("INSERT INTO records(kind, slug, payload) VALUES (?, ?, ?)", (kind, slug, json.dumps(payload, ensure_ascii=False)))
|
||||
else:
|
||||
raise ValueError(f"不支持的导出记录类型:{kind}")
|
||||
db.execute("CREATE INDEX by_kind_slug ON records(kind, slug)")
|
||||
db.commit()
|
||||
source = self.get("source", {})
|
||||
if source.get("package_version") != 1 or (source.get("provider") or source.get("type")) not in {"weknora", "assistant"}:
|
||||
raise ValueError("仅接受 WeKnora / 助手知识库生成的兼容 Wiki 向量包(v1)")
|
||||
for key, kind in (("wiki_pages", "wiki_page"), ("documents", "document"), ("chunks", "chunk"), ("vectors", "vector"), ("entities", "entity"), ("relations", "relation")):
|
||||
self[key] = Records(self.path, kind)
|
||||
self["graph"] = {"nodes": Records(self.path, "graph_node"), "edges": Records(self.path, "graph_edge")}
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def vectors_for(self, slug: str) -> Records:
|
||||
return Records(self.path, "vector", slug)
|
||||
|
||||
def close(self):
|
||||
self.path.unlink(missing_ok=True)
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Normalize heterogeneous sources into the assistant import-package shape."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def export_package_from_skill_ir(ir: dict[str, Any]) -> dict[str, Any]:
|
||||
skill = ir.get("skill") or {}
|
||||
return {
|
||||
"source": {
|
||||
"type": "skill",
|
||||
"skill_name": skill.get("name"),
|
||||
"source_digest": skill.get("source_digest"),
|
||||
},
|
||||
"wiki_pages": [ir.get("wiki", {}).get("skill_page")],
|
||||
"documents": ir.get("files") or [],
|
||||
"chunks": [],
|
||||
"entities": ir.get("entities") or [],
|
||||
"relations": ir.get("relations") or [],
|
||||
"graph": {"nodes": ir.get("entities") or [], "edges": ir.get("relations") or []},
|
||||
}
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""File-backed assistant knowledge import/export packages.
|
||||
|
||||
The assistant knowledge base is fed by WeKnora-exported data. Keep that
|
||||
boundary explicit: a WeKnora export is first materialized as a durable JSONL
|
||||
package, then imported asynchronously from that package. JSONL keeps large
|
||||
exports append-friendly and avoids treating the HTTP request body as the job
|
||||
state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable, Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
PACKAGE_VERSION = 1
|
||||
PACKAGE_RECORD_TYPES = {
|
||||
"source",
|
||||
"wiki_page",
|
||||
"document",
|
||||
"chunk",
|
||||
"vector",
|
||||
"entity",
|
||||
"relation",
|
||||
"graph",
|
||||
}
|
||||
|
||||
|
||||
def package_path(root: Path, job_id: str) -> Path:
|
||||
safe = "".join(ch for ch in str(job_id) if ch.isalnum() or ch in {"-", "_"})
|
||||
return root / f"{safe}.jsonl"
|
||||
|
||||
|
||||
def write_jsonl_record(handle: Any, record_type: str, payload: Mapping[str, Any]) -> None:
|
||||
handle.write(json.dumps({"record_type": record_type, "payload": dict(payload)}, ensure_ascii=False))
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def write_package(path: Path, records: Iterable[tuple[str, Mapping[str, Any]]]) -> dict[str, int]:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
counts: dict[str, int] = {key: 0 for key in PACKAGE_RECORD_TYPES}
|
||||
tmp_path = path.with_suffix(path.suffix + ".tmp")
|
||||
with tmp_path.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
write_jsonl_record(
|
||||
handle,
|
||||
"source",
|
||||
{
|
||||
"package_version": PACKAGE_VERSION,
|
||||
},
|
||||
)
|
||||
counts["source"] += 1
|
||||
for record_type, payload in records:
|
||||
if record_type not in PACKAGE_RECORD_TYPES:
|
||||
continue
|
||||
write_jsonl_record(handle, record_type, payload)
|
||||
counts[record_type] = counts.get(record_type, 0) + 1
|
||||
tmp_path.replace(path)
|
||||
return counts
|
||||
|
||||
|
||||
def read_package(path: Path) -> dict[str, Any]:
|
||||
package: dict[str, Any] = {
|
||||
"source": {},
|
||||
"wiki_pages": [],
|
||||
"documents": [],
|
||||
"chunks": [],
|
||||
"vectors": [],
|
||||
"entities": [],
|
||||
"relations": [],
|
||||
"graph": {"nodes": [], "edges": []},
|
||||
}
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
record = json.loads(line)
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
record_type = str(record.get("record_type") or "")
|
||||
payload = record.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
if record_type == "source":
|
||||
package["source"] = {**package.get("source", {}), **payload}
|
||||
elif record_type == "wiki_page":
|
||||
package["wiki_pages"].append(payload)
|
||||
elif record_type == "document":
|
||||
package["documents"].append(payload)
|
||||
elif record_type == "chunk":
|
||||
package["chunks"].append(payload)
|
||||
elif record_type == "vector":
|
||||
package["vectors"].append(payload)
|
||||
elif record_type == "entity":
|
||||
package["entities"].append(payload)
|
||||
elif record_type == "relation":
|
||||
package["relations"].append(payload)
|
||||
elif record_type == "graph":
|
||||
nodes = payload.get("nodes")
|
||||
edges = payload.get("edges")
|
||||
if isinstance(nodes, list):
|
||||
package["graph"]["nodes"].extend(item for item in nodes if isinstance(item, dict))
|
||||
if isinstance(edges, list):
|
||||
package["graph"]["edges"].extend(item for item in edges if isinstance(item, dict))
|
||||
for key, value in payload.items():
|
||||
if key not in {"nodes", "edges"}:
|
||||
package["graph"][key] = value
|
||||
return package
|
||||
Reference in new issue
Block a user