feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3

Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
吉鑫升 committed 2026-09-03 19:21:10 +08:00
1 parent 49d9ced04f
commit 60e2c72514
38 files changed
+2536 -581

No files matched your search

@@ -27,7 +27,7 @@ def _skill_directory(app_config: Any, skill_name: str) -> Path:
root = storage.get_skills_root_path().resolve()
for category in SkillCategory:
direct = root / category.value / skill_name
if (direct / "SKILL.md").is_file():
if direct.resolve().is_relative_to((root / category.value).resolve()) and (direct / "SKILL.md").is_file():
return direct
for category in SkillCategory:
category_root = root / category.value
@@ -235,6 +235,9 @@ class SkillKnowledgeJobExecutor:
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
scan = await enrich_converted_documents(scan)
await self._assert_source_unchanged(item, expected_digest)
if item["target_type"] == "weknora" and item["target_mode"] == "wiki":
await self._generate_vectorize_and_deposit(item, snapshot_id, ir, scan)
return
await self.store.update_item(str(item["id"]), phase="writing_wiki", progress=65)
target_type = str(item["target_type"])
artifacts: list[dict[str, Any]] = []
@@ -401,3 +404,63 @@ class SkillKnowledgeJobExecutor:
await self._assert_source_unchanged(item, expected_digest)
await self.store.update_item(str(item["id"]), phase="activating", progress=90)
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status=binding_status)
async def _generate_vectorize_and_deposit(self, item, snapshot_id, ir, scan):
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
if mapping is None:
raise ValueError("目标普通知识库不存在")
sync = getattr(self.app.state, "llmwiki_sync_service", None)
if sync is None:
raise ValueError("请先配置 Wiki 编码模型并启用本地向量索引;归纳需要完成向量化后同步知识梳理")
client = build_weknora_client(get_resolved_llmwiki_runtime(self.app.state.config))
remote_id = mapping["weknora_id"]
remote = await client.get_knowledge_base(remote_id)
if not (remote.get("indexing_strategy") or {}).get("wiki_enabled"):
raise ValueError("目标普通知识库尚未开启 Wiki")
artifacts = []
document_ids = []
await self.store.update_item(str(item["id"]), phase="writing_sources", progress=55)
for file in scan.files:
if await self._cancel_if_requested(item):
return
if not file.text:
continue
document = await client.create_manual_document(remote_id, title=f"{ir['skill'].get('display_name') or item['skill_name']} · {file.path}", content=file.text)
doc_id = str(document.get("id") or document.get("knowledge_id") or "")
if not doc_id:
raise ValueError("资料上传未返回文档标识")
document_ids.append(doc_id)
artifacts.append({"kind": "raw_source", "slug": file.path, "remote_id": doc_id})
if not document_ids:
raise ValueError("该技能没有可归纳的知识正文,代码、脚本和代码块不参与归纳")
await self.store.update_item(str(item["id"]), phase="generating_wiki", progress=65)
# WeKnora performs model extraction, categorization and linking. Wait
# for both parsing and Wiki's debounced finalize before vectorization.
stable = 0
for _ in range(720):
if await self._cancel_if_requested(item):
return
docs = [await client.get_document(doc_id) for doc_id in document_ids]
if any(doc.get("parse_status") in {"failed", "error"} for doc in docs):
raise RuntimeError("WeKnora 资料解析失败,请查看普通知识库中的文档状态")
stats = await client.get_wiki_stats(remote_id)
done = all(doc.get("parse_status") == "completed" for doc in docs) and not stats.get("is_active") and not stats.get("pending_tasks")
stable = stable + 1 if done else 0
if stable >= 3:
if not int(stats.get("total_pages") or 0):
raise RuntimeError("WeKnora 解析已结束,但未生成 Wiki;请检查 Wiki 生成模型的调用错误、余额及资料是否含实质知识,再重试")
break
await asyncio.sleep(5)
else:
raise TimeoutError("Wiki 分析尚未完成;请检查普通知识库的生成任务后重新归纳")
await self.store.update_item(str(item["id"]), phase="vectorizing_wiki", progress=80)
result = await sync.sync_mapping(mapping, force=False, notify=False)
if result.get("status") != "completed":
raise RuntimeError("Wiki 向量化未全部完成,请检查向量化进度")
await self.store.update_item(str(item["id"]), phase="syncing_assistant", progress=92)
from app.gateway.knowledge_transfer import sync_to_global
deposited = await sync_to_global(self.app, mapping)
artifacts.append({"kind": "assistant_deposit", "slug": "knowledge-summary", "metadata": deposited})
await self._assert_source_unchanged(item, str(ir["skill"]["source_digest"]))
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status="synced")