feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3
Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
1 parent
49d9ced04f
commit
60e2c72514
38 files changed
+2536
-581
No files matched your search
@@ -27,7 +27,7 @@ def _skill_directory(app_config: Any, skill_name: str) -> Path:
|
||||
root = storage.get_skills_root_path().resolve()
|
||||
for category in SkillCategory:
|
||||
direct = root / category.value / skill_name
|
||||
if (direct / "SKILL.md").is_file():
|
||||
if direct.resolve().is_relative_to((root / category.value).resolve()) and (direct / "SKILL.md").is_file():
|
||||
return direct
|
||||
for category in SkillCategory:
|
||||
category_root = root / category.value
|
||||
@@ -235,6 +235,9 @@ class SkillKnowledgeJobExecutor:
|
||||
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
await self._assert_source_unchanged(item, expected_digest)
|
||||
if item["target_type"] == "weknora" and item["target_mode"] == "wiki":
|
||||
await self._generate_vectorize_and_deposit(item, snapshot_id, ir, scan)
|
||||
return
|
||||
await self.store.update_item(str(item["id"]), phase="writing_wiki", progress=65)
|
||||
target_type = str(item["target_type"])
|
||||
artifacts: list[dict[str, Any]] = []
|
||||
@@ -401,3 +404,63 @@ class SkillKnowledgeJobExecutor:
|
||||
await self._assert_source_unchanged(item, expected_digest)
|
||||
await self.store.update_item(str(item["id"]), phase="activating", progress=90)
|
||||
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status=binding_status)
|
||||
|
||||
async def _generate_vectorize_and_deposit(self, item, snapshot_id, ir, scan):
|
||||
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
|
||||
if mapping is None:
|
||||
raise ValueError("目标普通知识库不存在")
|
||||
sync = getattr(self.app.state, "llmwiki_sync_service", None)
|
||||
if sync is None:
|
||||
raise ValueError("请先配置 Wiki 编码模型并启用本地向量索引;归纳需要完成向量化后同步知识梳理")
|
||||
client = build_weknora_client(get_resolved_llmwiki_runtime(self.app.state.config))
|
||||
remote_id = mapping["weknora_id"]
|
||||
remote = await client.get_knowledge_base(remote_id)
|
||||
if not (remote.get("indexing_strategy") or {}).get("wiki_enabled"):
|
||||
raise ValueError("目标普通知识库尚未开启 Wiki")
|
||||
artifacts = []
|
||||
document_ids = []
|
||||
await self.store.update_item(str(item["id"]), phase="writing_sources", progress=55)
|
||||
for file in scan.files:
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
if not file.text:
|
||||
continue
|
||||
document = await client.create_manual_document(remote_id, title=f"{ir['skill'].get('display_name') or item['skill_name']} · {file.path}", content=file.text)
|
||||
doc_id = str(document.get("id") or document.get("knowledge_id") or "")
|
||||
if not doc_id:
|
||||
raise ValueError("资料上传未返回文档标识")
|
||||
document_ids.append(doc_id)
|
||||
artifacts.append({"kind": "raw_source", "slug": file.path, "remote_id": doc_id})
|
||||
if not document_ids:
|
||||
raise ValueError("该技能没有可归纳的知识正文,代码、脚本和代码块不参与归纳")
|
||||
await self.store.update_item(str(item["id"]), phase="generating_wiki", progress=65)
|
||||
# WeKnora performs model extraction, categorization and linking. Wait
|
||||
# for both parsing and Wiki's debounced finalize before vectorization.
|
||||
stable = 0
|
||||
for _ in range(720):
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
docs = [await client.get_document(doc_id) for doc_id in document_ids]
|
||||
if any(doc.get("parse_status") in {"failed", "error"} for doc in docs):
|
||||
raise RuntimeError("WeKnora 资料解析失败,请查看普通知识库中的文档状态")
|
||||
stats = await client.get_wiki_stats(remote_id)
|
||||
done = all(doc.get("parse_status") == "completed" for doc in docs) and not stats.get("is_active") and not stats.get("pending_tasks")
|
||||
stable = stable + 1 if done else 0
|
||||
if stable >= 3:
|
||||
if not int(stats.get("total_pages") or 0):
|
||||
raise RuntimeError("WeKnora 解析已结束,但未生成 Wiki;请检查 Wiki 生成模型的调用错误、余额及资料是否含实质知识,再重试")
|
||||
break
|
||||
await asyncio.sleep(5)
|
||||
else:
|
||||
raise TimeoutError("Wiki 分析尚未完成;请检查普通知识库的生成任务后重新归纳")
|
||||
await self.store.update_item(str(item["id"]), phase="vectorizing_wiki", progress=80)
|
||||
result = await sync.sync_mapping(mapping, force=False, notify=False)
|
||||
if result.get("status") != "completed":
|
||||
raise RuntimeError("Wiki 向量化未全部完成,请检查向量化进度")
|
||||
await self.store.update_item(str(item["id"]), phase="syncing_assistant", progress=92)
|
||||
from app.gateway.knowledge_transfer import sync_to_global
|
||||
|
||||
deposited = await sync_to_global(self.app, mapping)
|
||||
artifacts.append({"kind": "assistant_deposit", "slug": "knowledge-summary", "metadata": deposited})
|
||||
await self._assert_source_unchanged(item, str(ir["skill"]["source_digest"]))
|
||||
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status="synced")
|
||||
Reference in new issue
Block a user