feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3

Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
吉鑫升 committed 2026-09-03 19:21:10 +08:00
1 parent 49d9ced04f
commit 60e2c72514
38 files changed
+2536 -581

No files matched your search

@@ -1586,6 +1586,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
app.include_router(skill_knowledge.router)
app.include_router(assistant_knowledge.router)
app.include_router(assistant_knowledge.export_router)
from app.gateway.routers import wiki_packages
app.include_router(wiki_packages.router)
# Artifacts API is mounted at /api/threads/{thread_id}/artifacts
app.include_router(artifacts.router)
@@ -381,7 +381,7 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
embedding_config = local_index_config.embedding
embedding_configured = bool(
embedding_config.enabled
and embedding_config.base_url
and (embedding_config.base_url or embedding_config.provider == "local")
and embedding_config.model.strip()
)
if embedding_configured:
@@ -404,13 +404,19 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.llmwiki_vector_cache,
)
if weknora_client is not None:
app.state.llmwiki_sync_service = WikiSyncService(
app.state.llmwiki_sync_service = WikiSyncService(
local_index_config,
app.state.llmwiki_index_store,
app.state.llmwiki_embedding,
app.state.llmwiki_vector_cache,
weknora_client,
)
weknora_client,
)
from app.gateway.knowledge_transfer import sync_to_global
async def on_wiki_vectorized(mapping):
return await sync_to_global(app, mapping)
app.state.llmwiki_sync_service.on_completed = on_wiki_vectorized
if weknora_client is not None or app.state.llmwiki_vector_search is not None:
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
@@ -0,0 +1,28 @@
"""Wiki package flow orchestration; all remote writes use WeKnora APIs."""
from __future__ import annotations
from starlette.requests import Request
async def sync_to_global(app, mapping):
from app.gateway.routers.assistant_knowledge import _export_weknora_then_import_for_job
if mapping.get("is_conversation_deposit") or mapping.get("owner_user_id") == "system":
return {"status": "excluded"}
store = app.state.assistant_knowledge_store
revision, fingerprint, _ = await app.state.llmwiki_index_store.get_index_revision(str(mapping["id"]))
base = await store.initialize(created_by=None)
for source in await store.list_sources(base_id=base["id"]):
meta = source.get("metadata") or {}
if source.get("source_key") == mapping["id"] and meta.get("index_revision") == revision:
job = await store.get_job(source.get("last_import_job_id")) if source.get("last_import_job_id") else None
if job and job.get("status") == "completed":
return {"status": "unchanged", "job_id": job["id"], "base_id": base["id"]}
source = {"provider": "weknora", "type": "weknora", "knowledge_base_id": mapping["id"], "knowledge_base_name": mapping["name"], "index_revision": revision, "embedding_fingerprint": fingerprint}
job = await store.create_queued_import_job(source_type="weknora", source_key=mapping["id"], source_name=mapping["name"], trigger="vectorization_completed", created_by=None, metadata=source, mapping_id=mapping["id"], base_id=base["id"])
await _export_weknora_then_import_for_job(request=Request({"type": "http", "app": app}), job_id=job["id"], mapping=mapping, remote_id=mapping["weknora_id"], source=source, actor=None, index_fresh=True)
result = await store.get_job(job["id"])
if not result or result.get("status") != "completed":
raise RuntimeError(f"向量化已完成,但知识梳理同步失败:{(result or {}).get('last_error_message') or job['id']}")
return {"status": "completed", "job_id": job["id"], "base_id": base["id"], "counts": result.get("counts")}
@@ -6,6 +6,7 @@ import asyncio
import base64
import json
import logging
import re
from collections.abc import AsyncIterable, AsyncIterator
from datetime import UTC, datetime
from pathlib import Path
@@ -17,7 +18,8 @@ from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from app.gateway.deps import get_optional_user_from_request
from deerflow.assistant_knowledge.package_io import PACKAGE_VERSION, package_path, read_package, write_jsonl_record
from deerflow.assistant_knowledge.archive import PackageArchive
from deerflow.assistant_knowledge.package_io import PACKAGE_VERSION, package_path, write_jsonl_record
from deerflow.config.runtime_paths import runtime_home
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
@@ -114,7 +116,9 @@ def _write_export_manifest(
}
path.parent.mkdir(parents=True, exist_ok=True)
payload = _normalize_export_manifest(payload)
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
temp = path.with_suffix(".tmp")
temp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
temp.replace(path)
return payload
@@ -227,10 +231,11 @@ async def list_wiki_pages(
q: str | None = Query(default=None),
page_type: str | None = Query(default=None),
limit: int = Query(default=100, ge=1, le=500),
offset: int = Query(default=0, ge=0),
) -> dict[str, Any]:
await _actor(request)
pages = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=limit)
return {"pages": pages, "total": len(pages)}
pages = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=limit + 1, offset=offset)
return {"pages": pages[:limit], "total": min(len(pages), limit), "next_offset": offset + limit if len(pages) > limit else None}
@router.get("/wiki-pages/{slug:path}/revisions/compare")
@@ -279,8 +284,11 @@ async def search(
page_type: str | None = Query(default=None),
) -> dict[str, Any]:
await _actor(request)
rows = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=100)
return {"results": rows, "total": len(rows)}
store = _store(request)
base = await store.get_base(base_id)
embedding = getattr(request.app.state, "llmwiki_embedding", None)
rows = await store.search_pages(base_ids=[base["id"]] if base else [], query=q, limit=20, include_chunks=False, embedding_client=embedding)
return {"results": rows, "total": len(rows), "retrieval_mode": rows[0].get("retrieval_mode", "keyword") if rows else "empty"}
@router.get("/sources")
@@ -399,7 +407,10 @@ async def import_weknora_package_file(
if not chunk:
break
size += len(chunk)
handle.write(chunk)
await asyncio.to_thread(handle.write, chunk)
except Exception as exc:
await _store(request).update_import_job(str(job["id"]), status="failed", phase="upload_failed", error=str(exc), completed=True)
raise
finally:
await file.close()
if size <= 0:
@@ -454,6 +465,7 @@ async def _write_weknora_export_package(
"source",
{
**source,
"provider": "weknora",
"package_version": PACKAGE_VERSION,
"export_format": "assistant-knowledge-jsonl",
},
@@ -462,6 +474,8 @@ async def _write_weknora_export_package(
# Wiki pages first: these are the human-readable pages that the
# assistant knowledge base presents in wiki form.
page_no = 1
graph_nodes = []
graph_edges = set()
while True:
page = await client.list_wiki_pages(remote_id, page=page_no, page_size=100)
batch = page.get("pages") or []
@@ -475,14 +489,16 @@ async def _write_weknora_export_package(
if isinstance(detail, dict):
payload = {**row, **detail}
except WeKnoraError:
logger.warning(
"Could not fetch full WeKnora wiki page during export: kb=%s slug=%s",
remote_id,
slug,
exc_info=True,
)
# A list summary is not a replacement for the full
# article. An incomplete package must fail visibly.
raise
write_jsonl_record(handle, "wiki_page", payload)
counts["wiki_page"] += 1
graph_nodes.append({"slug": slug, "title": payload.get("title"), "page_type": payload.get("page_type"), "aliases": payload.get("aliases") or []})
for target in payload.get("out_links") or []:
graph_edges.add((slug, str(target)))
for match in re.finditer(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]", str(payload.get("content") or "")):
graph_edges.add((slug, match.group(1).strip()))
if counts["wiki_page"] >= int(page.get("total") or counts["wiki_page"]) or not batch:
break
page_no += 1
@@ -526,23 +542,25 @@ async def _write_weknora_export_package(
write_jsonl_record(handle, "vector", row)
counts["vector"] += 1
try:
graph = await client.get_wiki_graph(remote_id, limit=10000)
except WeKnoraError:
graph = {"nodes": [], "edges": [], "capability_warning": "graph API unavailable"}
# Build the complete link graph from the paginated full pages. The
# provider's overview graph endpoint is capped and can omit nodes.
known_slugs = {row["slug"] for row in graph_nodes}
graph = {"nodes": graph_nodes, "edges": [{"source": a, "target": b} for a, b in sorted(graph_edges) if a in known_slugs and b in known_slugs]}
nodes = [row for row in graph.get("nodes") or [] if isinstance(row, dict)]
edges = [row for row in graph.get("edges") or [] if isinstance(row, dict)]
write_jsonl_record(handle, "graph", {**graph, "nodes": nodes, "edges": edges})
counts["graph"] += 1
for row in nodes:
if row.get("id") and (row.get("title") or row.get("name")):
if (row.get("slug") or row.get("id")) and (row.get("title") or row.get("name")):
write_jsonl_record(
handle,
"entity",
{
"id": row.get("id"),
"id": row.get("slug") or row.get("id"),
"name": row.get("title") or row.get("name"),
"type": row.get("type") or "concept",
"type": row.get("page_type") or row.get("type") or "concept",
"source_wiki_slug": row.get("slug") or row.get("id"),
"aliases": row.get("aliases") or [],
"confidence": 1.0,
},
)
@@ -555,6 +573,7 @@ async def _write_weknora_export_package(
"source_entity_id": row.get("source"),
"target_entity_id": row.get("target"),
"predicate": row.get("predicate") or "references",
"source_wiki_slug": row.get("source"),
"confidence": 1.0,
"evidence": [{"source": "weknora_graph"}],
},
@@ -627,8 +646,9 @@ async def _iter_local_wiki_vectors_for_export(
payload = _local_wiki_vector_export_payload(row, model=model, fingerprint=fingerprint)
if payload is not None:
yield payload
except Exception: # noqa: BLE001
logger.debug("No local wiki vector snapshot available for %s", mapping_id, exc_info=True)
except Exception:
logger.exception("Wiki vector export failed for %s", mapping_id)
raise
async def _has_local_wiki_vectors_for_export(request: Request, mapping_id: str) -> bool:
@@ -646,12 +666,11 @@ async def _ensure_local_wiki_vectors_for_export(
mapping: dict[str, Any],
) -> None:
mapping_id = str(mapping.get("id") or mapping.get("knowledge_base_id") or "")
if not mapping_id or await _has_local_wiki_vectors_for_export(request, mapping_id):
return
service = getattr(request.app.state, "llmwiki_sync_service", None)
if service is None or not mapping_id:
return
raise RuntimeError("请先配置 Wiki 编码模型并启用本地向量索引,再生成向量数据包")
# Refresh remote metadata/content hashes even when the previous generation
# was complete. Unchanged articles reuse their existing vectors.
sync_mapping = dict(mapping)
sync_mapping.setdefault("id", mapping_id)
sync_mapping.setdefault(
@@ -660,15 +679,9 @@ async def _ensure_local_wiki_vectors_for_export(
)
if not sync_mapping.get("weknora_id"):
return
try:
logger.info("Building local wiki vectors before export: mapping=%s", mapping_id)
await service.sync_mapping(sync_mapping, force=False)
except Exception: # noqa: BLE001
logger.warning(
"Could not build local wiki vectors before export: mapping=%s",
mapping_id,
exc_info=True,
)
result = await service.sync_mapping(sync_mapping, force=False, notify=False)
if result.get("status") != "completed":
raise RuntimeError("Wiki 向量化尚未全部完成,请查看向量化进度后重试")
async def _import_package_file_for_job(
@@ -679,9 +692,10 @@ async def _import_package_file_for_job(
actor: str | None,
) -> None:
store = _store(request)
package = None
try:
await store.update_import_job(job_id, status="running", phase="loading_package")
package = await asyncio.to_thread(read_package, path)
package = await asyncio.to_thread(PackageArchive, path)
await store.update_import_job(job_id, status="running", phase="importing")
await store.import_package_for_job(
job_id,
@@ -698,6 +712,9 @@ async def _import_package_file_for_job(
error=str(exc),
completed=True,
)
finally:
if package is not None:
await asyncio.to_thread(package.close)
async def _export_weknora_then_import_for_job(
@@ -708,6 +725,7 @@ async def _export_weknora_then_import_for_job(
remote_id: str,
source: dict[str, Any],
actor: str | None,
index_fresh: bool = False,
) -> None:
store = _store(request)
path = package_path(_package_root(), job_id)
@@ -717,7 +735,8 @@ async def _export_weknora_then_import_for_job(
if not runtime.weknora_enabled:
raise RuntimeError("未配置普通知识库服务")
mapping_id = str(mapping.get("id") or mapping.get("knowledge_base_id") or "")
await _ensure_local_wiki_vectors_for_export(request, mapping)
if not index_fresh:
await _ensure_local_wiki_vectors_for_export(request, mapping)
counts = await _write_weknora_export_package(
build_weknora_client(runtime),
remote_id,
@@ -2041,6 +2041,7 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
if not slug and not title and not content and not summary:
return None
return {
"page_metadata": page.get("page_metadata") if isinstance(page.get("page_metadata"), dict) else {},
"id": str(page.get("id") or ""),
"slug": slug,
"title": title,
@@ -2987,12 +2988,11 @@ async def list_knowledge_bases(
knowledge_bases: list[dict[str, Any]] = []
for row in rows:
remote_item = remote.get(str(row["weknora_id"]))
row, remote_item = await _sync_conversation_deposit_view(_store(request), client, row, remote_item)
# Owners need to see stale personal mappings so they can clean them up.
# Public/selector views must never expose a resource that no longer exists
# in WeKnora.
if remote_item is None and scope != "personal":
# A successful remote list is authoritative for every scope. Keep the
# mapping for existing references, but never display a deleted base.
if remote_item is None:
continue
row, remote_item = await _sync_conversation_deposit_view(_store(request), client, row, remote_item)
knowledge_bases.append(_mapping_view(row, remote_item, actor_user_id=user_id, is_admin=is_admin))
return {"knowledge_bases": knowledge_bases}
@@ -3077,13 +3077,19 @@ async def get_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any
async def update_knowledge_base(request: Request, mapping_id: str, body: KnowledgeBaseUpdate) -> dict[str, Any]:
row, user_id, is_admin = await _authorized_mapping(request, mapping_id, write=True)
changes = body.model_dump(exclude_none=True)
if "name" in changes:
changes["name"] = changes["name"].strip()
if not changes["name"]:
raise HTTPException(status_code=422, detail="知识库名称不能为空")
if changes["name"] == CONVERSATION_DEPOSIT_KB_NAME:
raise HTTPException(status_code=409, detail="对话沉淀是 cmzs 系统保留知识库")
if not changes:
return _mapping_view(row, None, actor_user_id=user_id, is_admin=is_admin)
try:
remote = await _client_or_503().update_knowledge_base(str(row["weknora_id"]), changes)
except WeKnoraError as exc:
_raise_upstream(exc)
updated = await _store(request).update_mapping(mapping_id, name=body.name, description=body.description)
updated = await _store(request).update_mapping(mapping_id, name=changes.get("name"), description=changes.get("description"))
return _mapping_view(updated or row, remote, actor_user_id=user_id, is_admin=is_admin)
@@ -313,7 +313,7 @@ async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[st
if not mapping.get("wiki_index_enabled", True):
mapping = await mapping_store.update_mapping(mapping_id, wiki_index_enabled=True)
job_id = str(uuid4())
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=True))
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=False))
return {"job_id": job_id, "status": "accepted", "knowledge_base_id": mapping_id}
@@ -52,9 +52,9 @@ class SyncTarget(BaseModel):
class CreateSyncJobRequest(BaseModel):
skill_names: list[str] = Field(min_length=1, max_length=200)
targets: list[SyncTarget] = Field(min_length=1, max_length=50)
targets: list[SyncTarget] = Field(min_length=1, max_length=1)
force: bool = False
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
review_mode: Literal["required", "auto_high_confidence", "off"] = "off"
@field_validator("skill_names")
@classmethod
@@ -165,8 +165,14 @@ async def disable_binding_auto_sync(request: Request, binding_id: str) -> dict[s
@router.post("/sync-jobs", status_code=202)
async def create_sync_job(request: Request, body: CreateSyncJobRequest) -> dict[str, Any]:
actor = await _require_admin(request)
skill_store = request.app.state.skill_store
missing = [name for name in body.skill_names if await skill_store.get_any(name) is None]
# Built-in and legacy skills need not have an ownership row. Validate
# against the same running filesystem used by the skills list/executor.
missing = []
for name in body.skill_names:
try:
await asyncio.to_thread(_skill_directory, request.app.state.config, name)
except (FileNotFoundError, ValueError):
missing.append(name)
if missing:
raise HTTPException(status_code=404, detail=f"技能不存在:{', '.join(missing[:10])}")
targets: list[dict[str, Any]] = []
@@ -0,0 +1,193 @@
"""Asynchronous, file-backed assistant export and ordinary Wiki restore."""
from __future__ import annotations
import asyncio
from pathlib import Path
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Request, UploadFile
from fastapi.responses import FileResponse
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
from app.gateway.routers.assistant_knowledge import _actor, _list_export_manifests, _package_root, _read_export_manifest, _store, _write_export_manifest
from deerflow.assistant_knowledge.archive import PackageArchive
from deerflow.assistant_knowledge.package_io import package_path, write_jsonl_record
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.persistence.assistant_knowledge.store import _vector_blob_from_payload
router = APIRouter(tags=["wiki-packages"])
def job_view(row):
if row["mapping_id"].startswith("assistant:"):
base_id = row["mapping_id"].split(":", 1)[1]
row["package_download_url"] = f"/api/assistant-knowledge/bases/{base_id}/export-package/{row['id']}/download"
elif row["mapping_id"].startswith("restore:"):
row["package_download_url"] = None
return row
async def export_assistant(app, base_id, job_id):
scope = f"assistant:{base_id}"
path = package_path(_package_root(), job_id)
temp = path.with_suffix(".tmp")
counts = {}
try:
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="exporting")
with temp.open("w", encoding="utf-8", newline="\n") as handle:
async for kind, payload in app.state.assistant_knowledge_store.iter_export_records(base_id):
await asyncio.to_thread(write_jsonl_record, handle, kind, payload)
counts[kind] = counts.get(kind, 0) + 1
if counts.get("wiki_page") and not counts.get("vector"):
raise ValueError("该助手知识库尚无可导出的 Wiki 向量,请先从已向量化的普通知识库同步")
await asyncio.to_thread(temp.replace, path)
_write_export_manifest(job_id=job_id, mapping_id=scope, status="completed", phase="package_ready", counts=counts)
except Exception as exc:
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="failed", counts=counts, error=str(exc))
@router.post("/api/assistant-knowledge/bases/{base_id}/export-package", status_code=202)
async def create_export(request: Request, base_id: str):
await _actor(request, admin=True)
if not await _store(request).get_base(base_id):
raise HTTPException(404, "助手知识库不存在")
job_id = str(uuid4())
row = _write_export_manifest(job_id=job_id, mapping_id=f"assistant:{base_id}", status="queued", phase="queued")
spawn_local_wiki_index_job(request.app, export_assistant(request.app, base_id, job_id))
return job_view(row)
@router.get("/api/assistant-knowledge/bases/{base_id}/export-package")
async def list_exports(request: Request, base_id: str):
await _actor(request, admin=True)
return {"jobs": [job_view(row) for row in await asyncio.to_thread(_list_export_manifests, f"assistant:{base_id}", limit=100)]}
@router.get("/api/assistant-knowledge/bases/{base_id}/export-package/{job_id}/download")
async def download_export(request: Request, base_id: str, job_id: str):
await _actor(request, admin=True)
row = _read_export_manifest(job_id)
if not row or row["mapping_id"] != f"assistant:{base_id}" or row["status"] != "completed" or not row["file_exists"]:
raise HTTPException(404, "导出包尚未就绪")
return FileResponse(package_path(_package_root(), job_id), media_type="application/x-ndjson", filename=f"wiki-vector-{job_id}.jsonl")
async def restore_package(app, mapping, job_id, path: Path):
scope = f"restore:{mapping['id']}"
archive = None
counts = {"wiki_page": 0, "vector": 0}
store = app.state.llmwiki_index_store
owner = f"package:{job_id}"
leased = False
try:
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="validating")
archive = await asyncio.to_thread(PackageArchive, path)
embedding = app.state.llmwiki_embedding
if embedding is None:
raise ValueError("请配置与数据包相同的 Wiki 编码模型,以便复用向量进行检索")
if embedding.dimensions is None:
await embedding.embed_query("Wiki embedding dimension probe")
# Validate the complete package before the first remote mutation.
slugs = {p["slug"] for p in archive["wiki_pages"]}
vector_slugs = set()
for raw in archive["vectors"]:
slug = str(raw.get("wiki_slug") or raw.get("page_slug") or raw.get("slug") or "")
if slug not in slugs or _vector_blob_from_payload(raw) is None:
raise ValueError(f"向量数据或 Wiki 引用无效:{slug}")
if raw.get("embedding_fingerprint") != embedding.fingerprint or raw.get("embedding_dimensions") != embedding.dimensions:
raise ValueError("数据包编码模型指纹与当前配置不一致;请使用同一模型配置,禁止静默重新编码")
vector_slugs.add(slug)
if not vector_slugs:
raise ValueError("数据包中没有 Wiki 向量")
for page in archive["wiki_pages"]:
if page["slug"] not in vector_slugs and page.get("page_type") not in {"index", "directory"} and (page.get("content") or "").strip():
raise ValueError(f"Wiki 缺少向量:{page['slug']}")
leased = await store.try_acquire_sync_lease(mapping["id"], owner=owner, sync_id=job_id, lease_seconds=900, fingerprint=embedding.fingerprint)
if not leased:
raise ValueError("目标知识库正在同步,请稍后重试")
client = build_weknora_client(get_resolved_llmwiki_runtime(app.state.config))
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="restoring")
folders = {(): ""}
for raw in archive["wiki_pages"]:
if not await store.renew_sync_lease(mapping["id"], owner=owner, lease_seconds=900):
raise RuntimeError("导入租约已失效,请重新发起")
# Restore the provider's Wiki schema, never raw documents; raw
# uploads would trigger generation and embedding a second time.
categories = raw.get("category_path") or []
if isinstance(categories, str):
categories = [part for part in categories.split("/") if part]
parent = ()
for name in categories:
path_key = (*parent, name)
if path_key not in folders:
folder = await client.create_wiki_folder(mapping["weknora_id"], name=name, parent_id=folders[parent])
folders[path_key] = folder["id"]
parent = path_key
payload = {key: value for key, value in raw.items() if key not in {"id", "tenant_id", "knowledge_base_id", "folder_id", "origins"}}
payload["folder_id"] = folders[parent]
if raw.get("tags"):
payload["page_metadata"] = {**(raw.get("page_metadata") or {}), "tags": raw["tags"]}
remote = await client.create_wiki_page(mapping["weknora_id"], payload)
page = normalize_wiki_page(mapping["id"], {**raw, **remote})
vectors = []
for value in archive.vectors_for(raw["slug"]):
blob, dimensions = _vector_blob_from_payload(value)
vectors.append({**value, "vector_blob": blob, "embedding_dimensions": dimensions})
await store.replace_page_vectors(page=page, vectors=vectors, fingerprint=embedding.fingerprint, sync_id=job_id)
counts["wiki_page"] += 1
counts["vector"] += len(vectors)
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="restoring", counts=counts)
await client.rebuild_wiki_links(mapping["weknora_id"])
await store.finish_sync(mapping["id"], owner=owner, sync_id=job_id, success=True, remote_page_count=counts["wiki_page"])
leased = False
if app.state.llmwiki_vector_cache:
app.state.llmwiki_vector_cache.invalidate(mapping["id"])
from app.gateway.knowledge_transfer import sync_to_global
await sync_to_global(app, mapping)
_write_export_manifest(job_id=job_id, mapping_id=scope, status="completed", phase="completed", counts=counts)
except Exception as exc:
if leased:
await store.finish_sync(mapping["id"], owner=owner, sync_id=job_id, success=False, remote_page_count=counts["wiki_page"], error=str(exc))
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="failed", counts=counts, error=str(exc))
finally:
if archive is not None:
await asyncio.to_thread(archive.close)
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/import-package", status_code=202)
async def upload_restore(request: Request, mapping_id: str, file: UploadFile = File(...)):
actor = await _actor(request, admin=True)
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, actor or "system", write=True, is_admin=True)
if not mapping or mapping.get("is_conversation_deposit") or mapping.get("owner_user_id") == "system":
raise HTTPException(404, "目标普通知识库不存在或不可导入")
client = build_weknora_client(get_resolved_llmwiki_runtime(request.app.state.config))
remote = await client.get_knowledge_base(mapping["weknora_id"])
if not (remote.get("indexing_strategy") or {}).get("wiki_enabled"):
raise HTTPException(422, "请先开启目标知识库的 Wiki")
existing = await client.list_wiki_pages(mapping["weknora_id"], page_size=1)
if existing.get("total"):
raise HTTPException(409, "请导入到新建的空 Wiki 知识库,避免覆盖已有页面")
job_id = str(uuid4())
scope = f"restore:{mapping_id}"
row = _write_export_manifest(job_id=job_id, mapping_id=scope, status="queued", phase="uploading")
path = package_path(_package_root(), job_id)
try:
with path.open("wb") as handle:
while data := await file.read(1024 * 1024):
await asyncio.to_thread(handle.write, data)
except Exception as exc:
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="upload_failed", error=str(exc))
raise
finally:
await file.close()
spawn_local_wiki_index_job(request.app, restore_package(request.app, mapping, job_id, path))
return job_view(row)
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/import-package")
async def list_restores(request: Request, mapping_id: str):
await _actor(request, admin=True)
return {"jobs": [job_view(row) for row in await asyncio.to_thread(_list_export_manifests, f"restore:{mapping_id}", limit=100)]}
@@ -27,7 +27,7 @@ def _skill_directory(app_config: Any, skill_name: str) -> Path:
root = storage.get_skills_root_path().resolve()
for category in SkillCategory:
direct = root / category.value / skill_name
if (direct / "SKILL.md").is_file():
if direct.resolve().is_relative_to((root / category.value).resolve()) and (direct / "SKILL.md").is_file():
return direct
for category in SkillCategory:
category_root = root / category.value
@@ -235,6 +235,9 @@ class SkillKnowledgeJobExecutor:
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
scan = await enrich_converted_documents(scan)
await self._assert_source_unchanged(item, expected_digest)
if item["target_type"] == "weknora" and item["target_mode"] == "wiki":
await self._generate_vectorize_and_deposit(item, snapshot_id, ir, scan)
return
await self.store.update_item(str(item["id"]), phase="writing_wiki", progress=65)
target_type = str(item["target_type"])
artifacts: list[dict[str, Any]] = []
@@ -401,3 +404,63 @@ class SkillKnowledgeJobExecutor:
await self._assert_source_unchanged(item, expected_digest)
await self.store.update_item(str(item["id"]), phase="activating", progress=90)
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status=binding_status)
async def _generate_vectorize_and_deposit(self, item, snapshot_id, ir, scan):
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
if mapping is None:
raise ValueError("目标普通知识库不存在")
sync = getattr(self.app.state, "llmwiki_sync_service", None)
if sync is None:
raise ValueError("请先配置 Wiki 编码模型并启用本地向量索引;归纳需要完成向量化后同步知识梳理")
client = build_weknora_client(get_resolved_llmwiki_runtime(self.app.state.config))
remote_id = mapping["weknora_id"]
remote = await client.get_knowledge_base(remote_id)
if not (remote.get("indexing_strategy") or {}).get("wiki_enabled"):
raise ValueError("目标普通知识库尚未开启 Wiki")
artifacts = []
document_ids = []
await self.store.update_item(str(item["id"]), phase="writing_sources", progress=55)
for file in scan.files:
if await self._cancel_if_requested(item):
return
if not file.text:
continue
document = await client.create_manual_document(remote_id, title=f"{ir['skill'].get('display_name') or item['skill_name']} · {file.path}", content=file.text)
doc_id = str(document.get("id") or document.get("knowledge_id") or "")
if not doc_id:
raise ValueError("资料上传未返回文档标识")
document_ids.append(doc_id)
artifacts.append({"kind": "raw_source", "slug": file.path, "remote_id": doc_id})
if not document_ids:
raise ValueError("该技能没有可归纳的知识正文,代码、脚本和代码块不参与归纳")
await self.store.update_item(str(item["id"]), phase="generating_wiki", progress=65)
# WeKnora performs model extraction, categorization and linking. Wait
# for both parsing and Wiki's debounced finalize before vectorization.
stable = 0
for _ in range(720):
if await self._cancel_if_requested(item):
return
docs = [await client.get_document(doc_id) for doc_id in document_ids]
if any(doc.get("parse_status") in {"failed", "error"} for doc in docs):
raise RuntimeError("WeKnora 资料解析失败,请查看普通知识库中的文档状态")
stats = await client.get_wiki_stats(remote_id)
done = all(doc.get("parse_status") == "completed" for doc in docs) and not stats.get("is_active") and not stats.get("pending_tasks")
stable = stable + 1 if done else 0
if stable >= 3:
if not int(stats.get("total_pages") or 0):
raise RuntimeError("WeKnora 解析已结束,但未生成 Wiki;请检查 Wiki 生成模型的调用错误、余额及资料是否含实质知识,再重试")
break
await asyncio.sleep(5)
else:
raise TimeoutError("Wiki 分析尚未完成;请检查普通知识库的生成任务后重新归纳")
await self.store.update_item(str(item["id"]), phase="vectorizing_wiki", progress=80)
result = await sync.sync_mapping(mapping, force=False, notify=False)
if result.get("status") != "completed":
raise RuntimeError("Wiki 向量化未全部完成,请检查向量化进度")
await self.store.update_item(str(item["id"]), phase="syncing_assistant", progress=92)
from app.gateway.knowledge_transfer import sync_to_global
deposited = await sync_to_global(self.app, mapping)
artifacts.append({"kind": "assistant_deposit", "slug": "knowledge-summary", "metadata": deposited})
await self._assert_source_unchanged(item, str(ir["skill"]["source_digest"]))
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status="synced")