feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3
Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
1 parent
49d9ced04f
commit
60e2c72514
38 files changed
+2536
-581
No files matched your search
@@ -1586,6 +1586,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
app.include_router(skill_knowledge.router)
|
||||
app.include_router(assistant_knowledge.router)
|
||||
app.include_router(assistant_knowledge.export_router)
|
||||
from app.gateway.routers import wiki_packages
|
||||
|
||||
app.include_router(wiki_packages.router)
|
||||
|
||||
# Artifacts API is mounted at /api/threads/{thread_id}/artifacts
|
||||
app.include_router(artifacts.router)
|
||||
|
||||
@@ -381,7 +381,7 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
embedding_config = local_index_config.embedding
|
||||
embedding_configured = bool(
|
||||
embedding_config.enabled
|
||||
and embedding_config.base_url
|
||||
and (embedding_config.base_url or embedding_config.provider == "local")
|
||||
and embedding_config.model.strip()
|
||||
)
|
||||
if embedding_configured:
|
||||
@@ -404,13 +404,19 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app.state.llmwiki_vector_cache,
|
||||
)
|
||||
if weknora_client is not None:
|
||||
app.state.llmwiki_sync_service = WikiSyncService(
|
||||
app.state.llmwiki_sync_service = WikiSyncService(
|
||||
local_index_config,
|
||||
app.state.llmwiki_index_store,
|
||||
app.state.llmwiki_embedding,
|
||||
app.state.llmwiki_vector_cache,
|
||||
weknora_client,
|
||||
)
|
||||
weknora_client,
|
||||
)
|
||||
from app.gateway.knowledge_transfer import sync_to_global
|
||||
|
||||
async def on_wiki_vectorized(mapping):
|
||||
return await sync_to_global(app, mapping)
|
||||
|
||||
app.state.llmwiki_sync_service.on_completed = on_wiki_vectorized
|
||||
if weknora_client is not None or app.state.llmwiki_vector_search is not None:
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Wiki package flow orchestration; all remote writes use WeKnora APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from starlette.requests import Request
|
||||
|
||||
|
||||
async def sync_to_global(app, mapping):
|
||||
from app.gateway.routers.assistant_knowledge import _export_weknora_then_import_for_job
|
||||
|
||||
if mapping.get("is_conversation_deposit") or mapping.get("owner_user_id") == "system":
|
||||
return {"status": "excluded"}
|
||||
store = app.state.assistant_knowledge_store
|
||||
revision, fingerprint, _ = await app.state.llmwiki_index_store.get_index_revision(str(mapping["id"]))
|
||||
base = await store.initialize(created_by=None)
|
||||
for source in await store.list_sources(base_id=base["id"]):
|
||||
meta = source.get("metadata") or {}
|
||||
if source.get("source_key") == mapping["id"] and meta.get("index_revision") == revision:
|
||||
job = await store.get_job(source.get("last_import_job_id")) if source.get("last_import_job_id") else None
|
||||
if job and job.get("status") == "completed":
|
||||
return {"status": "unchanged", "job_id": job["id"], "base_id": base["id"]}
|
||||
source = {"provider": "weknora", "type": "weknora", "knowledge_base_id": mapping["id"], "knowledge_base_name": mapping["name"], "index_revision": revision, "embedding_fingerprint": fingerprint}
|
||||
job = await store.create_queued_import_job(source_type="weknora", source_key=mapping["id"], source_name=mapping["name"], trigger="vectorization_completed", created_by=None, metadata=source, mapping_id=mapping["id"], base_id=base["id"])
|
||||
await _export_weknora_then_import_for_job(request=Request({"type": "http", "app": app}), job_id=job["id"], mapping=mapping, remote_id=mapping["weknora_id"], source=source, actor=None, index_fresh=True)
|
||||
result = await store.get_job(job["id"])
|
||||
if not result or result.get("status") != "completed":
|
||||
raise RuntimeError(f"向量化已完成,但知识梳理同步失败:{(result or {}).get('last_error_message') or job['id']}")
|
||||
return {"status": "completed", "job_id": job["id"], "base_id": base["id"], "counts": result.get("counts")}
|
||||
@@ -6,6 +6,7 @@ import asyncio
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import AsyncIterable, AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
@@ -17,7 +18,8 @@ from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from deerflow.assistant_knowledge.package_io import PACKAGE_VERSION, package_path, read_package, write_jsonl_record
|
||||
from deerflow.assistant_knowledge.archive import PackageArchive
|
||||
from deerflow.assistant_knowledge.package_io import PACKAGE_VERSION, package_path, write_jsonl_record
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
@@ -114,7 +116,9 @@ def _write_export_manifest(
|
||||
}
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = _normalize_export_manifest(payload)
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
temp = path.with_suffix(".tmp")
|
||||
temp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
temp.replace(path)
|
||||
return payload
|
||||
|
||||
|
||||
@@ -227,10 +231,11 @@ async def list_wiki_pages(
|
||||
q: str | None = Query(default=None),
|
||||
page_type: str | None = Query(default=None),
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
pages = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=limit)
|
||||
return {"pages": pages, "total": len(pages)}
|
||||
pages = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=limit + 1, offset=offset)
|
||||
return {"pages": pages[:limit], "total": min(len(pages), limit), "next_offset": offset + limit if len(pages) > limit else None}
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}/revisions/compare")
|
||||
@@ -279,8 +284,11 @@ async def search(
|
||||
page_type: str | None = Query(default=None),
|
||||
) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
rows = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=100)
|
||||
return {"results": rows, "total": len(rows)}
|
||||
store = _store(request)
|
||||
base = await store.get_base(base_id)
|
||||
embedding = getattr(request.app.state, "llmwiki_embedding", None)
|
||||
rows = await store.search_pages(base_ids=[base["id"]] if base else [], query=q, limit=20, include_chunks=False, embedding_client=embedding)
|
||||
return {"results": rows, "total": len(rows), "retrieval_mode": rows[0].get("retrieval_mode", "keyword") if rows else "empty"}
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
@@ -399,7 +407,10 @@ async def import_weknora_package_file(
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
handle.write(chunk)
|
||||
await asyncio.to_thread(handle.write, chunk)
|
||||
except Exception as exc:
|
||||
await _store(request).update_import_job(str(job["id"]), status="failed", phase="upload_failed", error=str(exc), completed=True)
|
||||
raise
|
||||
finally:
|
||||
await file.close()
|
||||
if size <= 0:
|
||||
@@ -454,6 +465,7 @@ async def _write_weknora_export_package(
|
||||
"source",
|
||||
{
|
||||
**source,
|
||||
"provider": "weknora",
|
||||
"package_version": PACKAGE_VERSION,
|
||||
"export_format": "assistant-knowledge-jsonl",
|
||||
},
|
||||
@@ -462,6 +474,8 @@ async def _write_weknora_export_package(
|
||||
# Wiki pages first: these are the human-readable pages that the
|
||||
# assistant knowledge base presents in wiki form.
|
||||
page_no = 1
|
||||
graph_nodes = []
|
||||
graph_edges = set()
|
||||
while True:
|
||||
page = await client.list_wiki_pages(remote_id, page=page_no, page_size=100)
|
||||
batch = page.get("pages") or []
|
||||
@@ -475,14 +489,16 @@ async def _write_weknora_export_package(
|
||||
if isinstance(detail, dict):
|
||||
payload = {**row, **detail}
|
||||
except WeKnoraError:
|
||||
logger.warning(
|
||||
"Could not fetch full WeKnora wiki page during export: kb=%s slug=%s",
|
||||
remote_id,
|
||||
slug,
|
||||
exc_info=True,
|
||||
)
|
||||
# A list summary is not a replacement for the full
|
||||
# article. An incomplete package must fail visibly.
|
||||
raise
|
||||
write_jsonl_record(handle, "wiki_page", payload)
|
||||
counts["wiki_page"] += 1
|
||||
graph_nodes.append({"slug": slug, "title": payload.get("title"), "page_type": payload.get("page_type"), "aliases": payload.get("aliases") or []})
|
||||
for target in payload.get("out_links") or []:
|
||||
graph_edges.add((slug, str(target)))
|
||||
for match in re.finditer(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]", str(payload.get("content") or "")):
|
||||
graph_edges.add((slug, match.group(1).strip()))
|
||||
if counts["wiki_page"] >= int(page.get("total") or counts["wiki_page"]) or not batch:
|
||||
break
|
||||
page_no += 1
|
||||
@@ -526,23 +542,25 @@ async def _write_weknora_export_package(
|
||||
write_jsonl_record(handle, "vector", row)
|
||||
counts["vector"] += 1
|
||||
|
||||
try:
|
||||
graph = await client.get_wiki_graph(remote_id, limit=10000)
|
||||
except WeKnoraError:
|
||||
graph = {"nodes": [], "edges": [], "capability_warning": "graph API unavailable"}
|
||||
# Build the complete link graph from the paginated full pages. The
|
||||
# provider's overview graph endpoint is capped and can omit nodes.
|
||||
known_slugs = {row["slug"] for row in graph_nodes}
|
||||
graph = {"nodes": graph_nodes, "edges": [{"source": a, "target": b} for a, b in sorted(graph_edges) if a in known_slugs and b in known_slugs]}
|
||||
nodes = [row for row in graph.get("nodes") or [] if isinstance(row, dict)]
|
||||
edges = [row for row in graph.get("edges") or [] if isinstance(row, dict)]
|
||||
write_jsonl_record(handle, "graph", {**graph, "nodes": nodes, "edges": edges})
|
||||
counts["graph"] += 1
|
||||
for row in nodes:
|
||||
if row.get("id") and (row.get("title") or row.get("name")):
|
||||
if (row.get("slug") or row.get("id")) and (row.get("title") or row.get("name")):
|
||||
write_jsonl_record(
|
||||
handle,
|
||||
"entity",
|
||||
{
|
||||
"id": row.get("id"),
|
||||
"id": row.get("slug") or row.get("id"),
|
||||
"name": row.get("title") or row.get("name"),
|
||||
"type": row.get("type") or "concept",
|
||||
"type": row.get("page_type") or row.get("type") or "concept",
|
||||
"source_wiki_slug": row.get("slug") or row.get("id"),
|
||||
"aliases": row.get("aliases") or [],
|
||||
"confidence": 1.0,
|
||||
},
|
||||
)
|
||||
@@ -555,6 +573,7 @@ async def _write_weknora_export_package(
|
||||
"source_entity_id": row.get("source"),
|
||||
"target_entity_id": row.get("target"),
|
||||
"predicate": row.get("predicate") or "references",
|
||||
"source_wiki_slug": row.get("source"),
|
||||
"confidence": 1.0,
|
||||
"evidence": [{"source": "weknora_graph"}],
|
||||
},
|
||||
@@ -627,8 +646,9 @@ async def _iter_local_wiki_vectors_for_export(
|
||||
payload = _local_wiki_vector_export_payload(row, model=model, fingerprint=fingerprint)
|
||||
if payload is not None:
|
||||
yield payload
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("No local wiki vector snapshot available for %s", mapping_id, exc_info=True)
|
||||
except Exception:
|
||||
logger.exception("Wiki vector export failed for %s", mapping_id)
|
||||
raise
|
||||
|
||||
|
||||
async def _has_local_wiki_vectors_for_export(request: Request, mapping_id: str) -> bool:
|
||||
@@ -646,12 +666,11 @@ async def _ensure_local_wiki_vectors_for_export(
|
||||
mapping: dict[str, Any],
|
||||
) -> None:
|
||||
mapping_id = str(mapping.get("id") or mapping.get("knowledge_base_id") or "")
|
||||
if not mapping_id or await _has_local_wiki_vectors_for_export(request, mapping_id):
|
||||
return
|
||||
|
||||
service = getattr(request.app.state, "llmwiki_sync_service", None)
|
||||
if service is None or not mapping_id:
|
||||
return
|
||||
raise RuntimeError("请先配置 Wiki 编码模型并启用本地向量索引,再生成向量数据包")
|
||||
# Refresh remote metadata/content hashes even when the previous generation
|
||||
# was complete. Unchanged articles reuse their existing vectors.
|
||||
sync_mapping = dict(mapping)
|
||||
sync_mapping.setdefault("id", mapping_id)
|
||||
sync_mapping.setdefault(
|
||||
@@ -660,15 +679,9 @@ async def _ensure_local_wiki_vectors_for_export(
|
||||
)
|
||||
if not sync_mapping.get("weknora_id"):
|
||||
return
|
||||
try:
|
||||
logger.info("Building local wiki vectors before export: mapping=%s", mapping_id)
|
||||
await service.sync_mapping(sync_mapping, force=False)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning(
|
||||
"Could not build local wiki vectors before export: mapping=%s",
|
||||
mapping_id,
|
||||
exc_info=True,
|
||||
)
|
||||
result = await service.sync_mapping(sync_mapping, force=False, notify=False)
|
||||
if result.get("status") != "completed":
|
||||
raise RuntimeError("Wiki 向量化尚未全部完成,请查看向量化进度后重试")
|
||||
|
||||
|
||||
async def _import_package_file_for_job(
|
||||
@@ -679,9 +692,10 @@ async def _import_package_file_for_job(
|
||||
actor: str | None,
|
||||
) -> None:
|
||||
store = _store(request)
|
||||
package = None
|
||||
try:
|
||||
await store.update_import_job(job_id, status="running", phase="loading_package")
|
||||
package = await asyncio.to_thread(read_package, path)
|
||||
package = await asyncio.to_thread(PackageArchive, path)
|
||||
await store.update_import_job(job_id, status="running", phase="importing")
|
||||
await store.import_package_for_job(
|
||||
job_id,
|
||||
@@ -698,6 +712,9 @@ async def _import_package_file_for_job(
|
||||
error=str(exc),
|
||||
completed=True,
|
||||
)
|
||||
finally:
|
||||
if package is not None:
|
||||
await asyncio.to_thread(package.close)
|
||||
|
||||
|
||||
async def _export_weknora_then_import_for_job(
|
||||
@@ -708,6 +725,7 @@ async def _export_weknora_then_import_for_job(
|
||||
remote_id: str,
|
||||
source: dict[str, Any],
|
||||
actor: str | None,
|
||||
index_fresh: bool = False,
|
||||
) -> None:
|
||||
store = _store(request)
|
||||
path = package_path(_package_root(), job_id)
|
||||
@@ -717,7 +735,8 @@ async def _export_weknora_then_import_for_job(
|
||||
if not runtime.weknora_enabled:
|
||||
raise RuntimeError("未配置普通知识库服务")
|
||||
mapping_id = str(mapping.get("id") or mapping.get("knowledge_base_id") or "")
|
||||
await _ensure_local_wiki_vectors_for_export(request, mapping)
|
||||
if not index_fresh:
|
||||
await _ensure_local_wiki_vectors_for_export(request, mapping)
|
||||
counts = await _write_weknora_export_package(
|
||||
build_weknora_client(runtime),
|
||||
remote_id,
|
||||
|
||||
@@ -2041,6 +2041,7 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
|
||||
if not slug and not title and not content and not summary:
|
||||
return None
|
||||
return {
|
||||
"page_metadata": page.get("page_metadata") if isinstance(page.get("page_metadata"), dict) else {},
|
||||
"id": str(page.get("id") or ""),
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
@@ -2987,12 +2988,11 @@ async def list_knowledge_bases(
|
||||
knowledge_bases: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
remote_item = remote.get(str(row["weknora_id"]))
|
||||
row, remote_item = await _sync_conversation_deposit_view(_store(request), client, row, remote_item)
|
||||
# Owners need to see stale personal mappings so they can clean them up.
|
||||
# Public/selector views must never expose a resource that no longer exists
|
||||
# in WeKnora.
|
||||
if remote_item is None and scope != "personal":
|
||||
# A successful remote list is authoritative for every scope. Keep the
|
||||
# mapping for existing references, but never display a deleted base.
|
||||
if remote_item is None:
|
||||
continue
|
||||
row, remote_item = await _sync_conversation_deposit_view(_store(request), client, row, remote_item)
|
||||
knowledge_bases.append(_mapping_view(row, remote_item, actor_user_id=user_id, is_admin=is_admin))
|
||||
return {"knowledge_bases": knowledge_bases}
|
||||
|
||||
@@ -3077,13 +3077,19 @@ async def get_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any
|
||||
async def update_knowledge_base(request: Request, mapping_id: str, body: KnowledgeBaseUpdate) -> dict[str, Any]:
|
||||
row, user_id, is_admin = await _authorized_mapping(request, mapping_id, write=True)
|
||||
changes = body.model_dump(exclude_none=True)
|
||||
if "name" in changes:
|
||||
changes["name"] = changes["name"].strip()
|
||||
if not changes["name"]:
|
||||
raise HTTPException(status_code=422, detail="知识库名称不能为空")
|
||||
if changes["name"] == CONVERSATION_DEPOSIT_KB_NAME:
|
||||
raise HTTPException(status_code=409, detail="对话沉淀是 cmzs 系统保留知识库")
|
||||
if not changes:
|
||||
return _mapping_view(row, None, actor_user_id=user_id, is_admin=is_admin)
|
||||
try:
|
||||
remote = await _client_or_503().update_knowledge_base(str(row["weknora_id"]), changes)
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
updated = await _store(request).update_mapping(mapping_id, name=body.name, description=body.description)
|
||||
updated = await _store(request).update_mapping(mapping_id, name=changes.get("name"), description=changes.get("description"))
|
||||
return _mapping_view(updated or row, remote, actor_user_id=user_id, is_admin=is_admin)
|
||||
|
||||
|
||||
|
||||
@@ -313,7 +313,7 @@ async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[st
|
||||
if not mapping.get("wiki_index_enabled", True):
|
||||
mapping = await mapping_store.update_mapping(mapping_id, wiki_index_enabled=True)
|
||||
job_id = str(uuid4())
|
||||
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=True))
|
||||
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=False))
|
||||
return {"job_id": job_id, "status": "accepted", "knowledge_base_id": mapping_id}
|
||||
|
||||
|
||||
|
||||
@@ -52,9 +52,9 @@ class SyncTarget(BaseModel):
|
||||
|
||||
class CreateSyncJobRequest(BaseModel):
|
||||
skill_names: list[str] = Field(min_length=1, max_length=200)
|
||||
targets: list[SyncTarget] = Field(min_length=1, max_length=50)
|
||||
targets: list[SyncTarget] = Field(min_length=1, max_length=1)
|
||||
force: bool = False
|
||||
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
|
||||
review_mode: Literal["required", "auto_high_confidence", "off"] = "off"
|
||||
|
||||
@field_validator("skill_names")
|
||||
@classmethod
|
||||
@@ -165,8 +165,14 @@ async def disable_binding_auto_sync(request: Request, binding_id: str) -> dict[s
|
||||
@router.post("/sync-jobs", status_code=202)
|
||||
async def create_sync_job(request: Request, body: CreateSyncJobRequest) -> dict[str, Any]:
|
||||
actor = await _require_admin(request)
|
||||
skill_store = request.app.state.skill_store
|
||||
missing = [name for name in body.skill_names if await skill_store.get_any(name) is None]
|
||||
# Built-in and legacy skills need not have an ownership row. Validate
|
||||
# against the same running filesystem used by the skills list/executor.
|
||||
missing = []
|
||||
for name in body.skill_names:
|
||||
try:
|
||||
await asyncio.to_thread(_skill_directory, request.app.state.config, name)
|
||||
except (FileNotFoundError, ValueError):
|
||||
missing.append(name)
|
||||
if missing:
|
||||
raise HTTPException(status_code=404, detail=f"技能不存在:{', '.join(missing[:10])}")
|
||||
targets: list[dict[str, Any]] = []
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Asynchronous, file-backed assistant export and ordinary Wiki restore."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
||||
from app.gateway.routers.assistant_knowledge import _actor, _list_export_manifests, _package_root, _read_export_manifest, _store, _write_export_manifest
|
||||
from deerflow.assistant_knowledge.archive import PackageArchive
|
||||
from deerflow.assistant_knowledge.package_io import package_path, write_jsonl_record
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.persistence.assistant_knowledge.store import _vector_blob_from_payload
|
||||
|
||||
router = APIRouter(tags=["wiki-packages"])
|
||||
|
||||
|
||||
def job_view(row):
|
||||
if row["mapping_id"].startswith("assistant:"):
|
||||
base_id = row["mapping_id"].split(":", 1)[1]
|
||||
row["package_download_url"] = f"/api/assistant-knowledge/bases/{base_id}/export-package/{row['id']}/download"
|
||||
elif row["mapping_id"].startswith("restore:"):
|
||||
row["package_download_url"] = None
|
||||
return row
|
||||
|
||||
|
||||
async def export_assistant(app, base_id, job_id):
|
||||
scope = f"assistant:{base_id}"
|
||||
path = package_path(_package_root(), job_id)
|
||||
temp = path.with_suffix(".tmp")
|
||||
counts = {}
|
||||
try:
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="exporting")
|
||||
with temp.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
async for kind, payload in app.state.assistant_knowledge_store.iter_export_records(base_id):
|
||||
await asyncio.to_thread(write_jsonl_record, handle, kind, payload)
|
||||
counts[kind] = counts.get(kind, 0) + 1
|
||||
if counts.get("wiki_page") and not counts.get("vector"):
|
||||
raise ValueError("该助手知识库尚无可导出的 Wiki 向量,请先从已向量化的普通知识库同步")
|
||||
await asyncio.to_thread(temp.replace, path)
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="completed", phase="package_ready", counts=counts)
|
||||
except Exception as exc:
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="failed", counts=counts, error=str(exc))
|
||||
|
||||
|
||||
@router.post("/api/assistant-knowledge/bases/{base_id}/export-package", status_code=202)
|
||||
async def create_export(request: Request, base_id: str):
|
||||
await _actor(request, admin=True)
|
||||
if not await _store(request).get_base(base_id):
|
||||
raise HTTPException(404, "助手知识库不存在")
|
||||
job_id = str(uuid4())
|
||||
row = _write_export_manifest(job_id=job_id, mapping_id=f"assistant:{base_id}", status="queued", phase="queued")
|
||||
spawn_local_wiki_index_job(request.app, export_assistant(request.app, base_id, job_id))
|
||||
return job_view(row)
|
||||
|
||||
|
||||
@router.get("/api/assistant-knowledge/bases/{base_id}/export-package")
|
||||
async def list_exports(request: Request, base_id: str):
|
||||
await _actor(request, admin=True)
|
||||
return {"jobs": [job_view(row) for row in await asyncio.to_thread(_list_export_manifests, f"assistant:{base_id}", limit=100)]}
|
||||
|
||||
|
||||
@router.get("/api/assistant-knowledge/bases/{base_id}/export-package/{job_id}/download")
|
||||
async def download_export(request: Request, base_id: str, job_id: str):
|
||||
await _actor(request, admin=True)
|
||||
row = _read_export_manifest(job_id)
|
||||
if not row or row["mapping_id"] != f"assistant:{base_id}" or row["status"] != "completed" or not row["file_exists"]:
|
||||
raise HTTPException(404, "导出包尚未就绪")
|
||||
return FileResponse(package_path(_package_root(), job_id), media_type="application/x-ndjson", filename=f"wiki-vector-{job_id}.jsonl")
|
||||
|
||||
|
||||
async def restore_package(app, mapping, job_id, path: Path):
|
||||
scope = f"restore:{mapping['id']}"
|
||||
archive = None
|
||||
counts = {"wiki_page": 0, "vector": 0}
|
||||
store = app.state.llmwiki_index_store
|
||||
owner = f"package:{job_id}"
|
||||
leased = False
|
||||
try:
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="validating")
|
||||
archive = await asyncio.to_thread(PackageArchive, path)
|
||||
embedding = app.state.llmwiki_embedding
|
||||
if embedding is None:
|
||||
raise ValueError("请配置与数据包相同的 Wiki 编码模型,以便复用向量进行检索")
|
||||
if embedding.dimensions is None:
|
||||
await embedding.embed_query("Wiki embedding dimension probe")
|
||||
# Validate the complete package before the first remote mutation.
|
||||
slugs = {p["slug"] for p in archive["wiki_pages"]}
|
||||
vector_slugs = set()
|
||||
for raw in archive["vectors"]:
|
||||
slug = str(raw.get("wiki_slug") or raw.get("page_slug") or raw.get("slug") or "")
|
||||
if slug not in slugs or _vector_blob_from_payload(raw) is None:
|
||||
raise ValueError(f"向量数据或 Wiki 引用无效:{slug}")
|
||||
if raw.get("embedding_fingerprint") != embedding.fingerprint or raw.get("embedding_dimensions") != embedding.dimensions:
|
||||
raise ValueError("数据包编码模型指纹与当前配置不一致;请使用同一模型配置,禁止静默重新编码")
|
||||
vector_slugs.add(slug)
|
||||
if not vector_slugs:
|
||||
raise ValueError("数据包中没有 Wiki 向量")
|
||||
for page in archive["wiki_pages"]:
|
||||
if page["slug"] not in vector_slugs and page.get("page_type") not in {"index", "directory"} and (page.get("content") or "").strip():
|
||||
raise ValueError(f"Wiki 缺少向量:{page['slug']}")
|
||||
leased = await store.try_acquire_sync_lease(mapping["id"], owner=owner, sync_id=job_id, lease_seconds=900, fingerprint=embedding.fingerprint)
|
||||
if not leased:
|
||||
raise ValueError("目标知识库正在同步,请稍后重试")
|
||||
client = build_weknora_client(get_resolved_llmwiki_runtime(app.state.config))
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="restoring")
|
||||
folders = {(): ""}
|
||||
for raw in archive["wiki_pages"]:
|
||||
if not await store.renew_sync_lease(mapping["id"], owner=owner, lease_seconds=900):
|
||||
raise RuntimeError("导入租约已失效,请重新发起")
|
||||
# Restore the provider's Wiki schema, never raw documents; raw
|
||||
# uploads would trigger generation and embedding a second time.
|
||||
categories = raw.get("category_path") or []
|
||||
if isinstance(categories, str):
|
||||
categories = [part for part in categories.split("/") if part]
|
||||
parent = ()
|
||||
for name in categories:
|
||||
path_key = (*parent, name)
|
||||
if path_key not in folders:
|
||||
folder = await client.create_wiki_folder(mapping["weknora_id"], name=name, parent_id=folders[parent])
|
||||
folders[path_key] = folder["id"]
|
||||
parent = path_key
|
||||
payload = {key: value for key, value in raw.items() if key not in {"id", "tenant_id", "knowledge_base_id", "folder_id", "origins"}}
|
||||
payload["folder_id"] = folders[parent]
|
||||
if raw.get("tags"):
|
||||
payload["page_metadata"] = {**(raw.get("page_metadata") or {}), "tags": raw["tags"]}
|
||||
remote = await client.create_wiki_page(mapping["weknora_id"], payload)
|
||||
page = normalize_wiki_page(mapping["id"], {**raw, **remote})
|
||||
vectors = []
|
||||
for value in archive.vectors_for(raw["slug"]):
|
||||
blob, dimensions = _vector_blob_from_payload(value)
|
||||
vectors.append({**value, "vector_blob": blob, "embedding_dimensions": dimensions})
|
||||
await store.replace_page_vectors(page=page, vectors=vectors, fingerprint=embedding.fingerprint, sync_id=job_id)
|
||||
counts["wiki_page"] += 1
|
||||
counts["vector"] += len(vectors)
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="restoring", counts=counts)
|
||||
await client.rebuild_wiki_links(mapping["weknora_id"])
|
||||
await store.finish_sync(mapping["id"], owner=owner, sync_id=job_id, success=True, remote_page_count=counts["wiki_page"])
|
||||
leased = False
|
||||
if app.state.llmwiki_vector_cache:
|
||||
app.state.llmwiki_vector_cache.invalidate(mapping["id"])
|
||||
from app.gateway.knowledge_transfer import sync_to_global
|
||||
|
||||
await sync_to_global(app, mapping)
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="completed", phase="completed", counts=counts)
|
||||
except Exception as exc:
|
||||
if leased:
|
||||
await store.finish_sync(mapping["id"], owner=owner, sync_id=job_id, success=False, remote_page_count=counts["wiki_page"], error=str(exc))
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="failed", counts=counts, error=str(exc))
|
||||
finally:
|
||||
if archive is not None:
|
||||
await asyncio.to_thread(archive.close)
|
||||
|
||||
|
||||
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/import-package", status_code=202)
|
||||
async def upload_restore(request: Request, mapping_id: str, file: UploadFile = File(...)):
|
||||
actor = await _actor(request, admin=True)
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, actor or "system", write=True, is_admin=True)
|
||||
if not mapping or mapping.get("is_conversation_deposit") or mapping.get("owner_user_id") == "system":
|
||||
raise HTTPException(404, "目标普通知识库不存在或不可导入")
|
||||
client = build_weknora_client(get_resolved_llmwiki_runtime(request.app.state.config))
|
||||
remote = await client.get_knowledge_base(mapping["weknora_id"])
|
||||
if not (remote.get("indexing_strategy") or {}).get("wiki_enabled"):
|
||||
raise HTTPException(422, "请先开启目标知识库的 Wiki")
|
||||
existing = await client.list_wiki_pages(mapping["weknora_id"], page_size=1)
|
||||
if existing.get("total"):
|
||||
raise HTTPException(409, "请导入到新建的空 Wiki 知识库,避免覆盖已有页面")
|
||||
job_id = str(uuid4())
|
||||
scope = f"restore:{mapping_id}"
|
||||
row = _write_export_manifest(job_id=job_id, mapping_id=scope, status="queued", phase="uploading")
|
||||
path = package_path(_package_root(), job_id)
|
||||
try:
|
||||
with path.open("wb") as handle:
|
||||
while data := await file.read(1024 * 1024):
|
||||
await asyncio.to_thread(handle.write, data)
|
||||
except Exception as exc:
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="upload_failed", error=str(exc))
|
||||
raise
|
||||
finally:
|
||||
await file.close()
|
||||
spawn_local_wiki_index_job(request.app, restore_package(request.app, mapping, job_id, path))
|
||||
return job_view(row)
|
||||
|
||||
|
||||
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/import-package")
|
||||
async def list_restores(request: Request, mapping_id: str):
|
||||
await _actor(request, admin=True)
|
||||
return {"jobs": [job_view(row) for row in await asyncio.to_thread(_list_export_manifests, f"restore:{mapping_id}", limit=100)]}
|
||||
@@ -27,7 +27,7 @@ def _skill_directory(app_config: Any, skill_name: str) -> Path:
|
||||
root = storage.get_skills_root_path().resolve()
|
||||
for category in SkillCategory:
|
||||
direct = root / category.value / skill_name
|
||||
if (direct / "SKILL.md").is_file():
|
||||
if direct.resolve().is_relative_to((root / category.value).resolve()) and (direct / "SKILL.md").is_file():
|
||||
return direct
|
||||
for category in SkillCategory:
|
||||
category_root = root / category.value
|
||||
@@ -235,6 +235,9 @@ class SkillKnowledgeJobExecutor:
|
||||
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
await self._assert_source_unchanged(item, expected_digest)
|
||||
if item["target_type"] == "weknora" and item["target_mode"] == "wiki":
|
||||
await self._generate_vectorize_and_deposit(item, snapshot_id, ir, scan)
|
||||
return
|
||||
await self.store.update_item(str(item["id"]), phase="writing_wiki", progress=65)
|
||||
target_type = str(item["target_type"])
|
||||
artifacts: list[dict[str, Any]] = []
|
||||
@@ -401,3 +404,63 @@ class SkillKnowledgeJobExecutor:
|
||||
await self._assert_source_unchanged(item, expected_digest)
|
||||
await self.store.update_item(str(item["id"]), phase="activating", progress=90)
|
||||
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status=binding_status)
|
||||
|
||||
async def _generate_vectorize_and_deposit(self, item, snapshot_id, ir, scan):
|
||||
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
|
||||
if mapping is None:
|
||||
raise ValueError("目标普通知识库不存在")
|
||||
sync = getattr(self.app.state, "llmwiki_sync_service", None)
|
||||
if sync is None:
|
||||
raise ValueError("请先配置 Wiki 编码模型并启用本地向量索引;归纳需要完成向量化后同步知识梳理")
|
||||
client = build_weknora_client(get_resolved_llmwiki_runtime(self.app.state.config))
|
||||
remote_id = mapping["weknora_id"]
|
||||
remote = await client.get_knowledge_base(remote_id)
|
||||
if not (remote.get("indexing_strategy") or {}).get("wiki_enabled"):
|
||||
raise ValueError("目标普通知识库尚未开启 Wiki")
|
||||
artifacts = []
|
||||
document_ids = []
|
||||
await self.store.update_item(str(item["id"]), phase="writing_sources", progress=55)
|
||||
for file in scan.files:
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
if not file.text:
|
||||
continue
|
||||
document = await client.create_manual_document(remote_id, title=f"{ir['skill'].get('display_name') or item['skill_name']} · {file.path}", content=file.text)
|
||||
doc_id = str(document.get("id") or document.get("knowledge_id") or "")
|
||||
if not doc_id:
|
||||
raise ValueError("资料上传未返回文档标识")
|
||||
document_ids.append(doc_id)
|
||||
artifacts.append({"kind": "raw_source", "slug": file.path, "remote_id": doc_id})
|
||||
if not document_ids:
|
||||
raise ValueError("该技能没有可归纳的知识正文,代码、脚本和代码块不参与归纳")
|
||||
await self.store.update_item(str(item["id"]), phase="generating_wiki", progress=65)
|
||||
# WeKnora performs model extraction, categorization and linking. Wait
|
||||
# for both parsing and Wiki's debounced finalize before vectorization.
|
||||
stable = 0
|
||||
for _ in range(720):
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
docs = [await client.get_document(doc_id) for doc_id in document_ids]
|
||||
if any(doc.get("parse_status") in {"failed", "error"} for doc in docs):
|
||||
raise RuntimeError("WeKnora 资料解析失败,请查看普通知识库中的文档状态")
|
||||
stats = await client.get_wiki_stats(remote_id)
|
||||
done = all(doc.get("parse_status") == "completed" for doc in docs) and not stats.get("is_active") and not stats.get("pending_tasks")
|
||||
stable = stable + 1 if done else 0
|
||||
if stable >= 3:
|
||||
if not int(stats.get("total_pages") or 0):
|
||||
raise RuntimeError("WeKnora 解析已结束,但未生成 Wiki;请检查 Wiki 生成模型的调用错误、余额及资料是否含实质知识,再重试")
|
||||
break
|
||||
await asyncio.sleep(5)
|
||||
else:
|
||||
raise TimeoutError("Wiki 分析尚未完成;请检查普通知识库的生成任务后重新归纳")
|
||||
await self.store.update_item(str(item["id"]), phase="vectorizing_wiki", progress=80)
|
||||
result = await sync.sync_mapping(mapping, force=False, notify=False)
|
||||
if result.get("status") != "completed":
|
||||
raise RuntimeError("Wiki 向量化未全部完成,请检查向量化进度")
|
||||
await self.store.update_item(str(item["id"]), phase="syncing_assistant", progress=92)
|
||||
from app.gateway.knowledge_transfer import sync_to_global
|
||||
|
||||
deposited = await sync_to_global(self.app, mapping)
|
||||
artifacts.append({"kind": "assistant_deposit", "slug": "knowledge-summary", "metadata": deposited})
|
||||
await self._assert_source_unchanged(item, str(ir["skill"]["source_digest"]))
|
||||
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status="synced")
|
||||
Reference in new issue
Block a user