feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3

Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
吉鑫升 committed 2026-09-03 19:21:10 +08:00
1 parent 49d9ced04f
commit 60e2c72514
38 files changed
+2536 -581

No files matched your search

@@ -6,6 +6,7 @@ import asyncio
import base64
import json
import logging
import re
from collections.abc import AsyncIterable, AsyncIterator
from datetime import UTC, datetime
from pathlib import Path
@@ -17,7 +18,8 @@ from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from app.gateway.deps import get_optional_user_from_request
from deerflow.assistant_knowledge.package_io import PACKAGE_VERSION, package_path, read_package, write_jsonl_record
from deerflow.assistant_knowledge.archive import PackageArchive
from deerflow.assistant_knowledge.package_io import PACKAGE_VERSION, package_path, write_jsonl_record
from deerflow.config.runtime_paths import runtime_home
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
@@ -114,7 +116,9 @@ def _write_export_manifest(
}
path.parent.mkdir(parents=True, exist_ok=True)
payload = _normalize_export_manifest(payload)
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
temp = path.with_suffix(".tmp")
temp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
temp.replace(path)
return payload
@@ -227,10 +231,11 @@ async def list_wiki_pages(
q: str | None = Query(default=None),
page_type: str | None = Query(default=None),
limit: int = Query(default=100, ge=1, le=500),
offset: int = Query(default=0, ge=0),
) -> dict[str, Any]:
await _actor(request)
pages = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=limit)
return {"pages": pages, "total": len(pages)}
pages = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=limit + 1, offset=offset)
return {"pages": pages[:limit], "total": min(len(pages), limit), "next_offset": offset + limit if len(pages) > limit else None}
@router.get("/wiki-pages/{slug:path}/revisions/compare")
@@ -279,8 +284,11 @@ async def search(
page_type: str | None = Query(default=None),
) -> dict[str, Any]:
await _actor(request)
rows = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=100)
return {"results": rows, "total": len(rows)}
store = _store(request)
base = await store.get_base(base_id)
embedding = getattr(request.app.state, "llmwiki_embedding", None)
rows = await store.search_pages(base_ids=[base["id"]] if base else [], query=q, limit=20, include_chunks=False, embedding_client=embedding)
return {"results": rows, "total": len(rows), "retrieval_mode": rows[0].get("retrieval_mode", "keyword") if rows else "empty"}
@router.get("/sources")
@@ -399,7 +407,10 @@ async def import_weknora_package_file(
if not chunk:
break
size += len(chunk)
handle.write(chunk)
await asyncio.to_thread(handle.write, chunk)
except Exception as exc:
await _store(request).update_import_job(str(job["id"]), status="failed", phase="upload_failed", error=str(exc), completed=True)
raise
finally:
await file.close()
if size <= 0:
@@ -454,6 +465,7 @@ async def _write_weknora_export_package(
"source",
{
**source,
"provider": "weknora",
"package_version": PACKAGE_VERSION,
"export_format": "assistant-knowledge-jsonl",
},
@@ -462,6 +474,8 @@ async def _write_weknora_export_package(
# Wiki pages first: these are the human-readable pages that the
# assistant knowledge base presents in wiki form.
page_no = 1
graph_nodes = []
graph_edges = set()
while True:
page = await client.list_wiki_pages(remote_id, page=page_no, page_size=100)
batch = page.get("pages") or []
@@ -475,14 +489,16 @@ async def _write_weknora_export_package(
if isinstance(detail, dict):
payload = {**row, **detail}
except WeKnoraError:
logger.warning(
"Could not fetch full WeKnora wiki page during export: kb=%s slug=%s",
remote_id,
slug,
exc_info=True,
)
# A list summary is not a replacement for the full
# article. An incomplete package must fail visibly.
raise
write_jsonl_record(handle, "wiki_page", payload)
counts["wiki_page"] += 1
graph_nodes.append({"slug": slug, "title": payload.get("title"), "page_type": payload.get("page_type"), "aliases": payload.get("aliases") or []})
for target in payload.get("out_links") or []:
graph_edges.add((slug, str(target)))
for match in re.finditer(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]", str(payload.get("content") or "")):
graph_edges.add((slug, match.group(1).strip()))
if counts["wiki_page"] >= int(page.get("total") or counts["wiki_page"]) or not batch:
break
page_no += 1
@@ -526,23 +542,25 @@ async def _write_weknora_export_package(
write_jsonl_record(handle, "vector", row)
counts["vector"] += 1
try:
graph = await client.get_wiki_graph(remote_id, limit=10000)
except WeKnoraError:
graph = {"nodes": [], "edges": [], "capability_warning": "graph API unavailable"}
# Build the complete link graph from the paginated full pages. The
# provider's overview graph endpoint is capped and can omit nodes.
known_slugs = {row["slug"] for row in graph_nodes}
graph = {"nodes": graph_nodes, "edges": [{"source": a, "target": b} for a, b in sorted(graph_edges) if a in known_slugs and b in known_slugs]}
nodes = [row for row in graph.get("nodes") or [] if isinstance(row, dict)]
edges = [row for row in graph.get("edges") or [] if isinstance(row, dict)]
write_jsonl_record(handle, "graph", {**graph, "nodes": nodes, "edges": edges})
counts["graph"] += 1
for row in nodes:
if row.get("id") and (row.get("title") or row.get("name")):
if (row.get("slug") or row.get("id")) and (row.get("title") or row.get("name")):
write_jsonl_record(
handle,
"entity",
{
"id": row.get("id"),
"id": row.get("slug") or row.get("id"),
"name": row.get("title") or row.get("name"),
"type": row.get("type") or "concept",
"type": row.get("page_type") or row.get("type") or "concept",
"source_wiki_slug": row.get("slug") or row.get("id"),
"aliases": row.get("aliases") or [],
"confidence": 1.0,
},
)
@@ -555,6 +573,7 @@ async def _write_weknora_export_package(
"source_entity_id": row.get("source"),
"target_entity_id": row.get("target"),
"predicate": row.get("predicate") or "references",
"source_wiki_slug": row.get("source"),
"confidence": 1.0,
"evidence": [{"source": "weknora_graph"}],
},
@@ -627,8 +646,9 @@ async def _iter_local_wiki_vectors_for_export(
payload = _local_wiki_vector_export_payload(row, model=model, fingerprint=fingerprint)
if payload is not None:
yield payload
except Exception: # noqa: BLE001
logger.debug("No local wiki vector snapshot available for %s", mapping_id, exc_info=True)
except Exception:
logger.exception("Wiki vector export failed for %s", mapping_id)
raise
async def _has_local_wiki_vectors_for_export(request: Request, mapping_id: str) -> bool:
@@ -646,12 +666,11 @@ async def _ensure_local_wiki_vectors_for_export(
mapping: dict[str, Any],
) -> None:
mapping_id = str(mapping.get("id") or mapping.get("knowledge_base_id") or "")
if not mapping_id or await _has_local_wiki_vectors_for_export(request, mapping_id):
return
service = getattr(request.app.state, "llmwiki_sync_service", None)
if service is None or not mapping_id:
return
raise RuntimeError("请先配置 Wiki 编码模型并启用本地向量索引,再生成向量数据包")
# Refresh remote metadata/content hashes even when the previous generation
# was complete. Unchanged articles reuse their existing vectors.
sync_mapping = dict(mapping)
sync_mapping.setdefault("id", mapping_id)
sync_mapping.setdefault(
@@ -660,15 +679,9 @@ async def _ensure_local_wiki_vectors_for_export(
)
if not sync_mapping.get("weknora_id"):
return
try:
logger.info("Building local wiki vectors before export: mapping=%s", mapping_id)
await service.sync_mapping(sync_mapping, force=False)
except Exception: # noqa: BLE001
logger.warning(
"Could not build local wiki vectors before export: mapping=%s",
mapping_id,
exc_info=True,
)
result = await service.sync_mapping(sync_mapping, force=False, notify=False)
if result.get("status") != "completed":
raise RuntimeError("Wiki 向量化尚未全部完成,请查看向量化进度后重试")
async def _import_package_file_for_job(
@@ -679,9 +692,10 @@ async def _import_package_file_for_job(
actor: str | None,
) -> None:
store = _store(request)
package = None
try:
await store.update_import_job(job_id, status="running", phase="loading_package")
package = await asyncio.to_thread(read_package, path)
package = await asyncio.to_thread(PackageArchive, path)
await store.update_import_job(job_id, status="running", phase="importing")
await store.import_package_for_job(
job_id,
@@ -698,6 +712,9 @@ async def _import_package_file_for_job(
error=str(exc),
completed=True,
)
finally:
if package is not None:
await asyncio.to_thread(package.close)
async def _export_weknora_then_import_for_job(
@@ -708,6 +725,7 @@ async def _export_weknora_then_import_for_job(
remote_id: str,
source: dict[str, Any],
actor: str | None,
index_fresh: bool = False,
) -> None:
store = _store(request)
path = package_path(_package_root(), job_id)
@@ -717,7 +735,8 @@ async def _export_weknora_then_import_for_job(
if not runtime.weknora_enabled:
raise RuntimeError("未配置普通知识库服务")
mapping_id = str(mapping.get("id") or mapping.get("knowledge_base_id") or "")
await _ensure_local_wiki_vectors_for_export(request, mapping)
if not index_fresh:
await _ensure_local_wiki_vectors_for_export(request, mapping)
counts = await _write_weknora_export_package(
build_weknora_client(runtime),
remote_id,