feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3
Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
1 parent
49d9ced04f
commit
60e2c72514
38 files changed
+2536
-581
No files matched your search
@@ -6,6 +6,7 @@ import asyncio
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import AsyncIterable, AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
@@ -17,7 +18,8 @@ from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from deerflow.assistant_knowledge.package_io import PACKAGE_VERSION, package_path, read_package, write_jsonl_record
|
||||
from deerflow.assistant_knowledge.archive import PackageArchive
|
||||
from deerflow.assistant_knowledge.package_io import PACKAGE_VERSION, package_path, write_jsonl_record
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
@@ -114,7 +116,9 @@ def _write_export_manifest(
|
||||
}
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = _normalize_export_manifest(payload)
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
temp = path.with_suffix(".tmp")
|
||||
temp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
temp.replace(path)
|
||||
return payload
|
||||
|
||||
|
||||
@@ -227,10 +231,11 @@ async def list_wiki_pages(
|
||||
q: str | None = Query(default=None),
|
||||
page_type: str | None = Query(default=None),
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
pages = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=limit)
|
||||
return {"pages": pages, "total": len(pages)}
|
||||
pages = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=limit + 1, offset=offset)
|
||||
return {"pages": pages[:limit], "total": min(len(pages), limit), "next_offset": offset + limit if len(pages) > limit else None}
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}/revisions/compare")
|
||||
@@ -279,8 +284,11 @@ async def search(
|
||||
page_type: str | None = Query(default=None),
|
||||
) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
rows = await _store(request).list_pages(base_id=base_id, query=q, page_type=page_type, limit=100)
|
||||
return {"results": rows, "total": len(rows)}
|
||||
store = _store(request)
|
||||
base = await store.get_base(base_id)
|
||||
embedding = getattr(request.app.state, "llmwiki_embedding", None)
|
||||
rows = await store.search_pages(base_ids=[base["id"]] if base else [], query=q, limit=20, include_chunks=False, embedding_client=embedding)
|
||||
return {"results": rows, "total": len(rows), "retrieval_mode": rows[0].get("retrieval_mode", "keyword") if rows else "empty"}
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
@@ -399,7 +407,10 @@ async def import_weknora_package_file(
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
handle.write(chunk)
|
||||
await asyncio.to_thread(handle.write, chunk)
|
||||
except Exception as exc:
|
||||
await _store(request).update_import_job(str(job["id"]), status="failed", phase="upload_failed", error=str(exc), completed=True)
|
||||
raise
|
||||
finally:
|
||||
await file.close()
|
||||
if size <= 0:
|
||||
@@ -454,6 +465,7 @@ async def _write_weknora_export_package(
|
||||
"source",
|
||||
{
|
||||
**source,
|
||||
"provider": "weknora",
|
||||
"package_version": PACKAGE_VERSION,
|
||||
"export_format": "assistant-knowledge-jsonl",
|
||||
},
|
||||
@@ -462,6 +474,8 @@ async def _write_weknora_export_package(
|
||||
# Wiki pages first: these are the human-readable pages that the
|
||||
# assistant knowledge base presents in wiki form.
|
||||
page_no = 1
|
||||
graph_nodes = []
|
||||
graph_edges = set()
|
||||
while True:
|
||||
page = await client.list_wiki_pages(remote_id, page=page_no, page_size=100)
|
||||
batch = page.get("pages") or []
|
||||
@@ -475,14 +489,16 @@ async def _write_weknora_export_package(
|
||||
if isinstance(detail, dict):
|
||||
payload = {**row, **detail}
|
||||
except WeKnoraError:
|
||||
logger.warning(
|
||||
"Could not fetch full WeKnora wiki page during export: kb=%s slug=%s",
|
||||
remote_id,
|
||||
slug,
|
||||
exc_info=True,
|
||||
)
|
||||
# A list summary is not a replacement for the full
|
||||
# article. An incomplete package must fail visibly.
|
||||
raise
|
||||
write_jsonl_record(handle, "wiki_page", payload)
|
||||
counts["wiki_page"] += 1
|
||||
graph_nodes.append({"slug": slug, "title": payload.get("title"), "page_type": payload.get("page_type"), "aliases": payload.get("aliases") or []})
|
||||
for target in payload.get("out_links") or []:
|
||||
graph_edges.add((slug, str(target)))
|
||||
for match in re.finditer(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]", str(payload.get("content") or "")):
|
||||
graph_edges.add((slug, match.group(1).strip()))
|
||||
if counts["wiki_page"] >= int(page.get("total") or counts["wiki_page"]) or not batch:
|
||||
break
|
||||
page_no += 1
|
||||
@@ -526,23 +542,25 @@ async def _write_weknora_export_package(
|
||||
write_jsonl_record(handle, "vector", row)
|
||||
counts["vector"] += 1
|
||||
|
||||
try:
|
||||
graph = await client.get_wiki_graph(remote_id, limit=10000)
|
||||
except WeKnoraError:
|
||||
graph = {"nodes": [], "edges": [], "capability_warning": "graph API unavailable"}
|
||||
# Build the complete link graph from the paginated full pages. The
|
||||
# provider's overview graph endpoint is capped and can omit nodes.
|
||||
known_slugs = {row["slug"] for row in graph_nodes}
|
||||
graph = {"nodes": graph_nodes, "edges": [{"source": a, "target": b} for a, b in sorted(graph_edges) if a in known_slugs and b in known_slugs]}
|
||||
nodes = [row for row in graph.get("nodes") or [] if isinstance(row, dict)]
|
||||
edges = [row for row in graph.get("edges") or [] if isinstance(row, dict)]
|
||||
write_jsonl_record(handle, "graph", {**graph, "nodes": nodes, "edges": edges})
|
||||
counts["graph"] += 1
|
||||
for row in nodes:
|
||||
if row.get("id") and (row.get("title") or row.get("name")):
|
||||
if (row.get("slug") or row.get("id")) and (row.get("title") or row.get("name")):
|
||||
write_jsonl_record(
|
||||
handle,
|
||||
"entity",
|
||||
{
|
||||
"id": row.get("id"),
|
||||
"id": row.get("slug") or row.get("id"),
|
||||
"name": row.get("title") or row.get("name"),
|
||||
"type": row.get("type") or "concept",
|
||||
"type": row.get("page_type") or row.get("type") or "concept",
|
||||
"source_wiki_slug": row.get("slug") or row.get("id"),
|
||||
"aliases": row.get("aliases") or [],
|
||||
"confidence": 1.0,
|
||||
},
|
||||
)
|
||||
@@ -555,6 +573,7 @@ async def _write_weknora_export_package(
|
||||
"source_entity_id": row.get("source"),
|
||||
"target_entity_id": row.get("target"),
|
||||
"predicate": row.get("predicate") or "references",
|
||||
"source_wiki_slug": row.get("source"),
|
||||
"confidence": 1.0,
|
||||
"evidence": [{"source": "weknora_graph"}],
|
||||
},
|
||||
@@ -627,8 +646,9 @@ async def _iter_local_wiki_vectors_for_export(
|
||||
payload = _local_wiki_vector_export_payload(row, model=model, fingerprint=fingerprint)
|
||||
if payload is not None:
|
||||
yield payload
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("No local wiki vector snapshot available for %s", mapping_id, exc_info=True)
|
||||
except Exception:
|
||||
logger.exception("Wiki vector export failed for %s", mapping_id)
|
||||
raise
|
||||
|
||||
|
||||
async def _has_local_wiki_vectors_for_export(request: Request, mapping_id: str) -> bool:
|
||||
@@ -646,12 +666,11 @@ async def _ensure_local_wiki_vectors_for_export(
|
||||
mapping: dict[str, Any],
|
||||
) -> None:
|
||||
mapping_id = str(mapping.get("id") or mapping.get("knowledge_base_id") or "")
|
||||
if not mapping_id or await _has_local_wiki_vectors_for_export(request, mapping_id):
|
||||
return
|
||||
|
||||
service = getattr(request.app.state, "llmwiki_sync_service", None)
|
||||
if service is None or not mapping_id:
|
||||
return
|
||||
raise RuntimeError("请先配置 Wiki 编码模型并启用本地向量索引,再生成向量数据包")
|
||||
# Refresh remote metadata/content hashes even when the previous generation
|
||||
# was complete. Unchanged articles reuse their existing vectors.
|
||||
sync_mapping = dict(mapping)
|
||||
sync_mapping.setdefault("id", mapping_id)
|
||||
sync_mapping.setdefault(
|
||||
@@ -660,15 +679,9 @@ async def _ensure_local_wiki_vectors_for_export(
|
||||
)
|
||||
if not sync_mapping.get("weknora_id"):
|
||||
return
|
||||
try:
|
||||
logger.info("Building local wiki vectors before export: mapping=%s", mapping_id)
|
||||
await service.sync_mapping(sync_mapping, force=False)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning(
|
||||
"Could not build local wiki vectors before export: mapping=%s",
|
||||
mapping_id,
|
||||
exc_info=True,
|
||||
)
|
||||
result = await service.sync_mapping(sync_mapping, force=False, notify=False)
|
||||
if result.get("status") != "completed":
|
||||
raise RuntimeError("Wiki 向量化尚未全部完成,请查看向量化进度后重试")
|
||||
|
||||
|
||||
async def _import_package_file_for_job(
|
||||
@@ -679,9 +692,10 @@ async def _import_package_file_for_job(
|
||||
actor: str | None,
|
||||
) -> None:
|
||||
store = _store(request)
|
||||
package = None
|
||||
try:
|
||||
await store.update_import_job(job_id, status="running", phase="loading_package")
|
||||
package = await asyncio.to_thread(read_package, path)
|
||||
package = await asyncio.to_thread(PackageArchive, path)
|
||||
await store.update_import_job(job_id, status="running", phase="importing")
|
||||
await store.import_package_for_job(
|
||||
job_id,
|
||||
@@ -698,6 +712,9 @@ async def _import_package_file_for_job(
|
||||
error=str(exc),
|
||||
completed=True,
|
||||
)
|
||||
finally:
|
||||
if package is not None:
|
||||
await asyncio.to_thread(package.close)
|
||||
|
||||
|
||||
async def _export_weknora_then_import_for_job(
|
||||
@@ -708,6 +725,7 @@ async def _export_weknora_then_import_for_job(
|
||||
remote_id: str,
|
||||
source: dict[str, Any],
|
||||
actor: str | None,
|
||||
index_fresh: bool = False,
|
||||
) -> None:
|
||||
store = _store(request)
|
||||
path = package_path(_package_root(), job_id)
|
||||
@@ -717,7 +735,8 @@ async def _export_weknora_then_import_for_job(
|
||||
if not runtime.weknora_enabled:
|
||||
raise RuntimeError("未配置普通知识库服务")
|
||||
mapping_id = str(mapping.get("id") or mapping.get("knowledge_base_id") or "")
|
||||
await _ensure_local_wiki_vectors_for_export(request, mapping)
|
||||
if not index_fresh:
|
||||
await _ensure_local_wiki_vectors_for_export(request, mapping)
|
||||
counts = await _write_weknora_export_package(
|
||||
build_weknora_client(runtime),
|
||||
remote_id,
|
||||
|
||||
Reference in new issue
Block a user