feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3

Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
吉鑫升 committed 2026-09-03 19:21:10 +08:00
1 parent 49d9ced04f
commit 60e2c72514
38 files changed
+2536 -581

No files matched your search

@@ -0,0 +1,79 @@
"""Disk-backed, replayable package records. Memory is bounded by one JSONL record.
The staging index is local SQLite, never a remote provider's database. It also
allows restoration to fetch only the vectors belonging to the current page.
"""
from __future__ import annotations
import json
import sqlite3
import tempfile
from collections.abc import Iterator
from contextlib import closing
from pathlib import Path
from typing import Any
class Records:
def __init__(self, path: Path, kind: str, slug: str | None = None):
self.path, self.kind, self.slug = path, kind, slug
def _where(self):
return ("kind=?", (self.kind,)) if self.slug is None else ("kind=? AND slug=?", (self.kind, self.slug))
def __iter__(self) -> Iterator[dict[str, Any]]:
where, args = self._where()
with closing(sqlite3.connect(self.path)) as db:
for (payload,) in db.execute(f"SELECT payload FROM records WHERE {where} ORDER BY seq", args):
yield json.loads(payload)
def __len__(self) -> int:
where, args = self._where()
with closing(sqlite3.connect(self.path)) as db:
return db.execute(f"SELECT count(*) FROM records WHERE {where}", args).fetchone()[0]
class PackageArchive(dict):
def __init__(self, path: Path):
super().__init__()
temporary = tempfile.NamedTemporaryFile(prefix="wiki-package-", suffix=".sqlite", dir=path.parent, delete=False)
self.path = Path(temporary.name)
temporary.close()
try:
with closing(sqlite3.connect(self.path)) as db, path.open(encoding="utf-8-sig") as handle:
db.execute("CREATE TABLE records(seq INTEGER PRIMARY KEY, kind TEXT, slug TEXT, payload TEXT)")
for line_no, line in enumerate(handle, 1):
if not line.strip():
continue
record = json.loads(line)
kind, payload = record.get("record_type"), record.get("payload")
if not isinstance(payload, dict):
raise ValueError(f"导出包第 {line_no} 行格式错误")
if kind == "source":
self["source"] = {**self.get("source", {}), **payload}
elif kind == "graph":
for key, subkind in (("nodes", "graph_node"), ("edges", "graph_edge")):
db.executemany("INSERT INTO records(kind, payload) VALUES (?, ?)", [(subkind, json.dumps(v, ensure_ascii=False)) for v in payload.get(key) or []])
elif kind in {"wiki_page", "document", "chunk", "vector", "entity", "relation"}:
slug = str(payload.get("wiki_slug") or payload.get("page_slug") or payload.get("slug") or "").strip("/")
db.execute("INSERT INTO records(kind, slug, payload) VALUES (?, ?, ?)", (kind, slug, json.dumps(payload, ensure_ascii=False)))
else:
raise ValueError(f"不支持的导出记录类型:{kind}")
db.execute("CREATE INDEX by_kind_slug ON records(kind, slug)")
db.commit()
source = self.get("source", {})
if source.get("package_version") != 1 or (source.get("provider") or source.get("type")) not in {"weknora", "assistant"}:
raise ValueError("仅接受 WeKnora / 助手知识库生成的兼容 Wiki 向量包(v1)")
for key, kind in (("wiki_pages", "wiki_page"), ("documents", "document"), ("chunks", "chunk"), ("vectors", "vector"), ("entities", "entity"), ("relations", "relation")):
self[key] = Records(self.path, kind)
self["graph"] = {"nodes": Records(self.path, "graph_node"), "edges": Records(self.path, "graph_edge")}
except BaseException:
self.close()
raise
def vectors_for(self, slug: str) -> Records:
return Records(self.path, "vector", slug)
def close(self):
self.path.unlink(missing_ok=True)
@@ -76,6 +76,9 @@ class LocalWikiEmbeddingConfig(BaseModel):
model: str = "bge-embedding-m3"
base_url: str = ""
api_key: str = Field(default="", repr=False)
local_model_path: str = ""
local_cache_dir: str = ""
local_threads: int = Field(default=2, ge=1, le=32)
dimensions: int | None = Field(
default=None,
ge=1,
@@ -91,8 +94,8 @@ class LocalWikiEmbeddingConfig(BaseModel):
@classmethod
def _validate_provider(cls, value: str) -> str:
normalized = value.strip().lower()
if normalized not in {"openai", "openai_compatible"}:
raise ValueError("local Wiki embedding provider must be openai or openai_compatible")
if normalized not in {"openai", "openai_compatible", "local"}:
raise ValueError("Wiki embedding provider must be openai, openai_compatible or local")
return normalized
@field_validator("base_url")
@@ -198,7 +201,7 @@ class LocalWikiIndexConfig(BaseModel):
raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
if self.enabled and not self.embedding.enabled:
raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
if self.enabled and (not self.embedding.base_url or not self.embedding.model.strip()):
if self.enabled and ((not self.embedding.base_url and self.embedding.provider != "local") or not self.embedding.model.strip()):
raise ValueError("local Wiki index embedding base_url and model are required when enabled")
return self
@@ -501,6 +501,21 @@ class WeKnoraClient:
raise WeKnoraError("WeKnora returned an invalid wiki graph")
return dict(value)
async def get_wiki_stats(self, knowledge_base_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/stats"))
if not isinstance(value, dict):
raise WeKnoraError("Invalid Wiki generation status")
return value
async def create_wiki_folder(self, knowledge_base_id: str, *, name: str, parent_id: str = "") -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders", json={"name": name, "parent_id": parent_id}))
if not isinstance(value, dict) or not value.get("id"):
raise WeKnoraError("Invalid Wiki folder response")
return value
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
async def list_wiki_pages(
self,
knowledge_base_id: str,
@@ -105,4 +105,12 @@ def chunk_wiki_page(
context.append(f"章节:{heading}")
embedding_text = "\n".join([*context, "", chunk]).strip()
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
# A short article with a short summary must not disappear merely because
# both sections fall below min_chars. Keep one combined, useful vector.
if not result and (content or summary):
fallback = "\n\n".join(dict.fromkeys(text for text in (summary, content) if text))
normalized = re.sub(r"\s+", " ", fallback).strip()
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
prefix = prefix_lines if title_prefix_enabled else []
result.append(WikiSection(0, None, fallback, "\n".join([*prefix, "", fallback]).strip(), digest))
return result
@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
import hashlib
import logging
from pathlib import Path
from time import monotonic
import httpx
@@ -43,6 +44,8 @@ class StrictWikiEmbeddingClient:
self._client = http_client
self._owns_client = http_client is None
self._resolved_dimensions = config.dimensions
self._local_model = None
self._local_lock = asyncio.Lock()
@property
def dimensions(self) -> int | None:
@@ -64,6 +67,11 @@ class StrictWikiEmbeddingClient:
return self._client
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
if self.config.provider == "local":
if not texts:
return []
async with self._local_lock:
return await asyncio.to_thread(self._embed_local, texts)
if not self.config.enabled or not self.config.base_url or not self.config.model:
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
if not texts:
@@ -76,6 +84,45 @@ class StrictWikiEmbeddingClient:
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
return vectors
def _embed_local(self, texts: list[str]) -> list[list[float]]:
if not self.config.enabled:
raise WikiEmbeddingError("本地编码已关闭", retryable=False)
if self._local_model is None:
try:
from fastembed import TextEmbedding
except ImportError as exc:
raise WikiEmbeddingError("请安装本地编码依赖:uv sync --extra local-embedding", retryable=False) from exc
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
if name == "BAAI/bge-m3" and not any(m["model"] == name for m in TextEmbedding.list_supported_models()):
from fastembed.common.model_description import ModelSource, PoolingType
TextEmbedding.add_custom_model(
model=name,
pooling=PoolingType.CLS,
normalization=True,
dim=1024,
sources=ModelSource(hf="BAAI/bge-m3"),
model_file="onnx/model.onnx",
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
license="mit",
size_in_gb=2.29,
)
cache_dir = Path(self.config.local_cache_dir).expanduser() if self.config.local_cache_dir else Path.home() / ".cache" / "deerflow" / "embeddings"
options = {"model_name": name, "threads": self.config.local_threads, "cache_dir": str(cache_dir), "providers": ["CPUExecutionProvider"]}
if self.config.local_model_path:
options["specific_model_path"] = str(Path(self.config.local_model_path).expanduser())
options["local_files_only"] = True
self._local_model = TextEmbedding(**options)
values = []
for vector in self._local_model.embed(texts, batch_size=min(self.config.batch_size, 4)):
normalized = normalize_vector(vector, dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(normalized.size)
values.append(normalized.tolist())
if len(values) != len(texts):
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
return values
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
url = self.config.base_url.rstrip("/") + "/embeddings"
headers = {"Content-Type": "application/json"}
@@ -36,8 +36,9 @@ class WikiSyncService:
self.cache = cache
self.client = client
self.owner = f"{platform.node()}:{uuid4()}"
self.on_completed = None
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False, notify: bool = True) -> dict[str, Any]:
mapping_id = str(mapping["id"])
sync_id = str(uuid4())
if self.embedding.dimensions is None:
@@ -111,6 +112,7 @@ class WikiSyncService:
and existing.get("content_hash") == page["content_hash"]
and existing.get("indexed_fingerprint") == active_fingerprint
and existing.get("index_status") == "ready"
and (existing.get("vector_count", 1) > 0 or page["status"] == "archived" or not (page["content_md"] or page["summary"]))
and not existing.get("is_remote_deleted")
):
await self.store.touch_page(str(existing["id"]), sync_id)
@@ -160,7 +162,10 @@ class WikiSyncService:
remote_page_count=remote_count,
error=f"{failed} page(s) failed" if failed else None,
)
return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
if not failed and remote_count and notify and self.on_completed:
result["assistant_sync"] = await self.on_completed(mapping)
return result
except asyncio.CancelledError:
await self.store.finish_sync(
mapping_id,
@@ -65,6 +65,7 @@ class AssistantKnowledgeImportJobRow(Base):
class AssistantKnowledgeWikiPageRow(Base):
__tablename__ = "assistant_knowledge_wiki_pages"
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
@@ -2,6 +2,7 @@
from __future__ import annotations
import asyncio
import base64
import binascii
import difflib
@@ -60,6 +61,12 @@ def _dict(row: Any) -> dict[str, Any]:
data["base_kind"] = data.get("base_kind") or "custom"
data["is_global"] = data["base_kind"] == "global"
data["description"] = data.get("description") or ""
if isinstance(row, AssistantKnowledgeWikiPageRow):
metadata = data.get("metadata") or {}
for key in ("aliases", "tags", "category_path", "parent_slug", "wiki_path", "in_links", "out_links", "source_refs", "origins"):
data[key] = metadata.get(key, [] if key not in {"parent_slug", "wiki_path"} else None)
data["tags"] = metadata.get("tags") or (metadata.get("page_metadata") or {}).get("tags") or []
data["content"] = data.get("content_markdown", "")
return data
@@ -95,7 +102,11 @@ def _vector_blob_from_payload(raw: dict[str, Any]) -> tuple[bytes, int] | None:
try:
decoded = base64.b64decode(str(blob64), validate=True)
vector = decode_vector(decoded, int(dimensions))
return encode_vector(vector)
# Our packages contain normalized float32 bytes. Preserve those
# bytes exactly: another normalization causes round-trip drift.
if not np.isclose(np.linalg.norm(vector), 1.0, rtol=1e-5, atol=1e-6):
return None
return decoded, int(dimensions)
except (ValueError, TypeError, binascii.Error):
return None
value = _first_vector_value(raw)
@@ -115,6 +126,66 @@ class AssistantKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def iter_export_records(self, base_id: str):
"""Export current Wiki articles and their original vectors, without encoding."""
base = await self.get_base(base_id)
if not base:
raise ValueError("助手知识库不存在")
yield "source", {"package_version": 1, "provider": "assistant", "type": "assistant", "knowledge_base_id": base_id, "knowledge_base_name": base["name"], "export_format": "assistant-knowledge-jsonl"}
async with self._sf() as session:
pages = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base_id, AssistantKnowledgeWikiPageRow.status == "active", AssistantKnowledgeWikiPageRow.page_type != "chunk")
stream = await session.stream_scalars(pages.execution_options(yield_per=100))
async for page in stream:
data = _dict(page)
yield "wiki_page", {**(page.metadata_json or {}), **{k: data[k] for k in ("slug", "title", "page_type", "summary", "content")}, "status": "published"}
vectors = (
select(AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow.slug)
.join(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeVectorRow.page_id)
.join(AssistantKnowledgeWikiRevisionRow, AssistantKnowledgeWikiRevisionRow.id == AssistantKnowledgeWikiPageRow.current_revision_id)
.where(
AssistantKnowledgeVectorRow.base_id == base_id,
AssistantKnowledgeWikiPageRow.status == "active",
AssistantKnowledgeWikiPageRow.page_type != "chunk",
AssistantKnowledgeVectorRow.import_job_id == AssistantKnowledgeWikiRevisionRow.import_job_id,
)
)
stream = await session.stream(vectors.execution_options(yield_per=100))
async for vector, slug in stream:
yield (
"vector",
{
"wiki_slug": slug,
"section_index": vector.section_index,
"heading": vector.heading,
"section_content": vector.section_content,
"content_hash": vector.content_hash,
"embedding_model": vector.embedding_model,
"embedding_fingerprint": vector.embedding_fingerprint,
"embedding_dimensions": vector.embedding_dimensions,
"vector_blob_base64": base64.b64encode(vector.vector_blob).decode("ascii"),
},
)
stream = await session.stream(
select(AssistantKnowledgeEntityRow, AssistantKnowledgeWikiPageRow.slug)
.outerjoin(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeEntityRow.wiki_page_id)
.where(AssistantKnowledgeEntityRow.base_id == base_id, AssistantKnowledgeEntityRow.status == "active")
.execution_options(yield_per=100)
)
async for entity, slug in stream:
yield "entity", {"id": entity.id, "name": entity.display_name, "type": entity.entity_type, "aliases": entity.aliases_json or [], "attributes": entity.attributes_json or {}, "source_wiki_slug": slug}
stream = await session.stream(
select(AssistantKnowledgeRelationRow, AssistantKnowledgeWikiPageRow.slug)
.join(AssistantKnowledgeEntityRow, AssistantKnowledgeEntityRow.id == AssistantKnowledgeRelationRow.source_entity_id)
.outerjoin(AssistantKnowledgeWikiPageRow, AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeEntityRow.wiki_page_id)
.where(AssistantKnowledgeRelationRow.base_id == base_id, AssistantKnowledgeRelationRow.status == "active")
.execution_options(yield_per=100)
)
async for relation, slug in stream:
yield (
"relation",
{"source_entity_id": relation.source_entity_id, "target_entity_id": relation.target_entity_id, "predicate": relation.predicate, "description": relation.description, "source_wiki_slug": slug},
)
async def _base_with_stats(
self,
session: AsyncSession,
@@ -138,6 +209,7 @@ class AssistantKnowledgeRepository:
select(func.count(AssistantKnowledgeWikiPageRow.id)).where(
AssistantKnowledgeWikiPageRow.base_id == row.id,
AssistantKnowledgeWikiPageRow.status == "active",
AssistantKnowledgeWikiPageRow.page_type != "chunk",
)
)
).scalar_one()
@@ -195,14 +267,7 @@ class AssistantKnowledgeRepository:
# Backward compatibility: earlier builds had a single active row
# and no ``base_kind``. Promote the first surviving base into the
# global full-library aggregation base.
row = (
await session.execute(
select(AssistantKnowledgeBaseRow)
.where(AssistantKnowledgeBaseRow.status != "deleted")
.order_by(AssistantKnowledgeBaseRow.created_at.asc())
.limit(1)
)
).scalar_one_or_none()
row = (await session.execute(select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status != "deleted").order_by(AssistantKnowledgeBaseRow.created_at.asc()).limit(1))).scalar_one_or_none()
if row is not None:
row.base_kind = "global"
if not row.description:
@@ -655,7 +720,7 @@ class AssistantKnowledgeRepository:
entity_map: dict[str, str] = {}
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
raw_entities = list(package.get("entities") or [])
raw_entities = package.get("entities") or []
for raw in raw_entities:
display_name = str(raw.get("name") or raw.get("display_name") or "").strip()
entity_type = str(raw.get("type") or raw.get("entity_type") or "concept")
@@ -698,7 +763,7 @@ class AssistantKnowledgeRepository:
)
)
raw_relations = list(package.get("relations") or [])
raw_relations = package.get("relations") or []
relation_rows: dict[str, tuple[AssistantKnowledgeRelationRow, dict[str, Any]]] = {}
for raw in raw_relations:
source_entity_id = entity_map.get(str(raw.get("source_entity_id") or raw.get("source_local_id") or raw.get("source")))
@@ -741,6 +806,10 @@ class AssistantKnowledgeRepository:
)
for entity, raw in entity_rows.values():
# Provider Wiki graph nodes already have real articles. Do not
# turn each node into a second, empty 'entity' article.
if raw.get("source_wiki_slug"):
continue
aliases = list(dict.fromkeys([*list(entity.aliases_json or []), *list(raw.get("aliases") or [])]))
entity.aliases_json = aliases
content = "\n".join(
@@ -775,6 +844,8 @@ class AssistantKnowledgeRepository:
entity.wiki_page_id = entity_page.id
for relation, raw in relation_rows.values():
if raw.get("source_wiki_slug"):
continue
source_entity = await session.get(AssistantKnowledgeEntityRow, relation.source_entity_id)
target_entity = await session.get(AssistantKnowledgeEntityRow, relation.target_entity_id)
source_name_display = source_entity.display_name if source_entity else relation.source_entity_id
@@ -796,7 +867,7 @@ class AssistantKnowledgeRepository:
)
relation.wiki_page_id = relation_page.id
pages = list(package.get("wiki_pages") or [])
pages = package.get("wiki_pages") or []
if not pages and package.get("wiki"):
wiki = package["wiki"]
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
@@ -807,7 +878,7 @@ class AssistantKnowledgeRepository:
continue
slug = str(raw.get("slug") or f"assistant/sources/{source.id}").strip("/")
title = str(raw.get("title") or source_name)
content = str(raw.get("content") or raw.get("content_markdown") or "")
content = str(raw.get("content") or raw.get("content_md") or raw.get("content_markdown") or "")
summary = str(raw.get("summary") or "")
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base["id"], AssistantKnowledgeWikiPageRow.slug == slug)
page = (await session.execute(stmt)).scalar_one_or_none()
@@ -850,32 +921,35 @@ class AssistantKnowledgeRepository:
page.current_revision_id = revision.id
page.status = "active"
page.updated_at = now
metadata = {key: raw[key] for key in ("aliases", "tags", "page_metadata", "category_path", "parent_slug", "wiki_path", "in_links", "out_links", "source_refs", "chunk_refs", "version") if key in raw}
source_meta = package.get("source") or {}
origin_mapping = mapping_id
if not origin_mapping and (source_meta.get("provider") or source_meta.get("type")) == "weknora":
origin_mapping = source_meta.get("knowledge_base_id")
origins = [*((page.metadata_json or {}).get("origins") or []), *(raw.get("origins") or [])]
origin = {"mapping_id": origin_mapping, "source_key": source_key, "source_name": source_name, "slug": slug}
origins = [value for value in origins if isinstance(value, dict) and value.get("source_key") != source_key]
metadata["origins"] = list({(v.get("mapping_id"), v.get("source_key"), v.get("slug")): v for v in [*origins, origin]}.values())
page.metadata_json = metadata
page_ids_by_slug[slug] = page.id
for entity, raw_entity in entity_rows.values():
if raw_entity.get("source_wiki_slug") == slug:
entity.wiki_page_id = page.id
if len(page_ids_by_slug) % 50 == 0:
await session.flush()
chunks = list(package.get("chunks") or [])
chunks = package.get("chunks") or []
chunk_pages = 0
imported_vectors = 0
for index, raw in enumerate(chunks):
if not isinstance(raw, dict):
continue
content = str(
raw.get("content")
or raw.get("section_content")
or raw.get("text")
or raw.get("chunk_content")
or ""
).strip()
content = str(raw.get("content") or raw.get("section_content") or raw.get("text") or raw.get("chunk_content") or "").strip()
if not content:
continue
raw_chunk_id = str(raw.get("id") or raw.get("chunk_id") or raw.get("remote_id") or f"{source.id}:{index}")
document_id = str(raw.get("source_document_id") or raw.get("knowledge_id") or raw.get("document_id") or "")
title = str(
raw.get("title")
or raw.get("knowledge_title")
or raw.get("filename")
or raw.get("knowledge_filename")
or f"{source_name} 片段 {index + 1}"
)
title = str(raw.get("title") or raw.get("knowledge_title") or raw.get("filename") or raw.get("knowledge_filename") or f"{source_name} 片段 {index + 1}")
heading = str(raw.get("heading") or raw.get("section_title") or "")
summary = content[:240]
page_content = "\n".join(
@@ -919,7 +993,7 @@ class AssistantKnowledgeRepository:
import_job_id=job.id,
source_vector_id=str(raw.get("vector_id") or raw.get("embedding_id") or raw_chunk_id),
source_chunk_id=raw_chunk_id,
section_index=int(raw.get("section_index") or raw.get("chunk_index") or index),
section_index=int(next((raw[key] for key in ("section_index", "chunk_index") if raw.get(key) is not None), index)),
heading=heading or None,
section_content=content,
content_hash=str(raw.get("content_hash") or _content_hash(content)),
@@ -933,19 +1007,19 @@ class AssistantKnowledgeRepository:
imported_vectors += 1
chunk_pages += 1
for index, raw in enumerate(list(package.get("vectors") or [])):
for index, raw in enumerate(package.get("vectors") or []):
if not isinstance(raw, dict):
continue
raise ValueError("向量记录必须为对象")
slug = str(raw.get("wiki_slug") or raw.get("page_slug") or raw.get("slug") or "").strip("/")
page_id = page_ids_by_slug.get(slug)
if not page_id:
continue
raise ValueError(f"向量引用不存在的 Wiki:{slug}")
content = str(raw.get("section_content") or raw.get("content") or raw.get("text") or "").strip()
if not content:
continue
vector_blob = _vector_blob_from_payload(raw)
if vector_blob is None:
continue
raise ValueError(f"Wiki 向量格式无效:{slug}")
blob, dimensions = vector_blob
raw_chunk_id = str(raw.get("source_chunk_id") or raw.get("chunk_id") or "")
session.add(
@@ -957,7 +1031,7 @@ class AssistantKnowledgeRepository:
import_job_id=job.id,
source_vector_id=str(raw.get("id") or raw.get("vector_id") or f"{slug}:{index}"),
source_chunk_id=raw_chunk_id or None,
section_index=int(raw.get("section_index") or index),
section_index=int(raw["section_index"]) if raw.get("section_index") is not None else index,
heading=str(raw.get("heading") or "") or None,
section_content=content,
content_hash=str(raw.get("content_hash") or _content_hash(content)),
@@ -969,9 +1043,11 @@ class AssistantKnowledgeRepository:
)
)
imported_vectors += 1
if imported_vectors % 50 == 0:
await session.flush()
counts = {
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
"wiki_pages": len(pages) + sum(not raw.get("source_wiki_slug") for _, raw in entity_rows.values()) + sum(not raw.get("source_wiki_slug") for _, raw in relation_rows.values()),
"documents": len(package.get("documents") or []),
"chunks": len(chunks),
"chunk_pages": chunk_pages,
@@ -1008,6 +1084,7 @@ class AssistantKnowledgeRepository:
query: str | None = None,
page_type: str | None = None,
limit: int = 100,
offset: int = 0,
include_chunks: bool = False,
) -> list[dict[str, Any]]:
base = await self.get_base(base_id)
@@ -1024,7 +1101,7 @@ class AssistantKnowledgeRepository:
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
if page_type:
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type == page_type)
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(limit)
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.slug, AssistantKnowledgeWikiPageRow.id).offset(offset).limit(limit)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
@@ -1070,10 +1147,7 @@ class AssistantKnowledgeRepository:
or_(
AssistantKnowledgeWikiPageRow.title.ilike(needle),
AssistantKnowledgeWikiPageRow.search_text.ilike(needle),
*[
AssistantKnowledgeWikiPageRow.search_text.ilike(f"%{token}%")
for token in tokens[:8]
],
*[AssistantKnowledgeWikiPageRow.search_text.ilike(f"%{token}%") for token in tokens[:8]],
)
)
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(max(limit * 8, limit))
@@ -1098,6 +1172,7 @@ class AssistantKnowledgeRepository:
results = rows[:limit]
for row in results:
row["score"] = score(row)
row["retrieval_mode"] = "keyword"
return results
async def _search_pages_by_vector(
@@ -1119,31 +1194,36 @@ class AssistantKnowledgeRepository:
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeVectorRow.page_id,
)
.join(AssistantKnowledgeWikiRevisionRow, AssistantKnowledgeWikiRevisionRow.id == AssistantKnowledgeWikiPageRow.current_revision_id)
.where(
AssistantKnowledgeVectorRow.base_id.in_(base_ids),
AssistantKnowledgeVectorRow.import_job_id == AssistantKnowledgeWikiRevisionRow.import_job_id,
AssistantKnowledgeVectorRow.embedding_dimensions == dimensions,
AssistantKnowledgeVectorRow.embedding_fingerprint == embedding_client.fingerprint,
AssistantKnowledgeWikiPageRow.status == "active",
)
.order_by(AssistantKnowledgeVectorRow.created_at.desc())
)
if not include_chunks:
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type != "chunk")
async with self._sf() as session:
rows = list((await session.execute(stmt)).all())
if not rows:
return []
tokens = [token for token in query.strip().lower().split() if token]
ranked: list[tuple[float, AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow]] = []
for vector, page in rows:
try:
stored = decode_vector(vector.vector_blob, int(vector.embedding_dimensions))
except ValueError:
continue
score = float(np.dot(query_vector, stored))
if tokens:
haystack = f"{page.title}\n{page.summary}\n{vector.heading or ''}\n{vector.section_content}".lower()
score += min(sum(1 for token in tokens if token in haystack), 4) * 0.02
ranked.append((score, vector, page))
# Bounded scan: keep only the best section per page, and at most limit
# candidates. Never load a multi-GB vector library into one matrix.
async with self._sf() as session:
stream = await session.stream(stmt.execution_options(yield_per=256))
async for batch in stream.partitions(256):
def score_batch():
values = []
for vector, page in batch:
values.append((float(np.dot(query_vector, decode_vector(vector.vector_blob, dimensions))), vector, page))
return values
candidates = ranked + await asyncio.to_thread(score_batch)
best = {}
for item in sorted(candidates, key=lambda item: item[0], reverse=True):
best.setdefault(item[2].id, item)
ranked = list(best.values())[:limit]
ranked.sort(key=lambda item: item[0], reverse=True)
results: list[dict[str, Any]] = []
@@ -1320,12 +1400,7 @@ class AssistantKnowledgeRepository:
if base is None:
return []
async with self._sf() as session:
stmt = (
select(AssistantKnowledgeImportJobRow)
.where(AssistantKnowledgeImportJobRow.base_id == base["id"])
.order_by(AssistantKnowledgeImportJobRow.created_at.desc())
.limit(limit)
)
stmt = select(AssistantKnowledgeImportJobRow).where(AssistantKnowledgeImportJobRow.base_id == base["id"]).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_job(self, job_id: str) -> dict[str, Any] | None:
@@ -24,7 +24,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
return dict(row) if row else None
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
@@ -41,10 +41,11 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
return row
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
result = (await session.execute(stmt)).one_or_none()
return {**_dict(result[0]), "vector_count": result[1]} if result else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
@@ -0,0 +1,21 @@
"""Preserve source Wiki metadata during package round trips."""
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20260903_01"
down_revision = ("20260902_01", "20260831_01")
branch_labels = None
depends_on = None
def upgrade():
columns = {col["name"] for col in sa.inspect(op.get_bind()).get_columns("assistant_knowledge_wiki_pages")}
if "metadata_json" not in columns:
op.add_column("assistant_knowledge_wiki_pages", sa.Column("metadata_json", PortableJSON(), nullable=True))
def downgrade():
op.drop_column("assistant_knowledge_wiki_pages", "metadata_json")