feat: add compact toolbox and public wiki vector search

This commit is contained in:
吉鑫升 committed 2026-08-26 14:40:57 +08:00
1 parent 9f29ecae9e
commit d35d075669
29 files changed
+757 -128

No files matched your search

@@ -108,6 +108,10 @@ class LocalWikiIndexConfig(BaseModel):
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
enabled: bool = False
frontend_base_url: str = Field(
default="http://127.0.0.1:5174",
description="DeerFlow frontend base URL used to build directly usable Wiki links in public vector-search results.",
)
auto_sync: bool = False
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
sync_page_size: int = Field(default=100, ge=1, le=500)
@@ -135,6 +139,14 @@ class LocalWikiIndexConfig(BaseModel):
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
model_config = ConfigDict(extra="ignore")
@field_validator("frontend_base_url")
@classmethod
def _validate_frontend_base_url(cls, value: str) -> str:
normalized = normalize_service_url(value)
if not normalized:
raise ValueError("local Wiki index frontend_base_url is required")
return normalized
@model_validator(mode="before")
@classmethod
def _migrate_legacy_flags(cls, value):
@@ -46,29 +46,12 @@ class WikiVectorSearchService:
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
if not mappings:
return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
try:
query_vector = await self._embed_query(normalized_query)
except (WikiEmbeddingError, ValueError) as exc:
raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
fingerprint = self.embedding.fingerprint
tasks = [self._search_mapping(normalized_query, query_vector, mapping, fingerprint, include_drafts=include_drafts, external=external) for mapping in mappings]
batches = await asyncio.gather(*tasks, return_exceptions=True)
candidates: list[dict[str, Any]] = []
warnings: list[str] = []
for mapping, batch in zip(mappings, batches, strict=True):
if isinstance(batch, Exception):
warnings.append(f"{mapping.get('id')}: {batch}")
else:
candidates.extend(batch)
if warnings and not candidates and len(warnings) == len(mappings):
if any("FINGERPRINT" in item for item in warnings):
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
elif any("rebuild" in item.lower() for item in warnings):
code = "WIKI_INDEX_REBUILDING"
else:
code = "WIKI_INDEX_NOT_READY"
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
candidates.sort(key=lambda item: item["score"], reverse=True)
fingerprint, candidates, warnings = await self._collect_candidates(
normalized_query,
mappings,
include_drafts=include_drafts,
external=external,
)
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
for candidate in candidates:
page = candidate["page"]
@@ -111,6 +94,141 @@ class WikiVectorSearchService:
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
async def search_sections(
self,
query: str,
mappings: list[dict[str, Any]],
*,
top_k_sections: int = 10,
external: bool = False,
) -> dict[str, Any]:
"""Return one result per matching Wiki vector section.
The regular ``search`` method intentionally groups several sections
under one Wiki page for citations. Public consumers of the standalone
vector API need the opposite contract: every matched vector is a
separate ranked item, while the complete page and knowledge-base
metadata remains attached to that item.
"""
normalized_query = query.strip()
if not normalized_query:
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
limit = max(1, min(int(top_k_sections), 100))
if not mappings:
return {
"query": normalized_query,
"mode": "vector",
"results": [],
"knowledge_base_count": 0,
"index_fingerprint": self.embedding.fingerprint[:12],
"partial": False,
"warnings": [],
}
fingerprint, candidates, warnings = await self._collect_candidates(
normalized_query,
mappings,
include_drafts=False,
external=external,
section_limit=limit,
)
results: list[dict[str, Any]] = []
for candidate in candidates[:limit]:
page = candidate["page"]
mapping = candidate["mapping"]
results.append(
{
"kind": "wiki_section",
"id": candidate.get("id") or f"{page['id']}:{candidate['section_index']}",
"section_id": candidate.get("id") or f"{page['id']}:{candidate['section_index']}",
"section_index": int(candidate["section_index"]),
"heading": candidate.get("heading"),
"content": str(candidate.get("section_content") or ""),
"section_content_hash": candidate.get("content_hash") or "",
"score": float(candidate["score"]),
"knowledge_base_id": mapping["id"],
"weknora_knowledge_base_id": mapping.get("weknora_id") or "",
"knowledge_base_name": mapping.get("name") or "",
"knowledge_base_description": mapping.get("description") or "",
"knowledge_base_type": mapping.get("kb_type") or "document",
"publication_status": mapping.get("publication_status") or "",
"published_at": mapping.get("published_at"),
"wiki_page_id": page["id"],
"weknora_wiki_page_id": page.get("remote_page_id"),
"wiki_slug": page["slug"],
"title": page.get("title") or page["slug"],
"summary": page.get("summary") or "",
"page_content": page.get("content_md") or "",
"page_type": page.get("page_type") or "article",
"page_status": page.get("status") or "published",
"page_version": int(page.get("page_version") or 0),
"aliases": page.get("aliases") or [],
"category_path": page.get("category_path") or [],
"parent_slug": page.get("parent_slug"),
"wiki_path": page.get("wiki_path"),
"source_refs": page.get("source_refs") or [],
"chunk_refs": page.get("chunk_refs") or [],
"page_content_hash": page.get("content_hash") or "",
"remote_created_at": page.get("remote_created_at"),
"remote_updated_at": page.get("remote_updated_at"),
"indexed_at": page.get("indexed_at"),
}
)
return {
"query": normalized_query,
"mode": "vector",
"results": results,
"knowledge_base_count": len(mappings),
"index_fingerprint": fingerprint[:12],
"partial": bool(warnings),
"warnings": warnings,
}
async def _collect_candidates(
self,
query: str,
mappings: list[dict[str, Any]],
*,
include_drafts: bool,
external: bool,
section_limit: int | None = None,
) -> tuple[str, list[dict[str, Any]], list[str]]:
try:
query_vector = await self._embed_query(query)
except (WikiEmbeddingError, ValueError) as exc:
raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
fingerprint = self.embedding.fingerprint
tasks = [
self._search_mapping(
query,
query_vector,
mapping,
fingerprint,
include_drafts=include_drafts,
external=external,
section_limit=section_limit,
)
for mapping in mappings
]
batches = await asyncio.gather(*tasks, return_exceptions=True)
candidates: list[dict[str, Any]] = []
warnings: list[str] = []
for mapping, batch in zip(mappings, batches, strict=True):
if isinstance(batch, Exception):
warnings.append(f"{mapping.get('id')}: {batch}")
else:
candidates.extend(batch)
if warnings and not candidates and len(warnings) == len(mappings):
if any("FINGERPRINT" in item for item in warnings):
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
elif any("rebuild" in item.lower() for item in warnings):
code = "WIKI_INDEX_REBUILDING"
else:
code = "WIKI_INDEX_NOT_READY"
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
candidates.sort(key=lambda item: item["score"], reverse=True)
return fingerprint, candidates, warnings
async def _embed_query(self, query: str) -> np.ndarray:
digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
key = f"{self.embedding.fingerprint}:{digest}"
@@ -126,7 +244,17 @@ class WikiVectorSearchService:
self._query_vectors.popitem(last=False)
return vector
async def _search_mapping(self, query: str, vector: np.ndarray, mapping: dict[str, Any], fingerprint: str, *, include_drafts: bool, external: bool) -> list[dict[str, Any]]:
async def _search_mapping(
self,
query: str,
vector: np.ndarray,
mapping: dict[str, Any],
fingerprint: str,
*,
include_drafts: bool,
external: bool,
section_limit: int | None = None,
) -> list[dict[str, Any]]:
mapping_id = str(mapping["id"])
try:
snapshot = await self.cache.get(mapping_id, fingerprint)
@@ -148,7 +276,13 @@ class WikiVectorSearchService:
)
if eligible.size == 0:
return []
indices, scores = await asyncio.to_thread(self._top_indices, snapshot, vector, eligible)
indices, scores = await asyncio.to_thread(
self._top_indices,
snapshot,
vector,
eligible,
section_limit,
)
terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
candidates = []
for index, score in zip(indices, scores, strict=True):
@@ -166,9 +300,15 @@ class WikiVectorSearchService:
candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
return candidates
def _top_indices(self, snapshot: WikiVectorSnapshot, vector: np.ndarray, eligible: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
def _top_indices(
self,
snapshot: WikiVectorSnapshot,
vector: np.ndarray,
eligible: np.ndarray,
section_limit: int | None = None,
) -> tuple[np.ndarray, np.ndarray]:
scores = snapshot.matrix[eligible] @ vector
count = min(self.config.top_k_sections, scores.size)
count = min(section_limit or self.config.top_k_sections, scores.size)
if count <= 0:
return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
positions = np.argpartition(scores, -count)[-count:]
@@ -19,7 +19,7 @@ def _normalize(data: dict[str, Any], menu_type: str, updated_by: str | None) ->
return {
"node_id": storage_node_id(menu_type, node_id),
"menu_type": menu_type,
"placement": placement if placement in {"sidebar", "settings"} else "sidebar",
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
"sort_order": int(data.get("sort_order", 0)),
"label": (data.get("label") or None),
"updated_by": updated_by,
@@ -35,7 +35,7 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C
return CompactMenuOverrideRow(
node_id=storage_node_id(menu_type, node_id),
menu_type=menu_type,
placement=placement if placement in {"sidebar", "settings"} else "sidebar",
placement=placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
sort_order=int(data.get("sort_order", 0)),
label=(data.get("label") or None),
updated_by=updated_by,