feat: add compact toolbox and public wiki vector search
This commit is contained in:
1 parent
9f29ecae9e
commit
d35d075669
29 files changed
+757
-128
No files matched your search
@@ -108,6 +108,10 @@ class LocalWikiIndexConfig(BaseModel):
|
||||
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
|
||||
|
||||
enabled: bool = False
|
||||
frontend_base_url: str = Field(
|
||||
default="http://127.0.0.1:5174",
|
||||
description="DeerFlow frontend base URL used to build directly usable Wiki links in public vector-search results.",
|
||||
)
|
||||
auto_sync: bool = False
|
||||
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
|
||||
sync_page_size: int = Field(default=100, ge=1, le=500)
|
||||
@@ -135,6 +139,14 @@ class LocalWikiIndexConfig(BaseModel):
|
||||
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("frontend_base_url")
|
||||
@classmethod
|
||||
def _validate_frontend_base_url(cls, value: str) -> str:
|
||||
normalized = normalize_service_url(value)
|
||||
if not normalized:
|
||||
raise ValueError("local Wiki index frontend_base_url is required")
|
||||
return normalized
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _migrate_legacy_flags(cls, value):
|
||||
|
||||
+167
-27
@@ -46,29 +46,12 @@ class WikiVectorSearchService:
|
||||
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
|
||||
if not mappings:
|
||||
return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
|
||||
try:
|
||||
query_vector = await self._embed_query(normalized_query)
|
||||
except (WikiEmbeddingError, ValueError) as exc:
|
||||
raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
|
||||
fingerprint = self.embedding.fingerprint
|
||||
tasks = [self._search_mapping(normalized_query, query_vector, mapping, fingerprint, include_drafts=include_drafts, external=external) for mapping in mappings]
|
||||
batches = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
candidates: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
for mapping, batch in zip(mappings, batches, strict=True):
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping.get('id')}: {batch}")
|
||||
else:
|
||||
candidates.extend(batch)
|
||||
if warnings and not candidates and len(warnings) == len(mappings):
|
||||
if any("FINGERPRINT" in item for item in warnings):
|
||||
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
|
||||
elif any("rebuild" in item.lower() for item in warnings):
|
||||
code = "WIKI_INDEX_REBUILDING"
|
||||
else:
|
||||
code = "WIKI_INDEX_NOT_READY"
|
||||
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
|
||||
candidates.sort(key=lambda item: item["score"], reverse=True)
|
||||
fingerprint, candidates, warnings = await self._collect_candidates(
|
||||
normalized_query,
|
||||
mappings,
|
||||
include_drafts=include_drafts,
|
||||
external=external,
|
||||
)
|
||||
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
||||
for candidate in candidates:
|
||||
page = candidate["page"]
|
||||
@@ -111,6 +94,141 @@ class WikiVectorSearchService:
|
||||
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
|
||||
return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
|
||||
|
||||
async def search_sections(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
top_k_sections: int = 10,
|
||||
external: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Return one result per matching Wiki vector section.
|
||||
|
||||
The regular ``search`` method intentionally groups several sections
|
||||
under one Wiki page for citations. Public consumers of the standalone
|
||||
vector API need the opposite contract: every matched vector is a
|
||||
separate ranked item, while the complete page and knowledge-base
|
||||
metadata remains attached to that item.
|
||||
"""
|
||||
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
|
||||
limit = max(1, min(int(top_k_sections), 100))
|
||||
if not mappings:
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": "vector",
|
||||
"results": [],
|
||||
"knowledge_base_count": 0,
|
||||
"index_fingerprint": self.embedding.fingerprint[:12],
|
||||
"partial": False,
|
||||
"warnings": [],
|
||||
}
|
||||
fingerprint, candidates, warnings = await self._collect_candidates(
|
||||
normalized_query,
|
||||
mappings,
|
||||
include_drafts=False,
|
||||
external=external,
|
||||
section_limit=limit,
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
for candidate in candidates[:limit]:
|
||||
page = candidate["page"]
|
||||
mapping = candidate["mapping"]
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_section",
|
||||
"id": candidate.get("id") or f"{page['id']}:{candidate['section_index']}",
|
||||
"section_id": candidate.get("id") or f"{page['id']}:{candidate['section_index']}",
|
||||
"section_index": int(candidate["section_index"]),
|
||||
"heading": candidate.get("heading"),
|
||||
"content": str(candidate.get("section_content") or ""),
|
||||
"section_content_hash": candidate.get("content_hash") or "",
|
||||
"score": float(candidate["score"]),
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"weknora_knowledge_base_id": mapping.get("weknora_id") or "",
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"knowledge_base_description": mapping.get("description") or "",
|
||||
"knowledge_base_type": mapping.get("kb_type") or "document",
|
||||
"publication_status": mapping.get("publication_status") or "",
|
||||
"published_at": mapping.get("published_at"),
|
||||
"wiki_page_id": page["id"],
|
||||
"weknora_wiki_page_id": page.get("remote_page_id"),
|
||||
"wiki_slug": page["slug"],
|
||||
"title": page.get("title") or page["slug"],
|
||||
"summary": page.get("summary") or "",
|
||||
"page_content": page.get("content_md") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_status": page.get("status") or "published",
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"aliases": page.get("aliases") or [],
|
||||
"category_path": page.get("category_path") or [],
|
||||
"parent_slug": page.get("parent_slug"),
|
||||
"wiki_path": page.get("wiki_path"),
|
||||
"source_refs": page.get("source_refs") or [],
|
||||
"chunk_refs": page.get("chunk_refs") or [],
|
||||
"page_content_hash": page.get("content_hash") or "",
|
||||
"remote_created_at": page.get("remote_created_at"),
|
||||
"remote_updated_at": page.get("remote_updated_at"),
|
||||
"indexed_at": page.get("indexed_at"),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": "vector",
|
||||
"results": results,
|
||||
"knowledge_base_count": len(mappings),
|
||||
"index_fingerprint": fingerprint[:12],
|
||||
"partial": bool(warnings),
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
async def _collect_candidates(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
include_drafts: bool,
|
||||
external: bool,
|
||||
section_limit: int | None = None,
|
||||
) -> tuple[str, list[dict[str, Any]], list[str]]:
|
||||
try:
|
||||
query_vector = await self._embed_query(query)
|
||||
except (WikiEmbeddingError, ValueError) as exc:
|
||||
raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
|
||||
fingerprint = self.embedding.fingerprint
|
||||
tasks = [
|
||||
self._search_mapping(
|
||||
query,
|
||||
query_vector,
|
||||
mapping,
|
||||
fingerprint,
|
||||
include_drafts=include_drafts,
|
||||
external=external,
|
||||
section_limit=section_limit,
|
||||
)
|
||||
for mapping in mappings
|
||||
]
|
||||
batches = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
candidates: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
for mapping, batch in zip(mappings, batches, strict=True):
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping.get('id')}: {batch}")
|
||||
else:
|
||||
candidates.extend(batch)
|
||||
if warnings and not candidates and len(warnings) == len(mappings):
|
||||
if any("FINGERPRINT" in item for item in warnings):
|
||||
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
|
||||
elif any("rebuild" in item.lower() for item in warnings):
|
||||
code = "WIKI_INDEX_REBUILDING"
|
||||
else:
|
||||
code = "WIKI_INDEX_NOT_READY"
|
||||
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
|
||||
candidates.sort(key=lambda item: item["score"], reverse=True)
|
||||
return fingerprint, candidates, warnings
|
||||
|
||||
async def _embed_query(self, query: str) -> np.ndarray:
|
||||
digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
|
||||
key = f"{self.embedding.fingerprint}:{digest}"
|
||||
@@ -126,7 +244,17 @@ class WikiVectorSearchService:
|
||||
self._query_vectors.popitem(last=False)
|
||||
return vector
|
||||
|
||||
async def _search_mapping(self, query: str, vector: np.ndarray, mapping: dict[str, Any], fingerprint: str, *, include_drafts: bool, external: bool) -> list[dict[str, Any]]:
|
||||
async def _search_mapping(
|
||||
self,
|
||||
query: str,
|
||||
vector: np.ndarray,
|
||||
mapping: dict[str, Any],
|
||||
fingerprint: str,
|
||||
*,
|
||||
include_drafts: bool,
|
||||
external: bool,
|
||||
section_limit: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
mapping_id = str(mapping["id"])
|
||||
try:
|
||||
snapshot = await self.cache.get(mapping_id, fingerprint)
|
||||
@@ -148,7 +276,13 @@ class WikiVectorSearchService:
|
||||
)
|
||||
if eligible.size == 0:
|
||||
return []
|
||||
indices, scores = await asyncio.to_thread(self._top_indices, snapshot, vector, eligible)
|
||||
indices, scores = await asyncio.to_thread(
|
||||
self._top_indices,
|
||||
snapshot,
|
||||
vector,
|
||||
eligible,
|
||||
section_limit,
|
||||
)
|
||||
terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
|
||||
candidates = []
|
||||
for index, score in zip(indices, scores, strict=True):
|
||||
@@ -166,9 +300,15 @@ class WikiVectorSearchService:
|
||||
candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
|
||||
return candidates
|
||||
|
||||
def _top_indices(self, snapshot: WikiVectorSnapshot, vector: np.ndarray, eligible: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
||||
def _top_indices(
|
||||
self,
|
||||
snapshot: WikiVectorSnapshot,
|
||||
vector: np.ndarray,
|
||||
eligible: np.ndarray,
|
||||
section_limit: int | None = None,
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
scores = snapshot.matrix[eligible] @ vector
|
||||
count = min(self.config.top_k_sections, scores.size)
|
||||
count = min(section_limit or self.config.top_k_sections, scores.size)
|
||||
if count <= 0:
|
||||
return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
|
||||
positions = np.argpartition(scores, -count)[-count:]
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ def _normalize(data: dict[str, Any], menu_type: str, updated_by: str | None) ->
|
||||
return {
|
||||
"node_id": storage_node_id(menu_type, node_id),
|
||||
"menu_type": menu_type,
|
||||
"placement": placement if placement in {"sidebar", "settings"} else "sidebar",
|
||||
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"label": (data.get("label") or None),
|
||||
"updated_by": updated_by,
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C
|
||||
return CompactMenuOverrideRow(
|
||||
node_id=storage_node_id(menu_type, node_id),
|
||||
menu_type=menu_type,
|
||||
placement=placement if placement in {"sidebar", "settings"} else "sidebar",
|
||||
placement=placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
sort_order=int(data.get("sort_order", 0)),
|
||||
label=(data.get("label") or None),
|
||||
updated_by=updated_by,
|
||||
|
||||
Reference in new issue
Block a user