sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -2,12 +2,22 @@
from __future__ import annotations
import asyncio
import hashlib
import time
from collections import deque
from collections.abc import Mapping, Sequence
from typing import Any
from urllib.parse import quote
import httpx
import regex
_GREP_CONTRACT = "weknora-v0.7.2-grep-v1"
_GREP_COMPATIBILITY_CONTRACT = "weknora-existing-api-compat-v1"
_GREP_MAX_CANDIDATES = 500
_GREP_MAX_RESULTS = 30
_GREP_REGEX_TIMEOUT_SECONDS = 0.05
class WeKnoraError(RuntimeError):
@@ -45,9 +55,13 @@ class WeKnoraClient:
tests. When omitted, each public method creates and closes a short-lived client.
If admin credentials are configured, DeerFlow authenticates to WeKnora with a
cached bearer token; otherwise it keeps the legacy trusted-network anonymous
mode for deployments that allow DeerFlow through a reverse proxy.
mode for deployments that allow DeerFlow through a reverse proxy. The
knowledge-base directory is also cached briefly because several Gateway
scopes request the same expensive upstream list at once.
"""
_KNOWLEDGE_BASES_CACHE_SECONDS = 15.0
def __init__(
self,
*,
@@ -65,6 +79,11 @@ class WeKnoraClient:
self._auth_token = ""
self._tenant_id = ""
self._auth_expires_at = 0.0
self._auth_generation = 0
self._auth_lock = asyncio.Lock()
self._knowledge_bases_cache: tuple[float, list[dict[str, Any]]] | None = None
self._knowledge_bases_cache_generation = 0
self._knowledge_bases_task: tuple[int, asyncio.Task[list[dict[str, Any]]]] | None = None
if not self.base_url:
raise ValueError("WeKnora base URL is required")
@@ -77,20 +96,8 @@ class WeKnoraClient:
if not isinstance(payload, dict):
return "", ""
source = payload.get("data") if isinstance(payload.get("data"), dict) else payload
token = str(
source.get("token")
or source.get("access_token")
or payload.get("token")
or payload.get("access_token")
or ""
)
tenant = (
source.get("tenant")
if isinstance(source.get("tenant"), dict)
else source.get("active_tenant")
if isinstance(source.get("active_tenant"), dict)
else payload.get("tenant")
)
token = str(source.get("token") or source.get("access_token") or payload.get("token") or payload.get("access_token") or "")
tenant = source.get("tenant") if isinstance(source.get("tenant"), dict) else source.get("active_tenant") if isinstance(source.get("active_tenant"), dict) else payload.get("tenant")
user = source.get("user") if isinstance(source.get("user"), dict) else payload.get("user")
tenant_id = ""
if isinstance(tenant, dict):
@@ -106,43 +113,51 @@ class WeKnoraClient:
if self._auth_token and not force_refresh and self._auth_expires_at > now + 60:
return
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=False,
trust_env=False,
)
try:
response = await client.post(
f"{self.base_url}/api/v1/auth/login",
json={"email": self.admin_email, "password": self.admin_password},
headers={"Accept": "application/json"},
observed_generation = self._auth_generation
async with self._auth_lock:
now = time.time()
token_is_fresh = bool(self._auth_token and self._auth_expires_at > now + 60)
if token_is_fresh and (not force_refresh or self._auth_generation != observed_generation):
return
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=False,
trust_env=False,
)
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
try:
response = await client.post(
f"{self.base_url}/api/v1/auth/login",
json={"email": self.admin_email, "password": self.admin_password},
headers={"Accept": "application/json"},
)
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
try:
payload = response.json()
except ValueError as exc:
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
if isinstance(payload, dict) and payload.get("success") is False:
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
token, tenant_id = self._extract_login_token(payload)
if not token:
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
try:
payload = response.json()
except ValueError as exc:
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
if isinstance(payload, dict) and payload.get("success") is False:
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
token, tenant_id = self._extract_login_token(payload)
if not token:
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
self._auth_token = token
self._tenant_id = tenant_id
# WeKnora login responses do not consistently expose expiry. Refresh
# periodically and retry once if upstream says the token is stale.
self._auth_expires_at = now + 25 * 60
self._auth_token = token
self._tenant_id = tenant_id
self._auth_generation += 1
# WeKnora login responses do not consistently expose expiry. Refresh
# periodically and retry once if upstream says the token is stale.
self._auth_expires_at = now + 25 * 60
async def _headers(self, *, force_refresh: bool = False, accept: str = "application/json") -> dict[str, str]:
headers = {"Accept": accept}
@@ -192,6 +207,8 @@ class WeKnoraClient:
await client.aclose()
if response.status_code == 404:
raise WeKnoraError("WeKnora resource was not found", status_code=404)
if response.status_code == 405:
raise WeKnoraError("WeKnora endpoint does not support this method", status_code=405)
if response.status_code in {401, 403}:
if self._has_admin_credentials:
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated internal request", status_code=502)
@@ -207,7 +224,14 @@ class WeKnoraClient:
detail = str(body.get("message") or body.get("error") or body.get("detail") or "")
except ValueError:
detail = ""
raise WeKnoraError(detail or f"WeKnora returned HTTP {response.status_code}", status_code=502)
# Folder creation uses 409 as an optimistic-concurrency signal.
# Preserve it so callers can re-read and adopt the concurrently
# created folder even if a deployment localizes the error text.
status_code = 409 if response.status_code == 409 else 502
raise WeKnoraError(
detail or f"WeKnora returned HTTP {response.status_code}",
status_code=status_code,
)
if response.status_code == 204 or not response.content:
return {"success": True}
try:
@@ -219,8 +243,49 @@ class WeKnoraClient:
payload = await self._request("GET", "/health")
return dict(payload) if isinstance(payload, dict) else {"status": "ok"}
async def list_knowledge_bases(self) -> list[dict[str, Any]]:
return _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
async def list_knowledge_bases(self, *, refresh: bool = False) -> list[dict[str, Any]]:
now = time.monotonic()
cached = self._knowledge_bases_cache
if not refresh and cached is not None and cached[0] > now:
return [dict(item) for item in cached[1]]
# Share both success and failure with the current wave of callers.
# A lock around an uncached failed read would make every waiter issue
# another full timeout, serializing an unavailable service's fan-out.
generation = self._knowledge_bases_cache_generation
pending = self._knowledge_bases_task
if pending is None or pending[0] != generation:
task = asyncio.create_task(self._load_knowledge_bases(generation))
self._knowledge_bases_task = (generation, task)
def clear_pending(completed: asyncio.Task[list[dict[str, Any]]]) -> None:
if self._knowledge_bases_task is not None and self._knowledge_bases_task[1] is completed:
self._knowledge_bases_task = None
if not completed.cancelled():
completed.exception()
task.add_done_callback(clear_pending)
else:
task = pending[1]
rows = await asyncio.shield(task)
return [dict(item) for item in rows]
async def _load_knowledge_bases(self, generation: int) -> list[dict[str, Any]]:
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
# A mutation can finish while the read is in flight. Its old snapshot
# must neither repopulate the cache nor be joined by post-write callers.
if self._knowledge_bases_cache_generation == generation:
self._knowledge_bases_cache = (
time.monotonic() + self._KNOWLEDGE_BASES_CACHE_SECONDS,
[dict(item) for item in rows],
)
return rows
def _invalidate_knowledge_bases_cache(self) -> None:
"""Discard the directory after a successful local mutation."""
self._knowledge_bases_cache = None
self._knowledge_bases_cache_generation += 1
async def get_knowledge_base(self, knowledge_base_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}"))
@@ -233,6 +298,23 @@ class WeKnoraClient:
return _as_list(_data(await self._request("GET", "/api/v1/models")))
async def debug_model(self, model_id: str) -> dict[str, Any]:
"""Ask WeKnora to verify one saved model with a small real request."""
normalized_id = str(model_id or "").strip()
if not normalized_id:
raise WeKnoraError("WeKnora model ID is required", status_code=400)
value = _data(
await self._request(
"POST",
f"/api/v1/models/{quote(normalized_id, safe='')}/debug",
files={"input": (None, "Reply with OK.")},
)
)
if not isinstance(value, dict) or not isinstance(value.get("ok"), bool):
raise WeKnoraError("WeKnora returned an invalid model debug result")
return dict(value)
async def get_default_embedding_model_id(self) -> str:
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "embedding"]
preferred = next((item for item in models if item.get("is_default") is True), None)
@@ -249,21 +331,58 @@ class WeKnoraClient:
return model_id
async def get_default_knowledge_qa_model_id(self) -> str:
"""Resolve WeKnora's own active synthesis model for Wiki indexing."""
"""Resolve a live WeKnora synthesis model for Wiki indexing.
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "knowledgeqa"]
preferred = next((item for item in models if item.get("is_default") is True), None)
selected = preferred or next(
(item for item in models if str(item.get("status") or "active").lower() == "active"),
None,
)
model_id = str((selected or {}).get("id") or "")
if not model_id:
WeKnora's ``active`` status means that the database configuration is
enabled; it does not prove that the upstream provider still serves the
model. Probe active candidates before binding a new Wiki so a disabled
provider model cannot silently send every ingest task to dead letters.
"""
models = [
item
for item in await self.list_models()
if str(item.get("type") or "").strip().lower() == "knowledgeqa"
and str(item.get("id") or "").strip()
and str(item.get("status") or "active").strip().lower() == "active"
]
if not models:
raise WeKnoraError(
"WeKnora has no available KnowledgeQA model; configure one in WeKnora before creating a Wiki",
status_code=400,
)
return model_id
# Prefer an explicit database default. When no default exists, sort by
# immutable persisted ID rather than trusting API row order or an
# editable display name.
models.sort(
key=lambda item: (
0 if item.get("is_default") is True else 1,
str(item.get("id") or "").strip(),
)
)
attempted: list[str] = []
for model in models:
model_id = str(model.get("id") or "").strip()
label = str(model.get("display_name") or model.get("name") or model_id).strip()
attempted.append(label or model_id)
# ``ok=false`` is a candidate-specific upstream-model failure. An
# HTTP/auth/timeout failure from the debug endpoint is a WeKnora
# service failure, so preserve it instead of multiplying probes
# and misreporting the incident as bad model configuration.
debug = await self.debug_model(model_id)
if debug.get("ok") is True:
return model_id
attempted_models = ", ".join(attempted[:12])
if len(attempted) > 12:
attempted_models = f"{attempted_models}, and {len(attempted) - 12} more"
raise WeKnoraError(
"WeKnora has no usable active KnowledgeQA model after connectivity checks"
+ (f": {attempted_models}" if attempted_models else "")
+ "; configure or repair a working default model before creating a Wiki",
status_code=400,
)
async def create_knowledge_base(
self,
@@ -273,6 +392,8 @@ class WeKnoraClient:
kb_type: str = "document",
embedding_model_id: str | None = None,
wiki_enabled: bool = False,
wiki_content_instructions: str = "",
wiki_extraction_instructions: str = "",
) -> dict[str, Any]:
model_id = embedding_model_id or await self.get_default_embedding_model_id()
create_payload: dict[str, Any] = {
@@ -295,6 +416,8 @@ class WeKnoraClient:
"wiki_config": {
"synthesis_model_id": synthesis_model_id,
"extraction_granularity": "standard",
"content_instructions": wiki_content_instructions.strip(),
"extraction_instructions": wiki_extraction_instructions.strip(),
},
}
)
@@ -306,16 +429,31 @@ class WeKnoraClient:
value = _data(payload)
if not isinstance(value, dict) or not value.get("id"):
raise WeKnoraError("WeKnora did not return the created knowledge base")
self._invalidate_knowledge_bases_cache()
return dict(value)
async def update_knowledge_base(self, knowledge_base_id: str, changes: Mapping[str, Any]) -> dict[str, Any]:
payload = dict(changes)
if not str(payload.get("name") or "").strip():
wiki_config_changes = payload.pop("wiki_config", None)
current: dict[str, Any] | None = None
if not str(payload.get("name") or "").strip() or wiki_config_changes is not None:
current = await self.get_knowledge_base(knowledge_base_id)
if not str(payload.get("name") or "").strip():
assert current is not None
current_name = str(current.get("name") or "").strip()
if not current_name:
raise WeKnoraError("WeKnora knowledge base name is required for updates")
payload["name"] = current_name
if wiki_config_changes is not None:
if not isinstance(wiki_config_changes, Mapping):
raise WeKnoraError("WeKnora Wiki instructions must be an object", status_code=422)
assert current is not None
current_wiki_config = current.get("wiki_config")
merged_wiki_config = dict(current_wiki_config) if isinstance(current_wiki_config, Mapping) else {}
for key in ("content_instructions", "extraction_instructions"):
if key in wiki_config_changes:
merged_wiki_config[key] = str(wiki_config_changes.get(key) or "").strip()
payload["config"] = {"wiki_config": merged_wiki_config}
value = _data(
await self._request(
"PUT",
@@ -323,10 +461,12 @@ class WeKnoraClient:
json=payload,
)
)
self._invalidate_knowledge_bases_cache()
return dict(value) if isinstance(value, dict) else {}
async def delete_knowledge_base(self, knowledge_base_id: str) -> None:
await self._request("DELETE", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}")
self._invalidate_knowledge_bases_cache()
async def list_documents(self, knowledge_base_id: str, *, page: int = 1, page_size: int = 100) -> dict[str, Any]:
payload = await self._request(
@@ -353,6 +493,7 @@ class WeKnoraClient:
files={"file": (filename, content, content_type)},
)
)
self._invalidate_knowledge_bases_cache()
return dict(value) if isinstance(value, dict) else {}
async def import_document_url(self, knowledge_base_id: str, *, url: str) -> dict[str, Any]:
@@ -363,6 +504,7 @@ class WeKnoraClient:
json={"url": url},
)
)
self._invalidate_knowledge_bases_cache()
return dict(value) if isinstance(value, dict) else {}
async def create_manual_document(
@@ -379,10 +521,12 @@ class WeKnoraClient:
json={"title": title, "content": content, "status": "publish"},
)
)
self._invalidate_knowledge_bases_cache()
return dict(value) if isinstance(value, dict) else {}
async def delete_document(self, knowledge_id: str) -> None:
await self._request("DELETE", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}")
self._invalidate_knowledge_bases_cache()
async def get_document(self, knowledge_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}"))
@@ -477,6 +621,7 @@ class WeKnoraClient:
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
self._invalidate_knowledge_bases_cache()
return dict(value) if isinstance(value, dict) else {}
async def list_chunks(self, knowledge_id: str, *, page: int = 1, page_size: int = 20) -> dict[str, Any]:
@@ -513,6 +658,79 @@ class WeKnoraClient:
raise WeKnoraError("Invalid Wiki folder response")
return value
async def list_wiki_folders(
self,
knowledge_base_id: str,
*,
parent_id: str = "",
page_types: Sequence[str] | None = None,
) -> dict[str, Any]:
params: dict[str, Any] = {"parent_id": parent_id or None}
if page_types:
params["page_types"] = ",".join(str(value) for value in page_types if str(value)) or None
payload = await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders",
params=params,
)
value = _data(payload)
source = value if isinstance(value, dict) else {}
raw_folders = source.get("folders") if isinstance(value, dict) else value
rows = _as_list(raw_folders)
return {
"parent_id": str(source.get("parent_id") or parent_id),
"folders": rows,
}
async def update_wiki_folder(
self,
knowledge_base_id: str,
folder_id: str,
changes: Mapping[str, Any] | None = None,
*,
name: str | None = None,
parent_id: str | None = None,
move_parent: bool | None = None,
) -> dict[str, Any]:
update = dict(changes or {})
if name is not None:
update["name"] = name
if parent_id is not None:
update["parent_id"] = parent_id
update.setdefault("move_parent", True)
if move_parent is not None:
update["move_parent"] = move_parent
elif "parent_id" in update:
update.setdefault("move_parent", True)
value = _data(
await self._request(
"PUT",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders/{quote(folder_id, safe='')}",
json=update,
)
)
if not isinstance(value, dict):
raise WeKnoraError("Invalid Wiki folder response")
return dict(value)
async def delete_wiki_folder(self, knowledge_base_id: str, folder_id: str) -> None:
await self._request(
"DELETE",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders/{quote(folder_id, safe='')}",
)
async def move_wiki_page(self, knowledge_base_id: str, slug: str, folder_id: str) -> dict[str, Any]:
value = _data(
await self._request(
"PUT",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/move-page",
json={"slug": slug, "folder_id": folder_id},
)
)
if not isinstance(value, dict):
raise WeKnoraError("Invalid Wiki page response")
return dict(value)
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
@@ -523,11 +741,17 @@ class WeKnoraClient:
page: int = 1,
page_size: int = 50,
query: str = "",
folder_id: str | None = None,
) -> dict[str, Any]:
params: dict[str, Any] = {"page": page, "page_size": page_size, "query": query or None}
# ``None`` means no placement filter; an explicit empty string means
# pages directly under the Wiki root.
if folder_id is not None:
params["folder_id"] = folder_id
payload = await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages",
params={"page": page, "page_size": page_size, "query": query or None},
params=params,
)
value = _data(payload)
source = value if isinstance(value, dict) else payload if isinstance(payload, dict) else {}
@@ -607,6 +831,327 @@ class WeKnoraClient:
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
)
@staticmethod
def _normalize_grep_chunk(row: Mapping[str, Any]) -> dict[str, Any]:
"""Normalize both the provider extension and compatibility rows."""
return {
"chunk_id": str(row.get("id") or row.get("chunk_id") or ""),
"content": str(row.get("content") or row.get("text") or row.get("match_snippet") or ""),
"knowledge_id": str(row.get("knowledge_id") or ""),
"knowledge_base_id": str(row.get("knowledge_base_id") or ""),
"title": str(
row.get("knowledge_title")
or row.get("title")
or row.get("knowledge_filename")
or ""
),
"chunk_index": int(row.get("chunk_index") or row.get("index") or 0),
"match_count": int(
row.get("matched_patterns")
or row.get("match_count")
or row.get("total_pattern_hits")
or 0
),
"title_match": bool(row.get("title_match")),
"score": float(row.get("match_score") or row.get("score") or 0.0),
"metadata": row.get("metadata") if isinstance(row.get("metadata"), dict) else {},
}
@staticmethod
def _grep_matches(expression: regex.Pattern, value: str) -> list[regex.Match]:
try:
return list(
expression.finditer(
value,
timeout=_GREP_REGEX_TIMEOUT_SECONDS,
)
)
except TimeoutError as exc:
raise WeKnoraError(
"WeKnora compatibility grep regular expression timed out",
status_code=422,
) from exc
async def _grep_chunks_compatibility(
self,
query: str,
knowledge_base_ids: Sequence[str],
) -> dict[str, Any]:
"""Bounded, fair fallback for older providers without ``chunks/grep``.
The native endpoint remains authoritative. This path round-robins both
knowledge bases and documents so one large first scope cannot consume
the global 500-chunk compatibility budget.
"""
ids = list(dict.fromkeys(str(value).strip() for value in knowledge_base_ids if str(value).strip()))
try:
expression = regex.compile(query, regex.IGNORECASE | regex.VERSION1)
except regex.error as exc:
raise WeKnoraError(f"Invalid grep regular expression: {exc}", status_code=422) from exc
per_scope_document_cap = max(1, (_GREP_MAX_CANDIDATES + len(ids) - 1) // max(1, len(ids)))
async def load_documents(knowledge_base_id: str) -> tuple[str, list[dict[str, Any]] | Exception]:
rows: list[dict[str, Any]] = []
page = 1
try:
while len(rows) < per_scope_document_cap:
page_size = min(100, per_scope_document_cap - len(rows))
payload = await self.list_documents(
knowledge_base_id,
page=page,
page_size=page_size,
)
batch = [
dict(item)
for item in payload.get("items") or []
if isinstance(item, Mapping)
]
rows.extend(batch)
if not batch or page * page_size >= int(payload.get("total") or len(rows)):
break
page += 1
return knowledge_base_id, rows
except Exception as exc: # noqa: BLE001 - returned as structured scope failure
return knowledge_base_id, exc
loaded = await asyncio.gather(*(load_documents(knowledge_base_id) for knowledge_base_id in ids))
documents_by_scope: dict[str, list[dict[str, Any]]] = {}
warnings: list[dict[str, Any]] = [
{
"code": "WEKNORA_GREP_COMPATIBILITY_FALLBACK",
"message": "Provider chunks/grep is unavailable; used bounded existing-API compatibility scan.",
}
]
failed_scope_ids: set[str] = set()
for knowledge_base_id, outcome in loaded:
if isinstance(outcome, Exception):
failed_scope_ids.add(knowledge_base_id)
warnings.append(
{
"code": "PROVIDER_SCOPE_FAILED",
"knowledge_base_id": knowledge_base_id,
"stage": "list_documents",
}
)
else:
documents_by_scope[knowledge_base_id] = outcome
if not documents_by_scope:
raise WeKnoraError("WeKnora compatibility grep failed for every knowledge base")
# Seed the work queue in KB/document round-robin order. Subsequent
# chunk pages are re-appended to the tail, preserving document fairness.
work: deque[tuple[str, dict[str, Any], int]] = deque()
max_documents = max((len(rows) for rows in documents_by_scope.values()), default=0)
for index in range(max_documents):
for knowledge_base_id in ids:
rows = documents_by_scope.get(knowledge_base_id) or []
if index < len(rows):
work.append((knowledge_base_id, rows[index], 1))
candidates: list[dict[str, Any]] = []
seen_content: set[str] = set()
scanned_chunks = 0
chunk_successes = {knowledge_base_id: 0 for knowledge_base_id in documents_by_scope}
chunk_failures = {knowledge_base_id: 0 for knowledge_base_id in documents_by_scope}
while work and scanned_chunks < _GREP_MAX_CANDIDATES:
knowledge_base_id, document, page = work.popleft()
knowledge_id = str(document.get("id") or document.get("knowledge_id") or "")
if not knowledge_id:
continue
page_size = min(20, _GREP_MAX_CANDIDATES - scanned_chunks)
try:
payload = await self.list_chunks(
knowledge_id,
page=page,
page_size=page_size,
)
except Exception: # noqa: BLE001 - one document must not fail the whole fallback
chunk_failures[knowledge_base_id] += 1
continue
chunk_successes[knowledge_base_id] += 1
chunks = [
dict(item)
for item in payload.get("items") or []
if isinstance(item, Mapping)
]
title = str(
document.get("title")
or document.get("name")
or document.get("filename")
or ""
)
title_matches = self._grep_matches(expression, title[:5000])
for chunk in chunks:
if scanned_chunks >= _GREP_MAX_CANDIDATES:
break
scanned_chunks += 1
content = str(chunk.get("content") or chunk.get("text") or "")
content_matches = self._grep_matches(expression, content[:50_000])
if not title_matches and not content_matches:
continue
signature = hashlib.sha256(" ".join(content.split()).encode()).hexdigest()
if signature in seen_content:
continue
seen_content.add(signature)
match_count = min(99, len(title_matches) * 4 + len(content_matches))
candidates.append(
self._normalize_grep_chunk(
{
**chunk,
"knowledge_id": knowledge_id,
"knowledge_base_id": knowledge_base_id,
"knowledge_title": title,
"match_count": match_count,
"title_match": bool(title_matches),
"match_score": float(match_count),
}
)
)
if chunks and page * page_size < int(payload.get("total") or len(chunks)):
work.append((knowledge_base_id, document, page + 1))
successful_scope_ids: list[str] = []
for knowledge_base_id, documents in documents_by_scope.items():
# An empty KB was successfully scanned. A non-empty KB for which
# every chunk read failed is a failed provider scope.
if not documents or chunk_successes[knowledge_base_id] > 0:
successful_scope_ids.append(knowledge_base_id)
else:
failed_scope_ids.add(knowledge_base_id)
if chunk_failures[knowledge_base_id]:
warnings.append(
{
"code": "PROVIDER_DOCUMENT_READ_FAILED",
"knowledge_base_id": knowledge_base_id,
"failed_documents": chunk_failures[knowledge_base_id],
}
)
if not successful_scope_ids:
raise WeKnoraError("WeKnora compatibility grep failed for every knowledge base")
candidates.sort(
key=lambda item: (
bool(item.get("title_match")),
int(item.get("match_count") or 0),
float(item.get("score") or 0),
-int(item.get("chunk_index") or 0),
),
reverse=True,
)
return {
"items": candidates[:_GREP_MAX_RESULTS],
"contract": _GREP_COMPATIBILITY_CONTRACT,
"compatibility_fallback": True,
"partial": bool(failed_scope_ids or any(chunk_failures.values())),
"warnings": warnings,
"successful_knowledge_base_ids": successful_scope_ids,
"failed_knowledge_base_ids": [value for value in ids if value in failed_scope_ids],
"scanned_chunks": scanned_chunks,
"scan_truncated": scanned_chunks >= _GREP_MAX_CANDIDATES,
}
async def grep_chunks(
self,
query: str,
knowledge_base_ids: Sequence[str],
) -> dict[str, Any]:
"""Run the v0.7.2 cross-KB grep extension, with a marked fallback."""
ids = list(dict.fromkeys(str(value).strip() for value in knowledge_base_ids if str(value).strip()))
if not ids:
return {
"items": [],
"contract": _GREP_CONTRACT,
"compatibility_fallback": False,
"partial": False,
"warnings": [],
"successful_knowledge_base_ids": [],
"failed_knowledge_base_ids": [],
"scanned_chunks": 0,
"scan_truncated": False,
}
try:
payload = await self._request(
"POST",
"/api/v1/chunks/grep",
json={"query": query, "knowledge_base_ids": ids},
)
except WeKnoraError as exc:
if exc.status_code not in {404, 405}:
raise
return await self._grep_chunks_compatibility(query, ids)
value = _data(payload)
source = value if isinstance(value, dict) else {}
contract = str(
source.get("contract")
or (payload.get("contract") if isinstance(payload, dict) else "")
or ""
)
if contract != _GREP_CONTRACT:
raise WeKnoraError(
f"WeKnora chunks/grep contract mismatch (expected {_GREP_CONTRACT})"
)
raw_rows = (
source.get("chunk_results")
or source.get("results")
or source.get("items")
or []
)
if not isinstance(raw_rows, list):
raise WeKnoraError("WeKnora chunks/grep returned invalid results")
rows = [
self._normalize_grep_chunk(row)
for row in raw_rows
if isinstance(row, Mapping)
]
warnings = [
dict(item)
for item in source.get("warnings") or []
if isinstance(item, Mapping)
]
failed_ids = [
str(value)
for value in source.get("failed_knowledge_base_ids") or []
if str(value) in ids
]
explicit_successes = source.get("successful_knowledge_base_ids")
successful_ids = (
[str(value) for value in explicit_successes if str(value) in ids]
if isinstance(explicit_successes, list)
else [value for value in ids if value not in failed_ids]
)
warned_scope_ids = {
str(item.get("knowledge_base_id") or "")
for item in warnings
}
warnings.extend(
{
"code": "PROVIDER_SCOPE_FAILED",
"knowledge_base_id": knowledge_base_id,
}
for knowledge_base_id in failed_ids
if knowledge_base_id not in warned_scope_ids
)
if source.get("partial") and not warnings:
warnings.append({"code": "PROVIDER_PARTIAL_RESPONSE"})
return {
"items": rows[:_GREP_MAX_RESULTS],
"contract": contract,
"compatibility_fallback": False,
"partial": bool(source.get("partial")) or bool(warnings) or bool(failed_ids),
"warnings": warnings,
"successful_knowledge_base_ids": successful_ids,
"failed_knowledge_base_ids": failed_ids,
"scanned_chunks": int(source.get("scanned_chunks") or len(rows)),
"scan_truncated": bool(source.get("scan_truncated")),
}
async def search(self, query: str, knowledge_base_ids: Sequence[str]) -> list[dict[str, Any]]:
ids = [str(value) for value in knowledge_base_ids if str(value)]
if not ids:
@@ -0,0 +1,82 @@
"""Product-owned configuration layered over a WeKnora knowledge base."""
from __future__ import annotations
from typing import Any
DEFAULT_RETRIEVAL_SETTINGS: dict[str, Any] = {
"entity_expansion_enabled": True,
"max_related_terms": 6,
"statistics_enabled": True,
}
def default_test_questions(name: str) -> list[str]:
label = str(name or "当前知识库").strip() or "当前知识库"
return [
f"「{label}」主要讲了什么?请概括资料范围并引用来源。",
f"请梳理「{label}」中的核心人物、机构、事件及其关联关系。",
"请统计指定日期范围内与关注对象有关的资料数量,并列出具体新闻或来源。",
f"「{label}」中哪些资料相互印证,哪些结论存在冲突或证据不足?",
]
def generated_test_questions_name(value: Any) -> str | None:
"""Return the embedded name when *value* is an untouched default set.
Older versions of the knowledge-base editor submitted the resolved default
questions on every save. That accidentally persisted them as custom
questions, so a later rename kept showing the old knowledge-base name. We
recognise only the exact four-question template here; genuinely edited
questions remain custom and are never rewritten.
"""
if not isinstance(value, list) or len(value) != 4:
return None
questions = [str(item).strip() for item in value]
prefix = "「"
suffix = "」主要讲了什么?请概括资料范围并引用来源。"
first = questions[0]
if not first.startswith(prefix) or not first.endswith(suffix):
return None
name = first[len(prefix) : -len(suffix)].strip()
if not name or questions != default_test_questions(name):
return None
return name
def resolved_test_questions(row: dict[str, Any]) -> list[str]:
configured = row.get("test_questions")
if isinstance(configured, list):
# Treat accidentally persisted legacy defaults as automatic. This
# repairs existing rows immediately and lets the questions follow a
# renamed knowledge base without a data migration.
if generated_test_questions_name(configured) is None:
return [str(item).strip() for item in configured if str(item).strip()][:12]
return default_test_questions(str(row.get("name") or ""))
def are_test_questions_customized(row: dict[str, Any]) -> bool:
configured = row.get("test_questions")
return isinstance(configured, list) and generated_test_questions_name(configured) is None
def resolved_retrieval_settings(row: dict[str, Any]) -> dict[str, Any]:
configured = row.get("retrieval_settings")
if not isinstance(configured, dict):
configured = {}
return {
"entity_expansion_enabled": bool(configured.get("entity_expansion_enabled", True)),
"max_related_terms": max(0, min(int(configured.get("max_related_terms", 6)), 12)),
"statistics_enabled": bool(configured.get("statistics_enabled", True)),
}
__all__ = [
"DEFAULT_RETRIEVAL_SETTINGS",
"default_test_questions",
"generated_test_questions_name",
"resolved_retrieval_settings",
"resolved_test_questions",
"are_test_questions_customized",
]
@@ -0,0 +1,518 @@
"""Entity-expanded and statistics-aware retrieval over the local Wiki mirror.
The language model is deliberately not asked to count search results. This
module parses an optional date range, expands first-degree Wiki links, filters
the stored pages/sources, and returns auditable counts plus concrete references.
"""
from __future__ import annotations
import asyncio
import re
from datetime import UTC, date, datetime
from typing import Any
from deerflow.integrations.weknora.knowledge_profile import resolved_retrieval_settings
from deerflow.integrations.weknora.local_index.overview import is_wiki_body_page, is_wiki_overview_query
from deerflow.integrations.weknora.local_index.ranking import (
title_boosted_rrf_score,
title_match_strength,
)
_DATE_KEY_PATTERN = re.compile(
r"(?im)^\s*(?:published_at|publish_time|published|date|发布日期|发布时间|新闻时间|日期|时间)\s*[::]\s*[\"']?([^\n\"']+)"
)
_ISO_DATE_PATTERN = re.compile(r"(?<!\d)(20\d{2})[-/.年](\d{1,2})[-/.月](\d{1,2})(?:日|号)?")
_MONTH_DAY_PATTERN = re.compile(r"(?<!\d)(\d{1,2})月(\d{1,2})(?:日|号)?")
_DAY_RANGE_PATTERN = re.compile(r"(?<!\d)(\d{1,2})(?:日|号)?\s*(?:到|至|[-~—–])\s*(\d{1,2})(?:日|号)")
_WIKILINK_PATTERN = re.compile(r"\[\[([^\]|#]{2,80})(?:#[^\]|]+)?(?:\|([^\]]{1,80}))?\]\]")
_MARKDOWN_LINK_PATTERN = re.compile(r"\[([^\]]{2,80})\]\(([^)]+)\)")
_STATISTICS_PATTERN = re.compile(r"(?:多少|几篇|几条|统计|数量|占比|汇总|合计)")
_SUBJECT_PATTERNS = (
re.compile(r"关于\s*([^,。;;??]{2,40}?)(?:的|有多少|相关新闻|相关资料|相关报告|新闻|报告|资料|$)"),
re.compile(r"(?:搜索|查询|查找|检索)\s*([^,。;;??]{2,30}?)(?:的|相关新闻|相关资料|新闻|报告|资料|$)"),
)
_QUERY_STOP_WORDS = {
"帮我", "总结", "一下", "报告", "新闻", "资料", "数据", "相关", "有关", "关于",
"多少", "统计", "查看", "具体", "知识库", "这个", "哪些", "时间", "日期", "月份",
}
def _safe_date(year: int, month: int, day: int) -> date | None:
try:
return date(year, month, day)
except ValueError:
return None
def _parse_date_value(value: Any, *, today: date) -> date | None:
if isinstance(value, datetime):
return value.date()
if isinstance(value, date):
return value
text = str(value or "").strip()
match = _ISO_DATE_PATTERN.search(text)
if match:
return _safe_date(int(match.group(1)), int(match.group(2)), int(match.group(3)))
match = _MONTH_DAY_PATTERN.search(text)
if match:
return _safe_date(today.year, int(match.group(1)), int(match.group(2)))
try:
return datetime.fromisoformat(text.replace("Z", "+00:00")).date()
except (TypeError, ValueError):
return None
def parse_date_range(
query: str,
*,
date_from: str | None = None,
date_to: str | None = None,
today: date | None = None,
) -> tuple[date | None, date | None]:
now = today or datetime.now(UTC).date()
explicit_start = _parse_date_value(date_from, today=now)
explicit_end = _parse_date_value(date_to, today=now)
if explicit_start or explicit_end:
return explicit_start, explicit_end
full = [_safe_date(int(y), int(m), int(d)) for y, m, d in _ISO_DATE_PATTERN.findall(query)]
full = [item for item in full if item is not None]
if len(full) >= 2:
return min(full[0], full[1]), max(full[0], full[1])
month_days = [_safe_date(now.year, int(m), int(d)) for m, d in _MONTH_DAY_PATTERN.findall(query)]
month_days = [item for item in month_days if item is not None]
if len(month_days) >= 2:
return min(month_days[0], month_days[1]), max(month_days[0], month_days[1])
day_range = _DAY_RANGE_PATTERN.search(query)
if day_range:
start = _safe_date(now.year, now.month, int(day_range.group(1)))
end = _safe_date(now.year, now.month, int(day_range.group(2)))
if start and end:
return min(start, end), max(start, end)
return (full[0], full[0]) if len(full) == 1 else (month_days[0], month_days[0]) if len(month_days) == 1 else (None, None)
def extract_subject_terms(query: str) -> list[str]:
terms: list[str] = []
terms.extend(item.strip() for item in re.findall(r"[「『\"']([^」』\"']{2,40})[」』\"']", query))
for pattern in _SUBJECT_PATTERNS:
match = pattern.search(query)
if match:
terms.append(match.group(1).strip(" 的之与和"))
if not terms:
cleaned = _ISO_DATE_PATTERN.sub(" ", query)
cleaned = _MONTH_DAY_PATTERN.sub(" ", cleaned)
for word in sorted(_QUERY_STOP_WORDS, key=len, reverse=True):
cleaned = cleaned.replace(word, " ")
candidates = re.findall(r"[A-Za-z][A-Za-z0-9_.-]{1,39}|[\u4e00-\u9fff·]{2,20}", cleaned)
if candidates:
terms.append(max(candidates, key=len))
return list(dict.fromkeys(term for term in terms if 1 < len(term) <= 40))[:4]
def _page_haystack(page: dict[str, Any]) -> str:
aliases = " ".join(str(item) for item in page.get("aliases") or [])
return f"{page.get('title') or ''}\n{aliases}\n{page.get('summary') or ''}\n{page.get('content_md') or ''}".casefold()
def expand_related_terms(
subject_terms: list[str],
pages: list[dict[str, Any]],
*,
limit: int,
) -> list[dict[str, str]]:
if limit <= 0:
return []
subjects = {item.casefold() for item in subject_terms}
expanded: list[dict[str, str]] = []
seen = set(subjects)
for page in pages:
haystack = _page_haystack(page)
if subjects and not any(subject in haystack for subject in subjects):
continue
candidates: list[tuple[str, str]] = []
candidates.extend((str(alias).strip(), "alias") for alias in page.get("aliases") or [])
candidates.extend(((label or target).strip(), "wikilink") for target, label in _WIKILINK_PATTERN.findall(str(page.get("content_md") or "")))
candidates.extend((label.strip(), "markdown_link") for label, _ in _MARKDOWN_LINK_PATTERN.findall(str(page.get("content_md") or "")))
for term, relation in candidates:
key = term.casefold()
if key in seen or not 1 < len(term) <= 40 or term.startswith(("http://", "https://")):
continue
seen.add(key)
expanded.append({"term": term, "relation": relation, "from_page": str(page.get("title") or page.get("slug") or "")})
if len(expanded) >= limit:
return expanded
return expanded
def extract_record_date(record: dict[str, Any], *, today: date | None = None) -> tuple[date | None, str | None]:
now = today or datetime.now(UTC).date()
for key in ("published_at", "publish_time", "published", "date", "发布日期", "发布时间", "新闻时间"):
parsed = _parse_date_value(record.get(key), today=now)
if parsed:
return parsed, key
metadata = record.get("metadata")
if isinstance(metadata, dict):
parsed, source = extract_record_date(metadata, today=now)
if parsed:
return parsed, f"metadata.{source}"
content = str(record.get("content_md") or record.get("content") or "")
match = _DATE_KEY_PATTERN.search(content[:8000])
if match:
parsed = _parse_date_value(match.group(1), today=now)
if parsed:
return parsed, "content_frontmatter"
for key in ("remote_created_at", "created_at", "remote_updated_at", "updated_at"):
parsed = _parse_date_value(record.get(key), today=now)
if parsed:
return parsed, key
return None, None
def _reference_identity(ref: dict[str, Any]) -> str:
return str(
ref.get("document_id")
or ref.get("knowledge_id")
or ref.get("source_id")
or ref.get("url")
or ref.get("title")
or ref.get("name")
or ""
).strip()
async def enrich_chunk_references(
pages: list[dict[str, Any]],
mappings: list[dict[str, Any]],
*,
client: Any | None,
max_chunks: int = 500,
) -> tuple[list[dict[str, Any]], list[str]]:
"""Resolve Wiki provenance chunk ids to source-document metadata.
This is not a second raw-document search: only chunks already cited by a
matched Wiki page are resolved, and every chunk is rechecked against the
authorized remote knowledge-base id.
"""
if client is None:
return pages, []
mapping_by_id = {str(item.get("id") or ""): item for item in mappings}
pending: list[tuple[int, str, str]] = []
copied = [{**page, "source_refs": list(page.get("source_refs") or [])} for page in pages]
for page_index, page in enumerate(copied):
mapping = mapping_by_id.get(str(page.get("knowledge_base_mapping_id") or ""))
if mapping is None:
continue
existing_ids = {
str(item.get("chunk_id") or item.get("id") or "")
for item in page["source_refs"]
if isinstance(item, dict)
}
raw_refs = [*page["source_refs"], *(page.get("chunk_refs") or [])]
for ref in raw_refs:
chunk_id = str(ref).strip() if not isinstance(ref, dict) else str(ref.get("chunk_id") or ref.get("id") or "").strip()
if chunk_id and chunk_id not in existing_ids:
existing_ids.add(chunk_id)
pending.append((page_index, str(mapping.get("weknora_id") or ""), chunk_id))
warnings: list[str] = []
if len(pending) > max_chunks:
warnings.append(f"source_reference_limit:{max_chunks}/{len(pending)}")
pending = pending[:max_chunks]
semaphore = asyncio.Semaphore(12)
async def resolve(expected_base_id: str, chunk_id: str) -> dict[str, Any] | None:
async with semaphore:
try:
chunk = await client.get_chunk(chunk_id)
except Exception:
return None
if expected_base_id and str(chunk.get("knowledge_base_id") or "") != expected_base_id:
return None
metadata = chunk.get("metadata") if isinstance(chunk.get("metadata"), dict) else {}
return {
"id": chunk_id,
"chunk_id": chunk_id,
"document_id": chunk.get("knowledge_id") or chunk.get("document_id"),
"knowledge_id": chunk.get("knowledge_id") or chunk.get("document_id"),
"title": chunk.get("knowledge_title") or chunk.get("title") or chunk.get("file_name") or chunk.get("filename"),
"content": chunk.get("content") or "",
"metadata": metadata,
"url": metadata.get("url") or metadata.get("source_url"),
"published_at": metadata.get("published_at") or metadata.get("publish_time") or metadata.get("date"),
}
resolved = await asyncio.gather(*(resolve(base_id, chunk_id) for _, base_id, chunk_id in pending))
for (page_index, _, _), item in zip(pending, resolved, strict=True):
if item is not None:
copied[page_index]["source_refs"].append(item)
return copied, warnings
def _page_reference(page: dict[str, Any], mapping_name: str) -> dict[str, Any]:
published, date_source = extract_record_date(page)
return {
"id": str(page.get("id") or page.get("slug") or ""),
"title": str(page.get("title") or page.get("slug") or "未命名 Wiki 页面"),
"published_at": published.isoformat() if published else None,
"date_source": date_source,
"knowledge_base_id": str(page.get("knowledge_base_mapping_id") or ""),
"knowledge_base_name": mapping_name,
"wiki_page_id": page.get("id"),
"wiki_slug": page.get("slug"),
"summary": str(page.get("summary") or ""),
"source_refs": page.get("source_refs") or [],
"url": f"/weknora-source-preview?knowledge_base_id={page.get('knowledge_base_mapping_id') or ''}&wiki_slug={page.get('slug') or ''}",
}
def build_statistics(
pages: list[dict[str, Any]],
mappings: list[dict[str, Any]],
*,
subject_terms: list[str],
date_start: date | None,
date_end: date | None,
requested: bool = False,
) -> dict[str, Any]:
mapping_names = {str(item.get("id") or ""): str(item.get("name") or "") for item in mappings}
direct_pages = [page for page in pages if not subject_terms or any(term.casefold() in _page_haystack(page) for term in subject_terms)]
source_items: list[dict[str, Any]] = []
for page in direct_pages:
for ref in page.get("source_refs") or []:
if not isinstance(ref, dict) or not _reference_identity(ref):
continue
published, date_source = extract_record_date(ref)
source_items.append(
{
"id": _reference_identity(ref),
"title": str(ref.get("title") or ref.get("name") or page.get("title") or "未命名来源"),
"published_at": published.isoformat() if published else None,
"date_source": date_source,
"url": ref.get("url") or ref.get("source_url"),
"document_id": ref.get("document_id") or ref.get("knowledge_id"),
"chunk_id": ref.get("chunk_id") or ref.get("id"),
"knowledge_base_id": page.get("knowledge_base_mapping_id"),
"knowledge_base_name": mapping_names.get(str(page.get("knowledge_base_mapping_id") or ""), ""),
"wiki_page_id": page.get("id"),
"wiki_slug": page.get("slug"),
"wiki_page_title": page.get("title"),
}
)
unit = "source_document" if source_items else "wiki_page"
raw_items = source_items or [
_page_reference(page, mapping_names.get(str(page.get("knowledge_base_mapping_id") or ""), ""))
for page in direct_pages
]
unique: dict[str, dict[str, Any]] = {}
for item in raw_items:
unique.setdefault(str(item.get("id") or f"{item.get('knowledge_base_id')}:{item.get('wiki_slug')}"), item)
undated = 0
matched: list[dict[str, Any]] = []
for item in unique.values():
published = _parse_date_value(item.get("published_at"), today=datetime.now(UTC).date())
if (date_start or date_end) and published is None:
undated += 1
continue
if date_start and published and published < date_start:
continue
if date_end and published and published > date_end:
continue
matched.append(item)
matched.sort(key=lambda item: (str(item.get("published_at") or ""), str(item.get("title") or "")), reverse=True)
return {
"requested": bool(requested or date_start or date_end),
"unit": unit,
"unit_label": "新闻/来源" if unit == "source_document" else "Wiki 页面",
"subject_terms": subject_terms,
"date_from": date_start.isoformat() if date_start else None,
"date_to": date_end.isoformat() if date_end else None,
"matched_count": len(matched),
"undated_excluded_count": undated,
"items": matched,
}
def _merge_results(
channels: list[tuple[str, list[dict[str, Any]]]],
*,
limit: int,
title_query: str | None = None,
title_boost_enabled: bool = True,
title_boost_weight: float = 0.25,
) -> list[dict[str, Any]]:
fused: dict[tuple[str, str], dict[str, Any]] = {}
scores: dict[tuple[str, str], float] = {}
matched_queries: dict[tuple[str, str], list[str]] = {}
title_scores: dict[tuple[str, str], float] = {}
for channel_index, (query, items) in enumerate(channels):
weight = 2.0 if channel_index == 0 else 1.0
for rank, item in enumerate(items, start=1):
key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
scores[key] = scores.get(key, 0.0) + weight / (60 + rank)
title_scores[key] = max(
title_scores.get(key, 0.0),
title_match_strength(
title_query if title_query is not None else query,
item.get("title"),
item.get("aliases") or [],
),
)
matched_queries.setdefault(key, []).append(query)
current = fused.get(key)
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
fused[key] = dict(item)
ranking_scores = {
key: title_boosted_rrf_score(
scores[key],
title_scores[key],
enabled=title_boost_enabled,
weight=title_boost_weight,
)
for key in fused
}
ordered = sorted(fused, key=lambda key: ranking_scores[key], reverse=True)[:limit]
return [
{
**fused[key],
"rrf_score": scores[key],
"title_ranking_score": ranking_scores[key],
"score": scores[key],
"title_match_score": title_scores[key],
"rank_score_semantics": "rrf",
"matched_queries": list(dict.fromkeys(matched_queries[key])),
}
for key in ordered
]
async def advanced_search(
*,
query: str,
mappings: list[dict[str, Any]],
retrieval_service: Any,
index_store: Any,
top_k: int = 20,
date_from: str | None = None,
date_to: str | None = None,
expand_entities: bool | None = None,
include_drafts: bool = False,
) -> dict[str, Any]:
normalized_query = query.strip()
settings = resolved_retrieval_settings(mappings[0]) if len(mappings) == 1 else {
"entity_expansion_enabled": True,
"max_related_terms": 6,
"statistics_enabled": True,
}
expansion_enabled = settings["entity_expansion_enabled"] if expand_entities is None else bool(expand_entities)
subjects = extract_subject_terms(normalized_query)
date_start, date_end = parse_date_range(normalized_query, date_from=date_from, date_to=date_to)
mapping_ids = [str(item["id"]) for item in mappings]
lexical_terms = subjects or [item for item in re.findall(r"[A-Za-z0-9_.-]{2,}|[\u4e00-\u9fff·]{2,20}", normalized_query) if item not in _QUERY_STOP_WORDS][:4]
seed_pages = await index_store.search_pages_by_keywords(mapping_ids, lexical_terms, limit=2000, include_drafts=include_drafts)
seed_pages = [item for item in seed_pages if is_wiki_body_page(item)]
related = (
expand_related_terms(subjects or lexical_terms, seed_pages, limit=settings["max_related_terms"])
if expansion_enabled and not is_wiki_overview_query(normalized_query)
else []
)
queries = [normalized_query]
anchor = " ".join(subjects)
queries.extend(f"{anchor} {item['term']}".strip() for item in related)
searches = await asyncio.gather(
*(retrieval_service.search(item, mappings, top_k_pages=max(top_k, 20), include_drafts=include_drafts) for item in queries),
return_exceptions=True,
)
channels: list[tuple[str, list[dict[str, Any]]]] = []
warnings: list[str] = []
modes: list[str] = []
diagnostics: list[dict[str, Any]] = []
for item_query, payload in zip(queries, searches, strict=True):
if isinstance(payload, Exception):
warnings.append(f"{item_query}:{payload}")
continue
channels.append((item_query, list(payload.get("results") or [])))
modes.append(str(payload.get("mode") or ""))
warnings.extend(str(item) for item in payload.get("warnings") or [])
diagnostics.append({"query": item_query, **dict(payload.get("diagnostics") or {})})
resolve_title_settings = getattr(retrieval_service, "_title_boost_settings", None)
if callable(resolve_title_settings):
title_boost_enabled, title_boost_weight = resolve_title_settings()
else:
vector_config = getattr(getattr(retrieval_service, "vector_search", None), "config", None)
title_boost_enabled = bool(getattr(vector_config, "title_boost_enabled", True))
title_boost_weight = float(getattr(vector_config, "title_boost_weight", 0.25))
results = _merge_results(
channels,
limit=max(1, min(top_k, 100)),
# Entity expansion queries add related terms for recall. They must
# not turn a related entity's title into a direct match for the
# user's original question (for example “特朗普” should not outrank
# a page titled “海湖庄园” when the user asked for the latter).
title_query=normalized_query,
title_boost_enabled=title_boost_enabled,
title_boost_weight=title_boost_weight,
)
enriched_pages, provenance_warnings = await enrich_chunk_references(
seed_pages,
mappings,
client=getattr(retrieval_service, "client", None),
)
warnings.extend(provenance_warnings)
stats = build_statistics(
enriched_pages,
mappings,
subject_terms=subjects or lexical_terms,
date_start=date_start,
date_end=date_end,
requested=bool(_STATISTICS_PATTERN.search(normalized_query)),
) if settings["statistics_enabled"] else None
references = [
{
"reference_id": f"wiki:{item.get('knowledge_base_id')}:{item.get('wiki_slug') or item.get('id')}",
"title": item.get("title"),
"knowledge_base_id": item.get("knowledge_base_id"),
"knowledge_base_name": item.get("knowledge_base_name"),
"wiki_page_id": item.get("wiki_page_id") or item.get("id"),
"wiki_slug": item.get("wiki_slug"),
"url": item.get("url"),
"snippet": item.get("snippet") or str(item.get("content") or "")[:360],
}
for item in results
]
return {
"query": normalized_query,
"mode": "entity_expanded" if related else (modes[0] if modes else "unavailable"),
"knowledge_base_count": len(mappings),
"knowledge_base_profiles": [
{"id": item.get("id"), "name": item.get("name"), "description": item.get("description") or ""}
for item in mappings
],
"query_plan": {
"subject_terms": subjects,
"date_from": date_start.isoformat() if date_start else None,
"date_to": date_end.isoformat() if date_end else None,
"entity_expansion_enabled": expansion_enabled,
"expanded_terms": related,
"executed_queries": queries,
},
"statistics": stats,
"results": results,
"references": references,
"partial": bool(warnings),
"warnings": warnings,
"diagnostics": diagnostics,
}
__all__ = [
"advanced_search",
"build_statistics",
"expand_related_terms",
"extract_record_date",
"extract_subject_terms",
"enrich_chunk_references",
"parse_date_range",
]
@@ -57,6 +57,9 @@ class StrictWikiEmbeddingClient:
self._local_started_at: str | None = None
self._local_ready_at: str | None = None
self._local_warmup_task: asyncio.Task[None] | None = None
self._lifecycle_condition = asyncio.Condition()
self._active_uses = 0
self._retired = False
@property
def dimensions(self) -> int | None:
@@ -64,7 +67,21 @@ class StrictWikiEmbeddingClient:
@property
def fingerprint(self) -> str:
raw = "\0".join((self.config.provider.strip().lower(), self.config.base_url.rstrip("/").lower(), self.config.model.strip(), str(self._resolved_dimensions or "auto")))
parts = [
self.config.provider.strip().lower(),
self.config.base_url.rstrip("/").lower(),
self.config.model.strip(),
str(self._resolved_dimensions or "auto"),
]
if self.config.provider == "local":
model_path = self._local_model_path()
parts.extend(
(
str(model_path) if model_path is not None else "",
self.config.local_model_revision,
)
)
raw = "\0".join(parts)
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
async def close(self) -> None:
@@ -73,6 +90,15 @@ class StrictWikiEmbeddingClient:
await self._client.aclose()
self._client = None
async def retire(self) -> None:
"""Reject future work, drain active calls, then release model resources."""
async with self._lifecycle_condition:
self._retired = True
async with self._lifecycle_condition:
await self._lifecycle_condition.wait_for(lambda: self._active_uses == 0)
await self.close()
async def reset_local_encoder(self, *, model_path: str | None = None) -> None:
"""Stop only the embedded worker and optionally switch its model path.
@@ -82,27 +108,80 @@ class StrictWikiEmbeddingClient:
"""
warmup = self._local_warmup_task
if warmup is not None and not warmup.done():
warmup.cancel()
# Local inference runs in a worker thread. Cancelling the asyncio
# wrapper does not stop that thread and could let it publish a new
# process/model after cleanup. Drain it before touching resources.
await asyncio.gather(warmup, return_exceptions=True)
self._local_warmup_task = None
process = self._local_process
self._local_process = None
if process is not None:
await asyncio.to_thread(self._stop_local_process, process)
self._local_model = None
self._local_state = "idle"
self._local_error = None
self._local_started_at = None
self._local_ready_at = None
if model_path is not None:
self.config = self.config.model_copy(update={"local_model_path": model_path})
# Wait for a non-warmup embedding batch to finish before replacing or
# stopping the local worker. This prevents an admin upload from
# breaking a concurrent vectorization request halfway through IPC.
async with self._local_lock:
process = self._local_process
self._local_process = None
if process is not None:
await self._run_thread_drained(self._stop_local_process, process)
self._local_model = None
self._local_state = "idle"
self._local_error = None
self._local_started_at = None
self._local_ready_at = None
if model_path is not None:
self.config = self.config.model_copy(update={"local_model_path": model_path})
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
return self._client
@staticmethod
async def _run_thread_drained(function, *args):
"""Do not release lifecycle locks while cancelled thread work still runs."""
worker = asyncio.create_task(asyncio.to_thread(function, *args))
cancelled = False
worker_error: BaseException | None = None
result = None
while not worker.done():
try:
result = await asyncio.shield(worker)
except asyncio.CancelledError:
# A caller can be cancelled more than once during shutdown.
# Swallow every cancellation until the non-cancellable thread
# has really stopped, then restore cancellation semantics.
cancelled = True
except BaseException as exc:
worker_error = exc
break
if worker.done() and worker_error is None:
try:
result = worker.result()
except BaseException as exc:
worker_error = exc
if cancelled:
raise asyncio.CancelledError from None
if worker_error is not None:
raise worker_error
return result
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
async with self._lifecycle_condition:
if self._retired:
raise WikiEmbeddingError(
"Wiki embedding runtime was replaced; retry with the active provider",
category="runtime_retired",
retryable=True,
)
self._active_uses += 1
try:
return await self._embed_texts_active(texts)
finally:
async with self._lifecycle_condition:
self._active_uses -= 1
if self._active_uses == 0:
self._lifecycle_condition.notify_all()
async def _embed_texts_active(self, texts: list[str]) -> list[list[float]]:
if self.config.provider == "local":
if not texts:
return []
@@ -110,7 +189,10 @@ class StrictWikiEmbeddingClient:
self._local_state = "starting"
self._local_started_at = self._local_started_at or datetime.now(UTC).isoformat()
try:
values = await asyncio.to_thread(self._embed_local, texts)
# `to_thread` keeps running after its awaiting task is
# cancelled. Keep the lifecycle lease and local lock until
# the real inference call has stopped.
values = await self._run_thread_drained(self._embed_local, texts)
except Exception as exc:
self._local_state = "failed"
self._local_error = " ".join(str(exc).split())[:1000] or type(exc).__name__
@@ -452,6 +534,8 @@ class StrictWikiEmbeddingClient:
def start_local_encoder(self) -> dict[str, object]:
"""Warm the embedded CPU encoder without coupling it to app startup."""
if self._retired:
raise ValueError("当前本地编码器运行时已切换,请刷新状态后重试")
if self.config.provider != "local":
raise ValueError("当前 embedding provider 不是 local")
if not self.config.enabled:
@@ -9,6 +9,23 @@ from datetime import datetime
from typing import Any
from urllib.parse import unquote
_WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#[^\]|]+)?(?:\|[^\]]+)?\]\]")
_LINK_REFERENCE_KEYS = (
"slug",
"wiki_slug",
"path",
"target",
"source",
"target_slug",
"source_slug",
"to",
"from",
"page_slug",
"id",
"title",
"name",
)
def normalize_slug(value: Any) -> str:
slug = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
@@ -44,6 +61,44 @@ def _datetime(value: Any) -> datetime | None:
return None
def normalize_wiki_link_reference(value: Any) -> str | None:
"""Normalize a provider link endpoint without assuming it is a slug yet."""
if isinstance(value, dict):
value = next(
(value.get(key) for key in _LINK_REFERENCE_KEYS if value.get(key) is not None and str(value.get(key)).strip()),
None,
)
raw = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
if not raw:
return None
raw = raw.split("#", 1)[0].strip().strip("/")
raw = re.sub(r"/{2,}", "/", raw)
return raw or None
def normalize_wiki_page_links(remote: dict[str, Any]) -> dict[str, list[str]]:
"""Return deterministic incoming/outgoing references from a full page.
WeKnora versions have returned both snake/camel-case fields and either
strings or small endpoint objects. Markdown Wiki links supplement explicit
``out_links`` so a complete page scan remains authoritative even when the
provider's overview graph is capped.
"""
def _references(value: Any) -> set[str]:
return {reference for item in _list(value) for reference in [normalize_wiki_link_reference(item)] if reference}
incoming = _references(remote.get("in_links")) | _references(remote.get("inLinks"))
outgoing = _references(remote.get("out_links")) | _references(remote.get("outLinks"))
content = str(remote.get("content_md") or remote.get("content") or remote.get("markdown") or "")
outgoing.update(reference for match in _WIKILINK_RE.finditer(content) for reference in [normalize_wiki_link_reference(match.group(1))] if reference)
return {
"in_links": sorted(incoming, key=str.casefold),
"out_links": sorted(outgoing, key=str.casefold),
}
def normalize_wiki_page(mapping_id: str, remote: dict[str, Any]) -> dict[str, Any]:
slug = normalize_slug(remote.get("slug") or remote.get("wiki_slug") or remote.get("path"))
aliases = sorted({str(item).strip() for item in _list(remote.get("aliases")) if str(item).strip()})
@@ -0,0 +1,58 @@
"""Shared intent and page eligibility rules for Wiki-library overviews."""
from __future__ import annotations
import re
from typing import Any
_OVERVIEW_PATTERNS = (
re.compile(
r"(?:(?:这个|这座|此|该|本)?(?:知识库|知识空间|资料库|wiki)|(?:这个|此|该|本)库).{0,12}"
r"(?:讲了?什么|有什么(?:内容|资料|主题)?|有哪些(?:内容|资料|主题)?|包含什么|"
r"内容?(?:概览|介绍)|(?:总体|整体)?(?:介绍|概览)|简介)",
re.IGNORECASE,
),
re.compile(r"^(?:概览|知识库概览|知识空间概览|库概览|介绍(?:一下)?(?:这个|此|该)?(?:知识库|知识空间|库)?)$", re.IGNORECASE),
re.compile(
r"(?:what(?:'s| is| does).{0,24}(?:knowledge base|wiki)|"
r"(?:knowledge base|wiki).{0,24}(?:overview|contents?|cover|about)|"
r"overview of (?:this |the )?(?:knowledge base|wiki))",
re.IGNORECASE,
),
)
_NON_BODY_PAGE_TYPES = frozenset({"directory", "index", "summary"})
_NON_BODY_SLUG_SEGMENTS = frozenset({"directory", "index", "summary", "目录", "索引", "摘要"})
_NON_BODY_TITLES = frozenset({"directory", "index", "summary", "目录", "索引", "摘要"})
def is_wiki_overview_query(query: str) -> bool:
"""Whether a request asks for a library-level orientation, not a fact lookup."""
normalized = " ".join(query.strip().split())
return bool(normalized) and any(pattern.search(normalized) for pattern in _OVERVIEW_PATTERNS)
def is_wiki_body_page(page: dict[str, Any]) -> bool:
"""Reject generated directory/index/summary shells from retrieval candidates."""
page_type = str(page.get("page_type") or page.get("type") or "article").strip().lower()
if page_type in _NON_BODY_PAGE_TYPES:
return False
slug = str(page.get("slug") or page.get("wiki_slug") or page.get("path") or "").strip("/ ").lower()
if any(segment in _NON_BODY_SLUG_SEGMENTS for segment in slug.split("/")):
return False
title = str(page.get("title") or "").strip().lower()
return title not in _NON_BODY_TITLES
def wiki_body_excerpt(page: dict[str, Any], *, max_chars: int = 1600) -> str:
"""Return a bounded real-body preview for an overview response."""
content = str(page.get("content_md") or page.get("content") or "").strip()
if len(content) <= max_chars:
return content
return f"{content[: max_chars - 1].rstrip()}…"
__all__ = ["is_wiki_body_page", "is_wiki_overview_query", "wiki_body_excerpt"]
@@ -0,0 +1,212 @@
"""Small, deterministic ranking helpers shared by Wiki retrieval channels.
The local Wiki index has several independent ranking paths (dense vectors,
keyword fallback, remote Wiki pages and raw document chunks). Keeping the
title signal in one helper makes it possible for the final channel fusion to
apply the same bias everywhere. This is intentionally not a tokenizer or an
LLM: it only detects a bounded, meaningful phrase from the user's query.
"""
from __future__ import annotations
import re
import unicodedata
from collections.abc import Callable, Iterable
# Natural-language wrappers which should not make a page title look relevant.
# The list is deliberately conservative; unknown words are retained as query
# terms rather than silently discarded.
_QUERY_NOISE = (
"知识空间",
"知识库",
"相关资料",
"相关文档",
"相关报告",
"相关信息",
"相关新闻",
"告诉我",
"我想知道",
"请问",
"请帮我",
"帮我",
"搜索",
"查询",
"查找",
"检索",
"查阅",
"介绍一下",
"介绍",
"解释一下",
"解释",
"有关",
"相关",
"请",
)
_QUERY_SUFFIXES = ("是什么", "是什麽", "怎么样", "如何", "怎么", "吗", "呢")
_ASCII_STOP_WORDS = {
"a",
"an",
"and",
"are",
"about",
"can",
"could",
"for",
"how",
"is",
"the",
"what",
"where",
"which",
"who",
"with",
}
def _compact(value: object) -> str:
"""Normalize text for phrase containment while retaining CJK/Latin chars."""
normalized = unicodedata.normalize("NFKC", str(value or "")).casefold()
# ``\w`` includes Chinese characters in Python's Unicode mode. Removing
# punctuation and whitespace makes ``Mar-a-Lago`` match ``maralago`` while
# leaving ordinary Chinese title phrases intact.
return re.sub(r"[^\w\u3400-\u9fff]+", "", normalized)
def _query_terms(query: str) -> list[str]:
"""Return bounded title-oriented phrases from a natural-language query."""
raw = unicodedata.normalize("NFKC", str(query or "")).casefold().strip()
if not raw:
return []
terms: list[str] = []
def add(value: str) -> None:
compact = _compact(value)
if len(compact) < 2 or compact in _ASCII_STOP_WORDS or compact in terms:
return
# Two-character CJK fragments are useful as a fallback, but we avoid
# treating a lone one-character fragment as a title match.
terms.append(compact)
# Quoted phrases are the strongest user signal.
for value in re.findall(r"[「『\"']([^」』\"']{2,60})[」』\"']", raw):
add(value)
cleaned = raw
for noise in sorted(_QUERY_NOISE, key=len, reverse=True):
cleaned = cleaned.replace(noise, " ")
# A possessive/conjunctive ``的`` often separates the subject from the
# question (``海湖庄园的历史``). Keep both the complete run and each side;
# do not delete it globally because it can be part of a real title.
for suffix in _QUERY_SUFFIXES:
cleaned = cleaned.replace(suffix, " ")
cleaned = re.sub(r"[,。;;?!?::,、()()\[\]{}<>]", " ", cleaned)
for token in re.findall(r"[a-z0-9][a-z0-9_.:/+\-]*", cleaned):
add(token)
for run in re.findall(r"[\u3400-\u4dbf\u4e00-\u9fff]+", cleaned):
add(run)
parts = [part for part in re.split(r"的|与|和|或|及|并", run) if part]
for part in parts:
add(part)
# Keep a small number of long contiguous phrases. The longest match
# wins later, so this handles a phrase embedded in a longer question
# without giving every two-character n-gram a large score.
if len(run) > 8:
for width in (8, 6, 4, 3):
for start in range(0, len(run) - width + 1):
add(run[start : start + width])
# Longer phrases are evaluated first and cap work for pathological input.
return sorted(terms, key=lambda value: (-len(value), value))[:24]
def _title_match_strength_from_terms(
terms: list[str],
title: object,
aliases: Iterable[object] | None = None,
) -> float:
title_value = _compact(title)
if not title_value:
return 0.0
if not terms:
return 0.0
aliases_compact = [_compact(item) for item in aliases or () if _compact(item)]
best = 0.0
for term in terms:
if len(term) < 3 and not re.fullmatch(r"[a-z0-9][a-z0-9_.:/+\-]*", term):
continue
for candidate, multiplier in ((title_value, 1.0), *[(item, 0.86) for item in aliases_compact]):
if not candidate:
continue
if term == candidate:
ratio = 1.0
elif term in candidate:
# The user supplied a concise title phrase while the stored
# title carries a qualifier, such as ``海湖庄园(美国)``.
ratio = min(1.0, len(term) / max(len(candidate), 1))
elif candidate in term and len(candidate) >= 3:
# Natural-language questions often keep the title inside a
# longer run (``海湖庄园最新消息``). Treat that as a title hit
# without requiring a fragile list of every question suffix.
ratio = min(1.0, len(candidate) / max(len(term), 1))
else:
continue
value = (0.55 + 0.45 * ratio) * multiplier
best = max(best, value)
return min(1.0, best)
def make_title_matcher(
query: str,
) -> Callable[[object, Iterable[object] | None], float]:
"""Compile query phrases once for repeated title checks in large indexes."""
terms = _query_terms(query)
def match(title: object, aliases: Iterable[object] | None = None) -> float:
return _title_match_strength_from_terms(terms, title, aliases)
return match
def title_match_strength(
query: str,
title: object,
aliases: Iterable[object] | None = None,
) -> float:
"""Return a bounded title relevance score in ``[0, 1]``.
``1`` means the most useful case for retrieval: the extracted query phrase
is exactly the title (or a long phrase is fully contained by the title).
A phrase occurring in an alias is intentionally discounted. The function
is cheap enough to run while fusing already-bounded result lists.
"""
return make_title_matcher(query)(title, aliases)
def title_boosted_rrf_score(
rrf_score: float,
title_score: float,
*,
enabled: bool = True,
weight: float = 0.25,
) -> float:
"""Apply a bounded multiplicative title boost to an RRF score.
A multiplier keeps the signal in the RRF score domain: an exact title hit
receives a 25% lift at the default weight, but it does not unconditionally
beat an item supported by substantially more retrieval channels.
"""
base = float(rrf_score)
if not enabled:
return base
strength = min(1.0, max(0.0, float(title_score)))
safe_weight = max(0.0, float(weight))
return base * (1.0 + safe_weight * strength)
__all__ = ["make_title_matcher", "title_boosted_rrf_score", "title_match_strength"]
@@ -8,31 +8,221 @@ Wiki pages even while a whole knowledge base is awaiting vectorization.
from __future__ import annotations
import asyncio
import re
from typing import Any
from urllib.parse import urlencode
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.overview import is_wiki_body_page, is_wiki_overview_query, wiki_body_excerpt
from deerflow.integrations.weknora.local_index.ranking import (
title_boosted_rrf_score,
title_match_strength,
)
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
_GENERIC_QUERY_TERMS = {
"知识库",
"知识空间",
"这个",
"此知识库",
"内容",
"哪些",
"什么",
"怎么",
"如何",
"请问",
"一下",
}
# The keyword channel is deliberately a small, database-portable fallback and
# currently uses substring predicates rather than a language-aware FTS index.
# Keep function words out of those predicates: otherwise an unrelated page
# containing e.g. ``is`` or ``on`` inside an English fragment is enough to
# survive a failed vector threshold and become the final answer source.
_ENGLISH_STOP_WORDS = {
"an",
"and",
"are",
"as",
"at",
"be",
"been",
"but",
"by",
"can",
"could",
"did",
"do",
"does",
"for",
"from",
"had",
"has",
"have",
"how",
"if",
"in",
"into",
"is",
"it",
"its",
"may",
"might",
"of",
"on",
"or",
"our",
"should",
"than",
"that",
"the",
"their",
"then",
"there",
"these",
"they",
"this",
"those",
"to",
"was",
"were",
"what",
"when",
"where",
"which",
"who",
"why",
"will",
"with",
"would",
"you",
"your",
}
_ASCII_KEYWORD = re.compile(r"[a-z0-9_.-]+")
def _keyword_tokens(query: str) -> list[str]:
"""Build bounded Latin terms and CJK sub-phrases for lexical recall."""
tokens: list[str] = []
for part in re.findall(r"[a-zA-Z0-9_.-]{2,}|[\u4e00-\u9fff]{2,}", query.lower()):
if re.fullmatch(r"[\u4e00-\u9fff]+", part):
for generic in sorted(_GENERIC_QUERY_TERMS, key=len, reverse=True):
part = part.replace(generic, "")
if part and part not in _GENERIC_QUERY_TERMS and part not in _ENGLISH_STOP_WORDS:
tokens.append(part)
if part and re.fullmatch(r"[\u4e00-\u9fff]+", part) and len(part) > 3:
for width in (3, 2):
tokens.extend(part[index : index + width] for index in range(len(part) - width + 1))
return list(
dict.fromkeys(
token
for token in tokens
if token not in _GENERIC_QUERY_TERMS and token not in _ENGLISH_STOP_WORDS
)
)[:32]
def _contains_keyword(haystack: str, token: str) -> bool:
"""Match Latin/code tokens on word boundaries and CJK tokens by substring."""
if _ASCII_KEYWORD.fullmatch(token):
# ``\b`` is not suitable here because underscores count as word
# characters and codes such as ``ZXQ-417`` contain punctuation. The
# explicit alphanumeric boundaries retain standalone two-letter terms
# (for example AI) and exact equipment codes without matching ``is``
# inside ``this`` or ``on`` inside ``controller``.
return re.search(rf"(?<![a-z0-9]){re.escape(token)}(?![a-z0-9])", haystack) is not None
return token in haystack
def _minimum_keyword_hits(tokens: list[str]) -> int:
"""Require corroboration for long, purely Latin natural-language queries."""
# A precise equipment/code token remains sufficient even when the caller
# also includes natural-language words (for example ``ZXQ-417 reset
# procedure``). Requiring two matches in that case would remove the most
# useful lexical fallback when vectors are unavailable or below threshold.
if any(any(char.isdigit() for char in token) or any(char in "_.-" for char in token) for token in tokens):
return 1
if len(tokens) >= 3 and all(_ASCII_KEYWORD.fullmatch(token) for token in tokens):
return 2
return 1
def _rrf_merge(
channels: list[list[dict[str, Any]]],
*,
limit: int,
title_boost_enabled: bool = True,
title_boost_weight: float = 0.25,
) -> list[dict[str, Any]]:
"""Fuse independently ranked channels without comparing raw score scales."""
fused: dict[tuple[str, str], dict[str, Any]] = {}
contributions: dict[tuple[str, str], float] = {}
modes: dict[tuple[str, str], set[str]] = {}
title_scores: dict[tuple[str, str], float] = {}
for channel in channels:
for rank, item in enumerate(channel, start=1):
key = (
str(item.get("knowledge_base_id") or ""),
str(item.get("wiki_slug") or item.get("id") or ""),
)
contributions[key] = contributions.get(key, 0.0) + 1.0 / (60 + rank)
title_scores[key] = max(title_scores.get(key, 0.0), float(item.get("title_match_score") or 0))
modes.setdefault(key, set()).add(str(item.get("retrieval_mode") or "vector"))
current = fused.get(key)
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
fused[key] = dict(item)
ranking_scores = {
key: title_boosted_rrf_score(
contributions[key],
title_scores[key],
enabled=title_boost_enabled,
weight=title_boost_weight,
)
for key in fused
}
ranked = sorted(fused, key=lambda key: ranking_scores[key], reverse=True)
results = []
for key in ranked[:limit]:
item = fused[key]
item["channel_score"] = float(item.get("score") or 0)
item["title_match_score"] = title_scores[key]
item["rrf_score"] = contributions[key]
item["title_ranking_score"] = ranking_scores[key]
item["score"] = contributions[key]
item["rank_score_semantics"] = "rrf"
item["retrieval_mode"] = "hybrid" if len(modes[key]) > 1 else next(iter(modes[key]))
results.append(item)
return results
def is_fully_vectorized(state: dict[str, Any] | None, fingerprint: str | None) -> bool:
"""Return whether a completed full scan produced one coherent active index."""
"""Return whether a coherent last-successful vector snapshot can serve.
A new incremental attempt may be running, cancelled or failed while every
page from the last successful snapshot remains readable. Do not take that
snapshot offline merely because the latest attempt did not finish.
"""
if not state or not fingerprint:
return False
if str(state.get("state") or "") != "idle" or not state.get("last_success_at"):
if str(state.get("state") or "") in {"rebuild_required", "rebuilding", "not_synced"} or not state.get("last_success_at"):
return False
if str(state.get("embedding_fingerprint") or "") != fingerprint:
return False
local_count = int(state.get("local_page_count") or 0)
ready_count = int(state.get("ready_page_count") or 0)
failed_count = int(state.get("failed_page_count") or 0)
vector_count = int(state.get("vector_count") or 0)
if failed_count or ready_count != local_count:
return False
return local_count == 0 or vector_count > 0
# ``failed_page_count`` includes a retained ready page with a newer failed
# attempt because ``last_error`` is diagnostic. Its atomically committed
# older vectors remain a valid serving snapshot. New/unindexed pages are
# still covered by the lexical Wiki channel.
return ready_count > 0 and vector_count > 0
class WikiRetrievalService:
@@ -44,10 +234,25 @@ class WikiRetrievalService:
*,
vector_search: WikiVectorSearchService | None,
client: WeKnoraClient | None,
title_boost_enabled: bool | None = None,
title_boost_weight: float | None = None,
) -> None:
self.store = store
self.vector_search = vector_search
self.client = client
self._title_boost_enabled = title_boost_enabled
self._title_boost_weight = title_boost_weight
def _title_boost_settings(self) -> tuple[bool, float]:
if self._title_boost_enabled is not None or self._title_boost_weight is not None:
return (
True if self._title_boost_enabled is None else bool(self._title_boost_enabled),
0.25 if self._title_boost_weight is None else float(self._title_boost_weight),
)
config = getattr(self.vector_search, "config", None)
if config is None:
return True, 0.25
return bool(config.title_boost_enabled), float(config.title_boost_weight)
async def search(
self,
@@ -71,11 +276,15 @@ class WikiRetrievalService:
"fallback_knowledge_base_ids": [],
}
overview = is_wiki_overview_query(normalized_query)
vector_mappings, fallback_mappings = await self._partition_mappings(mappings)
results: list[dict[str, Any]] = []
vector_results: list[dict[str, Any]] = []
keyword_results: list[dict[str, Any]] = []
remote_results: list[dict[str, Any]] = []
warnings: list[str] = []
vector_used = False
fallback_used = False
vector_diagnostics: dict[str, Any] = {}
if vector_mappings and self.vector_search is not None:
try:
@@ -85,8 +294,9 @@ class WikiRetrievalService:
top_k_pages=max(top_k_pages, len(vector_mappings)),
include_drafts=include_drafts,
)
results.extend(payload.get("results") or [])
vector_results.extend(payload.get("results") or [])
warnings.extend(str(item) for item in payload.get("warnings") or [])
vector_diagnostics = dict(payload.get("diagnostics") or {})
vector_used = True
except WikiIndexError as exc:
# A status row can become stale between classification and
@@ -95,6 +305,14 @@ class WikiRetrievalService:
fallback_mappings.extend(vector_mappings)
vector_mappings = []
if not overview:
keyword_results = await self._search_local_keywords(
normalized_query,
mappings,
limit=max(top_k_pages * 4, 20),
include_drafts=include_drafts,
)
fallback_ids = [str(mapping["id"]) for mapping in fallback_mappings]
if fallback_mappings:
if self.client is None:
@@ -107,6 +325,7 @@ class WikiRetrievalService:
mapping,
limit=max(top_k_pages, 1),
include_drafts=include_drafts,
overview=overview,
)
for mapping in fallback_mappings
),
@@ -116,53 +335,162 @@ class WikiRetrievalService:
if isinstance(batch, Exception):
warnings.append(f"{mapping['id']}:WIKI_API_SEARCH_FAILED")
else:
results.extend(batch)
remote_results.extend(batch)
fallback_used = True
if warnings and not results and not vector_used and not fallback_used:
if warnings and not vector_results and not keyword_results and not remote_results and not vector_used and not fallback_used:
raise WikiIndexError(
"WIKI_API_UNAVAILABLE",
"Wiki page retrieval is unavailable",
status_code=502 if self.client is not None else 503,
)
# De-duplicate by the public Wiki citation identity. Vector scores and
# Wiki-API rank scores are both descending relevance hints.
unique: dict[tuple[str, str], dict[str, Any]] = {}
for item in results:
key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
current = unique.get(key)
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
unique[key] = item
ranked = sorted(unique.values(), key=lambda item: float(item.get("score") or 0), reverse=True)
mode = "mixed" if vector_used and fallback_used else "vector" if vector_used else "wiki_api"
title_boost_enabled, title_boost_weight = self._title_boost_settings()
ranked = _rrf_merge(
[channel for channel in (vector_results, keyword_results, remote_results) if channel],
limit=top_k_pages,
title_boost_enabled=title_boost_enabled,
title_boost_weight=title_boost_weight,
)
if overview:
mode = "overview"
elif vector_results and keyword_results:
mode = "hybrid"
elif keyword_results:
mode = "keyword"
elif vector_used and fallback_used:
mode = "mixed"
elif vector_used:
mode = "vector"
else:
mode = "wiki_api"
fallback_reason = None
if keyword_results and not vector_results:
raw_top = vector_diagnostics.get("top_raw_vector_score")
fallback_reason = "VECTOR_BELOW_THRESHOLD" if raw_top is not None else "VECTOR_UNAVAILABLE"
fingerprint = self.vector_search.embedding.fingerprint[:12] if vector_used and self.vector_search is not None else None
vector_ids = {str(mapping["id"]) for mapping in vector_mappings}
vector_states = {
str(state.get("knowledge_base_mapping_id") or ""): str(state.get("state") or "")
for state in await self.store.list_index_status(list(vector_ids))
}
stale_serving_ids = sorted(
mapping_id
for mapping_id in vector_ids
if vector_states.get(mapping_id) not in {None, "", "idle"}
)
return {
"query": normalized_query,
"mode": mode,
"results": ranked[:top_k_pages],
"results": ranked,
"knowledge_base_count": len(mappings),
"index_fingerprint": fingerprint,
"partial": bool(warnings),
"warnings": warnings,
"fallback_knowledge_base_ids": fallback_ids,
"diagnostics": {
**vector_diagnostics,
"vector_candidates": len(vector_results),
"keyword_candidates": len(keyword_results),
"wiki_api_candidates": len(remote_results),
"fallback_reason": fallback_reason,
"stale_serving_knowledge_base_ids": stale_serving_ids,
},
}
async def _search_local_keywords(
self,
query: str,
mappings: list[dict[str, Any]],
*,
limit: int,
include_drafts: bool,
) -> list[dict[str, Any]]:
tokens = _keyword_tokens(query)
if not tokens:
return []
search_pages = getattr(self.store, "search_pages_by_keywords", None)
if search_pages is None:
return []
pages = await search_pages(
[str(mapping["id"]) for mapping in mappings],
tokens,
limit=limit,
include_drafts=include_drafts,
)
mapping_by_id = {str(mapping["id"]): mapping for mapping in mappings}
title_boost_enabled, title_boost_weight = self._title_boost_settings()
title_hit_weight = 3.0 + (20.0 * title_boost_weight if title_boost_enabled else 0.0)
results: list[dict[str, Any]] = []
for page in pages:
if not is_wiki_body_page(page):
continue
content = str(page.get("content_md") or "").strip()
if not content:
continue
mapping_id = str(page.get("knowledge_base_mapping_id") or "")
mapping = mapping_by_id.get(mapping_id)
if mapping is None:
continue
title = str(page.get("title") or page.get("slug") or "")
aliases = " ".join(str(item) for item in page.get("aliases") or [])
title_haystack = f"{title} {aliases}".lower()
summary_haystack = str(page.get("summary") or "").lower()
content_haystack = content.lower()
title_matches = {token for token in tokens if _contains_keyword(title_haystack, token)}
summary_matches = {token for token in tokens if _contains_keyword(summary_haystack, token)}
content_matches = {token for token in tokens if _contains_keyword(content_haystack, token)}
if len(title_matches | summary_matches | content_matches) < _minimum_keyword_hits(tokens):
continue
title_hits = len(title_matches)
summary_hits = len(summary_matches)
content_hits = len(content_matches)
score = (title_hit_weight * title_hits + 2 * summary_hits + content_hits) / max(len(tokens), 1)
title_match_score = title_match_strength(query, title, page.get("aliases") or [])
params = urlencode({"knowledge_base_id": mapping_id, "wiki_slug": page.get("slug") or ""})
results.append(
{
"kind": "wiki_page",
"id": page.get("id"),
"knowledge_base_id": mapping_id,
"knowledge_base_name": mapping.get("name") or "",
"wiki_page_id": page.get("id"),
"wiki_slug": page.get("slug") or "",
"title": title,
"summary": page.get("summary") or "",
"page_type": page.get("page_type") or "article",
"page_version": int(page.get("page_version") or 0),
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
"score": float(score),
"title_match_score": float(title_match_score),
"content": content[:4000],
"snippet": content[:360],
"matched_heading": None,
"matched_sections": [
{
"section_index": 0,
"heading": None,
"content": content[:4000],
"score": float(score),
}
],
"url": f"/weknora-source-preview?{params}",
"source": mapping.get("name") or "LLMWiki",
"type": "llmwiki_wiki",
"mode": "citation",
"retrieval_mode": "keyword",
}
)
results.sort(key=lambda item: float(item.get("score") or 0), reverse=True)
return results[:limit]
async def _partition_mappings(self, mappings: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
if self.vector_search is None:
return [], list(mappings)
mapping_ids = [str(mapping["id"]) for mapping in mappings]
states = {
str(state["knowledge_base_mapping_id"]): state
for state in await self.store.list_index_status(mapping_ids)
}
states = {str(state["knowledge_base_mapping_id"]): state for state in await self.store.list_index_status(mapping_ids)}
fingerprint = self.vector_search.embedding.fingerprint
vector_mappings = [
mapping
for mapping in mappings
if mapping.get("wiki_index_enabled", True)
and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)
]
vector_mappings = [mapping for mapping in mappings if mapping.get("wiki_index_enabled", True) and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)]
ready_ids = {str(mapping["id"]) for mapping in vector_mappings}
return vector_mappings, [mapping for mapping in mappings if str(mapping["id"]) not in ready_ids]
@@ -173,6 +501,7 @@ class WikiRetrievalService:
*,
limit: int,
include_drafts: bool,
overview: bool,
) -> list[dict[str, Any]]:
if self.client is None:
return []
@@ -181,21 +510,21 @@ class WikiRetrievalService:
remote_id,
page=1,
page_size=min(max(limit * 3, 20), 100),
query=query,
# An overview is not a text match. Asking the remote service to
# match its wording first is exactly what made “this library
# covers what?” return zero while a mirror was still indexing.
query="" if overview else query,
)
summaries = [item for item in listing.get("pages", []) if isinstance(item, dict)]
slugs = list(
dict.fromkeys(
str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
for item in summaries
if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
)
)[: max(limit * 2, limit)]
summaries = [item for item in listing.get("pages", []) if isinstance(item, dict) and is_wiki_body_page(item)]
slugs = list(dict.fromkeys(str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/") for item in summaries if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")))[
: limit if overview else max(limit * 2, limit)
]
details = await asyncio.gather(
*(self.client.get_wiki_page(remote_id, slug) for slug in slugs),
return_exceptions=True,
)
results: list[dict[str, Any]] = []
title_boost_enabled, title_boost_weight = self._title_boost_settings()
for rank, (slug, detail) in enumerate(zip(slugs, details, strict=True)):
if isinstance(detail, Exception):
continue
@@ -205,9 +534,12 @@ class WikiRetrievalService:
continue
if page["status"] == "archived" or (page["status"] == "draft" and not include_drafts):
continue
content = wiki_body_excerpt(page) if overview else str(page.get("content_md") or "").strip()
if not is_wiki_body_page(page) or not content:
continue
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": slug})
content = str(page.get("content_md") or "")
score = float(detail.get("score") or max(0.01, 1.0 - rank * 0.05))
title_match_score = title_match_strength(query, page.get("title") or slug, page.get("aliases") or [])
results.append(
{
"kind": "wiki_page",
@@ -222,6 +554,7 @@ class WikiRetrievalService:
"page_version": int(page.get("page_version") or 0),
"updated_at": page.get("remote_updated_at"),
"score": score,
"title_match_score": float(title_match_score),
"content": content,
"snippet": content[:360],
"matched_heading": None,
@@ -237,12 +570,20 @@ class WikiRetrievalService:
"source": mapping.get("name") or "LLMWiki",
"type": "llmwiki_wiki",
"mode": "citation",
"retrieval_mode": "wiki_api",
"retrieval_mode": "overview" if overview else "wiki_api",
}
)
if len(results) >= limit:
break
return results
if not overview:
results.sort(
key=lambda item: float(item.get("score") or 0)
+ (
title_boost_weight * float(item.get("title_match_score") or 0)
if title_boost_enabled
else 0.0
),
reverse=True,
)
return results[:limit]
__all__ = ["WikiRetrievalService", "is_fully_vectorized"]
@@ -14,9 +14,16 @@ import numpy as np
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
from deerflow.integrations.weknora.local_index.overview import is_wiki_body_page, is_wiki_overview_query, wiki_body_excerpt
from deerflow.integrations.weknora.local_index.ranking import (
make_title_matcher,
title_match_strength,
)
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache, WikiVectorSnapshot
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
_TITLE_RECALL_MIN_STRENGTH = 0.7
class WikiIndexError(RuntimeError):
def __init__(self, code: str, message: str, *, status_code: int) -> None:
@@ -46,7 +53,15 @@ class WikiVectorSearchService:
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
if not mappings:
return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
fingerprint, candidates, warnings = await self._collect_candidates(
if is_wiki_overview_query(normalized_query):
return await self._search_overview(
normalized_query,
mappings,
top_k_pages=top_k_pages,
include_drafts=include_drafts,
external=external,
)
fingerprint, candidates, warnings, diagnostics = await self._collect_candidates(
normalized_query,
mappings,
include_drafts=include_drafts,
@@ -80,6 +95,10 @@ class WikiVectorSearchService:
"page_version": int(page.get("page_version") or 0),
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
"score": float(page_score),
"vector_score": float(first.get("vector_score") or 0),
# Preserve the lexical signal through page grouping. A
# direct title hit ranks before broad semantic neighbours.
"title_match_score": float(first.get("title_match_score") or 0),
"content": "\n\n".join(str(item["section_content"]) for item in sections),
"snippet": str(sections[0]["section_content"])[:360],
"matched_heading": sections[0].get("heading"),
@@ -90,9 +109,21 @@ class WikiVectorSearchService:
"mode": "citation",
}
)
results.sort(key=lambda item: item["score"], reverse=True)
results.sort(
key=lambda item: float(item["score"]),
reverse=True,
)
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
return {
"query": normalized_query,
"mode": "vector",
"results": results[:limit],
"knowledge_base_count": len(mappings),
"index_fingerprint": fingerprint[:12],
"partial": bool(warnings),
"warnings": warnings,
"diagnostics": diagnostics,
}
async def search_sections(
self,
@@ -125,7 +156,7 @@ class WikiVectorSearchService:
"partial": False,
"warnings": [],
}
fingerprint, candidates, warnings = await self._collect_candidates(
fingerprint, candidates, warnings, diagnostics = await self._collect_candidates(
normalized_query,
mappings,
include_drafts=False,
@@ -146,6 +177,8 @@ class WikiVectorSearchService:
"content": str(candidate.get("section_content") or ""),
"section_content_hash": candidate.get("content_hash") or "",
"score": float(candidate["score"]),
"vector_score": float(candidate.get("vector_score") or 0),
"title_match_score": float(candidate.get("title_match_score") or 0),
"knowledge_base_id": mapping["id"],
"weknora_knowledge_base_id": mapping.get("weknora_id") or "",
"knowledge_base_name": mapping.get("name") or "",
@@ -182,8 +215,127 @@ class WikiVectorSearchService:
"index_fingerprint": fingerprint[:12],
"partial": bool(warnings),
"warnings": warnings,
"diagnostics": diagnostics,
}
async def _search_overview(
self,
query: str,
mappings: list[dict[str, Any]],
*,
top_k_pages: int | None,
include_drafts: bool,
external: bool,
) -> dict[str, Any]:
"""Return representative page bodies without embedding a broad library question.
A query such as “此知识库都讲了什么” is an orientation request rather
than a semantic similarity request. Selecting actual body pages keeps
it useful even when every section scores below ``min_similarity``.
"""
fingerprint = self.embedding.fingerprint
batches = await asyncio.gather(
*(
self._overview_mapping(
mapping,
fingerprint,
include_drafts=include_drafts,
external=external,
)
for mapping in mappings
),
return_exceptions=True,
)
candidates: list[dict[str, Any]] = []
warnings: list[str] = []
for mapping, batch in zip(mappings, batches, strict=True):
if isinstance(batch, Exception):
warnings.append(f"{mapping.get('id')}: {batch}")
else:
candidates.extend(batch)
if warnings and not candidates and len(warnings) == len(mappings):
if any("FINGERPRINT" in item for item in warnings):
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
elif any("rebuild" in item.lower() for item in warnings):
code = "WIKI_INDEX_REBUILDING"
else:
code = "WIKI_INDEX_NOT_READY"
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
results: list[dict[str, Any]] = []
for rank, candidate in enumerate(candidates):
page = candidate["page"]
mapping = candidate["mapping"]
content = wiki_body_excerpt(page)
score = max(0.01, 1.0 - rank * 0.05)
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": page["slug"]})
results.append(
{
"kind": "wiki_page",
"id": page["id"],
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"wiki_page_id": page["id"],
"wiki_slug": page["slug"],
"title": page.get("title") or page["slug"],
"summary": page.get("summary") or "",
"page_type": page.get("page_type") or "article",
"page_version": int(page.get("page_version") or 0),
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
"score": score,
"content": content,
"snippet": content[:360],
"matched_heading": None,
"matched_sections": [{"section_index": 0, "heading": None, "content": content, "score": score}],
"url": f"/weknora-source-preview?{params}",
"source": mapping.get("name") or "LLMWiki",
"type": "llmwiki_wiki",
"mode": "citation",
"retrieval_mode": "overview",
}
)
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
return {
"query": query,
"mode": "overview",
"results": results[:limit],
"knowledge_base_count": len(mappings),
"index_fingerprint": fingerprint[:12],
"partial": bool(warnings),
"warnings": warnings,
}
async def _overview_mapping(
self,
mapping: dict[str, Any],
fingerprint: str,
*,
include_drafts: bool,
external: bool,
) -> list[dict[str, Any]]:
mapping_id = str(mapping["id"])
try:
snapshot = await self.cache.get(mapping_id, fingerprint)
except ValueError as exc:
await self.cache.store.mark_rebuild_required([mapping_id], fingerprint)
self.cache.invalidate(mapping_id)
raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
seen_pages: set[str] = set()
candidates: list[dict[str, Any]] = []
for row in snapshot.rows:
page = row["page"]
page_id = str(page.get("id") or "")
if not page_id or page_id in seen_pages:
continue
seen_pages.add(page_id)
if page.get("status") == "draft" and not (include_drafts and not external):
continue
if not is_wiki_body_page(page) or not str(page.get("content_md") or "").strip():
continue
candidates.append({"page": page, "mapping": mapping})
return candidates
async def _collect_candidates(
self,
query: str,
@@ -192,7 +344,7 @@ class WikiVectorSearchService:
include_drafts: bool,
external: bool,
section_limit: int | None = None,
) -> tuple[str, list[dict[str, Any]], list[str]]:
) -> tuple[str, list[dict[str, Any]], list[str], dict[str, Any]]:
try:
query_vector = await self._embed_query(query)
except (WikiEmbeddingError, ValueError) as exc:
@@ -213,11 +365,17 @@ class WikiVectorSearchService:
batches = await asyncio.gather(*tasks, return_exceptions=True)
candidates: list[dict[str, Any]] = []
warnings: list[str] = []
raw_scores: list[float] = []
threshold_filtered = 0
for mapping, batch in zip(mappings, batches, strict=True):
if isinstance(batch, Exception):
warnings.append(f"{mapping.get('id')}: {batch}")
else:
candidates.extend(batch)
mapping_candidates, raw_top_score, mapping_filtered = batch
candidates.extend(mapping_candidates)
threshold_filtered += mapping_filtered
if raw_top_score is not None:
raw_scores.append(raw_top_score)
if warnings and not candidates and len(warnings) == len(mappings):
if any("FINGERPRINT" in item for item in warnings):
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
@@ -226,8 +384,20 @@ class WikiVectorSearchService:
else:
code = "WIKI_INDEX_NOT_READY"
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
candidates.sort(key=lambda item: item["score"], reverse=True)
return fingerprint, candidates, warnings
candidates.sort(
key=lambda item: float(item["score"]),
reverse=True,
)
return (
fingerprint,
candidates,
warnings,
{
"top_raw_vector_score": max(raw_scores) if raw_scores else None,
"min_similarity": self.config.min_similarity,
"threshold_filtered": threshold_filtered,
},
)
async def _embed_query(self, query: str) -> np.ndarray:
digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
@@ -254,7 +424,7 @@ class WikiVectorSearchService:
include_drafts: bool,
external: bool,
section_limit: int | None = None,
) -> list[dict[str, Any]]:
) -> tuple[list[dict[str, Any]], float | None, int]:
mapping_id = str(mapping["id"])
try:
snapshot = await self.cache.get(mapping_id, fingerprint)
@@ -263,33 +433,45 @@ class WikiVectorSearchService:
self.cache.invalidate(mapping_id)
raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
if snapshot.matrix.size == 0:
return []
return [], None, 0
if snapshot.dimensions != vector.size:
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
eligible = np.asarray(
[
index
for index, row in enumerate(snapshot.rows)
if row["page"].get("status") != "draft" or (include_drafts and not external)
],
[index for index, row in enumerate(snapshot.rows) if row["page"].get("status") != "draft" or (include_drafts and not external)],
dtype=int,
)
if eligible.size == 0:
return []
return [], None, 0
indices, scores = await asyncio.to_thread(
self._top_indices,
snapshot,
vector,
eligible,
query,
section_limit,
)
terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
candidates = []
raw_top_score = float(scores[0]) if scores.size else None
threshold_filtered = int(np.count_nonzero(scores < self.config.min_similarity))
for index, score in zip(indices, scores, strict=True):
row = snapshot.rows[int(index)]
page = row["page"]
vector_score = float(score)
if vector_score < self.config.min_similarity:
title_boost_active = (
self.config.title_boost_enabled
and self.config.title_boost_weight > 0
)
title_score = (
title_match_strength(query, page.get("title"), page.get("aliases") or [])
if title_boost_active
else 0.0
)
# An explicit title/alias hit is itself a high-confidence lexical
# recall signal. Keep strong hits even when their dense vector is
# below the semantic threshold; keyword/hybrid callers then see
# the same title-first result as the direct vector endpoint.
if vector_score < self.config.min_similarity and title_score < _TITLE_RECALL_MIN_STRENGTH:
continue
final_score = vector_score
if self.config.keyword_boost_enabled and terms:
@@ -297,20 +479,92 @@ class WikiVectorSearchService:
keyword_score = sum(term in haystack for term in terms) / len(terms)
weight = self.config.keyword_boost_weight
final_score = (1 - weight) * vector_score + weight * keyword_score
candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
return candidates
if title_score:
# Cosine scores and lexical scores are both roughly bounded by
# one. A modest additive title boost keeps dense retrieval
# useful while ensuring an exact title is not lost to a broad
# semantic neighbour (for example “特朗普” vs “海湖庄园”).
final_score += self.config.title_boost_weight * title_score
candidates.append(
{
**row,
"page": page,
"mapping": mapping,
"score": final_score,
"vector_score": vector_score,
"title_match_score": title_score,
}
)
requested_limit = section_limit or self.config.top_k_sections
candidates.sort(
key=lambda item: float(item["score"]),
reverse=True,
)
if section_limit is None:
# Page search only needs a bounded number of sections from each
# page. Apply the cap before truncation so one exact-title page
# cannot consume every candidate slot and hide useful backfill.
per_page: dict[str, int] = defaultdict(int)
diversified: list[dict[str, Any]] = []
for candidate in candidates:
page_key = str(candidate["page"].get("id") or candidate["page"].get("slug") or "")
if per_page[page_key] >= self.config.max_sections_per_page:
continue
per_page[page_key] += 1
diversified.append(candidate)
if len(diversified) >= requested_limit:
break
candidates = diversified
return candidates[:requested_limit], raw_top_score, threshold_filtered
def _top_indices(
self,
snapshot: WikiVectorSnapshot,
vector: np.ndarray,
eligible: np.ndarray,
query: str,
section_limit: int | None = None,
) -> tuple[np.ndarray, np.ndarray]:
scores = snapshot.matrix[eligible] @ vector
count = min(section_limit or self.config.top_k_sections, scores.size)
if count <= 0:
return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
positions = np.argpartition(scores, -count)[-count:]
# Page grouping happens after section retrieval, so keep a wider dense
# pool for page searches. This lets the per-page cap backfill from
# other pages instead of returning fewer than ``top_k_sections``.
pool_count = count if section_limit is not None else min(scores.size, max(count * 4, count + 32))
positions = np.argpartition(scores, -pool_count)[-pool_count:]
if self.config.title_boost_enabled and self.config.title_boost_weight > 0:
title_positions: list[tuple[float, float, int]] = []
per_page: dict[str, int] = defaultdict(int)
title_scores_by_page: dict[str, float] = {}
match_title = make_title_matcher(query)
for local_position, row_index in enumerate(eligible):
page = snapshot.rows[int(row_index)]["page"]
page_key = str(page.get("id") or page.get("slug") or row_index)
if page_key not in title_scores_by_page:
title_scores_by_page[page_key] = match_title(
page.get("title"),
page.get("aliases") or [],
)
strength = title_scores_by_page[page_key]
if strength >= _TITLE_RECALL_MIN_STRENGTH:
title_positions.append((strength, float(scores[local_position]), local_position))
title_positions.sort(reverse=True)
extra_limit = max(count, self.config.top_k_pages * self.config.max_sections_per_page)
extras: list[int] = []
for _strength, _score, local_position in title_positions:
page = snapshot.rows[int(eligible[local_position])]["page"]
page_key = str(page.get("id") or page.get("slug") or "")
if per_page[page_key] >= self.config.max_sections_per_page:
continue
per_page[page_key] += 1
extras.append(local_position)
if len(extras) >= extra_limit:
break
if extras:
positions = np.asarray(list(dict.fromkeys([*positions.tolist(), *extras])), dtype=int)
positions = positions[np.argsort(scores[positions])[::-1]]
return eligible[positions], scores[positions]
@@ -5,20 +5,36 @@ from __future__ import annotations
import asyncio
import logging
import platform
from datetime import UTC, datetime
from math import ceil
from time import monotonic
from typing import Any
from uuid import uuid4
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
from deerflow.integrations.weknora.local_index.normalize import (
normalize_wiki_link_reference,
normalize_wiki_page,
normalize_wiki_page_links,
)
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
logger = logging.getLogger(__name__)
# This breaker is intentionally process-local. The durable sync row already
# records the failed run and error, while a transient retry deadline does not
# belong in release files or administrator configuration. Rebuilding the
# runtime after a provider/config change (or restarting Gateway) resets the
# breaker and therefore allows one immediate recovery probe. The state is a
# fixed-size provider record rather than a per-page/per-mapping collection.
_EMBEDDING_PROVIDER_BACKOFF_INITIAL_SECONDS = 60.0
_EMBEDDING_PROVIDER_BACKOFF_MAX_SECONDS = 30.0 * 60.0
class WikiSyncLeaseLost(RuntimeError):
pass
@@ -28,6 +44,215 @@ def _safe_error(exc: BaseException) -> str:
return f"{type(exc).__name__}: {str(exc)[:1000]}"
def _iso_now() -> str:
return datetime.now(UTC).isoformat()
def _same_instant(left: Any, right: Any) -> bool:
"""Compare provider/database timestamps without depending on one timezone."""
def _parse(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value
raw = str(value or "").strip()
if not raw:
return None
try:
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
except ValueError:
return None
left_dt = _parse(left)
right_dt = _parse(right)
if left_dt is None or right_dt is None:
return left == right
if left_dt.tzinfo is None or right_dt.tzinfo is None:
return left_dt.replace(tzinfo=None) == right_dt.replace(tzinfo=None)
return left_dt.astimezone(UTC) == right_dt.astimezone(UTC)
def _summary_has_page_body(summary: dict[str, Any]) -> bool:
"""Whether a WeKnora list item already carries the complete Wiki body."""
return any(key in summary and summary.get(key) is not None for key in ("content_md", "content", "markdown"))
def _knowledge_base_is_processing(remote: dict[str, Any]) -> bool:
raw_flag = remote.get("is_processing", remote.get("isProcessing"))
if isinstance(raw_flag, str):
is_processing = raw_flag.strip().lower() in {"1", "true", "yes", "on"}
else:
is_processing = bool(raw_flag)
raw_count = remote.get("processing_count", remote.get("processingCount"))
try:
processing_count = int(raw_count or 0)
except (TypeError, ValueError):
processing_count = 0
return is_processing or processing_count > 0
def _summary_matches_existing(
mapping_id: str,
summary: dict[str, Any],
existing: dict[str, Any],
) -> bool:
"""Trust only explicit revision markers when skipping the detail request.
Older WeKnora releases omitted both markers. Those responses deliberately
fall back to ``get_wiki_page`` so an unreported edit can never be missed.
Explicit metadata included in the summary is also compared, catching a
provider that failed to bump its version for a directory/status change.
"""
try:
normalized = normalize_wiki_page(mapping_id, summary)
except (TypeError, ValueError):
return False
has_marker = False
raw_version = summary.get("page_version", summary.get("version"))
if raw_version not in (None, ""):
try:
version = int(raw_version)
except (TypeError, ValueError):
return False
if version > 0:
has_marker = True
if int(existing.get("page_version") or 0) != version:
return False
raw_updated_at = summary.get("updated_at", summary.get("updatedAt"))
if raw_updated_at not in (None, ""):
updated_at = normalized.get("remote_updated_at")
if updated_at is None:
return False
has_marker = True
if not _same_instant(existing.get("remote_updated_at"), updated_at):
return False
if not has_marker:
return False
explicit_fields: tuple[tuple[tuple[str, ...], str], ...] = (
(("id", "page_id"), "remote_page_id"),
(("title",), "title"),
(("summary", "description"), "summary"),
(("page_type", "type"), "page_type"),
(("status",), "status"),
(("aliases",), "aliases"),
(("category_path", "categories"), "category_path"),
(("parent_slug",), "parent_slug"),
(("wiki_path",), "wiki_path"),
(("source_refs", "sources"), "source_refs"),
(("chunk_refs", "chunks"), "chunk_refs"),
)
for source_keys, local_key in explicit_fields:
if any(key in summary for key in source_keys) and existing.get(local_key) != normalized.get(local_key):
return False
if _summary_has_page_body(summary) and existing.get("content_hash") != normalized.get("content_hash"):
return False
return True
def _existing_snapshot_reusable(existing: dict[str, Any] | None, fingerprint: str) -> bool:
if not existing:
return False
has_searchable_body = bool(existing.get("content_md") or existing.get("summary"))
return bool(
existing.get("indexed_fingerprint") == fingerprint
and existing.get("index_status") == "ready"
and not existing.get("last_error")
and not existing.get("is_remote_deleted")
and (
int(existing.get("vector_count") or 0) > 0
or existing.get("status") == "archived"
or not has_searchable_body
)
)
def _build_graph_edges(
page_records: list[dict[str, Any]],
provider_graph: dict[str, Any] | None,
) -> list[dict[str, str]]:
"""Resolve page/link aliases to mapping-local canonical slugs."""
identities: dict[str, set[str]] = {}
canonical_slugs: dict[str, str] = {}
def _remember(reference: Any, slug: str) -> None:
normalized = normalize_wiki_link_reference(reference)
if normalized:
identities.setdefault(normalized.casefold(), set()).add(slug)
for record in page_records:
page = record["page"]
slug = str(page["slug"])
canonical_slugs[slug] = slug
for value in (
slug,
page.get("title"),
page.get("remote_page_id"),
*(page.get("aliases") or []),
):
_remember(value, slug)
graph = provider_graph if isinstance(provider_graph, dict) else {}
for node in graph.get("nodes") or []:
if not isinstance(node, dict):
continue
canonical_reference = node.get("slug") or node.get("wiki_slug") or node.get("path")
normalized = normalize_wiki_link_reference(canonical_reference)
if normalized in canonical_slugs:
canonical_slug = canonical_slugs[normalized]
else:
candidates = identities.get(normalized.casefold(), set()) if normalized else set()
if len(candidates) != 1:
continue
canonical_slug = next(iter(candidates))
for value in (
node.get("id"),
node.get("slug"),
node.get("wiki_slug"),
node.get("path"),
node.get("title"),
node.get("name"),
):
_remember(value, canonical_slug)
def _resolve(reference: Any) -> str | None:
normalized = normalize_wiki_link_reference(reference)
if not normalized:
return None
if normalized in canonical_slugs:
return canonical_slugs[normalized]
candidates = identities.get(normalized.casefold(), set())
return next(iter(candidates)) if len(candidates) == 1 else None
pairs: set[tuple[str, str]] = set()
for record in page_records:
page = record["page"]
links = record["links"]
slug = str(page["slug"])
for target_reference in links.get("out_links") or []:
target = _resolve(target_reference)
if target and target != slug:
pairs.add((slug, target))
for source_reference in links.get("in_links") or []:
source = _resolve(source_reference)
if source and source != slug:
pairs.add((source, slug))
for edge in graph.get("edges") or []:
if not isinstance(edge, dict):
continue
source = _resolve(edge.get("source") or edge.get("source_id"))
target = _resolve(edge.get("target") or edge.get("target_id"))
if source and target and source != target:
pairs.add((source, target))
return [{"source_slug": source, "target_slug": target, "edge_type": "wiki_link"} for source, target in sorted(pairs)]
class WikiSyncService:
def __init__(self, config: LocalWikiIndexConfig, store: LlmWikiIndexStore, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache, client: WeKnoraClient) -> None:
self.config = config
@@ -37,14 +262,188 @@ class WikiSyncService:
self.client = client
self.owner = f"{platform.node()}:{uuid4()}"
self.on_completed = None
self._embedding_probe_lock = asyncio.Lock()
self._embedding_failure_count = 0
self._embedding_retry_at = 0.0
self._embedding_last_error: str | None = None
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False, notify: bool = True) -> dict[str, Any]:
def _embedding_backoff_result(
self,
mapping_id: str,
*,
now: float | None = None,
) -> dict[str, Any] | None:
checked_at = monotonic() if now is None else now
remaining = self._embedding_retry_at - checked_at
if self._embedding_failure_count <= 0 or remaining <= 0:
return None
return {
"mapping_id": mapping_id,
"status": "provider_backoff",
"failure_scope": "embedding_provider",
"failure_count": self._embedding_failure_count,
"retry_after_seconds": max(1, ceil(remaining)),
"error": self._embedding_last_error,
}
def _record_embedding_failure(self, exc: WikiEmbeddingError) -> int:
"""Open or extend one provider-wide exponential breaker window.
Several mappings may already be running under the scheduler semaphore
when the provider goes away. Failures arriving inside an already-open
window share that window instead of multiplying the exponent.
"""
now = monotonic()
if self._embedding_retry_at <= now:
self._embedding_failure_count = min(
self._embedding_failure_count + 1,
31,
)
exponent = min(self._embedding_failure_count - 1, 30)
delay = min(
_EMBEDDING_PROVIDER_BACKOFF_MAX_SECONDS,
_EMBEDDING_PROVIDER_BACKOFF_INITIAL_SECONDS * (2**exponent),
)
self._embedding_retry_at = now + delay
self._embedding_last_error = _safe_error(exc)
return max(1, ceil(self._embedding_retry_at - now))
def _reset_embedding_breaker(self) -> None:
self._embedding_failure_count = 0
self._embedding_retry_at = 0.0
self._embedding_last_error = None
async def _embedding_preflight(
self,
mapping_id: str,
*,
force: bool,
) -> dict[str, Any] | None:
"""Skip a known outage or perform one serialized recovery probe."""
if not force:
backoff = self._embedding_backoff_result(mapping_id)
if backoff is not None:
return backoff
async with self._embedding_probe_lock:
if not force:
backoff = self._embedding_backoff_result(mapping_id)
if backoff is not None:
return backoff
needs_probe = (
self.embedding.dimensions is None
or self._embedding_failure_count > 0
)
if not needs_probe:
return None
try:
# Auto-dimension configurations need one successful response
# for a stable fingerprint. After an outage the same tiny
# request proves provider recovery before any Wiki directory
# or page is enumerated.
await self.embedding.embed_texts(
["DeerFlow local Wiki embedding provider health probe"]
)
except WikiEmbeddingError as exc:
if exc.category != "runtime_retired":
retry_after = self._record_embedding_failure(exc)
logger.warning(
"Local Wiki embedding provider probe failed mapping_id=%s retry_after_seconds=%s: %s",
mapping_id,
retry_after,
_safe_error(exc),
)
raise
self._reset_embedding_breaker()
return None
async def _persist_requested_backoff(
self,
mapping_id: str,
*,
sync_id: str,
mode: str,
started_at: str,
backoff: dict[str, Any],
) -> dict[str, Any]:
"""Finish an explicitly requested job with a frontend-compatible state."""
states = await self.store.list_index_status([mapping_id])
prior = states[0] if states else {}
remote_count = int(prior.get("remote_page_count") or 0)
acquired = await self.store.try_acquire_sync_lease(
mapping_id,
owner=self.owner,
sync_id=sync_id,
lease_seconds=self.config.sync_lease_seconds,
fingerprint=self.embedding.fingerprint,
)
if not acquired:
return backoff
error = str(
backoff.get("error")
or "Wiki embedding provider is temporarily unavailable"
)
result = {
"sync_id": sync_id,
"mapping_id": mapping_id,
"mode": mode,
"status": "failed",
"remote_pages": remote_count,
"added_pages": 0,
"updated_pages": 0,
"metadata_updated_pages": 0,
"revectorized_pages": 0,
"changed_pages": 0,
"skipped_pages": 0,
"deleted_pages": 0,
"failed_pages": 0,
"started_at": started_at,
"completed_at": _iso_now(),
"error": error,
"failure_scope": backoff.get("failure_scope")
or "embedding_provider",
"failure_count": int(backoff.get("failure_count") or 0),
"retry_after_seconds": int(
backoff.get("retry_after_seconds") or 0
),
}
persisted = await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=False,
remote_page_count=remote_count,
error=error,
result=result,
)
return result if persisted else backoff
async def sync_mapping(
self,
mapping: dict[str, Any],
*,
force: bool = False,
notify: bool = True,
sync_id: str | None = None,
) -> dict[str, Any]:
mapping_id = str(mapping["id"])
sync_id = str(uuid4())
if self.embedding.dimensions is None:
# Auto-dimension configurations need one successful response to
# produce a stable fingerprint before lease/rebuild decisions.
await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
requested_sync_id = sync_id
sync_id = sync_id or str(uuid4())
mode = "full" if force else "incremental"
started_at = _iso_now()
backoff = await self._embedding_preflight(mapping_id, force=force)
if backoff is not None:
if requested_sync_id is not None:
return await self._persist_requested_backoff(
mapping_id,
sync_id=sync_id,
mode=mode,
started_at=started_at,
backoff=backoff,
)
return backoff
fingerprint = self.embedding.fingerprint
revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
@@ -82,40 +481,145 @@ class WikiSyncService:
remote_count = 0
changed = 0
added = 0
updated = 0
metadata_updated = 0
skipped = 0
failed = 0
graph_snapshot_complete = True
graph_page_records: list[dict[str, Any]] = []
def _sync_result(
status: str,
*,
deleted_pages: int = 0,
error: str | None = None,
) -> dict[str, Any]:
result: dict[str, Any] = {
"sync_id": sync_id,
"mapping_id": mapping_id,
"mode": mode,
"status": status,
"remote_pages": remote_count,
"added_pages": added,
"updated_pages": updated,
"metadata_updated_pages": metadata_updated,
"revectorized_pages": changed,
"changed_pages": changed,
"skipped_pages": skipped,
"deleted_pages": deleted_pages,
"failed_pages": failed,
"started_at": started_at,
"completed_at": _iso_now(),
}
if error:
result["error"] = error
return result
try:
remote_id = str(mapping["weknora_id"])
await self.client.get_knowledge_base(remote_id)
initial_remote = await self.client.get_knowledge_base(remote_id)
processing_observed_during_scan = _knowledge_base_is_processing(
initial_remote
)
page_number = 1
while True:
batch = await self.client.list_wiki_pages(remote_id, page=page_number, page_size=self.config.sync_page_size)
pages = [page for page in batch.get("pages", []) if isinstance(page, dict)]
remote_count += len(pages)
reported_total = int(batch.get("total") or batch.get("total_count") or remote_count)
await self.store.update_sync_progress(
progress_updated = await self.store.update_sync_progress(
mapping_id,
owner=self.owner,
sync_id=sync_id,
remote_page_count=max(remote_count, reported_total),
)
if not progress_updated:
lease_lost.set()
raise WikiSyncLeaseLost("Local Wiki sync lease was lost while updating progress")
for summary in pages:
graph_page_captured = False
try:
_ensure_lease()
slug = str(summary.get("slug") or summary.get("wiki_slug") or summary.get("path") or "")
remote_page = await self.client.get_wiki_page(remote_id, slug)
page = normalize_wiki_page(mapping_id, {**summary, **remote_page})
existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
summary_page = normalize_wiki_page(mapping_id, summary)
existing = await self.store.get_page_by_slug_hash(
mapping_id,
summary_page["slug_hash"],
)
active_fingerprint = self.embedding.fingerprint
if (
not force
and existing
and existing.get("content_hash") == page["content_hash"]
and existing.get("indexed_fingerprint") == active_fingerprint
and existing.get("index_status") == "ready"
and (existing.get("vector_count", 1) > 0 or page["status"] == "archived" or not (page["content_md"] or page["summary"]))
and not existing.get("is_remote_deleted")
and not _summary_has_page_body(summary)
and _existing_snapshot_reusable(existing, active_fingerprint)
and _summary_matches_existing(mapping_id, summary, existing)
):
await self.store.touch_page(str(existing["id"]), sync_id)
touched = await self.store.touch_page(
str(existing["id"]),
sync_id,
mapping_id=mapping_id,
owner=self.owner,
)
if not touched:
lease_lost.set()
raise WikiSyncLeaseLost(
"Local Wiki sync lease was lost before touching an unchanged page"
)
graph_page_records.append(
{
"page": existing,
"links": normalize_wiki_page_links(existing),
}
)
# Old WeKnora list shapes do not expose explicit
# incoming/outgoing links. Markdown links from the
# local body are useful for lookup, but are not an
# authoritative whole-graph snapshot.
graph_snapshot_complete = False
graph_page_captured = True
skipped += 1
continue
if _summary_has_page_body(summary):
remote_payload = dict(summary)
else:
remote_page = await self.client.get_wiki_page(remote_id, slug)
remote_payload = {**summary, **remote_page}
page = normalize_wiki_page(mapping_id, remote_payload)
graph_page_records.append(
{
"page": page,
"links": normalize_wiki_page_links(remote_payload),
}
)
graph_page_captured = True
# A provider may canonicalize the slug in the detail
# response. Resolve the canonical row before deciding
# whether vectors can be reused.
if existing is None or existing.get("slug_hash") != page["slug_hash"]:
existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
active_fingerprint = self.embedding.fingerprint
if (
not force
and _existing_snapshot_reusable(existing, active_fingerprint)
and existing.get("content_hash") == page["content_hash"]
):
metadata_result = await self.store.refresh_page_metadata(
page=page,
sync_id=sync_id,
owner=self.owner,
)
if metadata_result is None:
lease_lost.set()
raise WikiSyncLeaseLost(
"Local Wiki sync lease was lost before refreshing page metadata"
)
if metadata_result.get("changed"):
metadata_updated += 1
updated += 1
self.cache.invalidate(mapping_id)
else:
skipped += 1
continue
sections = (
[]
@@ -131,16 +635,50 @@ class WikiSyncService:
{"section_index": section.section_index, "heading": section.heading, "section_content": section.content, "content_hash": section.content_hash, "embedding_dimensions": dimensions, "vector_blob": blob}
)
_ensure_lease()
await self.store.replace_page_vectors(page=page, vectors=vectors, fingerprint=fingerprint, sync_id=sync_id)
saved_page = await self.store.replace_page_vectors(
page=page,
vectors=vectors,
fingerprint=fingerprint,
sync_id=sync_id,
owner=self.owner,
)
if saved_page is None:
lease_lost.set()
raise WikiSyncLeaseLost(
"Local Wiki sync lease was lost before replacing page vectors"
)
self.cache.invalidate(mapping_id)
changed += 1
if existing is None:
added += 1
else:
updated += 1
except Exception as exc:
if isinstance(exc, WikiSyncLeaseLost):
raise
# Embedding failures describe the shared provider, not
# one malformed Wiki page. Continuing would repeat the
# client's own retry loop for every remaining page.
if isinstance(exc, WikiEmbeddingError):
raise
if not graph_page_captured:
graph_snapshot_complete = False
failed += 1
try:
fallback = normalize_wiki_page(mapping_id, summary)
await self.store.record_page_failure(page=fallback, sync_id=sync_id, error=_safe_error(exc))
failure_recorded = await self.store.record_page_failure(
page=fallback,
sync_id=sync_id,
error=_safe_error(exc),
owner=self.owner,
)
if not failure_recorded:
lease_lost.set()
raise WikiSyncLeaseLost(
"Local Wiki sync lease was lost before recording a page failure"
) from exc
except WikiSyncLeaseLost:
raise
except Exception as record_exc:
logger.exception("Could not record local Wiki page failure mapping_id=%s", mapping_id)
raise RuntimeError("Could not safely retain a failed Wiki page during the full scan") from record_exc
@@ -151,18 +689,138 @@ class WikiSyncService:
break
page_number += 1
_ensure_lease()
deleted = await self.store.mark_missing_pages_deleted(mapping_id, sync_id)
# WeKnora can enter (or still be in) asynchronous processing after
# the pre-scan scheduler check. A page directory observed during
# that window is not deletion-authoritative. Re-check immediately
# before the only destructive phase and preserve both missing
# pages and graph edges whenever the status is processing or
# cannot be verified. Persist ``partial`` so the periodic scanner
# necessarily retries even when the KB timestamp is unchanged.
deferred_reason: str | None = None
deferred_code: str | None = None
try:
current_remote = await self.client.get_knowledge_base(remote_id)
except Exception as exc:
deferred_code = "processing_status_unavailable"
deferred_reason = (
"Could not verify WeKnora processing status; deletion and graph replacement deferred: "
f"{_safe_error(exc)}"
)
else:
if processing_observed_during_scan or _knowledge_base_is_processing(
current_remote
):
deferred_code = "upstream_processing"
deferred_reason = (
"WeKnora is still processing; deletion and graph replacement deferred"
)
_ensure_lease()
if deferred_reason is not None:
terminal_error = (
f"{deferred_reason}; {failed} page(s) failed"
if failed
else deferred_reason
)
result = {
**_sync_result("partial", error=terminal_error),
"graph_edges": None,
"graph_changed": False,
"graph_preserved": True,
"deferred_deletion": True,
"deferred_reason": deferred_code,
}
persisted = await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=True,
remote_page_count=remote_count,
error=terminal_error,
result=result,
)
if not persisted:
lease_lost.set()
raise WikiSyncLeaseLost(
"Local Wiki sync lease was lost before persisting the deferred scan result"
)
return result
provider_graph: dict[str, Any] | None = None
graph_api_succeeded = False
get_wiki_graph = getattr(self.client, "get_wiki_graph", None)
if callable(get_wiki_graph):
try:
provider_graph = await get_wiki_graph(
remote_id,
limit=max(200, min(remote_count, 10000)),
)
graph_api_succeeded = isinstance(provider_graph, dict)
except Exception as exc:
logger.warning(
"Local Wiki graph endpoint unavailable mapping_id=%s: %s",
mapping_id,
_safe_error(exc),
)
_ensure_lease()
deleted = await self.store.mark_missing_pages_deleted(
mapping_id,
sync_id,
owner=self.owner,
)
if deleted is None:
lease_lost.set()
raise WikiSyncLeaseLost("Local Wiki sync lease was lost before retiring missing pages")
if deleted:
self.cache.invalidate(mapping_id)
await self.store.finish_sync(
graph_result: dict[str, Any]
if remote_count == 0 or graph_api_succeeded or graph_snapshot_complete:
graph_edges = _build_graph_edges(graph_page_records, provider_graph)
_ensure_lease()
fenced_graph_result = await self.store.replace_graph_edges(
mapping_id,
graph_edges,
owner=self.owner,
sync_id=sync_id,
)
if fenced_graph_result is None:
lease_lost.set()
raise WikiSyncLeaseLost("Local Wiki sync lease was lost before replacing graph edges")
graph_result = fenced_graph_result
else:
# Do not replace a previously complete snapshot with a graph
# assembled from a partial page scan when the graph API also
# failed. The next successful full sync will repair it.
graph_result = {
"edge_count": None,
"changed": False,
"preserved": True,
}
_ensure_lease()
result = {
**_sync_result(
"partial" if failed else "completed",
deleted_pages=deleted,
),
"graph_edges": graph_result.get("edge_count"),
"graph_changed": bool(graph_result.get("changed")),
"graph_preserved": bool(graph_result.get("preserved")),
}
# Persist the completed counters only while this worker still owns
# the same lease. A stale worker can therefore never overwrite a
# newer run's status payload.
persisted = await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=not (rebuilding and failed),
remote_page_count=remote_count,
error=f"{failed} page(s) failed" if failed else None,
result=result,
)
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
if not persisted:
lease_lost.set()
raise WikiSyncLeaseLost("Local Wiki sync lease was lost before persisting the scan result")
# Notify even when the remote library is now empty. The global
# assistant base must then remove stale contributions from this
# source instead of retaining deleted Wiki pages forever.
@@ -170,6 +828,7 @@ class WikiSyncService:
result["assistant_sync"] = await self.on_completed(mapping)
return result
except asyncio.CancelledError:
cancelled = _sync_result("cancelled", error="Sync cancelled")
await self.store.finish_sync(
mapping_id,
owner=self.owner,
@@ -177,11 +836,44 @@ class WikiSyncService:
success=False,
remote_page_count=remote_count,
error="Sync cancelled",
result=cancelled,
)
raise
except Exception as exc:
await self.store.finish_sync(mapping_id, owner=self.owner, sync_id=sync_id, success=False, remote_page_count=remote_count, error=_safe_error(exc))
logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
safe_error = _safe_error(exc)
failed_result = _sync_result("failed", error=safe_error)
retry_after: int | None = None
if (
isinstance(exc, WikiEmbeddingError)
and exc.category != "runtime_retired"
):
retry_after = self._record_embedding_failure(exc)
failed_result.update(
{
"failure_scope": "embedding_provider",
"failure_count": self._embedding_failure_count,
"retry_after_seconds": retry_after,
}
)
await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=False,
remote_page_count=remote_count,
error=safe_error,
result=failed_result,
)
logger.warning(
"Local Wiki sync failed mapping_id=%s%s: %s",
mapping_id,
(
f" embedding_provider_retry_after_seconds={retry_after}"
if retry_after is not None
else ""
),
safe_error,
)
raise
finally:
heartbeat.cancel()
@@ -2,6 +2,8 @@
from __future__ import annotations
import asyncio
import httpx
from deerflow.config import get_app_config
@@ -37,9 +39,63 @@ def build_weknora_client(
) -> WeKnoraClient:
if not runtime.weknora_enabled:
raise RuntimeError("WeKnora mode is not enabled")
if http_client is None:
return _shared_weknora_client(
runtime.api_base_url,
runtime.admin_email,
runtime.admin_password,
)
return WeKnoraClient(
base_url=runtime.api_base_url,
http_client=http_client,
admin_email=runtime.admin_email,
admin_password=runtime.admin_password,
)
def _shared_weknora_client(
base_url: str,
admin_email: str,
admin_password: str,
) -> WeKnoraClient:
"""Reuse auth state only inside the current event loop.
``WeKnoraClient`` owns asyncio primitives (an authentication lock and a
cached directory task). A process-wide ``lru_cache`` therefore lets a
client first used by the Gateway loop escape into a LangGraph/tool loop.
Once either primitive has been awaited, reusing it from the other loop can
fail with ``... is bound to a different event loop``. Smart retrieval
used to normalize that internal exception to ``PROVIDER_UNAVAILABLE``, so
an otherwise healthy selected knowledge base appeared empty.
Store the small configuration cache on the loop itself. The loop owns the
clients for exactly its lifetime, equivalent configurations still share
their bearer token and knowledge-base directory within that loop, and no
async primitive crosses loop boundaries. Synchronous callers have no
safe loop ownership yet, so they receive an uncached client.
"""
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return WeKnoraClient(
base_url=base_url,
admin_email=admin_email,
admin_password=admin_password,
)
cache_name = "_deerflow_weknora_clients"
cache = getattr(loop, cache_name, None)
if not isinstance(cache, dict):
cache = {}
setattr(loop, cache_name, cache)
key = (base_url, admin_email, admin_password)
client = cache.get(key)
if client is None:
client = WeKnoraClient(
base_url=base_url,
admin_email=admin_email,
admin_password=admin_password,
)
cache[key] = client
return client