sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+609
-64
@@ -2,12 +2,22 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
import regex
|
||||
|
||||
_GREP_CONTRACT = "weknora-v0.7.2-grep-v1"
|
||||
_GREP_COMPATIBILITY_CONTRACT = "weknora-existing-api-compat-v1"
|
||||
_GREP_MAX_CANDIDATES = 500
|
||||
_GREP_MAX_RESULTS = 30
|
||||
_GREP_REGEX_TIMEOUT_SECONDS = 0.05
|
||||
|
||||
|
||||
class WeKnoraError(RuntimeError):
|
||||
@@ -45,9 +55,13 @@ class WeKnoraClient:
|
||||
tests. When omitted, each public method creates and closes a short-lived client.
|
||||
If admin credentials are configured, DeerFlow authenticates to WeKnora with a
|
||||
cached bearer token; otherwise it keeps the legacy trusted-network anonymous
|
||||
mode for deployments that allow DeerFlow through a reverse proxy.
|
||||
mode for deployments that allow DeerFlow through a reverse proxy. The
|
||||
knowledge-base directory is also cached briefly because several Gateway
|
||||
scopes request the same expensive upstream list at once.
|
||||
"""
|
||||
|
||||
_KNOWLEDGE_BASES_CACHE_SECONDS = 15.0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -65,6 +79,11 @@ class WeKnoraClient:
|
||||
self._auth_token = ""
|
||||
self._tenant_id = ""
|
||||
self._auth_expires_at = 0.0
|
||||
self._auth_generation = 0
|
||||
self._auth_lock = asyncio.Lock()
|
||||
self._knowledge_bases_cache: tuple[float, list[dict[str, Any]]] | None = None
|
||||
self._knowledge_bases_cache_generation = 0
|
||||
self._knowledge_bases_task: tuple[int, asyncio.Task[list[dict[str, Any]]]] | None = None
|
||||
if not self.base_url:
|
||||
raise ValueError("WeKnora base URL is required")
|
||||
|
||||
@@ -77,20 +96,8 @@ class WeKnoraClient:
|
||||
if not isinstance(payload, dict):
|
||||
return "", ""
|
||||
source = payload.get("data") if isinstance(payload.get("data"), dict) else payload
|
||||
token = str(
|
||||
source.get("token")
|
||||
or source.get("access_token")
|
||||
or payload.get("token")
|
||||
or payload.get("access_token")
|
||||
or ""
|
||||
)
|
||||
tenant = (
|
||||
source.get("tenant")
|
||||
if isinstance(source.get("tenant"), dict)
|
||||
else source.get("active_tenant")
|
||||
if isinstance(source.get("active_tenant"), dict)
|
||||
else payload.get("tenant")
|
||||
)
|
||||
token = str(source.get("token") or source.get("access_token") or payload.get("token") or payload.get("access_token") or "")
|
||||
tenant = source.get("tenant") if isinstance(source.get("tenant"), dict) else source.get("active_tenant") if isinstance(source.get("active_tenant"), dict) else payload.get("tenant")
|
||||
user = source.get("user") if isinstance(source.get("user"), dict) else payload.get("user")
|
||||
tenant_id = ""
|
||||
if isinstance(tenant, dict):
|
||||
@@ -106,43 +113,51 @@ class WeKnoraClient:
|
||||
if self._auth_token and not force_refresh and self._auth_expires_at > now + 60:
|
||||
return
|
||||
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
follow_redirects=False,
|
||||
trust_env=False,
|
||||
)
|
||||
try:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/v1/auth/login",
|
||||
json={"email": self.admin_email, "password": self.admin_password},
|
||||
headers={"Accept": "application/json"},
|
||||
observed_generation = self._auth_generation
|
||||
async with self._auth_lock:
|
||||
now = time.time()
|
||||
token_is_fresh = bool(self._auth_token and self._auth_expires_at > now + 60)
|
||||
if token_is_fresh and (not force_refresh or self._auth_generation != observed_generation):
|
||||
return
|
||||
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
follow_redirects=False,
|
||||
trust_env=False,
|
||||
)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
try:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/v1/auth/login",
|
||||
json={"email": self.admin_email, "password": self.admin_password},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
|
||||
if response.status_code >= 400:
|
||||
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
|
||||
if isinstance(payload, dict) and payload.get("success") is False:
|
||||
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
|
||||
token, tenant_id = self._extract_login_token(payload)
|
||||
if not token:
|
||||
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
|
||||
if response.status_code >= 400:
|
||||
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
|
||||
if isinstance(payload, dict) and payload.get("success") is False:
|
||||
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
|
||||
token, tenant_id = self._extract_login_token(payload)
|
||||
if not token:
|
||||
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
|
||||
|
||||
self._auth_token = token
|
||||
self._tenant_id = tenant_id
|
||||
# WeKnora login responses do not consistently expose expiry. Refresh
|
||||
# periodically and retry once if upstream says the token is stale.
|
||||
self._auth_expires_at = now + 25 * 60
|
||||
self._auth_token = token
|
||||
self._tenant_id = tenant_id
|
||||
self._auth_generation += 1
|
||||
# WeKnora login responses do not consistently expose expiry. Refresh
|
||||
# periodically and retry once if upstream says the token is stale.
|
||||
self._auth_expires_at = now + 25 * 60
|
||||
|
||||
async def _headers(self, *, force_refresh: bool = False, accept: str = "application/json") -> dict[str, str]:
|
||||
headers = {"Accept": accept}
|
||||
@@ -192,6 +207,8 @@ class WeKnoraClient:
|
||||
await client.aclose()
|
||||
if response.status_code == 404:
|
||||
raise WeKnoraError("WeKnora resource was not found", status_code=404)
|
||||
if response.status_code == 405:
|
||||
raise WeKnoraError("WeKnora endpoint does not support this method", status_code=405)
|
||||
if response.status_code in {401, 403}:
|
||||
if self._has_admin_credentials:
|
||||
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated internal request", status_code=502)
|
||||
@@ -207,7 +224,14 @@ class WeKnoraClient:
|
||||
detail = str(body.get("message") or body.get("error") or body.get("detail") or "")
|
||||
except ValueError:
|
||||
detail = ""
|
||||
raise WeKnoraError(detail or f"WeKnora returned HTTP {response.status_code}", status_code=502)
|
||||
# Folder creation uses 409 as an optimistic-concurrency signal.
|
||||
# Preserve it so callers can re-read and adopt the concurrently
|
||||
# created folder even if a deployment localizes the error text.
|
||||
status_code = 409 if response.status_code == 409 else 502
|
||||
raise WeKnoraError(
|
||||
detail or f"WeKnora returned HTTP {response.status_code}",
|
||||
status_code=status_code,
|
||||
)
|
||||
if response.status_code == 204 or not response.content:
|
||||
return {"success": True}
|
||||
try:
|
||||
@@ -219,8 +243,49 @@ class WeKnoraClient:
|
||||
payload = await self._request("GET", "/health")
|
||||
return dict(payload) if isinstance(payload, dict) else {"status": "ok"}
|
||||
|
||||
async def list_knowledge_bases(self) -> list[dict[str, Any]]:
|
||||
return _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
|
||||
async def list_knowledge_bases(self, *, refresh: bool = False) -> list[dict[str, Any]]:
|
||||
now = time.monotonic()
|
||||
cached = self._knowledge_bases_cache
|
||||
if not refresh and cached is not None and cached[0] > now:
|
||||
return [dict(item) for item in cached[1]]
|
||||
|
||||
# Share both success and failure with the current wave of callers.
|
||||
# A lock around an uncached failed read would make every waiter issue
|
||||
# another full timeout, serializing an unavailable service's fan-out.
|
||||
generation = self._knowledge_bases_cache_generation
|
||||
pending = self._knowledge_bases_task
|
||||
if pending is None or pending[0] != generation:
|
||||
task = asyncio.create_task(self._load_knowledge_bases(generation))
|
||||
self._knowledge_bases_task = (generation, task)
|
||||
|
||||
def clear_pending(completed: asyncio.Task[list[dict[str, Any]]]) -> None:
|
||||
if self._knowledge_bases_task is not None and self._knowledge_bases_task[1] is completed:
|
||||
self._knowledge_bases_task = None
|
||||
if not completed.cancelled():
|
||||
completed.exception()
|
||||
|
||||
task.add_done_callback(clear_pending)
|
||||
else:
|
||||
task = pending[1]
|
||||
rows = await asyncio.shield(task)
|
||||
return [dict(item) for item in rows]
|
||||
|
||||
async def _load_knowledge_bases(self, generation: int) -> list[dict[str, Any]]:
|
||||
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
|
||||
# A mutation can finish while the read is in flight. Its old snapshot
|
||||
# must neither repopulate the cache nor be joined by post-write callers.
|
||||
if self._knowledge_bases_cache_generation == generation:
|
||||
self._knowledge_bases_cache = (
|
||||
time.monotonic() + self._KNOWLEDGE_BASES_CACHE_SECONDS,
|
||||
[dict(item) for item in rows],
|
||||
)
|
||||
return rows
|
||||
|
||||
def _invalidate_knowledge_bases_cache(self) -> None:
|
||||
"""Discard the directory after a successful local mutation."""
|
||||
|
||||
self._knowledge_bases_cache = None
|
||||
self._knowledge_bases_cache_generation += 1
|
||||
|
||||
async def get_knowledge_base(self, knowledge_base_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("GET", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}"))
|
||||
@@ -233,6 +298,23 @@ class WeKnoraClient:
|
||||
|
||||
return _as_list(_data(await self._request("GET", "/api/v1/models")))
|
||||
|
||||
async def debug_model(self, model_id: str) -> dict[str, Any]:
|
||||
"""Ask WeKnora to verify one saved model with a small real request."""
|
||||
|
||||
normalized_id = str(model_id or "").strip()
|
||||
if not normalized_id:
|
||||
raise WeKnoraError("WeKnora model ID is required", status_code=400)
|
||||
value = _data(
|
||||
await self._request(
|
||||
"POST",
|
||||
f"/api/v1/models/{quote(normalized_id, safe='')}/debug",
|
||||
files={"input": (None, "Reply with OK.")},
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict) or not isinstance(value.get("ok"), bool):
|
||||
raise WeKnoraError("WeKnora returned an invalid model debug result")
|
||||
return dict(value)
|
||||
|
||||
async def get_default_embedding_model_id(self) -> str:
|
||||
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "embedding"]
|
||||
preferred = next((item for item in models if item.get("is_default") is True), None)
|
||||
@@ -249,21 +331,58 @@ class WeKnoraClient:
|
||||
return model_id
|
||||
|
||||
async def get_default_knowledge_qa_model_id(self) -> str:
|
||||
"""Resolve WeKnora's own active synthesis model for Wiki indexing."""
|
||||
"""Resolve a live WeKnora synthesis model for Wiki indexing.
|
||||
|
||||
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "knowledgeqa"]
|
||||
preferred = next((item for item in models if item.get("is_default") is True), None)
|
||||
selected = preferred or next(
|
||||
(item for item in models if str(item.get("status") or "active").lower() == "active"),
|
||||
None,
|
||||
)
|
||||
model_id = str((selected or {}).get("id") or "")
|
||||
if not model_id:
|
||||
WeKnora's ``active`` status means that the database configuration is
|
||||
enabled; it does not prove that the upstream provider still serves the
|
||||
model. Probe active candidates before binding a new Wiki so a disabled
|
||||
provider model cannot silently send every ingest task to dead letters.
|
||||
"""
|
||||
|
||||
models = [
|
||||
item
|
||||
for item in await self.list_models()
|
||||
if str(item.get("type") or "").strip().lower() == "knowledgeqa"
|
||||
and str(item.get("id") or "").strip()
|
||||
and str(item.get("status") or "active").strip().lower() == "active"
|
||||
]
|
||||
if not models:
|
||||
raise WeKnoraError(
|
||||
"WeKnora has no available KnowledgeQA model; configure one in WeKnora before creating a Wiki",
|
||||
status_code=400,
|
||||
)
|
||||
return model_id
|
||||
|
||||
# Prefer an explicit database default. When no default exists, sort by
|
||||
# immutable persisted ID rather than trusting API row order or an
|
||||
# editable display name.
|
||||
models.sort(
|
||||
key=lambda item: (
|
||||
0 if item.get("is_default") is True else 1,
|
||||
str(item.get("id") or "").strip(),
|
||||
)
|
||||
)
|
||||
attempted: list[str] = []
|
||||
for model in models:
|
||||
model_id = str(model.get("id") or "").strip()
|
||||
label = str(model.get("display_name") or model.get("name") or model_id).strip()
|
||||
attempted.append(label or model_id)
|
||||
# ``ok=false`` is a candidate-specific upstream-model failure. An
|
||||
# HTTP/auth/timeout failure from the debug endpoint is a WeKnora
|
||||
# service failure, so preserve it instead of multiplying probes
|
||||
# and misreporting the incident as bad model configuration.
|
||||
debug = await self.debug_model(model_id)
|
||||
if debug.get("ok") is True:
|
||||
return model_id
|
||||
|
||||
attempted_models = ", ".join(attempted[:12])
|
||||
if len(attempted) > 12:
|
||||
attempted_models = f"{attempted_models}, and {len(attempted) - 12} more"
|
||||
raise WeKnoraError(
|
||||
"WeKnora has no usable active KnowledgeQA model after connectivity checks"
|
||||
+ (f": {attempted_models}" if attempted_models else "")
|
||||
+ "; configure or repair a working default model before creating a Wiki",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
async def create_knowledge_base(
|
||||
self,
|
||||
@@ -273,6 +392,8 @@ class WeKnoraClient:
|
||||
kb_type: str = "document",
|
||||
embedding_model_id: str | None = None,
|
||||
wiki_enabled: bool = False,
|
||||
wiki_content_instructions: str = "",
|
||||
wiki_extraction_instructions: str = "",
|
||||
) -> dict[str, Any]:
|
||||
model_id = embedding_model_id or await self.get_default_embedding_model_id()
|
||||
create_payload: dict[str, Any] = {
|
||||
@@ -295,6 +416,8 @@ class WeKnoraClient:
|
||||
"wiki_config": {
|
||||
"synthesis_model_id": synthesis_model_id,
|
||||
"extraction_granularity": "standard",
|
||||
"content_instructions": wiki_content_instructions.strip(),
|
||||
"extraction_instructions": wiki_extraction_instructions.strip(),
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -306,16 +429,31 @@ class WeKnoraClient:
|
||||
value = _data(payload)
|
||||
if not isinstance(value, dict) or not value.get("id"):
|
||||
raise WeKnoraError("WeKnora did not return the created knowledge base")
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value)
|
||||
|
||||
async def update_knowledge_base(self, knowledge_base_id: str, changes: Mapping[str, Any]) -> dict[str, Any]:
|
||||
payload = dict(changes)
|
||||
if not str(payload.get("name") or "").strip():
|
||||
wiki_config_changes = payload.pop("wiki_config", None)
|
||||
current: dict[str, Any] | None = None
|
||||
if not str(payload.get("name") or "").strip() or wiki_config_changes is not None:
|
||||
current = await self.get_knowledge_base(knowledge_base_id)
|
||||
if not str(payload.get("name") or "").strip():
|
||||
assert current is not None
|
||||
current_name = str(current.get("name") or "").strip()
|
||||
if not current_name:
|
||||
raise WeKnoraError("WeKnora knowledge base name is required for updates")
|
||||
payload["name"] = current_name
|
||||
if wiki_config_changes is not None:
|
||||
if not isinstance(wiki_config_changes, Mapping):
|
||||
raise WeKnoraError("WeKnora Wiki instructions must be an object", status_code=422)
|
||||
assert current is not None
|
||||
current_wiki_config = current.get("wiki_config")
|
||||
merged_wiki_config = dict(current_wiki_config) if isinstance(current_wiki_config, Mapping) else {}
|
||||
for key in ("content_instructions", "extraction_instructions"):
|
||||
if key in wiki_config_changes:
|
||||
merged_wiki_config[key] = str(wiki_config_changes.get(key) or "").strip()
|
||||
payload["config"] = {"wiki_config": merged_wiki_config}
|
||||
value = _data(
|
||||
await self._request(
|
||||
"PUT",
|
||||
@@ -323,10 +461,12 @@ class WeKnoraClient:
|
||||
json=payload,
|
||||
)
|
||||
)
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def delete_knowledge_base(self, knowledge_base_id: str) -> None:
|
||||
await self._request("DELETE", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}")
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
|
||||
async def list_documents(self, knowledge_base_id: str, *, page: int = 1, page_size: int = 100) -> dict[str, Any]:
|
||||
payload = await self._request(
|
||||
@@ -353,6 +493,7 @@ class WeKnoraClient:
|
||||
files={"file": (filename, content, content_type)},
|
||||
)
|
||||
)
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def import_document_url(self, knowledge_base_id: str, *, url: str) -> dict[str, Any]:
|
||||
@@ -363,6 +504,7 @@ class WeKnoraClient:
|
||||
json={"url": url},
|
||||
)
|
||||
)
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def create_manual_document(
|
||||
@@ -379,10 +521,12 @@ class WeKnoraClient:
|
||||
json={"title": title, "content": content, "status": "publish"},
|
||||
)
|
||||
)
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def delete_document(self, knowledge_id: str) -> None:
|
||||
await self._request("DELETE", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}")
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
|
||||
async def get_document(self, knowledge_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("GET", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}"))
|
||||
@@ -477,6 +621,7 @@ class WeKnoraClient:
|
||||
|
||||
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def list_chunks(self, knowledge_id: str, *, page: int = 1, page_size: int = 20) -> dict[str, Any]:
|
||||
@@ -513,6 +658,79 @@ class WeKnoraClient:
|
||||
raise WeKnoraError("Invalid Wiki folder response")
|
||||
return value
|
||||
|
||||
async def list_wiki_folders(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
*,
|
||||
parent_id: str = "",
|
||||
page_types: Sequence[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
params: dict[str, Any] = {"parent_id": parent_id or None}
|
||||
if page_types:
|
||||
params["page_types"] = ",".join(str(value) for value in page_types if str(value)) or None
|
||||
payload = await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders",
|
||||
params=params,
|
||||
)
|
||||
value = _data(payload)
|
||||
source = value if isinstance(value, dict) else {}
|
||||
raw_folders = source.get("folders") if isinstance(value, dict) else value
|
||||
rows = _as_list(raw_folders)
|
||||
return {
|
||||
"parent_id": str(source.get("parent_id") or parent_id),
|
||||
"folders": rows,
|
||||
}
|
||||
|
||||
async def update_wiki_folder(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
folder_id: str,
|
||||
changes: Mapping[str, Any] | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
parent_id: str | None = None,
|
||||
move_parent: bool | None = None,
|
||||
) -> dict[str, Any]:
|
||||
update = dict(changes or {})
|
||||
if name is not None:
|
||||
update["name"] = name
|
||||
if parent_id is not None:
|
||||
update["parent_id"] = parent_id
|
||||
update.setdefault("move_parent", True)
|
||||
if move_parent is not None:
|
||||
update["move_parent"] = move_parent
|
||||
elif "parent_id" in update:
|
||||
update.setdefault("move_parent", True)
|
||||
value = _data(
|
||||
await self._request(
|
||||
"PUT",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders/{quote(folder_id, safe='')}",
|
||||
json=update,
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("Invalid Wiki folder response")
|
||||
return dict(value)
|
||||
|
||||
async def delete_wiki_folder(self, knowledge_base_id: str, folder_id: str) -> None:
|
||||
await self._request(
|
||||
"DELETE",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders/{quote(folder_id, safe='')}",
|
||||
)
|
||||
|
||||
async def move_wiki_page(self, knowledge_base_id: str, slug: str, folder_id: str) -> dict[str, Any]:
|
||||
value = _data(
|
||||
await self._request(
|
||||
"PUT",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/move-page",
|
||||
json={"slug": slug, "folder_id": folder_id},
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("Invalid Wiki page response")
|
||||
return dict(value)
|
||||
|
||||
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
|
||||
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
|
||||
|
||||
@@ -523,11 +741,17 @@ class WeKnoraClient:
|
||||
page: int = 1,
|
||||
page_size: int = 50,
|
||||
query: str = "",
|
||||
folder_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
params: dict[str, Any] = {"page": page, "page_size": page_size, "query": query or None}
|
||||
# ``None`` means no placement filter; an explicit empty string means
|
||||
# pages directly under the Wiki root.
|
||||
if folder_id is not None:
|
||||
params["folder_id"] = folder_id
|
||||
payload = await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages",
|
||||
params={"page": page, "page_size": page_size, "query": query or None},
|
||||
params=params,
|
||||
)
|
||||
value = _data(payload)
|
||||
source = value if isinstance(value, dict) else payload if isinstance(payload, dict) else {}
|
||||
@@ -607,6 +831,327 @@ class WeKnoraClient:
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_grep_chunk(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Normalize both the provider extension and compatibility rows."""
|
||||
|
||||
return {
|
||||
"chunk_id": str(row.get("id") or row.get("chunk_id") or ""),
|
||||
"content": str(row.get("content") or row.get("text") or row.get("match_snippet") or ""),
|
||||
"knowledge_id": str(row.get("knowledge_id") or ""),
|
||||
"knowledge_base_id": str(row.get("knowledge_base_id") or ""),
|
||||
"title": str(
|
||||
row.get("knowledge_title")
|
||||
or row.get("title")
|
||||
or row.get("knowledge_filename")
|
||||
or ""
|
||||
),
|
||||
"chunk_index": int(row.get("chunk_index") or row.get("index") or 0),
|
||||
"match_count": int(
|
||||
row.get("matched_patterns")
|
||||
or row.get("match_count")
|
||||
or row.get("total_pattern_hits")
|
||||
or 0
|
||||
),
|
||||
"title_match": bool(row.get("title_match")),
|
||||
"score": float(row.get("match_score") or row.get("score") or 0.0),
|
||||
"metadata": row.get("metadata") if isinstance(row.get("metadata"), dict) else {},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _grep_matches(expression: regex.Pattern, value: str) -> list[regex.Match]:
|
||||
try:
|
||||
return list(
|
||||
expression.finditer(
|
||||
value,
|
||||
timeout=_GREP_REGEX_TIMEOUT_SECONDS,
|
||||
)
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise WeKnoraError(
|
||||
"WeKnora compatibility grep regular expression timed out",
|
||||
status_code=422,
|
||||
) from exc
|
||||
|
||||
async def _grep_chunks_compatibility(
|
||||
self,
|
||||
query: str,
|
||||
knowledge_base_ids: Sequence[str],
|
||||
) -> dict[str, Any]:
|
||||
"""Bounded, fair fallback for older providers without ``chunks/grep``.
|
||||
|
||||
The native endpoint remains authoritative. This path round-robins both
|
||||
knowledge bases and documents so one large first scope cannot consume
|
||||
the global 500-chunk compatibility budget.
|
||||
"""
|
||||
|
||||
ids = list(dict.fromkeys(str(value).strip() for value in knowledge_base_ids if str(value).strip()))
|
||||
try:
|
||||
expression = regex.compile(query, regex.IGNORECASE | regex.VERSION1)
|
||||
except regex.error as exc:
|
||||
raise WeKnoraError(f"Invalid grep regular expression: {exc}", status_code=422) from exc
|
||||
|
||||
per_scope_document_cap = max(1, (_GREP_MAX_CANDIDATES + len(ids) - 1) // max(1, len(ids)))
|
||||
|
||||
async def load_documents(knowledge_base_id: str) -> tuple[str, list[dict[str, Any]] | Exception]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
page = 1
|
||||
try:
|
||||
while len(rows) < per_scope_document_cap:
|
||||
page_size = min(100, per_scope_document_cap - len(rows))
|
||||
payload = await self.list_documents(
|
||||
knowledge_base_id,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
batch = [
|
||||
dict(item)
|
||||
for item in payload.get("items") or []
|
||||
if isinstance(item, Mapping)
|
||||
]
|
||||
rows.extend(batch)
|
||||
if not batch or page * page_size >= int(payload.get("total") or len(rows)):
|
||||
break
|
||||
page += 1
|
||||
return knowledge_base_id, rows
|
||||
except Exception as exc: # noqa: BLE001 - returned as structured scope failure
|
||||
return knowledge_base_id, exc
|
||||
|
||||
loaded = await asyncio.gather(*(load_documents(knowledge_base_id) for knowledge_base_id in ids))
|
||||
documents_by_scope: dict[str, list[dict[str, Any]]] = {}
|
||||
warnings: list[dict[str, Any]] = [
|
||||
{
|
||||
"code": "WEKNORA_GREP_COMPATIBILITY_FALLBACK",
|
||||
"message": "Provider chunks/grep is unavailable; used bounded existing-API compatibility scan.",
|
||||
}
|
||||
]
|
||||
failed_scope_ids: set[str] = set()
|
||||
for knowledge_base_id, outcome in loaded:
|
||||
if isinstance(outcome, Exception):
|
||||
failed_scope_ids.add(knowledge_base_id)
|
||||
warnings.append(
|
||||
{
|
||||
"code": "PROVIDER_SCOPE_FAILED",
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
"stage": "list_documents",
|
||||
}
|
||||
)
|
||||
else:
|
||||
documents_by_scope[knowledge_base_id] = outcome
|
||||
|
||||
if not documents_by_scope:
|
||||
raise WeKnoraError("WeKnora compatibility grep failed for every knowledge base")
|
||||
|
||||
# Seed the work queue in KB/document round-robin order. Subsequent
|
||||
# chunk pages are re-appended to the tail, preserving document fairness.
|
||||
work: deque[tuple[str, dict[str, Any], int]] = deque()
|
||||
max_documents = max((len(rows) for rows in documents_by_scope.values()), default=0)
|
||||
for index in range(max_documents):
|
||||
for knowledge_base_id in ids:
|
||||
rows = documents_by_scope.get(knowledge_base_id) or []
|
||||
if index < len(rows):
|
||||
work.append((knowledge_base_id, rows[index], 1))
|
||||
|
||||
candidates: list[dict[str, Any]] = []
|
||||
seen_content: set[str] = set()
|
||||
scanned_chunks = 0
|
||||
chunk_successes = {knowledge_base_id: 0 for knowledge_base_id in documents_by_scope}
|
||||
chunk_failures = {knowledge_base_id: 0 for knowledge_base_id in documents_by_scope}
|
||||
while work and scanned_chunks < _GREP_MAX_CANDIDATES:
|
||||
knowledge_base_id, document, page = work.popleft()
|
||||
knowledge_id = str(document.get("id") or document.get("knowledge_id") or "")
|
||||
if not knowledge_id:
|
||||
continue
|
||||
page_size = min(20, _GREP_MAX_CANDIDATES - scanned_chunks)
|
||||
try:
|
||||
payload = await self.list_chunks(
|
||||
knowledge_id,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - one document must not fail the whole fallback
|
||||
chunk_failures[knowledge_base_id] += 1
|
||||
continue
|
||||
chunk_successes[knowledge_base_id] += 1
|
||||
chunks = [
|
||||
dict(item)
|
||||
for item in payload.get("items") or []
|
||||
if isinstance(item, Mapping)
|
||||
]
|
||||
title = str(
|
||||
document.get("title")
|
||||
or document.get("name")
|
||||
or document.get("filename")
|
||||
or ""
|
||||
)
|
||||
title_matches = self._grep_matches(expression, title[:5000])
|
||||
for chunk in chunks:
|
||||
if scanned_chunks >= _GREP_MAX_CANDIDATES:
|
||||
break
|
||||
scanned_chunks += 1
|
||||
content = str(chunk.get("content") or chunk.get("text") or "")
|
||||
content_matches = self._grep_matches(expression, content[:50_000])
|
||||
if not title_matches and not content_matches:
|
||||
continue
|
||||
signature = hashlib.sha256(" ".join(content.split()).encode()).hexdigest()
|
||||
if signature in seen_content:
|
||||
continue
|
||||
seen_content.add(signature)
|
||||
match_count = min(99, len(title_matches) * 4 + len(content_matches))
|
||||
candidates.append(
|
||||
self._normalize_grep_chunk(
|
||||
{
|
||||
**chunk,
|
||||
"knowledge_id": knowledge_id,
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
"knowledge_title": title,
|
||||
"match_count": match_count,
|
||||
"title_match": bool(title_matches),
|
||||
"match_score": float(match_count),
|
||||
}
|
||||
)
|
||||
)
|
||||
if chunks and page * page_size < int(payload.get("total") or len(chunks)):
|
||||
work.append((knowledge_base_id, document, page + 1))
|
||||
|
||||
successful_scope_ids: list[str] = []
|
||||
for knowledge_base_id, documents in documents_by_scope.items():
|
||||
# An empty KB was successfully scanned. A non-empty KB for which
|
||||
# every chunk read failed is a failed provider scope.
|
||||
if not documents or chunk_successes[knowledge_base_id] > 0:
|
||||
successful_scope_ids.append(knowledge_base_id)
|
||||
else:
|
||||
failed_scope_ids.add(knowledge_base_id)
|
||||
if chunk_failures[knowledge_base_id]:
|
||||
warnings.append(
|
||||
{
|
||||
"code": "PROVIDER_DOCUMENT_READ_FAILED",
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
"failed_documents": chunk_failures[knowledge_base_id],
|
||||
}
|
||||
)
|
||||
|
||||
if not successful_scope_ids:
|
||||
raise WeKnoraError("WeKnora compatibility grep failed for every knowledge base")
|
||||
|
||||
candidates.sort(
|
||||
key=lambda item: (
|
||||
bool(item.get("title_match")),
|
||||
int(item.get("match_count") or 0),
|
||||
float(item.get("score") or 0),
|
||||
-int(item.get("chunk_index") or 0),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return {
|
||||
"items": candidates[:_GREP_MAX_RESULTS],
|
||||
"contract": _GREP_COMPATIBILITY_CONTRACT,
|
||||
"compatibility_fallback": True,
|
||||
"partial": bool(failed_scope_ids or any(chunk_failures.values())),
|
||||
"warnings": warnings,
|
||||
"successful_knowledge_base_ids": successful_scope_ids,
|
||||
"failed_knowledge_base_ids": [value for value in ids if value in failed_scope_ids],
|
||||
"scanned_chunks": scanned_chunks,
|
||||
"scan_truncated": scanned_chunks >= _GREP_MAX_CANDIDATES,
|
||||
}
|
||||
|
||||
async def grep_chunks(
|
||||
self,
|
||||
query: str,
|
||||
knowledge_base_ids: Sequence[str],
|
||||
) -> dict[str, Any]:
|
||||
"""Run the v0.7.2 cross-KB grep extension, with a marked fallback."""
|
||||
|
||||
ids = list(dict.fromkeys(str(value).strip() for value in knowledge_base_ids if str(value).strip()))
|
||||
if not ids:
|
||||
return {
|
||||
"items": [],
|
||||
"contract": _GREP_CONTRACT,
|
||||
"compatibility_fallback": False,
|
||||
"partial": False,
|
||||
"warnings": [],
|
||||
"successful_knowledge_base_ids": [],
|
||||
"failed_knowledge_base_ids": [],
|
||||
"scanned_chunks": 0,
|
||||
"scan_truncated": False,
|
||||
}
|
||||
try:
|
||||
payload = await self._request(
|
||||
"POST",
|
||||
"/api/v1/chunks/grep",
|
||||
json={"query": query, "knowledge_base_ids": ids},
|
||||
)
|
||||
except WeKnoraError as exc:
|
||||
if exc.status_code not in {404, 405}:
|
||||
raise
|
||||
return await self._grep_chunks_compatibility(query, ids)
|
||||
|
||||
value = _data(payload)
|
||||
source = value if isinstance(value, dict) else {}
|
||||
contract = str(
|
||||
source.get("contract")
|
||||
or (payload.get("contract") if isinstance(payload, dict) else "")
|
||||
or ""
|
||||
)
|
||||
if contract != _GREP_CONTRACT:
|
||||
raise WeKnoraError(
|
||||
f"WeKnora chunks/grep contract mismatch (expected {_GREP_CONTRACT})"
|
||||
)
|
||||
raw_rows = (
|
||||
source.get("chunk_results")
|
||||
or source.get("results")
|
||||
or source.get("items")
|
||||
or []
|
||||
)
|
||||
if not isinstance(raw_rows, list):
|
||||
raise WeKnoraError("WeKnora chunks/grep returned invalid results")
|
||||
rows = [
|
||||
self._normalize_grep_chunk(row)
|
||||
for row in raw_rows
|
||||
if isinstance(row, Mapping)
|
||||
]
|
||||
warnings = [
|
||||
dict(item)
|
||||
for item in source.get("warnings") or []
|
||||
if isinstance(item, Mapping)
|
||||
]
|
||||
failed_ids = [
|
||||
str(value)
|
||||
for value in source.get("failed_knowledge_base_ids") or []
|
||||
if str(value) in ids
|
||||
]
|
||||
explicit_successes = source.get("successful_knowledge_base_ids")
|
||||
successful_ids = (
|
||||
[str(value) for value in explicit_successes if str(value) in ids]
|
||||
if isinstance(explicit_successes, list)
|
||||
else [value for value in ids if value not in failed_ids]
|
||||
)
|
||||
warned_scope_ids = {
|
||||
str(item.get("knowledge_base_id") or "")
|
||||
for item in warnings
|
||||
}
|
||||
warnings.extend(
|
||||
{
|
||||
"code": "PROVIDER_SCOPE_FAILED",
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
}
|
||||
for knowledge_base_id in failed_ids
|
||||
if knowledge_base_id not in warned_scope_ids
|
||||
)
|
||||
if source.get("partial") and not warnings:
|
||||
warnings.append({"code": "PROVIDER_PARTIAL_RESPONSE"})
|
||||
return {
|
||||
"items": rows[:_GREP_MAX_RESULTS],
|
||||
"contract": contract,
|
||||
"compatibility_fallback": False,
|
||||
"partial": bool(source.get("partial")) or bool(warnings) or bool(failed_ids),
|
||||
"warnings": warnings,
|
||||
"successful_knowledge_base_ids": successful_ids,
|
||||
"failed_knowledge_base_ids": failed_ids,
|
||||
"scanned_chunks": int(source.get("scanned_chunks") or len(rows)),
|
||||
"scan_truncated": bool(source.get("scan_truncated")),
|
||||
}
|
||||
|
||||
async def search(self, query: str, knowledge_base_ids: Sequence[str]) -> list[dict[str, Any]]:
|
||||
ids = [str(value) for value in knowledge_base_ids if str(value)]
|
||||
if not ids:
|
||||
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
"""Product-owned configuration layered over a WeKnora knowledge base."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_RETRIEVAL_SETTINGS: dict[str, Any] = {
|
||||
"entity_expansion_enabled": True,
|
||||
"max_related_terms": 6,
|
||||
"statistics_enabled": True,
|
||||
}
|
||||
|
||||
|
||||
def default_test_questions(name: str) -> list[str]:
|
||||
label = str(name or "当前知识库").strip() or "当前知识库"
|
||||
return [
|
||||
f"「{label}」主要讲了什么?请概括资料范围并引用来源。",
|
||||
f"请梳理「{label}」中的核心人物、机构、事件及其关联关系。",
|
||||
"请统计指定日期范围内与关注对象有关的资料数量,并列出具体新闻或来源。",
|
||||
f"「{label}」中哪些资料相互印证,哪些结论存在冲突或证据不足?",
|
||||
]
|
||||
|
||||
|
||||
def generated_test_questions_name(value: Any) -> str | None:
|
||||
"""Return the embedded name when *value* is an untouched default set.
|
||||
|
||||
Older versions of the knowledge-base editor submitted the resolved default
|
||||
questions on every save. That accidentally persisted them as custom
|
||||
questions, so a later rename kept showing the old knowledge-base name. We
|
||||
recognise only the exact four-question template here; genuinely edited
|
||||
questions remain custom and are never rewritten.
|
||||
"""
|
||||
|
||||
if not isinstance(value, list) or len(value) != 4:
|
||||
return None
|
||||
questions = [str(item).strip() for item in value]
|
||||
prefix = "「"
|
||||
suffix = "」主要讲了什么?请概括资料范围并引用来源。"
|
||||
first = questions[0]
|
||||
if not first.startswith(prefix) or not first.endswith(suffix):
|
||||
return None
|
||||
name = first[len(prefix) : -len(suffix)].strip()
|
||||
if not name or questions != default_test_questions(name):
|
||||
return None
|
||||
return name
|
||||
|
||||
|
||||
def resolved_test_questions(row: dict[str, Any]) -> list[str]:
|
||||
configured = row.get("test_questions")
|
||||
if isinstance(configured, list):
|
||||
# Treat accidentally persisted legacy defaults as automatic. This
|
||||
# repairs existing rows immediately and lets the questions follow a
|
||||
# renamed knowledge base without a data migration.
|
||||
if generated_test_questions_name(configured) is None:
|
||||
return [str(item).strip() for item in configured if str(item).strip()][:12]
|
||||
return default_test_questions(str(row.get("name") or ""))
|
||||
|
||||
|
||||
def are_test_questions_customized(row: dict[str, Any]) -> bool:
|
||||
configured = row.get("test_questions")
|
||||
return isinstance(configured, list) and generated_test_questions_name(configured) is None
|
||||
|
||||
|
||||
def resolved_retrieval_settings(row: dict[str, Any]) -> dict[str, Any]:
|
||||
configured = row.get("retrieval_settings")
|
||||
if not isinstance(configured, dict):
|
||||
configured = {}
|
||||
return {
|
||||
"entity_expansion_enabled": bool(configured.get("entity_expansion_enabled", True)),
|
||||
"max_related_terms": max(0, min(int(configured.get("max_related_terms", 6)), 12)),
|
||||
"statistics_enabled": bool(configured.get("statistics_enabled", True)),
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_RETRIEVAL_SETTINGS",
|
||||
"default_test_questions",
|
||||
"generated_test_questions_name",
|
||||
"resolved_retrieval_settings",
|
||||
"resolved_test_questions",
|
||||
"are_test_questions_customized",
|
||||
]
|
||||
+518
@@ -0,0 +1,518 @@
|
||||
"""Entity-expanded and statistics-aware retrieval over the local Wiki mirror.
|
||||
|
||||
The language model is deliberately not asked to count search results. This
|
||||
module parses an optional date range, expands first-degree Wiki links, filters
|
||||
the stored pages/sources, and returns auditable counts plus concrete references.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from datetime import UTC, date, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.integrations.weknora.knowledge_profile import resolved_retrieval_settings
|
||||
from deerflow.integrations.weknora.local_index.overview import is_wiki_body_page, is_wiki_overview_query
|
||||
from deerflow.integrations.weknora.local_index.ranking import (
|
||||
title_boosted_rrf_score,
|
||||
title_match_strength,
|
||||
)
|
||||
|
||||
_DATE_KEY_PATTERN = re.compile(
|
||||
r"(?im)^\s*(?:published_at|publish_time|published|date|发布日期|发布时间|新闻时间|日期|时间)\s*[::]\s*[\"']?([^\n\"']+)"
|
||||
)
|
||||
_ISO_DATE_PATTERN = re.compile(r"(?<!\d)(20\d{2})[-/.年](\d{1,2})[-/.月](\d{1,2})(?:日|号)?")
|
||||
_MONTH_DAY_PATTERN = re.compile(r"(?<!\d)(\d{1,2})月(\d{1,2})(?:日|号)?")
|
||||
_DAY_RANGE_PATTERN = re.compile(r"(?<!\d)(\d{1,2})(?:日|号)?\s*(?:到|至|[-~—–])\s*(\d{1,2})(?:日|号)")
|
||||
_WIKILINK_PATTERN = re.compile(r"\[\[([^\]|#]{2,80})(?:#[^\]|]+)?(?:\|([^\]]{1,80}))?\]\]")
|
||||
_MARKDOWN_LINK_PATTERN = re.compile(r"\[([^\]]{2,80})\]\(([^)]+)\)")
|
||||
_STATISTICS_PATTERN = re.compile(r"(?:多少|几篇|几条|统计|数量|占比|汇总|合计)")
|
||||
_SUBJECT_PATTERNS = (
|
||||
re.compile(r"关于\s*([^,。;;??]{2,40}?)(?:的|有多少|相关新闻|相关资料|相关报告|新闻|报告|资料|$)"),
|
||||
re.compile(r"(?:搜索|查询|查找|检索)\s*([^,。;;??]{2,30}?)(?:的|相关新闻|相关资料|新闻|报告|资料|$)"),
|
||||
)
|
||||
_QUERY_STOP_WORDS = {
|
||||
"帮我", "总结", "一下", "报告", "新闻", "资料", "数据", "相关", "有关", "关于",
|
||||
"多少", "统计", "查看", "具体", "知识库", "这个", "哪些", "时间", "日期", "月份",
|
||||
}
|
||||
|
||||
|
||||
def _safe_date(year: int, month: int, day: int) -> date | None:
|
||||
try:
|
||||
return date(year, month, day)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_date_value(value: Any, *, today: date) -> date | None:
|
||||
if isinstance(value, datetime):
|
||||
return value.date()
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
text = str(value or "").strip()
|
||||
match = _ISO_DATE_PATTERN.search(text)
|
||||
if match:
|
||||
return _safe_date(int(match.group(1)), int(match.group(2)), int(match.group(3)))
|
||||
match = _MONTH_DAY_PATTERN.search(text)
|
||||
if match:
|
||||
return _safe_date(today.year, int(match.group(1)), int(match.group(2)))
|
||||
try:
|
||||
return datetime.fromisoformat(text.replace("Z", "+00:00")).date()
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def parse_date_range(
|
||||
query: str,
|
||||
*,
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
today: date | None = None,
|
||||
) -> tuple[date | None, date | None]:
|
||||
now = today or datetime.now(UTC).date()
|
||||
explicit_start = _parse_date_value(date_from, today=now)
|
||||
explicit_end = _parse_date_value(date_to, today=now)
|
||||
if explicit_start or explicit_end:
|
||||
return explicit_start, explicit_end
|
||||
|
||||
full = [_safe_date(int(y), int(m), int(d)) for y, m, d in _ISO_DATE_PATTERN.findall(query)]
|
||||
full = [item for item in full if item is not None]
|
||||
if len(full) >= 2:
|
||||
return min(full[0], full[1]), max(full[0], full[1])
|
||||
month_days = [_safe_date(now.year, int(m), int(d)) for m, d in _MONTH_DAY_PATTERN.findall(query)]
|
||||
month_days = [item for item in month_days if item is not None]
|
||||
if len(month_days) >= 2:
|
||||
return min(month_days[0], month_days[1]), max(month_days[0], month_days[1])
|
||||
day_range = _DAY_RANGE_PATTERN.search(query)
|
||||
if day_range:
|
||||
start = _safe_date(now.year, now.month, int(day_range.group(1)))
|
||||
end = _safe_date(now.year, now.month, int(day_range.group(2)))
|
||||
if start and end:
|
||||
return min(start, end), max(start, end)
|
||||
return (full[0], full[0]) if len(full) == 1 else (month_days[0], month_days[0]) if len(month_days) == 1 else (None, None)
|
||||
|
||||
|
||||
def extract_subject_terms(query: str) -> list[str]:
|
||||
terms: list[str] = []
|
||||
terms.extend(item.strip() for item in re.findall(r"[「『\"']([^」』\"']{2,40})[」』\"']", query))
|
||||
for pattern in _SUBJECT_PATTERNS:
|
||||
match = pattern.search(query)
|
||||
if match:
|
||||
terms.append(match.group(1).strip(" 的之与和"))
|
||||
if not terms:
|
||||
cleaned = _ISO_DATE_PATTERN.sub(" ", query)
|
||||
cleaned = _MONTH_DAY_PATTERN.sub(" ", cleaned)
|
||||
for word in sorted(_QUERY_STOP_WORDS, key=len, reverse=True):
|
||||
cleaned = cleaned.replace(word, " ")
|
||||
candidates = re.findall(r"[A-Za-z][A-Za-z0-9_.-]{1,39}|[\u4e00-\u9fff·]{2,20}", cleaned)
|
||||
if candidates:
|
||||
terms.append(max(candidates, key=len))
|
||||
return list(dict.fromkeys(term for term in terms if 1 < len(term) <= 40))[:4]
|
||||
|
||||
|
||||
def _page_haystack(page: dict[str, Any]) -> str:
|
||||
aliases = " ".join(str(item) for item in page.get("aliases") or [])
|
||||
return f"{page.get('title') or ''}\n{aliases}\n{page.get('summary') or ''}\n{page.get('content_md') or ''}".casefold()
|
||||
|
||||
|
||||
def expand_related_terms(
|
||||
subject_terms: list[str],
|
||||
pages: list[dict[str, Any]],
|
||||
*,
|
||||
limit: int,
|
||||
) -> list[dict[str, str]]:
|
||||
if limit <= 0:
|
||||
return []
|
||||
subjects = {item.casefold() for item in subject_terms}
|
||||
expanded: list[dict[str, str]] = []
|
||||
seen = set(subjects)
|
||||
for page in pages:
|
||||
haystack = _page_haystack(page)
|
||||
if subjects and not any(subject in haystack for subject in subjects):
|
||||
continue
|
||||
candidates: list[tuple[str, str]] = []
|
||||
candidates.extend((str(alias).strip(), "alias") for alias in page.get("aliases") or [])
|
||||
candidates.extend(((label or target).strip(), "wikilink") for target, label in _WIKILINK_PATTERN.findall(str(page.get("content_md") or "")))
|
||||
candidates.extend((label.strip(), "markdown_link") for label, _ in _MARKDOWN_LINK_PATTERN.findall(str(page.get("content_md") or "")))
|
||||
for term, relation in candidates:
|
||||
key = term.casefold()
|
||||
if key in seen or not 1 < len(term) <= 40 or term.startswith(("http://", "https://")):
|
||||
continue
|
||||
seen.add(key)
|
||||
expanded.append({"term": term, "relation": relation, "from_page": str(page.get("title") or page.get("slug") or "")})
|
||||
if len(expanded) >= limit:
|
||||
return expanded
|
||||
return expanded
|
||||
|
||||
|
||||
def extract_record_date(record: dict[str, Any], *, today: date | None = None) -> tuple[date | None, str | None]:
|
||||
now = today or datetime.now(UTC).date()
|
||||
for key in ("published_at", "publish_time", "published", "date", "发布日期", "发布时间", "新闻时间"):
|
||||
parsed = _parse_date_value(record.get(key), today=now)
|
||||
if parsed:
|
||||
return parsed, key
|
||||
metadata = record.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
parsed, source = extract_record_date(metadata, today=now)
|
||||
if parsed:
|
||||
return parsed, f"metadata.{source}"
|
||||
content = str(record.get("content_md") or record.get("content") or "")
|
||||
match = _DATE_KEY_PATTERN.search(content[:8000])
|
||||
if match:
|
||||
parsed = _parse_date_value(match.group(1), today=now)
|
||||
if parsed:
|
||||
return parsed, "content_frontmatter"
|
||||
for key in ("remote_created_at", "created_at", "remote_updated_at", "updated_at"):
|
||||
parsed = _parse_date_value(record.get(key), today=now)
|
||||
if parsed:
|
||||
return parsed, key
|
||||
return None, None
|
||||
|
||||
|
||||
def _reference_identity(ref: dict[str, Any]) -> str:
|
||||
return str(
|
||||
ref.get("document_id")
|
||||
or ref.get("knowledge_id")
|
||||
or ref.get("source_id")
|
||||
or ref.get("url")
|
||||
or ref.get("title")
|
||||
or ref.get("name")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
|
||||
async def enrich_chunk_references(
|
||||
pages: list[dict[str, Any]],
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
client: Any | None,
|
||||
max_chunks: int = 500,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
"""Resolve Wiki provenance chunk ids to source-document metadata.
|
||||
|
||||
This is not a second raw-document search: only chunks already cited by a
|
||||
matched Wiki page are resolved, and every chunk is rechecked against the
|
||||
authorized remote knowledge-base id.
|
||||
"""
|
||||
|
||||
if client is None:
|
||||
return pages, []
|
||||
mapping_by_id = {str(item.get("id") or ""): item for item in mappings}
|
||||
pending: list[tuple[int, str, str]] = []
|
||||
copied = [{**page, "source_refs": list(page.get("source_refs") or [])} for page in pages]
|
||||
for page_index, page in enumerate(copied):
|
||||
mapping = mapping_by_id.get(str(page.get("knowledge_base_mapping_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
existing_ids = {
|
||||
str(item.get("chunk_id") or item.get("id") or "")
|
||||
for item in page["source_refs"]
|
||||
if isinstance(item, dict)
|
||||
}
|
||||
raw_refs = [*page["source_refs"], *(page.get("chunk_refs") or [])]
|
||||
for ref in raw_refs:
|
||||
chunk_id = str(ref).strip() if not isinstance(ref, dict) else str(ref.get("chunk_id") or ref.get("id") or "").strip()
|
||||
if chunk_id and chunk_id not in existing_ids:
|
||||
existing_ids.add(chunk_id)
|
||||
pending.append((page_index, str(mapping.get("weknora_id") or ""), chunk_id))
|
||||
warnings: list[str] = []
|
||||
if len(pending) > max_chunks:
|
||||
warnings.append(f"source_reference_limit:{max_chunks}/{len(pending)}")
|
||||
pending = pending[:max_chunks]
|
||||
semaphore = asyncio.Semaphore(12)
|
||||
|
||||
async def resolve(expected_base_id: str, chunk_id: str) -> dict[str, Any] | None:
|
||||
async with semaphore:
|
||||
try:
|
||||
chunk = await client.get_chunk(chunk_id)
|
||||
except Exception:
|
||||
return None
|
||||
if expected_base_id and str(chunk.get("knowledge_base_id") or "") != expected_base_id:
|
||||
return None
|
||||
metadata = chunk.get("metadata") if isinstance(chunk.get("metadata"), dict) else {}
|
||||
return {
|
||||
"id": chunk_id,
|
||||
"chunk_id": chunk_id,
|
||||
"document_id": chunk.get("knowledge_id") or chunk.get("document_id"),
|
||||
"knowledge_id": chunk.get("knowledge_id") or chunk.get("document_id"),
|
||||
"title": chunk.get("knowledge_title") or chunk.get("title") or chunk.get("file_name") or chunk.get("filename"),
|
||||
"content": chunk.get("content") or "",
|
||||
"metadata": metadata,
|
||||
"url": metadata.get("url") or metadata.get("source_url"),
|
||||
"published_at": metadata.get("published_at") or metadata.get("publish_time") or metadata.get("date"),
|
||||
}
|
||||
|
||||
resolved = await asyncio.gather(*(resolve(base_id, chunk_id) for _, base_id, chunk_id in pending))
|
||||
for (page_index, _, _), item in zip(pending, resolved, strict=True):
|
||||
if item is not None:
|
||||
copied[page_index]["source_refs"].append(item)
|
||||
return copied, warnings
|
||||
|
||||
|
||||
def _page_reference(page: dict[str, Any], mapping_name: str) -> dict[str, Any]:
|
||||
published, date_source = extract_record_date(page)
|
||||
return {
|
||||
"id": str(page.get("id") or page.get("slug") or ""),
|
||||
"title": str(page.get("title") or page.get("slug") or "未命名 Wiki 页面"),
|
||||
"published_at": published.isoformat() if published else None,
|
||||
"date_source": date_source,
|
||||
"knowledge_base_id": str(page.get("knowledge_base_mapping_id") or ""),
|
||||
"knowledge_base_name": mapping_name,
|
||||
"wiki_page_id": page.get("id"),
|
||||
"wiki_slug": page.get("slug"),
|
||||
"summary": str(page.get("summary") or ""),
|
||||
"source_refs": page.get("source_refs") or [],
|
||||
"url": f"/weknora-source-preview?knowledge_base_id={page.get('knowledge_base_mapping_id') or ''}&wiki_slug={page.get('slug') or ''}",
|
||||
}
|
||||
|
||||
|
||||
def build_statistics(
|
||||
pages: list[dict[str, Any]],
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
subject_terms: list[str],
|
||||
date_start: date | None,
|
||||
date_end: date | None,
|
||||
requested: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
mapping_names = {str(item.get("id") or ""): str(item.get("name") or "") for item in mappings}
|
||||
direct_pages = [page for page in pages if not subject_terms or any(term.casefold() in _page_haystack(page) for term in subject_terms)]
|
||||
source_items: list[dict[str, Any]] = []
|
||||
for page in direct_pages:
|
||||
for ref in page.get("source_refs") or []:
|
||||
if not isinstance(ref, dict) or not _reference_identity(ref):
|
||||
continue
|
||||
published, date_source = extract_record_date(ref)
|
||||
source_items.append(
|
||||
{
|
||||
"id": _reference_identity(ref),
|
||||
"title": str(ref.get("title") or ref.get("name") or page.get("title") or "未命名来源"),
|
||||
"published_at": published.isoformat() if published else None,
|
||||
"date_source": date_source,
|
||||
"url": ref.get("url") or ref.get("source_url"),
|
||||
"document_id": ref.get("document_id") or ref.get("knowledge_id"),
|
||||
"chunk_id": ref.get("chunk_id") or ref.get("id"),
|
||||
"knowledge_base_id": page.get("knowledge_base_mapping_id"),
|
||||
"knowledge_base_name": mapping_names.get(str(page.get("knowledge_base_mapping_id") or ""), ""),
|
||||
"wiki_page_id": page.get("id"),
|
||||
"wiki_slug": page.get("slug"),
|
||||
"wiki_page_title": page.get("title"),
|
||||
}
|
||||
)
|
||||
unit = "source_document" if source_items else "wiki_page"
|
||||
raw_items = source_items or [
|
||||
_page_reference(page, mapping_names.get(str(page.get("knowledge_base_mapping_id") or ""), ""))
|
||||
for page in direct_pages
|
||||
]
|
||||
unique: dict[str, dict[str, Any]] = {}
|
||||
for item in raw_items:
|
||||
unique.setdefault(str(item.get("id") or f"{item.get('knowledge_base_id')}:{item.get('wiki_slug')}"), item)
|
||||
undated = 0
|
||||
matched: list[dict[str, Any]] = []
|
||||
for item in unique.values():
|
||||
published = _parse_date_value(item.get("published_at"), today=datetime.now(UTC).date())
|
||||
if (date_start or date_end) and published is None:
|
||||
undated += 1
|
||||
continue
|
||||
if date_start and published and published < date_start:
|
||||
continue
|
||||
if date_end and published and published > date_end:
|
||||
continue
|
||||
matched.append(item)
|
||||
matched.sort(key=lambda item: (str(item.get("published_at") or ""), str(item.get("title") or "")), reverse=True)
|
||||
return {
|
||||
"requested": bool(requested or date_start or date_end),
|
||||
"unit": unit,
|
||||
"unit_label": "新闻/来源" if unit == "source_document" else "Wiki 页面",
|
||||
"subject_terms": subject_terms,
|
||||
"date_from": date_start.isoformat() if date_start else None,
|
||||
"date_to": date_end.isoformat() if date_end else None,
|
||||
"matched_count": len(matched),
|
||||
"undated_excluded_count": undated,
|
||||
"items": matched,
|
||||
}
|
||||
|
||||
|
||||
def _merge_results(
|
||||
channels: list[tuple[str, list[dict[str, Any]]]],
|
||||
*,
|
||||
limit: int,
|
||||
title_query: str | None = None,
|
||||
title_boost_enabled: bool = True,
|
||||
title_boost_weight: float = 0.25,
|
||||
) -> list[dict[str, Any]]:
|
||||
fused: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
scores: dict[tuple[str, str], float] = {}
|
||||
matched_queries: dict[tuple[str, str], list[str]] = {}
|
||||
title_scores: dict[tuple[str, str], float] = {}
|
||||
for channel_index, (query, items) in enumerate(channels):
|
||||
weight = 2.0 if channel_index == 0 else 1.0
|
||||
for rank, item in enumerate(items, start=1):
|
||||
key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
|
||||
scores[key] = scores.get(key, 0.0) + weight / (60 + rank)
|
||||
title_scores[key] = max(
|
||||
title_scores.get(key, 0.0),
|
||||
title_match_strength(
|
||||
title_query if title_query is not None else query,
|
||||
item.get("title"),
|
||||
item.get("aliases") or [],
|
||||
),
|
||||
)
|
||||
matched_queries.setdefault(key, []).append(query)
|
||||
current = fused.get(key)
|
||||
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
|
||||
fused[key] = dict(item)
|
||||
ranking_scores = {
|
||||
key: title_boosted_rrf_score(
|
||||
scores[key],
|
||||
title_scores[key],
|
||||
enabled=title_boost_enabled,
|
||||
weight=title_boost_weight,
|
||||
)
|
||||
for key in fused
|
||||
}
|
||||
ordered = sorted(fused, key=lambda key: ranking_scores[key], reverse=True)[:limit]
|
||||
return [
|
||||
{
|
||||
**fused[key],
|
||||
"rrf_score": scores[key],
|
||||
"title_ranking_score": ranking_scores[key],
|
||||
"score": scores[key],
|
||||
"title_match_score": title_scores[key],
|
||||
"rank_score_semantics": "rrf",
|
||||
"matched_queries": list(dict.fromkeys(matched_queries[key])),
|
||||
}
|
||||
for key in ordered
|
||||
]
|
||||
|
||||
|
||||
async def advanced_search(
|
||||
*,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
retrieval_service: Any,
|
||||
index_store: Any,
|
||||
top_k: int = 20,
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
expand_entities: bool | None = None,
|
||||
include_drafts: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
normalized_query = query.strip()
|
||||
settings = resolved_retrieval_settings(mappings[0]) if len(mappings) == 1 else {
|
||||
"entity_expansion_enabled": True,
|
||||
"max_related_terms": 6,
|
||||
"statistics_enabled": True,
|
||||
}
|
||||
expansion_enabled = settings["entity_expansion_enabled"] if expand_entities is None else bool(expand_entities)
|
||||
subjects = extract_subject_terms(normalized_query)
|
||||
date_start, date_end = parse_date_range(normalized_query, date_from=date_from, date_to=date_to)
|
||||
mapping_ids = [str(item["id"]) for item in mappings]
|
||||
lexical_terms = subjects or [item for item in re.findall(r"[A-Za-z0-9_.-]{2,}|[\u4e00-\u9fff·]{2,20}", normalized_query) if item not in _QUERY_STOP_WORDS][:4]
|
||||
seed_pages = await index_store.search_pages_by_keywords(mapping_ids, lexical_terms, limit=2000, include_drafts=include_drafts)
|
||||
seed_pages = [item for item in seed_pages if is_wiki_body_page(item)]
|
||||
related = (
|
||||
expand_related_terms(subjects or lexical_terms, seed_pages, limit=settings["max_related_terms"])
|
||||
if expansion_enabled and not is_wiki_overview_query(normalized_query)
|
||||
else []
|
||||
)
|
||||
|
||||
queries = [normalized_query]
|
||||
anchor = " ".join(subjects)
|
||||
queries.extend(f"{anchor} {item['term']}".strip() for item in related)
|
||||
searches = await asyncio.gather(
|
||||
*(retrieval_service.search(item, mappings, top_k_pages=max(top_k, 20), include_drafts=include_drafts) for item in queries),
|
||||
return_exceptions=True,
|
||||
)
|
||||
channels: list[tuple[str, list[dict[str, Any]]]] = []
|
||||
warnings: list[str] = []
|
||||
modes: list[str] = []
|
||||
diagnostics: list[dict[str, Any]] = []
|
||||
for item_query, payload in zip(queries, searches, strict=True):
|
||||
if isinstance(payload, Exception):
|
||||
warnings.append(f"{item_query}:{payload}")
|
||||
continue
|
||||
channels.append((item_query, list(payload.get("results") or [])))
|
||||
modes.append(str(payload.get("mode") or ""))
|
||||
warnings.extend(str(item) for item in payload.get("warnings") or [])
|
||||
diagnostics.append({"query": item_query, **dict(payload.get("diagnostics") or {})})
|
||||
resolve_title_settings = getattr(retrieval_service, "_title_boost_settings", None)
|
||||
if callable(resolve_title_settings):
|
||||
title_boost_enabled, title_boost_weight = resolve_title_settings()
|
||||
else:
|
||||
vector_config = getattr(getattr(retrieval_service, "vector_search", None), "config", None)
|
||||
title_boost_enabled = bool(getattr(vector_config, "title_boost_enabled", True))
|
||||
title_boost_weight = float(getattr(vector_config, "title_boost_weight", 0.25))
|
||||
results = _merge_results(
|
||||
channels,
|
||||
limit=max(1, min(top_k, 100)),
|
||||
# Entity expansion queries add related terms for recall. They must
|
||||
# not turn a related entity's title into a direct match for the
|
||||
# user's original question (for example “特朗普” should not outrank
|
||||
# a page titled “海湖庄园” when the user asked for the latter).
|
||||
title_query=normalized_query,
|
||||
title_boost_enabled=title_boost_enabled,
|
||||
title_boost_weight=title_boost_weight,
|
||||
)
|
||||
enriched_pages, provenance_warnings = await enrich_chunk_references(
|
||||
seed_pages,
|
||||
mappings,
|
||||
client=getattr(retrieval_service, "client", None),
|
||||
)
|
||||
warnings.extend(provenance_warnings)
|
||||
stats = build_statistics(
|
||||
enriched_pages,
|
||||
mappings,
|
||||
subject_terms=subjects or lexical_terms,
|
||||
date_start=date_start,
|
||||
date_end=date_end,
|
||||
requested=bool(_STATISTICS_PATTERN.search(normalized_query)),
|
||||
) if settings["statistics_enabled"] else None
|
||||
references = [
|
||||
{
|
||||
"reference_id": f"wiki:{item.get('knowledge_base_id')}:{item.get('wiki_slug') or item.get('id')}",
|
||||
"title": item.get("title"),
|
||||
"knowledge_base_id": item.get("knowledge_base_id"),
|
||||
"knowledge_base_name": item.get("knowledge_base_name"),
|
||||
"wiki_page_id": item.get("wiki_page_id") or item.get("id"),
|
||||
"wiki_slug": item.get("wiki_slug"),
|
||||
"url": item.get("url"),
|
||||
"snippet": item.get("snippet") or str(item.get("content") or "")[:360],
|
||||
}
|
||||
for item in results
|
||||
]
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": "entity_expanded" if related else (modes[0] if modes else "unavailable"),
|
||||
"knowledge_base_count": len(mappings),
|
||||
"knowledge_base_profiles": [
|
||||
{"id": item.get("id"), "name": item.get("name"), "description": item.get("description") or ""}
|
||||
for item in mappings
|
||||
],
|
||||
"query_plan": {
|
||||
"subject_terms": subjects,
|
||||
"date_from": date_start.isoformat() if date_start else None,
|
||||
"date_to": date_end.isoformat() if date_end else None,
|
||||
"entity_expansion_enabled": expansion_enabled,
|
||||
"expanded_terms": related,
|
||||
"executed_queries": queries,
|
||||
},
|
||||
"statistics": stats,
|
||||
"results": results,
|
||||
"references": references,
|
||||
"partial": bool(warnings),
|
||||
"warnings": warnings,
|
||||
"diagnostics": diagnostics,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"advanced_search",
|
||||
"build_statistics",
|
||||
"expand_related_terms",
|
||||
"extract_record_date",
|
||||
"extract_subject_terms",
|
||||
"enrich_chunk_references",
|
||||
"parse_date_range",
|
||||
]
|
||||
+98
-14
@@ -57,6 +57,9 @@ class StrictWikiEmbeddingClient:
|
||||
self._local_started_at: str | None = None
|
||||
self._local_ready_at: str | None = None
|
||||
self._local_warmup_task: asyncio.Task[None] | None = None
|
||||
self._lifecycle_condition = asyncio.Condition()
|
||||
self._active_uses = 0
|
||||
self._retired = False
|
||||
|
||||
@property
|
||||
def dimensions(self) -> int | None:
|
||||
@@ -64,7 +67,21 @@ class StrictWikiEmbeddingClient:
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
raw = "\0".join((self.config.provider.strip().lower(), self.config.base_url.rstrip("/").lower(), self.config.model.strip(), str(self._resolved_dimensions or "auto")))
|
||||
parts = [
|
||||
self.config.provider.strip().lower(),
|
||||
self.config.base_url.rstrip("/").lower(),
|
||||
self.config.model.strip(),
|
||||
str(self._resolved_dimensions or "auto"),
|
||||
]
|
||||
if self.config.provider == "local":
|
||||
model_path = self._local_model_path()
|
||||
parts.extend(
|
||||
(
|
||||
str(model_path) if model_path is not None else "",
|
||||
self.config.local_model_revision,
|
||||
)
|
||||
)
|
||||
raw = "\0".join(parts)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
async def close(self) -> None:
|
||||
@@ -73,6 +90,15 @@ class StrictWikiEmbeddingClient:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def retire(self) -> None:
|
||||
"""Reject future work, drain active calls, then release model resources."""
|
||||
|
||||
async with self._lifecycle_condition:
|
||||
self._retired = True
|
||||
async with self._lifecycle_condition:
|
||||
await self._lifecycle_condition.wait_for(lambda: self._active_uses == 0)
|
||||
await self.close()
|
||||
|
||||
async def reset_local_encoder(self, *, model_path: str | None = None) -> None:
|
||||
"""Stop only the embedded worker and optionally switch its model path.
|
||||
|
||||
@@ -82,27 +108,80 @@ class StrictWikiEmbeddingClient:
|
||||
"""
|
||||
warmup = self._local_warmup_task
|
||||
if warmup is not None and not warmup.done():
|
||||
warmup.cancel()
|
||||
# Local inference runs in a worker thread. Cancelling the asyncio
|
||||
# wrapper does not stop that thread and could let it publish a new
|
||||
# process/model after cleanup. Drain it before touching resources.
|
||||
await asyncio.gather(warmup, return_exceptions=True)
|
||||
self._local_warmup_task = None
|
||||
process = self._local_process
|
||||
self._local_process = None
|
||||
if process is not None:
|
||||
await asyncio.to_thread(self._stop_local_process, process)
|
||||
self._local_model = None
|
||||
self._local_state = "idle"
|
||||
self._local_error = None
|
||||
self._local_started_at = None
|
||||
self._local_ready_at = None
|
||||
if model_path is not None:
|
||||
self.config = self.config.model_copy(update={"local_model_path": model_path})
|
||||
# Wait for a non-warmup embedding batch to finish before replacing or
|
||||
# stopping the local worker. This prevents an admin upload from
|
||||
# breaking a concurrent vectorization request halfway through IPC.
|
||||
async with self._local_lock:
|
||||
process = self._local_process
|
||||
self._local_process = None
|
||||
if process is not None:
|
||||
await self._run_thread_drained(self._stop_local_process, process)
|
||||
self._local_model = None
|
||||
self._local_state = "idle"
|
||||
self._local_error = None
|
||||
self._local_started_at = None
|
||||
self._local_ready_at = None
|
||||
if model_path is not None:
|
||||
self.config = self.config.model_copy(update={"local_model_path": model_path})
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
if self._client is None:
|
||||
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
|
||||
return self._client
|
||||
|
||||
@staticmethod
|
||||
async def _run_thread_drained(function, *args):
|
||||
"""Do not release lifecycle locks while cancelled thread work still runs."""
|
||||
|
||||
worker = asyncio.create_task(asyncio.to_thread(function, *args))
|
||||
cancelled = False
|
||||
worker_error: BaseException | None = None
|
||||
result = None
|
||||
while not worker.done():
|
||||
try:
|
||||
result = await asyncio.shield(worker)
|
||||
except asyncio.CancelledError:
|
||||
# A caller can be cancelled more than once during shutdown.
|
||||
# Swallow every cancellation until the non-cancellable thread
|
||||
# has really stopped, then restore cancellation semantics.
|
||||
cancelled = True
|
||||
except BaseException as exc:
|
||||
worker_error = exc
|
||||
break
|
||||
if worker.done() and worker_error is None:
|
||||
try:
|
||||
result = worker.result()
|
||||
except BaseException as exc:
|
||||
worker_error = exc
|
||||
if cancelled:
|
||||
raise asyncio.CancelledError from None
|
||||
if worker_error is not None:
|
||||
raise worker_error
|
||||
return result
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
async with self._lifecycle_condition:
|
||||
if self._retired:
|
||||
raise WikiEmbeddingError(
|
||||
"Wiki embedding runtime was replaced; retry with the active provider",
|
||||
category="runtime_retired",
|
||||
retryable=True,
|
||||
)
|
||||
self._active_uses += 1
|
||||
try:
|
||||
return await self._embed_texts_active(texts)
|
||||
finally:
|
||||
async with self._lifecycle_condition:
|
||||
self._active_uses -= 1
|
||||
if self._active_uses == 0:
|
||||
self._lifecycle_condition.notify_all()
|
||||
|
||||
async def _embed_texts_active(self, texts: list[str]) -> list[list[float]]:
|
||||
if self.config.provider == "local":
|
||||
if not texts:
|
||||
return []
|
||||
@@ -110,7 +189,10 @@ class StrictWikiEmbeddingClient:
|
||||
self._local_state = "starting"
|
||||
self._local_started_at = self._local_started_at or datetime.now(UTC).isoformat()
|
||||
try:
|
||||
values = await asyncio.to_thread(self._embed_local, texts)
|
||||
# `to_thread` keeps running after its awaiting task is
|
||||
# cancelled. Keep the lifecycle lease and local lock until
|
||||
# the real inference call has stopped.
|
||||
values = await self._run_thread_drained(self._embed_local, texts)
|
||||
except Exception as exc:
|
||||
self._local_state = "failed"
|
||||
self._local_error = " ".join(str(exc).split())[:1000] or type(exc).__name__
|
||||
@@ -452,6 +534,8 @@ class StrictWikiEmbeddingClient:
|
||||
|
||||
def start_local_encoder(self) -> dict[str, object]:
|
||||
"""Warm the embedded CPU encoder without coupling it to app startup."""
|
||||
if self._retired:
|
||||
raise ValueError("当前本地编码器运行时已切换,请刷新状态后重试")
|
||||
if self.config.provider != "local":
|
||||
raise ValueError("当前 embedding provider 不是 local")
|
||||
if not self.config.enabled:
|
||||
|
||||
+55
@@ -9,6 +9,23 @@ from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
_WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#[^\]|]+)?(?:\|[^\]]+)?\]\]")
|
||||
_LINK_REFERENCE_KEYS = (
|
||||
"slug",
|
||||
"wiki_slug",
|
||||
"path",
|
||||
"target",
|
||||
"source",
|
||||
"target_slug",
|
||||
"source_slug",
|
||||
"to",
|
||||
"from",
|
||||
"page_slug",
|
||||
"id",
|
||||
"title",
|
||||
"name",
|
||||
)
|
||||
|
||||
|
||||
def normalize_slug(value: Any) -> str:
|
||||
slug = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
|
||||
@@ -44,6 +61,44 @@ def _datetime(value: Any) -> datetime | None:
|
||||
return None
|
||||
|
||||
|
||||
def normalize_wiki_link_reference(value: Any) -> str | None:
|
||||
"""Normalize a provider link endpoint without assuming it is a slug yet."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
value = next(
|
||||
(value.get(key) for key in _LINK_REFERENCE_KEYS if value.get(key) is not None and str(value.get(key)).strip()),
|
||||
None,
|
||||
)
|
||||
raw = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
|
||||
if not raw:
|
||||
return None
|
||||
raw = raw.split("#", 1)[0].strip().strip("/")
|
||||
raw = re.sub(r"/{2,}", "/", raw)
|
||||
return raw or None
|
||||
|
||||
|
||||
def normalize_wiki_page_links(remote: dict[str, Any]) -> dict[str, list[str]]:
|
||||
"""Return deterministic incoming/outgoing references from a full page.
|
||||
|
||||
WeKnora versions have returned both snake/camel-case fields and either
|
||||
strings or small endpoint objects. Markdown Wiki links supplement explicit
|
||||
``out_links`` so a complete page scan remains authoritative even when the
|
||||
provider's overview graph is capped.
|
||||
"""
|
||||
|
||||
def _references(value: Any) -> set[str]:
|
||||
return {reference for item in _list(value) for reference in [normalize_wiki_link_reference(item)] if reference}
|
||||
|
||||
incoming = _references(remote.get("in_links")) | _references(remote.get("inLinks"))
|
||||
outgoing = _references(remote.get("out_links")) | _references(remote.get("outLinks"))
|
||||
content = str(remote.get("content_md") or remote.get("content") or remote.get("markdown") or "")
|
||||
outgoing.update(reference for match in _WIKILINK_RE.finditer(content) for reference in [normalize_wiki_link_reference(match.group(1))] if reference)
|
||||
return {
|
||||
"in_links": sorted(incoming, key=str.casefold),
|
||||
"out_links": sorted(outgoing, key=str.casefold),
|
||||
}
|
||||
|
||||
|
||||
def normalize_wiki_page(mapping_id: str, remote: dict[str, Any]) -> dict[str, Any]:
|
||||
slug = normalize_slug(remote.get("slug") or remote.get("wiki_slug") or remote.get("path"))
|
||||
aliases = sorted({str(item).strip() for item in _list(remote.get("aliases")) if str(item).strip()})
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
"""Shared intent and page eligibility rules for Wiki-library overviews."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
_OVERVIEW_PATTERNS = (
|
||||
re.compile(
|
||||
r"(?:(?:这个|这座|此|该|本)?(?:知识库|知识空间|资料库|wiki)|(?:这个|此|该|本)库).{0,12}"
|
||||
r"(?:讲了?什么|有什么(?:内容|资料|主题)?|有哪些(?:内容|资料|主题)?|包含什么|"
|
||||
r"内容?(?:概览|介绍)|(?:总体|整体)?(?:介绍|概览)|简介)",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
re.compile(r"^(?:概览|知识库概览|知识空间概览|库概览|介绍(?:一下)?(?:这个|此|该)?(?:知识库|知识空间|库)?)$", re.IGNORECASE),
|
||||
re.compile(
|
||||
r"(?:what(?:'s| is| does).{0,24}(?:knowledge base|wiki)|"
|
||||
r"(?:knowledge base|wiki).{0,24}(?:overview|contents?|cover|about)|"
|
||||
r"overview of (?:this |the )?(?:knowledge base|wiki))",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
)
|
||||
|
||||
_NON_BODY_PAGE_TYPES = frozenset({"directory", "index", "summary"})
|
||||
_NON_BODY_SLUG_SEGMENTS = frozenset({"directory", "index", "summary", "目录", "索引", "摘要"})
|
||||
_NON_BODY_TITLES = frozenset({"directory", "index", "summary", "目录", "索引", "摘要"})
|
||||
|
||||
|
||||
def is_wiki_overview_query(query: str) -> bool:
|
||||
"""Whether a request asks for a library-level orientation, not a fact lookup."""
|
||||
|
||||
normalized = " ".join(query.strip().split())
|
||||
return bool(normalized) and any(pattern.search(normalized) for pattern in _OVERVIEW_PATTERNS)
|
||||
|
||||
|
||||
def is_wiki_body_page(page: dict[str, Any]) -> bool:
|
||||
"""Reject generated directory/index/summary shells from retrieval candidates."""
|
||||
|
||||
page_type = str(page.get("page_type") or page.get("type") or "article").strip().lower()
|
||||
if page_type in _NON_BODY_PAGE_TYPES:
|
||||
return False
|
||||
slug = str(page.get("slug") or page.get("wiki_slug") or page.get("path") or "").strip("/ ").lower()
|
||||
if any(segment in _NON_BODY_SLUG_SEGMENTS for segment in slug.split("/")):
|
||||
return False
|
||||
title = str(page.get("title") or "").strip().lower()
|
||||
return title not in _NON_BODY_TITLES
|
||||
|
||||
|
||||
def wiki_body_excerpt(page: dict[str, Any], *, max_chars: int = 1600) -> str:
|
||||
"""Return a bounded real-body preview for an overview response."""
|
||||
|
||||
content = str(page.get("content_md") or page.get("content") or "").strip()
|
||||
if len(content) <= max_chars:
|
||||
return content
|
||||
return f"{content[: max_chars - 1].rstrip()}…"
|
||||
|
||||
|
||||
__all__ = ["is_wiki_body_page", "is_wiki_overview_query", "wiki_body_excerpt"]
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
"""Small, deterministic ranking helpers shared by Wiki retrieval channels.
|
||||
|
||||
The local Wiki index has several independent ranking paths (dense vectors,
|
||||
keyword fallback, remote Wiki pages and raw document chunks). Keeping the
|
||||
title signal in one helper makes it possible for the final channel fusion to
|
||||
apply the same bias everywhere. This is intentionally not a tokenizer or an
|
||||
LLM: it only detects a bounded, meaningful phrase from the user's query.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
# Natural-language wrappers which should not make a page title look relevant.
|
||||
# The list is deliberately conservative; unknown words are retained as query
|
||||
# terms rather than silently discarded.
|
||||
_QUERY_NOISE = (
|
||||
"知识空间",
|
||||
"知识库",
|
||||
"相关资料",
|
||||
"相关文档",
|
||||
"相关报告",
|
||||
"相关信息",
|
||||
"相关新闻",
|
||||
"告诉我",
|
||||
"我想知道",
|
||||
"请问",
|
||||
"请帮我",
|
||||
"帮我",
|
||||
"搜索",
|
||||
"查询",
|
||||
"查找",
|
||||
"检索",
|
||||
"查阅",
|
||||
"介绍一下",
|
||||
"介绍",
|
||||
"解释一下",
|
||||
"解释",
|
||||
"有关",
|
||||
"相关",
|
||||
"请",
|
||||
)
|
||||
_QUERY_SUFFIXES = ("是什么", "是什麽", "怎么样", "如何", "怎么", "吗", "呢")
|
||||
_ASCII_STOP_WORDS = {
|
||||
"a",
|
||||
"an",
|
||||
"and",
|
||||
"are",
|
||||
"about",
|
||||
"can",
|
||||
"could",
|
||||
"for",
|
||||
"how",
|
||||
"is",
|
||||
"the",
|
||||
"what",
|
||||
"where",
|
||||
"which",
|
||||
"who",
|
||||
"with",
|
||||
}
|
||||
|
||||
|
||||
def _compact(value: object) -> str:
|
||||
"""Normalize text for phrase containment while retaining CJK/Latin chars."""
|
||||
|
||||
normalized = unicodedata.normalize("NFKC", str(value or "")).casefold()
|
||||
# ``\w`` includes Chinese characters in Python's Unicode mode. Removing
|
||||
# punctuation and whitespace makes ``Mar-a-Lago`` match ``maralago`` while
|
||||
# leaving ordinary Chinese title phrases intact.
|
||||
return re.sub(r"[^\w\u3400-\u9fff]+", "", normalized)
|
||||
|
||||
|
||||
def _query_terms(query: str) -> list[str]:
|
||||
"""Return bounded title-oriented phrases from a natural-language query."""
|
||||
|
||||
raw = unicodedata.normalize("NFKC", str(query or "")).casefold().strip()
|
||||
if not raw:
|
||||
return []
|
||||
terms: list[str] = []
|
||||
|
||||
def add(value: str) -> None:
|
||||
compact = _compact(value)
|
||||
if len(compact) < 2 or compact in _ASCII_STOP_WORDS or compact in terms:
|
||||
return
|
||||
# Two-character CJK fragments are useful as a fallback, but we avoid
|
||||
# treating a lone one-character fragment as a title match.
|
||||
terms.append(compact)
|
||||
|
||||
# Quoted phrases are the strongest user signal.
|
||||
for value in re.findall(r"[「『\"']([^」』\"']{2,60})[」』\"']", raw):
|
||||
add(value)
|
||||
|
||||
cleaned = raw
|
||||
for noise in sorted(_QUERY_NOISE, key=len, reverse=True):
|
||||
cleaned = cleaned.replace(noise, " ")
|
||||
# A possessive/conjunctive ``的`` often separates the subject from the
|
||||
# question (``海湖庄园的历史``). Keep both the complete run and each side;
|
||||
# do not delete it globally because it can be part of a real title.
|
||||
for suffix in _QUERY_SUFFIXES:
|
||||
cleaned = cleaned.replace(suffix, " ")
|
||||
cleaned = re.sub(r"[,。;;?!?::,、()()\[\]{}<>]", " ", cleaned)
|
||||
|
||||
for token in re.findall(r"[a-z0-9][a-z0-9_.:/+\-]*", cleaned):
|
||||
add(token)
|
||||
for run in re.findall(r"[\u3400-\u4dbf\u4e00-\u9fff]+", cleaned):
|
||||
add(run)
|
||||
parts = [part for part in re.split(r"的|与|和|或|及|并", run) if part]
|
||||
for part in parts:
|
||||
add(part)
|
||||
# Keep a small number of long contiguous phrases. The longest match
|
||||
# wins later, so this handles a phrase embedded in a longer question
|
||||
# without giving every two-character n-gram a large score.
|
||||
if len(run) > 8:
|
||||
for width in (8, 6, 4, 3):
|
||||
for start in range(0, len(run) - width + 1):
|
||||
add(run[start : start + width])
|
||||
|
||||
# Longer phrases are evaluated first and cap work for pathological input.
|
||||
return sorted(terms, key=lambda value: (-len(value), value))[:24]
|
||||
|
||||
|
||||
def _title_match_strength_from_terms(
|
||||
terms: list[str],
|
||||
title: object,
|
||||
aliases: Iterable[object] | None = None,
|
||||
) -> float:
|
||||
title_value = _compact(title)
|
||||
if not title_value:
|
||||
return 0.0
|
||||
if not terms:
|
||||
return 0.0
|
||||
aliases_compact = [_compact(item) for item in aliases or () if _compact(item)]
|
||||
best = 0.0
|
||||
for term in terms:
|
||||
if len(term) < 3 and not re.fullmatch(r"[a-z0-9][a-z0-9_.:/+\-]*", term):
|
||||
continue
|
||||
for candidate, multiplier in ((title_value, 1.0), *[(item, 0.86) for item in aliases_compact]):
|
||||
if not candidate:
|
||||
continue
|
||||
if term == candidate:
|
||||
ratio = 1.0
|
||||
elif term in candidate:
|
||||
# The user supplied a concise title phrase while the stored
|
||||
# title carries a qualifier, such as ``海湖庄园(美国)``.
|
||||
ratio = min(1.0, len(term) / max(len(candidate), 1))
|
||||
elif candidate in term and len(candidate) >= 3:
|
||||
# Natural-language questions often keep the title inside a
|
||||
# longer run (``海湖庄园最新消息``). Treat that as a title hit
|
||||
# without requiring a fragile list of every question suffix.
|
||||
ratio = min(1.0, len(candidate) / max(len(term), 1))
|
||||
else:
|
||||
continue
|
||||
value = (0.55 + 0.45 * ratio) * multiplier
|
||||
best = max(best, value)
|
||||
return min(1.0, best)
|
||||
|
||||
|
||||
def make_title_matcher(
|
||||
query: str,
|
||||
) -> Callable[[object, Iterable[object] | None], float]:
|
||||
"""Compile query phrases once for repeated title checks in large indexes."""
|
||||
|
||||
terms = _query_terms(query)
|
||||
|
||||
def match(title: object, aliases: Iterable[object] | None = None) -> float:
|
||||
return _title_match_strength_from_terms(terms, title, aliases)
|
||||
|
||||
return match
|
||||
|
||||
|
||||
def title_match_strength(
|
||||
query: str,
|
||||
title: object,
|
||||
aliases: Iterable[object] | None = None,
|
||||
) -> float:
|
||||
"""Return a bounded title relevance score in ``[0, 1]``.
|
||||
|
||||
``1`` means the most useful case for retrieval: the extracted query phrase
|
||||
is exactly the title (or a long phrase is fully contained by the title).
|
||||
A phrase occurring in an alias is intentionally discounted. The function
|
||||
is cheap enough to run while fusing already-bounded result lists.
|
||||
"""
|
||||
|
||||
return make_title_matcher(query)(title, aliases)
|
||||
|
||||
|
||||
def title_boosted_rrf_score(
|
||||
rrf_score: float,
|
||||
title_score: float,
|
||||
*,
|
||||
enabled: bool = True,
|
||||
weight: float = 0.25,
|
||||
) -> float:
|
||||
"""Apply a bounded multiplicative title boost to an RRF score.
|
||||
|
||||
A multiplier keeps the signal in the RRF score domain: an exact title hit
|
||||
receives a 25% lift at the default weight, but it does not unconditionally
|
||||
beat an item supported by substantially more retrieval channels.
|
||||
"""
|
||||
|
||||
base = float(rrf_score)
|
||||
if not enabled:
|
||||
return base
|
||||
strength = min(1.0, max(0.0, float(title_score)))
|
||||
safe_weight = max(0.0, float(weight))
|
||||
return base * (1.0 + safe_weight * strength)
|
||||
|
||||
|
||||
__all__ = ["make_title_matcher", "title_boosted_rrf_score", "title_match_strength"]
|
||||
+387
-46
@@ -8,31 +8,221 @@ Wiki pages even while a whole knowledge base is awaiting vectorization.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.overview import is_wiki_body_page, is_wiki_overview_query, wiki_body_excerpt
|
||||
from deerflow.integrations.weknora.local_index.ranking import (
|
||||
title_boosted_rrf_score,
|
||||
title_match_strength,
|
||||
)
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
_GENERIC_QUERY_TERMS = {
|
||||
"知识库",
|
||||
"知识空间",
|
||||
"这个",
|
||||
"此知识库",
|
||||
"内容",
|
||||
"哪些",
|
||||
"什么",
|
||||
"怎么",
|
||||
"如何",
|
||||
"请问",
|
||||
"一下",
|
||||
}
|
||||
|
||||
# The keyword channel is deliberately a small, database-portable fallback and
|
||||
# currently uses substring predicates rather than a language-aware FTS index.
|
||||
# Keep function words out of those predicates: otherwise an unrelated page
|
||||
# containing e.g. ``is`` or ``on`` inside an English fragment is enough to
|
||||
# survive a failed vector threshold and become the final answer source.
|
||||
_ENGLISH_STOP_WORDS = {
|
||||
"an",
|
||||
"and",
|
||||
"are",
|
||||
"as",
|
||||
"at",
|
||||
"be",
|
||||
"been",
|
||||
"but",
|
||||
"by",
|
||||
"can",
|
||||
"could",
|
||||
"did",
|
||||
"do",
|
||||
"does",
|
||||
"for",
|
||||
"from",
|
||||
"had",
|
||||
"has",
|
||||
"have",
|
||||
"how",
|
||||
"if",
|
||||
"in",
|
||||
"into",
|
||||
"is",
|
||||
"it",
|
||||
"its",
|
||||
"may",
|
||||
"might",
|
||||
"of",
|
||||
"on",
|
||||
"or",
|
||||
"our",
|
||||
"should",
|
||||
"than",
|
||||
"that",
|
||||
"the",
|
||||
"their",
|
||||
"then",
|
||||
"there",
|
||||
"these",
|
||||
"they",
|
||||
"this",
|
||||
"those",
|
||||
"to",
|
||||
"was",
|
||||
"were",
|
||||
"what",
|
||||
"when",
|
||||
"where",
|
||||
"which",
|
||||
"who",
|
||||
"why",
|
||||
"will",
|
||||
"with",
|
||||
"would",
|
||||
"you",
|
||||
"your",
|
||||
}
|
||||
|
||||
_ASCII_KEYWORD = re.compile(r"[a-z0-9_.-]+")
|
||||
|
||||
|
||||
def _keyword_tokens(query: str) -> list[str]:
|
||||
"""Build bounded Latin terms and CJK sub-phrases for lexical recall."""
|
||||
|
||||
tokens: list[str] = []
|
||||
for part in re.findall(r"[a-zA-Z0-9_.-]{2,}|[\u4e00-\u9fff]{2,}", query.lower()):
|
||||
if re.fullmatch(r"[\u4e00-\u9fff]+", part):
|
||||
for generic in sorted(_GENERIC_QUERY_TERMS, key=len, reverse=True):
|
||||
part = part.replace(generic, "")
|
||||
if part and part not in _GENERIC_QUERY_TERMS and part not in _ENGLISH_STOP_WORDS:
|
||||
tokens.append(part)
|
||||
if part and re.fullmatch(r"[\u4e00-\u9fff]+", part) and len(part) > 3:
|
||||
for width in (3, 2):
|
||||
tokens.extend(part[index : index + width] for index in range(len(part) - width + 1))
|
||||
return list(
|
||||
dict.fromkeys(
|
||||
token
|
||||
for token in tokens
|
||||
if token not in _GENERIC_QUERY_TERMS and token not in _ENGLISH_STOP_WORDS
|
||||
)
|
||||
)[:32]
|
||||
|
||||
|
||||
def _contains_keyword(haystack: str, token: str) -> bool:
|
||||
"""Match Latin/code tokens on word boundaries and CJK tokens by substring."""
|
||||
|
||||
if _ASCII_KEYWORD.fullmatch(token):
|
||||
# ``\b`` is not suitable here because underscores count as word
|
||||
# characters and codes such as ``ZXQ-417`` contain punctuation. The
|
||||
# explicit alphanumeric boundaries retain standalone two-letter terms
|
||||
# (for example AI) and exact equipment codes without matching ``is``
|
||||
# inside ``this`` or ``on`` inside ``controller``.
|
||||
return re.search(rf"(?<![a-z0-9]){re.escape(token)}(?![a-z0-9])", haystack) is not None
|
||||
return token in haystack
|
||||
|
||||
|
||||
def _minimum_keyword_hits(tokens: list[str]) -> int:
|
||||
"""Require corroboration for long, purely Latin natural-language queries."""
|
||||
|
||||
# A precise equipment/code token remains sufficient even when the caller
|
||||
# also includes natural-language words (for example ``ZXQ-417 reset
|
||||
# procedure``). Requiring two matches in that case would remove the most
|
||||
# useful lexical fallback when vectors are unavailable or below threshold.
|
||||
if any(any(char.isdigit() for char in token) or any(char in "_.-" for char in token) for token in tokens):
|
||||
return 1
|
||||
if len(tokens) >= 3 and all(_ASCII_KEYWORD.fullmatch(token) for token in tokens):
|
||||
return 2
|
||||
return 1
|
||||
|
||||
|
||||
def _rrf_merge(
|
||||
channels: list[list[dict[str, Any]]],
|
||||
*,
|
||||
limit: int,
|
||||
title_boost_enabled: bool = True,
|
||||
title_boost_weight: float = 0.25,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Fuse independently ranked channels without comparing raw score scales."""
|
||||
|
||||
fused: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
contributions: dict[tuple[str, str], float] = {}
|
||||
modes: dict[tuple[str, str], set[str]] = {}
|
||||
title_scores: dict[tuple[str, str], float] = {}
|
||||
for channel in channels:
|
||||
for rank, item in enumerate(channel, start=1):
|
||||
key = (
|
||||
str(item.get("knowledge_base_id") or ""),
|
||||
str(item.get("wiki_slug") or item.get("id") or ""),
|
||||
)
|
||||
contributions[key] = contributions.get(key, 0.0) + 1.0 / (60 + rank)
|
||||
title_scores[key] = max(title_scores.get(key, 0.0), float(item.get("title_match_score") or 0))
|
||||
modes.setdefault(key, set()).add(str(item.get("retrieval_mode") or "vector"))
|
||||
current = fused.get(key)
|
||||
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
|
||||
fused[key] = dict(item)
|
||||
ranking_scores = {
|
||||
key: title_boosted_rrf_score(
|
||||
contributions[key],
|
||||
title_scores[key],
|
||||
enabled=title_boost_enabled,
|
||||
weight=title_boost_weight,
|
||||
)
|
||||
for key in fused
|
||||
}
|
||||
ranked = sorted(fused, key=lambda key: ranking_scores[key], reverse=True)
|
||||
results = []
|
||||
for key in ranked[:limit]:
|
||||
item = fused[key]
|
||||
item["channel_score"] = float(item.get("score") or 0)
|
||||
item["title_match_score"] = title_scores[key]
|
||||
item["rrf_score"] = contributions[key]
|
||||
item["title_ranking_score"] = ranking_scores[key]
|
||||
item["score"] = contributions[key]
|
||||
item["rank_score_semantics"] = "rrf"
|
||||
item["retrieval_mode"] = "hybrid" if len(modes[key]) > 1 else next(iter(modes[key]))
|
||||
results.append(item)
|
||||
return results
|
||||
|
||||
|
||||
def is_fully_vectorized(state: dict[str, Any] | None, fingerprint: str | None) -> bool:
|
||||
"""Return whether a completed full scan produced one coherent active index."""
|
||||
"""Return whether a coherent last-successful vector snapshot can serve.
|
||||
|
||||
A new incremental attempt may be running, cancelled or failed while every
|
||||
page from the last successful snapshot remains readable. Do not take that
|
||||
snapshot offline merely because the latest attempt did not finish.
|
||||
"""
|
||||
|
||||
if not state or not fingerprint:
|
||||
return False
|
||||
if str(state.get("state") or "") != "idle" or not state.get("last_success_at"):
|
||||
if str(state.get("state") or "") in {"rebuild_required", "rebuilding", "not_synced"} or not state.get("last_success_at"):
|
||||
return False
|
||||
if str(state.get("embedding_fingerprint") or "") != fingerprint:
|
||||
return False
|
||||
local_count = int(state.get("local_page_count") or 0)
|
||||
ready_count = int(state.get("ready_page_count") or 0)
|
||||
failed_count = int(state.get("failed_page_count") or 0)
|
||||
vector_count = int(state.get("vector_count") or 0)
|
||||
if failed_count or ready_count != local_count:
|
||||
return False
|
||||
return local_count == 0 or vector_count > 0
|
||||
# ``failed_page_count`` includes a retained ready page with a newer failed
|
||||
# attempt because ``last_error`` is diagnostic. Its atomically committed
|
||||
# older vectors remain a valid serving snapshot. New/unindexed pages are
|
||||
# still covered by the lexical Wiki channel.
|
||||
return ready_count > 0 and vector_count > 0
|
||||
|
||||
|
||||
class WikiRetrievalService:
|
||||
@@ -44,10 +234,25 @@ class WikiRetrievalService:
|
||||
*,
|
||||
vector_search: WikiVectorSearchService | None,
|
||||
client: WeKnoraClient | None,
|
||||
title_boost_enabled: bool | None = None,
|
||||
title_boost_weight: float | None = None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.vector_search = vector_search
|
||||
self.client = client
|
||||
self._title_boost_enabled = title_boost_enabled
|
||||
self._title_boost_weight = title_boost_weight
|
||||
|
||||
def _title_boost_settings(self) -> tuple[bool, float]:
|
||||
if self._title_boost_enabled is not None or self._title_boost_weight is not None:
|
||||
return (
|
||||
True if self._title_boost_enabled is None else bool(self._title_boost_enabled),
|
||||
0.25 if self._title_boost_weight is None else float(self._title_boost_weight),
|
||||
)
|
||||
config = getattr(self.vector_search, "config", None)
|
||||
if config is None:
|
||||
return True, 0.25
|
||||
return bool(config.title_boost_enabled), float(config.title_boost_weight)
|
||||
|
||||
async def search(
|
||||
self,
|
||||
@@ -71,11 +276,15 @@ class WikiRetrievalService:
|
||||
"fallback_knowledge_base_ids": [],
|
||||
}
|
||||
|
||||
overview = is_wiki_overview_query(normalized_query)
|
||||
vector_mappings, fallback_mappings = await self._partition_mappings(mappings)
|
||||
results: list[dict[str, Any]] = []
|
||||
vector_results: list[dict[str, Any]] = []
|
||||
keyword_results: list[dict[str, Any]] = []
|
||||
remote_results: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
vector_used = False
|
||||
fallback_used = False
|
||||
vector_diagnostics: dict[str, Any] = {}
|
||||
|
||||
if vector_mappings and self.vector_search is not None:
|
||||
try:
|
||||
@@ -85,8 +294,9 @@ class WikiRetrievalService:
|
||||
top_k_pages=max(top_k_pages, len(vector_mappings)),
|
||||
include_drafts=include_drafts,
|
||||
)
|
||||
results.extend(payload.get("results") or [])
|
||||
vector_results.extend(payload.get("results") or [])
|
||||
warnings.extend(str(item) for item in payload.get("warnings") or [])
|
||||
vector_diagnostics = dict(payload.get("diagnostics") or {})
|
||||
vector_used = True
|
||||
except WikiIndexError as exc:
|
||||
# A status row can become stale between classification and
|
||||
@@ -95,6 +305,14 @@ class WikiRetrievalService:
|
||||
fallback_mappings.extend(vector_mappings)
|
||||
vector_mappings = []
|
||||
|
||||
if not overview:
|
||||
keyword_results = await self._search_local_keywords(
|
||||
normalized_query,
|
||||
mappings,
|
||||
limit=max(top_k_pages * 4, 20),
|
||||
include_drafts=include_drafts,
|
||||
)
|
||||
|
||||
fallback_ids = [str(mapping["id"]) for mapping in fallback_mappings]
|
||||
if fallback_mappings:
|
||||
if self.client is None:
|
||||
@@ -107,6 +325,7 @@ class WikiRetrievalService:
|
||||
mapping,
|
||||
limit=max(top_k_pages, 1),
|
||||
include_drafts=include_drafts,
|
||||
overview=overview,
|
||||
)
|
||||
for mapping in fallback_mappings
|
||||
),
|
||||
@@ -116,53 +335,162 @@ class WikiRetrievalService:
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping['id']}:WIKI_API_SEARCH_FAILED")
|
||||
else:
|
||||
results.extend(batch)
|
||||
remote_results.extend(batch)
|
||||
fallback_used = True
|
||||
|
||||
if warnings and not results and not vector_used and not fallback_used:
|
||||
if warnings and not vector_results and not keyword_results and not remote_results and not vector_used and not fallback_used:
|
||||
raise WikiIndexError(
|
||||
"WIKI_API_UNAVAILABLE",
|
||||
"Wiki page retrieval is unavailable",
|
||||
status_code=502 if self.client is not None else 503,
|
||||
)
|
||||
|
||||
# De-duplicate by the public Wiki citation identity. Vector scores and
|
||||
# Wiki-API rank scores are both descending relevance hints.
|
||||
unique: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for item in results:
|
||||
key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
|
||||
current = unique.get(key)
|
||||
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
|
||||
unique[key] = item
|
||||
ranked = sorted(unique.values(), key=lambda item: float(item.get("score") or 0), reverse=True)
|
||||
mode = "mixed" if vector_used and fallback_used else "vector" if vector_used else "wiki_api"
|
||||
title_boost_enabled, title_boost_weight = self._title_boost_settings()
|
||||
ranked = _rrf_merge(
|
||||
[channel for channel in (vector_results, keyword_results, remote_results) if channel],
|
||||
limit=top_k_pages,
|
||||
title_boost_enabled=title_boost_enabled,
|
||||
title_boost_weight=title_boost_weight,
|
||||
)
|
||||
if overview:
|
||||
mode = "overview"
|
||||
elif vector_results and keyword_results:
|
||||
mode = "hybrid"
|
||||
elif keyword_results:
|
||||
mode = "keyword"
|
||||
elif vector_used and fallback_used:
|
||||
mode = "mixed"
|
||||
elif vector_used:
|
||||
mode = "vector"
|
||||
else:
|
||||
mode = "wiki_api"
|
||||
fallback_reason = None
|
||||
if keyword_results and not vector_results:
|
||||
raw_top = vector_diagnostics.get("top_raw_vector_score")
|
||||
fallback_reason = "VECTOR_BELOW_THRESHOLD" if raw_top is not None else "VECTOR_UNAVAILABLE"
|
||||
fingerprint = self.vector_search.embedding.fingerprint[:12] if vector_used and self.vector_search is not None else None
|
||||
vector_ids = {str(mapping["id"]) for mapping in vector_mappings}
|
||||
vector_states = {
|
||||
str(state.get("knowledge_base_mapping_id") or ""): str(state.get("state") or "")
|
||||
for state in await self.store.list_index_status(list(vector_ids))
|
||||
}
|
||||
stale_serving_ids = sorted(
|
||||
mapping_id
|
||||
for mapping_id in vector_ids
|
||||
if vector_states.get(mapping_id) not in {None, "", "idle"}
|
||||
)
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": mode,
|
||||
"results": ranked[:top_k_pages],
|
||||
"results": ranked,
|
||||
"knowledge_base_count": len(mappings),
|
||||
"index_fingerprint": fingerprint,
|
||||
"partial": bool(warnings),
|
||||
"warnings": warnings,
|
||||
"fallback_knowledge_base_ids": fallback_ids,
|
||||
"diagnostics": {
|
||||
**vector_diagnostics,
|
||||
"vector_candidates": len(vector_results),
|
||||
"keyword_candidates": len(keyword_results),
|
||||
"wiki_api_candidates": len(remote_results),
|
||||
"fallback_reason": fallback_reason,
|
||||
"stale_serving_knowledge_base_ids": stale_serving_ids,
|
||||
},
|
||||
}
|
||||
|
||||
async def _search_local_keywords(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
limit: int,
|
||||
include_drafts: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
tokens = _keyword_tokens(query)
|
||||
if not tokens:
|
||||
return []
|
||||
search_pages = getattr(self.store, "search_pages_by_keywords", None)
|
||||
if search_pages is None:
|
||||
return []
|
||||
pages = await search_pages(
|
||||
[str(mapping["id"]) for mapping in mappings],
|
||||
tokens,
|
||||
limit=limit,
|
||||
include_drafts=include_drafts,
|
||||
)
|
||||
mapping_by_id = {str(mapping["id"]): mapping for mapping in mappings}
|
||||
title_boost_enabled, title_boost_weight = self._title_boost_settings()
|
||||
title_hit_weight = 3.0 + (20.0 * title_boost_weight if title_boost_enabled else 0.0)
|
||||
results: list[dict[str, Any]] = []
|
||||
for page in pages:
|
||||
if not is_wiki_body_page(page):
|
||||
continue
|
||||
content = str(page.get("content_md") or "").strip()
|
||||
if not content:
|
||||
continue
|
||||
mapping_id = str(page.get("knowledge_base_mapping_id") or "")
|
||||
mapping = mapping_by_id.get(mapping_id)
|
||||
if mapping is None:
|
||||
continue
|
||||
title = str(page.get("title") or page.get("slug") or "")
|
||||
aliases = " ".join(str(item) for item in page.get("aliases") or [])
|
||||
title_haystack = f"{title} {aliases}".lower()
|
||||
summary_haystack = str(page.get("summary") or "").lower()
|
||||
content_haystack = content.lower()
|
||||
title_matches = {token for token in tokens if _contains_keyword(title_haystack, token)}
|
||||
summary_matches = {token for token in tokens if _contains_keyword(summary_haystack, token)}
|
||||
content_matches = {token for token in tokens if _contains_keyword(content_haystack, token)}
|
||||
if len(title_matches | summary_matches | content_matches) < _minimum_keyword_hits(tokens):
|
||||
continue
|
||||
title_hits = len(title_matches)
|
||||
summary_hits = len(summary_matches)
|
||||
content_hits = len(content_matches)
|
||||
score = (title_hit_weight * title_hits + 2 * summary_hits + content_hits) / max(len(tokens), 1)
|
||||
title_match_score = title_match_strength(query, title, page.get("aliases") or [])
|
||||
params = urlencode({"knowledge_base_id": mapping_id, "wiki_slug": page.get("slug") or ""})
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"id": page.get("id"),
|
||||
"knowledge_base_id": mapping_id,
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"wiki_page_id": page.get("id"),
|
||||
"wiki_slug": page.get("slug") or "",
|
||||
"title": title,
|
||||
"summary": page.get("summary") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
|
||||
"score": float(score),
|
||||
"title_match_score": float(title_match_score),
|
||||
"content": content[:4000],
|
||||
"snippet": content[:360],
|
||||
"matched_heading": None,
|
||||
"matched_sections": [
|
||||
{
|
||||
"section_index": 0,
|
||||
"heading": None,
|
||||
"content": content[:4000],
|
||||
"score": float(score),
|
||||
}
|
||||
],
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"type": "llmwiki_wiki",
|
||||
"mode": "citation",
|
||||
"retrieval_mode": "keyword",
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: float(item.get("score") or 0), reverse=True)
|
||||
return results[:limit]
|
||||
|
||||
async def _partition_mappings(self, mappings: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
if self.vector_search is None:
|
||||
return [], list(mappings)
|
||||
mapping_ids = [str(mapping["id"]) for mapping in mappings]
|
||||
states = {
|
||||
str(state["knowledge_base_mapping_id"]): state
|
||||
for state in await self.store.list_index_status(mapping_ids)
|
||||
}
|
||||
states = {str(state["knowledge_base_mapping_id"]): state for state in await self.store.list_index_status(mapping_ids)}
|
||||
fingerprint = self.vector_search.embedding.fingerprint
|
||||
vector_mappings = [
|
||||
mapping
|
||||
for mapping in mappings
|
||||
if mapping.get("wiki_index_enabled", True)
|
||||
and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)
|
||||
]
|
||||
vector_mappings = [mapping for mapping in mappings if mapping.get("wiki_index_enabled", True) and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)]
|
||||
ready_ids = {str(mapping["id"]) for mapping in vector_mappings}
|
||||
return vector_mappings, [mapping for mapping in mappings if str(mapping["id"]) not in ready_ids]
|
||||
|
||||
@@ -173,6 +501,7 @@ class WikiRetrievalService:
|
||||
*,
|
||||
limit: int,
|
||||
include_drafts: bool,
|
||||
overview: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
if self.client is None:
|
||||
return []
|
||||
@@ -181,21 +510,21 @@ class WikiRetrievalService:
|
||||
remote_id,
|
||||
page=1,
|
||||
page_size=min(max(limit * 3, 20), 100),
|
||||
query=query,
|
||||
# An overview is not a text match. Asking the remote service to
|
||||
# match its wording first is exactly what made “this library
|
||||
# covers what?” return zero while a mirror was still indexing.
|
||||
query="" if overview else query,
|
||||
)
|
||||
summaries = [item for item in listing.get("pages", []) if isinstance(item, dict)]
|
||||
slugs = list(
|
||||
dict.fromkeys(
|
||||
str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
for item in summaries
|
||||
if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
)
|
||||
)[: max(limit * 2, limit)]
|
||||
summaries = [item for item in listing.get("pages", []) if isinstance(item, dict) and is_wiki_body_page(item)]
|
||||
slugs = list(dict.fromkeys(str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/") for item in summaries if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")))[
|
||||
: limit if overview else max(limit * 2, limit)
|
||||
]
|
||||
details = await asyncio.gather(
|
||||
*(self.client.get_wiki_page(remote_id, slug) for slug in slugs),
|
||||
return_exceptions=True,
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
title_boost_enabled, title_boost_weight = self._title_boost_settings()
|
||||
for rank, (slug, detail) in enumerate(zip(slugs, details, strict=True)):
|
||||
if isinstance(detail, Exception):
|
||||
continue
|
||||
@@ -205,9 +534,12 @@ class WikiRetrievalService:
|
||||
continue
|
||||
if page["status"] == "archived" or (page["status"] == "draft" and not include_drafts):
|
||||
continue
|
||||
content = wiki_body_excerpt(page) if overview else str(page.get("content_md") or "").strip()
|
||||
if not is_wiki_body_page(page) or not content:
|
||||
continue
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": slug})
|
||||
content = str(page.get("content_md") or "")
|
||||
score = float(detail.get("score") or max(0.01, 1.0 - rank * 0.05))
|
||||
title_match_score = title_match_strength(query, page.get("title") or slug, page.get("aliases") or [])
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
@@ -222,6 +554,7 @@ class WikiRetrievalService:
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"updated_at": page.get("remote_updated_at"),
|
||||
"score": score,
|
||||
"title_match_score": float(title_match_score),
|
||||
"content": content,
|
||||
"snippet": content[:360],
|
||||
"matched_heading": None,
|
||||
@@ -237,12 +570,20 @@ class WikiRetrievalService:
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"type": "llmwiki_wiki",
|
||||
"mode": "citation",
|
||||
"retrieval_mode": "wiki_api",
|
||||
"retrieval_mode": "overview" if overview else "wiki_api",
|
||||
}
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
if not overview:
|
||||
results.sort(
|
||||
key=lambda item: float(item.get("score") or 0)
|
||||
+ (
|
||||
title_boost_weight * float(item.get("title_match_score") or 0)
|
||||
if title_boost_enabled
|
||||
else 0.0
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return results[:limit]
|
||||
|
||||
|
||||
__all__ = ["WikiRetrievalService", "is_fully_vectorized"]
|
||||
+274
-20
@@ -14,9 +14,16 @@ import numpy as np
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
|
||||
from deerflow.integrations.weknora.local_index.overview import is_wiki_body_page, is_wiki_overview_query, wiki_body_excerpt
|
||||
from deerflow.integrations.weknora.local_index.ranking import (
|
||||
make_title_matcher,
|
||||
title_match_strength,
|
||||
)
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache, WikiVectorSnapshot
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
|
||||
|
||||
_TITLE_RECALL_MIN_STRENGTH = 0.7
|
||||
|
||||
|
||||
class WikiIndexError(RuntimeError):
|
||||
def __init__(self, code: str, message: str, *, status_code: int) -> None:
|
||||
@@ -46,7 +53,15 @@ class WikiVectorSearchService:
|
||||
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
|
||||
if not mappings:
|
||||
return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
|
||||
fingerprint, candidates, warnings = await self._collect_candidates(
|
||||
if is_wiki_overview_query(normalized_query):
|
||||
return await self._search_overview(
|
||||
normalized_query,
|
||||
mappings,
|
||||
top_k_pages=top_k_pages,
|
||||
include_drafts=include_drafts,
|
||||
external=external,
|
||||
)
|
||||
fingerprint, candidates, warnings, diagnostics = await self._collect_candidates(
|
||||
normalized_query,
|
||||
mappings,
|
||||
include_drafts=include_drafts,
|
||||
@@ -80,6 +95,10 @@ class WikiVectorSearchService:
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
|
||||
"score": float(page_score),
|
||||
"vector_score": float(first.get("vector_score") or 0),
|
||||
# Preserve the lexical signal through page grouping. A
|
||||
# direct title hit ranks before broad semantic neighbours.
|
||||
"title_match_score": float(first.get("title_match_score") or 0),
|
||||
"content": "\n\n".join(str(item["section_content"]) for item in sections),
|
||||
"snippet": str(sections[0]["section_content"])[:360],
|
||||
"matched_heading": sections[0].get("heading"),
|
||||
@@ -90,9 +109,21 @@ class WikiVectorSearchService:
|
||||
"mode": "citation",
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
results.sort(
|
||||
key=lambda item: float(item["score"]),
|
||||
reverse=True,
|
||||
)
|
||||
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
|
||||
return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": "vector",
|
||||
"results": results[:limit],
|
||||
"knowledge_base_count": len(mappings),
|
||||
"index_fingerprint": fingerprint[:12],
|
||||
"partial": bool(warnings),
|
||||
"warnings": warnings,
|
||||
"diagnostics": diagnostics,
|
||||
}
|
||||
|
||||
async def search_sections(
|
||||
self,
|
||||
@@ -125,7 +156,7 @@ class WikiVectorSearchService:
|
||||
"partial": False,
|
||||
"warnings": [],
|
||||
}
|
||||
fingerprint, candidates, warnings = await self._collect_candidates(
|
||||
fingerprint, candidates, warnings, diagnostics = await self._collect_candidates(
|
||||
normalized_query,
|
||||
mappings,
|
||||
include_drafts=False,
|
||||
@@ -146,6 +177,8 @@ class WikiVectorSearchService:
|
||||
"content": str(candidate.get("section_content") or ""),
|
||||
"section_content_hash": candidate.get("content_hash") or "",
|
||||
"score": float(candidate["score"]),
|
||||
"vector_score": float(candidate.get("vector_score") or 0),
|
||||
"title_match_score": float(candidate.get("title_match_score") or 0),
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"weknora_knowledge_base_id": mapping.get("weknora_id") or "",
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
@@ -182,8 +215,127 @@ class WikiVectorSearchService:
|
||||
"index_fingerprint": fingerprint[:12],
|
||||
"partial": bool(warnings),
|
||||
"warnings": warnings,
|
||||
"diagnostics": diagnostics,
|
||||
}
|
||||
|
||||
async def _search_overview(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
top_k_pages: int | None,
|
||||
include_drafts: bool,
|
||||
external: bool,
|
||||
) -> dict[str, Any]:
|
||||
"""Return representative page bodies without embedding a broad library question.
|
||||
|
||||
A query such as “此知识库都讲了什么” is an orientation request rather
|
||||
than a semantic similarity request. Selecting actual body pages keeps
|
||||
it useful even when every section scores below ``min_similarity``.
|
||||
"""
|
||||
|
||||
fingerprint = self.embedding.fingerprint
|
||||
batches = await asyncio.gather(
|
||||
*(
|
||||
self._overview_mapping(
|
||||
mapping,
|
||||
fingerprint,
|
||||
include_drafts=include_drafts,
|
||||
external=external,
|
||||
)
|
||||
for mapping in mappings
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
candidates: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
for mapping, batch in zip(mappings, batches, strict=True):
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping.get('id')}: {batch}")
|
||||
else:
|
||||
candidates.extend(batch)
|
||||
if warnings and not candidates and len(warnings) == len(mappings):
|
||||
if any("FINGERPRINT" in item for item in warnings):
|
||||
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
|
||||
elif any("rebuild" in item.lower() for item in warnings):
|
||||
code = "WIKI_INDEX_REBUILDING"
|
||||
else:
|
||||
code = "WIKI_INDEX_NOT_READY"
|
||||
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for rank, candidate in enumerate(candidates):
|
||||
page = candidate["page"]
|
||||
mapping = candidate["mapping"]
|
||||
content = wiki_body_excerpt(page)
|
||||
score = max(0.01, 1.0 - rank * 0.05)
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": page["slug"]})
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"id": page["id"],
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"wiki_page_id": page["id"],
|
||||
"wiki_slug": page["slug"],
|
||||
"title": page.get("title") or page["slug"],
|
||||
"summary": page.get("summary") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
|
||||
"score": score,
|
||||
"content": content,
|
||||
"snippet": content[:360],
|
||||
"matched_heading": None,
|
||||
"matched_sections": [{"section_index": 0, "heading": None, "content": content, "score": score}],
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"type": "llmwiki_wiki",
|
||||
"mode": "citation",
|
||||
"retrieval_mode": "overview",
|
||||
}
|
||||
)
|
||||
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
|
||||
return {
|
||||
"query": query,
|
||||
"mode": "overview",
|
||||
"results": results[:limit],
|
||||
"knowledge_base_count": len(mappings),
|
||||
"index_fingerprint": fingerprint[:12],
|
||||
"partial": bool(warnings),
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
async def _overview_mapping(
|
||||
self,
|
||||
mapping: dict[str, Any],
|
||||
fingerprint: str,
|
||||
*,
|
||||
include_drafts: bool,
|
||||
external: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
mapping_id = str(mapping["id"])
|
||||
try:
|
||||
snapshot = await self.cache.get(mapping_id, fingerprint)
|
||||
except ValueError as exc:
|
||||
await self.cache.store.mark_rebuild_required([mapping_id], fingerprint)
|
||||
self.cache.invalidate(mapping_id)
|
||||
raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
|
||||
seen_pages: set[str] = set()
|
||||
candidates: list[dict[str, Any]] = []
|
||||
for row in snapshot.rows:
|
||||
page = row["page"]
|
||||
page_id = str(page.get("id") or "")
|
||||
if not page_id or page_id in seen_pages:
|
||||
continue
|
||||
seen_pages.add(page_id)
|
||||
if page.get("status") == "draft" and not (include_drafts and not external):
|
||||
continue
|
||||
if not is_wiki_body_page(page) or not str(page.get("content_md") or "").strip():
|
||||
continue
|
||||
candidates.append({"page": page, "mapping": mapping})
|
||||
return candidates
|
||||
|
||||
async def _collect_candidates(
|
||||
self,
|
||||
query: str,
|
||||
@@ -192,7 +344,7 @@ class WikiVectorSearchService:
|
||||
include_drafts: bool,
|
||||
external: bool,
|
||||
section_limit: int | None = None,
|
||||
) -> tuple[str, list[dict[str, Any]], list[str]]:
|
||||
) -> tuple[str, list[dict[str, Any]], list[str], dict[str, Any]]:
|
||||
try:
|
||||
query_vector = await self._embed_query(query)
|
||||
except (WikiEmbeddingError, ValueError) as exc:
|
||||
@@ -213,11 +365,17 @@ class WikiVectorSearchService:
|
||||
batches = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
candidates: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
raw_scores: list[float] = []
|
||||
threshold_filtered = 0
|
||||
for mapping, batch in zip(mappings, batches, strict=True):
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping.get('id')}: {batch}")
|
||||
else:
|
||||
candidates.extend(batch)
|
||||
mapping_candidates, raw_top_score, mapping_filtered = batch
|
||||
candidates.extend(mapping_candidates)
|
||||
threshold_filtered += mapping_filtered
|
||||
if raw_top_score is not None:
|
||||
raw_scores.append(raw_top_score)
|
||||
if warnings and not candidates and len(warnings) == len(mappings):
|
||||
if any("FINGERPRINT" in item for item in warnings):
|
||||
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
|
||||
@@ -226,8 +384,20 @@ class WikiVectorSearchService:
|
||||
else:
|
||||
code = "WIKI_INDEX_NOT_READY"
|
||||
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
|
||||
candidates.sort(key=lambda item: item["score"], reverse=True)
|
||||
return fingerprint, candidates, warnings
|
||||
candidates.sort(
|
||||
key=lambda item: float(item["score"]),
|
||||
reverse=True,
|
||||
)
|
||||
return (
|
||||
fingerprint,
|
||||
candidates,
|
||||
warnings,
|
||||
{
|
||||
"top_raw_vector_score": max(raw_scores) if raw_scores else None,
|
||||
"min_similarity": self.config.min_similarity,
|
||||
"threshold_filtered": threshold_filtered,
|
||||
},
|
||||
)
|
||||
|
||||
async def _embed_query(self, query: str) -> np.ndarray:
|
||||
digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
|
||||
@@ -254,7 +424,7 @@ class WikiVectorSearchService:
|
||||
include_drafts: bool,
|
||||
external: bool,
|
||||
section_limit: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
) -> tuple[list[dict[str, Any]], float | None, int]:
|
||||
mapping_id = str(mapping["id"])
|
||||
try:
|
||||
snapshot = await self.cache.get(mapping_id, fingerprint)
|
||||
@@ -263,33 +433,45 @@ class WikiVectorSearchService:
|
||||
self.cache.invalidate(mapping_id)
|
||||
raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
|
||||
if snapshot.matrix.size == 0:
|
||||
return []
|
||||
return [], None, 0
|
||||
if snapshot.dimensions != vector.size:
|
||||
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
|
||||
eligible = np.asarray(
|
||||
[
|
||||
index
|
||||
for index, row in enumerate(snapshot.rows)
|
||||
if row["page"].get("status") != "draft" or (include_drafts and not external)
|
||||
],
|
||||
[index for index, row in enumerate(snapshot.rows) if row["page"].get("status") != "draft" or (include_drafts and not external)],
|
||||
dtype=int,
|
||||
)
|
||||
if eligible.size == 0:
|
||||
return []
|
||||
return [], None, 0
|
||||
indices, scores = await asyncio.to_thread(
|
||||
self._top_indices,
|
||||
snapshot,
|
||||
vector,
|
||||
eligible,
|
||||
query,
|
||||
section_limit,
|
||||
)
|
||||
terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
|
||||
candidates = []
|
||||
raw_top_score = float(scores[0]) if scores.size else None
|
||||
threshold_filtered = int(np.count_nonzero(scores < self.config.min_similarity))
|
||||
for index, score in zip(indices, scores, strict=True):
|
||||
row = snapshot.rows[int(index)]
|
||||
page = row["page"]
|
||||
vector_score = float(score)
|
||||
if vector_score < self.config.min_similarity:
|
||||
title_boost_active = (
|
||||
self.config.title_boost_enabled
|
||||
and self.config.title_boost_weight > 0
|
||||
)
|
||||
title_score = (
|
||||
title_match_strength(query, page.get("title"), page.get("aliases") or [])
|
||||
if title_boost_active
|
||||
else 0.0
|
||||
)
|
||||
# An explicit title/alias hit is itself a high-confidence lexical
|
||||
# recall signal. Keep strong hits even when their dense vector is
|
||||
# below the semantic threshold; keyword/hybrid callers then see
|
||||
# the same title-first result as the direct vector endpoint.
|
||||
if vector_score < self.config.min_similarity and title_score < _TITLE_RECALL_MIN_STRENGTH:
|
||||
continue
|
||||
final_score = vector_score
|
||||
if self.config.keyword_boost_enabled and terms:
|
||||
@@ -297,20 +479,92 @@ class WikiVectorSearchService:
|
||||
keyword_score = sum(term in haystack for term in terms) / len(terms)
|
||||
weight = self.config.keyword_boost_weight
|
||||
final_score = (1 - weight) * vector_score + weight * keyword_score
|
||||
candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
|
||||
return candidates
|
||||
if title_score:
|
||||
# Cosine scores and lexical scores are both roughly bounded by
|
||||
# one. A modest additive title boost keeps dense retrieval
|
||||
# useful while ensuring an exact title is not lost to a broad
|
||||
# semantic neighbour (for example “特朗普” vs “海湖庄园”).
|
||||
final_score += self.config.title_boost_weight * title_score
|
||||
candidates.append(
|
||||
{
|
||||
**row,
|
||||
"page": page,
|
||||
"mapping": mapping,
|
||||
"score": final_score,
|
||||
"vector_score": vector_score,
|
||||
"title_match_score": title_score,
|
||||
}
|
||||
)
|
||||
requested_limit = section_limit or self.config.top_k_sections
|
||||
candidates.sort(
|
||||
key=lambda item: float(item["score"]),
|
||||
reverse=True,
|
||||
)
|
||||
if section_limit is None:
|
||||
# Page search only needs a bounded number of sections from each
|
||||
# page. Apply the cap before truncation so one exact-title page
|
||||
# cannot consume every candidate slot and hide useful backfill.
|
||||
per_page: dict[str, int] = defaultdict(int)
|
||||
diversified: list[dict[str, Any]] = []
|
||||
for candidate in candidates:
|
||||
page_key = str(candidate["page"].get("id") or candidate["page"].get("slug") or "")
|
||||
if per_page[page_key] >= self.config.max_sections_per_page:
|
||||
continue
|
||||
per_page[page_key] += 1
|
||||
diversified.append(candidate)
|
||||
if len(diversified) >= requested_limit:
|
||||
break
|
||||
candidates = diversified
|
||||
return candidates[:requested_limit], raw_top_score, threshold_filtered
|
||||
|
||||
def _top_indices(
|
||||
self,
|
||||
snapshot: WikiVectorSnapshot,
|
||||
vector: np.ndarray,
|
||||
eligible: np.ndarray,
|
||||
query: str,
|
||||
section_limit: int | None = None,
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
scores = snapshot.matrix[eligible] @ vector
|
||||
count = min(section_limit or self.config.top_k_sections, scores.size)
|
||||
if count <= 0:
|
||||
return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
|
||||
positions = np.argpartition(scores, -count)[-count:]
|
||||
# Page grouping happens after section retrieval, so keep a wider dense
|
||||
# pool for page searches. This lets the per-page cap backfill from
|
||||
# other pages instead of returning fewer than ``top_k_sections``.
|
||||
pool_count = count if section_limit is not None else min(scores.size, max(count * 4, count + 32))
|
||||
positions = np.argpartition(scores, -pool_count)[-pool_count:]
|
||||
|
||||
if self.config.title_boost_enabled and self.config.title_boost_weight > 0:
|
||||
title_positions: list[tuple[float, float, int]] = []
|
||||
per_page: dict[str, int] = defaultdict(int)
|
||||
title_scores_by_page: dict[str, float] = {}
|
||||
match_title = make_title_matcher(query)
|
||||
for local_position, row_index in enumerate(eligible):
|
||||
page = snapshot.rows[int(row_index)]["page"]
|
||||
page_key = str(page.get("id") or page.get("slug") or row_index)
|
||||
if page_key not in title_scores_by_page:
|
||||
title_scores_by_page[page_key] = match_title(
|
||||
page.get("title"),
|
||||
page.get("aliases") or [],
|
||||
)
|
||||
strength = title_scores_by_page[page_key]
|
||||
if strength >= _TITLE_RECALL_MIN_STRENGTH:
|
||||
title_positions.append((strength, float(scores[local_position]), local_position))
|
||||
title_positions.sort(reverse=True)
|
||||
extra_limit = max(count, self.config.top_k_pages * self.config.max_sections_per_page)
|
||||
extras: list[int] = []
|
||||
for _strength, _score, local_position in title_positions:
|
||||
page = snapshot.rows[int(eligible[local_position])]["page"]
|
||||
page_key = str(page.get("id") or page.get("slug") or "")
|
||||
if per_page[page_key] >= self.config.max_sections_per_page:
|
||||
continue
|
||||
per_page[page_key] += 1
|
||||
extras.append(local_position)
|
||||
if len(extras) >= extra_limit:
|
||||
break
|
||||
if extras:
|
||||
positions = np.asarray(list(dict.fromkeys([*positions.tolist(), *extras])), dtype=int)
|
||||
|
||||
positions = positions[np.argsort(scores[positions])[::-1]]
|
||||
return eligible[positions], scores[positions]
|
||||
+719
-27
@@ -5,20 +5,36 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import platform
|
||||
from datetime import UTC, datetime
|
||||
from math import ceil
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
|
||||
from deerflow.integrations.weknora.local_index.normalize import (
|
||||
normalize_wiki_link_reference,
|
||||
normalize_wiki_page,
|
||||
normalize_wiki_page_links,
|
||||
)
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# This breaker is intentionally process-local. The durable sync row already
|
||||
# records the failed run and error, while a transient retry deadline does not
|
||||
# belong in release files or administrator configuration. Rebuilding the
|
||||
# runtime after a provider/config change (or restarting Gateway) resets the
|
||||
# breaker and therefore allows one immediate recovery probe. The state is a
|
||||
# fixed-size provider record rather than a per-page/per-mapping collection.
|
||||
_EMBEDDING_PROVIDER_BACKOFF_INITIAL_SECONDS = 60.0
|
||||
_EMBEDDING_PROVIDER_BACKOFF_MAX_SECONDS = 30.0 * 60.0
|
||||
|
||||
|
||||
class WikiSyncLeaseLost(RuntimeError):
|
||||
pass
|
||||
@@ -28,6 +44,215 @@ def _safe_error(exc: BaseException) -> str:
|
||||
return f"{type(exc).__name__}: {str(exc)[:1000]}"
|
||||
|
||||
|
||||
def _iso_now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _same_instant(left: Any, right: Any) -> bool:
|
||||
"""Compare provider/database timestamps without depending on one timezone."""
|
||||
|
||||
def _parse(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
left_dt = _parse(left)
|
||||
right_dt = _parse(right)
|
||||
if left_dt is None or right_dt is None:
|
||||
return left == right
|
||||
if left_dt.tzinfo is None or right_dt.tzinfo is None:
|
||||
return left_dt.replace(tzinfo=None) == right_dt.replace(tzinfo=None)
|
||||
return left_dt.astimezone(UTC) == right_dt.astimezone(UTC)
|
||||
|
||||
|
||||
def _summary_has_page_body(summary: dict[str, Any]) -> bool:
|
||||
"""Whether a WeKnora list item already carries the complete Wiki body."""
|
||||
|
||||
return any(key in summary and summary.get(key) is not None for key in ("content_md", "content", "markdown"))
|
||||
|
||||
|
||||
def _knowledge_base_is_processing(remote: dict[str, Any]) -> bool:
|
||||
raw_flag = remote.get("is_processing", remote.get("isProcessing"))
|
||||
if isinstance(raw_flag, str):
|
||||
is_processing = raw_flag.strip().lower() in {"1", "true", "yes", "on"}
|
||||
else:
|
||||
is_processing = bool(raw_flag)
|
||||
raw_count = remote.get("processing_count", remote.get("processingCount"))
|
||||
try:
|
||||
processing_count = int(raw_count or 0)
|
||||
except (TypeError, ValueError):
|
||||
processing_count = 0
|
||||
return is_processing or processing_count > 0
|
||||
|
||||
|
||||
def _summary_matches_existing(
|
||||
mapping_id: str,
|
||||
summary: dict[str, Any],
|
||||
existing: dict[str, Any],
|
||||
) -> bool:
|
||||
"""Trust only explicit revision markers when skipping the detail request.
|
||||
|
||||
Older WeKnora releases omitted both markers. Those responses deliberately
|
||||
fall back to ``get_wiki_page`` so an unreported edit can never be missed.
|
||||
Explicit metadata included in the summary is also compared, catching a
|
||||
provider that failed to bump its version for a directory/status change.
|
||||
"""
|
||||
|
||||
try:
|
||||
normalized = normalize_wiki_page(mapping_id, summary)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
has_marker = False
|
||||
raw_version = summary.get("page_version", summary.get("version"))
|
||||
if raw_version not in (None, ""):
|
||||
try:
|
||||
version = int(raw_version)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if version > 0:
|
||||
has_marker = True
|
||||
if int(existing.get("page_version") or 0) != version:
|
||||
return False
|
||||
|
||||
raw_updated_at = summary.get("updated_at", summary.get("updatedAt"))
|
||||
if raw_updated_at not in (None, ""):
|
||||
updated_at = normalized.get("remote_updated_at")
|
||||
if updated_at is None:
|
||||
return False
|
||||
has_marker = True
|
||||
if not _same_instant(existing.get("remote_updated_at"), updated_at):
|
||||
return False
|
||||
|
||||
if not has_marker:
|
||||
return False
|
||||
|
||||
explicit_fields: tuple[tuple[tuple[str, ...], str], ...] = (
|
||||
(("id", "page_id"), "remote_page_id"),
|
||||
(("title",), "title"),
|
||||
(("summary", "description"), "summary"),
|
||||
(("page_type", "type"), "page_type"),
|
||||
(("status",), "status"),
|
||||
(("aliases",), "aliases"),
|
||||
(("category_path", "categories"), "category_path"),
|
||||
(("parent_slug",), "parent_slug"),
|
||||
(("wiki_path",), "wiki_path"),
|
||||
(("source_refs", "sources"), "source_refs"),
|
||||
(("chunk_refs", "chunks"), "chunk_refs"),
|
||||
)
|
||||
for source_keys, local_key in explicit_fields:
|
||||
if any(key in summary for key in source_keys) and existing.get(local_key) != normalized.get(local_key):
|
||||
return False
|
||||
if _summary_has_page_body(summary) and existing.get("content_hash") != normalized.get("content_hash"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _existing_snapshot_reusable(existing: dict[str, Any] | None, fingerprint: str) -> bool:
|
||||
if not existing:
|
||||
return False
|
||||
has_searchable_body = bool(existing.get("content_md") or existing.get("summary"))
|
||||
return bool(
|
||||
existing.get("indexed_fingerprint") == fingerprint
|
||||
and existing.get("index_status") == "ready"
|
||||
and not existing.get("last_error")
|
||||
and not existing.get("is_remote_deleted")
|
||||
and (
|
||||
int(existing.get("vector_count") or 0) > 0
|
||||
or existing.get("status") == "archived"
|
||||
or not has_searchable_body
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _build_graph_edges(
|
||||
page_records: list[dict[str, Any]],
|
||||
provider_graph: dict[str, Any] | None,
|
||||
) -> list[dict[str, str]]:
|
||||
"""Resolve page/link aliases to mapping-local canonical slugs."""
|
||||
|
||||
identities: dict[str, set[str]] = {}
|
||||
canonical_slugs: dict[str, str] = {}
|
||||
|
||||
def _remember(reference: Any, slug: str) -> None:
|
||||
normalized = normalize_wiki_link_reference(reference)
|
||||
if normalized:
|
||||
identities.setdefault(normalized.casefold(), set()).add(slug)
|
||||
|
||||
for record in page_records:
|
||||
page = record["page"]
|
||||
slug = str(page["slug"])
|
||||
canonical_slugs[slug] = slug
|
||||
for value in (
|
||||
slug,
|
||||
page.get("title"),
|
||||
page.get("remote_page_id"),
|
||||
*(page.get("aliases") or []),
|
||||
):
|
||||
_remember(value, slug)
|
||||
|
||||
graph = provider_graph if isinstance(provider_graph, dict) else {}
|
||||
for node in graph.get("nodes") or []:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
canonical_reference = node.get("slug") or node.get("wiki_slug") or node.get("path")
|
||||
normalized = normalize_wiki_link_reference(canonical_reference)
|
||||
if normalized in canonical_slugs:
|
||||
canonical_slug = canonical_slugs[normalized]
|
||||
else:
|
||||
candidates = identities.get(normalized.casefold(), set()) if normalized else set()
|
||||
if len(candidates) != 1:
|
||||
continue
|
||||
canonical_slug = next(iter(candidates))
|
||||
for value in (
|
||||
node.get("id"),
|
||||
node.get("slug"),
|
||||
node.get("wiki_slug"),
|
||||
node.get("path"),
|
||||
node.get("title"),
|
||||
node.get("name"),
|
||||
):
|
||||
_remember(value, canonical_slug)
|
||||
|
||||
def _resolve(reference: Any) -> str | None:
|
||||
normalized = normalize_wiki_link_reference(reference)
|
||||
if not normalized:
|
||||
return None
|
||||
if normalized in canonical_slugs:
|
||||
return canonical_slugs[normalized]
|
||||
candidates = identities.get(normalized.casefold(), set())
|
||||
return next(iter(candidates)) if len(candidates) == 1 else None
|
||||
|
||||
pairs: set[tuple[str, str]] = set()
|
||||
for record in page_records:
|
||||
page = record["page"]
|
||||
links = record["links"]
|
||||
slug = str(page["slug"])
|
||||
for target_reference in links.get("out_links") or []:
|
||||
target = _resolve(target_reference)
|
||||
if target and target != slug:
|
||||
pairs.add((slug, target))
|
||||
for source_reference in links.get("in_links") or []:
|
||||
source = _resolve(source_reference)
|
||||
if source and source != slug:
|
||||
pairs.add((source, slug))
|
||||
|
||||
for edge in graph.get("edges") or []:
|
||||
if not isinstance(edge, dict):
|
||||
continue
|
||||
source = _resolve(edge.get("source") or edge.get("source_id"))
|
||||
target = _resolve(edge.get("target") or edge.get("target_id"))
|
||||
if source and target and source != target:
|
||||
pairs.add((source, target))
|
||||
return [{"source_slug": source, "target_slug": target, "edge_type": "wiki_link"} for source, target in sorted(pairs)]
|
||||
|
||||
|
||||
class WikiSyncService:
|
||||
def __init__(self, config: LocalWikiIndexConfig, store: LlmWikiIndexStore, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache, client: WeKnoraClient) -> None:
|
||||
self.config = config
|
||||
@@ -37,14 +262,188 @@ class WikiSyncService:
|
||||
self.client = client
|
||||
self.owner = f"{platform.node()}:{uuid4()}"
|
||||
self.on_completed = None
|
||||
self._embedding_probe_lock = asyncio.Lock()
|
||||
self._embedding_failure_count = 0
|
||||
self._embedding_retry_at = 0.0
|
||||
self._embedding_last_error: str | None = None
|
||||
|
||||
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False, notify: bool = True) -> dict[str, Any]:
|
||||
def _embedding_backoff_result(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
now: float | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
checked_at = monotonic() if now is None else now
|
||||
remaining = self._embedding_retry_at - checked_at
|
||||
if self._embedding_failure_count <= 0 or remaining <= 0:
|
||||
return None
|
||||
return {
|
||||
"mapping_id": mapping_id,
|
||||
"status": "provider_backoff",
|
||||
"failure_scope": "embedding_provider",
|
||||
"failure_count": self._embedding_failure_count,
|
||||
"retry_after_seconds": max(1, ceil(remaining)),
|
||||
"error": self._embedding_last_error,
|
||||
}
|
||||
|
||||
def _record_embedding_failure(self, exc: WikiEmbeddingError) -> int:
|
||||
"""Open or extend one provider-wide exponential breaker window.
|
||||
|
||||
Several mappings may already be running under the scheduler semaphore
|
||||
when the provider goes away. Failures arriving inside an already-open
|
||||
window share that window instead of multiplying the exponent.
|
||||
"""
|
||||
|
||||
now = monotonic()
|
||||
if self._embedding_retry_at <= now:
|
||||
self._embedding_failure_count = min(
|
||||
self._embedding_failure_count + 1,
|
||||
31,
|
||||
)
|
||||
exponent = min(self._embedding_failure_count - 1, 30)
|
||||
delay = min(
|
||||
_EMBEDDING_PROVIDER_BACKOFF_MAX_SECONDS,
|
||||
_EMBEDDING_PROVIDER_BACKOFF_INITIAL_SECONDS * (2**exponent),
|
||||
)
|
||||
self._embedding_retry_at = now + delay
|
||||
self._embedding_last_error = _safe_error(exc)
|
||||
return max(1, ceil(self._embedding_retry_at - now))
|
||||
|
||||
def _reset_embedding_breaker(self) -> None:
|
||||
self._embedding_failure_count = 0
|
||||
self._embedding_retry_at = 0.0
|
||||
self._embedding_last_error = None
|
||||
|
||||
async def _embedding_preflight(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
force: bool,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Skip a known outage or perform one serialized recovery probe."""
|
||||
|
||||
if not force:
|
||||
backoff = self._embedding_backoff_result(mapping_id)
|
||||
if backoff is not None:
|
||||
return backoff
|
||||
async with self._embedding_probe_lock:
|
||||
if not force:
|
||||
backoff = self._embedding_backoff_result(mapping_id)
|
||||
if backoff is not None:
|
||||
return backoff
|
||||
needs_probe = (
|
||||
self.embedding.dimensions is None
|
||||
or self._embedding_failure_count > 0
|
||||
)
|
||||
if not needs_probe:
|
||||
return None
|
||||
try:
|
||||
# Auto-dimension configurations need one successful response
|
||||
# for a stable fingerprint. After an outage the same tiny
|
||||
# request proves provider recovery before any Wiki directory
|
||||
# or page is enumerated.
|
||||
await self.embedding.embed_texts(
|
||||
["DeerFlow local Wiki embedding provider health probe"]
|
||||
)
|
||||
except WikiEmbeddingError as exc:
|
||||
if exc.category != "runtime_retired":
|
||||
retry_after = self._record_embedding_failure(exc)
|
||||
logger.warning(
|
||||
"Local Wiki embedding provider probe failed mapping_id=%s retry_after_seconds=%s: %s",
|
||||
mapping_id,
|
||||
retry_after,
|
||||
_safe_error(exc),
|
||||
)
|
||||
raise
|
||||
self._reset_embedding_breaker()
|
||||
return None
|
||||
|
||||
async def _persist_requested_backoff(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
sync_id: str,
|
||||
mode: str,
|
||||
started_at: str,
|
||||
backoff: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Finish an explicitly requested job with a frontend-compatible state."""
|
||||
|
||||
states = await self.store.list_index_status([mapping_id])
|
||||
prior = states[0] if states else {}
|
||||
remote_count = int(prior.get("remote_page_count") or 0)
|
||||
acquired = await self.store.try_acquire_sync_lease(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
lease_seconds=self.config.sync_lease_seconds,
|
||||
fingerprint=self.embedding.fingerprint,
|
||||
)
|
||||
if not acquired:
|
||||
return backoff
|
||||
error = str(
|
||||
backoff.get("error")
|
||||
or "Wiki embedding provider is temporarily unavailable"
|
||||
)
|
||||
result = {
|
||||
"sync_id": sync_id,
|
||||
"mapping_id": mapping_id,
|
||||
"mode": mode,
|
||||
"status": "failed",
|
||||
"remote_pages": remote_count,
|
||||
"added_pages": 0,
|
||||
"updated_pages": 0,
|
||||
"metadata_updated_pages": 0,
|
||||
"revectorized_pages": 0,
|
||||
"changed_pages": 0,
|
||||
"skipped_pages": 0,
|
||||
"deleted_pages": 0,
|
||||
"failed_pages": 0,
|
||||
"started_at": started_at,
|
||||
"completed_at": _iso_now(),
|
||||
"error": error,
|
||||
"failure_scope": backoff.get("failure_scope")
|
||||
or "embedding_provider",
|
||||
"failure_count": int(backoff.get("failure_count") or 0),
|
||||
"retry_after_seconds": int(
|
||||
backoff.get("retry_after_seconds") or 0
|
||||
),
|
||||
}
|
||||
persisted = await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=False,
|
||||
remote_page_count=remote_count,
|
||||
error=error,
|
||||
result=result,
|
||||
)
|
||||
return result if persisted else backoff
|
||||
|
||||
async def sync_mapping(
|
||||
self,
|
||||
mapping: dict[str, Any],
|
||||
*,
|
||||
force: bool = False,
|
||||
notify: bool = True,
|
||||
sync_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
mapping_id = str(mapping["id"])
|
||||
sync_id = str(uuid4())
|
||||
if self.embedding.dimensions is None:
|
||||
# Auto-dimension configurations need one successful response to
|
||||
# produce a stable fingerprint before lease/rebuild decisions.
|
||||
await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
|
||||
requested_sync_id = sync_id
|
||||
sync_id = sync_id or str(uuid4())
|
||||
mode = "full" if force else "incremental"
|
||||
started_at = _iso_now()
|
||||
backoff = await self._embedding_preflight(mapping_id, force=force)
|
||||
if backoff is not None:
|
||||
if requested_sync_id is not None:
|
||||
return await self._persist_requested_backoff(
|
||||
mapping_id,
|
||||
sync_id=sync_id,
|
||||
mode=mode,
|
||||
started_at=started_at,
|
||||
backoff=backoff,
|
||||
)
|
||||
return backoff
|
||||
fingerprint = self.embedding.fingerprint
|
||||
revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
|
||||
if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
|
||||
@@ -82,40 +481,145 @@ class WikiSyncService:
|
||||
|
||||
remote_count = 0
|
||||
changed = 0
|
||||
added = 0
|
||||
updated = 0
|
||||
metadata_updated = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
graph_snapshot_complete = True
|
||||
graph_page_records: list[dict[str, Any]] = []
|
||||
|
||||
def _sync_result(
|
||||
status: str,
|
||||
*,
|
||||
deleted_pages: int = 0,
|
||||
error: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {
|
||||
"sync_id": sync_id,
|
||||
"mapping_id": mapping_id,
|
||||
"mode": mode,
|
||||
"status": status,
|
||||
"remote_pages": remote_count,
|
||||
"added_pages": added,
|
||||
"updated_pages": updated,
|
||||
"metadata_updated_pages": metadata_updated,
|
||||
"revectorized_pages": changed,
|
||||
"changed_pages": changed,
|
||||
"skipped_pages": skipped,
|
||||
"deleted_pages": deleted_pages,
|
||||
"failed_pages": failed,
|
||||
"started_at": started_at,
|
||||
"completed_at": _iso_now(),
|
||||
}
|
||||
if error:
|
||||
result["error"] = error
|
||||
return result
|
||||
|
||||
try:
|
||||
remote_id = str(mapping["weknora_id"])
|
||||
await self.client.get_knowledge_base(remote_id)
|
||||
initial_remote = await self.client.get_knowledge_base(remote_id)
|
||||
processing_observed_during_scan = _knowledge_base_is_processing(
|
||||
initial_remote
|
||||
)
|
||||
page_number = 1
|
||||
while True:
|
||||
batch = await self.client.list_wiki_pages(remote_id, page=page_number, page_size=self.config.sync_page_size)
|
||||
pages = [page for page in batch.get("pages", []) if isinstance(page, dict)]
|
||||
remote_count += len(pages)
|
||||
reported_total = int(batch.get("total") or batch.get("total_count") or remote_count)
|
||||
await self.store.update_sync_progress(
|
||||
progress_updated = await self.store.update_sync_progress(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
remote_page_count=max(remote_count, reported_total),
|
||||
)
|
||||
if not progress_updated:
|
||||
lease_lost.set()
|
||||
raise WikiSyncLeaseLost("Local Wiki sync lease was lost while updating progress")
|
||||
for summary in pages:
|
||||
graph_page_captured = False
|
||||
try:
|
||||
_ensure_lease()
|
||||
slug = str(summary.get("slug") or summary.get("wiki_slug") or summary.get("path") or "")
|
||||
remote_page = await self.client.get_wiki_page(remote_id, slug)
|
||||
page = normalize_wiki_page(mapping_id, {**summary, **remote_page})
|
||||
existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
|
||||
summary_page = normalize_wiki_page(mapping_id, summary)
|
||||
existing = await self.store.get_page_by_slug_hash(
|
||||
mapping_id,
|
||||
summary_page["slug_hash"],
|
||||
)
|
||||
active_fingerprint = self.embedding.fingerprint
|
||||
if (
|
||||
not force
|
||||
and existing
|
||||
and existing.get("content_hash") == page["content_hash"]
|
||||
and existing.get("indexed_fingerprint") == active_fingerprint
|
||||
and existing.get("index_status") == "ready"
|
||||
and (existing.get("vector_count", 1) > 0 or page["status"] == "archived" or not (page["content_md"] or page["summary"]))
|
||||
and not existing.get("is_remote_deleted")
|
||||
and not _summary_has_page_body(summary)
|
||||
and _existing_snapshot_reusable(existing, active_fingerprint)
|
||||
and _summary_matches_existing(mapping_id, summary, existing)
|
||||
):
|
||||
await self.store.touch_page(str(existing["id"]), sync_id)
|
||||
touched = await self.store.touch_page(
|
||||
str(existing["id"]),
|
||||
sync_id,
|
||||
mapping_id=mapping_id,
|
||||
owner=self.owner,
|
||||
)
|
||||
if not touched:
|
||||
lease_lost.set()
|
||||
raise WikiSyncLeaseLost(
|
||||
"Local Wiki sync lease was lost before touching an unchanged page"
|
||||
)
|
||||
graph_page_records.append(
|
||||
{
|
||||
"page": existing,
|
||||
"links": normalize_wiki_page_links(existing),
|
||||
}
|
||||
)
|
||||
# Old WeKnora list shapes do not expose explicit
|
||||
# incoming/outgoing links. Markdown links from the
|
||||
# local body are useful for lookup, but are not an
|
||||
# authoritative whole-graph snapshot.
|
||||
graph_snapshot_complete = False
|
||||
graph_page_captured = True
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if _summary_has_page_body(summary):
|
||||
remote_payload = dict(summary)
|
||||
else:
|
||||
remote_page = await self.client.get_wiki_page(remote_id, slug)
|
||||
remote_payload = {**summary, **remote_page}
|
||||
page = normalize_wiki_page(mapping_id, remote_payload)
|
||||
graph_page_records.append(
|
||||
{
|
||||
"page": page,
|
||||
"links": normalize_wiki_page_links(remote_payload),
|
||||
}
|
||||
)
|
||||
graph_page_captured = True
|
||||
# A provider may canonicalize the slug in the detail
|
||||
# response. Resolve the canonical row before deciding
|
||||
# whether vectors can be reused.
|
||||
if existing is None or existing.get("slug_hash") != page["slug_hash"]:
|
||||
existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
|
||||
active_fingerprint = self.embedding.fingerprint
|
||||
if (
|
||||
not force
|
||||
and _existing_snapshot_reusable(existing, active_fingerprint)
|
||||
and existing.get("content_hash") == page["content_hash"]
|
||||
):
|
||||
metadata_result = await self.store.refresh_page_metadata(
|
||||
page=page,
|
||||
sync_id=sync_id,
|
||||
owner=self.owner,
|
||||
)
|
||||
if metadata_result is None:
|
||||
lease_lost.set()
|
||||
raise WikiSyncLeaseLost(
|
||||
"Local Wiki sync lease was lost before refreshing page metadata"
|
||||
)
|
||||
if metadata_result.get("changed"):
|
||||
metadata_updated += 1
|
||||
updated += 1
|
||||
self.cache.invalidate(mapping_id)
|
||||
else:
|
||||
skipped += 1
|
||||
continue
|
||||
sections = (
|
||||
[]
|
||||
@@ -131,16 +635,50 @@ class WikiSyncService:
|
||||
{"section_index": section.section_index, "heading": section.heading, "section_content": section.content, "content_hash": section.content_hash, "embedding_dimensions": dimensions, "vector_blob": blob}
|
||||
)
|
||||
_ensure_lease()
|
||||
await self.store.replace_page_vectors(page=page, vectors=vectors, fingerprint=fingerprint, sync_id=sync_id)
|
||||
saved_page = await self.store.replace_page_vectors(
|
||||
page=page,
|
||||
vectors=vectors,
|
||||
fingerprint=fingerprint,
|
||||
sync_id=sync_id,
|
||||
owner=self.owner,
|
||||
)
|
||||
if saved_page is None:
|
||||
lease_lost.set()
|
||||
raise WikiSyncLeaseLost(
|
||||
"Local Wiki sync lease was lost before replacing page vectors"
|
||||
)
|
||||
self.cache.invalidate(mapping_id)
|
||||
changed += 1
|
||||
if existing is None:
|
||||
added += 1
|
||||
else:
|
||||
updated += 1
|
||||
except Exception as exc:
|
||||
if isinstance(exc, WikiSyncLeaseLost):
|
||||
raise
|
||||
# Embedding failures describe the shared provider, not
|
||||
# one malformed Wiki page. Continuing would repeat the
|
||||
# client's own retry loop for every remaining page.
|
||||
if isinstance(exc, WikiEmbeddingError):
|
||||
raise
|
||||
if not graph_page_captured:
|
||||
graph_snapshot_complete = False
|
||||
failed += 1
|
||||
try:
|
||||
fallback = normalize_wiki_page(mapping_id, summary)
|
||||
await self.store.record_page_failure(page=fallback, sync_id=sync_id, error=_safe_error(exc))
|
||||
failure_recorded = await self.store.record_page_failure(
|
||||
page=fallback,
|
||||
sync_id=sync_id,
|
||||
error=_safe_error(exc),
|
||||
owner=self.owner,
|
||||
)
|
||||
if not failure_recorded:
|
||||
lease_lost.set()
|
||||
raise WikiSyncLeaseLost(
|
||||
"Local Wiki sync lease was lost before recording a page failure"
|
||||
) from exc
|
||||
except WikiSyncLeaseLost:
|
||||
raise
|
||||
except Exception as record_exc:
|
||||
logger.exception("Could not record local Wiki page failure mapping_id=%s", mapping_id)
|
||||
raise RuntimeError("Could not safely retain a failed Wiki page during the full scan") from record_exc
|
||||
@@ -151,18 +689,138 @@ class WikiSyncService:
|
||||
break
|
||||
page_number += 1
|
||||
_ensure_lease()
|
||||
deleted = await self.store.mark_missing_pages_deleted(mapping_id, sync_id)
|
||||
|
||||
# WeKnora can enter (or still be in) asynchronous processing after
|
||||
# the pre-scan scheduler check. A page directory observed during
|
||||
# that window is not deletion-authoritative. Re-check immediately
|
||||
# before the only destructive phase and preserve both missing
|
||||
# pages and graph edges whenever the status is processing or
|
||||
# cannot be verified. Persist ``partial`` so the periodic scanner
|
||||
# necessarily retries even when the KB timestamp is unchanged.
|
||||
deferred_reason: str | None = None
|
||||
deferred_code: str | None = None
|
||||
try:
|
||||
current_remote = await self.client.get_knowledge_base(remote_id)
|
||||
except Exception as exc:
|
||||
deferred_code = "processing_status_unavailable"
|
||||
deferred_reason = (
|
||||
"Could not verify WeKnora processing status; deletion and graph replacement deferred: "
|
||||
f"{_safe_error(exc)}"
|
||||
)
|
||||
else:
|
||||
if processing_observed_during_scan or _knowledge_base_is_processing(
|
||||
current_remote
|
||||
):
|
||||
deferred_code = "upstream_processing"
|
||||
deferred_reason = (
|
||||
"WeKnora is still processing; deletion and graph replacement deferred"
|
||||
)
|
||||
_ensure_lease()
|
||||
if deferred_reason is not None:
|
||||
terminal_error = (
|
||||
f"{deferred_reason}; {failed} page(s) failed"
|
||||
if failed
|
||||
else deferred_reason
|
||||
)
|
||||
result = {
|
||||
**_sync_result("partial", error=terminal_error),
|
||||
"graph_edges": None,
|
||||
"graph_changed": False,
|
||||
"graph_preserved": True,
|
||||
"deferred_deletion": True,
|
||||
"deferred_reason": deferred_code,
|
||||
}
|
||||
persisted = await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=True,
|
||||
remote_page_count=remote_count,
|
||||
error=terminal_error,
|
||||
result=result,
|
||||
)
|
||||
if not persisted:
|
||||
lease_lost.set()
|
||||
raise WikiSyncLeaseLost(
|
||||
"Local Wiki sync lease was lost before persisting the deferred scan result"
|
||||
)
|
||||
return result
|
||||
|
||||
provider_graph: dict[str, Any] | None = None
|
||||
graph_api_succeeded = False
|
||||
get_wiki_graph = getattr(self.client, "get_wiki_graph", None)
|
||||
if callable(get_wiki_graph):
|
||||
try:
|
||||
provider_graph = await get_wiki_graph(
|
||||
remote_id,
|
||||
limit=max(200, min(remote_count, 10000)),
|
||||
)
|
||||
graph_api_succeeded = isinstance(provider_graph, dict)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Local Wiki graph endpoint unavailable mapping_id=%s: %s",
|
||||
mapping_id,
|
||||
_safe_error(exc),
|
||||
)
|
||||
_ensure_lease()
|
||||
deleted = await self.store.mark_missing_pages_deleted(
|
||||
mapping_id,
|
||||
sync_id,
|
||||
owner=self.owner,
|
||||
)
|
||||
if deleted is None:
|
||||
lease_lost.set()
|
||||
raise WikiSyncLeaseLost("Local Wiki sync lease was lost before retiring missing pages")
|
||||
if deleted:
|
||||
self.cache.invalidate(mapping_id)
|
||||
await self.store.finish_sync(
|
||||
graph_result: dict[str, Any]
|
||||
if remote_count == 0 or graph_api_succeeded or graph_snapshot_complete:
|
||||
graph_edges = _build_graph_edges(graph_page_records, provider_graph)
|
||||
_ensure_lease()
|
||||
fenced_graph_result = await self.store.replace_graph_edges(
|
||||
mapping_id,
|
||||
graph_edges,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
)
|
||||
if fenced_graph_result is None:
|
||||
lease_lost.set()
|
||||
raise WikiSyncLeaseLost("Local Wiki sync lease was lost before replacing graph edges")
|
||||
graph_result = fenced_graph_result
|
||||
else:
|
||||
# Do not replace a previously complete snapshot with a graph
|
||||
# assembled from a partial page scan when the graph API also
|
||||
# failed. The next successful full sync will repair it.
|
||||
graph_result = {
|
||||
"edge_count": None,
|
||||
"changed": False,
|
||||
"preserved": True,
|
||||
}
|
||||
_ensure_lease()
|
||||
result = {
|
||||
**_sync_result(
|
||||
"partial" if failed else "completed",
|
||||
deleted_pages=deleted,
|
||||
),
|
||||
"graph_edges": graph_result.get("edge_count"),
|
||||
"graph_changed": bool(graph_result.get("changed")),
|
||||
"graph_preserved": bool(graph_result.get("preserved")),
|
||||
}
|
||||
# Persist the completed counters only while this worker still owns
|
||||
# the same lease. A stale worker can therefore never overwrite a
|
||||
# newer run's status payload.
|
||||
persisted = await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=not (rebuilding and failed),
|
||||
remote_page_count=remote_count,
|
||||
error=f"{failed} page(s) failed" if failed else None,
|
||||
result=result,
|
||||
)
|
||||
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
|
||||
if not persisted:
|
||||
lease_lost.set()
|
||||
raise WikiSyncLeaseLost("Local Wiki sync lease was lost before persisting the scan result")
|
||||
# Notify even when the remote library is now empty. The global
|
||||
# assistant base must then remove stale contributions from this
|
||||
# source instead of retaining deleted Wiki pages forever.
|
||||
@@ -170,6 +828,7 @@ class WikiSyncService:
|
||||
result["assistant_sync"] = await self.on_completed(mapping)
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
cancelled = _sync_result("cancelled", error="Sync cancelled")
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
@@ -177,11 +836,44 @@ class WikiSyncService:
|
||||
success=False,
|
||||
remote_page_count=remote_count,
|
||||
error="Sync cancelled",
|
||||
result=cancelled,
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
await self.store.finish_sync(mapping_id, owner=self.owner, sync_id=sync_id, success=False, remote_page_count=remote_count, error=_safe_error(exc))
|
||||
logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
|
||||
safe_error = _safe_error(exc)
|
||||
failed_result = _sync_result("failed", error=safe_error)
|
||||
retry_after: int | None = None
|
||||
if (
|
||||
isinstance(exc, WikiEmbeddingError)
|
||||
and exc.category != "runtime_retired"
|
||||
):
|
||||
retry_after = self._record_embedding_failure(exc)
|
||||
failed_result.update(
|
||||
{
|
||||
"failure_scope": "embedding_provider",
|
||||
"failure_count": self._embedding_failure_count,
|
||||
"retry_after_seconds": retry_after,
|
||||
}
|
||||
)
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=False,
|
||||
remote_page_count=remote_count,
|
||||
error=safe_error,
|
||||
result=failed_result,
|
||||
)
|
||||
logger.warning(
|
||||
"Local Wiki sync failed mapping_id=%s%s: %s",
|
||||
mapping_id,
|
||||
(
|
||||
f" embedding_provider_retry_after_seconds={retry_after}"
|
||||
if retry_after is not None
|
||||
else ""
|
||||
),
|
||||
safe_error,
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
|
||||
+56
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
@@ -37,9 +39,63 @@ def build_weknora_client(
|
||||
) -> WeKnoraClient:
|
||||
if not runtime.weknora_enabled:
|
||||
raise RuntimeError("WeKnora mode is not enabled")
|
||||
if http_client is None:
|
||||
return _shared_weknora_client(
|
||||
runtime.api_base_url,
|
||||
runtime.admin_email,
|
||||
runtime.admin_password,
|
||||
)
|
||||
return WeKnoraClient(
|
||||
base_url=runtime.api_base_url,
|
||||
http_client=http_client,
|
||||
admin_email=runtime.admin_email,
|
||||
admin_password=runtime.admin_password,
|
||||
)
|
||||
|
||||
|
||||
def _shared_weknora_client(
|
||||
base_url: str,
|
||||
admin_email: str,
|
||||
admin_password: str,
|
||||
) -> WeKnoraClient:
|
||||
"""Reuse auth state only inside the current event loop.
|
||||
|
||||
``WeKnoraClient`` owns asyncio primitives (an authentication lock and a
|
||||
cached directory task). A process-wide ``lru_cache`` therefore lets a
|
||||
client first used by the Gateway loop escape into a LangGraph/tool loop.
|
||||
Once either primitive has been awaited, reusing it from the other loop can
|
||||
fail with ``... is bound to a different event loop``. Smart retrieval
|
||||
used to normalize that internal exception to ``PROVIDER_UNAVAILABLE``, so
|
||||
an otherwise healthy selected knowledge base appeared empty.
|
||||
|
||||
Store the small configuration cache on the loop itself. The loop owns the
|
||||
clients for exactly its lifetime, equivalent configurations still share
|
||||
their bearer token and knowledge-base directory within that loop, and no
|
||||
async primitive crosses loop boundaries. Synchronous callers have no
|
||||
safe loop ownership yet, so they receive an uncached client.
|
||||
"""
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return WeKnoraClient(
|
||||
base_url=base_url,
|
||||
admin_email=admin_email,
|
||||
admin_password=admin_password,
|
||||
)
|
||||
|
||||
cache_name = "_deerflow_weknora_clients"
|
||||
cache = getattr(loop, cache_name, None)
|
||||
if not isinstance(cache, dict):
|
||||
cache = {}
|
||||
setattr(loop, cache_name, cache)
|
||||
key = (base_url, admin_email, admin_password)
|
||||
client = cache.get(key)
|
||||
if client is None:
|
||||
client = WeKnoraClient(
|
||||
base_url=base_url,
|
||||
admin_email=admin_email,
|
||||
admin_password=admin_password,
|
||||
)
|
||||
cache[key] = client
|
||||
return client
|
||||
Reference in new issue
Block a user