sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
+615
-41
@@ -3,12 +3,32 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
import regex
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _slow_diagnostic_threshold_ms() -> float:
|
||||
try:
|
||||
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_REQUEST_LOG_MS", "500")), 60_000.0))
|
||||
except (TypeError, ValueError):
|
||||
return 500.0
|
||||
|
||||
_GREP_CONTRACT = "weknora-v0.7.2-grep-v1"
|
||||
_GREP_COMPATIBILITY_CONTRACT = "weknora-existing-api-compat-v1"
|
||||
_GREP_MAX_CANDIDATES = 500
|
||||
_GREP_MAX_RESULTS = 30
|
||||
_GREP_REGEX_TIMEOUT_SECONDS = 0.05
|
||||
|
||||
|
||||
class WeKnoraError(RuntimeError):
|
||||
@@ -74,7 +94,7 @@ class WeKnoraClient:
|
||||
self._auth_lock = asyncio.Lock()
|
||||
self._knowledge_bases_cache: tuple[float, list[dict[str, Any]]] | None = None
|
||||
self._knowledge_bases_cache_generation = 0
|
||||
self._knowledge_bases_lock = asyncio.Lock()
|
||||
self._knowledge_bases_task: tuple[int, asyncio.Task[list[dict[str, Any]]]] | None = None
|
||||
if not self.base_url:
|
||||
raise ValueError("WeKnora base URL is required")
|
||||
|
||||
@@ -170,6 +190,11 @@ class WeKnoraClient:
|
||||
files: Any = None,
|
||||
data: Mapping[str, Any] | None = None,
|
||||
) -> Any:
|
||||
diagnose_knowledge_base_list = method.upper() == "GET" and path == "/api/v1/knowledge-bases"
|
||||
diagnostic_started = time.perf_counter() if diagnose_knowledge_base_list else None
|
||||
auth_ms = 0.0
|
||||
network_ms = 0.0
|
||||
attempts = 0
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
@@ -178,26 +203,59 @@ class WeKnoraClient:
|
||||
)
|
||||
try:
|
||||
for attempt in range(2):
|
||||
attempts += 1
|
||||
auth_started = time.perf_counter()
|
||||
request_headers = await self._headers(force_refresh=attempt > 0)
|
||||
if diagnose_knowledge_base_list:
|
||||
auth_ms += (time.perf_counter() - auth_started) * 1000
|
||||
request_started = time.perf_counter()
|
||||
response = await client.request(
|
||||
method,
|
||||
f"{self.base_url}{path}",
|
||||
headers=await self._headers(force_refresh=attempt > 0),
|
||||
headers=request_headers,
|
||||
json=json,
|
||||
params=params,
|
||||
files=files,
|
||||
data=data,
|
||||
)
|
||||
if diagnose_knowledge_base_list:
|
||||
network_ms += (time.perf_counter() - request_started) * 1000
|
||||
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
|
||||
break
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
|
||||
except Exception as exc:
|
||||
if diagnose_knowledge_base_list:
|
||||
logger.warning(
|
||||
"PERF weknora_http_request operation=knowledge_base_directory method=GET total_ms=%.2f auth_ms=%.2f network_ms=%.2f attempts=%d outcome=error error_type=%s",
|
||||
(time.perf_counter() - diagnostic_started) * 1000,
|
||||
auth_ms,
|
||||
network_ms,
|
||||
attempts,
|
||||
type(exc).__name__,
|
||||
)
|
||||
if isinstance(exc, httpx.TimeoutException):
|
||||
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
|
||||
if isinstance(exc, httpx.HTTPError):
|
||||
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
|
||||
raise
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
if diagnose_knowledge_base_list:
|
||||
total_ms = (time.perf_counter() - diagnostic_started) * 1000
|
||||
if total_ms >= _slow_diagnostic_threshold_ms() or response.status_code >= 400:
|
||||
logger.warning(
|
||||
"PERF weknora_http_request operation=knowledge_base_directory method=GET total_ms=%.2f auth_ms=%.2f network_ms=%.2f attempts=%d status=%d outcome=%s",
|
||||
total_ms,
|
||||
auth_ms,
|
||||
network_ms,
|
||||
attempts,
|
||||
response.status_code,
|
||||
"http_error" if response.status_code >= 400 else "ok",
|
||||
)
|
||||
if response.status_code == 404:
|
||||
raise WeKnoraError("WeKnora resource was not found", status_code=404)
|
||||
if response.status_code == 405:
|
||||
raise WeKnoraError("WeKnora endpoint does not support this method", status_code=405)
|
||||
if response.status_code in {401, 403}:
|
||||
if self._has_admin_credentials:
|
||||
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated internal request", status_code=502)
|
||||
@@ -213,7 +271,14 @@ class WeKnoraClient:
|
||||
detail = str(body.get("message") or body.get("error") or body.get("detail") or "")
|
||||
except ValueError:
|
||||
detail = ""
|
||||
raise WeKnoraError(detail or f"WeKnora returned HTTP {response.status_code}", status_code=502)
|
||||
# Folder creation uses 409 as an optimistic-concurrency signal.
|
||||
# Preserve it so callers can re-read and adopt the concurrently
|
||||
# created folder even if a deployment localizes the error text.
|
||||
status_code = 409 if response.status_code == 409 else 502
|
||||
raise WeKnoraError(
|
||||
detail or f"WeKnora returned HTTP {response.status_code}",
|
||||
status_code=status_code,
|
||||
)
|
||||
if response.status_code == 204 or not response.content:
|
||||
return {"success": True}
|
||||
try:
|
||||
@@ -226,33 +291,59 @@ class WeKnoraClient:
|
||||
return dict(payload) if isinstance(payload, dict) else {"status": "ok"}
|
||||
|
||||
async def list_knowledge_bases(self, *, refresh: bool = False) -> list[dict[str, Any]]:
|
||||
if refresh:
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
now = time.monotonic()
|
||||
cached = self._knowledge_bases_cache
|
||||
if cached is not None and cached[0] > now:
|
||||
if not refresh and cached is not None and cached[0] > now:
|
||||
return [dict(item) for item in cached[1]]
|
||||
|
||||
# Coalesce requests that reach the Gateway at the same time. Keeping
|
||||
# the upstream read under this narrow lock is intentional: every
|
||||
# waiter receives the same fresh snapshot, while a failed request does
|
||||
# not populate the cache and the next caller can retry normally.
|
||||
async with self._knowledge_bases_lock:
|
||||
now = time.monotonic()
|
||||
cached = self._knowledge_bases_cache
|
||||
if cached is not None and cached[0] > now:
|
||||
return [dict(item) for item in cached[1]]
|
||||
observed_generation = self._knowledge_bases_cache_generation
|
||||
# Share both success and failure with the current wave of callers.
|
||||
# A lock around an uncached failed read would make every waiter issue
|
||||
# another full timeout, serializing an unavailable service's fan-out.
|
||||
generation = self._knowledge_bases_cache_generation
|
||||
pending = self._knowledge_bases_task
|
||||
if pending is None or pending[0] != generation:
|
||||
task = asyncio.create_task(self._load_knowledge_bases(generation))
|
||||
self._knowledge_bases_task = (generation, task)
|
||||
|
||||
def clear_pending(completed: asyncio.Task[list[dict[str, Any]]]) -> None:
|
||||
if self._knowledge_bases_task is not None and self._knowledge_bases_task[1] is completed:
|
||||
self._knowledge_bases_task = None
|
||||
if not completed.cancelled():
|
||||
completed.exception()
|
||||
|
||||
task.add_done_callback(clear_pending)
|
||||
else:
|
||||
task = pending[1]
|
||||
rows = await asyncio.shield(task)
|
||||
return [dict(item) for item in rows]
|
||||
|
||||
async def _load_knowledge_bases(self, generation: int) -> list[dict[str, Any]]:
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
|
||||
# A mutation can finish while this request is in flight. Do not
|
||||
# let the preceding snapshot repopulate the cache after that
|
||||
# mutation has explicitly invalidated it.
|
||||
if self._knowledge_bases_cache_generation == observed_generation:
|
||||
self._knowledge_bases_cache = (
|
||||
time.monotonic() + self._KNOWLEDGE_BASES_CACHE_SECONDS,
|
||||
[dict(item) for item in rows],
|
||||
)
|
||||
return [dict(item) for item in rows]
|
||||
except Exception as exc:
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
logger.warning(
|
||||
"PERF weknora_knowledge_base_list upstream_ms=%.2f outcome=error error_type=%s",
|
||||
elapsed_ms,
|
||||
type(exc).__name__,
|
||||
)
|
||||
raise
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
if elapsed_ms >= _slow_diagnostic_threshold_ms():
|
||||
logger.warning(
|
||||
"PERF weknora_knowledge_base_list upstream_ms=%.2f item_count=%d outcome=ok",
|
||||
elapsed_ms,
|
||||
len(rows),
|
||||
)
|
||||
# A mutation can finish while the read is in flight. Its old snapshot
|
||||
# must neither repopulate the cache nor be joined by post-write callers.
|
||||
if self._knowledge_bases_cache_generation == generation:
|
||||
self._knowledge_bases_cache = (
|
||||
time.monotonic() + self._KNOWLEDGE_BASES_CACHE_SECONDS,
|
||||
[dict(item) for item in rows],
|
||||
)
|
||||
return rows
|
||||
|
||||
def _invalidate_knowledge_bases_cache(self) -> None:
|
||||
"""Discard the directory after a successful local mutation."""
|
||||
@@ -271,6 +362,23 @@ class WeKnoraClient:
|
||||
|
||||
return _as_list(_data(await self._request("GET", "/api/v1/models")))
|
||||
|
||||
async def debug_model(self, model_id: str) -> dict[str, Any]:
|
||||
"""Ask WeKnora to verify one saved model with a small real request."""
|
||||
|
||||
normalized_id = str(model_id or "").strip()
|
||||
if not normalized_id:
|
||||
raise WeKnoraError("WeKnora model ID is required", status_code=400)
|
||||
value = _data(
|
||||
await self._request(
|
||||
"POST",
|
||||
f"/api/v1/models/{quote(normalized_id, safe='')}/debug",
|
||||
files={"input": (None, "Reply with OK.")},
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict) or not isinstance(value.get("ok"), bool):
|
||||
raise WeKnoraError("WeKnora returned an invalid model debug result")
|
||||
return dict(value)
|
||||
|
||||
async def get_default_embedding_model_id(self) -> str:
|
||||
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "embedding"]
|
||||
preferred = next((item for item in models if item.get("is_default") is True), None)
|
||||
@@ -287,21 +395,58 @@ class WeKnoraClient:
|
||||
return model_id
|
||||
|
||||
async def get_default_knowledge_qa_model_id(self) -> str:
|
||||
"""Resolve WeKnora's own active synthesis model for Wiki indexing."""
|
||||
"""Resolve a live WeKnora synthesis model for Wiki indexing.
|
||||
|
||||
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "knowledgeqa"]
|
||||
preferred = next((item for item in models if item.get("is_default") is True), None)
|
||||
selected = preferred or next(
|
||||
(item for item in models if str(item.get("status") or "active").lower() == "active"),
|
||||
None,
|
||||
)
|
||||
model_id = str((selected or {}).get("id") or "")
|
||||
if not model_id:
|
||||
WeKnora's ``active`` status means that the database configuration is
|
||||
enabled; it does not prove that the upstream provider still serves the
|
||||
model. Probe active candidates before binding a new Wiki so a disabled
|
||||
provider model cannot silently send every ingest task to dead letters.
|
||||
"""
|
||||
|
||||
models = [
|
||||
item
|
||||
for item in await self.list_models()
|
||||
if str(item.get("type") or "").strip().lower() == "knowledgeqa"
|
||||
and str(item.get("id") or "").strip()
|
||||
and str(item.get("status") or "active").strip().lower() == "active"
|
||||
]
|
||||
if not models:
|
||||
raise WeKnoraError(
|
||||
"WeKnora has no available KnowledgeQA model; configure one in WeKnora before creating a Wiki",
|
||||
status_code=400,
|
||||
)
|
||||
return model_id
|
||||
|
||||
# Prefer an explicit database default. When no default exists, sort by
|
||||
# immutable persisted ID rather than trusting API row order or an
|
||||
# editable display name.
|
||||
models.sort(
|
||||
key=lambda item: (
|
||||
0 if item.get("is_default") is True else 1,
|
||||
str(item.get("id") or "").strip(),
|
||||
)
|
||||
)
|
||||
attempted: list[str] = []
|
||||
for model in models:
|
||||
model_id = str(model.get("id") or "").strip()
|
||||
label = str(model.get("display_name") or model.get("name") or model_id).strip()
|
||||
attempted.append(label or model_id)
|
||||
# ``ok=false`` is a candidate-specific upstream-model failure. An
|
||||
# HTTP/auth/timeout failure from the debug endpoint is a WeKnora
|
||||
# service failure, so preserve it instead of multiplying probes
|
||||
# and misreporting the incident as bad model configuration.
|
||||
debug = await self.debug_model(model_id)
|
||||
if debug.get("ok") is True:
|
||||
return model_id
|
||||
|
||||
attempted_models = ", ".join(attempted[:12])
|
||||
if len(attempted) > 12:
|
||||
attempted_models = f"{attempted_models}, and {len(attempted) - 12} more"
|
||||
raise WeKnoraError(
|
||||
"WeKnora has no usable active KnowledgeQA model after connectivity checks"
|
||||
+ (f": {attempted_models}" if attempted_models else "")
|
||||
+ "; configure or repair a working default model before creating a Wiki",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
async def create_knowledge_base(
|
||||
self,
|
||||
@@ -310,6 +455,7 @@ class WeKnoraClient:
|
||||
description: str = "",
|
||||
kb_type: str = "document",
|
||||
embedding_model_id: str | None = None,
|
||||
knowledge_qa_model_id: str | None = None,
|
||||
wiki_enabled: bool = False,
|
||||
wiki_content_instructions: str = "",
|
||||
wiki_extraction_instructions: str = "",
|
||||
@@ -322,7 +468,7 @@ class WeKnoraClient:
|
||||
"embedding_model_id": model_id,
|
||||
}
|
||||
if wiki_enabled:
|
||||
synthesis_model_id = await self.get_default_knowledge_qa_model_id()
|
||||
synthesis_model_id = knowledge_qa_model_id or await self.get_default_knowledge_qa_model_id()
|
||||
create_payload.update(
|
||||
{
|
||||
"summary_model_id": synthesis_model_id,
|
||||
@@ -577,6 +723,79 @@ class WeKnoraClient:
|
||||
raise WeKnoraError("Invalid Wiki folder response")
|
||||
return value
|
||||
|
||||
async def list_wiki_folders(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
*,
|
||||
parent_id: str = "",
|
||||
page_types: Sequence[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
params: dict[str, Any] = {"parent_id": parent_id or None}
|
||||
if page_types:
|
||||
params["page_types"] = ",".join(str(value) for value in page_types if str(value)) or None
|
||||
payload = await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders",
|
||||
params=params,
|
||||
)
|
||||
value = _data(payload)
|
||||
source = value if isinstance(value, dict) else {}
|
||||
raw_folders = source.get("folders") if isinstance(value, dict) else value
|
||||
rows = _as_list(raw_folders)
|
||||
return {
|
||||
"parent_id": str(source.get("parent_id") or parent_id),
|
||||
"folders": rows,
|
||||
}
|
||||
|
||||
async def update_wiki_folder(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
folder_id: str,
|
||||
changes: Mapping[str, Any] | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
parent_id: str | None = None,
|
||||
move_parent: bool | None = None,
|
||||
) -> dict[str, Any]:
|
||||
update = dict(changes or {})
|
||||
if name is not None:
|
||||
update["name"] = name
|
||||
if parent_id is not None:
|
||||
update["parent_id"] = parent_id
|
||||
update.setdefault("move_parent", True)
|
||||
if move_parent is not None:
|
||||
update["move_parent"] = move_parent
|
||||
elif "parent_id" in update:
|
||||
update.setdefault("move_parent", True)
|
||||
value = _data(
|
||||
await self._request(
|
||||
"PUT",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders/{quote(folder_id, safe='')}",
|
||||
json=update,
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("Invalid Wiki folder response")
|
||||
return dict(value)
|
||||
|
||||
async def delete_wiki_folder(self, knowledge_base_id: str, folder_id: str) -> None:
|
||||
await self._request(
|
||||
"DELETE",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders/{quote(folder_id, safe='')}",
|
||||
)
|
||||
|
||||
async def move_wiki_page(self, knowledge_base_id: str, slug: str, folder_id: str) -> dict[str, Any]:
|
||||
value = _data(
|
||||
await self._request(
|
||||
"PUT",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/move-page",
|
||||
json={"slug": slug, "folder_id": folder_id},
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("Invalid Wiki page response")
|
||||
return dict(value)
|
||||
|
||||
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
|
||||
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
|
||||
|
||||
@@ -587,11 +806,17 @@ class WeKnoraClient:
|
||||
page: int = 1,
|
||||
page_size: int = 50,
|
||||
query: str = "",
|
||||
folder_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
params: dict[str, Any] = {"page": page, "page_size": page_size, "query": query or None}
|
||||
# ``None`` means no placement filter; an explicit empty string means
|
||||
# pages directly under the Wiki root.
|
||||
if folder_id is not None:
|
||||
params["folder_id"] = folder_id
|
||||
payload = await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages",
|
||||
params={"page": page, "page_size": page_size, "query": query or None},
|
||||
params=params,
|
||||
)
|
||||
value = _data(payload)
|
||||
source = value if isinstance(value, dict) else payload if isinstance(payload, dict) else {}
|
||||
@@ -604,6 +829,34 @@ class WeKnoraClient:
|
||||
"total_pages": int(source.get("total_pages") or max(1, (int(source.get("total") or len(rows)) + page_size - 1) // page_size)),
|
||||
}
|
||||
|
||||
async def search_wiki_pages(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
query: str,
|
||||
*,
|
||||
limit: int = 30,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Use WeKnora's native, ranked Wiki regex search.
|
||||
|
||||
WeKnora v0.7.2 exposes this as a single-library ``GET`` endpoint with
|
||||
the regex in ``q`` (not the list endpoint's ``query`` parameter). It
|
||||
returns complete WikiPage rows, so callers do not need to enumerate a
|
||||
directory and issue one detail request per page.
|
||||
"""
|
||||
|
||||
payload = await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/search",
|
||||
params={
|
||||
"q": str(query or ""),
|
||||
"limit": max(1, min(int(limit), 50)),
|
||||
},
|
||||
)
|
||||
value = _data(payload)
|
||||
if isinstance(value, dict):
|
||||
return _as_list(value.get("pages"))
|
||||
return _as_list(value)
|
||||
|
||||
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict[str, Any]:
|
||||
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
|
||||
value = _data(
|
||||
@@ -671,6 +924,327 @@ class WeKnoraClient:
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_grep_chunk(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Normalize both the provider extension and compatibility rows."""
|
||||
|
||||
return {
|
||||
"chunk_id": str(row.get("id") or row.get("chunk_id") or ""),
|
||||
"content": str(row.get("content") or row.get("text") or row.get("match_snippet") or ""),
|
||||
"knowledge_id": str(row.get("knowledge_id") or ""),
|
||||
"knowledge_base_id": str(row.get("knowledge_base_id") or ""),
|
||||
"title": str(
|
||||
row.get("knowledge_title")
|
||||
or row.get("title")
|
||||
or row.get("knowledge_filename")
|
||||
or ""
|
||||
),
|
||||
"chunk_index": int(row.get("chunk_index") or row.get("index") or 0),
|
||||
"match_count": int(
|
||||
row.get("matched_patterns")
|
||||
or row.get("match_count")
|
||||
or row.get("total_pattern_hits")
|
||||
or 0
|
||||
),
|
||||
"title_match": bool(row.get("title_match")),
|
||||
"score": float(row.get("match_score") or row.get("score") or 0.0),
|
||||
"metadata": row.get("metadata") if isinstance(row.get("metadata"), dict) else {},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _grep_matches(expression: regex.Pattern, value: str) -> list[regex.Match]:
|
||||
try:
|
||||
return list(
|
||||
expression.finditer(
|
||||
value,
|
||||
timeout=_GREP_REGEX_TIMEOUT_SECONDS,
|
||||
)
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise WeKnoraError(
|
||||
"WeKnora compatibility grep regular expression timed out",
|
||||
status_code=422,
|
||||
) from exc
|
||||
|
||||
async def _grep_chunks_compatibility(
|
||||
self,
|
||||
query: str,
|
||||
knowledge_base_ids: Sequence[str],
|
||||
) -> dict[str, Any]:
|
||||
"""Bounded, fair fallback for older providers without ``chunks/grep``.
|
||||
|
||||
The native endpoint remains authoritative. This path round-robins both
|
||||
knowledge bases and documents so one large first scope cannot consume
|
||||
the global 500-chunk compatibility budget.
|
||||
"""
|
||||
|
||||
ids = list(dict.fromkeys(str(value).strip() for value in knowledge_base_ids if str(value).strip()))
|
||||
try:
|
||||
expression = regex.compile(query, regex.IGNORECASE | regex.VERSION1)
|
||||
except regex.error as exc:
|
||||
raise WeKnoraError(f"Invalid grep regular expression: {exc}", status_code=422) from exc
|
||||
|
||||
per_scope_document_cap = max(1, (_GREP_MAX_CANDIDATES + len(ids) - 1) // max(1, len(ids)))
|
||||
|
||||
async def load_documents(knowledge_base_id: str) -> tuple[str, list[dict[str, Any]] | Exception]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
page = 1
|
||||
try:
|
||||
while len(rows) < per_scope_document_cap:
|
||||
page_size = min(100, per_scope_document_cap - len(rows))
|
||||
payload = await self.list_documents(
|
||||
knowledge_base_id,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
batch = [
|
||||
dict(item)
|
||||
for item in payload.get("items") or []
|
||||
if isinstance(item, Mapping)
|
||||
]
|
||||
rows.extend(batch)
|
||||
if not batch or page * page_size >= int(payload.get("total") or len(rows)):
|
||||
break
|
||||
page += 1
|
||||
return knowledge_base_id, rows
|
||||
except Exception as exc: # noqa: BLE001 - returned as structured scope failure
|
||||
return knowledge_base_id, exc
|
||||
|
||||
loaded = await asyncio.gather(*(load_documents(knowledge_base_id) for knowledge_base_id in ids))
|
||||
documents_by_scope: dict[str, list[dict[str, Any]]] = {}
|
||||
warnings: list[dict[str, Any]] = [
|
||||
{
|
||||
"code": "WEKNORA_GREP_COMPATIBILITY_FALLBACK",
|
||||
"message": "Provider chunks/grep is unavailable; used bounded existing-API compatibility scan.",
|
||||
}
|
||||
]
|
||||
failed_scope_ids: set[str] = set()
|
||||
for knowledge_base_id, outcome in loaded:
|
||||
if isinstance(outcome, Exception):
|
||||
failed_scope_ids.add(knowledge_base_id)
|
||||
warnings.append(
|
||||
{
|
||||
"code": "PROVIDER_SCOPE_FAILED",
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
"stage": "list_documents",
|
||||
}
|
||||
)
|
||||
else:
|
||||
documents_by_scope[knowledge_base_id] = outcome
|
||||
|
||||
if not documents_by_scope:
|
||||
raise WeKnoraError("WeKnora compatibility grep failed for every knowledge base")
|
||||
|
||||
# Seed the work queue in KB/document round-robin order. Subsequent
|
||||
# chunk pages are re-appended to the tail, preserving document fairness.
|
||||
work: deque[tuple[str, dict[str, Any], int]] = deque()
|
||||
max_documents = max((len(rows) for rows in documents_by_scope.values()), default=0)
|
||||
for index in range(max_documents):
|
||||
for knowledge_base_id in ids:
|
||||
rows = documents_by_scope.get(knowledge_base_id) or []
|
||||
if index < len(rows):
|
||||
work.append((knowledge_base_id, rows[index], 1))
|
||||
|
||||
candidates: list[dict[str, Any]] = []
|
||||
seen_content: set[str] = set()
|
||||
scanned_chunks = 0
|
||||
chunk_successes = {knowledge_base_id: 0 for knowledge_base_id in documents_by_scope}
|
||||
chunk_failures = {knowledge_base_id: 0 for knowledge_base_id in documents_by_scope}
|
||||
while work and scanned_chunks < _GREP_MAX_CANDIDATES:
|
||||
knowledge_base_id, document, page = work.popleft()
|
||||
knowledge_id = str(document.get("id") or document.get("knowledge_id") or "")
|
||||
if not knowledge_id:
|
||||
continue
|
||||
page_size = min(20, _GREP_MAX_CANDIDATES - scanned_chunks)
|
||||
try:
|
||||
payload = await self.list_chunks(
|
||||
knowledge_id,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - one document must not fail the whole fallback
|
||||
chunk_failures[knowledge_base_id] += 1
|
||||
continue
|
||||
chunk_successes[knowledge_base_id] += 1
|
||||
chunks = [
|
||||
dict(item)
|
||||
for item in payload.get("items") or []
|
||||
if isinstance(item, Mapping)
|
||||
]
|
||||
title = str(
|
||||
document.get("title")
|
||||
or document.get("name")
|
||||
or document.get("filename")
|
||||
or ""
|
||||
)
|
||||
title_matches = self._grep_matches(expression, title[:5000])
|
||||
for chunk in chunks:
|
||||
if scanned_chunks >= _GREP_MAX_CANDIDATES:
|
||||
break
|
||||
scanned_chunks += 1
|
||||
content = str(chunk.get("content") or chunk.get("text") or "")
|
||||
content_matches = self._grep_matches(expression, content[:50_000])
|
||||
if not title_matches and not content_matches:
|
||||
continue
|
||||
signature = hashlib.sha256(" ".join(content.split()).encode()).hexdigest()
|
||||
if signature in seen_content:
|
||||
continue
|
||||
seen_content.add(signature)
|
||||
match_count = min(99, len(title_matches) * 4 + len(content_matches))
|
||||
candidates.append(
|
||||
self._normalize_grep_chunk(
|
||||
{
|
||||
**chunk,
|
||||
"knowledge_id": knowledge_id,
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
"knowledge_title": title,
|
||||
"match_count": match_count,
|
||||
"title_match": bool(title_matches),
|
||||
"match_score": float(match_count),
|
||||
}
|
||||
)
|
||||
)
|
||||
if chunks and page * page_size < int(payload.get("total") or len(chunks)):
|
||||
work.append((knowledge_base_id, document, page + 1))
|
||||
|
||||
successful_scope_ids: list[str] = []
|
||||
for knowledge_base_id, documents in documents_by_scope.items():
|
||||
# An empty KB was successfully scanned. A non-empty KB for which
|
||||
# every chunk read failed is a failed provider scope.
|
||||
if not documents or chunk_successes[knowledge_base_id] > 0:
|
||||
successful_scope_ids.append(knowledge_base_id)
|
||||
else:
|
||||
failed_scope_ids.add(knowledge_base_id)
|
||||
if chunk_failures[knowledge_base_id]:
|
||||
warnings.append(
|
||||
{
|
||||
"code": "PROVIDER_DOCUMENT_READ_FAILED",
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
"failed_documents": chunk_failures[knowledge_base_id],
|
||||
}
|
||||
)
|
||||
|
||||
if not successful_scope_ids:
|
||||
raise WeKnoraError("WeKnora compatibility grep failed for every knowledge base")
|
||||
|
||||
candidates.sort(
|
||||
key=lambda item: (
|
||||
bool(item.get("title_match")),
|
||||
int(item.get("match_count") or 0),
|
||||
float(item.get("score") or 0),
|
||||
-int(item.get("chunk_index") or 0),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return {
|
||||
"items": candidates[:_GREP_MAX_RESULTS],
|
||||
"contract": _GREP_COMPATIBILITY_CONTRACT,
|
||||
"compatibility_fallback": True,
|
||||
"partial": bool(failed_scope_ids or any(chunk_failures.values())),
|
||||
"warnings": warnings,
|
||||
"successful_knowledge_base_ids": successful_scope_ids,
|
||||
"failed_knowledge_base_ids": [value for value in ids if value in failed_scope_ids],
|
||||
"scanned_chunks": scanned_chunks,
|
||||
"scan_truncated": scanned_chunks >= _GREP_MAX_CANDIDATES,
|
||||
}
|
||||
|
||||
async def grep_chunks(
|
||||
self,
|
||||
query: str,
|
||||
knowledge_base_ids: Sequence[str],
|
||||
) -> dict[str, Any]:
|
||||
"""Run the v0.7.2 cross-KB grep extension, with a marked fallback."""
|
||||
|
||||
ids = list(dict.fromkeys(str(value).strip() for value in knowledge_base_ids if str(value).strip()))
|
||||
if not ids:
|
||||
return {
|
||||
"items": [],
|
||||
"contract": _GREP_CONTRACT,
|
||||
"compatibility_fallback": False,
|
||||
"partial": False,
|
||||
"warnings": [],
|
||||
"successful_knowledge_base_ids": [],
|
||||
"failed_knowledge_base_ids": [],
|
||||
"scanned_chunks": 0,
|
||||
"scan_truncated": False,
|
||||
}
|
||||
try:
|
||||
payload = await self._request(
|
||||
"POST",
|
||||
"/api/v1/chunks/grep",
|
||||
json={"query": query, "knowledge_base_ids": ids},
|
||||
)
|
||||
except WeKnoraError as exc:
|
||||
if exc.status_code not in {404, 405}:
|
||||
raise
|
||||
return await self._grep_chunks_compatibility(query, ids)
|
||||
|
||||
value = _data(payload)
|
||||
source = value if isinstance(value, dict) else {}
|
||||
contract = str(
|
||||
source.get("contract")
|
||||
or (payload.get("contract") if isinstance(payload, dict) else "")
|
||||
or ""
|
||||
)
|
||||
if contract != _GREP_CONTRACT:
|
||||
raise WeKnoraError(
|
||||
f"WeKnora chunks/grep contract mismatch (expected {_GREP_CONTRACT})"
|
||||
)
|
||||
raw_rows = (
|
||||
source.get("chunk_results")
|
||||
or source.get("results")
|
||||
or source.get("items")
|
||||
or []
|
||||
)
|
||||
if not isinstance(raw_rows, list):
|
||||
raise WeKnoraError("WeKnora chunks/grep returned invalid results")
|
||||
rows = [
|
||||
self._normalize_grep_chunk(row)
|
||||
for row in raw_rows
|
||||
if isinstance(row, Mapping)
|
||||
]
|
||||
warnings = [
|
||||
dict(item)
|
||||
for item in source.get("warnings") or []
|
||||
if isinstance(item, Mapping)
|
||||
]
|
||||
failed_ids = [
|
||||
str(value)
|
||||
for value in source.get("failed_knowledge_base_ids") or []
|
||||
if str(value) in ids
|
||||
]
|
||||
explicit_successes = source.get("successful_knowledge_base_ids")
|
||||
successful_ids = (
|
||||
[str(value) for value in explicit_successes if str(value) in ids]
|
||||
if isinstance(explicit_successes, list)
|
||||
else [value for value in ids if value not in failed_ids]
|
||||
)
|
||||
warned_scope_ids = {
|
||||
str(item.get("knowledge_base_id") or "")
|
||||
for item in warnings
|
||||
}
|
||||
warnings.extend(
|
||||
{
|
||||
"code": "PROVIDER_SCOPE_FAILED",
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
}
|
||||
for knowledge_base_id in failed_ids
|
||||
if knowledge_base_id not in warned_scope_ids
|
||||
)
|
||||
if source.get("partial") and not warnings:
|
||||
warnings.append({"code": "PROVIDER_PARTIAL_RESPONSE"})
|
||||
return {
|
||||
"items": rows[:_GREP_MAX_RESULTS],
|
||||
"contract": contract,
|
||||
"compatibility_fallback": False,
|
||||
"partial": bool(source.get("partial")) or bool(warnings) or bool(failed_ids),
|
||||
"warnings": warnings,
|
||||
"successful_knowledge_base_ids": successful_ids,
|
||||
"failed_knowledge_base_ids": failed_ids,
|
||||
"scanned_chunks": int(source.get("scanned_chunks") or len(rows)),
|
||||
"scan_truncated": bool(source.get("scan_truncated")),
|
||||
}
|
||||
|
||||
async def search(self, query: str, knowledge_base_ids: Sequence[str]) -> list[dict[str, Any]]:
|
||||
ids = [str(value) for value in knowledge_base_ids if str(value)]
|
||||
if not ids:
|
||||
|
||||
+209
-7
@@ -6,6 +6,8 @@ import asyncio
|
||||
import logging
|
||||
import platform
|
||||
from datetime import UTC, datetime
|
||||
from math import ceil
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -24,6 +26,15 @@ from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# This breaker is intentionally process-local. The durable sync row already
|
||||
# records the failed run and error, while a transient retry deadline does not
|
||||
# belong in release files or administrator configuration. Rebuilding the
|
||||
# runtime after a provider/config change (or restarting Gateway) resets the
|
||||
# breaker and therefore allows one immediate recovery probe. The state is a
|
||||
# fixed-size provider record rather than a per-page/per-mapping collection.
|
||||
_EMBEDDING_PROVIDER_BACKOFF_INITIAL_SECONDS = 60.0
|
||||
_EMBEDDING_PROVIDER_BACKOFF_MAX_SECONDS = 30.0 * 60.0
|
||||
|
||||
|
||||
class WikiSyncLeaseLost(RuntimeError):
|
||||
pass
|
||||
@@ -251,6 +262,163 @@ class WikiSyncService:
|
||||
self.client = client
|
||||
self.owner = f"{platform.node()}:{uuid4()}"
|
||||
self.on_completed = None
|
||||
self._embedding_probe_lock = asyncio.Lock()
|
||||
self._embedding_failure_count = 0
|
||||
self._embedding_retry_at = 0.0
|
||||
self._embedding_last_error: str | None = None
|
||||
|
||||
def _embedding_backoff_result(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
now: float | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
checked_at = monotonic() if now is None else now
|
||||
remaining = self._embedding_retry_at - checked_at
|
||||
if self._embedding_failure_count <= 0 or remaining <= 0:
|
||||
return None
|
||||
return {
|
||||
"mapping_id": mapping_id,
|
||||
"status": "provider_backoff",
|
||||
"failure_scope": "embedding_provider",
|
||||
"failure_count": self._embedding_failure_count,
|
||||
"retry_after_seconds": max(1, ceil(remaining)),
|
||||
"error": self._embedding_last_error,
|
||||
}
|
||||
|
||||
def _record_embedding_failure(self, exc: WikiEmbeddingError) -> int:
|
||||
"""Open or extend one provider-wide exponential breaker window.
|
||||
|
||||
Several mappings may already be running under the scheduler semaphore
|
||||
when the provider goes away. Failures arriving inside an already-open
|
||||
window share that window instead of multiplying the exponent.
|
||||
"""
|
||||
|
||||
now = monotonic()
|
||||
if self._embedding_retry_at <= now:
|
||||
self._embedding_failure_count = min(
|
||||
self._embedding_failure_count + 1,
|
||||
31,
|
||||
)
|
||||
exponent = min(self._embedding_failure_count - 1, 30)
|
||||
delay = min(
|
||||
_EMBEDDING_PROVIDER_BACKOFF_MAX_SECONDS,
|
||||
_EMBEDDING_PROVIDER_BACKOFF_INITIAL_SECONDS * (2**exponent),
|
||||
)
|
||||
self._embedding_retry_at = now + delay
|
||||
self._embedding_last_error = _safe_error(exc)
|
||||
return max(1, ceil(self._embedding_retry_at - now))
|
||||
|
||||
def _reset_embedding_breaker(self) -> None:
|
||||
self._embedding_failure_count = 0
|
||||
self._embedding_retry_at = 0.0
|
||||
self._embedding_last_error = None
|
||||
|
||||
async def _embedding_preflight(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
force: bool,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Skip a known outage or perform one serialized recovery probe."""
|
||||
|
||||
if not force:
|
||||
backoff = self._embedding_backoff_result(mapping_id)
|
||||
if backoff is not None:
|
||||
return backoff
|
||||
async with self._embedding_probe_lock:
|
||||
if not force:
|
||||
backoff = self._embedding_backoff_result(mapping_id)
|
||||
if backoff is not None:
|
||||
return backoff
|
||||
needs_probe = (
|
||||
self.embedding.dimensions is None
|
||||
or self._embedding_failure_count > 0
|
||||
)
|
||||
if not needs_probe:
|
||||
return None
|
||||
try:
|
||||
# Auto-dimension configurations need one successful response
|
||||
# for a stable fingerprint. After an outage the same tiny
|
||||
# request proves provider recovery before any Wiki directory
|
||||
# or page is enumerated.
|
||||
await self.embedding.embed_texts(
|
||||
["DeerFlow local Wiki embedding provider health probe"]
|
||||
)
|
||||
except WikiEmbeddingError as exc:
|
||||
if exc.category != "runtime_retired":
|
||||
retry_after = self._record_embedding_failure(exc)
|
||||
logger.warning(
|
||||
"Local Wiki embedding provider probe failed mapping_id=%s retry_after_seconds=%s: %s",
|
||||
mapping_id,
|
||||
retry_after,
|
||||
_safe_error(exc),
|
||||
)
|
||||
raise
|
||||
self._reset_embedding_breaker()
|
||||
return None
|
||||
|
||||
async def _persist_requested_backoff(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
sync_id: str,
|
||||
mode: str,
|
||||
started_at: str,
|
||||
backoff: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Finish an explicitly requested job with a frontend-compatible state."""
|
||||
|
||||
states = await self.store.list_index_status([mapping_id])
|
||||
prior = states[0] if states else {}
|
||||
remote_count = int(prior.get("remote_page_count") or 0)
|
||||
acquired = await self.store.try_acquire_sync_lease(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
lease_seconds=self.config.sync_lease_seconds,
|
||||
fingerprint=self.embedding.fingerprint,
|
||||
)
|
||||
if not acquired:
|
||||
return backoff
|
||||
error = str(
|
||||
backoff.get("error")
|
||||
or "Wiki embedding provider is temporarily unavailable"
|
||||
)
|
||||
result = {
|
||||
"sync_id": sync_id,
|
||||
"mapping_id": mapping_id,
|
||||
"mode": mode,
|
||||
"status": "failed",
|
||||
"remote_pages": remote_count,
|
||||
"added_pages": 0,
|
||||
"updated_pages": 0,
|
||||
"metadata_updated_pages": 0,
|
||||
"revectorized_pages": 0,
|
||||
"changed_pages": 0,
|
||||
"skipped_pages": 0,
|
||||
"deleted_pages": 0,
|
||||
"failed_pages": 0,
|
||||
"started_at": started_at,
|
||||
"completed_at": _iso_now(),
|
||||
"error": error,
|
||||
"failure_scope": backoff.get("failure_scope")
|
||||
or "embedding_provider",
|
||||
"failure_count": int(backoff.get("failure_count") or 0),
|
||||
"retry_after_seconds": int(
|
||||
backoff.get("retry_after_seconds") or 0
|
||||
),
|
||||
}
|
||||
persisted = await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=False,
|
||||
remote_page_count=remote_count,
|
||||
error=error,
|
||||
result=result,
|
||||
)
|
||||
return result if persisted else backoff
|
||||
|
||||
async def sync_mapping(
|
||||
self,
|
||||
@@ -261,13 +429,21 @@ class WikiSyncService:
|
||||
sync_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
mapping_id = str(mapping["id"])
|
||||
requested_sync_id = sync_id
|
||||
sync_id = sync_id or str(uuid4())
|
||||
mode = "full" if force else "incremental"
|
||||
started_at = _iso_now()
|
||||
if self.embedding.dimensions is None:
|
||||
# Auto-dimension configurations need one successful response to
|
||||
# produce a stable fingerprint before lease/rebuild decisions.
|
||||
await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
|
||||
backoff = await self._embedding_preflight(mapping_id, force=force)
|
||||
if backoff is not None:
|
||||
if requested_sync_id is not None:
|
||||
return await self._persist_requested_backoff(
|
||||
mapping_id,
|
||||
sync_id=sync_id,
|
||||
mode=mode,
|
||||
started_at=started_at,
|
||||
backoff=backoff,
|
||||
)
|
||||
return backoff
|
||||
fingerprint = self.embedding.fingerprint
|
||||
revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
|
||||
if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
|
||||
@@ -480,7 +656,10 @@ class WikiSyncService:
|
||||
except Exception as exc:
|
||||
if isinstance(exc, WikiSyncLeaseLost):
|
||||
raise
|
||||
if isinstance(exc, WikiEmbeddingError) and exc.category == "runtime_retired":
|
||||
# Embedding failures describe the shared provider, not
|
||||
# one malformed Wiki page. Continuing would repeat the
|
||||
# client's own retry loop for every remaining page.
|
||||
if isinstance(exc, WikiEmbeddingError):
|
||||
raise
|
||||
if not graph_page_captured:
|
||||
graph_snapshot_complete = False
|
||||
@@ -662,6 +841,20 @@ class WikiSyncService:
|
||||
raise
|
||||
except Exception as exc:
|
||||
safe_error = _safe_error(exc)
|
||||
failed_result = _sync_result("failed", error=safe_error)
|
||||
retry_after: int | None = None
|
||||
if (
|
||||
isinstance(exc, WikiEmbeddingError)
|
||||
and exc.category != "runtime_retired"
|
||||
):
|
||||
retry_after = self._record_embedding_failure(exc)
|
||||
failed_result.update(
|
||||
{
|
||||
"failure_scope": "embedding_provider",
|
||||
"failure_count": self._embedding_failure_count,
|
||||
"retry_after_seconds": retry_after,
|
||||
}
|
||||
)
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
@@ -669,9 +862,18 @@ class WikiSyncService:
|
||||
success=False,
|
||||
remote_page_count=remote_count,
|
||||
error=safe_error,
|
||||
result=_sync_result("failed", error=safe_error),
|
||||
result=failed_result,
|
||||
)
|
||||
logger.warning(
|
||||
"Local Wiki sync failed mapping_id=%s%s: %s",
|
||||
mapping_id,
|
||||
(
|
||||
f" embedding_provider_retry_after_seconds={retry_after}"
|
||||
if retry_after is not None
|
||||
else ""
|
||||
),
|
||||
safe_error,
|
||||
)
|
||||
logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, safe_error)
|
||||
raise
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
|
||||
+41
-8
@@ -2,7 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -53,16 +53,49 @@ def build_weknora_client(
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def _shared_weknora_client(
|
||||
base_url: str,
|
||||
admin_email: str,
|
||||
admin_password: str,
|
||||
) -> WeKnoraClient:
|
||||
"""Reuse auth state for equivalent runtime configurations."""
|
||||
"""Reuse auth state only inside the current event loop.
|
||||
|
||||
return WeKnoraClient(
|
||||
base_url=base_url,
|
||||
admin_email=admin_email,
|
||||
admin_password=admin_password,
|
||||
)
|
||||
``WeKnoraClient`` owns asyncio primitives (an authentication lock and a
|
||||
cached directory task). A process-wide ``lru_cache`` therefore lets a
|
||||
client first used by the Gateway loop escape into a LangGraph/tool loop.
|
||||
Once either primitive has been awaited, reusing it from the other loop can
|
||||
fail with ``... is bound to a different event loop``. Smart retrieval
|
||||
used to normalize that internal exception to ``PROVIDER_UNAVAILABLE``, so
|
||||
an otherwise healthy selected knowledge base appeared empty.
|
||||
|
||||
Store the small configuration cache on the loop itself. The loop owns the
|
||||
clients for exactly its lifetime, equivalent configurations still share
|
||||
their bearer token and knowledge-base directory within that loop, and no
|
||||
async primitive crosses loop boundaries. Synchronous callers have no
|
||||
safe loop ownership yet, so they receive an uncached client.
|
||||
"""
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return WeKnoraClient(
|
||||
base_url=base_url,
|
||||
admin_email=admin_email,
|
||||
admin_password=admin_password,
|
||||
)
|
||||
|
||||
cache_name = "_deerflow_weknora_clients"
|
||||
cache = getattr(loop, cache_name, None)
|
||||
if not isinstance(cache, dict):
|
||||
cache = {}
|
||||
setattr(loop, cache_name, cache)
|
||||
key = (base_url, admin_email, admin_password)
|
||||
client = cache.get(key)
|
||||
if client is None:
|
||||
client = WeKnoraClient(
|
||||
base_url=base_url,
|
||||
admin_email=admin_email,
|
||||
admin_password=admin_password,
|
||||
)
|
||||
cache[key] = client
|
||||
return client
|
||||
Reference in new issue
Block a user