sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -3,12 +3,32 @@
from __future__ import annotations
import asyncio
import hashlib
import logging
import os
import time
from collections import deque
from collections.abc import Mapping, Sequence
from typing import Any
from urllib.parse import quote
import httpx
import regex
logger = logging.getLogger(__name__)
def _slow_diagnostic_threshold_ms() -> float:
try:
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_REQUEST_LOG_MS", "500")), 60_000.0))
except (TypeError, ValueError):
return 500.0
_GREP_CONTRACT = "weknora-v0.7.2-grep-v1"
_GREP_COMPATIBILITY_CONTRACT = "weknora-existing-api-compat-v1"
_GREP_MAX_CANDIDATES = 500
_GREP_MAX_RESULTS = 30
_GREP_REGEX_TIMEOUT_SECONDS = 0.05
class WeKnoraError(RuntimeError):
@@ -74,7 +94,7 @@ class WeKnoraClient:
self._auth_lock = asyncio.Lock()
self._knowledge_bases_cache: tuple[float, list[dict[str, Any]]] | None = None
self._knowledge_bases_cache_generation = 0
self._knowledge_bases_lock = asyncio.Lock()
self._knowledge_bases_task: tuple[int, asyncio.Task[list[dict[str, Any]]]] | None = None
if not self.base_url:
raise ValueError("WeKnora base URL is required")
@@ -170,6 +190,11 @@ class WeKnoraClient:
files: Any = None,
data: Mapping[str, Any] | None = None,
) -> Any:
diagnose_knowledge_base_list = method.upper() == "GET" and path == "/api/v1/knowledge-bases"
diagnostic_started = time.perf_counter() if diagnose_knowledge_base_list else None
auth_ms = 0.0
network_ms = 0.0
attempts = 0
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
@@ -178,26 +203,59 @@ class WeKnoraClient:
)
try:
for attempt in range(2):
attempts += 1
auth_started = time.perf_counter()
request_headers = await self._headers(force_refresh=attempt > 0)
if diagnose_knowledge_base_list:
auth_ms += (time.perf_counter() - auth_started) * 1000
request_started = time.perf_counter()
response = await client.request(
method,
f"{self.base_url}{path}",
headers=await self._headers(force_refresh=attempt > 0),
headers=request_headers,
json=json,
params=params,
files=files,
data=data,
)
if diagnose_knowledge_base_list:
network_ms += (time.perf_counter() - request_started) * 1000
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
break
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
except Exception as exc:
if diagnose_knowledge_base_list:
logger.warning(
"PERF weknora_http_request operation=knowledge_base_directory method=GET total_ms=%.2f auth_ms=%.2f network_ms=%.2f attempts=%d outcome=error error_type=%s",
(time.perf_counter() - diagnostic_started) * 1000,
auth_ms,
network_ms,
attempts,
type(exc).__name__,
)
if isinstance(exc, httpx.TimeoutException):
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
if isinstance(exc, httpx.HTTPError):
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
raise
finally:
if owns_client:
await client.aclose()
if diagnose_knowledge_base_list:
total_ms = (time.perf_counter() - diagnostic_started) * 1000
if total_ms >= _slow_diagnostic_threshold_ms() or response.status_code >= 400:
logger.warning(
"PERF weknora_http_request operation=knowledge_base_directory method=GET total_ms=%.2f auth_ms=%.2f network_ms=%.2f attempts=%d status=%d outcome=%s",
total_ms,
auth_ms,
network_ms,
attempts,
response.status_code,
"http_error" if response.status_code >= 400 else "ok",
)
if response.status_code == 404:
raise WeKnoraError("WeKnora resource was not found", status_code=404)
if response.status_code == 405:
raise WeKnoraError("WeKnora endpoint does not support this method", status_code=405)
if response.status_code in {401, 403}:
if self._has_admin_credentials:
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated internal request", status_code=502)
@@ -213,7 +271,14 @@ class WeKnoraClient:
detail = str(body.get("message") or body.get("error") or body.get("detail") or "")
except ValueError:
detail = ""
raise WeKnoraError(detail or f"WeKnora returned HTTP {response.status_code}", status_code=502)
# Folder creation uses 409 as an optimistic-concurrency signal.
# Preserve it so callers can re-read and adopt the concurrently
# created folder even if a deployment localizes the error text.
status_code = 409 if response.status_code == 409 else 502
raise WeKnoraError(
detail or f"WeKnora returned HTTP {response.status_code}",
status_code=status_code,
)
if response.status_code == 204 or not response.content:
return {"success": True}
try:
@@ -226,33 +291,59 @@ class WeKnoraClient:
return dict(payload) if isinstance(payload, dict) else {"status": "ok"}
async def list_knowledge_bases(self, *, refresh: bool = False) -> list[dict[str, Any]]:
if refresh:
self._invalidate_knowledge_bases_cache()
now = time.monotonic()
cached = self._knowledge_bases_cache
if cached is not None and cached[0] > now:
if not refresh and cached is not None and cached[0] > now:
return [dict(item) for item in cached[1]]
# Coalesce requests that reach the Gateway at the same time. Keeping
# the upstream read under this narrow lock is intentional: every
# waiter receives the same fresh snapshot, while a failed request does
# not populate the cache and the next caller can retry normally.
async with self._knowledge_bases_lock:
now = time.monotonic()
cached = self._knowledge_bases_cache
if cached is not None and cached[0] > now:
return [dict(item) for item in cached[1]]
observed_generation = self._knowledge_bases_cache_generation
# Share both success and failure with the current wave of callers.
# A lock around an uncached failed read would make every waiter issue
# another full timeout, serializing an unavailable service's fan-out.
generation = self._knowledge_bases_cache_generation
pending = self._knowledge_bases_task
if pending is None or pending[0] != generation:
task = asyncio.create_task(self._load_knowledge_bases(generation))
self._knowledge_bases_task = (generation, task)
def clear_pending(completed: asyncio.Task[list[dict[str, Any]]]) -> None:
if self._knowledge_bases_task is not None and self._knowledge_bases_task[1] is completed:
self._knowledge_bases_task = None
if not completed.cancelled():
completed.exception()
task.add_done_callback(clear_pending)
else:
task = pending[1]
rows = await asyncio.shield(task)
return [dict(item) for item in rows]
async def _load_knowledge_bases(self, generation: int) -> list[dict[str, Any]]:
started = time.perf_counter()
try:
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
# A mutation can finish while this request is in flight. Do not
# let the preceding snapshot repopulate the cache after that
# mutation has explicitly invalidated it.
if self._knowledge_bases_cache_generation == observed_generation:
self._knowledge_bases_cache = (
time.monotonic() + self._KNOWLEDGE_BASES_CACHE_SECONDS,
[dict(item) for item in rows],
)
return [dict(item) for item in rows]
except Exception as exc:
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
logger.warning(
"PERF weknora_knowledge_base_list upstream_ms=%.2f outcome=error error_type=%s",
elapsed_ms,
type(exc).__name__,
)
raise
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
if elapsed_ms >= _slow_diagnostic_threshold_ms():
logger.warning(
"PERF weknora_knowledge_base_list upstream_ms=%.2f item_count=%d outcome=ok",
elapsed_ms,
len(rows),
)
# A mutation can finish while the read is in flight. Its old snapshot
# must neither repopulate the cache nor be joined by post-write callers.
if self._knowledge_bases_cache_generation == generation:
self._knowledge_bases_cache = (
time.monotonic() + self._KNOWLEDGE_BASES_CACHE_SECONDS,
[dict(item) for item in rows],
)
return rows
def _invalidate_knowledge_bases_cache(self) -> None:
"""Discard the directory after a successful local mutation."""
@@ -271,6 +362,23 @@ class WeKnoraClient:
return _as_list(_data(await self._request("GET", "/api/v1/models")))
async def debug_model(self, model_id: str) -> dict[str, Any]:
"""Ask WeKnora to verify one saved model with a small real request."""
normalized_id = str(model_id or "").strip()
if not normalized_id:
raise WeKnoraError("WeKnora model ID is required", status_code=400)
value = _data(
await self._request(
"POST",
f"/api/v1/models/{quote(normalized_id, safe='')}/debug",
files={"input": (None, "Reply with OK.")},
)
)
if not isinstance(value, dict) or not isinstance(value.get("ok"), bool):
raise WeKnoraError("WeKnora returned an invalid model debug result")
return dict(value)
async def get_default_embedding_model_id(self) -> str:
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "embedding"]
preferred = next((item for item in models if item.get("is_default") is True), None)
@@ -287,21 +395,58 @@ class WeKnoraClient:
return model_id
async def get_default_knowledge_qa_model_id(self) -> str:
"""Resolve WeKnora's own active synthesis model for Wiki indexing."""
"""Resolve a live WeKnora synthesis model for Wiki indexing.
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "knowledgeqa"]
preferred = next((item for item in models if item.get("is_default") is True), None)
selected = preferred or next(
(item for item in models if str(item.get("status") or "active").lower() == "active"),
None,
)
model_id = str((selected or {}).get("id") or "")
if not model_id:
WeKnora's ``active`` status means that the database configuration is
enabled; it does not prove that the upstream provider still serves the
model. Probe active candidates before binding a new Wiki so a disabled
provider model cannot silently send every ingest task to dead letters.
"""
models = [
item
for item in await self.list_models()
if str(item.get("type") or "").strip().lower() == "knowledgeqa"
and str(item.get("id") or "").strip()
and str(item.get("status") or "active").strip().lower() == "active"
]
if not models:
raise WeKnoraError(
"WeKnora has no available KnowledgeQA model; configure one in WeKnora before creating a Wiki",
status_code=400,
)
return model_id
# Prefer an explicit database default. When no default exists, sort by
# immutable persisted ID rather than trusting API row order or an
# editable display name.
models.sort(
key=lambda item: (
0 if item.get("is_default") is True else 1,
str(item.get("id") or "").strip(),
)
)
attempted: list[str] = []
for model in models:
model_id = str(model.get("id") or "").strip()
label = str(model.get("display_name") or model.get("name") or model_id).strip()
attempted.append(label or model_id)
# ``ok=false`` is a candidate-specific upstream-model failure. An
# HTTP/auth/timeout failure from the debug endpoint is a WeKnora
# service failure, so preserve it instead of multiplying probes
# and misreporting the incident as bad model configuration.
debug = await self.debug_model(model_id)
if debug.get("ok") is True:
return model_id
attempted_models = ", ".join(attempted[:12])
if len(attempted) > 12:
attempted_models = f"{attempted_models}, and {len(attempted) - 12} more"
raise WeKnoraError(
"WeKnora has no usable active KnowledgeQA model after connectivity checks"
+ (f": {attempted_models}" if attempted_models else "")
+ "; configure or repair a working default model before creating a Wiki",
status_code=400,
)
async def create_knowledge_base(
self,
@@ -310,6 +455,7 @@ class WeKnoraClient:
description: str = "",
kb_type: str = "document",
embedding_model_id: str | None = None,
knowledge_qa_model_id: str | None = None,
wiki_enabled: bool = False,
wiki_content_instructions: str = "",
wiki_extraction_instructions: str = "",
@@ -322,7 +468,7 @@ class WeKnoraClient:
"embedding_model_id": model_id,
}
if wiki_enabled:
synthesis_model_id = await self.get_default_knowledge_qa_model_id()
synthesis_model_id = knowledge_qa_model_id or await self.get_default_knowledge_qa_model_id()
create_payload.update(
{
"summary_model_id": synthesis_model_id,
@@ -577,6 +723,79 @@ class WeKnoraClient:
raise WeKnoraError("Invalid Wiki folder response")
return value
async def list_wiki_folders(
self,
knowledge_base_id: str,
*,
parent_id: str = "",
page_types: Sequence[str] | None = None,
) -> dict[str, Any]:
params: dict[str, Any] = {"parent_id": parent_id or None}
if page_types:
params["page_types"] = ",".join(str(value) for value in page_types if str(value)) or None
payload = await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders",
params=params,
)
value = _data(payload)
source = value if isinstance(value, dict) else {}
raw_folders = source.get("folders") if isinstance(value, dict) else value
rows = _as_list(raw_folders)
return {
"parent_id": str(source.get("parent_id") or parent_id),
"folders": rows,
}
async def update_wiki_folder(
self,
knowledge_base_id: str,
folder_id: str,
changes: Mapping[str, Any] | None = None,
*,
name: str | None = None,
parent_id: str | None = None,
move_parent: bool | None = None,
) -> dict[str, Any]:
update = dict(changes or {})
if name is not None:
update["name"] = name
if parent_id is not None:
update["parent_id"] = parent_id
update.setdefault("move_parent", True)
if move_parent is not None:
update["move_parent"] = move_parent
elif "parent_id" in update:
update.setdefault("move_parent", True)
value = _data(
await self._request(
"PUT",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders/{quote(folder_id, safe='')}",
json=update,
)
)
if not isinstance(value, dict):
raise WeKnoraError("Invalid Wiki folder response")
return dict(value)
async def delete_wiki_folder(self, knowledge_base_id: str, folder_id: str) -> None:
await self._request(
"DELETE",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders/{quote(folder_id, safe='')}",
)
async def move_wiki_page(self, knowledge_base_id: str, slug: str, folder_id: str) -> dict[str, Any]:
value = _data(
await self._request(
"PUT",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/move-page",
json={"slug": slug, "folder_id": folder_id},
)
)
if not isinstance(value, dict):
raise WeKnoraError("Invalid Wiki page response")
return dict(value)
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
@@ -587,11 +806,17 @@ class WeKnoraClient:
page: int = 1,
page_size: int = 50,
query: str = "",
folder_id: str | None = None,
) -> dict[str, Any]:
params: dict[str, Any] = {"page": page, "page_size": page_size, "query": query or None}
# ``None`` means no placement filter; an explicit empty string means
# pages directly under the Wiki root.
if folder_id is not None:
params["folder_id"] = folder_id
payload = await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages",
params={"page": page, "page_size": page_size, "query": query or None},
params=params,
)
value = _data(payload)
source = value if isinstance(value, dict) else payload if isinstance(payload, dict) else {}
@@ -604,6 +829,34 @@ class WeKnoraClient:
"total_pages": int(source.get("total_pages") or max(1, (int(source.get("total") or len(rows)) + page_size - 1) // page_size)),
}
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
limit: int = 30,
) -> list[dict[str, Any]]:
"""Use WeKnora's native, ranked Wiki regex search.
WeKnora v0.7.2 exposes this as a single-library ``GET`` endpoint with
the regex in ``q`` (not the list endpoint's ``query`` parameter). It
returns complete WikiPage rows, so callers do not need to enumerate a
directory and issue one detail request per page.
"""
payload = await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/search",
params={
"q": str(query or ""),
"limit": max(1, min(int(limit), 50)),
},
)
value = _data(payload)
if isinstance(value, dict):
return _as_list(value.get("pages"))
return _as_list(value)
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict[str, Any]:
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
value = _data(
@@ -671,6 +924,327 @@ class WeKnoraClient:
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
)
@staticmethod
def _normalize_grep_chunk(row: Mapping[str, Any]) -> dict[str, Any]:
"""Normalize both the provider extension and compatibility rows."""
return {
"chunk_id": str(row.get("id") or row.get("chunk_id") or ""),
"content": str(row.get("content") or row.get("text") or row.get("match_snippet") or ""),
"knowledge_id": str(row.get("knowledge_id") or ""),
"knowledge_base_id": str(row.get("knowledge_base_id") or ""),
"title": str(
row.get("knowledge_title")
or row.get("title")
or row.get("knowledge_filename")
or ""
),
"chunk_index": int(row.get("chunk_index") or row.get("index") or 0),
"match_count": int(
row.get("matched_patterns")
or row.get("match_count")
or row.get("total_pattern_hits")
or 0
),
"title_match": bool(row.get("title_match")),
"score": float(row.get("match_score") or row.get("score") or 0.0),
"metadata": row.get("metadata") if isinstance(row.get("metadata"), dict) else {},
}
@staticmethod
def _grep_matches(expression: regex.Pattern, value: str) -> list[regex.Match]:
try:
return list(
expression.finditer(
value,
timeout=_GREP_REGEX_TIMEOUT_SECONDS,
)
)
except TimeoutError as exc:
raise WeKnoraError(
"WeKnora compatibility grep regular expression timed out",
status_code=422,
) from exc
async def _grep_chunks_compatibility(
self,
query: str,
knowledge_base_ids: Sequence[str],
) -> dict[str, Any]:
"""Bounded, fair fallback for older providers without ``chunks/grep``.
The native endpoint remains authoritative. This path round-robins both
knowledge bases and documents so one large first scope cannot consume
the global 500-chunk compatibility budget.
"""
ids = list(dict.fromkeys(str(value).strip() for value in knowledge_base_ids if str(value).strip()))
try:
expression = regex.compile(query, regex.IGNORECASE | regex.VERSION1)
except regex.error as exc:
raise WeKnoraError(f"Invalid grep regular expression: {exc}", status_code=422) from exc
per_scope_document_cap = max(1, (_GREP_MAX_CANDIDATES + len(ids) - 1) // max(1, len(ids)))
async def load_documents(knowledge_base_id: str) -> tuple[str, list[dict[str, Any]] | Exception]:
rows: list[dict[str, Any]] = []
page = 1
try:
while len(rows) < per_scope_document_cap:
page_size = min(100, per_scope_document_cap - len(rows))
payload = await self.list_documents(
knowledge_base_id,
page=page,
page_size=page_size,
)
batch = [
dict(item)
for item in payload.get("items") or []
if isinstance(item, Mapping)
]
rows.extend(batch)
if not batch or page * page_size >= int(payload.get("total") or len(rows)):
break
page += 1
return knowledge_base_id, rows
except Exception as exc: # noqa: BLE001 - returned as structured scope failure
return knowledge_base_id, exc
loaded = await asyncio.gather(*(load_documents(knowledge_base_id) for knowledge_base_id in ids))
documents_by_scope: dict[str, list[dict[str, Any]]] = {}
warnings: list[dict[str, Any]] = [
{
"code": "WEKNORA_GREP_COMPATIBILITY_FALLBACK",
"message": "Provider chunks/grep is unavailable; used bounded existing-API compatibility scan.",
}
]
failed_scope_ids: set[str] = set()
for knowledge_base_id, outcome in loaded:
if isinstance(outcome, Exception):
failed_scope_ids.add(knowledge_base_id)
warnings.append(
{
"code": "PROVIDER_SCOPE_FAILED",
"knowledge_base_id": knowledge_base_id,
"stage": "list_documents",
}
)
else:
documents_by_scope[knowledge_base_id] = outcome
if not documents_by_scope:
raise WeKnoraError("WeKnora compatibility grep failed for every knowledge base")
# Seed the work queue in KB/document round-robin order. Subsequent
# chunk pages are re-appended to the tail, preserving document fairness.
work: deque[tuple[str, dict[str, Any], int]] = deque()
max_documents = max((len(rows) for rows in documents_by_scope.values()), default=0)
for index in range(max_documents):
for knowledge_base_id in ids:
rows = documents_by_scope.get(knowledge_base_id) or []
if index < len(rows):
work.append((knowledge_base_id, rows[index], 1))
candidates: list[dict[str, Any]] = []
seen_content: set[str] = set()
scanned_chunks = 0
chunk_successes = {knowledge_base_id: 0 for knowledge_base_id in documents_by_scope}
chunk_failures = {knowledge_base_id: 0 for knowledge_base_id in documents_by_scope}
while work and scanned_chunks < _GREP_MAX_CANDIDATES:
knowledge_base_id, document, page = work.popleft()
knowledge_id = str(document.get("id") or document.get("knowledge_id") or "")
if not knowledge_id:
continue
page_size = min(20, _GREP_MAX_CANDIDATES - scanned_chunks)
try:
payload = await self.list_chunks(
knowledge_id,
page=page,
page_size=page_size,
)
except Exception: # noqa: BLE001 - one document must not fail the whole fallback
chunk_failures[knowledge_base_id] += 1
continue
chunk_successes[knowledge_base_id] += 1
chunks = [
dict(item)
for item in payload.get("items") or []
if isinstance(item, Mapping)
]
title = str(
document.get("title")
or document.get("name")
or document.get("filename")
or ""
)
title_matches = self._grep_matches(expression, title[:5000])
for chunk in chunks:
if scanned_chunks >= _GREP_MAX_CANDIDATES:
break
scanned_chunks += 1
content = str(chunk.get("content") or chunk.get("text") or "")
content_matches = self._grep_matches(expression, content[:50_000])
if not title_matches and not content_matches:
continue
signature = hashlib.sha256(" ".join(content.split()).encode()).hexdigest()
if signature in seen_content:
continue
seen_content.add(signature)
match_count = min(99, len(title_matches) * 4 + len(content_matches))
candidates.append(
self._normalize_grep_chunk(
{
**chunk,
"knowledge_id": knowledge_id,
"knowledge_base_id": knowledge_base_id,
"knowledge_title": title,
"match_count": match_count,
"title_match": bool(title_matches),
"match_score": float(match_count),
}
)
)
if chunks and page * page_size < int(payload.get("total") or len(chunks)):
work.append((knowledge_base_id, document, page + 1))
successful_scope_ids: list[str] = []
for knowledge_base_id, documents in documents_by_scope.items():
# An empty KB was successfully scanned. A non-empty KB for which
# every chunk read failed is a failed provider scope.
if not documents or chunk_successes[knowledge_base_id] > 0:
successful_scope_ids.append(knowledge_base_id)
else:
failed_scope_ids.add(knowledge_base_id)
if chunk_failures[knowledge_base_id]:
warnings.append(
{
"code": "PROVIDER_DOCUMENT_READ_FAILED",
"knowledge_base_id": knowledge_base_id,
"failed_documents": chunk_failures[knowledge_base_id],
}
)
if not successful_scope_ids:
raise WeKnoraError("WeKnora compatibility grep failed for every knowledge base")
candidates.sort(
key=lambda item: (
bool(item.get("title_match")),
int(item.get("match_count") or 0),
float(item.get("score") or 0),
-int(item.get("chunk_index") or 0),
),
reverse=True,
)
return {
"items": candidates[:_GREP_MAX_RESULTS],
"contract": _GREP_COMPATIBILITY_CONTRACT,
"compatibility_fallback": True,
"partial": bool(failed_scope_ids or any(chunk_failures.values())),
"warnings": warnings,
"successful_knowledge_base_ids": successful_scope_ids,
"failed_knowledge_base_ids": [value for value in ids if value in failed_scope_ids],
"scanned_chunks": scanned_chunks,
"scan_truncated": scanned_chunks >= _GREP_MAX_CANDIDATES,
}
async def grep_chunks(
self,
query: str,
knowledge_base_ids: Sequence[str],
) -> dict[str, Any]:
"""Run the v0.7.2 cross-KB grep extension, with a marked fallback."""
ids = list(dict.fromkeys(str(value).strip() for value in knowledge_base_ids if str(value).strip()))
if not ids:
return {
"items": [],
"contract": _GREP_CONTRACT,
"compatibility_fallback": False,
"partial": False,
"warnings": [],
"successful_knowledge_base_ids": [],
"failed_knowledge_base_ids": [],
"scanned_chunks": 0,
"scan_truncated": False,
}
try:
payload = await self._request(
"POST",
"/api/v1/chunks/grep",
json={"query": query, "knowledge_base_ids": ids},
)
except WeKnoraError as exc:
if exc.status_code not in {404, 405}:
raise
return await self._grep_chunks_compatibility(query, ids)
value = _data(payload)
source = value if isinstance(value, dict) else {}
contract = str(
source.get("contract")
or (payload.get("contract") if isinstance(payload, dict) else "")
or ""
)
if contract != _GREP_CONTRACT:
raise WeKnoraError(
f"WeKnora chunks/grep contract mismatch (expected {_GREP_CONTRACT})"
)
raw_rows = (
source.get("chunk_results")
or source.get("results")
or source.get("items")
or []
)
if not isinstance(raw_rows, list):
raise WeKnoraError("WeKnora chunks/grep returned invalid results")
rows = [
self._normalize_grep_chunk(row)
for row in raw_rows
if isinstance(row, Mapping)
]
warnings = [
dict(item)
for item in source.get("warnings") or []
if isinstance(item, Mapping)
]
failed_ids = [
str(value)
for value in source.get("failed_knowledge_base_ids") or []
if str(value) in ids
]
explicit_successes = source.get("successful_knowledge_base_ids")
successful_ids = (
[str(value) for value in explicit_successes if str(value) in ids]
if isinstance(explicit_successes, list)
else [value for value in ids if value not in failed_ids]
)
warned_scope_ids = {
str(item.get("knowledge_base_id") or "")
for item in warnings
}
warnings.extend(
{
"code": "PROVIDER_SCOPE_FAILED",
"knowledge_base_id": knowledge_base_id,
}
for knowledge_base_id in failed_ids
if knowledge_base_id not in warned_scope_ids
)
if source.get("partial") and not warnings:
warnings.append({"code": "PROVIDER_PARTIAL_RESPONSE"})
return {
"items": rows[:_GREP_MAX_RESULTS],
"contract": contract,
"compatibility_fallback": False,
"partial": bool(source.get("partial")) or bool(warnings) or bool(failed_ids),
"warnings": warnings,
"successful_knowledge_base_ids": successful_ids,
"failed_knowledge_base_ids": failed_ids,
"scanned_chunks": int(source.get("scanned_chunks") or len(rows)),
"scan_truncated": bool(source.get("scan_truncated")),
}
async def search(self, query: str, knowledge_base_ids: Sequence[str]) -> list[dict[str, Any]]:
ids = [str(value) for value in knowledge_base_ids if str(value)]
if not ids:
@@ -6,6 +6,8 @@ import asyncio
import logging
import platform
from datetime import UTC, datetime
from math import ceil
from time import monotonic
from typing import Any
from uuid import uuid4
@@ -24,6 +26,15 @@ from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
logger = logging.getLogger(__name__)
# This breaker is intentionally process-local. The durable sync row already
# records the failed run and error, while a transient retry deadline does not
# belong in release files or administrator configuration. Rebuilding the
# runtime after a provider/config change (or restarting Gateway) resets the
# breaker and therefore allows one immediate recovery probe. The state is a
# fixed-size provider record rather than a per-page/per-mapping collection.
_EMBEDDING_PROVIDER_BACKOFF_INITIAL_SECONDS = 60.0
_EMBEDDING_PROVIDER_BACKOFF_MAX_SECONDS = 30.0 * 60.0
class WikiSyncLeaseLost(RuntimeError):
pass
@@ -251,6 +262,163 @@ class WikiSyncService:
self.client = client
self.owner = f"{platform.node()}:{uuid4()}"
self.on_completed = None
self._embedding_probe_lock = asyncio.Lock()
self._embedding_failure_count = 0
self._embedding_retry_at = 0.0
self._embedding_last_error: str | None = None
def _embedding_backoff_result(
self,
mapping_id: str,
*,
now: float | None = None,
) -> dict[str, Any] | None:
checked_at = monotonic() if now is None else now
remaining = self._embedding_retry_at - checked_at
if self._embedding_failure_count <= 0 or remaining <= 0:
return None
return {
"mapping_id": mapping_id,
"status": "provider_backoff",
"failure_scope": "embedding_provider",
"failure_count": self._embedding_failure_count,
"retry_after_seconds": max(1, ceil(remaining)),
"error": self._embedding_last_error,
}
def _record_embedding_failure(self, exc: WikiEmbeddingError) -> int:
"""Open or extend one provider-wide exponential breaker window.
Several mappings may already be running under the scheduler semaphore
when the provider goes away. Failures arriving inside an already-open
window share that window instead of multiplying the exponent.
"""
now = monotonic()
if self._embedding_retry_at <= now:
self._embedding_failure_count = min(
self._embedding_failure_count + 1,
31,
)
exponent = min(self._embedding_failure_count - 1, 30)
delay = min(
_EMBEDDING_PROVIDER_BACKOFF_MAX_SECONDS,
_EMBEDDING_PROVIDER_BACKOFF_INITIAL_SECONDS * (2**exponent),
)
self._embedding_retry_at = now + delay
self._embedding_last_error = _safe_error(exc)
return max(1, ceil(self._embedding_retry_at - now))
def _reset_embedding_breaker(self) -> None:
self._embedding_failure_count = 0
self._embedding_retry_at = 0.0
self._embedding_last_error = None
async def _embedding_preflight(
self,
mapping_id: str,
*,
force: bool,
) -> dict[str, Any] | None:
"""Skip a known outage or perform one serialized recovery probe."""
if not force:
backoff = self._embedding_backoff_result(mapping_id)
if backoff is not None:
return backoff
async with self._embedding_probe_lock:
if not force:
backoff = self._embedding_backoff_result(mapping_id)
if backoff is not None:
return backoff
needs_probe = (
self.embedding.dimensions is None
or self._embedding_failure_count > 0
)
if not needs_probe:
return None
try:
# Auto-dimension configurations need one successful response
# for a stable fingerprint. After an outage the same tiny
# request proves provider recovery before any Wiki directory
# or page is enumerated.
await self.embedding.embed_texts(
["DeerFlow local Wiki embedding provider health probe"]
)
except WikiEmbeddingError as exc:
if exc.category != "runtime_retired":
retry_after = self._record_embedding_failure(exc)
logger.warning(
"Local Wiki embedding provider probe failed mapping_id=%s retry_after_seconds=%s: %s",
mapping_id,
retry_after,
_safe_error(exc),
)
raise
self._reset_embedding_breaker()
return None
async def _persist_requested_backoff(
self,
mapping_id: str,
*,
sync_id: str,
mode: str,
started_at: str,
backoff: dict[str, Any],
) -> dict[str, Any]:
"""Finish an explicitly requested job with a frontend-compatible state."""
states = await self.store.list_index_status([mapping_id])
prior = states[0] if states else {}
remote_count = int(prior.get("remote_page_count") or 0)
acquired = await self.store.try_acquire_sync_lease(
mapping_id,
owner=self.owner,
sync_id=sync_id,
lease_seconds=self.config.sync_lease_seconds,
fingerprint=self.embedding.fingerprint,
)
if not acquired:
return backoff
error = str(
backoff.get("error")
or "Wiki embedding provider is temporarily unavailable"
)
result = {
"sync_id": sync_id,
"mapping_id": mapping_id,
"mode": mode,
"status": "failed",
"remote_pages": remote_count,
"added_pages": 0,
"updated_pages": 0,
"metadata_updated_pages": 0,
"revectorized_pages": 0,
"changed_pages": 0,
"skipped_pages": 0,
"deleted_pages": 0,
"failed_pages": 0,
"started_at": started_at,
"completed_at": _iso_now(),
"error": error,
"failure_scope": backoff.get("failure_scope")
or "embedding_provider",
"failure_count": int(backoff.get("failure_count") or 0),
"retry_after_seconds": int(
backoff.get("retry_after_seconds") or 0
),
}
persisted = await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=False,
remote_page_count=remote_count,
error=error,
result=result,
)
return result if persisted else backoff
async def sync_mapping(
self,
@@ -261,13 +429,21 @@ class WikiSyncService:
sync_id: str | None = None,
) -> dict[str, Any]:
mapping_id = str(mapping["id"])
requested_sync_id = sync_id
sync_id = sync_id or str(uuid4())
mode = "full" if force else "incremental"
started_at = _iso_now()
if self.embedding.dimensions is None:
# Auto-dimension configurations need one successful response to
# produce a stable fingerprint before lease/rebuild decisions.
await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
backoff = await self._embedding_preflight(mapping_id, force=force)
if backoff is not None:
if requested_sync_id is not None:
return await self._persist_requested_backoff(
mapping_id,
sync_id=sync_id,
mode=mode,
started_at=started_at,
backoff=backoff,
)
return backoff
fingerprint = self.embedding.fingerprint
revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
@@ -480,7 +656,10 @@ class WikiSyncService:
except Exception as exc:
if isinstance(exc, WikiSyncLeaseLost):
raise
if isinstance(exc, WikiEmbeddingError) and exc.category == "runtime_retired":
# Embedding failures describe the shared provider, not
# one malformed Wiki page. Continuing would repeat the
# client's own retry loop for every remaining page.
if isinstance(exc, WikiEmbeddingError):
raise
if not graph_page_captured:
graph_snapshot_complete = False
@@ -662,6 +841,20 @@ class WikiSyncService:
raise
except Exception as exc:
safe_error = _safe_error(exc)
failed_result = _sync_result("failed", error=safe_error)
retry_after: int | None = None
if (
isinstance(exc, WikiEmbeddingError)
and exc.category != "runtime_retired"
):
retry_after = self._record_embedding_failure(exc)
failed_result.update(
{
"failure_scope": "embedding_provider",
"failure_count": self._embedding_failure_count,
"retry_after_seconds": retry_after,
}
)
await self.store.finish_sync(
mapping_id,
owner=self.owner,
@@ -669,9 +862,18 @@ class WikiSyncService:
success=False,
remote_page_count=remote_count,
error=safe_error,
result=_sync_result("failed", error=safe_error),
result=failed_result,
)
logger.warning(
"Local Wiki sync failed mapping_id=%s%s: %s",
mapping_id,
(
f" embedding_provider_retry_after_seconds={retry_after}"
if retry_after is not None
else ""
),
safe_error,
)
logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, safe_error)
raise
finally:
heartbeat.cancel()
@@ -2,7 +2,7 @@
from __future__ import annotations
from functools import lru_cache
import asyncio
import httpx
@@ -53,16 +53,49 @@ def build_weknora_client(
)
@lru_cache(maxsize=4)
def _shared_weknora_client(
base_url: str,
admin_email: str,
admin_password: str,
) -> WeKnoraClient:
"""Reuse auth state for equivalent runtime configurations."""
"""Reuse auth state only inside the current event loop.
return WeKnoraClient(
base_url=base_url,
admin_email=admin_email,
admin_password=admin_password,
)
``WeKnoraClient`` owns asyncio primitives (an authentication lock and a
cached directory task). A process-wide ``lru_cache`` therefore lets a
client first used by the Gateway loop escape into a LangGraph/tool loop.
Once either primitive has been awaited, reusing it from the other loop can
fail with ``... is bound to a different event loop``. Smart retrieval
used to normalize that internal exception to ``PROVIDER_UNAVAILABLE``, so
an otherwise healthy selected knowledge base appeared empty.
Store the small configuration cache on the loop itself. The loop owns the
clients for exactly its lifetime, equivalent configurations still share
their bearer token and knowledge-base directory within that loop, and no
async primitive crosses loop boundaries. Synchronous callers have no
safe loop ownership yet, so they receive an uncached client.
"""
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return WeKnoraClient(
base_url=base_url,
admin_email=admin_email,
admin_password=admin_password,
)
cache_name = "_deerflow_weknora_clients"
cache = getattr(loop, cache_name, None)
if not isinstance(cache, dict):
cache = {}
setattr(loop, cache_name, cache)
key = (base_url, admin_email, admin_password)
client = cache.get(key)
if client is None:
client = WeKnoraClient(
base_url=base_url,
admin_email=admin_email,
admin_password=admin_password,
)
cache[key] = client
return client