sync update from LAN repo (source master @ ca48c4ec)

This commit is contained in:
gozeros committed 2026-11-05 10:53:54 +08:00
1 parent 8bc8b0e082
commit ea9dcff9c0
873 files changed
+107866 -14011

No files matched your search

@@ -4,6 +4,8 @@ from __future__ import annotations
import asyncio
import hashlib
import logging
import os
import time
from collections import deque
from collections.abc import Mapping, Sequence
@@ -13,6 +15,15 @@ from urllib.parse import quote
import httpx
import regex
logger = logging.getLogger(__name__)
def _slow_diagnostic_threshold_ms() -> float:
try:
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_REQUEST_LOG_MS", "500")), 60_000.0))
except (TypeError, ValueError):
return 500.0
_GREP_CONTRACT = "weknora-v0.7.2-grep-v1"
_GREP_COMPATIBILITY_CONTRACT = "weknora-existing-api-compat-v1"
_GREP_MAX_CANDIDATES = 500
@@ -179,6 +190,11 @@ class WeKnoraClient:
files: Any = None,
data: Mapping[str, Any] | None = None,
) -> Any:
diagnose_knowledge_base_list = method.upper() == "GET" and path == "/api/v1/knowledge-bases"
diagnostic_started = time.perf_counter() if diagnose_knowledge_base_list else None
auth_ms = 0.0
network_ms = 0.0
attempts = 0
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
@@ -187,24 +203,55 @@ class WeKnoraClient:
)
try:
for attempt in range(2):
attempts += 1
auth_started = time.perf_counter()
request_headers = await self._headers(force_refresh=attempt > 0)
if diagnose_knowledge_base_list:
auth_ms += (time.perf_counter() - auth_started) * 1000
request_started = time.perf_counter()
response = await client.request(
method,
f"{self.base_url}{path}",
headers=await self._headers(force_refresh=attempt > 0),
headers=request_headers,
json=json,
params=params,
files=files,
data=data,
)
if diagnose_knowledge_base_list:
network_ms += (time.perf_counter() - request_started) * 1000
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
break
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
except Exception as exc:
if diagnose_knowledge_base_list:
logger.warning(
"PERF weknora_http_request operation=knowledge_base_directory method=GET total_ms=%.2f auth_ms=%.2f network_ms=%.2f attempts=%d outcome=error error_type=%s",
(time.perf_counter() - diagnostic_started) * 1000,
auth_ms,
network_ms,
attempts,
type(exc).__name__,
)
if isinstance(exc, httpx.TimeoutException):
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
if isinstance(exc, httpx.HTTPError):
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
raise
finally:
if owns_client:
await client.aclose()
if diagnose_knowledge_base_list:
total_ms = (time.perf_counter() - diagnostic_started) * 1000
if total_ms >= _slow_diagnostic_threshold_ms() or response.status_code >= 400:
logger.warning(
"PERF weknora_http_request operation=knowledge_base_directory method=GET total_ms=%.2f auth_ms=%.2f network_ms=%.2f attempts=%d status=%d outcome=%s",
total_ms,
auth_ms,
network_ms,
attempts,
response.status_code,
"http_error" if response.status_code >= 400 else "ok",
)
if response.status_code == 404:
raise WeKnoraError("WeKnora resource was not found", status_code=404)
if response.status_code == 405:
@@ -271,7 +318,24 @@ class WeKnoraClient:
return [dict(item) for item in rows]
async def _load_knowledge_bases(self, generation: int) -> list[dict[str, Any]]:
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
started = time.perf_counter()
try:
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
except Exception as exc:
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
logger.warning(
"PERF weknora_knowledge_base_list upstream_ms=%.2f outcome=error error_type=%s",
elapsed_ms,
type(exc).__name__,
)
raise
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
if elapsed_ms >= _slow_diagnostic_threshold_ms():
logger.warning(
"PERF weknora_knowledge_base_list upstream_ms=%.2f item_count=%d outcome=ok",
elapsed_ms,
len(rows),
)
# A mutation can finish while the read is in flight. Its old snapshot
# must neither repopulate the cache nor be joined by post-write callers.
if self._knowledge_bases_cache_generation == generation:
@@ -391,6 +455,7 @@ class WeKnoraClient:
description: str = "",
kb_type: str = "document",
embedding_model_id: str | None = None,
knowledge_qa_model_id: str | None = None,
wiki_enabled: bool = False,
wiki_content_instructions: str = "",
wiki_extraction_instructions: str = "",
@@ -403,7 +468,7 @@ class WeKnoraClient:
"embedding_model_id": model_id,
}
if wiki_enabled:
synthesis_model_id = await self.get_default_knowledge_qa_model_id()
synthesis_model_id = knowledge_qa_model_id or await self.get_default_knowledge_qa_model_id()
create_payload.update(
{
"summary_model_id": synthesis_model_id,
@@ -764,6 +829,34 @@ class WeKnoraClient:
"total_pages": int(source.get("total_pages") or max(1, (int(source.get("total") or len(rows)) + page_size - 1) // page_size)),
}
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
limit: int = 30,
) -> list[dict[str, Any]]:
"""Use WeKnora's native, ranked Wiki regex search.
WeKnora v0.7.2 exposes this as a single-library ``GET`` endpoint with
the regex in ``q`` (not the list endpoint's ``query`` parameter). It
returns complete WikiPage rows, so callers do not need to enumerate a
directory and issue one detail request per page.
"""
payload = await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/search",
params={
"q": str(query or ""),
"limit": max(1, min(int(limit), 50)),
},
)
value = _data(payload)
if isinstance(value, dict):
return _as_list(value.get("pages"))
return _as_list(value)
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict[str, Any]:
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
value = _data(