sync update from LAN repo (source master @ ca48c4ec)
This commit is contained in:
1 parent
8bc8b0e082
commit
ea9dcff9c0
873 files changed
+107866
-14011
No files matched your search
+100
-7
@@ -4,6 +4,8 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Mapping, Sequence
|
||||
@@ -13,6 +15,15 @@ from urllib.parse import quote
|
||||
import httpx
|
||||
import regex
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _slow_diagnostic_threshold_ms() -> float:
|
||||
try:
|
||||
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_REQUEST_LOG_MS", "500")), 60_000.0))
|
||||
except (TypeError, ValueError):
|
||||
return 500.0
|
||||
|
||||
_GREP_CONTRACT = "weknora-v0.7.2-grep-v1"
|
||||
_GREP_COMPATIBILITY_CONTRACT = "weknora-existing-api-compat-v1"
|
||||
_GREP_MAX_CANDIDATES = 500
|
||||
@@ -179,6 +190,11 @@ class WeKnoraClient:
|
||||
files: Any = None,
|
||||
data: Mapping[str, Any] | None = None,
|
||||
) -> Any:
|
||||
diagnose_knowledge_base_list = method.upper() == "GET" and path == "/api/v1/knowledge-bases"
|
||||
diagnostic_started = time.perf_counter() if diagnose_knowledge_base_list else None
|
||||
auth_ms = 0.0
|
||||
network_ms = 0.0
|
||||
attempts = 0
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
@@ -187,24 +203,55 @@ class WeKnoraClient:
|
||||
)
|
||||
try:
|
||||
for attempt in range(2):
|
||||
attempts += 1
|
||||
auth_started = time.perf_counter()
|
||||
request_headers = await self._headers(force_refresh=attempt > 0)
|
||||
if diagnose_knowledge_base_list:
|
||||
auth_ms += (time.perf_counter() - auth_started) * 1000
|
||||
request_started = time.perf_counter()
|
||||
response = await client.request(
|
||||
method,
|
||||
f"{self.base_url}{path}",
|
||||
headers=await self._headers(force_refresh=attempt > 0),
|
||||
headers=request_headers,
|
||||
json=json,
|
||||
params=params,
|
||||
files=files,
|
||||
data=data,
|
||||
)
|
||||
if diagnose_knowledge_base_list:
|
||||
network_ms += (time.perf_counter() - request_started) * 1000
|
||||
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
|
||||
break
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
|
||||
except Exception as exc:
|
||||
if diagnose_knowledge_base_list:
|
||||
logger.warning(
|
||||
"PERF weknora_http_request operation=knowledge_base_directory method=GET total_ms=%.2f auth_ms=%.2f network_ms=%.2f attempts=%d outcome=error error_type=%s",
|
||||
(time.perf_counter() - diagnostic_started) * 1000,
|
||||
auth_ms,
|
||||
network_ms,
|
||||
attempts,
|
||||
type(exc).__name__,
|
||||
)
|
||||
if isinstance(exc, httpx.TimeoutException):
|
||||
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
|
||||
if isinstance(exc, httpx.HTTPError):
|
||||
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
|
||||
raise
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
if diagnose_knowledge_base_list:
|
||||
total_ms = (time.perf_counter() - diagnostic_started) * 1000
|
||||
if total_ms >= _slow_diagnostic_threshold_ms() or response.status_code >= 400:
|
||||
logger.warning(
|
||||
"PERF weknora_http_request operation=knowledge_base_directory method=GET total_ms=%.2f auth_ms=%.2f network_ms=%.2f attempts=%d status=%d outcome=%s",
|
||||
total_ms,
|
||||
auth_ms,
|
||||
network_ms,
|
||||
attempts,
|
||||
response.status_code,
|
||||
"http_error" if response.status_code >= 400 else "ok",
|
||||
)
|
||||
if response.status_code == 404:
|
||||
raise WeKnoraError("WeKnora resource was not found", status_code=404)
|
||||
if response.status_code == 405:
|
||||
@@ -271,7 +318,24 @@ class WeKnoraClient:
|
||||
return [dict(item) for item in rows]
|
||||
|
||||
async def _load_knowledge_bases(self, generation: int) -> list[dict[str, Any]]:
|
||||
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
|
||||
except Exception as exc:
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
logger.warning(
|
||||
"PERF weknora_knowledge_base_list upstream_ms=%.2f outcome=error error_type=%s",
|
||||
elapsed_ms,
|
||||
type(exc).__name__,
|
||||
)
|
||||
raise
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
if elapsed_ms >= _slow_diagnostic_threshold_ms():
|
||||
logger.warning(
|
||||
"PERF weknora_knowledge_base_list upstream_ms=%.2f item_count=%d outcome=ok",
|
||||
elapsed_ms,
|
||||
len(rows),
|
||||
)
|
||||
# A mutation can finish while the read is in flight. Its old snapshot
|
||||
# must neither repopulate the cache nor be joined by post-write callers.
|
||||
if self._knowledge_bases_cache_generation == generation:
|
||||
@@ -391,6 +455,7 @@ class WeKnoraClient:
|
||||
description: str = "",
|
||||
kb_type: str = "document",
|
||||
embedding_model_id: str | None = None,
|
||||
knowledge_qa_model_id: str | None = None,
|
||||
wiki_enabled: bool = False,
|
||||
wiki_content_instructions: str = "",
|
||||
wiki_extraction_instructions: str = "",
|
||||
@@ -403,7 +468,7 @@ class WeKnoraClient:
|
||||
"embedding_model_id": model_id,
|
||||
}
|
||||
if wiki_enabled:
|
||||
synthesis_model_id = await self.get_default_knowledge_qa_model_id()
|
||||
synthesis_model_id = knowledge_qa_model_id or await self.get_default_knowledge_qa_model_id()
|
||||
create_payload.update(
|
||||
{
|
||||
"summary_model_id": synthesis_model_id,
|
||||
@@ -764,6 +829,34 @@ class WeKnoraClient:
|
||||
"total_pages": int(source.get("total_pages") or max(1, (int(source.get("total") or len(rows)) + page_size - 1) // page_size)),
|
||||
}
|
||||
|
||||
async def search_wiki_pages(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
query: str,
|
||||
*,
|
||||
limit: int = 30,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Use WeKnora's native, ranked Wiki regex search.
|
||||
|
||||
WeKnora v0.7.2 exposes this as a single-library ``GET`` endpoint with
|
||||
the regex in ``q`` (not the list endpoint's ``query`` parameter). It
|
||||
returns complete WikiPage rows, so callers do not need to enumerate a
|
||||
directory and issue one detail request per page.
|
||||
"""
|
||||
|
||||
payload = await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/search",
|
||||
params={
|
||||
"q": str(query or ""),
|
||||
"limit": max(1, min(int(limit), 50)),
|
||||
},
|
||||
)
|
||||
value = _data(payload)
|
||||
if isinstance(value, dict):
|
||||
return _as_list(value.get("pages"))
|
||||
return _as_list(value)
|
||||
|
||||
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict[str, Any]:
|
||||
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
|
||||
value = _data(
|
||||
|
||||
Reference in new issue
Block a user