sync snapshot from LAN repo (source codex/merge-tjlnew-classic-retrieval-20261019 @ ded012d9)
This commit is contained in:
1 parent
ea9dcff9c0
commit
c753e6c9b5
818 files changed
+14037
-108991
No files matched your search
+7
-100
@@ -4,8 +4,6 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Mapping, Sequence
|
||||
@@ -15,15 +13,6 @@ from urllib.parse import quote
|
||||
import httpx
|
||||
import regex
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _slow_diagnostic_threshold_ms() -> float:
|
||||
try:
|
||||
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_REQUEST_LOG_MS", "500")), 60_000.0))
|
||||
except (TypeError, ValueError):
|
||||
return 500.0
|
||||
|
||||
_GREP_CONTRACT = "weknora-v0.7.2-grep-v1"
|
||||
_GREP_COMPATIBILITY_CONTRACT = "weknora-existing-api-compat-v1"
|
||||
_GREP_MAX_CANDIDATES = 500
|
||||
@@ -190,11 +179,6 @@ class WeKnoraClient:
|
||||
files: Any = None,
|
||||
data: Mapping[str, Any] | None = None,
|
||||
) -> Any:
|
||||
diagnose_knowledge_base_list = method.upper() == "GET" and path == "/api/v1/knowledge-bases"
|
||||
diagnostic_started = time.perf_counter() if diagnose_knowledge_base_list else None
|
||||
auth_ms = 0.0
|
||||
network_ms = 0.0
|
||||
attempts = 0
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
@@ -203,55 +187,24 @@ class WeKnoraClient:
|
||||
)
|
||||
try:
|
||||
for attempt in range(2):
|
||||
attempts += 1
|
||||
auth_started = time.perf_counter()
|
||||
request_headers = await self._headers(force_refresh=attempt > 0)
|
||||
if diagnose_knowledge_base_list:
|
||||
auth_ms += (time.perf_counter() - auth_started) * 1000
|
||||
request_started = time.perf_counter()
|
||||
response = await client.request(
|
||||
method,
|
||||
f"{self.base_url}{path}",
|
||||
headers=request_headers,
|
||||
headers=await self._headers(force_refresh=attempt > 0),
|
||||
json=json,
|
||||
params=params,
|
||||
files=files,
|
||||
data=data,
|
||||
)
|
||||
if diagnose_knowledge_base_list:
|
||||
network_ms += (time.perf_counter() - request_started) * 1000
|
||||
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
|
||||
break
|
||||
except Exception as exc:
|
||||
if diagnose_knowledge_base_list:
|
||||
logger.warning(
|
||||
"PERF weknora_http_request operation=knowledge_base_directory method=GET total_ms=%.2f auth_ms=%.2f network_ms=%.2f attempts=%d outcome=error error_type=%s",
|
||||
(time.perf_counter() - diagnostic_started) * 1000,
|
||||
auth_ms,
|
||||
network_ms,
|
||||
attempts,
|
||||
type(exc).__name__,
|
||||
)
|
||||
if isinstance(exc, httpx.TimeoutException):
|
||||
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
|
||||
if isinstance(exc, httpx.HTTPError):
|
||||
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
|
||||
raise
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
if diagnose_knowledge_base_list:
|
||||
total_ms = (time.perf_counter() - diagnostic_started) * 1000
|
||||
if total_ms >= _slow_diagnostic_threshold_ms() or response.status_code >= 400:
|
||||
logger.warning(
|
||||
"PERF weknora_http_request operation=knowledge_base_directory method=GET total_ms=%.2f auth_ms=%.2f network_ms=%.2f attempts=%d status=%d outcome=%s",
|
||||
total_ms,
|
||||
auth_ms,
|
||||
network_ms,
|
||||
attempts,
|
||||
response.status_code,
|
||||
"http_error" if response.status_code >= 400 else "ok",
|
||||
)
|
||||
if response.status_code == 404:
|
||||
raise WeKnoraError("WeKnora resource was not found", status_code=404)
|
||||
if response.status_code == 405:
|
||||
@@ -318,24 +271,7 @@ class WeKnoraClient:
|
||||
return [dict(item) for item in rows]
|
||||
|
||||
async def _load_knowledge_bases(self, generation: int) -> list[dict[str, Any]]:
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
|
||||
except Exception as exc:
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
logger.warning(
|
||||
"PERF weknora_knowledge_base_list upstream_ms=%.2f outcome=error error_type=%s",
|
||||
elapsed_ms,
|
||||
type(exc).__name__,
|
||||
)
|
||||
raise
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
if elapsed_ms >= _slow_diagnostic_threshold_ms():
|
||||
logger.warning(
|
||||
"PERF weknora_knowledge_base_list upstream_ms=%.2f item_count=%d outcome=ok",
|
||||
elapsed_ms,
|
||||
len(rows),
|
||||
)
|
||||
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
|
||||
# A mutation can finish while the read is in flight. Its old snapshot
|
||||
# must neither repopulate the cache nor be joined by post-write callers.
|
||||
if self._knowledge_bases_cache_generation == generation:
|
||||
@@ -455,7 +391,6 @@ class WeKnoraClient:
|
||||
description: str = "",
|
||||
kb_type: str = "document",
|
||||
embedding_model_id: str | None = None,
|
||||
knowledge_qa_model_id: str | None = None,
|
||||
wiki_enabled: bool = False,
|
||||
wiki_content_instructions: str = "",
|
||||
wiki_extraction_instructions: str = "",
|
||||
@@ -468,7 +403,7 @@ class WeKnoraClient:
|
||||
"embedding_model_id": model_id,
|
||||
}
|
||||
if wiki_enabled:
|
||||
synthesis_model_id = knowledge_qa_model_id or await self.get_default_knowledge_qa_model_id()
|
||||
synthesis_model_id = await self.get_default_knowledge_qa_model_id()
|
||||
create_payload.update(
|
||||
{
|
||||
"summary_model_id": synthesis_model_id,
|
||||
@@ -829,34 +764,6 @@ class WeKnoraClient:
|
||||
"total_pages": int(source.get("total_pages") or max(1, (int(source.get("total") or len(rows)) + page_size - 1) // page_size)),
|
||||
}
|
||||
|
||||
async def search_wiki_pages(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
query: str,
|
||||
*,
|
||||
limit: int = 30,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Use WeKnora's native, ranked Wiki regex search.
|
||||
|
||||
WeKnora v0.7.2 exposes this as a single-library ``GET`` endpoint with
|
||||
the regex in ``q`` (not the list endpoint's ``query`` parameter). It
|
||||
returns complete WikiPage rows, so callers do not need to enumerate a
|
||||
directory and issue one detail request per page.
|
||||
"""
|
||||
|
||||
payload = await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/search",
|
||||
params={
|
||||
"q": str(query or ""),
|
||||
"limit": max(1, min(int(limit), 50)),
|
||||
},
|
||||
)
|
||||
value = _data(payload)
|
||||
if isinstance(value, dict):
|
||||
return _as_list(value.get("pages"))
|
||||
return _as_list(value)
|
||||
|
||||
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict[str, Any]:
|
||||
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
|
||||
value = _data(
|
||||
|
||||
Reference in new issue
Block a user