sync update from LAN repo (source dhk-1 @ 3e87be55)

This commit is contained in:
gozeros committed 2026-10-21 15:06:08 +08:00
1 parent ab6068a7c1
commit 42f2a26fa8
1482 files changed
+227759 -9584

No files matched your search

@@ -2,6 +2,7 @@
from __future__ import annotations
import asyncio
import time
from collections.abc import Mapping, Sequence
from typing import Any
@@ -45,9 +46,13 @@ class WeKnoraClient:
tests. When omitted, each public method creates and closes a short-lived client.
If admin credentials are configured, DeerFlow authenticates to WeKnora with a
cached bearer token; otherwise it keeps the legacy trusted-network anonymous
mode for deployments that allow DeerFlow through a reverse proxy.
mode for deployments that allow DeerFlow through a reverse proxy. The
knowledge-base directory is also cached briefly because several Gateway
scopes request the same expensive upstream list at once.
"""
_KNOWLEDGE_BASES_CACHE_SECONDS = 15.0
def __init__(
self,
*,
@@ -65,6 +70,11 @@ class WeKnoraClient:
self._auth_token = ""
self._tenant_id = ""
self._auth_expires_at = 0.0
self._auth_generation = 0
self._auth_lock = asyncio.Lock()
self._knowledge_bases_cache: tuple[float, list[dict[str, Any]]] | None = None
self._knowledge_bases_cache_generation = 0
self._knowledge_bases_lock = asyncio.Lock()
if not self.base_url:
raise ValueError("WeKnora base URL is required")
@@ -77,20 +87,8 @@ class WeKnoraClient:
if not isinstance(payload, dict):
return "", ""
source = payload.get("data") if isinstance(payload.get("data"), dict) else payload
token = str(
source.get("token")
or source.get("access_token")
or payload.get("token")
or payload.get("access_token")
or ""
)
tenant = (
source.get("tenant")
if isinstance(source.get("tenant"), dict)
else source.get("active_tenant")
if isinstance(source.get("active_tenant"), dict)
else payload.get("tenant")
)
token = str(source.get("token") or source.get("access_token") or payload.get("token") or payload.get("access_token") or "")
tenant = source.get("tenant") if isinstance(source.get("tenant"), dict) else source.get("active_tenant") if isinstance(source.get("active_tenant"), dict) else payload.get("tenant")
user = source.get("user") if isinstance(source.get("user"), dict) else payload.get("user")
tenant_id = ""
if isinstance(tenant, dict):
@@ -106,43 +104,51 @@ class WeKnoraClient:
if self._auth_token and not force_refresh and self._auth_expires_at > now + 60:
return
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=False,
trust_env=False,
)
try:
response = await client.post(
f"{self.base_url}/api/v1/auth/login",
json={"email": self.admin_email, "password": self.admin_password},
headers={"Accept": "application/json"},
observed_generation = self._auth_generation
async with self._auth_lock:
now = time.time()
token_is_fresh = bool(self._auth_token and self._auth_expires_at > now + 60)
if token_is_fresh and (not force_refresh or self._auth_generation != observed_generation):
return
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=False,
trust_env=False,
)
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
try:
response = await client.post(
f"{self.base_url}/api/v1/auth/login",
json={"email": self.admin_email, "password": self.admin_password},
headers={"Accept": "application/json"},
)
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
try:
payload = response.json()
except ValueError as exc:
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
if isinstance(payload, dict) and payload.get("success") is False:
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
token, tenant_id = self._extract_login_token(payload)
if not token:
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
try:
payload = response.json()
except ValueError as exc:
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
if isinstance(payload, dict) and payload.get("success") is False:
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
token, tenant_id = self._extract_login_token(payload)
if not token:
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
self._auth_token = token
self._tenant_id = tenant_id
# WeKnora login responses do not consistently expose expiry. Refresh
# periodically and retry once if upstream says the token is stale.
self._auth_expires_at = now + 25 * 60
self._auth_token = token
self._tenant_id = tenant_id
self._auth_generation += 1
# WeKnora login responses do not consistently expose expiry. Refresh
# periodically and retry once if upstream says the token is stale.
self._auth_expires_at = now + 25 * 60
async def _headers(self, *, force_refresh: bool = False, accept: str = "application/json") -> dict[str, str]:
headers = {"Accept": accept}
@@ -219,8 +225,40 @@ class WeKnoraClient:
payload = await self._request("GET", "/health")
return dict(payload) if isinstance(payload, dict) else {"status": "ok"}
async def list_knowledge_bases(self) -> list[dict[str, Any]]:
return _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
async def list_knowledge_bases(self, *, refresh: bool = False) -> list[dict[str, Any]]:
if refresh:
self._invalidate_knowledge_bases_cache()
now = time.monotonic()
cached = self._knowledge_bases_cache
if cached is not None and cached[0] > now:
return [dict(item) for item in cached[1]]
# Coalesce requests that reach the Gateway at the same time. Keeping
# the upstream read under this narrow lock is intentional: every
# waiter receives the same fresh snapshot, while a failed request does
# not populate the cache and the next caller can retry normally.
async with self._knowledge_bases_lock:
now = time.monotonic()
cached = self._knowledge_bases_cache
if cached is not None and cached[0] > now:
return [dict(item) for item in cached[1]]
observed_generation = self._knowledge_bases_cache_generation
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
# A mutation can finish while this request is in flight. Do not
# let the preceding snapshot repopulate the cache after that
# mutation has explicitly invalidated it.
if self._knowledge_bases_cache_generation == observed_generation:
self._knowledge_bases_cache = (
time.monotonic() + self._KNOWLEDGE_BASES_CACHE_SECONDS,
[dict(item) for item in rows],
)
return [dict(item) for item in rows]
def _invalidate_knowledge_bases_cache(self) -> None:
"""Discard the directory after a successful local mutation."""
self._knowledge_bases_cache = None
self._knowledge_bases_cache_generation += 1
async def get_knowledge_base(self, knowledge_base_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}"))
@@ -273,6 +311,8 @@ class WeKnoraClient:
kb_type: str = "document",
embedding_model_id: str | None = None,
wiki_enabled: bool = False,
wiki_content_instructions: str = "",
wiki_extraction_instructions: str = "",
) -> dict[str, Any]:
model_id = embedding_model_id or await self.get_default_embedding_model_id()
create_payload: dict[str, Any] = {
@@ -295,6 +335,8 @@ class WeKnoraClient:
"wiki_config": {
"synthesis_model_id": synthesis_model_id,
"extraction_granularity": "standard",
"content_instructions": wiki_content_instructions.strip(),
"extraction_instructions": wiki_extraction_instructions.strip(),
},
}
)
@@ -306,16 +348,31 @@ class WeKnoraClient:
value = _data(payload)
if not isinstance(value, dict) or not value.get("id"):
raise WeKnoraError("WeKnora did not return the created knowledge base")
self._invalidate_knowledge_bases_cache()
return dict(value)
async def update_knowledge_base(self, knowledge_base_id: str, changes: Mapping[str, Any]) -> dict[str, Any]:
payload = dict(changes)
if not str(payload.get("name") or "").strip():
wiki_config_changes = payload.pop("wiki_config", None)
current: dict[str, Any] | None = None
if not str(payload.get("name") or "").strip() or wiki_config_changes is not None:
current = await self.get_knowledge_base(knowledge_base_id)
if not str(payload.get("name") or "").strip():
assert current is not None
current_name = str(current.get("name") or "").strip()
if not current_name:
raise WeKnoraError("WeKnora knowledge base name is required for updates")
payload["name"] = current_name
if wiki_config_changes is not None:
if not isinstance(wiki_config_changes, Mapping):
raise WeKnoraError("WeKnora Wiki instructions must be an object", status_code=422)
assert current is not None
current_wiki_config = current.get("wiki_config")
merged_wiki_config = dict(current_wiki_config) if isinstance(current_wiki_config, Mapping) else {}
for key in ("content_instructions", "extraction_instructions"):
if key in wiki_config_changes:
merged_wiki_config[key] = str(wiki_config_changes.get(key) or "").strip()
payload["config"] = {"wiki_config": merged_wiki_config}
value = _data(
await self._request(
"PUT",
@@ -323,10 +380,12 @@ class WeKnoraClient:
json=payload,
)
)
self._invalidate_knowledge_bases_cache()
return dict(value) if isinstance(value, dict) else {}
async def delete_knowledge_base(self, knowledge_base_id: str) -> None:
await self._request("DELETE", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}")
self._invalidate_knowledge_bases_cache()
async def list_documents(self, knowledge_base_id: str, *, page: int = 1, page_size: int = 100) -> dict[str, Any]:
payload = await self._request(
@@ -353,6 +412,7 @@ class WeKnoraClient:
files={"file": (filename, content, content_type)},
)
)
self._invalidate_knowledge_bases_cache()
return dict(value) if isinstance(value, dict) else {}
async def import_document_url(self, knowledge_base_id: str, *, url: str) -> dict[str, Any]:
@@ -363,6 +423,7 @@ class WeKnoraClient:
json={"url": url},
)
)
self._invalidate_knowledge_bases_cache()
return dict(value) if isinstance(value, dict) else {}
async def create_manual_document(
@@ -379,10 +440,12 @@ class WeKnoraClient:
json={"title": title, "content": content, "status": "publish"},
)
)
self._invalidate_knowledge_bases_cache()
return dict(value) if isinstance(value, dict) else {}
async def delete_document(self, knowledge_id: str) -> None:
await self._request("DELETE", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}")
self._invalidate_knowledge_bases_cache()
async def get_document(self, knowledge_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}"))
@@ -477,6 +540,7 @@ class WeKnoraClient:
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
self._invalidate_knowledge_bases_cache()
return dict(value) if isinstance(value, dict) else {}
async def list_chunks(self, knowledge_id: str, *, page: int = 1, page_size: int = 20) -> dict[str, Any]: