sync update from LAN repo (source dhk-1 @ 3e87be55)
This commit is contained in:
1 parent
ab6068a7c1
commit
42f2a26fa8
1482 files changed
+227759
-9584
No files matched your search
+116
-52
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
@@ -45,9 +46,13 @@ class WeKnoraClient:
|
||||
tests. When omitted, each public method creates and closes a short-lived client.
|
||||
If admin credentials are configured, DeerFlow authenticates to WeKnora with a
|
||||
cached bearer token; otherwise it keeps the legacy trusted-network anonymous
|
||||
mode for deployments that allow DeerFlow through a reverse proxy.
|
||||
mode for deployments that allow DeerFlow through a reverse proxy. The
|
||||
knowledge-base directory is also cached briefly because several Gateway
|
||||
scopes request the same expensive upstream list at once.
|
||||
"""
|
||||
|
||||
_KNOWLEDGE_BASES_CACHE_SECONDS = 15.0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -65,6 +70,11 @@ class WeKnoraClient:
|
||||
self._auth_token = ""
|
||||
self._tenant_id = ""
|
||||
self._auth_expires_at = 0.0
|
||||
self._auth_generation = 0
|
||||
self._auth_lock = asyncio.Lock()
|
||||
self._knowledge_bases_cache: tuple[float, list[dict[str, Any]]] | None = None
|
||||
self._knowledge_bases_cache_generation = 0
|
||||
self._knowledge_bases_lock = asyncio.Lock()
|
||||
if not self.base_url:
|
||||
raise ValueError("WeKnora base URL is required")
|
||||
|
||||
@@ -77,20 +87,8 @@ class WeKnoraClient:
|
||||
if not isinstance(payload, dict):
|
||||
return "", ""
|
||||
source = payload.get("data") if isinstance(payload.get("data"), dict) else payload
|
||||
token = str(
|
||||
source.get("token")
|
||||
or source.get("access_token")
|
||||
or payload.get("token")
|
||||
or payload.get("access_token")
|
||||
or ""
|
||||
)
|
||||
tenant = (
|
||||
source.get("tenant")
|
||||
if isinstance(source.get("tenant"), dict)
|
||||
else source.get("active_tenant")
|
||||
if isinstance(source.get("active_tenant"), dict)
|
||||
else payload.get("tenant")
|
||||
)
|
||||
token = str(source.get("token") or source.get("access_token") or payload.get("token") or payload.get("access_token") or "")
|
||||
tenant = source.get("tenant") if isinstance(source.get("tenant"), dict) else source.get("active_tenant") if isinstance(source.get("active_tenant"), dict) else payload.get("tenant")
|
||||
user = source.get("user") if isinstance(source.get("user"), dict) else payload.get("user")
|
||||
tenant_id = ""
|
||||
if isinstance(tenant, dict):
|
||||
@@ -106,43 +104,51 @@ class WeKnoraClient:
|
||||
if self._auth_token and not force_refresh and self._auth_expires_at > now + 60:
|
||||
return
|
||||
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
follow_redirects=False,
|
||||
trust_env=False,
|
||||
)
|
||||
try:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/v1/auth/login",
|
||||
json={"email": self.admin_email, "password": self.admin_password},
|
||||
headers={"Accept": "application/json"},
|
||||
observed_generation = self._auth_generation
|
||||
async with self._auth_lock:
|
||||
now = time.time()
|
||||
token_is_fresh = bool(self._auth_token and self._auth_expires_at > now + 60)
|
||||
if token_is_fresh and (not force_refresh or self._auth_generation != observed_generation):
|
||||
return
|
||||
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
follow_redirects=False,
|
||||
trust_env=False,
|
||||
)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
try:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/v1/auth/login",
|
||||
json={"email": self.admin_email, "password": self.admin_password},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
|
||||
if response.status_code >= 400:
|
||||
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
|
||||
if isinstance(payload, dict) and payload.get("success") is False:
|
||||
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
|
||||
token, tenant_id = self._extract_login_token(payload)
|
||||
if not token:
|
||||
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
|
||||
if response.status_code >= 400:
|
||||
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
|
||||
if isinstance(payload, dict) and payload.get("success") is False:
|
||||
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
|
||||
token, tenant_id = self._extract_login_token(payload)
|
||||
if not token:
|
||||
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
|
||||
|
||||
self._auth_token = token
|
||||
self._tenant_id = tenant_id
|
||||
# WeKnora login responses do not consistently expose expiry. Refresh
|
||||
# periodically and retry once if upstream says the token is stale.
|
||||
self._auth_expires_at = now + 25 * 60
|
||||
self._auth_token = token
|
||||
self._tenant_id = tenant_id
|
||||
self._auth_generation += 1
|
||||
# WeKnora login responses do not consistently expose expiry. Refresh
|
||||
# periodically and retry once if upstream says the token is stale.
|
||||
self._auth_expires_at = now + 25 * 60
|
||||
|
||||
async def _headers(self, *, force_refresh: bool = False, accept: str = "application/json") -> dict[str, str]:
|
||||
headers = {"Accept": accept}
|
||||
@@ -219,8 +225,40 @@ class WeKnoraClient:
|
||||
payload = await self._request("GET", "/health")
|
||||
return dict(payload) if isinstance(payload, dict) else {"status": "ok"}
|
||||
|
||||
async def list_knowledge_bases(self) -> list[dict[str, Any]]:
|
||||
return _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
|
||||
async def list_knowledge_bases(self, *, refresh: bool = False) -> list[dict[str, Any]]:
|
||||
if refresh:
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
now = time.monotonic()
|
||||
cached = self._knowledge_bases_cache
|
||||
if cached is not None and cached[0] > now:
|
||||
return [dict(item) for item in cached[1]]
|
||||
|
||||
# Coalesce requests that reach the Gateway at the same time. Keeping
|
||||
# the upstream read under this narrow lock is intentional: every
|
||||
# waiter receives the same fresh snapshot, while a failed request does
|
||||
# not populate the cache and the next caller can retry normally.
|
||||
async with self._knowledge_bases_lock:
|
||||
now = time.monotonic()
|
||||
cached = self._knowledge_bases_cache
|
||||
if cached is not None and cached[0] > now:
|
||||
return [dict(item) for item in cached[1]]
|
||||
observed_generation = self._knowledge_bases_cache_generation
|
||||
rows = _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
|
||||
# A mutation can finish while this request is in flight. Do not
|
||||
# let the preceding snapshot repopulate the cache after that
|
||||
# mutation has explicitly invalidated it.
|
||||
if self._knowledge_bases_cache_generation == observed_generation:
|
||||
self._knowledge_bases_cache = (
|
||||
time.monotonic() + self._KNOWLEDGE_BASES_CACHE_SECONDS,
|
||||
[dict(item) for item in rows],
|
||||
)
|
||||
return [dict(item) for item in rows]
|
||||
|
||||
def _invalidate_knowledge_bases_cache(self) -> None:
|
||||
"""Discard the directory after a successful local mutation."""
|
||||
|
||||
self._knowledge_bases_cache = None
|
||||
self._knowledge_bases_cache_generation += 1
|
||||
|
||||
async def get_knowledge_base(self, knowledge_base_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("GET", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}"))
|
||||
@@ -273,6 +311,8 @@ class WeKnoraClient:
|
||||
kb_type: str = "document",
|
||||
embedding_model_id: str | None = None,
|
||||
wiki_enabled: bool = False,
|
||||
wiki_content_instructions: str = "",
|
||||
wiki_extraction_instructions: str = "",
|
||||
) -> dict[str, Any]:
|
||||
model_id = embedding_model_id or await self.get_default_embedding_model_id()
|
||||
create_payload: dict[str, Any] = {
|
||||
@@ -295,6 +335,8 @@ class WeKnoraClient:
|
||||
"wiki_config": {
|
||||
"synthesis_model_id": synthesis_model_id,
|
||||
"extraction_granularity": "standard",
|
||||
"content_instructions": wiki_content_instructions.strip(),
|
||||
"extraction_instructions": wiki_extraction_instructions.strip(),
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -306,16 +348,31 @@ class WeKnoraClient:
|
||||
value = _data(payload)
|
||||
if not isinstance(value, dict) or not value.get("id"):
|
||||
raise WeKnoraError("WeKnora did not return the created knowledge base")
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value)
|
||||
|
||||
async def update_knowledge_base(self, knowledge_base_id: str, changes: Mapping[str, Any]) -> dict[str, Any]:
|
||||
payload = dict(changes)
|
||||
if not str(payload.get("name") or "").strip():
|
||||
wiki_config_changes = payload.pop("wiki_config", None)
|
||||
current: dict[str, Any] | None = None
|
||||
if not str(payload.get("name") or "").strip() or wiki_config_changes is not None:
|
||||
current = await self.get_knowledge_base(knowledge_base_id)
|
||||
if not str(payload.get("name") or "").strip():
|
||||
assert current is not None
|
||||
current_name = str(current.get("name") or "").strip()
|
||||
if not current_name:
|
||||
raise WeKnoraError("WeKnora knowledge base name is required for updates")
|
||||
payload["name"] = current_name
|
||||
if wiki_config_changes is not None:
|
||||
if not isinstance(wiki_config_changes, Mapping):
|
||||
raise WeKnoraError("WeKnora Wiki instructions must be an object", status_code=422)
|
||||
assert current is not None
|
||||
current_wiki_config = current.get("wiki_config")
|
||||
merged_wiki_config = dict(current_wiki_config) if isinstance(current_wiki_config, Mapping) else {}
|
||||
for key in ("content_instructions", "extraction_instructions"):
|
||||
if key in wiki_config_changes:
|
||||
merged_wiki_config[key] = str(wiki_config_changes.get(key) or "").strip()
|
||||
payload["config"] = {"wiki_config": merged_wiki_config}
|
||||
value = _data(
|
||||
await self._request(
|
||||
"PUT",
|
||||
@@ -323,10 +380,12 @@ class WeKnoraClient:
|
||||
json=payload,
|
||||
)
|
||||
)
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def delete_knowledge_base(self, knowledge_base_id: str) -> None:
|
||||
await self._request("DELETE", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}")
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
|
||||
async def list_documents(self, knowledge_base_id: str, *, page: int = 1, page_size: int = 100) -> dict[str, Any]:
|
||||
payload = await self._request(
|
||||
@@ -353,6 +412,7 @@ class WeKnoraClient:
|
||||
files={"file": (filename, content, content_type)},
|
||||
)
|
||||
)
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def import_document_url(self, knowledge_base_id: str, *, url: str) -> dict[str, Any]:
|
||||
@@ -363,6 +423,7 @@ class WeKnoraClient:
|
||||
json={"url": url},
|
||||
)
|
||||
)
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def create_manual_document(
|
||||
@@ -379,10 +440,12 @@ class WeKnoraClient:
|
||||
json={"title": title, "content": content, "status": "publish"},
|
||||
)
|
||||
)
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def delete_document(self, knowledge_id: str) -> None:
|
||||
await self._request("DELETE", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}")
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
|
||||
async def get_document(self, knowledge_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("GET", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}"))
|
||||
@@ -477,6 +540,7 @@ class WeKnoraClient:
|
||||
|
||||
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
|
||||
self._invalidate_knowledge_bases_cache()
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def list_chunks(self, knowledge_id: str, *, page: int = 1, page_size: int = 20) -> dict[str, Any]:
|
||||
|
||||
Reference in new issue
Block a user