fix: restore embedded wiki images and optimize vector sync
This commit is contained in:
1 parent
30943159bf
commit
0c5e2e8cb0
23 files changed
+609
-136
No files matched your search
+46
-1
@@ -2,8 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
@@ -430,6 +430,51 @@ class WeKnoraClient:
|
||||
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
|
||||
return response.content, media_type or "application/octet-stream"
|
||||
|
||||
async def get_knowledge_base_file(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
file_path: str,
|
||||
) -> tuple[bytes, str]:
|
||||
"""Return one protected Wiki attachment using DeerFlow's WeKnora identity."""
|
||||
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
follow_redirects=True,
|
||||
trust_env=False,
|
||||
)
|
||||
try:
|
||||
for attempt in range(2):
|
||||
response = await client.request(
|
||||
"GET",
|
||||
f"{self.base_url}/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/files",
|
||||
params={"file_path": file_path},
|
||||
headers=await self._headers(force_refresh=attempt > 0, accept="*/*"),
|
||||
follow_redirects=True,
|
||||
)
|
||||
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
|
||||
break
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora Wiki image request timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora Wiki image is unavailable", status_code=502) from exc
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
if response.status_code == 404:
|
||||
raise WeKnoraError("WeKnora Wiki image was not found", status_code=404)
|
||||
if response.status_code in {401, 403}:
|
||||
if self._has_admin_credentials:
|
||||
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated Wiki image request", status_code=502)
|
||||
raise WeKnoraError(
|
||||
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
|
||||
status_code=502,
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502)
|
||||
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
|
||||
return response.content, media_type or "application/octet-stream"
|
||||
|
||||
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
+37
-6
@@ -15,6 +15,21 @@ from deerflow.integrations.weknora.local_index.vector_codec import normalize_vec
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _upstream_error_detail(response: httpx.Response) -> str:
|
||||
"""Return a bounded upstream reason without hiding the useful response body."""
|
||||
|
||||
try:
|
||||
payload = response.json()
|
||||
if isinstance(payload, dict):
|
||||
detail = payload.get("detail") or payload.get("message") or payload.get("error") or payload
|
||||
else:
|
||||
detail = payload
|
||||
value = str(detail)
|
||||
except (ValueError, TypeError):
|
||||
value = response.text
|
||||
return " ".join(value.split())[:2000] or "no response body"
|
||||
|
||||
|
||||
class WikiEmbeddingError(RuntimeError):
|
||||
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
|
||||
super().__init__(message)
|
||||
@@ -76,11 +91,15 @@ class StrictWikiEmbeddingClient:
|
||||
started = monotonic()
|
||||
try:
|
||||
response = await (await self._get_client()).post(url, headers=headers, json=payload)
|
||||
if response.status_code in {401, 403}:
|
||||
raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False)
|
||||
if response.status_code == 429 or response.status_code >= 500:
|
||||
raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True)
|
||||
response.raise_for_status()
|
||||
if response.status_code >= 400:
|
||||
detail = _upstream_error_detail(response)
|
||||
category = "authentication" if response.status_code in {401, 403} else "upstream"
|
||||
retryable = response.status_code == 429 or response.status_code >= 500
|
||||
raise WikiEmbeddingError(
|
||||
f"Embedding service returned HTTP {response.status_code}: {detail}",
|
||||
category=category,
|
||||
retryable=retryable,
|
||||
)
|
||||
data = response.json()
|
||||
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
|
||||
if len(items) != len(texts):
|
||||
@@ -98,8 +117,20 @@ class StrictWikiEmbeddingClient:
|
||||
except httpx.TimeoutException:
|
||||
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
|
||||
except (httpx.HTTPError, ValueError, TypeError) as exc:
|
||||
last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError))
|
||||
reason = " ".join(str(exc).split())[:2000] or type(exc).__name__
|
||||
last_error = WikiEmbeddingError(
|
||||
f"Embedding request failed ({type(exc).__name__}): {reason}",
|
||||
category="request",
|
||||
retryable=isinstance(exc, httpx.HTTPError),
|
||||
)
|
||||
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
|
||||
logger.error(
|
||||
"Wiki embedding batch failed model=%s batch=%d category=%s reason=%s",
|
||||
self.config.model,
|
||||
len(texts),
|
||||
last_error.category,
|
||||
last_error,
|
||||
)
|
||||
raise last_error
|
||||
await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
|
||||
raise last_error or WikiEmbeddingError("Embedding request failed")
|
||||
|
||||
Reference in new issue
Block a user