初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
@@ -0,0 +1,2 @@
|
||||
"""External service integrations owned by the DeerFlow harness."""
|
||||
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
"""WeKnora REST integration."""
|
||||
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
|
||||
|
||||
__all__ = ["WeKnoraClient", "WeKnoraError"]
|
||||
+674
@@ -0,0 +1,674 @@
|
||||
"""Small, version-pinned adapter over WeKnora's official REST API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
class WeKnoraError(RuntimeError):
|
||||
"""Normalized upstream failure that is safe for DeerFlow to translate."""
|
||||
|
||||
def __init__(self, message: str, *, status_code: int = 502) -> None:
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
def _data(payload: Any) -> Any:
|
||||
if not isinstance(payload, dict):
|
||||
raise WeKnoraError("WeKnora returned an invalid response")
|
||||
if payload.get("success") is False:
|
||||
message = payload.get("message") or payload.get("error") or "WeKnora rejected the request"
|
||||
raise WeKnoraError(str(message), status_code=400)
|
||||
return payload.get("data", payload)
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list[dict[str, Any]]:
|
||||
if isinstance(value, list):
|
||||
return [dict(item) for item in value if isinstance(item, Mapping)]
|
||||
if isinstance(value, dict):
|
||||
for key in ("items", "list", "knowledge_bases", "knowledge", "records"):
|
||||
nested = value.get(key)
|
||||
if isinstance(nested, list):
|
||||
return [dict(item) for item in nested if isinstance(item, Mapping)]
|
||||
return []
|
||||
|
||||
|
||||
class WeKnoraClient:
|
||||
"""Async client for a trusted-network WeKnora endpoint.
|
||||
|
||||
The caller may inject an ``httpx.AsyncClient`` for connection pooling and
|
||||
tests. When omitted, each public method creates and closes a short-lived client.
|
||||
If admin credentials are configured, DeerFlow authenticates to WeKnora with a
|
||||
cached bearer token; otherwise it keeps the legacy trusted-network anonymous
|
||||
mode for deployments that allow DeerFlow through a reverse proxy.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
timeout_seconds: float = 30.0,
|
||||
http_client: httpx.AsyncClient | None = None,
|
||||
admin_email: str = "",
|
||||
admin_password: str = "",
|
||||
) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self._http_client = http_client
|
||||
self.admin_email = admin_email.strip()
|
||||
self.admin_password = admin_password
|
||||
self._auth_token = ""
|
||||
self._tenant_id = ""
|
||||
self._auth_expires_at = 0.0
|
||||
if not self.base_url:
|
||||
raise ValueError("WeKnora base URL is required")
|
||||
|
||||
@property
|
||||
def _has_admin_credentials(self) -> bool:
|
||||
return bool(self.admin_email and self.admin_password)
|
||||
|
||||
@staticmethod
|
||||
def _extract_login_token(payload: Any) -> tuple[str, str]:
|
||||
if not isinstance(payload, dict):
|
||||
return "", ""
|
||||
source = payload.get("data") if isinstance(payload.get("data"), dict) else payload
|
||||
token = str(
|
||||
source.get("token")
|
||||
or source.get("access_token")
|
||||
or payload.get("token")
|
||||
or payload.get("access_token")
|
||||
or ""
|
||||
)
|
||||
tenant = (
|
||||
source.get("tenant")
|
||||
if isinstance(source.get("tenant"), dict)
|
||||
else source.get("active_tenant")
|
||||
if isinstance(source.get("active_tenant"), dict)
|
||||
else payload.get("tenant")
|
||||
)
|
||||
user = source.get("user") if isinstance(source.get("user"), dict) else payload.get("user")
|
||||
tenant_id = ""
|
||||
if isinstance(tenant, dict):
|
||||
tenant_id = str(tenant.get("id") or "")
|
||||
if not tenant_id and isinstance(user, dict):
|
||||
tenant_id = str(user.get("tenant_id") or "")
|
||||
return token, tenant_id
|
||||
|
||||
async def _login(self, *, force_refresh: bool = False) -> None:
|
||||
if not self._has_admin_credentials:
|
||||
return
|
||||
now = time.time()
|
||||
if self._auth_token and not force_refresh and self._auth_expires_at > now + 60:
|
||||
return
|
||||
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
follow_redirects=False,
|
||||
trust_env=False,
|
||||
)
|
||||
try:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/v1/auth/login",
|
||||
json={"email": self.admin_email, "password": self.admin_password},
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
|
||||
if response.status_code >= 400:
|
||||
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
|
||||
if isinstance(payload, dict) and payload.get("success") is False:
|
||||
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
|
||||
token, tenant_id = self._extract_login_token(payload)
|
||||
if not token:
|
||||
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
|
||||
|
||||
self._auth_token = token
|
||||
self._tenant_id = tenant_id
|
||||
# WeKnora login responses do not consistently expose expiry. Refresh
|
||||
# periodically and retry once if upstream says the token is stale.
|
||||
self._auth_expires_at = now + 25 * 60
|
||||
|
||||
async def _headers(self, *, force_refresh: bool = False, accept: str = "application/json") -> dict[str, str]:
|
||||
headers = {"Accept": accept}
|
||||
if self._has_admin_credentials:
|
||||
await self._login(force_refresh=force_refresh)
|
||||
if self._auth_token:
|
||||
headers["Authorization"] = f"Bearer {self._auth_token}"
|
||||
if self._tenant_id:
|
||||
headers["X-Tenant-ID"] = self._tenant_id
|
||||
return headers
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
json: Any = None,
|
||||
params: Mapping[str, Any] | None = None,
|
||||
files: Any = None,
|
||||
data: Mapping[str, Any] | None = None,
|
||||
) -> Any:
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
follow_redirects=False,
|
||||
trust_env=False,
|
||||
)
|
||||
try:
|
||||
for attempt in range(2):
|
||||
response = await client.request(
|
||||
method,
|
||||
f"{self.base_url}{path}",
|
||||
headers=await self._headers(force_refresh=attempt > 0),
|
||||
json=json,
|
||||
params=params,
|
||||
files=files,
|
||||
data=data,
|
||||
)
|
||||
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
|
||||
break
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
if response.status_code == 404:
|
||||
raise WeKnoraError("WeKnora resource was not found", status_code=404)
|
||||
if response.status_code in {401, 403}:
|
||||
if self._has_admin_credentials:
|
||||
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated internal request", status_code=502)
|
||||
raise WeKnoraError(
|
||||
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
|
||||
status_code=502,
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
detail = ""
|
||||
try:
|
||||
body = response.json()
|
||||
if isinstance(body, dict):
|
||||
detail = str(body.get("message") or body.get("error") or body.get("detail") or "")
|
||||
except ValueError:
|
||||
detail = ""
|
||||
raise WeKnoraError(detail or f"WeKnora returned HTTP {response.status_code}", status_code=502)
|
||||
if response.status_code == 204 or not response.content:
|
||||
return {"success": True}
|
||||
try:
|
||||
return response.json()
|
||||
except ValueError as exc:
|
||||
raise WeKnoraError("WeKnora returned invalid JSON") from exc
|
||||
|
||||
async def health(self) -> dict[str, Any]:
|
||||
payload = await self._request("GET", "/health")
|
||||
return dict(payload) if isinstance(payload, dict) else {"status": "ok"}
|
||||
|
||||
async def list_knowledge_bases(self) -> list[dict[str, Any]]:
|
||||
return _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
|
||||
|
||||
async def get_knowledge_base(self, knowledge_base_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("GET", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}"))
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("WeKnora returned an invalid knowledge base")
|
||||
return dict(value)
|
||||
|
||||
async def list_models(self) -> list[dict[str, Any]]:
|
||||
"""List models from WeKnora; model credentials are never returned."""
|
||||
|
||||
return _as_list(_data(await self._request("GET", "/api/v1/models")))
|
||||
|
||||
async def get_default_embedding_model_id(self) -> str:
|
||||
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "embedding"]
|
||||
preferred = next((item for item in models if item.get("is_default") is True), None)
|
||||
selected = preferred or next(
|
||||
(item for item in models if str(item.get("status") or "active").lower() == "active"),
|
||||
None,
|
||||
)
|
||||
model_id = str((selected or {}).get("id") or "")
|
||||
if not model_id:
|
||||
raise WeKnoraError(
|
||||
"WeKnora has no available embedding model; configure its default model in WeKnora",
|
||||
status_code=400,
|
||||
)
|
||||
return model_id
|
||||
|
||||
async def get_default_knowledge_qa_model_id(self) -> str:
|
||||
"""Resolve WeKnora's own active synthesis model for Wiki indexing."""
|
||||
|
||||
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "knowledgeqa"]
|
||||
preferred = next((item for item in models if item.get("is_default") is True), None)
|
||||
selected = preferred or next(
|
||||
(item for item in models if str(item.get("status") or "active").lower() == "active"),
|
||||
None,
|
||||
)
|
||||
model_id = str((selected or {}).get("id") or "")
|
||||
if not model_id:
|
||||
raise WeKnoraError(
|
||||
"WeKnora has no available KnowledgeQA model; configure one in WeKnora before creating a Wiki",
|
||||
status_code=400,
|
||||
)
|
||||
return model_id
|
||||
|
||||
async def create_knowledge_base(
|
||||
self,
|
||||
*,
|
||||
name: str,
|
||||
description: str = "",
|
||||
kb_type: str = "document",
|
||||
embedding_model_id: str | None = None,
|
||||
wiki_enabled: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
model_id = embedding_model_id or await self.get_default_embedding_model_id()
|
||||
create_payload: dict[str, Any] = {
|
||||
"name": name,
|
||||
"description": description,
|
||||
"type": kb_type,
|
||||
"embedding_model_id": model_id,
|
||||
}
|
||||
if wiki_enabled:
|
||||
synthesis_model_id = await self.get_default_knowledge_qa_model_id()
|
||||
create_payload.update(
|
||||
{
|
||||
"summary_model_id": synthesis_model_id,
|
||||
"indexing_strategy": {
|
||||
"vector_enabled": True,
|
||||
"keyword_enabled": True,
|
||||
"wiki_enabled": True,
|
||||
"graph_enabled": False,
|
||||
},
|
||||
"wiki_config": {
|
||||
"synthesis_model_id": synthesis_model_id,
|
||||
"extraction_granularity": "standard",
|
||||
},
|
||||
}
|
||||
)
|
||||
payload = await self._request(
|
||||
"POST",
|
||||
"/api/v1/knowledge-bases",
|
||||
json=create_payload,
|
||||
)
|
||||
value = _data(payload)
|
||||
if not isinstance(value, dict) or not value.get("id"):
|
||||
raise WeKnoraError("WeKnora did not return the created knowledge base")
|
||||
return dict(value)
|
||||
|
||||
async def update_knowledge_base(self, knowledge_base_id: str, changes: Mapping[str, Any]) -> dict[str, Any]:
|
||||
payload = dict(changes)
|
||||
if not str(payload.get("name") or "").strip():
|
||||
current = await self.get_knowledge_base(knowledge_base_id)
|
||||
current_name = str(current.get("name") or "").strip()
|
||||
if not current_name:
|
||||
raise WeKnoraError("WeKnora knowledge base name is required for updates")
|
||||
payload["name"] = current_name
|
||||
value = _data(
|
||||
await self._request(
|
||||
"PUT",
|
||||
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}",
|
||||
json=payload,
|
||||
)
|
||||
)
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def delete_knowledge_base(self, knowledge_base_id: str) -> None:
|
||||
await self._request("DELETE", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}")
|
||||
|
||||
async def list_documents(self, knowledge_base_id: str, *, page: int = 1, page_size: int = 100) -> dict[str, Any]:
|
||||
payload = await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge",
|
||||
params={"page": page, "page_size": page_size},
|
||||
)
|
||||
rows = _as_list(_data(payload))
|
||||
total = payload.get("total", len(rows)) if isinstance(payload, dict) else len(rows)
|
||||
return {"items": rows, "total": int(total or 0), "page": page, "page_size": page_size}
|
||||
|
||||
async def upload_document(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
*,
|
||||
filename: str,
|
||||
content: bytes,
|
||||
content_type: str = "application/octet-stream",
|
||||
) -> dict[str, Any]:
|
||||
value = _data(
|
||||
await self._request(
|
||||
"POST",
|
||||
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge/file",
|
||||
files={"file": (filename, content, content_type)},
|
||||
)
|
||||
)
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def import_document_url(self, knowledge_base_id: str, *, url: str) -> dict[str, Any]:
|
||||
value = _data(
|
||||
await self._request(
|
||||
"POST",
|
||||
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge/url",
|
||||
json={"url": url},
|
||||
)
|
||||
)
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def create_manual_document(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
*,
|
||||
title: str,
|
||||
content: str,
|
||||
) -> dict[str, Any]:
|
||||
value = _data(
|
||||
await self._request(
|
||||
"POST",
|
||||
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge/manual",
|
||||
json={"title": title, "content": content, "status": "publish"},
|
||||
)
|
||||
)
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def delete_document(self, knowledge_id: str) -> None:
|
||||
await self._request("DELETE", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}")
|
||||
|
||||
async def get_document(self, knowledge_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("GET", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}"))
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("WeKnora returned an invalid document")
|
||||
return dict(value)
|
||||
|
||||
async def get_document_preview(self, knowledge_id: str) -> tuple[bytes, str]:
|
||||
"""Return the original document preview bytes and its media type."""
|
||||
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
follow_redirects=True,
|
||||
trust_env=False,
|
||||
)
|
||||
try:
|
||||
for attempt in range(2):
|
||||
response = await client.request(
|
||||
"GET",
|
||||
f"{self.base_url}/api/v1/knowledge/{quote(knowledge_id, safe='')}/preview",
|
||||
headers=await self._headers(force_refresh=attempt > 0, accept="*/*"),
|
||||
follow_redirects=True,
|
||||
)
|
||||
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
|
||||
break
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora preview request timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora preview is unavailable", status_code=502) from exc
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
if response.status_code == 404:
|
||||
raise WeKnoraError("WeKnora document preview was not found", status_code=404)
|
||||
if response.status_code in {401, 403}:
|
||||
if self._has_admin_credentials:
|
||||
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated preview request", status_code=502)
|
||||
raise WeKnoraError(
|
||||
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
|
||||
status_code=502,
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502)
|
||||
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
|
||||
return response.content, media_type or "application/octet-stream"
|
||||
|
||||
async def get_knowledge_base_file(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
file_path: str,
|
||||
) -> tuple[bytes, str]:
|
||||
"""Return one protected Wiki attachment using DeerFlow's WeKnora identity."""
|
||||
|
||||
owns_client = self._http_client is None
|
||||
client = self._http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(self.timeout_seconds),
|
||||
follow_redirects=True,
|
||||
trust_env=False,
|
||||
)
|
||||
try:
|
||||
for attempt in range(2):
|
||||
response = await client.request(
|
||||
"GET",
|
||||
f"{self.base_url}/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/files",
|
||||
params={"file_path": file_path},
|
||||
headers=await self._headers(force_refresh=attempt > 0, accept="*/*"),
|
||||
follow_redirects=True,
|
||||
)
|
||||
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
|
||||
break
|
||||
except httpx.TimeoutException as exc:
|
||||
raise WeKnoraError("WeKnora Wiki image request timed out", status_code=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WeKnoraError("WeKnora Wiki image is unavailable", status_code=502) from exc
|
||||
finally:
|
||||
if owns_client:
|
||||
await client.aclose()
|
||||
if response.status_code == 404:
|
||||
raise WeKnoraError("WeKnora Wiki image was not found", status_code=404)
|
||||
if response.status_code in {401, 403}:
|
||||
if self._has_admin_credentials:
|
||||
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated Wiki image request", status_code=502)
|
||||
raise WeKnoraError(
|
||||
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
|
||||
status_code=502,
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502)
|
||||
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
|
||||
return response.content, media_type or "application/octet-stream"
|
||||
|
||||
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def list_chunks(self, knowledge_id: str, *, page: int = 1, page_size: int = 20) -> dict[str, Any]:
|
||||
payload = await self._request(
|
||||
"GET",
|
||||
f"/api/v1/chunks/{quote(knowledge_id, safe='')}",
|
||||
params={"page": page, "page_size": page_size},
|
||||
)
|
||||
rows = _as_list(_data(payload))
|
||||
total = payload.get("total", len(rows)) if isinstance(payload, dict) else len(rows)
|
||||
return {"items": rows, "total": int(total or 0), "page": page, "page_size": page_size}
|
||||
|
||||
async def get_wiki_graph(self, knowledge_base_id: str, *, limit: int = 200) -> dict[str, Any]:
|
||||
value = _data(
|
||||
await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/graph",
|
||||
params={"mode": "overview", "limit": max(1, min(limit, 10000))},
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("WeKnora returned an invalid wiki graph")
|
||||
return dict(value)
|
||||
|
||||
async def get_wiki_stats(self, knowledge_base_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("GET", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/stats"))
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("Invalid Wiki generation status")
|
||||
return value
|
||||
|
||||
async def create_wiki_folder(self, knowledge_base_id: str, *, name: str, parent_id: str = "") -> dict[str, Any]:
|
||||
value = _data(await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders", json={"name": name, "parent_id": parent_id}))
|
||||
if not isinstance(value, dict) or not value.get("id"):
|
||||
raise WeKnoraError("Invalid Wiki folder response")
|
||||
return value
|
||||
|
||||
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
|
||||
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
|
||||
|
||||
async def list_wiki_pages(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
*,
|
||||
page: int = 1,
|
||||
page_size: int = 50,
|
||||
query: str = "",
|
||||
) -> dict[str, Any]:
|
||||
payload = await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages",
|
||||
params={"page": page, "page_size": page_size, "query": query or None},
|
||||
)
|
||||
value = _data(payload)
|
||||
source = value if isinstance(value, dict) else payload if isinstance(payload, dict) else {}
|
||||
rows = _as_list(source.get("pages") if isinstance(source, dict) else value)
|
||||
return {
|
||||
"pages": rows,
|
||||
"total": int(source.get("total") or len(rows)),
|
||||
"page": int(source.get("page") or page),
|
||||
"page_size": int(source.get("page_size") or page_size),
|
||||
"total_pages": int(source.get("total_pages") or max(1, (int(source.get("total") or len(rows)) + page_size - 1) // page_size)),
|
||||
}
|
||||
|
||||
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict[str, Any]:
|
||||
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
|
||||
value = _data(
|
||||
await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("WeKnora returned an invalid Wiki page")
|
||||
return dict(value)
|
||||
|
||||
async def get_wiki_index(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
*,
|
||||
types: Sequence[str],
|
||||
limit: int = 50,
|
||||
cursor: str = "",
|
||||
) -> dict[str, Any]:
|
||||
params: dict[str, Any] = {
|
||||
"types": ",".join(value for value in types if value),
|
||||
"limit": max(1, min(limit, 500)),
|
||||
}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
value = _data(
|
||||
await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/index",
|
||||
params=params,
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("WeKnora returned an invalid Wiki index")
|
||||
return dict(value)
|
||||
|
||||
async def create_wiki_page(self, knowledge_base_id: str, payload: Mapping[str, Any]) -> dict[str, Any]:
|
||||
value = _data(
|
||||
await self._request(
|
||||
"POST",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages",
|
||||
json=dict(payload),
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("WeKnora returned an invalid Wiki page")
|
||||
return dict(value)
|
||||
|
||||
async def update_wiki_page(self, knowledge_base_id: str, slug: str, changes: Mapping[str, Any]) -> dict[str, Any]:
|
||||
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
|
||||
value = _data(
|
||||
await self._request(
|
||||
"PUT",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
|
||||
json=dict(changes),
|
||||
)
|
||||
)
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
async def delete_wiki_page(self, knowledge_base_id: str, slug: str) -> None:
|
||||
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
|
||||
await self._request(
|
||||
"DELETE",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
|
||||
)
|
||||
|
||||
async def search(self, query: str, knowledge_base_ids: Sequence[str]) -> list[dict[str, Any]]:
|
||||
ids = [str(value) for value in knowledge_base_ids if str(value)]
|
||||
if not ids:
|
||||
return []
|
||||
rows = _as_list(
|
||||
_data(
|
||||
await self._request(
|
||||
"POST",
|
||||
"/api/v1/knowledge-search",
|
||||
json={"query": query, "knowledge_base_ids": ids},
|
||||
)
|
||||
)
|
||||
)
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
normalized.append(
|
||||
{
|
||||
"chunk_id": str(row.get("id") or row.get("chunk_id") or ""),
|
||||
"content": str(row.get("content") or ""),
|
||||
"knowledge_id": str(row.get("knowledge_id") or ""),
|
||||
"knowledge_base_id": str(row.get("knowledge_base_id") or ""),
|
||||
"title": str(row.get("knowledge_title") or row.get("title") or row.get("knowledge_filename") or ""),
|
||||
"filename": str(row.get("knowledge_filename") or ""),
|
||||
"score": float(row.get("score") or 0),
|
||||
"chunk_index": int(row.get("chunk_index") or 0),
|
||||
"metadata": row.get("metadata") if isinstance(row.get("metadata"), dict) else {},
|
||||
}
|
||||
)
|
||||
return normalized
|
||||
|
||||
async def get_chunk(self, chunk_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("GET", f"/api/v1/chunks/by-id/{quote(chunk_id, safe='')}"))
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("WeKnora returned an invalid chunk")
|
||||
return dict(value)
|
||||
|
||||
async def get_chunk_context(self, chunk_id: str, *, expected_knowledge_base_id: str, radius: int = 2) -> list[dict[str, Any]]:
|
||||
center = await self.get_chunk(chunk_id)
|
||||
if str(center.get("knowledge_base_id") or "") != expected_knowledge_base_id:
|
||||
raise WeKnoraError("Chunk does not belong to the authorized knowledge base", status_code=404)
|
||||
|
||||
before: list[dict[str, Any]] = []
|
||||
cursor = center
|
||||
for _ in range(max(0, min(radius, 4))):
|
||||
previous_id = str(cursor.get("pre_chunk_id") or "")
|
||||
if not previous_id:
|
||||
break
|
||||
previous = await self.get_chunk(previous_id)
|
||||
if str(previous.get("knowledge_base_id") or "") != expected_knowledge_base_id:
|
||||
break
|
||||
before.append(previous)
|
||||
cursor = previous
|
||||
|
||||
after: list[dict[str, Any]] = []
|
||||
cursor = center
|
||||
for _ in range(max(0, min(radius, 4))):
|
||||
next_id = str(cursor.get("next_chunk_id") or "")
|
||||
if not next_id:
|
||||
break
|
||||
following = await self.get_chunk(next_id)
|
||||
if str(following.get("knowledge_base_id") or "") != expected_knowledge_base_id:
|
||||
break
|
||||
after.append(following)
|
||||
cursor = following
|
||||
return [*reversed(before), center, *after]
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.integrations.weknora.local_index.retrieval import (
|
||||
WikiRetrievalService,
|
||||
is_fully_vectorized,
|
||||
)
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
|
||||
__all__ = [
|
||||
"WikiIndexError",
|
||||
"WikiRetrievalService",
|
||||
"WikiVectorSearchService",
|
||||
"WikiSyncService",
|
||||
"is_fully_vectorized",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name in {"WikiRetrievalService", "is_fully_vectorized"}:
|
||||
from deerflow.integrations.weknora.local_index import retrieval
|
||||
|
||||
return getattr(retrieval, name)
|
||||
if name in {"WikiIndexError", "WikiVectorSearchService"}:
|
||||
from deerflow.integrations.weknora.local_index import search
|
||||
|
||||
return getattr(search, name)
|
||||
if name == "WikiSyncService":
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
|
||||
return WikiSyncService
|
||||
raise AttributeError(name)
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
"""Markdown-aware Wiki section chunking."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
_FRONTMATTER_RE = re.compile(r"\A\s*---\s*\n.*?\n---\s*(?:\n|\Z)", re.DOTALL)
|
||||
_SCRIPT_STYLE_RE = re.compile(r"<\s*(script|style)\b[^>]*>.*?<\s*/\s*\1\s*>", re.IGNORECASE | re.DOTALL)
|
||||
_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\([^)]*\)")
|
||||
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WikiSection:
|
||||
section_index: int
|
||||
heading: str | None
|
||||
content: str
|
||||
embedding_text: str
|
||||
content_hash: str
|
||||
|
||||
|
||||
def _clean_markdown(value: str) -> str:
|
||||
text = value.replace("\r\n", "\n")
|
||||
text = _FRONTMATTER_RE.sub("", text)
|
||||
text = _SCRIPT_STYLE_RE.sub("", text)
|
||||
text = _IMAGE_RE.sub(lambda match: match.group(1), text)
|
||||
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
||||
|
||||
|
||||
def _windows(text: str, max_chars: int, overlap: int) -> list[str]:
|
||||
paragraphs = [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
|
||||
chunks: list[str] = []
|
||||
buffer = ""
|
||||
for paragraph in paragraphs:
|
||||
if len(buffer) + len(paragraph) + 2 <= max_chars:
|
||||
buffer = f"{buffer}\n\n{paragraph}".strip()
|
||||
continue
|
||||
if buffer:
|
||||
chunks.append(buffer)
|
||||
buffer = ""
|
||||
if len(paragraph) <= max_chars:
|
||||
buffer = paragraph
|
||||
continue
|
||||
step = max(1, max_chars - overlap)
|
||||
chunks.extend(paragraph[start : start + max_chars] for start in range(0, len(paragraph), step))
|
||||
if buffer:
|
||||
chunks.append(buffer)
|
||||
return chunks
|
||||
|
||||
|
||||
def chunk_wiki_page(
|
||||
page: dict,
|
||||
*,
|
||||
max_chars: int,
|
||||
overlap: int,
|
||||
min_chars: int,
|
||||
title_prefix_enabled: bool,
|
||||
) -> list[WikiSection]:
|
||||
content = _clean_markdown(str(page.get("content_md") or ""))
|
||||
heading_stack: list[str] = []
|
||||
sections: list[tuple[str | None, str]] = []
|
||||
current_heading: str | None = None
|
||||
body: list[str] = []
|
||||
for line in content.splitlines():
|
||||
match = _HEADING_RE.match(line)
|
||||
if not match:
|
||||
body.append(line)
|
||||
continue
|
||||
if any(item.strip() for item in body):
|
||||
sections.append((current_heading, "\n".join(body).strip()))
|
||||
level = len(match.group(1))
|
||||
heading_stack = heading_stack[: level - 1]
|
||||
heading_stack.append(match.group(2).strip())
|
||||
current_heading = " / ".join(heading_stack)
|
||||
body = []
|
||||
if any(item.strip() for item in body):
|
||||
sections.append((current_heading, "\n".join(body).strip()))
|
||||
if content and not sections:
|
||||
sections.append((None, content))
|
||||
summary = _clean_markdown(str(page.get("summary") or ""))
|
||||
if summary:
|
||||
sections.insert(0, ("页面摘要", summary))
|
||||
|
||||
prefix_lines = [f"页面:{page.get('title') or page.get('slug') or ''}"]
|
||||
aliases = [str(item) for item in page.get("aliases") or [] if str(item).strip()]
|
||||
if aliases:
|
||||
prefix_lines.append(f"别名:{'、'.join(aliases)}")
|
||||
if page.get("page_type"):
|
||||
prefix_lines.append(f"类型:{page['page_type']}")
|
||||
seen: set[str] = set()
|
||||
result: list[WikiSection] = []
|
||||
for heading, section_text in sections:
|
||||
for chunk in _windows(section_text, max_chars, overlap):
|
||||
normalized = re.sub(r"\s+", " ", chunk).strip()
|
||||
if len(normalized) < min_chars and len(sections) > 1:
|
||||
continue
|
||||
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
if digest in seen:
|
||||
continue
|
||||
seen.add(digest)
|
||||
context = list(prefix_lines) if title_prefix_enabled else []
|
||||
if heading:
|
||||
context.append(f"章节:{heading}")
|
||||
embedding_text = "\n".join([*context, "", chunk]).strip()
|
||||
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
|
||||
# A short article with a short summary must not disappear merely because
|
||||
# both sections fall below min_chars. Keep one combined, useful vector.
|
||||
if not result and (content or summary):
|
||||
fallback = "\n\n".join(dict.fromkeys(text for text in (summary, content) if text))
|
||||
normalized = re.sub(r"\s+", " ", fallback).strip()
|
||||
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
prefix = prefix_lines if title_prefix_enabled else []
|
||||
result.append(WikiSection(0, None, fallback, "\n".join([*prefix, "", fallback]).strip(), digest))
|
||||
return result
|
||||
+480
@@ -0,0 +1,480 @@
|
||||
"""Strict OpenAI-compatible embedding client for the Wiki index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from time import monotonic
|
||||
|
||||
import httpx
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _upstream_error_detail(response: httpx.Response) -> str:
|
||||
"""Return a bounded upstream reason without hiding the useful response body."""
|
||||
|
||||
try:
|
||||
payload = response.json()
|
||||
if isinstance(payload, dict):
|
||||
detail = payload.get("detail") or payload.get("message") or payload.get("error") or payload
|
||||
else:
|
||||
detail = payload
|
||||
value = str(detail)
|
||||
except (ValueError, TypeError):
|
||||
value = response.text
|
||||
return " ".join(value.split())[:2000] or "no response body"
|
||||
|
||||
|
||||
class WikiEmbeddingError(RuntimeError):
|
||||
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
|
||||
super().__init__(message)
|
||||
self.category = category
|
||||
self.retryable = retryable
|
||||
|
||||
|
||||
class StrictWikiEmbeddingClient:
|
||||
def __init__(self, config: LocalWikiEmbeddingConfig, *, http_client: httpx.AsyncClient | None = None) -> None:
|
||||
self.config = config
|
||||
self._client = http_client
|
||||
self._owns_client = http_client is None
|
||||
self._resolved_dimensions = config.dimensions
|
||||
self._local_model = None
|
||||
self._local_process: subprocess.Popen[str] | None = None
|
||||
self._local_lock = asyncio.Lock()
|
||||
self._local_state = "idle"
|
||||
self._local_error: str | None = None
|
||||
self._local_started_at: str | None = None
|
||||
self._local_ready_at: str | None = None
|
||||
self._local_warmup_task: asyncio.Task[None] | None = None
|
||||
|
||||
@property
|
||||
def dimensions(self) -> int | None:
|
||||
return self._resolved_dimensions
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
raw = "\0".join((self.config.provider.strip().lower(), self.config.base_url.rstrip("/").lower(), self.config.model.strip(), str(self._resolved_dimensions or "auto")))
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.reset_local_encoder()
|
||||
if self._owns_client and self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def reset_local_encoder(self, *, model_path: str | None = None) -> None:
|
||||
"""Stop only the embedded worker and optionally switch its model path.
|
||||
|
||||
This is intentionally independent from the Gateway lifecycle: an
|
||||
administrator can replace the offline model without restarting or
|
||||
stopping the rest of DeerFlow.
|
||||
"""
|
||||
warmup = self._local_warmup_task
|
||||
if warmup is not None and not warmup.done():
|
||||
warmup.cancel()
|
||||
await asyncio.gather(warmup, return_exceptions=True)
|
||||
self._local_warmup_task = None
|
||||
process = self._local_process
|
||||
self._local_process = None
|
||||
if process is not None:
|
||||
await asyncio.to_thread(self._stop_local_process, process)
|
||||
self._local_model = None
|
||||
self._local_state = "idle"
|
||||
self._local_error = None
|
||||
self._local_started_at = None
|
||||
self._local_ready_at = None
|
||||
if model_path is not None:
|
||||
self.config = self.config.model_copy(update={"local_model_path": model_path})
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
if self._client is None:
|
||||
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
|
||||
return self._client
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
if self.config.provider == "local":
|
||||
if not texts:
|
||||
return []
|
||||
async with self._local_lock:
|
||||
self._local_state = "starting"
|
||||
self._local_started_at = self._local_started_at or datetime.now(UTC).isoformat()
|
||||
try:
|
||||
values = await asyncio.to_thread(self._embed_local, texts)
|
||||
except Exception as exc:
|
||||
self._local_state = "failed"
|
||||
self._local_error = " ".join(str(exc).split())[:1000] or type(exc).__name__
|
||||
raise
|
||||
self._local_state = "ready"
|
||||
self._local_error = None
|
||||
self._local_ready_at = datetime.now(UTC).isoformat()
|
||||
return values
|
||||
if not self.config.enabled or not self.config.base_url or not self.config.model:
|
||||
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
|
||||
if not texts:
|
||||
return []
|
||||
vectors: list[list[float]] = []
|
||||
batch_size = max(1, self.config.batch_size)
|
||||
for start in range(0, len(texts), batch_size):
|
||||
vectors.extend(await self._embed_batch(texts[start : start + batch_size]))
|
||||
if len(vectors) != len(texts):
|
||||
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
|
||||
return vectors
|
||||
|
||||
def _embed_local(self, texts: list[str]) -> list[list[float]]:
|
||||
if not self.config.enabled:
|
||||
raise WikiEmbeddingError("本地编码已关闭", retryable=False)
|
||||
model_path = self._validated_local_model_path()
|
||||
if self.config.local_process_isolation:
|
||||
return self._embed_local_process(texts, model_path)
|
||||
return self._embed_local_in_process(texts, model_path)
|
||||
|
||||
def _validated_local_model_path(self) -> Path:
|
||||
model_path = self._local_model_path()
|
||||
if model_path is None:
|
||||
raise WikiEmbeddingError(
|
||||
"离线本地编码器目录未配置;请设置 local_model_path 或 DEERFLOW_BGE_M3_MODEL_PATH",
|
||||
category="configuration",
|
||||
retryable=False,
|
||||
)
|
||||
if not model_path.is_dir() or not (
|
||||
(model_path / "onnx" / "model.onnx").is_file()
|
||||
or (model_path / "model.onnx").is_file()
|
||||
):
|
||||
raise WikiEmbeddingError(
|
||||
f"离线本地编码器目录无效或缺少 ONNX 模型:{model_path}",
|
||||
category="configuration",
|
||||
retryable=False,
|
||||
)
|
||||
return model_path
|
||||
|
||||
def _embed_local_in_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
|
||||
if self._local_model is None:
|
||||
try:
|
||||
from fastembed import TextEmbedding
|
||||
except ImportError as exc:
|
||||
raise WikiEmbeddingError("标准后端依赖缺少 fastembed,请检查离线 wheelhouse 和安装结果", retryable=False) from exc
|
||||
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
|
||||
if name == "BAAI/bge-m3" and not any(m["model"] == name for m in TextEmbedding.list_supported_models()):
|
||||
from fastembed.common.model_description import ModelSource, PoolingType
|
||||
|
||||
TextEmbedding.add_custom_model(
|
||||
model=name,
|
||||
pooling=PoolingType.CLS,
|
||||
normalization=True,
|
||||
dim=1024,
|
||||
sources=ModelSource(hf="BAAI/bge-m3"),
|
||||
model_file="onnx/model.onnx",
|
||||
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
|
||||
license="mit",
|
||||
size_in_gb=2.29,
|
||||
)
|
||||
cache_dir = Path(self.config.local_cache_dir).expanduser() if self.config.local_cache_dir else Path.home() / ".cache" / "deerflow" / "embeddings"
|
||||
options = {
|
||||
"model_name": name,
|
||||
"threads": self.config.local_threads,
|
||||
"cache_dir": str(cache_dir),
|
||||
"providers": ["CPUExecutionProvider"],
|
||||
"specific_model_path": str(model_path),
|
||||
"local_files_only": True,
|
||||
}
|
||||
self._local_model = TextEmbedding(**options)
|
||||
values = []
|
||||
for vector in self._local_model.embed(texts, batch_size=min(self.config.batch_size, 4)):
|
||||
normalized = normalize_vector(vector, dimensions=self._resolved_dimensions)
|
||||
if self._resolved_dimensions is None:
|
||||
self._resolved_dimensions = int(normalized.size)
|
||||
values.append(normalized.tolist())
|
||||
if len(values) != len(texts):
|
||||
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
|
||||
return values
|
||||
|
||||
def _embed_local_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
|
||||
process = self._local_process
|
||||
if process is None or process.poll() is not None:
|
||||
process = self._start_local_process(model_path)
|
||||
self._local_process = process
|
||||
if process.stdin is None or process.stdout is None:
|
||||
raise WikiEmbeddingError("本地编码器子进程通信通道不可用", retryable=False)
|
||||
request_id = hashlib.sha256(f"{monotonic()}:{len(texts)}".encode()).hexdigest()[:16]
|
||||
try:
|
||||
process.stdin.write(
|
||||
json.dumps({"id": request_id, "texts": texts}, ensure_ascii=False) + "\n"
|
||||
)
|
||||
process.stdin.flush()
|
||||
response = self._read_local_process_message(process, request_id=request_id)
|
||||
except (BrokenPipeError, OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
self._stop_local_process(process)
|
||||
self._local_process = None
|
||||
raise WikiEmbeddingError(
|
||||
f"本地编码器子进程异常退出:{str(exc)[:500]}",
|
||||
category="local_process",
|
||||
retryable=True,
|
||||
) from exc
|
||||
if response.get("status") != "ok":
|
||||
error = str(response.get("error") or "本地编码失败")[:1000]
|
||||
if process.poll() is not None:
|
||||
self._local_process = None
|
||||
raise WikiEmbeddingError(error, category="local_process", retryable=False)
|
||||
raw_vectors = response.get("vectors")
|
||||
if not isinstance(raw_vectors, list) or len(raw_vectors) != len(texts):
|
||||
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
|
||||
values: list[list[float]] = []
|
||||
for raw in raw_vectors:
|
||||
normalized = normalize_vector(raw, dimensions=self._resolved_dimensions)
|
||||
if self._resolved_dimensions is None:
|
||||
self._resolved_dimensions = int(normalized.size)
|
||||
values.append(normalized.tolist())
|
||||
return values
|
||||
|
||||
def _start_local_process(self, model_path: Path) -> subprocess.Popen[str]:
|
||||
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
|
||||
cache_dir = (
|
||||
Path(self.config.local_cache_dir).expanduser()
|
||||
if self.config.local_cache_dir
|
||||
else Path.home() / ".cache" / "deerflow" / "embeddings"
|
||||
)
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"deerflow.integrations.weknora.local_index.embedding_worker",
|
||||
"--model-name",
|
||||
name,
|
||||
"--model-path",
|
||||
str(model_path),
|
||||
"--cache-dir",
|
||||
str(cache_dir),
|
||||
"--threads",
|
||||
str(self.config.local_threads),
|
||||
"--batch-size",
|
||||
str(min(self.config.batch_size, 4)),
|
||||
]
|
||||
environment = os.environ.copy()
|
||||
environment.update(
|
||||
{
|
||||
"HF_HUB_OFFLINE": "1",
|
||||
"TRANSFORMERS_OFFLINE": "1",
|
||||
"HF_DATASETS_OFFLINE": "1",
|
||||
"PYTHONIOENCODING": "utf-8",
|
||||
"PYTHONUTF8": "1",
|
||||
}
|
||||
)
|
||||
creation_flags = 0
|
||||
if os.name == "nt":
|
||||
creation_flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) | int(
|
||||
getattr(subprocess, "BELOW_NORMAL_PRIORITY_CLASS", 0)
|
||||
)
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
bufsize=1,
|
||||
env=environment,
|
||||
creationflags=creation_flags,
|
||||
)
|
||||
response = self._read_local_process_message(process, ready=True)
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
if "process" in locals():
|
||||
self._stop_local_process(process)
|
||||
raise WikiEmbeddingError(
|
||||
f"无法启动本地编码器子进程:{str(exc)[:500]}",
|
||||
category="local_process",
|
||||
retryable=False,
|
||||
) from exc
|
||||
if response.get("status") != "ready":
|
||||
error = str(response.get("error") or "本地编码器初始化失败")[:1000]
|
||||
self._stop_local_process(process)
|
||||
raise WikiEmbeddingError(error, category="local_process", retryable=False)
|
||||
return process
|
||||
|
||||
@staticmethod
|
||||
def _read_local_process_message(
|
||||
process: subprocess.Popen[str],
|
||||
*,
|
||||
request_id: str | None = None,
|
||||
ready: bool = False,
|
||||
) -> dict[str, object]:
|
||||
if process.stdout is None:
|
||||
raise ValueError("本地编码器没有输出通道")
|
||||
for _ in range(20):
|
||||
line = process.stdout.readline()
|
||||
if not line:
|
||||
raise ValueError(f"本地编码器已退出(exit={process.poll()})")
|
||||
payload = json.loads(line)
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
if ready and payload.get("status") in {"ready", "error"}:
|
||||
return payload
|
||||
if request_id is not None and payload.get("id") == request_id:
|
||||
return payload
|
||||
raise ValueError("本地编码器返回了无法识别的响应")
|
||||
|
||||
@staticmethod
|
||||
def _stop_local_process(process: subprocess.Popen[str]) -> None:
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
if process.stdin is not None:
|
||||
process.stdin.write('{"command":"shutdown"}\n')
|
||||
process.stdin.flush()
|
||||
process.wait(timeout=2)
|
||||
except Exception:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
except Exception:
|
||||
process.kill()
|
||||
|
||||
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
|
||||
url = self.config.base_url.rstrip("/") + "/embeddings"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if self.config.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.config.api_key}"
|
||||
# ``dimensions`` is a local invariant, not a portable request option.
|
||||
# OpenAI-compatible BGE endpoints commonly reject that extra field
|
||||
# with HTTP 400 even though they always return a fixed-size vector.
|
||||
# Keep validating the response against the configured value below.
|
||||
payload: dict = {"model": self.config.model, "input": texts}
|
||||
last_error: WikiEmbeddingError | None = None
|
||||
for attempt in range(self.config.max_retries + 1):
|
||||
started = monotonic()
|
||||
try:
|
||||
response = await (await self._get_client()).post(url, headers=headers, json=payload)
|
||||
if response.status_code >= 400:
|
||||
detail = _upstream_error_detail(response)
|
||||
category = "authentication" if response.status_code in {401, 403} else "upstream"
|
||||
retryable = response.status_code == 429 or response.status_code >= 500
|
||||
raise WikiEmbeddingError(
|
||||
f"Embedding service returned HTTP {response.status_code}: {detail}",
|
||||
category=category,
|
||||
retryable=retryable,
|
||||
)
|
||||
data = response.json()
|
||||
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
|
||||
if len(items) != len(texts):
|
||||
raise WikiEmbeddingError("Embedding response count mismatch", category="response", retryable=False)
|
||||
result: list[list[float]] = []
|
||||
for item in items:
|
||||
vector = normalize_vector(item.get("embedding") or [], dimensions=self._resolved_dimensions)
|
||||
if self._resolved_dimensions is None:
|
||||
self._resolved_dimensions = int(vector.size)
|
||||
result.append(vector.astype(float).tolist())
|
||||
logger.info("Wiki embedding batch completed model=%s batch=%d duration_ms=%d", self.config.model, len(texts), int((monotonic() - started) * 1000))
|
||||
return result
|
||||
except WikiEmbeddingError as exc:
|
||||
last_error = exc
|
||||
except httpx.TimeoutException:
|
||||
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
|
||||
except (httpx.HTTPError, ValueError, TypeError) as exc:
|
||||
reason = " ".join(str(exc).split())[:2000] or type(exc).__name__
|
||||
last_error = WikiEmbeddingError(
|
||||
f"Embedding request failed ({type(exc).__name__}): {reason}",
|
||||
category="request",
|
||||
retryable=isinstance(exc, httpx.HTTPError),
|
||||
)
|
||||
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
|
||||
logger.error(
|
||||
"Wiki embedding batch failed model=%s batch=%d category=%s reason=%s",
|
||||
self.config.model,
|
||||
len(texts),
|
||||
last_error.category,
|
||||
last_error,
|
||||
)
|
||||
raise last_error
|
||||
await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
|
||||
raise last_error or WikiEmbeddingError("Embedding request failed")
|
||||
|
||||
async def embed_query(self, text: str) -> list[float]:
|
||||
result = await self.embed_texts([text])
|
||||
return result[0]
|
||||
|
||||
def local_encoder_status(self) -> dict[str, object]:
|
||||
running = self._local_warmup_task is not None and not self._local_warmup_task.done()
|
||||
process_ready = self._local_process is not None and self._local_process.poll() is None
|
||||
process_failed = self._local_process is not None and not process_ready
|
||||
state = (
|
||||
"starting"
|
||||
if running
|
||||
else "ready"
|
||||
if (self._local_model is not None or process_ready)
|
||||
else "failed"
|
||||
if process_failed
|
||||
else self._local_state
|
||||
)
|
||||
last_error = self._local_error
|
||||
if process_failed and not last_error:
|
||||
last_error = f"本地编码器子进程已退出(exit={self._local_process.poll()})"
|
||||
model_path = self._local_model_path()
|
||||
model_files_ready = bool(
|
||||
model_path
|
||||
and model_path.is_dir()
|
||||
and ((model_path / "onnx" / "model.onnx").is_file() or (model_path / "model.onnx").is_file())
|
||||
)
|
||||
return {
|
||||
"provider": self.config.provider,
|
||||
"model": self.config.model,
|
||||
"dimensions": self.dimensions,
|
||||
"state": state,
|
||||
"running": running,
|
||||
"last_error": last_error,
|
||||
"started_at": self._local_started_at,
|
||||
"ready_at": self._local_ready_at,
|
||||
"offline_only": self.config.provider == "local",
|
||||
"model_path_configured": model_path is not None,
|
||||
"model_path_exists": bool(model_path and model_path.is_dir()),
|
||||
"model_files_ready": model_files_ready,
|
||||
"process_isolation": self.config.local_process_isolation,
|
||||
"model_path": str(model_path) if model_path is not None else "",
|
||||
}
|
||||
|
||||
def _local_model_path(self) -> Path | None:
|
||||
raw = self.config.local_model_path.strip() or os.environ.get(
|
||||
"DEERFLOW_BGE_M3_MODEL_PATH", ""
|
||||
).strip()
|
||||
if not raw:
|
||||
return None
|
||||
path = Path(raw).expanduser()
|
||||
return path.resolve() if path.is_absolute() else (Path.cwd() / path).resolve()
|
||||
|
||||
def start_local_encoder(self) -> dict[str, object]:
|
||||
"""Warm the embedded CPU encoder without coupling it to app startup."""
|
||||
if self.config.provider != "local":
|
||||
raise ValueError("当前 embedding provider 不是 local")
|
||||
if not self.config.enabled:
|
||||
raise ValueError("本地编码器未启用")
|
||||
process_ready = self._local_process is not None and self._local_process.poll() is None
|
||||
if self._local_model is not None or process_ready:
|
||||
self._local_state = "ready"
|
||||
return self.local_encoder_status()
|
||||
if self._local_warmup_task is None or self._local_warmup_task.done():
|
||||
self._local_state = "starting"
|
||||
self._local_error = None
|
||||
self._local_started_at = datetime.now(UTC).isoformat()
|
||||
self._local_warmup_task = asyncio.create_task(
|
||||
self._warm_local_encoder(),
|
||||
name="llmwiki-local-encoder-warmup",
|
||||
)
|
||||
return self.local_encoder_status()
|
||||
|
||||
async def _warm_local_encoder(self) -> None:
|
||||
try:
|
||||
await self.embed_query("本地编码器启动检查")
|
||||
except asyncio.CancelledError:
|
||||
self._local_state = "idle"
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Local Wiki encoder warmup failed model=%s", self.config.model)
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
"""Offline FastEmbed worker isolated from the DeerFlow gateway process.
|
||||
|
||||
The protocol is UTF-8 JSON Lines over stdin/stdout. Model weights must already
|
||||
exist on disk; every offline flag is enforced before FastEmbed is imported.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _write(payload: dict[str, object]) -> None:
|
||||
sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _build_model(args: argparse.Namespace):
|
||||
os.environ["HF_HUB_OFFLINE"] = "1"
|
||||
os.environ["TRANSFORMERS_OFFLINE"] = "1"
|
||||
os.environ["HF_DATASETS_OFFLINE"] = "1"
|
||||
from fastembed import TextEmbedding
|
||||
|
||||
if args.model_name == "BAAI/bge-m3" and not any(
|
||||
item["model"] == args.model_name for item in TextEmbedding.list_supported_models()
|
||||
):
|
||||
from fastembed.common.model_description import ModelSource, PoolingType
|
||||
|
||||
TextEmbedding.add_custom_model(
|
||||
model=args.model_name,
|
||||
pooling=PoolingType.CLS,
|
||||
normalization=True,
|
||||
dim=1024,
|
||||
sources=ModelSource(hf="BAAI/bge-m3"),
|
||||
model_file="onnx/model.onnx",
|
||||
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
|
||||
license="mit",
|
||||
size_in_gb=2.29,
|
||||
)
|
||||
return TextEmbedding(
|
||||
model_name=args.model_name,
|
||||
threads=args.threads,
|
||||
cache_dir=args.cache_dir,
|
||||
providers=["CPUExecutionProvider"],
|
||||
specific_model_path=args.model_path,
|
||||
local_files_only=True,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model-name", required=True)
|
||||
parser.add_argument("--model-path", required=True)
|
||||
parser.add_argument("--cache-dir", required=True)
|
||||
parser.add_argument("--threads", type=int, default=2)
|
||||
parser.add_argument("--batch-size", type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
model_path = Path(args.model_path)
|
||||
if not model_path.is_dir():
|
||||
_write({"status": "error", "error": f"离线模型目录不存在:{model_path}"})
|
||||
return 2
|
||||
try:
|
||||
model = _build_model(args)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_write({"status": "error", "error": f"离线模型加载失败:{type(exc).__name__}: {exc}"})
|
||||
return 2
|
||||
_write({"status": "ready"})
|
||||
for line in sys.stdin:
|
||||
request: dict[str, object] = {}
|
||||
try:
|
||||
request = json.loads(line)
|
||||
if request.get("command") == "shutdown":
|
||||
return 0
|
||||
request_id = str(request.get("id") or "")
|
||||
texts = request.get("texts")
|
||||
if not request_id or not isinstance(texts, list) or not all(isinstance(value, str) for value in texts):
|
||||
raise ValueError("编码请求格式无效")
|
||||
vectors = [
|
||||
value.tolist()
|
||||
for value in model.embed(texts, batch_size=max(1, min(args.batch_size, 4)))
|
||||
]
|
||||
_write({"id": request_id, "status": "ok", "vectors": vectors})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_write(
|
||||
{
|
||||
"id": str(request.get("id") or "") if isinstance(request, dict) else "",
|
||||
"status": "error",
|
||||
"error": f"本地编码失败:{type(exc).__name__}: {exc}",
|
||||
}
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
"""Normalize unstable WeKnora Wiki response shapes into the local contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
|
||||
def normalize_slug(value: Any) -> str:
|
||||
slug = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
|
||||
slug = re.sub(r"/{2,}", "/", slug)
|
||||
if not slug:
|
||||
raise ValueError("Wiki page slug is empty")
|
||||
return slug
|
||||
|
||||
|
||||
def slug_hash(slug: str) -> str:
|
||||
return hashlib.sha256(normalize_slug(slug).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _list(value: Any) -> list:
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if isinstance(value, tuple):
|
||||
return list(value)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return [value.strip()]
|
||||
return []
|
||||
|
||||
|
||||
def _datetime(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def normalize_wiki_page(mapping_id: str, remote: dict[str, Any]) -> dict[str, Any]:
|
||||
slug = normalize_slug(remote.get("slug") or remote.get("wiki_slug") or remote.get("path"))
|
||||
aliases = sorted({str(item).strip() for item in _list(remote.get("aliases")) if str(item).strip()})
|
||||
category_path = [str(item).strip() for item in _list(remote.get("category_path") or remote.get("categories")) if str(item).strip()]
|
||||
title = str(remote.get("title") or slug.rsplit("/", 1)[-1]).strip()
|
||||
summary = str(remote.get("summary") or remote.get("description") or "").replace("\r\n", "\n").strip()
|
||||
content = str(remote.get("content_md") or remote.get("content") or remote.get("markdown") or "").replace("\r\n", "\n").strip()
|
||||
status = str(remote.get("status") or "published").strip().lower()
|
||||
if status not in {"draft", "published", "archived"}:
|
||||
status = "published"
|
||||
hash_payload = {
|
||||
"title": title,
|
||||
"aliases": aliases,
|
||||
"summary": summary,
|
||||
"content_md": content,
|
||||
"page_type": str(remote.get("page_type") or remote.get("type") or "article").strip().lower(),
|
||||
"status": status,
|
||||
}
|
||||
content_hash = hashlib.sha256(json.dumps(hash_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
|
||||
return {
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"remote_page_id": str(remote.get("id") or remote.get("page_id") or "").strip() or None,
|
||||
"slug": slug,
|
||||
"slug_hash": slug_hash(slug),
|
||||
"title": title,
|
||||
"summary": summary,
|
||||
"content_md": content,
|
||||
"page_type": hash_payload["page_type"],
|
||||
"status": status,
|
||||
"aliases": aliases,
|
||||
"category_path": category_path,
|
||||
"parent_slug": str(remote.get("parent_slug") or "").strip() or None,
|
||||
"wiki_path": str(remote.get("wiki_path") or remote.get("path") or "").strip() or None,
|
||||
"source_refs": _list(remote.get("source_refs") or remote.get("sources")),
|
||||
"chunk_refs": _list(remote.get("chunk_refs") or remote.get("chunks")),
|
||||
"page_version": int(remote.get("page_version") or remote.get("version") or 0),
|
||||
"content_hash": content_hash,
|
||||
"remote_created_at": _datetime(remote.get("created_at") or remote.get("createdAt")),
|
||||
"remote_updated_at": _datetime(remote.get("updated_at") or remote.get("updatedAt")),
|
||||
}
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
"""Wiki-only retrieval with local-vector preference and remote Wiki fallback.
|
||||
|
||||
The fallback deliberately calls WeKnora's ``/wiki/pages`` API. It never calls
|
||||
the raw knowledge/chunk search endpoint, so callers always receive processed
|
||||
Wiki pages even while a whole knowledge base is awaiting vectorization.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
|
||||
def is_fully_vectorized(state: dict[str, Any] | None, fingerprint: str | None) -> bool:
|
||||
"""Return whether a completed full scan produced one coherent active index."""
|
||||
|
||||
if not state or not fingerprint:
|
||||
return False
|
||||
if str(state.get("state") or "") != "idle" or not state.get("last_success_at"):
|
||||
return False
|
||||
if str(state.get("embedding_fingerprint") or "") != fingerprint:
|
||||
return False
|
||||
local_count = int(state.get("local_page_count") or 0)
|
||||
ready_count = int(state.get("ready_page_count") or 0)
|
||||
failed_count = int(state.get("failed_page_count") or 0)
|
||||
vector_count = int(state.get("vector_count") or 0)
|
||||
if failed_count or ready_count != local_count:
|
||||
return False
|
||||
return local_count == 0 or vector_count > 0
|
||||
|
||||
|
||||
class WikiRetrievalService:
|
||||
"""Search complete indexes locally and incomplete indexes through Wiki API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: LlmWikiIndexStore,
|
||||
*,
|
||||
vector_search: WikiVectorSearchService | None,
|
||||
client: WeKnoraClient | None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.vector_search = vector_search
|
||||
self.client = client
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
top_k_pages: int = 8,
|
||||
include_drafts: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
|
||||
if not mappings:
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": "wiki_api",
|
||||
"results": [],
|
||||
"knowledge_base_count": 0,
|
||||
"partial": False,
|
||||
"warnings": [],
|
||||
"fallback_knowledge_base_ids": [],
|
||||
}
|
||||
|
||||
vector_mappings, fallback_mappings = await self._partition_mappings(mappings)
|
||||
results: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
vector_used = False
|
||||
fallback_used = False
|
||||
|
||||
if vector_mappings and self.vector_search is not None:
|
||||
try:
|
||||
payload = await self.vector_search.search(
|
||||
normalized_query,
|
||||
vector_mappings,
|
||||
top_k_pages=max(top_k_pages, len(vector_mappings)),
|
||||
include_drafts=include_drafts,
|
||||
)
|
||||
results.extend(payload.get("results") or [])
|
||||
warnings.extend(str(item) for item in payload.get("warnings") or [])
|
||||
vector_used = True
|
||||
except WikiIndexError as exc:
|
||||
# A status row can become stale between classification and
|
||||
# matrix loading. Wiki API remains a safe Wiki-only fallback.
|
||||
warnings.append(f"local_vector:{exc.code}")
|
||||
fallback_mappings.extend(vector_mappings)
|
||||
vector_mappings = []
|
||||
|
||||
fallback_ids = [str(mapping["id"]) for mapping in fallback_mappings]
|
||||
if fallback_mappings:
|
||||
if self.client is None:
|
||||
warnings.extend(f"{mapping['id']}:WIKI_API_UNAVAILABLE" for mapping in fallback_mappings)
|
||||
else:
|
||||
batches = await asyncio.gather(
|
||||
*(
|
||||
self._search_remote_mapping(
|
||||
normalized_query,
|
||||
mapping,
|
||||
limit=max(top_k_pages, 1),
|
||||
include_drafts=include_drafts,
|
||||
)
|
||||
for mapping in fallback_mappings
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
for mapping, batch in zip(fallback_mappings, batches, strict=True):
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping['id']}:WIKI_API_SEARCH_FAILED")
|
||||
else:
|
||||
results.extend(batch)
|
||||
fallback_used = True
|
||||
|
||||
if warnings and not results and not vector_used and not fallback_used:
|
||||
raise WikiIndexError(
|
||||
"WIKI_API_UNAVAILABLE",
|
||||
"Wiki page retrieval is unavailable",
|
||||
status_code=502 if self.client is not None else 503,
|
||||
)
|
||||
|
||||
# De-duplicate by the public Wiki citation identity. Vector scores and
|
||||
# Wiki-API rank scores are both descending relevance hints.
|
||||
unique: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for item in results:
|
||||
key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
|
||||
current = unique.get(key)
|
||||
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
|
||||
unique[key] = item
|
||||
ranked = sorted(unique.values(), key=lambda item: float(item.get("score") or 0), reverse=True)
|
||||
mode = "mixed" if vector_used and fallback_used else "vector" if vector_used else "wiki_api"
|
||||
fingerprint = self.vector_search.embedding.fingerprint[:12] if vector_used and self.vector_search is not None else None
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": mode,
|
||||
"results": ranked[:top_k_pages],
|
||||
"knowledge_base_count": len(mappings),
|
||||
"index_fingerprint": fingerprint,
|
||||
"partial": bool(warnings),
|
||||
"warnings": warnings,
|
||||
"fallback_knowledge_base_ids": fallback_ids,
|
||||
}
|
||||
|
||||
async def _partition_mappings(self, mappings: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
if self.vector_search is None:
|
||||
return [], list(mappings)
|
||||
mapping_ids = [str(mapping["id"]) for mapping in mappings]
|
||||
states = {
|
||||
str(state["knowledge_base_mapping_id"]): state
|
||||
for state in await self.store.list_index_status(mapping_ids)
|
||||
}
|
||||
fingerprint = self.vector_search.embedding.fingerprint
|
||||
vector_mappings = [
|
||||
mapping
|
||||
for mapping in mappings
|
||||
if mapping.get("wiki_index_enabled", True)
|
||||
and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)
|
||||
]
|
||||
ready_ids = {str(mapping["id"]) for mapping in vector_mappings}
|
||||
return vector_mappings, [mapping for mapping in mappings if str(mapping["id"]) not in ready_ids]
|
||||
|
||||
async def _search_remote_mapping(
|
||||
self,
|
||||
query: str,
|
||||
mapping: dict[str, Any],
|
||||
*,
|
||||
limit: int,
|
||||
include_drafts: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
if self.client is None:
|
||||
return []
|
||||
remote_id = str(mapping["weknora_id"])
|
||||
listing = await self.client.list_wiki_pages(
|
||||
remote_id,
|
||||
page=1,
|
||||
page_size=min(max(limit * 3, 20), 100),
|
||||
query=query,
|
||||
)
|
||||
summaries = [item for item in listing.get("pages", []) if isinstance(item, dict)]
|
||||
slugs = list(
|
||||
dict.fromkeys(
|
||||
str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
for item in summaries
|
||||
if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
)
|
||||
)[: max(limit * 2, limit)]
|
||||
details = await asyncio.gather(
|
||||
*(self.client.get_wiki_page(remote_id, slug) for slug in slugs),
|
||||
return_exceptions=True,
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
for rank, (slug, detail) in enumerate(zip(slugs, details, strict=True)):
|
||||
if isinstance(detail, Exception):
|
||||
continue
|
||||
try:
|
||||
page = normalize_wiki_page(str(mapping["id"]), detail)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if page["status"] == "archived" or (page["status"] == "draft" and not include_drafts):
|
||||
continue
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": slug})
|
||||
content = str(page.get("content_md") or "")
|
||||
score = float(detail.get("score") or max(0.01, 1.0 - rank * 0.05))
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"wiki_page_id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
|
||||
"wiki_slug": slug,
|
||||
"title": page.get("title") or slug,
|
||||
"summary": page.get("summary") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"updated_at": page.get("remote_updated_at"),
|
||||
"score": score,
|
||||
"content": content,
|
||||
"snippet": content[:360],
|
||||
"matched_heading": None,
|
||||
"matched_sections": [
|
||||
{
|
||||
"section_index": 0,
|
||||
"heading": None,
|
||||
"content": content,
|
||||
"score": score,
|
||||
}
|
||||
],
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"type": "llmwiki_wiki",
|
||||
"mode": "citation",
|
||||
"retrieval_mode": "wiki_api",
|
||||
}
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
__all__ = ["WikiRetrievalService", "is_fully_vectorized"]
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""App-to-harness runtime injection for local Wiki index services."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
|
||||
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
_store: LlmWikiIndexStore | None = None
|
||||
_search: WikiVectorSearchService | None = None
|
||||
_sync: WikiSyncService | None = None
|
||||
_retrieval: WikiRetrievalService | None = None
|
||||
|
||||
|
||||
def register_local_wiki_index_runtime(
|
||||
*,
|
||||
store: LlmWikiIndexStore | None,
|
||||
search: WikiVectorSearchService | None,
|
||||
sync: WikiSyncService | None,
|
||||
retrieval: WikiRetrievalService | None = None,
|
||||
) -> None:
|
||||
global _store, _search, _sync, _retrieval
|
||||
_store, _search, _sync, _retrieval = store, search, sync, retrieval
|
||||
|
||||
|
||||
def get_local_wiki_index_store() -> LlmWikiIndexStore:
|
||||
if _store is None:
|
||||
raise RuntimeError("Local Wiki index store is not initialized")
|
||||
return _store
|
||||
|
||||
|
||||
def get_local_wiki_search_service() -> WikiVectorSearchService:
|
||||
if _search is None:
|
||||
raise RuntimeError("Local Wiki vector search is not initialized")
|
||||
return _search
|
||||
|
||||
|
||||
def get_local_wiki_sync_service() -> WikiSyncService:
|
||||
if _sync is None:
|
||||
raise RuntimeError("Local Wiki sync service is not initialized")
|
||||
return _sync
|
||||
|
||||
|
||||
def get_local_wiki_retrieval_service() -> WikiRetrievalService:
|
||||
if _retrieval is None:
|
||||
raise RuntimeError("Wiki-only retrieval service is not initialized")
|
||||
return _retrieval
|
||||
+316
@@ -0,0 +1,316 @@
|
||||
"""Unified exact-vector search over DeerFlow-local Wiki section snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import re
|
||||
from collections import OrderedDict, defaultdict
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import numpy as np
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache, WikiVectorSnapshot
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
|
||||
|
||||
|
||||
class WikiIndexError(RuntimeError):
|
||||
def __init__(self, code: str, message: str, *, status_code: int) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class WikiVectorSearchService:
|
||||
def __init__(self, config: LocalWikiIndexConfig, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache) -> None:
|
||||
self.config = config
|
||||
self.embedding = embedding
|
||||
self.cache = cache
|
||||
self._query_vectors: OrderedDict[str, tuple[float, np.ndarray]] = OrderedDict()
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
top_k_pages: int | None = None,
|
||||
include_drafts: bool = False,
|
||||
external: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
|
||||
if not mappings:
|
||||
return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
|
||||
fingerprint, candidates, warnings = await self._collect_candidates(
|
||||
normalized_query,
|
||||
mappings,
|
||||
include_drafts=include_drafts,
|
||||
external=external,
|
||||
)
|
||||
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
||||
for candidate in candidates:
|
||||
page = candidate["page"]
|
||||
key = (str(page["knowledge_base_mapping_id"]), str(page["id"]))
|
||||
if len(grouped[key]) < self.config.max_sections_per_page:
|
||||
grouped[key].append(candidate)
|
||||
results: list[dict[str, Any]] = []
|
||||
for sections in grouped.values():
|
||||
first = sections[0]
|
||||
page = first["page"]
|
||||
mapping = first["mapping"]
|
||||
second_score = sections[1]["score"] if len(sections) > 1 else 0.0
|
||||
page_score = first["score"] + (0.05 * second_score if second_score else 0)
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": page["slug"]})
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"id": page["id"],
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"wiki_page_id": page["id"],
|
||||
"wiki_slug": page["slug"],
|
||||
"title": page.get("title") or page["slug"],
|
||||
"summary": page.get("summary") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
|
||||
"score": float(page_score),
|
||||
"content": "\n\n".join(str(item["section_content"]) for item in sections),
|
||||
"snippet": str(sections[0]["section_content"])[:360],
|
||||
"matched_heading": sections[0].get("heading"),
|
||||
"matched_sections": [{"section_index": int(item["section_index"]), "heading": item.get("heading"), "content": item["section_content"], "score": float(item["score"])} for item in sections],
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"type": "llmwiki_wiki",
|
||||
"mode": "citation",
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
|
||||
return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
|
||||
|
||||
async def search_sections(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
top_k_sections: int = 10,
|
||||
external: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Return one result per matching Wiki vector section.
|
||||
|
||||
The regular ``search`` method intentionally groups several sections
|
||||
under one Wiki page for citations. Public consumers of the standalone
|
||||
vector API need the opposite contract: every matched vector is a
|
||||
separate ranked item, while the complete page and knowledge-base
|
||||
metadata remains attached to that item.
|
||||
"""
|
||||
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
|
||||
limit = max(1, min(int(top_k_sections), 100))
|
||||
if not mappings:
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": "vector",
|
||||
"results": [],
|
||||
"knowledge_base_count": 0,
|
||||
"index_fingerprint": self.embedding.fingerprint[:12],
|
||||
"partial": False,
|
||||
"warnings": [],
|
||||
}
|
||||
fingerprint, candidates, warnings = await self._collect_candidates(
|
||||
normalized_query,
|
||||
mappings,
|
||||
include_drafts=False,
|
||||
external=external,
|
||||
section_limit=limit,
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
for candidate in candidates[:limit]:
|
||||
page = candidate["page"]
|
||||
mapping = candidate["mapping"]
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_section",
|
||||
"id": candidate.get("id") or f"{page['id']}:{candidate['section_index']}",
|
||||
"section_id": candidate.get("id") or f"{page['id']}:{candidate['section_index']}",
|
||||
"section_index": int(candidate["section_index"]),
|
||||
"heading": candidate.get("heading"),
|
||||
"content": str(candidate.get("section_content") or ""),
|
||||
"section_content_hash": candidate.get("content_hash") or "",
|
||||
"score": float(candidate["score"]),
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"weknora_knowledge_base_id": mapping.get("weknora_id") or "",
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"knowledge_base_description": mapping.get("description") or "",
|
||||
"knowledge_base_type": mapping.get("kb_type") or "document",
|
||||
"publication_status": mapping.get("publication_status") or "",
|
||||
"published_at": mapping.get("published_at"),
|
||||
"wiki_page_id": page["id"],
|
||||
"weknora_wiki_page_id": page.get("remote_page_id"),
|
||||
"wiki_slug": page["slug"],
|
||||
"title": page.get("title") or page["slug"],
|
||||
"summary": page.get("summary") or "",
|
||||
"page_content": page.get("content_md") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_status": page.get("status") or "published",
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"aliases": page.get("aliases") or [],
|
||||
"category_path": page.get("category_path") or [],
|
||||
"parent_slug": page.get("parent_slug"),
|
||||
"wiki_path": page.get("wiki_path"),
|
||||
"source_refs": page.get("source_refs") or [],
|
||||
"chunk_refs": page.get("chunk_refs") or [],
|
||||
"page_content_hash": page.get("content_hash") or "",
|
||||
"remote_created_at": page.get("remote_created_at"),
|
||||
"remote_updated_at": page.get("remote_updated_at"),
|
||||
"indexed_at": page.get("indexed_at"),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": "vector",
|
||||
"results": results,
|
||||
"knowledge_base_count": len(mappings),
|
||||
"index_fingerprint": fingerprint[:12],
|
||||
"partial": bool(warnings),
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
async def _collect_candidates(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
include_drafts: bool,
|
||||
external: bool,
|
||||
section_limit: int | None = None,
|
||||
) -> tuple[str, list[dict[str, Any]], list[str]]:
|
||||
try:
|
||||
query_vector = await self._embed_query(query)
|
||||
except (WikiEmbeddingError, ValueError) as exc:
|
||||
raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
|
||||
fingerprint = self.embedding.fingerprint
|
||||
tasks = [
|
||||
self._search_mapping(
|
||||
query,
|
||||
query_vector,
|
||||
mapping,
|
||||
fingerprint,
|
||||
include_drafts=include_drafts,
|
||||
external=external,
|
||||
section_limit=section_limit,
|
||||
)
|
||||
for mapping in mappings
|
||||
]
|
||||
batches = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
candidates: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
for mapping, batch in zip(mappings, batches, strict=True):
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping.get('id')}: {batch}")
|
||||
else:
|
||||
candidates.extend(batch)
|
||||
if warnings and not candidates and len(warnings) == len(mappings):
|
||||
if any("FINGERPRINT" in item for item in warnings):
|
||||
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
|
||||
elif any("rebuild" in item.lower() for item in warnings):
|
||||
code = "WIKI_INDEX_REBUILDING"
|
||||
else:
|
||||
code = "WIKI_INDEX_NOT_READY"
|
||||
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
|
||||
candidates.sort(key=lambda item: item["score"], reverse=True)
|
||||
return fingerprint, candidates, warnings
|
||||
|
||||
async def _embed_query(self, query: str) -> np.ndarray:
|
||||
digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
|
||||
key = f"{self.embedding.fingerprint}:{digest}"
|
||||
cached = self._query_vectors.get(key)
|
||||
if cached is not None and monotonic() - cached[0] <= 300:
|
||||
self._query_vectors.move_to_end(key)
|
||||
return cached[1]
|
||||
vector = normalize_vector(await self.embedding.embed_query(query), dimensions=self.embedding.dimensions)
|
||||
key = f"{self.embedding.fingerprint}:{digest}"
|
||||
self._query_vectors[key] = (monotonic(), vector)
|
||||
self._query_vectors.move_to_end(key)
|
||||
while len(self._query_vectors) > 256:
|
||||
self._query_vectors.popitem(last=False)
|
||||
return vector
|
||||
|
||||
async def _search_mapping(
|
||||
self,
|
||||
query: str,
|
||||
vector: np.ndarray,
|
||||
mapping: dict[str, Any],
|
||||
fingerprint: str,
|
||||
*,
|
||||
include_drafts: bool,
|
||||
external: bool,
|
||||
section_limit: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
mapping_id = str(mapping["id"])
|
||||
try:
|
||||
snapshot = await self.cache.get(mapping_id, fingerprint)
|
||||
except ValueError as exc:
|
||||
await self.cache.store.mark_rebuild_required([mapping_id], fingerprint)
|
||||
self.cache.invalidate(mapping_id)
|
||||
raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
|
||||
if snapshot.matrix.size == 0:
|
||||
return []
|
||||
if snapshot.dimensions != vector.size:
|
||||
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
|
||||
eligible = np.asarray(
|
||||
[
|
||||
index
|
||||
for index, row in enumerate(snapshot.rows)
|
||||
if row["page"].get("status") != "draft" or (include_drafts and not external)
|
||||
],
|
||||
dtype=int,
|
||||
)
|
||||
if eligible.size == 0:
|
||||
return []
|
||||
indices, scores = await asyncio.to_thread(
|
||||
self._top_indices,
|
||||
snapshot,
|
||||
vector,
|
||||
eligible,
|
||||
section_limit,
|
||||
)
|
||||
terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
|
||||
candidates = []
|
||||
for index, score in zip(indices, scores, strict=True):
|
||||
row = snapshot.rows[int(index)]
|
||||
page = row["page"]
|
||||
vector_score = float(score)
|
||||
if vector_score < self.config.min_similarity:
|
||||
continue
|
||||
final_score = vector_score
|
||||
if self.config.keyword_boost_enabled and terms:
|
||||
haystack = f"{page.get('title', '')} {row.get('heading', '')} {row.get('section_content', '')}".lower()
|
||||
keyword_score = sum(term in haystack for term in terms) / len(terms)
|
||||
weight = self.config.keyword_boost_weight
|
||||
final_score = (1 - weight) * vector_score + weight * keyword_score
|
||||
candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
|
||||
return candidates
|
||||
|
||||
def _top_indices(
|
||||
self,
|
||||
snapshot: WikiVectorSnapshot,
|
||||
vector: np.ndarray,
|
||||
eligible: np.ndarray,
|
||||
section_limit: int | None = None,
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
scores = snapshot.matrix[eligible] @ vector
|
||||
count = min(section_limit or self.config.top_k_sections, scores.size)
|
||||
if count <= 0:
|
||||
return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
|
||||
positions = np.argpartition(scores, -count)[-count:]
|
||||
positions = positions[np.argsort(scores[positions])[::-1]]
|
||||
return eligible[positions], scores[positions]
|
||||
+188
@@ -0,0 +1,188 @@
|
||||
"""Lease-protected full Wiki mirror scan and page-level vector replacement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import platform
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WikiSyncLeaseLost(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _safe_error(exc: BaseException) -> str:
|
||||
return f"{type(exc).__name__}: {str(exc)[:1000]}"
|
||||
|
||||
|
||||
class WikiSyncService:
|
||||
def __init__(self, config: LocalWikiIndexConfig, store: LlmWikiIndexStore, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache, client: WeKnoraClient) -> None:
|
||||
self.config = config
|
||||
self.store = store
|
||||
self.embedding = embedding
|
||||
self.cache = cache
|
||||
self.client = client
|
||||
self.owner = f"{platform.node()}:{uuid4()}"
|
||||
self.on_completed = None
|
||||
|
||||
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False, notify: bool = True) -> dict[str, Any]:
|
||||
mapping_id = str(mapping["id"])
|
||||
sync_id = str(uuid4())
|
||||
if self.embedding.dimensions is None:
|
||||
# Auto-dimension configurations need one successful response to
|
||||
# produce a stable fingerprint before lease/rebuild decisions.
|
||||
await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
|
||||
fingerprint = self.embedding.fingerprint
|
||||
revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
|
||||
if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
|
||||
await self.store.mark_rebuild_required([mapping_id], fingerprint)
|
||||
self.cache.invalidate(mapping_id)
|
||||
prior_state = "rebuild_required"
|
||||
rebuilding = prior_state == "rebuild_required"
|
||||
acquired = await self.store.try_acquire_sync_lease(mapping_id, owner=self.owner, sync_id=sync_id, lease_seconds=self.config.sync_lease_seconds, fingerprint=fingerprint)
|
||||
if not acquired:
|
||||
return {"mapping_id": mapping_id, "status": "already_running"}
|
||||
lease_lost = asyncio.Event()
|
||||
|
||||
async def _lease_heartbeat() -> None:
|
||||
interval = max(5.0, min(60.0, self.config.sync_lease_seconds / 3))
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
renewed = await self.store.renew_sync_lease(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
lease_seconds=self.config.sync_lease_seconds,
|
||||
)
|
||||
except Exception:
|
||||
renewed = False
|
||||
logger.exception("Local Wiki sync lease heartbeat failed mapping_id=%s", mapping_id)
|
||||
if not renewed:
|
||||
lease_lost.set()
|
||||
return
|
||||
|
||||
heartbeat = asyncio.create_task(_lease_heartbeat())
|
||||
|
||||
def _ensure_lease() -> None:
|
||||
if lease_lost.is_set():
|
||||
raise WikiSyncLeaseLost("Local Wiki sync lease was lost")
|
||||
|
||||
remote_count = 0
|
||||
changed = 0
|
||||
failed = 0
|
||||
try:
|
||||
remote_id = str(mapping["weknora_id"])
|
||||
await self.client.get_knowledge_base(remote_id)
|
||||
page_number = 1
|
||||
while True:
|
||||
batch = await self.client.list_wiki_pages(remote_id, page=page_number, page_size=self.config.sync_page_size)
|
||||
pages = [page for page in batch.get("pages", []) if isinstance(page, dict)]
|
||||
remote_count += len(pages)
|
||||
reported_total = int(batch.get("total") or batch.get("total_count") or remote_count)
|
||||
await self.store.update_sync_progress(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
remote_page_count=max(remote_count, reported_total),
|
||||
)
|
||||
for summary in pages:
|
||||
try:
|
||||
_ensure_lease()
|
||||
slug = str(summary.get("slug") or summary.get("wiki_slug") or summary.get("path") or "")
|
||||
remote_page = await self.client.get_wiki_page(remote_id, slug)
|
||||
page = normalize_wiki_page(mapping_id, {**summary, **remote_page})
|
||||
existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
|
||||
active_fingerprint = self.embedding.fingerprint
|
||||
if (
|
||||
not force
|
||||
and existing
|
||||
and existing.get("content_hash") == page["content_hash"]
|
||||
and existing.get("indexed_fingerprint") == active_fingerprint
|
||||
and existing.get("index_status") == "ready"
|
||||
and (existing.get("vector_count", 1) > 0 or page["status"] == "archived" or not (page["content_md"] or page["summary"]))
|
||||
and not existing.get("is_remote_deleted")
|
||||
):
|
||||
await self.store.touch_page(str(existing["id"]), sync_id)
|
||||
continue
|
||||
sections = (
|
||||
[]
|
||||
if page["status"] == "archived"
|
||||
else chunk_wiki_page(page, max_chars=self.config.chunk_max_chars, overlap=self.config.chunk_overlap_chars, min_chars=self.config.chunk_min_chars, title_prefix_enabled=self.config.title_prefix_enabled)
|
||||
)
|
||||
embeddings = await self.embedding.embed_texts([section.embedding_text for section in sections]) if sections else []
|
||||
fingerprint = self.embedding.fingerprint
|
||||
vectors = []
|
||||
for section, vector in zip(sections, embeddings, strict=True):
|
||||
blob, dimensions = encode_vector(vector, dimensions=self.embedding.dimensions)
|
||||
vectors.append(
|
||||
{"section_index": section.section_index, "heading": section.heading, "section_content": section.content, "content_hash": section.content_hash, "embedding_dimensions": dimensions, "vector_blob": blob}
|
||||
)
|
||||
_ensure_lease()
|
||||
await self.store.replace_page_vectors(page=page, vectors=vectors, fingerprint=fingerprint, sync_id=sync_id)
|
||||
self.cache.invalidate(mapping_id)
|
||||
changed += 1
|
||||
except Exception as exc:
|
||||
if isinstance(exc, WikiSyncLeaseLost):
|
||||
raise
|
||||
failed += 1
|
||||
try:
|
||||
fallback = normalize_wiki_page(mapping_id, summary)
|
||||
await self.store.record_page_failure(page=fallback, sync_id=sync_id, error=_safe_error(exc))
|
||||
except Exception as record_exc:
|
||||
logger.exception("Could not record local Wiki page failure mapping_id=%s", mapping_id)
|
||||
raise RuntimeError("Could not safely retain a failed Wiki page during the full scan") from record_exc
|
||||
logger.warning("Local Wiki page sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
|
||||
_ensure_lease()
|
||||
total_pages = int(batch.get("total_pages") or 1)
|
||||
if page_number >= total_pages:
|
||||
break
|
||||
page_number += 1
|
||||
_ensure_lease()
|
||||
deleted = await self.store.mark_missing_pages_deleted(mapping_id, sync_id)
|
||||
if deleted:
|
||||
self.cache.invalidate(mapping_id)
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=not (rebuilding and failed),
|
||||
remote_page_count=remote_count,
|
||||
error=f"{failed} page(s) failed" if failed else None,
|
||||
)
|
||||
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
|
||||
# Notify even when the remote library is now empty. The global
|
||||
# assistant base must then remove stale contributions from this
|
||||
# source instead of retaining deleted Wiki pages forever.
|
||||
if not failed and notify and self.on_completed:
|
||||
result["assistant_sync"] = await self.on_completed(mapping)
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=False,
|
||||
remote_page_count=remote_count,
|
||||
error="Sync cancelled",
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
await self.store.finish_sync(mapping_id, owner=self.owner, sync_id=sync_id, success=False, remote_page_count=remote_count, error=_safe_error(exc))
|
||||
logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
|
||||
raise
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
await asyncio.gather(heartbeat, return_exceptions=True)
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
"""Per-knowledge-base LRU snapshots for NumPy exact search."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WikiVectorSnapshot:
|
||||
mapping_id: str
|
||||
index_revision: int
|
||||
embedding_fingerprint: str
|
||||
dimensions: int
|
||||
matrix: np.ndarray
|
||||
rows: list[dict[str, Any]]
|
||||
|
||||
|
||||
class WikiVectorCache:
|
||||
def __init__(self, store: LlmWikiIndexStore, *, max_knowledge_bases: int = 20, revision_check_seconds: float = 5) -> None:
|
||||
self.store = store
|
||||
self.max_knowledge_bases = max_knowledge_bases
|
||||
self.revision_check_seconds = revision_check_seconds
|
||||
self._snapshots: OrderedDict[str, WikiVectorSnapshot] = OrderedDict()
|
||||
self._checked_at: dict[str, float] = {}
|
||||
self._locks: dict[str, asyncio.Lock] = {}
|
||||
|
||||
def invalidate(self, mapping_id: str) -> None:
|
||||
self._snapshots.pop(mapping_id, None)
|
||||
self._checked_at.pop(mapping_id, None)
|
||||
|
||||
def clear(self) -> None:
|
||||
self._snapshots.clear()
|
||||
self._checked_at.clear()
|
||||
|
||||
async def get(self, mapping_id: str, fingerprint: str) -> WikiVectorSnapshot:
|
||||
cached = self._snapshots.get(mapping_id)
|
||||
now = monotonic()
|
||||
if cached is not None and cached.embedding_fingerprint == fingerprint and now - self._checked_at.get(mapping_id, 0) < self.revision_check_seconds:
|
||||
self._snapshots.move_to_end(mapping_id)
|
||||
return cached
|
||||
lock = self._locks.setdefault(mapping_id, asyncio.Lock())
|
||||
async with lock:
|
||||
revision, active_fingerprint, state = await self.store.get_index_revision(mapping_id)
|
||||
self._checked_at[mapping_id] = monotonic()
|
||||
cached = self._snapshots.get(mapping_id)
|
||||
if state in {"not_synced", "rebuild_required", "rebuilding", "disabled"} or revision <= 0:
|
||||
raise RuntimeError(f"WIKI_INDEX_NOT_READY:{state}")
|
||||
if active_fingerprint != fingerprint:
|
||||
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
|
||||
if cached is not None and cached.index_revision == revision and cached.embedding_fingerprint == fingerprint:
|
||||
self._snapshots.move_to_end(mapping_id)
|
||||
return cached
|
||||
raw = await self.store.load_vector_snapshot(mapping_id, fingerprint)
|
||||
snapshot = await asyncio.to_thread(self._decode_snapshot, mapping_id, raw)
|
||||
self._snapshots[mapping_id] = snapshot
|
||||
self._snapshots.move_to_end(mapping_id)
|
||||
while len(self._snapshots) > self.max_knowledge_bases:
|
||||
evicted, _ = self._snapshots.popitem(last=False)
|
||||
self._checked_at.pop(evicted, None)
|
||||
return snapshot
|
||||
|
||||
@staticmethod
|
||||
def _decode_snapshot(mapping_id: str, raw: dict[str, Any]) -> WikiVectorSnapshot:
|
||||
rows = list(raw.get("rows") or [])
|
||||
if not rows:
|
||||
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), 0, np.empty((0, 0), dtype=np.float32), [])
|
||||
dimensions = int(rows[0]["embedding_dimensions"])
|
||||
vectors = []
|
||||
valid_rows = []
|
||||
for row in rows:
|
||||
if int(row["embedding_dimensions"]) != dimensions:
|
||||
raise ValueError("Mixed vector dimensions in one Wiki snapshot")
|
||||
vectors.append(decode_vector(row["vector_blob"], dimensions))
|
||||
valid_rows.append(row)
|
||||
matrix = np.ascontiguousarray(np.vstack(vectors), dtype=np.float32)
|
||||
matrix.setflags(write=False)
|
||||
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), dimensions, matrix, valid_rows)
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
"""Validated little-endian normalized float32 Wiki vector codec."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def normalize_vector(values, *, dimensions: int | None = None) -> np.ndarray:
|
||||
vector = np.asarray(values, dtype=np.float32)
|
||||
if vector.ndim != 1 or vector.size == 0:
|
||||
raise ValueError("Embedding vector must be a non-empty one-dimensional array")
|
||||
if dimensions is not None and vector.size != dimensions:
|
||||
raise ValueError(f"Embedding dimension mismatch: expected {dimensions}, got {vector.size}")
|
||||
if not np.isfinite(vector).all():
|
||||
raise ValueError("Embedding vector contains NaN or infinity")
|
||||
norm = float(np.linalg.norm(vector))
|
||||
if not np.isfinite(norm) or norm <= 0:
|
||||
raise ValueError("Embedding vector has zero norm")
|
||||
return np.asarray(vector / norm, dtype="<f4")
|
||||
|
||||
|
||||
def encode_vector(values, *, dimensions: int | None = None) -> tuple[bytes, int]:
|
||||
vector = normalize_vector(values, dimensions=dimensions)
|
||||
return vector.tobytes(order="C"), int(vector.size)
|
||||
|
||||
|
||||
def decode_vector(blob: bytes, dimensions: int) -> np.ndarray:
|
||||
if len(blob) != dimensions * 4:
|
||||
raise ValueError(f"Corrupt vector BLOB: expected {dimensions * 4} bytes, got {len(blob)}")
|
||||
vector = np.frombuffer(blob, dtype="<f4")
|
||||
if not np.isfinite(vector).all():
|
||||
raise ValueError("Stored vector contains NaN or infinity")
|
||||
return vector.copy()
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
"""Runtime composition shared by Gateway routes and the read-only agent tool."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.llmwiki_config import ResolvedLlmWikiRuntime, resolve_llmwiki_runtime
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.persistence.llmwiki import LlmWikiStore
|
||||
|
||||
_default_store: LlmWikiStore | None = None
|
||||
|
||||
|
||||
def register_llmwiki_store(store: LlmWikiStore | None) -> None:
|
||||
global _default_store
|
||||
_default_store = store
|
||||
|
||||
|
||||
def get_llmwiki_store() -> LlmWikiStore:
|
||||
if _default_store is None:
|
||||
raise RuntimeError("LLMWiki store is not initialized")
|
||||
return _default_store
|
||||
|
||||
|
||||
def get_resolved_llmwiki_runtime(config: AppConfig | None = None) -> ResolvedLlmWikiRuntime:
|
||||
current = config or get_app_config()
|
||||
return resolve_llmwiki_runtime(current.llmwiki, load_system_settings().llmwiki)
|
||||
|
||||
|
||||
def build_weknora_client(
|
||||
runtime: ResolvedLlmWikiRuntime,
|
||||
*,
|
||||
http_client: httpx.AsyncClient | None = None,
|
||||
) -> WeKnoraClient:
|
||||
if not runtime.weknora_enabled:
|
||||
raise RuntimeError("WeKnora mode is not enabled")
|
||||
return WeKnoraClient(
|
||||
base_url=runtime.api_base_url,
|
||||
http_client=http_client,
|
||||
admin_email=runtime.admin_email,
|
||||
admin_password=runtime.admin_password,
|
||||
)
|
||||
Reference in new issue
Block a user