feat: integrate WeKnora llmwiki knowledge flow

This commit is contained in:
吉鑫升 committed 2026-08-11 18:13:30 +08:00
1 parent a0a75a9964
commit e0c09f6704
63 files changed
+7045 -197

No files matched your search

@@ -0,0 +1,3 @@
from .tools import web_search_tool
__all__ = ["web_search_tool"]
@@ -0,0 +1,202 @@
"""Configurable JSON-over-HTTP implementation of the ``web_search`` tool."""
from __future__ import annotations
import json
import logging
from copy import deepcopy
from dataclasses import dataclass
from typing import Any
from urllib.parse import quote
import httpx
from langchain.tools import tool
from deerflow.config import get_app_config
from deerflow.config.tool_config import ToolConfig
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class ConfigurableSearchSettings:
"""Validated runtime settings read from the ``web_search`` tool entry."""
enabled: bool
endpoint: str
payload: dict[str, Any]
verify_ssl: bool
timeout: float
max_results: int
result_url_template: str
@classmethod
def from_tool_config(cls, config: ToolConfig | None) -> ConfigurableSearchSettings:
if config is None:
raise ValueError("web_search is not configured")
extra = config.model_extra or {}
endpoint = str(extra.get("endpoint") or "").strip()
if not endpoint.startswith(("http://", "https://")):
raise ValueError("web_search.endpoint must use http:// or https://")
raw_payload = extra.get("payload", {})
if not isinstance(raw_payload, dict):
raise ValueError("web_search.payload must be an object")
try:
timeout = float(extra.get("timeout", 30))
except (TypeError, ValueError) as exc:
raise ValueError("web_search.timeout must be a number") from exc
if timeout <= 0:
raise ValueError("web_search.timeout must be greater than zero")
try:
max_results = int(extra.get("max_results", 5))
except (TypeError, ValueError) as exc:
raise ValueError("web_search.max_results must be an integer") from exc
if max_results <= 0:
raise ValueError("web_search.max_results must be greater than zero")
result_url_template = str(extra.get("result_url_template") or "").strip()
if result_url_template and "{recUuid}" not in result_url_template:
raise ValueError("web_search.result_url_template must contain {recUuid}")
verify_ssl = extra.get("verify_ssl", True)
if not isinstance(verify_ssl, bool):
raise ValueError("web_search.verify_ssl must be true or false")
return cls(
enabled=config.enabled,
endpoint=endpoint,
payload=deepcopy(raw_payload),
verify_ssl=verify_ssl,
timeout=timeout,
max_results=max_results,
result_url_template=result_url_template,
)
def _text(value: Any) -> str:
"""Normalize optional scalar result fields without leaking ``None``."""
if value is None:
return ""
if isinstance(value, str):
return value
return str(value)
def _error_result(query: str, message: str) -> str:
return json.dumps(
{
"query": query,
"total_results": 0,
"results": [],
"error": message,
},
ensure_ascii=False,
)
def _normalize_results(data: Any, settings: ConfigurableSearchSettings) -> list[dict[str, str]]:
if not isinstance(data, dict):
raise ValueError("search API response must be a JSON object")
raw_results = data.get("results")
if not isinstance(raw_results, list):
raise ValueError("search API response field 'results' must be an array")
normalized: list[dict[str, str]] = []
for item in raw_results[: settings.max_results]:
if not isinstance(item, dict):
logger.warning("Ignoring non-object search result: %r", item)
continue
rec_uuid = _text(item.get("recUuid")).strip()
original_url = _text(item.get("url")).strip()
result_url = original_url
if rec_uuid and settings.result_url_template:
result_url = settings.result_url_template.replace(
"{recUuid}",
quote(rec_uuid, safe=""),
)
normalized.append(
{
"recUuid": rec_uuid,
"content": _text(item.get("content")),
"title": _text(item.get("title")),
"url": result_url,
}
)
return normalized
async def execute_search(
query: str,
settings: ConfigurableSearchSettings,
*,
transport: httpx.AsyncBaseTransport | None = None,
) -> str:
"""Execute a search request; ``transport`` exists for network-free tests."""
normalized_query = query.strip()
if not normalized_query:
return _error_result(query, "Search query must not be empty")
if not settings.enabled:
return _error_result(normalized_query, "web_search is disabled")
payload = deepcopy(settings.payload)
payload["query"] = normalized_query
try:
async with httpx.AsyncClient(
timeout=settings.timeout,
verify=settings.verify_ssl,
transport=transport,
) as client:
response = await client.post(settings.endpoint, json=payload)
response.raise_for_status()
data = response.json()
results = _normalize_results(data, settings)
except httpx.TimeoutException:
logger.warning("Configured web search timed out for endpoint %s", settings.endpoint)
return _error_result(normalized_query, "Search API request timed out")
except httpx.HTTPStatusError as exc:
logger.warning(
"Configured web search returned HTTP %s for endpoint %s",
exc.response.status_code,
settings.endpoint,
)
return _error_result(normalized_query, f"Search API returned HTTP {exc.response.status_code}")
except (httpx.HTTPError, json.JSONDecodeError) as exc:
logger.warning("Configured web search request failed: %s", exc)
return _error_result(normalized_query, f"Search API request failed: {type(exc).__name__}")
except ValueError as exc:
logger.warning("Configured web search returned an invalid response: %s", exc)
return _error_result(normalized_query, str(exc))
return json.dumps(
{
"query": normalized_query,
"total_results": len(results),
"results": results,
},
ensure_ascii=False,
)
@tool("web_search", parse_docstring=True)
async def web_search_tool(query: str) -> str:
"""Search the configured information service for current facts and source passages.
Args:
query: Focused search keywords or a natural-language retrieval question.
"""
try:
settings = ConfigurableSearchSettings.from_tool_config(get_app_config().get_tool_config("web_search"))
except ValueError as exc:
return _error_result(query, str(exc))
return await execute_search(query, settings)
@@ -7,7 +7,7 @@ import threading
from typing import Any
import yaml
from pydantic import BaseModel, model_validator
from pydantic import BaseModel, field_validator, model_validator
from deerflow.config.paths import get_paths
@@ -82,7 +82,7 @@ class CommonQuestion(BaseModel):
return value
class AgentConfig(BaseModel):
class AgentConfig(BaseModel):
"""Configuration for a custom agent."""
id: str | None = None
@@ -108,7 +108,20 @@ class AgentConfig(BaseModel):
# 拼进本轮任务,避免弱模型无视系统提示里的技能块。None/False = 关(默认);True = 开。
# 与「业务链条配置」里的同名开关取 OR:任一打开即对该席位生效。见
# ``app/gateway/roundtable_seat_skills.py``。
seat_skill_directive: bool | None = None
seat_skill_directive: bool | None = None
# DeerFlow mapping ids this agent may offer in its new-conversation
# knowledge selector. WeKnora ids and credentials never enter agent files.
llmwiki_knowledge_base_ids: list[str] | None = None
@field_validator("llmwiki_knowledge_base_ids")
@classmethod
def _normalize_llmwiki_ids(cls, value: list[str] | None) -> list[str] | None:
if value is None:
return None
normalized = list(dict.fromkeys(str(item).strip() for item in value if str(item).strip()))
if len(normalized) > 100:
raise ValueError("An agent can bind at most 100 LLMWiki knowledge bases")
return normalized
def load_agent_config(agent_id: str | None) -> AgentConfig | None:
@@ -17,10 +17,11 @@ from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.knowledge_ingest_config import KnowledgeIngestConfig
from deerflow.config.log_system_config import LogSystemConfig
from deerflow.config.login_config import AuthLoginConfig
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.knowledge_ingest_config import KnowledgeIngestConfig
from deerflow.config.llmwiki_config import LlmWikiConfig
from deerflow.config.log_system_config import LogSystemConfig
from deerflow.config.login_config import AuthLoginConfig
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
from deerflow.config.model_config import ModelConfig
from deerflow.config.run_events_config import RunEventsConfig
@@ -42,7 +43,8 @@ from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_searc
from deerflow.config.vision_config import VisionConfig
from deerflow.config.y_log_config import YLogConfig
load_dotenv()
load_dotenv()
load_dotenv(Path(__file__).resolve().parents[4] / ".env.local", override=True)
logger = logging.getLogger(__name__)
@@ -115,7 +117,8 @@ class AppConfig(BaseModel):
models: list[ModelConfig] = Field(default_factory=list, description="Available models")
vision: VisionConfig = Field(default_factory=VisionConfig, description="Image recognition (识图) configuration")
auth_login: AuthLoginConfig = Field(default_factory=AuthLoginConfig, description="Custom login modes (username password gate, token exchange)")
log_system: LogSystemConfig = Field(default_factory=LogSystemConfig, description="Admin-only external log-system entry shown in the settings menu")
log_system: LogSystemConfig = Field(default_factory=LogSystemConfig, description="Admin-only external log-system entry shown in the settings menu")
llmwiki: LlmWikiConfig = Field(default_factory=LlmWikiConfig, description="Conditional legacy/WeKnora LLMWiki integration")
sandbox: SandboxConfig = Field(description="Sandbox configuration")
tools: list[ToolConfig] = Field(default_factory=list, description="Available tools")
tool_groups: list[ToolGroupConfig] = Field(default_factory=list, description="Available tool groups")
@@ -0,0 +1,117 @@
"""Conditional LLMWiki provider configuration.
DeerFlow owns only the connection to WeKnora. Database, Redis, document
parsing, storage and model configuration stay entirely in WeKnora's own
deployment files. An effective non-empty API URL is the sole provider switch.
"""
from __future__ import annotations
from dataclasses import dataclass
from urllib.parse import urlsplit, urlunsplit
from pydantic import BaseModel, ConfigDict, Field, field_validator
def normalize_service_url(value: str | None) -> str:
"""Normalize an optional HTTP(S) service URL and reject embedded secrets."""
raw = str(value or "").strip()
if not raw:
return ""
parsed = urlsplit(raw)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise ValueError("WeKnora addresses must be absolute http(s) URLs")
if parsed.username is not None or parsed.password is not None:
raise ValueError("WeKnora addresses must not contain credentials")
if parsed.query or parsed.fragment:
raise ValueError("WeKnora addresses must not contain query strings or fragments")
path = parsed.path.rstrip("/")
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
class WeKnoraConnectionConfig(BaseModel):
"""The complete DeerFlow-owned connection surface for WeKnora."""
api_base_url: str = Field(default="", description="WeKnora app API base URL; non-empty enables WeKnora mode")
web_base_url: str = Field(default="", description="Optional internal WeKnora administration URL")
admin_email: str = Field(
default="",
description=(
"Optional WeKnora administrator email used only by DeerFlow's "
"server-side detail-page iframe proxy. Prefer $ENV references."
),
)
admin_password: str = Field(
default="",
repr=False,
description=(
"Optional WeKnora administrator password used only by DeerFlow's "
"server-side detail-page iframe proxy. Prefer $ENV references."
),
)
model_config = ConfigDict(extra="ignore")
@field_validator("api_base_url", "web_base_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
class LlmWikiConfig(BaseModel):
"""Top-level ``config.yaml`` section for conditional WeKnora mode."""
weknora: WeKnoraConnectionConfig = Field(default_factory=WeKnoraConnectionConfig)
model_config = ConfigDict(extra="ignore")
class LlmWikiRuntimeOverride(BaseModel):
"""Administrator-written URL override stored in system settings.
``enabled=False`` means "use config.yaml". When enabled, an empty API URL
deliberately forces legacy mode even if the source file has an address.
Secrets are never stored here.
"""
enabled: bool = False
api_base_url: str = ""
web_base_url: str = ""
model_config = ConfigDict(extra="ignore")
@field_validator("api_base_url", "web_base_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
@dataclass(frozen=True)
class ResolvedLlmWikiRuntime:
provider: str
api_base_url: str
web_base_url: str
admin_email: str
admin_password: str
override_enabled: bool
@property
def weknora_enabled(self) -> bool:
return self.provider == "weknora"
def resolve_llmwiki_runtime(
config: LlmWikiConfig,
override: LlmWikiRuntimeOverride | None = None,
) -> ResolvedLlmWikiRuntime:
"""Resolve the deterministic provider from file config plus URL override."""
use_override = bool(override and override.enabled)
api_base_url = override.api_base_url if use_override and override else config.weknora.api_base_url
web_base_url = override.web_base_url if use_override and override else config.weknora.web_base_url
return ResolvedLlmWikiRuntime(
provider="weknora" if api_base_url else "legacy",
api_base_url=api_base_url,
web_base_url=web_base_url,
admin_email=config.weknora.admin_email,
admin_password=config.weknora.admin_password,
override_enabled=use_override,
)
@@ -23,6 +23,7 @@ from uuid import uuid4
from pydantic import BaseModel, ConfigDict, Field, field_validator
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride
from deerflow.config.runtime_paths import runtime_home
logger = logging.getLogger(__name__)
@@ -499,6 +500,7 @@ class SystemSettings(BaseModel):
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
model_config = ConfigDict(extra="ignore")
@@ -1,20 +1,21 @@
from pydantic import BaseModel, ConfigDict, Field
class ToolGroupConfig(BaseModel):
"""Config section for a tool group"""
name: str = Field(..., description="Unique name for the tool group")
model_config = ConfigDict(extra="allow")
class ToolConfig(BaseModel):
"""Config section for a tool"""
name: str = Field(..., description="Unique name for the tool")
group: str = Field(..., description="Group name for the tool")
use: str = Field(
...,
description="Variable name of the tool provider(e.g. deerflow.sandbox.tools:bash_tool)",
)
model_config = ConfigDict(extra="allow")
from pydantic import BaseModel, ConfigDict, Field
class ToolGroupConfig(BaseModel):
"""Config section for a tool group"""
name: str = Field(..., description="Unique name for the tool group")
model_config = ConfigDict(extra="allow")
class ToolConfig(BaseModel):
"""Config section for a tool"""
name: str = Field(..., description="Unique name for the tool")
group: str = Field(..., description="Group name for the tool")
enabled: bool = Field(default=True, description="Whether the tool is registered with the agent")
use: str = Field(
...,
description="Variable name of the tool provider(e.g. deerflow.sandbox.tools:bash_tool)",
)
model_config = ConfigDict(extra="allow")
@@ -0,0 +1,2 @@
"""External service integrations owned by the DeerFlow harness."""
@@ -0,0 +1,5 @@
"""WeKnora REST integration."""
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
__all__ = ["WeKnoraClient", "WeKnoraError"]
@@ -0,0 +1,443 @@
"""Small, version-pinned adapter over WeKnora's official REST API."""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from typing import Any
from urllib.parse import quote
import httpx
class WeKnoraError(RuntimeError):
"""Normalized upstream failure that is safe for DeerFlow to translate."""
def __init__(self, message: str, *, status_code: int = 502) -> None:
super().__init__(message)
self.status_code = status_code
def _data(payload: Any) -> Any:
if not isinstance(payload, dict):
raise WeKnoraError("WeKnora returned an invalid response")
if payload.get("success") is False:
message = payload.get("message") or payload.get("error") or "WeKnora rejected the request"
raise WeKnoraError(str(message), status_code=400)
return payload.get("data", payload)
def _as_list(value: Any) -> list[dict[str, Any]]:
if isinstance(value, list):
return [dict(item) for item in value if isinstance(item, Mapping)]
if isinstance(value, dict):
for key in ("items", "list", "knowledge_bases", "knowledge", "records"):
nested = value.get(key)
if isinstance(nested, list):
return [dict(item) for item in nested if isinstance(item, Mapping)]
return []
class WeKnoraClient:
"""Async client for a trusted-network WeKnora endpoint.
The caller may inject an ``httpx.AsyncClient`` for connection pooling and
tests. When omitted, each public method creates and closes a short-lived
client. DeerFlow deliberately sends no WeKnora credential; the independently
deployed service or its reverse proxy must allow DeerFlow's internal traffic.
"""
def __init__(
self,
*,
base_url: str,
timeout_seconds: float = 30.0,
http_client: httpx.AsyncClient | None = None,
) -> None:
self.base_url = base_url.rstrip("/")
self.timeout_seconds = timeout_seconds
self._http_client = http_client
if not self.base_url:
raise ValueError("WeKnora base URL is required")
async def _request(
self,
method: str,
path: str,
*,
json: Any = None,
params: Mapping[str, Any] | None = None,
files: Any = None,
data: Mapping[str, Any] | None = None,
) -> Any:
headers = {"Accept": "application/json"}
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(timeout=httpx.Timeout(self.timeout_seconds), follow_redirects=False)
try:
response = await client.request(
method,
f"{self.base_url}{path}",
headers=headers,
json=json,
params=params,
files=files,
data=data,
)
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code == 404:
raise WeKnoraError("WeKnora resource was not found", status_code=404)
if response.status_code in {401, 403}:
raise WeKnoraError(
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
status_code=502,
)
if response.status_code >= 400:
detail = ""
try:
body = response.json()
if isinstance(body, dict):
detail = str(body.get("message") or body.get("error") or body.get("detail") or "")
except ValueError:
detail = ""
raise WeKnoraError(detail or f"WeKnora returned HTTP {response.status_code}", status_code=502)
if response.status_code == 204 or not response.content:
return {"success": True}
try:
return response.json()
except ValueError as exc:
raise WeKnoraError("WeKnora returned invalid JSON") from exc
async def health(self) -> dict[str, Any]:
payload = await self._request("GET", "/health")
return dict(payload) if isinstance(payload, dict) else {"status": "ok"}
async def list_knowledge_bases(self) -> list[dict[str, Any]]:
return _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
async def get_knowledge_base(self, knowledge_base_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}"))
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid knowledge base")
return dict(value)
async def list_models(self) -> list[dict[str, Any]]:
"""List models from WeKnora; model credentials are never returned."""
return _as_list(_data(await self._request("GET", "/api/v1/models")))
async def get_default_embedding_model_id(self) -> str:
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "embedding"]
preferred = next((item for item in models if item.get("is_default") is True), None)
selected = preferred or next(
(item for item in models if str(item.get("status") or "active").lower() == "active"),
None,
)
model_id = str((selected or {}).get("id") or "")
if not model_id:
raise WeKnoraError(
"WeKnora has no available embedding model; configure its default model in WeKnora",
status_code=400,
)
return model_id
async def get_default_knowledge_qa_model_id(self) -> str:
"""Resolve WeKnora's own active synthesis model for Wiki indexing."""
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "knowledgeqa"]
preferred = next((item for item in models if item.get("is_default") is True), None)
selected = preferred or next(
(item for item in models if str(item.get("status") or "active").lower() == "active"),
None,
)
model_id = str((selected or {}).get("id") or "")
if not model_id:
raise WeKnoraError(
"WeKnora has no available KnowledgeQA model; configure one in WeKnora before creating a Wiki",
status_code=400,
)
return model_id
async def create_knowledge_base(
self,
*,
name: str,
description: str = "",
kb_type: str = "document",
embedding_model_id: str | None = None,
wiki_enabled: bool = False,
) -> dict[str, Any]:
model_id = embedding_model_id or await self.get_default_embedding_model_id()
create_payload: dict[str, Any] = {
"name": name,
"description": description,
"type": kb_type,
"embedding_model_id": model_id,
}
if wiki_enabled:
synthesis_model_id = await self.get_default_knowledge_qa_model_id()
create_payload.update(
{
"summary_model_id": synthesis_model_id,
"indexing_strategy": {
"vector_enabled": True,
"keyword_enabled": True,
"wiki_enabled": True,
"graph_enabled": False,
},
"wiki_config": {
"synthesis_model_id": synthesis_model_id,
"extraction_granularity": "standard",
},
}
)
payload = await self._request(
"POST",
"/api/v1/knowledge-bases",
json=create_payload,
)
value = _data(payload)
if not isinstance(value, dict) or not value.get("id"):
raise WeKnoraError("WeKnora did not return the created knowledge base")
return dict(value)
async def update_knowledge_base(self, knowledge_base_id: str, changes: Mapping[str, Any]) -> dict[str, Any]:
value = _data(
await self._request(
"PUT",
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}",
json=dict(changes),
)
)
return dict(value) if isinstance(value, dict) else {}
async def delete_knowledge_base(self, knowledge_base_id: str) -> None:
await self._request("DELETE", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}")
async def list_documents(self, knowledge_base_id: str, *, page: int = 1, page_size: int = 100) -> dict[str, Any]:
payload = await self._request(
"GET",
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge",
params={"page": page, "page_size": page_size},
)
rows = _as_list(_data(payload))
total = payload.get("total", len(rows)) if isinstance(payload, dict) else len(rows)
return {"items": rows, "total": int(total or 0), "page": page, "page_size": page_size}
async def upload_document(
self,
knowledge_base_id: str,
*,
filename: str,
content: bytes,
content_type: str = "application/octet-stream",
) -> dict[str, Any]:
value = _data(
await self._request(
"POST",
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge/file",
files={"file": (filename, content, content_type)},
)
)
return dict(value) if isinstance(value, dict) else {}
async def import_document_url(self, knowledge_base_id: str, *, url: str) -> dict[str, Any]:
value = _data(
await self._request(
"POST",
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge/url",
json={"url": url},
)
)
return dict(value) if isinstance(value, dict) else {}
async def create_manual_document(
self,
knowledge_base_id: str,
*,
title: str,
content: str,
) -> dict[str, Any]:
value = _data(
await self._request(
"POST",
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge/manual",
json={"title": title, "content": content, "status": "publish"},
)
)
return dict(value) if isinstance(value, dict) else {}
async def delete_document(self, knowledge_id: str) -> None:
await self._request("DELETE", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}")
async def get_document(self, knowledge_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}"))
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid document")
return dict(value)
async def get_document_preview(self, knowledge_id: str) -> tuple[bytes, str]:
"""Return the original document preview bytes and its media type."""
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=True,
)
try:
response = await client.request(
"GET",
f"{self.base_url}/api/v1/knowledge/{quote(knowledge_id, safe='')}/preview",
headers={"Accept": "*/*"},
follow_redirects=True,
)
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora preview request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora preview is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code == 404:
raise WeKnoraError("WeKnora document preview was not found", status_code=404)
if response.status_code in {401, 403}:
raise WeKnoraError(
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
status_code=502,
)
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502)
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
return response.content, media_type or "application/octet-stream"
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
return dict(value) if isinstance(value, dict) else {}
async def list_chunks(self, knowledge_id: str, *, page: int = 1, page_size: int = 20) -> dict[str, Any]:
payload = await self._request(
"GET",
f"/api/v1/chunks/{quote(knowledge_id, safe='')}",
params={"page": page, "page_size": page_size},
)
rows = _as_list(_data(payload))
total = payload.get("total", len(rows)) if isinstance(payload, dict) else len(rows)
return {"items": rows, "total": int(total or 0), "page": page, "page_size": page_size}
async def get_wiki_graph(self, knowledge_base_id: str, *, limit: int = 200) -> dict[str, Any]:
value = _data(
await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/graph",
params={"mode": "overview", "limit": max(1, min(limit, 500))},
)
)
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid wiki graph")
return dict(value)
async def list_wiki_pages(
self,
knowledge_base_id: str,
*,
page: int = 1,
page_size: int = 50,
query: str = "",
) -> dict[str, Any]:
payload = await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages",
params={"page": page, "page_size": page_size, "query": query or None},
)
value = _data(payload)
source = value if isinstance(value, dict) else payload if isinstance(payload, dict) else {}
rows = _as_list(source.get("pages") if isinstance(source, dict) else value)
return {
"pages": rows,
"total": int(source.get("total") or len(rows)),
"page": int(source.get("page") or page),
"page_size": int(source.get("page_size") or page_size),
"total_pages": int(source.get("total_pages") or max(1, (int(source.get("total") or len(rows)) + page_size - 1) // page_size)),
}
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict[str, Any]:
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
value = _data(
await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
)
)
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid Wiki page")
return dict(value)
async def search(self, query: str, knowledge_base_ids: Sequence[str]) -> list[dict[str, Any]]:
ids = [str(value) for value in knowledge_base_ids if str(value)]
if not ids:
return []
rows = _as_list(
_data(
await self._request(
"POST",
"/api/v1/knowledge-search",
json={"query": query, "knowledge_base_ids": ids},
)
)
)
normalized: list[dict[str, Any]] = []
for row in rows:
normalized.append(
{
"chunk_id": str(row.get("id") or row.get("chunk_id") or ""),
"content": str(row.get("content") or ""),
"knowledge_id": str(row.get("knowledge_id") or ""),
"knowledge_base_id": str(row.get("knowledge_base_id") or ""),
"title": str(row.get("knowledge_title") or row.get("title") or row.get("knowledge_filename") or ""),
"filename": str(row.get("knowledge_filename") or ""),
"score": float(row.get("score") or 0),
"chunk_index": int(row.get("chunk_index") or 0),
"metadata": row.get("metadata") if isinstance(row.get("metadata"), dict) else {},
}
)
return normalized
async def get_chunk(self, chunk_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/chunks/by-id/{quote(chunk_id, safe='')}"))
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid chunk")
return dict(value)
async def get_chunk_context(self, chunk_id: str, *, expected_knowledge_base_id: str, radius: int = 2) -> list[dict[str, Any]]:
center = await self.get_chunk(chunk_id)
if str(center.get("knowledge_base_id") or "") != expected_knowledge_base_id:
raise WeKnoraError("Chunk does not belong to the authorized knowledge base", status_code=404)
before: list[dict[str, Any]] = []
cursor = center
for _ in range(max(0, min(radius, 4))):
previous_id = str(cursor.get("pre_chunk_id") or "")
if not previous_id:
break
previous = await self.get_chunk(previous_id)
if str(previous.get("knowledge_base_id") or "") != expected_knowledge_base_id:
break
before.append(previous)
cursor = previous
after: list[dict[str, Any]] = []
cursor = center
for _ in range(max(0, min(radius, 4))):
next_id = str(cursor.get("next_chunk_id") or "")
if not next_id:
break
following = await self.get_chunk(next_id)
if str(following.get("knowledge_base_id") or "") != expected_knowledge_base_id:
break
after.append(following)
cursor = following
return [*reversed(before), center, *after]
@@ -0,0 +1,43 @@
"""Runtime composition shared by Gateway routes and the read-only agent tool."""
from __future__ import annotations
import httpx
from deerflow.config import get_app_config
from deerflow.config.app_config import AppConfig
from deerflow.config.llmwiki_config import ResolvedLlmWikiRuntime, resolve_llmwiki_runtime
from deerflow.config.system_settings import load_system_settings
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.persistence.llmwiki import LlmWikiStore
_default_store: LlmWikiStore | None = None
def register_llmwiki_store(store: LlmWikiStore | None) -> None:
global _default_store
_default_store = store
def get_llmwiki_store() -> LlmWikiStore:
if _default_store is None:
raise RuntimeError("LLMWiki store is not initialized")
return _default_store
def get_resolved_llmwiki_runtime(config: AppConfig | None = None) -> ResolvedLlmWikiRuntime:
current = config or get_app_config()
return resolve_llmwiki_runtime(current.llmwiki, load_system_settings().llmwiki)
def build_weknora_client(
runtime: ResolvedLlmWikiRuntime,
*,
http_client: httpx.AsyncClient | None = None,
) -> WeKnoraClient:
if not runtime.weknora_enabled:
raise RuntimeError("WeKnora mode is not enabled")
return WeKnoraClient(
base_url=runtime.api_base_url,
http_client=http_client,
)
@@ -0,0 +1,22 @@
"""DeerFlow-owned LLMWiki authorization metadata."""
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.memory import MemoryLlmWikiStore
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki.sql import LlmWikiRepository
__all__ = [
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
"LlmWikiRepository",
"LlmWikiScope",
"LlmWikiStore",
"MemoryLlmWikiStore",
"make_llmwiki_store",
]
def make_llmwiki_store(session_factory):
if session_factory is None:
return MemoryLlmWikiStore()
return LlmWikiRepository(session_factory)
@@ -0,0 +1,72 @@
"""Persistence contract for DeerFlow-owned LLMWiki authorization metadata."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, Literal
LlmWikiScope = Literal["personal", "public", "all"]
class LlmWikiStore(ABC):
@abstractmethod
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]: ...
@abstractmethod
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]: ...
@abstractmethod
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None: ...
@abstractmethod
async def delete_mapping(self, mapping_id: str) -> bool: ...
@abstractmethod
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]: ...
@abstractmethod
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None: ...
@abstractmethod
async def get_conversation_deposit(
self,
*,
thread_id: str,
assistant_message_id: str,
) -> dict[str, Any] | None: ...
@abstractmethod
async def upsert_conversation_deposit(
self,
*,
thread_id: str,
human_message_id: str | None,
assistant_message_id: str,
knowledge_base_mapping_id: str,
weknora_knowledge_id: str | None,
title: str,
created_by_user_id: str | None,
) -> dict[str, Any]: ...
@abstractmethod
async def delete_conversation_deposits(
self,
*,
thread_id: str,
message_ids: list[str] | None = None,
) -> list[dict[str, Any]]: ...
@@ -0,0 +1,190 @@
"""In-memory LLMWiki metadata store for tests and DB-less deployments."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
def _now() -> str:
return datetime.now(UTC).isoformat()
class MemoryLlmWikiStore(LlmWikiStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._conversation_deposits: dict[str, dict[str, Any]] = {}
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
if any(row["weknora_id"] == weknora_id for row in self._rows.values()):
raise ValueError("WeKnora knowledge base is already mapped")
now = _now()
row = {
"id": str(uuid4()),
"weknora_id": weknora_id,
"owner_user_id": owner_user_id,
"name": name,
"description": description,
"kb_type": kb_type,
"publication_status": "private",
"published_by": None,
"reviewed_by": None,
"published_at": None,
"created_at": now,
"updated_at": now,
}
self._rows[row["id"]] = row
return dict(row)
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
rows = list(self._rows.values())
if scope == "personal":
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id]
elif scope == "public":
rows = [row for row in rows if row["publication_status"] == "published"]
else:
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id or row["publication_status"] == "published"]
return [dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
if is_admin or row["owner_user_id"] == user_id:
return dict(row)
if not write and row["publication_status"] == "published":
return dict(row)
return None
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
for row in self._rows.values():
if row["weknora_id"] == weknora_id:
return dict(row)
return None
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
if name is not None:
row["name"] = name
if description is not None:
row["description"] = description
row["updated_at"] = _now()
return dict(row)
async def delete_mapping(self, mapping_id: str) -> bool:
return self._rows.pop(mapping_id, None) is not None
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None or (not is_admin and row["owner_user_id"] != actor_user_id):
return None
row["publication_status"] = "published"
row["published_by"] = actor_user_id
row["reviewed_by"] = None
row["published_at"] = _now()
row["updated_at"] = _now()
return dict(row)
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None or row["publication_status"] != "pending":
return None
row["publication_status"] = "published" if approved else "rejected"
row["reviewed_by"] = reviewer_user_id
row["published_at"] = _now() if approved else None
row["updated_at"] = _now()
return dict(row)
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None or (not is_admin and row["owner_user_id"] != actor_user_id):
return None
row["publication_status"] = "private"
row["reviewed_by"] = actor_user_id if is_admin else row.get("reviewed_by")
row["published_at"] = None
row["updated_at"] = _now()
return dict(row)
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
rows = [row for row in self._rows.values() if row["publication_status"] != "private"]
if status:
rows = [row for row in rows if row["publication_status"] == status]
return [dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
for row in self._rows.values():
if row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
return dict(row)
return None
async def get_conversation_deposit(
self,
*,
thread_id: str,
assistant_message_id: str,
) -> dict[str, Any] | None:
key = f"{thread_id}:{assistant_message_id}"
row = self._conversation_deposits.get(key)
return dict(row) if row else None
async def upsert_conversation_deposit(
self,
*,
thread_id: str,
human_message_id: str | None,
assistant_message_id: str,
knowledge_base_mapping_id: str,
weknora_knowledge_id: str | None,
title: str,
created_by_user_id: str | None,
) -> dict[str, Any]:
key = f"{thread_id}:{assistant_message_id}"
now = _now()
row = self._conversation_deposits.get(key)
if row is None:
row = {
"id": str(uuid4()),
"thread_id": thread_id,
"human_message_id": human_message_id,
"assistant_message_id": assistant_message_id,
"knowledge_base_mapping_id": knowledge_base_mapping_id,
"weknora_knowledge_id": weknora_knowledge_id,
"title": title,
"created_by_user_id": created_by_user_id,
"created_at": now,
"updated_at": now,
}
self._conversation_deposits[key] = row
else:
row.update(
{
"human_message_id": human_message_id,
"knowledge_base_mapping_id": knowledge_base_mapping_id,
"weknora_knowledge_id": weknora_knowledge_id,
"title": title,
"updated_at": now,
}
)
return dict(row)
async def delete_conversation_deposits(
self,
*,
thread_id: str,
message_ids: list[str] | None = None,
) -> list[dict[str, Any]]:
ids = set(message_ids or [])
deleted: list[dict[str, Any]] = []
for key, row in list(self._conversation_deposits.items()):
if row.get("thread_id") != thread_id:
continue
if ids and row.get("human_message_id") not in ids and row.get("assistant_message_id") not in ids:
continue
deleted.append(dict(row))
self._conversation_deposits.pop(key, None)
return deleted
@@ -0,0 +1,63 @@
"""ORM mapping between DeerFlow ownership and WeKnora knowledge bases."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class LlmWikiKnowledgeBaseRow(Base):
__tablename__ = "llmwiki_knowledge_bases"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
weknora_id: Mapped[str] = mapped_column(String(128), unique=True, index=True)
owner_user_id: Mapped[str] = mapped_column(String(64), index=True)
name: Mapped[str] = mapped_column(String(255), default="")
description: Mapped[str] = mapped_column(String(2048), default="")
kb_type: Mapped[str] = mapped_column(String(32), default="document")
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
def to_dict(self) -> dict:
return {column.name: getattr(self, column.name) for column in self.__table__.columns}
class LlmWikiConversationDepositRow(Base):
"""Mapping from one DeerFlow answer turn to its WeKnora manual document.
The actual content lives in WeKnora. DeerFlow keeps only the stable ids it
needs to rewrite/delete the document when the user regenerates or removes a
conversation turn.
"""
__tablename__ = "llmwiki_conversation_deposits"
__table_args__ = (
UniqueConstraint(
"thread_id",
"assistant_message_id",
name="uq_llmwiki_conversation_deposit_answer",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
thread_id: Mapped[str] = mapped_column(String(128), index=True)
human_message_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
assistant_message_id: Mapped[str] = mapped_column(String(128), index=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), index=True)
weknora_knowledge_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
title: Mapped[str] = mapped_column(String(255), default="")
created_by_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
def to_dict(self) -> dict:
return {column.name: getattr(self, column.name) for column in self.__table__.columns}
@@ -0,0 +1,244 @@
"""SQLAlchemy LLMWiki metadata repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
def _now() -> datetime:
return datetime.now(UTC)
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("published_at", "created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
return data
class LlmWikiRepository(LlmWikiStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
row = LlmWikiKnowledgeBaseRow(
id=str(uuid4()),
weknora_id=weknora_id,
owner_user_id=owner_user_id,
name=name,
description=description,
kb_type=kb_type,
publication_status="private",
created_at=_now(),
updated_at=_now(),
)
async with self._sf() as session:
session.add(row)
await session.commit()
return _to_dict(row)
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
stmt = select(LlmWikiKnowledgeBaseRow)
if scope == "personal" and not is_admin:
stmt = stmt.where(LlmWikiKnowledgeBaseRow.owner_user_id == user_id)
elif scope == "public":
stmt = stmt.where(LlmWikiKnowledgeBaseRow.publication_status == "published")
elif scope == "all" and not is_admin:
stmt = stmt.where(
or_(
LlmWikiKnowledgeBaseRow.owner_user_id == user_id,
LlmWikiKnowledgeBaseRow.publication_status == "published",
)
)
stmt = stmt.order_by(LlmWikiKnowledgeBaseRow.created_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.id == mapping_id)
if not is_admin:
condition = LlmWikiKnowledgeBaseRow.owner_user_id == user_id
if not write:
condition = or_(condition, LlmWikiKnowledgeBaseRow.publication_status == "published")
stmt = stmt.where(condition)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.weknora_id == weknora_id)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
return None
if name is not None:
row.name = name
if description is not None:
row.description = description
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def delete_mapping(self, mapping_id: str) -> bool:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None or (not is_admin and row.owner_user_id != actor_user_id):
return None
row.publication_status = "published"
row.published_by = actor_user_id
row.reviewed_by = None
row.published_at = _now()
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None or row.publication_status != "pending":
return None
row.publication_status = "published" if approved else "rejected"
row.reviewed_by = reviewer_user_id
row.published_at = _now() if approved else None
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None or (not is_admin and row.owner_user_id != actor_user_id):
return None
row.publication_status = "private"
if is_admin:
row.reviewed_by = actor_user_id
row.published_at = None
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.publication_status != "private")
if status:
stmt = stmt.where(LlmWikiKnowledgeBaseRow.publication_status == status)
stmt = stmt.order_by(LlmWikiKnowledgeBaseRow.updated_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
stmt = (
select(LlmWikiKnowledgeBaseRow)
.where(
LlmWikiKnowledgeBaseRow.name == "对话沉淀",
LlmWikiKnowledgeBaseRow.publication_status == "published",
)
.order_by(LlmWikiKnowledgeBaseRow.created_at.asc())
.limit(1)
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def get_conversation_deposit(
self,
*,
thread_id: str,
assistant_message_id: str,
) -> dict[str, Any] | None:
stmt = select(LlmWikiConversationDepositRow).where(
LlmWikiConversationDepositRow.thread_id == thread_id,
LlmWikiConversationDepositRow.assistant_message_id == assistant_message_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def upsert_conversation_deposit(
self,
*,
thread_id: str,
human_message_id: str | None,
assistant_message_id: str,
knowledge_base_mapping_id: str,
weknora_knowledge_id: str | None,
title: str,
created_by_user_id: str | None,
) -> dict[str, Any]:
now = _now()
async with self._sf() as session:
stmt = select(LlmWikiConversationDepositRow).where(
LlmWikiConversationDepositRow.thread_id == thread_id,
LlmWikiConversationDepositRow.assistant_message_id == assistant_message_id,
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = LlmWikiConversationDepositRow(
id=str(uuid4()),
thread_id=thread_id,
human_message_id=human_message_id,
assistant_message_id=assistant_message_id,
knowledge_base_mapping_id=knowledge_base_mapping_id,
weknora_knowledge_id=weknora_knowledge_id,
title=title,
created_by_user_id=created_by_user_id,
created_at=now,
updated_at=now,
)
session.add(row)
else:
row.human_message_id = human_message_id
row.knowledge_base_mapping_id = knowledge_base_mapping_id
row.weknora_knowledge_id = weknora_knowledge_id
row.title = title
row.updated_at = now
await session.commit()
return _to_dict(row)
async def delete_conversation_deposits(
self,
*,
thread_id: str,
message_ids: list[str] | None = None,
) -> list[dict[str, Any]]:
conditions = [LlmWikiConversationDepositRow.thread_id == thread_id]
if message_ids:
conditions.append(
or_(
LlmWikiConversationDepositRow.human_message_id.in_(message_ids),
LlmWikiConversationDepositRow.assistant_message_id.in_(message_ids),
)
)
stmt = select(LlmWikiConversationDepositRow).where(*conditions)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
deleted = [_to_dict(row) for row in rows]
if rows:
await session.execute(delete(LlmWikiConversationDepositRow).where(LlmWikiConversationDepositRow.id.in_([row.id for row in rows])))
await session.commit()
return deleted
@@ -42,6 +42,7 @@ from deerflow.persistence.embed_sessions.model import EmbedSessionRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
from deerflow.persistence.models.run_event import RunEventRow
@@ -108,6 +109,8 @@ __all__ = [
"KnowledgeTagRow",
"LightAppRow",
"LlmCallMetricRow",
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
"MenuOverrideRow",
"NotificationRow",
"ParallelAgentTaskRow",
@@ -13,7 +13,6 @@ from deerflow.config.extensions_config import (
get_extensions_config,
)
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
@@ -27,7 +26,11 @@ def build_reference_batches(messages: list[Any]) -> list[dict[str, Any]]:
if _message_type(messages[cursor]) == "human":
end = cursor
break
sources = _collect_sources(messages[index + 1 : end])
preloaded_sources = _collect_preloaded_sources(message, start_index=1)
sources = [
*preloaded_sources,
*_collect_sources(messages[index + 1 : end], start_index=len(preloaded_sources) + 1),
]
if sources:
anchor = _find_reference_batch_anchor(messages, index + 1, end)
message_id = _message_id(anchor) or _message_id(message) or str(index)
@@ -46,7 +49,11 @@ def build_latest_reference_batch(messages: list[Any]) -> dict[str, Any] | None:
human_index = _find_latest_human_index(messages)
if human_index is None:
return None
sources = _collect_sources(messages[human_index + 1 :])
preloaded_sources = _collect_preloaded_sources(messages[human_index], start_index=1)
sources = [
*preloaded_sources,
*_collect_sources(messages[human_index + 1 :], start_index=len(preloaded_sources) + 1),
]
if not sources:
return None
assistant = _find_reference_batch_anchor(messages, human_index + 1, len(messages))
@@ -365,7 +372,7 @@ def _source_debug_item(source: dict[str, Any]) -> dict[str, Any]:
}
def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
def _collect_sources(messages: list[Any], *, start_index: int = 1) -> list[dict[str, Any]]:
sources: list[dict[str, Any]] = []
pending_tool_contexts: list[dict[str, Any]] = []
tool_contexts_by_id: dict[str, dict[str, Any]] = {}
@@ -393,7 +400,7 @@ def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
)
if configured_sources is not None:
for normalized in configured_sources:
normalized["index"] = len(sources) + 1
normalized["index"] = start_index + len(sources)
sources.append(normalized)
continue
for raw_item in _first_list(payload):
@@ -405,11 +412,37 @@ def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
)
if not normalized:
continue
normalized["index"] = len(sources) + 1
normalized["index"] = start_index + len(sources)
sources.append(normalized)
return sources
def _collect_preloaded_sources(message: Any, *, start_index: int = 1) -> list[dict[str, Any]]:
extra = _message_additional_kwargs(message)
raw_sources = extra.get("llmwiki_reference_sources")
if not isinstance(raw_sources, list):
return []
sources: list[dict[str, Any]] = []
for raw in raw_sources:
if not isinstance(raw, dict):
continue
source = dict(raw)
index = start_index + len(sources)
source["index"] = index
source.setdefault("skillName", "llmwiki_search")
source.setdefault("mode", "citation")
source.setdefault("type", "llmwiki")
title = str(source.get("title") or "").strip()
if not title:
source["title"] = f"LLMWiki 片段 {index}"
snippet = str(source.get("snippet") or source.get("content") or "").strip()
source["snippet"] = snippet[:360]
if "raw" not in source or not isinstance(source.get("raw"), dict):
source["raw"] = {key: value for key, value in source.items() if key != "raw"}
sources.append(source)
return sources
def _collect_configured_sources(
payload: Any,
fallback_index: int,
@@ -486,6 +519,14 @@ def _message_content(message: Any) -> Any:
return message.get("content") if isinstance(message, dict) else getattr(message, "content", None)
def _message_additional_kwargs(message: Any) -> dict[str, Any]:
if isinstance(message, dict):
value = message.get("additional_kwargs") or message.get("additionalKwargs")
else:
value = getattr(message, "additional_kwargs", None)
return value if isinstance(value, dict) else {}
def _message_name(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("name") or (message.get("additional_kwargs") or {}).get("name") or "tool")
@@ -0,0 +1,132 @@
"""Authorized read-only WeKnora retrieval for DeerFlow Q&A."""
from __future__ import annotations
import json
from collections.abc import Mapping
from typing import Any
from urllib.parse import urlencode
from langchain.tools import ToolRuntime, tool
from langgraph.typing import ContextT
from deerflow.agents.thread_state import ThreadState
from deerflow.config.agents_config import load_agent_config
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_llmwiki_store, get_resolved_llmwiki_runtime
from deerflow.runtime.user_context import get_current_user, get_effective_user_id
def selected_knowledge_base_ids(context: Mapping[str, Any] | None) -> list[str] | None:
"""Read the conversation selection; ``None`` means legacy/no selection key."""
if not context or "llmwiki_knowledge_base_ids" not in context:
return None
raw = context.get("llmwiki_knowledge_base_ids")
if not isinstance(raw, list):
return []
return list(dict.fromkeys(str(item).strip() for item in raw if str(item).strip()))[:100]
def _agent_allowed_ids(context: Mapping[str, Any] | None) -> list[str] | None:
agent_id = str((context or {}).get("agent_id") or "").strip()
if not agent_id or agent_id in {"default", "lead_agent"}:
return None
try:
config = load_agent_config(agent_id)
except (FileNotFoundError, ValueError):
return []
return list(config.llmwiki_knowledge_base_ids or []) if config else []
@tool("llmwiki_search", parse_docstring=True)
async def llmwiki_search_tool(
query: str,
runtime: ToolRuntime[ContextT, ThreadState],
) -> str:
"""Search the LLMWiki knowledge selected for this conversation.
Use this tool when an answer can benefit from the user's managed knowledge
bases or the shared public LLMWiki. It only retrieves passages; it cannot
upload, edit, delete, publish or change permissions.
Args:
query: Focused natural-language retrieval query.
"""
resolved = get_resolved_llmwiki_runtime()
if not resolved.weknora_enabled:
return "LLMWiki retrieval is unavailable because WeKnora mode is not enabled."
store = get_llmwiki_store()
user_id = get_effective_user_id()
is_admin = getattr(get_current_user(), "system_role", None) == "admin"
context = runtime.context if runtime.context else {}
selected_ids = selected_knowledge_base_ids(context)
allowed_ids = _agent_allowed_ids(context)
if selected_ids is None:
selected_ids = allowed_ids
elif allowed_ids is not None and not set(selected_ids).issubset(set(allowed_ids)):
return "The conversation requested knowledge outside this agent's configured LLMWiki scope."
if selected_ids == []:
return "No LLMWiki knowledge base is selected for this conversation."
if selected_ids is not None:
rows = []
for mapping_id in selected_ids:
row = await store.get_authorized(
mapping_id,
user_id,
write=False,
is_admin=is_admin,
)
if row is None:
return f"Knowledge base {mapping_id!r} is not available to the current user."
rows.append(row)
else:
rows = await store.list_visible(user_id, scope="all", is_admin=is_admin)
if not rows:
return "No authorized LLMWiki knowledge bases are available."
try:
results = await build_weknora_client(resolved).search(
query.strip(),
[str(row["weknora_id"]) for row in rows],
)
except (RuntimeError, WeKnoraError) as exc:
return f"LLMWiki retrieval failed: {exc}"
local_by_remote = {str(row["weknora_id"]): row for row in rows}
citations = []
for result in results:
mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
if mapping is None:
continue
params = urlencode(
{
"knowledge_base_id": mapping["id"],
"chunk_id": result.get("chunk_id") or "",
}
)
citations.append(
{
"title": result.get("title") or result.get("filename") or mapping.get("name") or "LLMWiki source",
"content": result.get("content") or "",
"score": result.get("score") or 0,
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"source": mapping.get("name") or "LLMWiki",
"knowledge_id": result.get("knowledge_id") or "",
"chunk_id": result.get("chunk_id") or "",
"url": f"/weknora-source-preview?{params}",
}
)
return json.dumps(
{
"query": query,
"selection": [row["id"] for row in rows],
"knowledge_base_count": len(rows),
"results": citations,
},
ensure_ascii=False,
)
@@ -75,7 +75,11 @@ def get_available_tools(
List of available tools.
"""
config = app_config or get_app_config()
tool_configs = [tool for tool in config.tools if groups is None or tool.group in groups]
selected_tool_configs = [tool for tool in config.tools if groups is None or tool.group in groups]
disabled_tool_names = [tool.name for tool in selected_tool_configs if not tool.enabled]
if disabled_tool_names:
logger.info("Skipping disabled configured tool(s): %s", disabled_tool_names)
tool_configs = [tool for tool in selected_tool_configs if tool.enabled]
browser_context_config = getattr(config, "browser_context", None)
if not getattr(browser_context_config, "enabled", False):
@@ -154,9 +158,21 @@ def get_available_tools(
# read_peer_delivery —— 圆桌席位「按需读取前序席位完整交付」的只读工具(从 run context
# 取网关注入的全文,非圆桌场景为空 → 工具自述「无」)。始终提供(只读、无副作用);圆桌
# 总控/报告角色由 run policy 显式排除,避免干扰它们。
from deerflow.tools.builtins.roundtable_peers_tool import read_peer_delivery_tool
builtin_tools.append(read_peer_delivery_tool)
from deerflow.tools.builtins.roundtable_peers_tool import read_peer_delivery_tool
builtin_tools.append(read_peer_delivery_tool)
# Conditional LLMWiki mode: the tool is absent when no effective WeKnora
# address is configured, keeping legacy deployments behavior-identical.
try:
from deerflow.integrations.weknora.runtime import get_resolved_llmwiki_runtime
if get_resolved_llmwiki_runtime(config).weknora_enabled:
from deerflow.tools.builtins.llmwiki_search_tool import llmwiki_search_tool
builtin_tools.append(llmwiki_search_tool)
except Exception: # noqa: BLE001 - configuration/store failures must not block agent creation
logger.debug("Failed to evaluate the conditional LLMWiki search tool", exc_info=True)
# skill_manage 能创建/修改/删除技能,属于"自我进化"写能力,按 skill_evolution.enabled
# 门控。该开关用户可在 per-user 配置里覆盖,所以按"生效配置(含 per-user 覆盖)"判断 ——
@@ -0,0 +1,229 @@
from __future__ import annotations
import json
import httpx
import pytest
from deerflow.community.configurable_search.tools import (
ConfigurableSearchSettings,
execute_search,
)
from deerflow.config.app_config import AppConfig
from deerflow.config.tool_config import ToolConfig
from deerflow.tools.tools import get_available_tools
def _settings(**overrides: object) -> ConfigurableSearchSettings:
values = {
"enabled": True,
"endpoint": "https://search.example.test/searchName",
"payload": {"query": "configured value", "xxx": "xxxx"},
"verify_ssl": False,
"timeout": 30.0,
"max_results": 5,
"result_url_template": "https://search.example.test/mindId/xxx?recUuid={recUuid}",
}
values.update(overrides)
return ConfigurableSearchSettings(**values) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_configurable_search_overrides_query_and_normalizes_nullable_results() -> None:
requests: list[httpx.Request] = []
async def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
assert request.method == "POST"
assert json.loads(request.content) == {"query": "测试检索词", "xxx": "xxxx"}
return httpx.Response(
200,
json={
"results": [
{
"recUuid": "record/123",
"content": "检索内容",
"title": "标题",
"url": "https://original.example.test/one",
},
{
"recUuid": None,
"content": None,
"title": None,
"url": "https://original.example.test/two",
},
"invalid row",
]
},
)
settings = _settings()
result = json.loads(
await execute_search(
" 测试检索词 ",
settings,
transport=httpx.MockTransport(handler),
)
)
assert len(requests) == 1
assert settings.payload == {"query": "configured value", "xxx": "xxxx"}
assert result == {
"query": "测试检索词",
"total_results": 2,
"results": [
{
"recUuid": "record/123",
"content": "检索内容",
"title": "标题",
"url": "https://search.example.test/mindId/xxx?recUuid=record%2F123",
},
{
"recUuid": "",
"content": "",
"title": "",
"url": "https://original.example.test/two",
},
],
}
@pytest.mark.asyncio
async def test_configurable_search_handles_empty_results() -> None:
async def handler(_: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={"results": []})
result = json.loads(
await execute_search(
"nothing",
_settings(),
transport=httpx.MockTransport(handler),
)
)
assert result == {"query": "nothing", "total_results": 0, "results": []}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("response_factory", "expected_error"),
[
(lambda request: httpx.Response(503, request=request), "Search API returned HTTP 503"),
(
lambda request: httpx.Response(200, content=b"not-json", request=request),
"Search API request failed: JSONDecodeError",
),
(
lambda request: httpx.Response(200, json={"results": None}, request=request),
"search API response field 'results' must be an array",
),
],
)
async def test_configurable_search_returns_safe_errors(
response_factory,
expected_error: str,
) -> None:
async def handler(request: httpx.Request) -> httpx.Response:
return response_factory(request)
result = json.loads(
await execute_search(
"query",
_settings(),
transport=httpx.MockTransport(handler),
)
)
assert result["results"] == []
assert result["total_results"] == 0
assert result["error"] == expected_error
@pytest.mark.asyncio
async def test_configurable_search_handles_timeout() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ReadTimeout("timed out", request=request)
result = json.loads(
await execute_search(
"query",
_settings(),
transport=httpx.MockTransport(handler),
)
)
assert result["error"] == "Search API request timed out"
@pytest.mark.asyncio
async def test_configurable_search_passes_verify_ssl_to_httpx(monkeypatch: pytest.MonkeyPatch) -> None:
captured: dict[str, object] = {}
class FakeAsyncClient:
def __init__(self, **kwargs: object) -> None:
captured.update(kwargs)
async def __aenter__(self):
return self
async def __aexit__(self, *_: object) -> None:
return None
async def post(self, url: str, *, json: dict[str, object]) -> httpx.Response:
request = httpx.Request("POST", url, json=json)
return httpx.Response(200, json={"results": []}, request=request)
monkeypatch.setattr(
"deerflow.community.configurable_search.tools.httpx.AsyncClient",
FakeAsyncClient,
)
await execute_search("query", _settings(verify_ssl=False))
assert captured["verify"] is False
assert captured["timeout"] == 30.0
def test_disabled_configured_tool_is_not_registered() -> None:
config = AppConfig(
sandbox={"use": "deerflow.sandbox.local:LocalSandboxProvider"},
tools=[
ToolConfig(
name="web_search",
group="web",
enabled=False,
use="deerflow.community.configurable_search.tools:web_search_tool",
endpoint="https://search.example.test/searchName",
)
],
)
available = get_available_tools(
groups=["web"],
include_mcp=False,
app_config=config,
)
assert "web_search" not in {item.name for item in available}
config.tools[0].enabled = True
enabled_tools = get_available_tools(
groups=["web"],
include_mcp=False,
app_config=config,
)
assert "web_search" in {item.name for item in enabled_tools}
def test_config_validation_requires_rec_uuid_url_placeholder() -> None:
config = ToolConfig(
name="web_search",
group="web",
use="deerflow.community.configurable_search.tools:web_search_tool",
endpoint="https://search.example.test/searchName",
result_url_template="https://search.example.test/mindId/xxx",
)
with pytest.raises(ValueError, match="must contain"):
ConfigurableSearchSettings.from_tool_config(config)