sync update from LAN repo (source dhk-1 @ 3e87be55)

This commit is contained in:
gozeros committed 2026-10-21 15:06:08 +08:00
1 parent ab6068a7c1
commit 42f2a26fa8
1482 files changed
+227759 -9584

No files matched your search

@@ -12,12 +12,14 @@ import time
from dataclasses import replace
from datetime import UTC, datetime
from io import BytesIO
from types import SimpleNamespace
from typing import Any, Literal
from urllib.parse import quote, unquote, urlencode, urlsplit
import httpx
from fastapi import APIRouter, BackgroundTasks, File, HTTPException, Query, Request, UploadFile
from fastapi.responses import JSONResponse, RedirectResponse, Response, StreamingResponse
from langchain_core.messages import HumanMessage
from pydantic import BaseModel, Field
from app.gateway.csrf_middleware import is_secure_request
@@ -30,6 +32,7 @@ from app.gateway.llmwiki_deposit import (
deposit_conversation_turn,
ensure_conversation_deposit_mapping,
)
from app.gateway.llmwiki_rag import build_llmwiki_rag_context
from app.gateway.weknora_embed import (
WEKNORA_EMBED_API_PREFIX as _WEKNORA_EMBED_API_PREFIX,
)
@@ -51,10 +54,16 @@ from app.gateway.weknora_embed import (
from app.gateway.weknora_embed import (
verify_weknora_signed_payload as _verify_signed_payload,
)
from deerflow.assistant_knowledge.package_io import looks_like_knowledge_package
from deerflow.config.agents_config import load_agent_config
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
from deerflow.config.system_settings import load_system_settings, save_system_settings
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
from deerflow.integrations.weknora.knowledge_profile import (
are_test_questions_customized,
resolved_retrieval_settings,
resolved_test_questions,
)
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
@@ -66,6 +75,10 @@ router = APIRouter(tags=["llmwiki"])
proxy_router = APIRouter(tags=["llmwiki-proxy"])
_MAX_UPLOAD_BYTES = 100 * 1024 * 1024
_KNOWLEDGE_PACKAGE_UPLOAD_DETAIL = (
"检测到知识库 JSONL 数据包。为避免大模型把数据包当普通文档重新识别,"
"请使用“导入 / 导出数据包”中的“上传并导入”;本次文件未送入模型。"
)
_WIKI_FILE_ACCESS_TTL_SECONDS = 24 * 60 * 60
_PROTECTED_WIKI_FILE_PATH = re.compile(
r"^(?:storage://[0-9A-Za-z_-]+/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs)://",
@@ -176,16 +189,34 @@ class LlmWikiConnectionTest(BaseModel):
api_base_url: str | None = None
class WikiGenerationInstructions(BaseModel):
"""User-owned guidance for WeKnora's stable Wiki ingest prompts."""
content_instructions: str | None = Field(default=None, max_length=8000)
extraction_instructions: str | None = Field(default=None, max_length=8000)
class KnowledgeBaseCreate(BaseModel):
name: str = Field(..., min_length=1, max_length=255)
description: str = Field(default="", max_length=2048)
type: Literal["document", "faq"] = "document"
wiki_enabled: bool = False
embedding_model_id: str | None = Field(default=None, min_length=1, max_length=255)
wiki_config: WikiGenerationInstructions | None = None
class KnowledgeRetrievalSettingsUpdate(BaseModel):
entity_expansion_enabled: bool = True
max_related_terms: int = Field(default=6, ge=0, le=12)
statistics_enabled: bool = True
class KnowledgeBaseUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=255)
description: str | None = Field(default=None, max_length=2048)
test_questions: list[str] | None = Field(default=None, max_length=12)
retrieval_settings: KnowledgeRetrievalSettingsUpdate | None = None
wiki_config: WikiGenerationInstructions | None = None
class SearchRequest(BaseModel):
@@ -196,6 +227,14 @@ class SearchRequest(BaseModel):
include_drafts: bool = False
class RetrievalDebugRequest(BaseModel):
"""Run the exact pre-answer knowledge retrieval path without calling an LLM."""
query: str = Field(..., min_length=1, max_length=4000)
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
assistant_knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
class KnowledgeUrlImport(BaseModel):
url: str = Field(..., min_length=1, max_length=4096)
@@ -251,14 +290,16 @@ def _store(request: Request) -> LlmWikiStore:
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
service = getattr(request.app.state, "llmwiki_sync_service", None)
config = request.app.state.config.llmwiki.local_wiki_index
config = getattr(request.app.state, "llmwiki_effective_config", None)
if config is None:
config = request.app.state.config.llmwiki.local_wiki_index
if service is not None and config.enabled and config.auto_sync and mapping.get("wiki_index_enabled", True):
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
from app.gateway.llmwiki_index_scheduler import schedule_local_wiki_mutation_sync
# Event-driven refresh is incremental. Content hash + embedding
# fingerprint decide which pages need encoding; unchanged Wiki pages
# must retain and reuse their existing vectors.
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=False))
# WeKnora parses documents and regenerates Wiki pages asynchronously.
# A debounced retry sequence therefore replaces the old one-shot scan;
# every scan remains metadata-first and lease-deduplicated.
schedule_local_wiki_mutation_sync(request.app, mapping)
async def _actor(request: Request) -> tuple[str, bool]:
@@ -300,6 +341,74 @@ def _raise_upstream(exc: WeKnoraError) -> None:
raise HTTPException(status_code=status, detail=str(exc)) from None
def _embedding_model_view(source: dict[str, Any]) -> dict[str, Any] | None:
"""Return the small, credential-free model shape needed by the create dialog."""
if str(source.get("type") or "").strip().lower() != "embedding":
return None
model_id = str(source.get("id") or "").strip()
if not model_id:
return None
status = str(source.get("status") or "active").strip().lower() or "active"
explicitly_available = source.get("available")
available = (
bool(explicitly_available)
if isinstance(explicitly_available, bool)
else status in {"active", "available", "enabled", "ready"}
)
parameters = source.get("parameters") if isinstance(source.get("parameters"), dict) else {}
def safe_text(*keys: str) -> str:
for key in keys:
value = source.get(key)
if isinstance(value, (str, int, float)) and str(value).strip():
return str(value).strip()
return ""
nested_provider = parameters.get("provider")
provider = safe_text("provider", "provider_type")
if not provider and isinstance(nested_provider, str):
provider = nested_provider.strip()
if not provider:
provider = safe_text("source", "platform")
nested_model = parameters.get("model") or parameters.get("model_name")
model_name = safe_text("model_name", "model")
if not model_name and isinstance(nested_model, str):
model_name = nested_model.strip()
return {
"id": model_id,
"name": safe_text("display_name", "name", "model_name", "model") or model_name or model_id,
"model_name": model_name,
"provider": provider,
"status": status,
"is_default": source.get("is_default") is True,
"available": available,
}
async def _embedding_models(client: WeKnoraClient) -> list[dict[str, Any]]:
return [view for item in await client.list_models() if (view := _embedding_model_view(item)) is not None]
def _select_embedding_model(models: list[dict[str, Any]], requested_id: str | None) -> str:
available = [item for item in models if item["available"]]
if requested_id:
selected = next((item for item in models if item["id"] == requested_id), None)
if selected is None:
raise WeKnoraError("The selected WeKnora embedding model does not exist", status_code=400)
if not selected["available"]:
raise WeKnoraError("The selected WeKnora embedding model is unavailable", status_code=400)
return requested_id
selected = next((item for item in available if item["is_default"]), None) or next(iter(available), None)
if selected is None:
raise WeKnoraError(
"WeKnora has no available embedding model; configure one in WeKnora before creating a knowledge base",
status_code=400,
)
return str(selected["id"])
def _json_response(payload: Any, *, status_code: int = 200) -> Response:
return Response(
content=json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
@@ -309,7 +418,10 @@ def _json_response(payload: Any, *, status_code: int = 200) -> Response:
def _runtime_for_iframe_proxy(request: Request | None = None):
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None) if request is not None else None)
source_config = None
if request is not None:
source_config = getattr(request.app.state, "llmwiki_source_config", None)
runtime = get_resolved_llmwiki_runtime(source_config)
if not runtime.weknora_enabled:
raise HTTPException(status_code=409, detail="WeKnora mode is not enabled")
if not runtime.web_base_url:
@@ -503,6 +615,41 @@ def _should_follow_weknora_binary_redirect(path: str) -> bool:
return (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge"] and segments[4] == "preview") or (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge-bases"] and segments[4] == "files")
def _is_weknora_content_mutation(method: str, path: str) -> bool:
if method.upper() not in {"POST", "PUT", "PATCH", "DELETE"}:
return False
segments = _path_segments(path)
if len(segments) < 3 or segments[:2] != ["api", "v1"]:
return False
return segments[2] in {"knowledge-bases", "knowledgebase", "knowledge", "chunks"}
async def _schedule_proxy_local_wiki_sync(
request: Request,
ctx: WeKnoraEmbedContext,
) -> None:
"""Bridge successful native WeKnora UI writes back to the local mirror."""
try:
mapping = await _store(request).get_authorized(
ctx.mapping_id,
ctx.user_id,
write=False,
is_admin=ctx.is_admin,
)
if mapping is not None and str(mapping.get("weknora_id") or "") == ctx.weknora_id:
_schedule_local_wiki_sync(request, mapping)
except Exception:
# The upstream mutation already succeeded. A scheduling failure must
# not turn that durable write into a false client-side failure; the
# periodic scanner remains the fallback.
logger.warning(
"Could not schedule local Wiki sync after proxied mutation mapping_id=%s",
ctx.mapping_id,
exc_info=True,
)
def _forward_request_headers(request: Request) -> dict[str, str]:
blocked = {
"host",
@@ -620,6 +767,12 @@ async def _authorize_weknora_api_proxy(
raise HTTPException(status_code=403, detail="Creating or deleting other WeKnora knowledge bases is not allowed")
if tail[1] != ctx.weknora_id:
raise HTTPException(status_code=403, detail="Only the current WeKnora knowledge base is allowed")
if (
method == "POST"
and tail[2:] == ["knowledge", "file"]
and looks_like_knowledge_package(body)
):
raise HTTPException(status_code=422, detail=_KNOWLEDGE_PACKAGE_UPLOAD_DETAIL)
if read_method or (write_method and ctx.can_write):
return
@@ -1746,6 +1899,12 @@ async def _proxy_weknora_request(
headers_out.pop("etag", None)
headers_out.pop("ETag", None)
headers_out["Cache-Control"] = "no-store" if is_api or "text/html" in content_type.lower() else "private, max-age=3600"
if (
is_api
and response.status_code < 400
and _is_weknora_content_mutation(request.method, upstream_path)
):
await _schedule_proxy_local_wiki_sync(request, ctx)
return Response(
content=content,
status_code=response.status_code,
@@ -1812,11 +1971,16 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
is_conversation_deposit = _is_conversation_deposit_mapping(row)
is_owner = row.get("owner_user_id") == actor_user_id
description = CONVERSATION_DEPOSIT_KB_DESCRIPTION if is_conversation_deposit else source.get("description") if source.get("description") is not None else row.get("description") or ""
display_name = str(source.get("name") or row.get("name") or "")
profile_row = {**row, "name": display_name}
return {
"id": row["id"],
"remote_status": "available" if remote is not None else "missing",
"name": source.get("name") or row.get("name") or "",
"name": display_name,
"description": description,
"test_questions": resolved_test_questions(profile_row),
"test_questions_customized": are_test_questions_customized(profile_row),
"retrieval_settings": resolved_retrieval_settings(row),
"type": source.get("type") or row.get("kb_type") or "document",
"publication_status": "published" if is_conversation_deposit else row.get("publication_status") or "private",
"wiki_index_enabled": row.get("wiki_index_enabled", True),
@@ -2485,7 +2649,9 @@ async def get_llmwiki_runtime() -> dict[str, Any]:
@router.get("/api/system-settings/llmwiki")
async def get_llmwiki_settings(request: Request) -> dict[str, Any]:
await _require_admin(request)
app_config = request.app.state.config
from deerflow.config import get_app_config
app_config = get_app_config()
runtime = get_resolved_llmwiki_runtime(app_config)
return {
**_runtime_or_legacy(),
@@ -2508,6 +2674,9 @@ async def update_llmwiki_settings(request: Request, body: LlmWikiSettingsUpdate)
raise HTTPException(status_code=422, detail=str(exc)) from None
current = load_system_settings()
save_system_settings(current.model_copy(update={"llmwiki": override}))
from app.gateway.llmwiki_index_runtime import rebuild_local_wiki_index_runtime
await rebuild_local_wiki_index_runtime(request.app, force=True)
return _runtime_or_legacy()
@@ -2516,13 +2685,16 @@ async def reset_llmwiki_settings(request: Request) -> dict[str, Any]:
await _require_admin(request)
current = load_system_settings()
save_system_settings(current.model_copy(update={"llmwiki": LlmWikiRuntimeOverride()}))
from app.gateway.llmwiki_index_runtime import rebuild_local_wiki_index_runtime
await rebuild_local_wiki_index_runtime(request.app, force=True)
return _runtime_or_legacy()
@router.post("/api/system-settings/llmwiki/test")
async def test_llmwiki_connection(request: Request, body: LlmWikiConnectionTest) -> dict[str, Any]:
await _require_admin(request)
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
runtime = get_resolved_llmwiki_runtime()
if body.api_base_url is not None:
try:
candidate = normalize_service_url(body.api_base_url)
@@ -2677,11 +2849,13 @@ def _public_local_index_summary(
state: dict[str, Any] | None,
) -> dict[str, Any]:
app_state = request.app.state
local_config = getattr(
getattr(getattr(app_state, "config", None), "llmwiki", None),
"local_wiki_index",
None,
)
local_config = getattr(app_state, "llmwiki_effective_config", None)
if local_config is None:
local_config = getattr(
getattr(getattr(app_state, "config", None), "llmwiki", None),
"local_wiki_index",
None,
)
enabled = bool(getattr(local_config, "enabled", False))
current = dict(state or {})
state_name = str(current.get("state") or "not_synced")
@@ -2965,7 +3139,7 @@ async def list_knowledge_bases(
client = _client_or_503()
user_id, is_admin = await _actor(request)
try:
remote_items = await client.list_knowledge_bases()
remote_items = await client.list_knowledge_bases(refresh=True)
remote = {str(item.get("id")): item for item in remote_items if item.get("id")}
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -3000,6 +3174,21 @@ async def list_knowledge_bases(
return {"knowledge_bases": knowledge_bases}
@router.get("/api/llmwiki/embedding-models")
async def list_embedding_models(request: Request) -> dict[str, Any]:
"""List selectable WeKnora embedding models without exposing model secrets."""
await _actor(request)
try:
models = await _embedding_models(_client_or_503())
except WeKnoraError as exc:
_raise_upstream(exc)
available = [item for item in models if item["available"]]
default_model = next((item for item in available if item["is_default"]), None) or next(iter(available), None)
default_model_id = str(default_model["id"]) if default_model is not None else None
return {"models": models, "default_model_id": default_model_id}
@router.get("/api/llmwiki/agents/{agent_id}/knowledge-bases")
async def list_agent_knowledge_bases(request: Request, agent_id: str) -> dict[str, Any]:
"""Return the agent-bound bases still visible to the current DeerFlow user."""
@@ -3038,11 +3227,19 @@ async def create_knowledge_base(request: Request, body: KnowledgeBaseCreate) ->
if body.name.strip() == CONVERSATION_DEPOSIT_KB_NAME:
raise HTTPException(status_code=409, detail="对话沉淀是 cmzs 系统保留知识库")
try:
embedding_model_id = _select_embedding_model(
await _embedding_models(client),
body.embedding_model_id,
)
wiki_config = body.wiki_config if body.wiki_enabled else None
remote = await client.create_knowledge_base(
name=body.name.strip(),
description=body.description,
kb_type=body.type,
wiki_enabled=body.wiki_enabled,
embedding_model_id=embedding_model_id,
wiki_content_instructions=(wiki_config.content_instructions or "") if wiki_config else "",
wiki_extraction_instructions=(wiki_config.extraction_instructions or "") if wiki_config else "",
)
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -3087,13 +3284,37 @@ async def update_knowledge_base(request: Request, mapping_id: str, body: Knowled
raise HTTPException(status_code=422, detail="知识库名称不能为空")
if changes["name"] == CONVERSATION_DEPOSIT_KB_NAME:
raise HTTPException(status_code=409, detail="对话沉淀是 cmzs 系统保留知识库")
if not changes:
questions_were_set = "test_questions" in body.model_fields_set
questions = None
if questions_were_set:
questions = list(dict.fromkeys(str(item).strip() for item in (body.test_questions or []) if str(item).strip()))
if any(len(item) > 500 for item in questions):
raise HTTPException(status_code=422, detail="测试问题不能超过 500 个字符")
retrieval_settings = None
if body.retrieval_settings is not None:
retrieval_settings = body.retrieval_settings.model_dump()
remote_changes = {key: value for key, value in changes.items() if key in {"name", "description", "wiki_config"}}
if not changes and not questions_were_set:
return _mapping_view(row, None, actor_user_id=user_id, is_admin=is_admin)
try:
remote = await _client_or_503().update_knowledge_base(str(row["weknora_id"]), changes)
except WeKnoraError as exc:
_raise_upstream(exc)
updated = await _store(request).update_mapping(mapping_id, name=changes.get("name"), description=changes.get("description"))
remote = None
if remote_changes:
try:
remote = await _client_or_503().update_knowledge_base(str(row["weknora_id"]), remote_changes)
except WeKnoraError as exc:
_raise_upstream(exc)
else:
try:
remote = await _client_or_503().get_knowledge_base(str(row["weknora_id"]))
except WeKnoraError as exc:
_raise_upstream(exc)
updated = await _store(request).update_mapping(
mapping_id,
name=remote_changes.get("name"),
description=remote_changes.get("description"),
test_questions=questions,
test_questions_set=questions_were_set,
retrieval_settings=retrieval_settings,
)
return _mapping_view(updated or row, remote, actor_user_id=user_id, is_admin=is_admin)
@@ -3348,6 +3569,8 @@ async def upload_document(request: Request, mapping_id: str, file: UploadFile =
raise HTTPException(status_code=413, detail="File exceeds the 100 MB cmzs upload limit")
if not file.filename:
raise HTTPException(status_code=422, detail="File name is required")
if looks_like_knowledge_package(content):
raise HTTPException(status_code=422, detail=_KNOWLEDGE_PACKAGE_UPLOAD_DETAIL)
try:
result = await _client_or_503().upload_document(
str(row["weknora_id"]),
@@ -3441,6 +3664,44 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
return result
@router.post("/api/llmwiki/retrieval-debug")
async def debug_llmwiki_retrieval(request: Request, body: RetrievalDebugRequest) -> dict[str, Any]:
"""Preview the same bounded evidence that a normal Q&A turn would receive.
The endpoint deliberately calls :func:`build_llmwiki_rag_context` instead
of the vector-only API. Its response therefore exposes the real
vector/keyword/Wiki-page/original-chunk mode, thresholds, warnings and
exact prompt-bounded evidence without starting a model run.
"""
await _require_admin(request)
clean_query = body.query.strip()
if not clean_query:
raise HTTPException(status_code=422, detail="检索问题不能为空")
ordinary_ids = list(dict.fromkeys(item.strip() for item in body.knowledge_base_ids if item.strip()))
assistant_ids = list(
dict.fromkeys(item.strip() for item in body.assistant_knowledge_base_ids if item.strip())
)
if not ordinary_ids and not assistant_ids:
raise HTTPException(status_code=422, detail="请至少选择一个知识空间")
synthetic_body = SimpleNamespace(
assistant_id=None,
metadata=None,
context={
"llmwiki_knowledge_base_ids": ordinary_ids,
"assistant_knowledge_base_ids": assistant_ids,
},
)
rag = await build_llmwiki_rag_context(
request=request,
body=synthetic_body,
graph_input={"messages": [HumanMessage(content=clean_query)]},
)
trace = dict(rag.trace)
trace["trigger"] = "debug"
return trace
@router.post("/api/llmwiki/conversation-deposits/turn", status_code=202)
async def create_conversation_deposit(
request: Request,