fix: harden wiki delivery and leaderboard performance

This commit is contained in:
吉鑫升 committed 2026-08-25 16:57:42 +08:00
1 parent 0c5e2e8cb0
commit 9f29ecae9e
28 files changed
+616 -68

No files matched your search

@@ -637,10 +637,16 @@ def _non_scheduled_run_filter():
def _non_scheduled_tool_filter():
return or_(
ToolCallMetricRow.run_id.is_(None),
RunRow.run_id.is_(None),
_non_scheduled_run_filter(),
"""Exclude legacy scheduled runs without joining runs into the metric scan."""
return ~(
select(1)
.select_from(RunRow)
.where(
RunRow.run_id == ToolCallMetricRow.run_id,
RunRow.first_human_message.like("【定时任务触发】%"),
)
.exists()
)
@@ -667,8 +673,15 @@ def _admin_user_ids_stmt():
return select(UserRow.id).where(UserRow.system_role == "admin")
def _scheduled_run_ids_stmt():
return select(ScheduledTaskRunRow.agent_run_id).where(ScheduledTaskRunRow.agent_run_id.is_not(None))
def _not_scheduled_run_id_filter(run_id_column):
"""Use a NULL-safe, index-friendly anti-semi join instead of NOT IN."""
return ~(
select(1)
.select_from(ScheduledTaskRunRow)
.where(ScheduledTaskRunRow.agent_run_id == run_id_column)
.exists()
)
def _apply_user_scope(stmt, column, *, exclude_user_ids: set[str], include_admins: bool):
@@ -692,7 +705,7 @@ def _apply_run_scope(
if not include_failed:
stmt = stmt.where(RunRow.status != "error")
if not include_scheduled:
stmt = stmt.where(or_(run_id_column.is_(None), run_id_column.not_in(_scheduled_run_ids_stmt())))
stmt = stmt.where(_not_scheduled_run_id_filter(run_id_column))
stmt = stmt.where(_non_scheduled_run_filter())
return stmt
@@ -706,8 +719,8 @@ def _apply_tool_scope(
):
stmt = _apply_user_scope(stmt, ToolCallMetricRow.user_id, exclude_user_ids=exclude_user_ids, include_admins=include_admins)
if not include_scheduled:
stmt = stmt.where(or_(ToolCallMetricRow.run_id.is_(None), ToolCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
stmt = stmt.outerjoin(RunRow, RunRow.run_id == ToolCallMetricRow.run_id).where(_non_scheduled_tool_filter())
stmt = stmt.where(_not_scheduled_run_id_filter(ToolCallMetricRow.run_id))
stmt = stmt.where(_non_scheduled_tool_filter())
return stmt
@@ -985,7 +998,7 @@ async def _query_model_rankings(
if not include_failed:
stmt = stmt.where(LlmCallMetricRow.status != "error")
if not include_scheduled:
stmt = stmt.where(or_(LlmCallMetricRow.run_id.is_(None), LlmCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
stmt = stmt.where(_not_scheduled_run_id_filter(LlmCallMetricRow.run_id))
rows = (await _execute_leaderboard_query(session, "model_rankings", stmt)).all()
return [
AdminLeaderboardDimension(
@@ -1080,8 +1093,8 @@ async def _query_overview(
if not include_failed:
run_where.append(RunRow.status != "error")
if not include_scheduled:
run_where.append(or_(RunRow.run_id.is_(None), RunRow.run_id.not_in(_scheduled_run_ids_stmt())))
tool_where.append(or_(ToolCallMetricRow.run_id.is_(None), ToolCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
run_where.append(_not_scheduled_run_id_filter(RunRow.run_id))
tool_where.append(_not_scheduled_run_id_filter(ToolCallMetricRow.run_id))
tool_where.append(_non_scheduled_tool_filter())
total_users_stmt = select(func.count(UserRow.id))
@@ -1104,7 +1117,6 @@ async def _query_overview(
func.count(ToolCallMetricRow.id),
func.sum(case((func.nullif(ToolCallMetricRow.skill_name, "").is_not(None), 1), else_=0)),
)
.outerjoin(RunRow, RunRow.run_id == ToolCallMetricRow.run_id)
.where(*tool_where)
)
tool_row = (await _execute_leaderboard_query(session, "overview.tool_summary", tool_summary_stmt)).one()
@@ -6,6 +6,7 @@ import asyncio
import hashlib
import json
import logging
import mimetypes
import re
import time
from dataclasses import replace
@@ -47,6 +48,9 @@ from app.gateway.weknora_embed import (
from app.gateway.weknora_embed import (
verify_weknora_embed_cookie as _verify_embed_cookie,
)
from app.gateway.weknora_embed import (
verify_weknora_signed_payload as _verify_signed_payload,
)
from deerflow.config.agents_config import load_agent_config
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
from deerflow.config.system_settings import load_system_settings, save_system_settings
@@ -61,6 +65,7 @@ router = APIRouter(tags=["llmwiki"])
proxy_router = APIRouter(tags=["llmwiki-proxy"])
_MAX_UPLOAD_BYTES = 100 * 1024 * 1024
_WIKI_FILE_ACCESS_TTL_SECONDS = 24 * 60 * 60
_PROTECTED_WIKI_FILE_PATH = re.compile(
r"^(?:storage://[0-9A-Za-z_-]+/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs)://",
re.IGNORECASE,
@@ -1909,19 +1914,23 @@ def _validate_protected_wiki_file_path(file_path: str) -> str:
async def _wiki_file_response(row: dict[str, Any], file_path: str, *, public: bool) -> Response:
safe_file_path = _validate_protected_wiki_file_path(file_path)
try:
content, media_type = await _client_or_503().get_knowledge_base_file(
str(row["weknora_id"]),
_validate_protected_wiki_file_path(file_path),
safe_file_path,
)
except WeKnoraError as exc:
_raise_upstream(exc)
filename = unquote(safe_file_path.rstrip("/").rsplit("/", 1)[-1]).replace("\r", "").replace("\n", "") or "wiki-file"
if "." not in filename:
filename += mimetypes.guess_extension(str(media_type).split(";", 1)[0].strip()) or ""
return Response(
content=content,
media_type=media_type,
headers={
"Cache-Control": "public, max-age=300" if public else "private, max-age=60",
"Content-Disposition": "inline",
"Cache-Control": "public, max-age=300" if public else "private, max-age=3600",
"Content-Disposition": f"inline; filename*=UTF-8''{quote(filename, safe='')}",
"X-Content-Type-Options": "nosniff",
},
)
@@ -2629,6 +2638,25 @@ async def get_public_wiki_file(
return await _wiki_file_response(row, file_path, public=True)
@router.get("/api/public/llmwiki/files/{token}")
async def get_signed_wiki_file(token: str) -> Response:
"""Serve a private Wiki image through a short-lived, signed HTTP URL."""
payload = _verify_signed_payload(token)
if payload.get("kind") != "wiki_file":
raise HTTPException(status_code=401, detail="Signed Wiki file token is invalid")
mapping_id = str(payload.get("mapping_id") or "")
weknora_id = str(payload.get("weknora_id") or "")
file_path = str(payload.get("file_path") or "")
if not mapping_id or not weknora_id or not file_path:
raise HTTPException(status_code=401, detail="Signed Wiki file token is invalid")
return await _wiki_file_response(
{"id": mapping_id, "weknora_id": weknora_id},
file_path,
public=False,
)
_PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison")
@@ -2951,6 +2979,28 @@ async def get_wiki_file(
return await _wiki_file_response(row, file_path, public=False)
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/files/access-url")
async def create_wiki_file_access(
request: Request,
mapping_id: str,
file_path: str = Query(..., min_length=1, max_length=8192),
) -> dict[str, Any]:
"""Issue a browser-usable HTTP capability after normal KB authorization."""
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
safe_file_path = _validate_protected_wiki_file_path(file_path)
token = _sign_embed_payload(
{
"kind": "wiki_file",
"mapping_id": mapping_id,
"weknora_id": str(row["weknora_id"]),
"file_path": safe_file_path,
"exp": int(time.time()) + _WIKI_FILE_ACCESS_TTL_SECONDS,
}
)
return {"token": token, "expires_in": _WIKI_FILE_ACCESS_TTL_SECONDS}
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph")
async def get_knowledge_base_graph(
request: Request,
@@ -245,6 +245,33 @@ async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) ->
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
states = await index_store.list_index_status([mapping_id])
state = states[0] if states else {"knowledge_base_mapping_id": mapping_id, "state": "not_synced", "index_revision": 0}
if (
state.get("state") == "idle"
and int(state.get("index_revision") or 0) == 0
and not state.get("last_completed_at")
and not state.get("last_success_at")
):
state = {**state, "state": "not_synced"}
pages = await index_store.list_page_index_status(mapping_id)
active_sync_id = str(state.get("active_sync_id") or "")
is_running = str(state.get("state") or "") in {"running", "rebuilding"}
for page in pages:
page["vectorized"] = page.get("index_status") == "ready" and int(page.get("vector_count") or 0) > 0
page["processed_in_current_sync"] = bool(active_sync_id) and page.get("last_seen_sync_id") == active_sync_id
total_pages = max(int(state.get("remote_page_count") or 0), len(pages))
processed_pages = (
sum(bool(page["processed_in_current_sync"]) for page in pages)
if is_running
else total_pages if str(state.get("state") or "") in {"idle", "partial"} else 0
)
completion_percent = (
min(100, round((processed_pages / total_pages) * 100))
if total_pages
else 100
if str(state.get("state") or "") in {"idle", "partial"}
and bool(state.get("last_completed_at") or state.get("last_success_at"))
else 0
)
embedding = getattr(request.app.state, "llmwiki_embedding", None)
fingerprint = embedding.fingerprint if embedding is not None else None
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(state, fingerprint)
@@ -255,6 +282,15 @@ async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) ->
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
"fully_vectorized": fully_vectorized,
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
"progress": {
"total_pages": total_pages,
"processed_pages": processed_pages,
"vectorized_pages": sum(bool(page["vectorized"]) for page in pages),
"failed_pages": sum(bool(page.get("last_error")) or page.get("index_status") in {"failed", "stale"} for page in pages),
"vector_count": sum(int(page.get("vector_count") or 0) for page in pages),
"completion_percent": completion_percent,
},
"pages": pages,
}
@@ -106,6 +106,26 @@ def sign_weknora_embed_payload(payload: dict[str, Any]) -> str:
return f"{body}.{_b64url_encode(sig)}"
def verify_weknora_signed_payload(raw: str | None) -> dict[str, Any]:
"""Verify a short-lived capability signed with the iframe proxy secret."""
if not raw or "." not in raw:
raise HTTPException(status_code=401, detail="Signed resource token is missing")
body, sig = raw.rsplit(".", 1)
expected = _b64url_encode(hmac.new(WEKNORA_PROXY_SECRET.encode("utf-8"), body.encode("ascii"), hashlib.sha256).digest())
if not hmac.compare_digest(sig, expected):
raise HTTPException(status_code=401, detail="Signed resource token is invalid")
try:
payload = json.loads(_b64url_decode(body))
except (ValueError, TypeError, json.JSONDecodeError):
raise HTTPException(status_code=401, detail="Signed resource token is invalid") from None
if not isinstance(payload, dict):
raise HTTPException(status_code=401, detail="Signed resource token is invalid")
if int(payload.get("exp") or 0) < int(time.time()):
raise HTTPException(status_code=401, detail="Signed resource token has expired")
return payload
def verify_weknora_embed_cookie(raw: str | None) -> WeKnoraEmbedContext:
if not raw or "." not in raw:
raise HTTPException(status_code=401, detail="WeKnora iframe session is missing")