fix: harden wiki delivery and leaderboard performance
This commit is contained in:
1 parent
0c5e2e8cb0
commit
9f29ecae9e
28 files changed
+616
-68
No files matched your search
@@ -637,10 +637,16 @@ def _non_scheduled_run_filter():
|
||||
|
||||
|
||||
def _non_scheduled_tool_filter():
|
||||
return or_(
|
||||
ToolCallMetricRow.run_id.is_(None),
|
||||
RunRow.run_id.is_(None),
|
||||
_non_scheduled_run_filter(),
|
||||
"""Exclude legacy scheduled runs without joining runs into the metric scan."""
|
||||
|
||||
return ~(
|
||||
select(1)
|
||||
.select_from(RunRow)
|
||||
.where(
|
||||
RunRow.run_id == ToolCallMetricRow.run_id,
|
||||
RunRow.first_human_message.like("【定时任务触发】%"),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
|
||||
|
||||
@@ -667,8 +673,15 @@ def _admin_user_ids_stmt():
|
||||
return select(UserRow.id).where(UserRow.system_role == "admin")
|
||||
|
||||
|
||||
def _scheduled_run_ids_stmt():
|
||||
return select(ScheduledTaskRunRow.agent_run_id).where(ScheduledTaskRunRow.agent_run_id.is_not(None))
|
||||
def _not_scheduled_run_id_filter(run_id_column):
|
||||
"""Use a NULL-safe, index-friendly anti-semi join instead of NOT IN."""
|
||||
|
||||
return ~(
|
||||
select(1)
|
||||
.select_from(ScheduledTaskRunRow)
|
||||
.where(ScheduledTaskRunRow.agent_run_id == run_id_column)
|
||||
.exists()
|
||||
)
|
||||
|
||||
|
||||
def _apply_user_scope(stmt, column, *, exclude_user_ids: set[str], include_admins: bool):
|
||||
@@ -692,7 +705,7 @@ def _apply_run_scope(
|
||||
if not include_failed:
|
||||
stmt = stmt.where(RunRow.status != "error")
|
||||
if not include_scheduled:
|
||||
stmt = stmt.where(or_(run_id_column.is_(None), run_id_column.not_in(_scheduled_run_ids_stmt())))
|
||||
stmt = stmt.where(_not_scheduled_run_id_filter(run_id_column))
|
||||
stmt = stmt.where(_non_scheduled_run_filter())
|
||||
return stmt
|
||||
|
||||
@@ -706,8 +719,8 @@ def _apply_tool_scope(
|
||||
):
|
||||
stmt = _apply_user_scope(stmt, ToolCallMetricRow.user_id, exclude_user_ids=exclude_user_ids, include_admins=include_admins)
|
||||
if not include_scheduled:
|
||||
stmt = stmt.where(or_(ToolCallMetricRow.run_id.is_(None), ToolCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
|
||||
stmt = stmt.outerjoin(RunRow, RunRow.run_id == ToolCallMetricRow.run_id).where(_non_scheduled_tool_filter())
|
||||
stmt = stmt.where(_not_scheduled_run_id_filter(ToolCallMetricRow.run_id))
|
||||
stmt = stmt.where(_non_scheduled_tool_filter())
|
||||
return stmt
|
||||
|
||||
|
||||
@@ -985,7 +998,7 @@ async def _query_model_rankings(
|
||||
if not include_failed:
|
||||
stmt = stmt.where(LlmCallMetricRow.status != "error")
|
||||
if not include_scheduled:
|
||||
stmt = stmt.where(or_(LlmCallMetricRow.run_id.is_(None), LlmCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
|
||||
stmt = stmt.where(_not_scheduled_run_id_filter(LlmCallMetricRow.run_id))
|
||||
rows = (await _execute_leaderboard_query(session, "model_rankings", stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardDimension(
|
||||
@@ -1080,8 +1093,8 @@ async def _query_overview(
|
||||
if not include_failed:
|
||||
run_where.append(RunRow.status != "error")
|
||||
if not include_scheduled:
|
||||
run_where.append(or_(RunRow.run_id.is_(None), RunRow.run_id.not_in(_scheduled_run_ids_stmt())))
|
||||
tool_where.append(or_(ToolCallMetricRow.run_id.is_(None), ToolCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
|
||||
run_where.append(_not_scheduled_run_id_filter(RunRow.run_id))
|
||||
tool_where.append(_not_scheduled_run_id_filter(ToolCallMetricRow.run_id))
|
||||
tool_where.append(_non_scheduled_tool_filter())
|
||||
|
||||
total_users_stmt = select(func.count(UserRow.id))
|
||||
@@ -1104,7 +1117,6 @@ async def _query_overview(
|
||||
func.count(ToolCallMetricRow.id),
|
||||
func.sum(case((func.nullif(ToolCallMetricRow.skill_name, "").is_not(None), 1), else_=0)),
|
||||
)
|
||||
.outerjoin(RunRow, RunRow.run_id == ToolCallMetricRow.run_id)
|
||||
.where(*tool_where)
|
||||
)
|
||||
tool_row = (await _execute_leaderboard_query(session, "overview.tool_summary", tool_summary_stmt)).one()
|
||||
|
||||
@@ -6,6 +6,7 @@ import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import mimetypes
|
||||
import re
|
||||
import time
|
||||
from dataclasses import replace
|
||||
@@ -47,6 +48,9 @@ from app.gateway.weknora_embed import (
|
||||
from app.gateway.weknora_embed import (
|
||||
verify_weknora_embed_cookie as _verify_embed_cookie,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
verify_weknora_signed_payload as _verify_signed_payload,
|
||||
)
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
|
||||
from deerflow.config.system_settings import load_system_settings, save_system_settings
|
||||
@@ -61,6 +65,7 @@ router = APIRouter(tags=["llmwiki"])
|
||||
proxy_router = APIRouter(tags=["llmwiki-proxy"])
|
||||
|
||||
_MAX_UPLOAD_BYTES = 100 * 1024 * 1024
|
||||
_WIKI_FILE_ACCESS_TTL_SECONDS = 24 * 60 * 60
|
||||
_PROTECTED_WIKI_FILE_PATH = re.compile(
|
||||
r"^(?:storage://[0-9A-Za-z_-]+/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs)://",
|
||||
re.IGNORECASE,
|
||||
@@ -1909,19 +1914,23 @@ def _validate_protected_wiki_file_path(file_path: str) -> str:
|
||||
|
||||
|
||||
async def _wiki_file_response(row: dict[str, Any], file_path: str, *, public: bool) -> Response:
|
||||
safe_file_path = _validate_protected_wiki_file_path(file_path)
|
||||
try:
|
||||
content, media_type = await _client_or_503().get_knowledge_base_file(
|
||||
str(row["weknora_id"]),
|
||||
_validate_protected_wiki_file_path(file_path),
|
||||
safe_file_path,
|
||||
)
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
filename = unquote(safe_file_path.rstrip("/").rsplit("/", 1)[-1]).replace("\r", "").replace("\n", "") or "wiki-file"
|
||||
if "." not in filename:
|
||||
filename += mimetypes.guess_extension(str(media_type).split(";", 1)[0].strip()) or ""
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=media_type,
|
||||
headers={
|
||||
"Cache-Control": "public, max-age=300" if public else "private, max-age=60",
|
||||
"Content-Disposition": "inline",
|
||||
"Cache-Control": "public, max-age=300" if public else "private, max-age=3600",
|
||||
"Content-Disposition": f"inline; filename*=UTF-8''{quote(filename, safe='')}",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
@@ -2629,6 +2638,25 @@ async def get_public_wiki_file(
|
||||
return await _wiki_file_response(row, file_path, public=True)
|
||||
|
||||
|
||||
@router.get("/api/public/llmwiki/files/{token}")
|
||||
async def get_signed_wiki_file(token: str) -> Response:
|
||||
"""Serve a private Wiki image through a short-lived, signed HTTP URL."""
|
||||
|
||||
payload = _verify_signed_payload(token)
|
||||
if payload.get("kind") != "wiki_file":
|
||||
raise HTTPException(status_code=401, detail="Signed Wiki file token is invalid")
|
||||
mapping_id = str(payload.get("mapping_id") or "")
|
||||
weknora_id = str(payload.get("weknora_id") or "")
|
||||
file_path = str(payload.get("file_path") or "")
|
||||
if not mapping_id or not weknora_id or not file_path:
|
||||
raise HTTPException(status_code=401, detail="Signed Wiki file token is invalid")
|
||||
return await _wiki_file_response(
|
||||
{"id": mapping_id, "weknora_id": weknora_id},
|
||||
file_path,
|
||||
public=False,
|
||||
)
|
||||
|
||||
|
||||
_PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison")
|
||||
|
||||
|
||||
@@ -2951,6 +2979,28 @@ async def get_wiki_file(
|
||||
return await _wiki_file_response(row, file_path, public=False)
|
||||
|
||||
|
||||
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/files/access-url")
|
||||
async def create_wiki_file_access(
|
||||
request: Request,
|
||||
mapping_id: str,
|
||||
file_path: str = Query(..., min_length=1, max_length=8192),
|
||||
) -> dict[str, Any]:
|
||||
"""Issue a browser-usable HTTP capability after normal KB authorization."""
|
||||
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
|
||||
safe_file_path = _validate_protected_wiki_file_path(file_path)
|
||||
token = _sign_embed_payload(
|
||||
{
|
||||
"kind": "wiki_file",
|
||||
"mapping_id": mapping_id,
|
||||
"weknora_id": str(row["weknora_id"]),
|
||||
"file_path": safe_file_path,
|
||||
"exp": int(time.time()) + _WIKI_FILE_ACCESS_TTL_SECONDS,
|
||||
}
|
||||
)
|
||||
return {"token": token, "expires_in": _WIKI_FILE_ACCESS_TTL_SECONDS}
|
||||
|
||||
|
||||
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph")
|
||||
async def get_knowledge_base_graph(
|
||||
request: Request,
|
||||
|
||||
@@ -245,6 +245,33 @@ async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) ->
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
|
||||
states = await index_store.list_index_status([mapping_id])
|
||||
state = states[0] if states else {"knowledge_base_mapping_id": mapping_id, "state": "not_synced", "index_revision": 0}
|
||||
if (
|
||||
state.get("state") == "idle"
|
||||
and int(state.get("index_revision") or 0) == 0
|
||||
and not state.get("last_completed_at")
|
||||
and not state.get("last_success_at")
|
||||
):
|
||||
state = {**state, "state": "not_synced"}
|
||||
pages = await index_store.list_page_index_status(mapping_id)
|
||||
active_sync_id = str(state.get("active_sync_id") or "")
|
||||
is_running = str(state.get("state") or "") in {"running", "rebuilding"}
|
||||
for page in pages:
|
||||
page["vectorized"] = page.get("index_status") == "ready" and int(page.get("vector_count") or 0) > 0
|
||||
page["processed_in_current_sync"] = bool(active_sync_id) and page.get("last_seen_sync_id") == active_sync_id
|
||||
total_pages = max(int(state.get("remote_page_count") or 0), len(pages))
|
||||
processed_pages = (
|
||||
sum(bool(page["processed_in_current_sync"]) for page in pages)
|
||||
if is_running
|
||||
else total_pages if str(state.get("state") or "") in {"idle", "partial"} else 0
|
||||
)
|
||||
completion_percent = (
|
||||
min(100, round((processed_pages / total_pages) * 100))
|
||||
if total_pages
|
||||
else 100
|
||||
if str(state.get("state") or "") in {"idle", "partial"}
|
||||
and bool(state.get("last_completed_at") or state.get("last_success_at"))
|
||||
else 0
|
||||
)
|
||||
embedding = getattr(request.app.state, "llmwiki_embedding", None)
|
||||
fingerprint = embedding.fingerprint if embedding is not None else None
|
||||
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(state, fingerprint)
|
||||
@@ -255,6 +282,15 @@ async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) ->
|
||||
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
|
||||
"fully_vectorized": fully_vectorized,
|
||||
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
|
||||
"progress": {
|
||||
"total_pages": total_pages,
|
||||
"processed_pages": processed_pages,
|
||||
"vectorized_pages": sum(bool(page["vectorized"]) for page in pages),
|
||||
"failed_pages": sum(bool(page.get("last_error")) or page.get("index_status") in {"failed", "stale"} for page in pages),
|
||||
"vector_count": sum(int(page.get("vector_count") or 0) for page in pages),
|
||||
"completion_percent": completion_percent,
|
||||
},
|
||||
"pages": pages,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -106,6 +106,26 @@ def sign_weknora_embed_payload(payload: dict[str, Any]) -> str:
|
||||
return f"{body}.{_b64url_encode(sig)}"
|
||||
|
||||
|
||||
def verify_weknora_signed_payload(raw: str | None) -> dict[str, Any]:
|
||||
"""Verify a short-lived capability signed with the iframe proxy secret."""
|
||||
|
||||
if not raw or "." not in raw:
|
||||
raise HTTPException(status_code=401, detail="Signed resource token is missing")
|
||||
body, sig = raw.rsplit(".", 1)
|
||||
expected = _b64url_encode(hmac.new(WEKNORA_PROXY_SECRET.encode("utf-8"), body.encode("ascii"), hashlib.sha256).digest())
|
||||
if not hmac.compare_digest(sig, expected):
|
||||
raise HTTPException(status_code=401, detail="Signed resource token is invalid")
|
||||
try:
|
||||
payload = json.loads(_b64url_decode(body))
|
||||
except (ValueError, TypeError, json.JSONDecodeError):
|
||||
raise HTTPException(status_code=401, detail="Signed resource token is invalid") from None
|
||||
if not isinstance(payload, dict):
|
||||
raise HTTPException(status_code=401, detail="Signed resource token is invalid")
|
||||
if int(payload.get("exp") or 0) < int(time.time()):
|
||||
raise HTTPException(status_code=401, detail="Signed resource token has expired")
|
||||
return payload
|
||||
|
||||
|
||||
def verify_weknora_embed_cookie(raw: str | None) -> WeKnoraEmbedContext:
|
||||
if not raw or "." not in raw:
|
||||
raise HTTPException(status_code=401, detail="WeKnora iframe session is missing")
|
||||
|
||||
Reference in new issue
Block a user