fix: harden wiki delivery and leaderboard performance
This commit is contained in:
1 parent
0c5e2e8cb0
commit
9f29ecae9e
28 files changed
+616
-68
No files matched your search
@@ -6,6 +6,7 @@ import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import mimetypes
|
||||
import re
|
||||
import time
|
||||
from dataclasses import replace
|
||||
@@ -47,6 +48,9 @@ from app.gateway.weknora_embed import (
|
||||
from app.gateway.weknora_embed import (
|
||||
verify_weknora_embed_cookie as _verify_embed_cookie,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
verify_weknora_signed_payload as _verify_signed_payload,
|
||||
)
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
|
||||
from deerflow.config.system_settings import load_system_settings, save_system_settings
|
||||
@@ -61,6 +65,7 @@ router = APIRouter(tags=["llmwiki"])
|
||||
proxy_router = APIRouter(tags=["llmwiki-proxy"])
|
||||
|
||||
_MAX_UPLOAD_BYTES = 100 * 1024 * 1024
|
||||
_WIKI_FILE_ACCESS_TTL_SECONDS = 24 * 60 * 60
|
||||
_PROTECTED_WIKI_FILE_PATH = re.compile(
|
||||
r"^(?:storage://[0-9A-Za-z_-]+/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs)://",
|
||||
re.IGNORECASE,
|
||||
@@ -1909,19 +1914,23 @@ def _validate_protected_wiki_file_path(file_path: str) -> str:
|
||||
|
||||
|
||||
async def _wiki_file_response(row: dict[str, Any], file_path: str, *, public: bool) -> Response:
|
||||
safe_file_path = _validate_protected_wiki_file_path(file_path)
|
||||
try:
|
||||
content, media_type = await _client_or_503().get_knowledge_base_file(
|
||||
str(row["weknora_id"]),
|
||||
_validate_protected_wiki_file_path(file_path),
|
||||
safe_file_path,
|
||||
)
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
filename = unquote(safe_file_path.rstrip("/").rsplit("/", 1)[-1]).replace("\r", "").replace("\n", "") or "wiki-file"
|
||||
if "." not in filename:
|
||||
filename += mimetypes.guess_extension(str(media_type).split(";", 1)[0].strip()) or ""
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=media_type,
|
||||
headers={
|
||||
"Cache-Control": "public, max-age=300" if public else "private, max-age=60",
|
||||
"Content-Disposition": "inline",
|
||||
"Cache-Control": "public, max-age=300" if public else "private, max-age=3600",
|
||||
"Content-Disposition": f"inline; filename*=UTF-8''{quote(filename, safe='')}",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
@@ -2629,6 +2638,25 @@ async def get_public_wiki_file(
|
||||
return await _wiki_file_response(row, file_path, public=True)
|
||||
|
||||
|
||||
@router.get("/api/public/llmwiki/files/{token}")
|
||||
async def get_signed_wiki_file(token: str) -> Response:
|
||||
"""Serve a private Wiki image through a short-lived, signed HTTP URL."""
|
||||
|
||||
payload = _verify_signed_payload(token)
|
||||
if payload.get("kind") != "wiki_file":
|
||||
raise HTTPException(status_code=401, detail="Signed Wiki file token is invalid")
|
||||
mapping_id = str(payload.get("mapping_id") or "")
|
||||
weknora_id = str(payload.get("weknora_id") or "")
|
||||
file_path = str(payload.get("file_path") or "")
|
||||
if not mapping_id or not weknora_id or not file_path:
|
||||
raise HTTPException(status_code=401, detail="Signed Wiki file token is invalid")
|
||||
return await _wiki_file_response(
|
||||
{"id": mapping_id, "weknora_id": weknora_id},
|
||||
file_path,
|
||||
public=False,
|
||||
)
|
||||
|
||||
|
||||
_PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison")
|
||||
|
||||
|
||||
@@ -2951,6 +2979,28 @@ async def get_wiki_file(
|
||||
return await _wiki_file_response(row, file_path, public=False)
|
||||
|
||||
|
||||
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/files/access-url")
|
||||
async def create_wiki_file_access(
|
||||
request: Request,
|
||||
mapping_id: str,
|
||||
file_path: str = Query(..., min_length=1, max_length=8192),
|
||||
) -> dict[str, Any]:
|
||||
"""Issue a browser-usable HTTP capability after normal KB authorization."""
|
||||
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
|
||||
safe_file_path = _validate_protected_wiki_file_path(file_path)
|
||||
token = _sign_embed_payload(
|
||||
{
|
||||
"kind": "wiki_file",
|
||||
"mapping_id": mapping_id,
|
||||
"weknora_id": str(row["weknora_id"]),
|
||||
"file_path": safe_file_path,
|
||||
"exp": int(time.time()) + _WIKI_FILE_ACCESS_TTL_SECONDS,
|
||||
}
|
||||
)
|
||||
return {"token": token, "expires_in": _WIKI_FILE_ACCESS_TTL_SECONDS}
|
||||
|
||||
|
||||
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph")
|
||||
async def get_knowledge_base_graph(
|
||||
request: Request,
|
||||
|
||||
Reference in new issue
Block a user