This commit is contained in:
董宏凯 committed 2026-08-25 17:33:40 +08:00
commit 5c0864bf0a
40 files changed
+1200 -179

No files matched your search

@@ -637,10 +637,16 @@ def _non_scheduled_run_filter():
def _non_scheduled_tool_filter():
return or_(
ToolCallMetricRow.run_id.is_(None),
RunRow.run_id.is_(None),
_non_scheduled_run_filter(),
"""Exclude legacy scheduled runs without joining runs into the metric scan."""
return ~(
select(1)
.select_from(RunRow)
.where(
RunRow.run_id == ToolCallMetricRow.run_id,
RunRow.first_human_message.like("【定时任务触发】%"),
)
.exists()
)
@@ -667,8 +673,15 @@ def _admin_user_ids_stmt():
return select(UserRow.id).where(UserRow.system_role == "admin")
def _scheduled_run_ids_stmt():
return select(ScheduledTaskRunRow.agent_run_id).where(ScheduledTaskRunRow.agent_run_id.is_not(None))
def _not_scheduled_run_id_filter(run_id_column):
"""Use a NULL-safe, index-friendly anti-semi join instead of NOT IN."""
return ~(
select(1)
.select_from(ScheduledTaskRunRow)
.where(ScheduledTaskRunRow.agent_run_id == run_id_column)
.exists()
)
def _apply_user_scope(stmt, column, *, exclude_user_ids: set[str], include_admins: bool):
@@ -692,7 +705,7 @@ def _apply_run_scope(
if not include_failed:
stmt = stmt.where(RunRow.status != "error")
if not include_scheduled:
stmt = stmt.where(or_(run_id_column.is_(None), run_id_column.not_in(_scheduled_run_ids_stmt())))
stmt = stmt.where(_not_scheduled_run_id_filter(run_id_column))
stmt = stmt.where(_non_scheduled_run_filter())
return stmt
@@ -706,8 +719,8 @@ def _apply_tool_scope(
):
stmt = _apply_user_scope(stmt, ToolCallMetricRow.user_id, exclude_user_ids=exclude_user_ids, include_admins=include_admins)
if not include_scheduled:
stmt = stmt.where(or_(ToolCallMetricRow.run_id.is_(None), ToolCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
stmt = stmt.outerjoin(RunRow, RunRow.run_id == ToolCallMetricRow.run_id).where(_non_scheduled_tool_filter())
stmt = stmt.where(_not_scheduled_run_id_filter(ToolCallMetricRow.run_id))
stmt = stmt.where(_non_scheduled_tool_filter())
return stmt
@@ -985,7 +998,7 @@ async def _query_model_rankings(
if not include_failed:
stmt = stmt.where(LlmCallMetricRow.status != "error")
if not include_scheduled:
stmt = stmt.where(or_(LlmCallMetricRow.run_id.is_(None), LlmCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
stmt = stmt.where(_not_scheduled_run_id_filter(LlmCallMetricRow.run_id))
rows = (await _execute_leaderboard_query(session, "model_rankings", stmt)).all()
return [
AdminLeaderboardDimension(
@@ -1080,8 +1093,8 @@ async def _query_overview(
if not include_failed:
run_where.append(RunRow.status != "error")
if not include_scheduled:
run_where.append(or_(RunRow.run_id.is_(None), RunRow.run_id.not_in(_scheduled_run_ids_stmt())))
tool_where.append(or_(ToolCallMetricRow.run_id.is_(None), ToolCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
run_where.append(_not_scheduled_run_id_filter(RunRow.run_id))
tool_where.append(_not_scheduled_run_id_filter(ToolCallMetricRow.run_id))
tool_where.append(_non_scheduled_tool_filter())
total_users_stmt = select(func.count(UserRow.id))
@@ -1104,7 +1117,6 @@ async def _query_overview(
func.count(ToolCallMetricRow.id),
func.sum(case((func.nullif(ToolCallMetricRow.skill_name, "").is_not(None), 1), else_=0)),
)
.outerjoin(RunRow, RunRow.run_id == ToolCallMetricRow.run_id)
.where(*tool_where)
)
tool_row = (await _execute_leaderboard_query(session, "overview.tool_summary", tool_summary_stmt)).one()
@@ -6,6 +6,7 @@ import asyncio
import hashlib
import json
import logging
import mimetypes
import re
import time
from dataclasses import replace
@@ -47,6 +48,9 @@ from app.gateway.weknora_embed import (
from app.gateway.weknora_embed import (
verify_weknora_embed_cookie as _verify_embed_cookie,
)
from app.gateway.weknora_embed import (
verify_weknora_signed_payload as _verify_signed_payload,
)
from deerflow.config.agents_config import load_agent_config
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
from deerflow.config.system_settings import load_system_settings, save_system_settings
@@ -61,6 +65,11 @@ router = APIRouter(tags=["llmwiki"])
proxy_router = APIRouter(tags=["llmwiki-proxy"])
_MAX_UPLOAD_BYTES = 100 * 1024 * 1024
_WIKI_FILE_ACCESS_TTL_SECONDS = 24 * 60 * 60
_PROTECTED_WIKI_FILE_PATH = re.compile(
r"^(?:storage://[0-9A-Za-z_-]+/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs)://",
re.IGNORECASE,
)
_WEKNORA_ADMIN_SESSION_CACHE: dict[str, dict[str, Any]] = {}
_WEKNORA_PUBLIC_PREFIX = "/deerflow"
_WEKNORA_PUBLIC_ROOT_PATHS = (
@@ -242,7 +251,7 @@ def _store(request: Request) -> LlmWikiStore:
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
service = getattr(request.app.state, "llmwiki_sync_service", None)
config = request.app.state.config.llmwiki.local_wiki_index
if service is not None and config.enabled and mapping.get("wiki_index_enabled", True):
if service is not None and config.enabled and config.auto_sync and mapping.get("wiki_index_enabled", True):
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
@@ -1085,6 +1094,8 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
function rewriteApiUrl(value) {{
try {{
const protectedUrl = protectedImageApiUrl(value);
if (protectedUrl) return protectedUrl;
const next = new URL(value, location.origin);
if (next.origin !== location.origin) return value;
const publicPrefix = cfg.publicPrefix || "";
@@ -1104,6 +1115,19 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
return value;
}}
}}
function protectedImageApiUrl(value) {{
const raw = String(value || "").trim();
if (!/^(?:storage:\\/\\/[0-9A-Za-z_-]+\\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\\/\\//i.test(raw)) {{
return "";
}}
const endpoint =
cfg.apiPrefix +
"/api/v1/knowledge-bases/" +
encodeURIComponent(cfg.weknoraId) +
"/files?file_path=" +
encodeURIComponent(raw);
return new URL(endpoint, location.origin).href;
}}
const nativeFetch = window.fetch.bind(window);
window.fetch = (input, init) => {{
if (input instanceof Request) {{
@@ -1116,6 +1140,67 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
return nativeXhrOpen.call(this, method, rewriteApiUrl(url), ...rest);
}};
// WeKnora renders protected Markdown images as transparent placeholders and
// hydrates them asynchronously. In an embedded page that hydration can run
// before its own API client has picked up our proxy prefix, leaving a 1x1
// GIF forever. Resolve the same protected source through the scoped proxy
// as a fallback. The backend validates the scheme and current KB again.
function hydrateProtectedImage(image) {{
const protectedSrc = String(image?.getAttribute?.("data-protected-src") || "").trim();
const proxyUrl = protectedImageApiUrl(protectedSrc);
if (!proxyUrl) return;
if (
image.dataset.deerflowProtectedSource === protectedSrc &&
String(image.getAttribute("src") || "").startsWith("blob:")
) return;
const attempts = Number(image.dataset.deerflowProtectedAttempts || "0");
if (attempts >= 3 || image.dataset.deerflowProtectedLoading === "1") return;
image.dataset.deerflowProtectedLoading = "1";
image.dataset.deerflowProtectedAttempts = String(attempts + 1);
nativeFetch(proxyUrl, {{ credentials: "same-origin", headers: {{ Accept: "image/*" }} }})
.then((response) => {{
if (!response.ok) throw new Error(`protected image request failed (${{response.status}})`);
return response.blob();
}})
.then((blob) => {{
const objectUrl = URL.createObjectURL(blob);
image.dataset.deerflowProtectedSource = protectedSrc;
image.removeAttribute("data-img-loading");
image.removeAttribute("data-wiki-image-error");
image.src = objectUrl;
image.addEventListener("load", () => URL.revokeObjectURL(objectUrl), {{ once: true }});
}})
.catch(() => {{
image.dataset.wikiImageError = "true";
window.setTimeout(() => {{
delete image.dataset.deerflowProtectedLoading;
hydrateProtectedImage(image);
}}, 1200 * (attempts + 1));
}})
.finally(() => {{
delete image.dataset.deerflowProtectedLoading;
}});
}}
function hydrateProtectedImages(scope = document) {{
if (scope?.matches?.("img[data-protected-src]")) hydrateProtectedImage(scope);
scope?.querySelectorAll?.("img[data-protected-src]").forEach(hydrateProtectedImage);
}}
const protectedImageObserver = new MutationObserver((mutations) => {{
mutations.forEach((mutation) => {{
if (mutation.type === "attributes") hydrateProtectedImage(mutation.target);
mutation.addedNodes?.forEach((node) => {{
if (node?.nodeType === Node.ELEMENT_NODE) hydrateProtectedImages(node);
}});
}});
}});
protectedImageObserver.observe(document.documentElement, {{
childList: true,
subtree: true,
attributes: true,
attributeFilter: ["data-protected-src"]
}});
hydrateProtectedImages();
function tone(color, alpha) {{
return `color-mix(in srgb, ${{color}} ${{Math.round(alpha * 100)}}%, transparent)`;
}}
@@ -1519,6 +1604,7 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
enforceLocation();
purgeGuides();
hideWeKnoraShell();
hydrateProtectedImages();
if (tickSourceDetail() && sourceDetailMode) {{
clearInterval(maintenanceTimer);
try {{ readyObserver.disconnect(); }} catch (_) {{}}
@@ -1820,6 +1906,36 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
return row
def _validate_protected_wiki_file_path(file_path: str) -> str:
value = file_path.strip()
if not value or "\x00" in value or not _PROTECTED_WIKI_FILE_PATH.match(value):
raise HTTPException(status_code=422, detail="Unsupported Wiki image path")
return value
async def _wiki_file_response(row: dict[str, Any], file_path: str, *, public: bool) -> Response:
safe_file_path = _validate_protected_wiki_file_path(file_path)
try:
content, media_type = await _client_or_503().get_knowledge_base_file(
str(row["weknora_id"]),
safe_file_path,
)
except WeKnoraError as exc:
_raise_upstream(exc)
filename = unquote(safe_file_path.rstrip("/").rsplit("/", 1)[-1]).replace("\r", "").replace("\n", "") or "wiki-file"
if "." not in filename:
filename += mimetypes.guess_extension(str(media_type).split(";", 1)[0].strip()) or ""
return Response(
content=content,
media_type=media_type,
headers={
"Cache-Control": "public, max-age=300" if public else "private, max-age=3600",
"Content-Disposition": f"inline; filename*=UTF-8''{quote(filename, safe='')}",
"X-Content-Type-Options": "nosniff",
},
)
async def _verify_document(client: WeKnoraClient, row: dict[str, Any], document_id: str) -> dict[str, Any]:
document = await client.get_document(document_id)
if str(document.get("knowledge_base_id") or "") != str(row["weknora_id"]):
@@ -2510,6 +2626,37 @@ async def list_public_wiki_pages(
return {**result, "pages": pages}
@router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}/files")
async def get_public_wiki_file(
request: Request,
mapping_id: str,
file_path: str = Query(..., min_length=1, max_length=8192),
) -> Response:
"""Serve one protected image belonging to a published Wiki."""
row = await _public_mapping(request, mapping_id)
return await _wiki_file_response(row, file_path, public=True)
@router.get("/api/public/llmwiki/files/{token}")
async def get_signed_wiki_file(token: str) -> Response:
"""Serve a private Wiki image through a short-lived, signed HTTP URL."""
payload = _verify_signed_payload(token)
if payload.get("kind") != "wiki_file":
raise HTTPException(status_code=401, detail="Signed Wiki file token is invalid")
mapping_id = str(payload.get("mapping_id") or "")
weknora_id = str(payload.get("weknora_id") or "")
file_path = str(payload.get("file_path") or "")
if not mapping_id or not weknora_id or not file_path:
raise HTTPException(status_code=401, detail="Signed Wiki file token is invalid")
return await _wiki_file_response(
{"id": mapping_id, "weknora_id": weknora_id},
file_path,
public=False,
)
_PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison")
@@ -2820,6 +2967,40 @@ async def preview_document(request: Request, mapping_id: str, document_id: str)
)
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/files")
async def get_wiki_file(
request: Request,
mapping_id: str,
file_path: str = Query(..., min_length=1, max_length=8192),
) -> Response:
"""Serve one protected Wiki image after normal knowledge-base authorization."""
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
return await _wiki_file_response(row, file_path, public=False)
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/files/access-url")
async def create_wiki_file_access(
request: Request,
mapping_id: str,
file_path: str = Query(..., min_length=1, max_length=8192),
) -> dict[str, Any]:
"""Issue a browser-usable HTTP capability after normal KB authorization."""
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
safe_file_path = _validate_protected_wiki_file_path(file_path)
token = _sign_embed_payload(
{
"kind": "wiki_file",
"mapping_id": mapping_id,
"weknora_id": str(row["weknora_id"]),
"file_path": safe_file_path,
"exp": int(time.time()) + _WIKI_FILE_ACCESS_TTL_SECONDS,
}
)
return {"token": token, "expires_in": _WIKI_FILE_ACCESS_TTL_SECONDS}
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph")
async def get_knowledge_base_graph(
request: Request,
@@ -2,6 +2,7 @@
from __future__ import annotations
import logging
from typing import Any
from uuid import uuid4
@@ -14,6 +15,7 @@ from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectori
from deerflow.integrations.weknora.local_index.search import WikiIndexError
router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"])
logger = logging.getLogger(__name__)
class WikiVectorSearchRequest(BaseModel):
@@ -181,6 +183,7 @@ async def _run_sync(app, mapping_ids: list[str], force: bool) -> None:
try:
await service.sync_mapping(mapping, force=force)
except Exception:
logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id"))
continue
@@ -191,6 +194,7 @@ async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
try:
await service.sync_mapping(mapping, force=force)
except Exception:
logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id"))
return
@@ -241,6 +245,33 @@ async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) ->
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
states = await index_store.list_index_status([mapping_id])
state = states[0] if states else {"knowledge_base_mapping_id": mapping_id, "state": "not_synced", "index_revision": 0}
if (
state.get("state") == "idle"
and int(state.get("index_revision") or 0) == 0
and not state.get("last_completed_at")
and not state.get("last_success_at")
):
state = {**state, "state": "not_synced"}
pages = await index_store.list_page_index_status(mapping_id)
active_sync_id = str(state.get("active_sync_id") or "")
is_running = str(state.get("state") or "") in {"running", "rebuilding"}
for page in pages:
page["vectorized"] = page.get("index_status") == "ready" and int(page.get("vector_count") or 0) > 0
page["processed_in_current_sync"] = bool(active_sync_id) and page.get("last_seen_sync_id") == active_sync_id
total_pages = max(int(state.get("remote_page_count") or 0), len(pages))
processed_pages = (
sum(bool(page["processed_in_current_sync"]) for page in pages)
if is_running
else total_pages if str(state.get("state") or "") in {"idle", "partial"} else 0
)
completion_percent = (
min(100, round((processed_pages / total_pages) * 100))
if total_pages
else 100
if str(state.get("state") or "") in {"idle", "partial"}
and bool(state.get("last_completed_at") or state.get("last_success_at"))
else 0
)
embedding = getattr(request.app.state, "llmwiki_embedding", None)
fingerprint = embedding.fingerprint if embedding is not None else None
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(state, fingerprint)
@@ -251,6 +282,15 @@ async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) ->
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
"fully_vectorized": fully_vectorized,
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
"progress": {
"total_pages": total_pages,
"processed_pages": processed_pages,
"vectorized_pages": sum(bool(page["vectorized"]) for page in pages),
"failed_pages": sum(bool(page.get("last_error")) or page.get("index_status") in {"failed", "stale"} for page in pages),
"vector_count": sum(int(page.get("vector_count") or 0) for page in pages),
"completion_percent": completion_percent,
},
"pages": pages,
}