This commit is contained in:
fengxiangdebaozi committed 2026-06-10 19:39:06 +08:00
commit d9e3a87911
129 files changed
+17888 -1703

No files matched your search

@@ -6,6 +6,8 @@ import sys
import threading
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from datetime import datetime
from time import perf_counter
from dotenv import load_dotenv
from fastapi import FastAPI
@@ -30,12 +32,14 @@ from app.gateway.routers import (
artifacts,
assistants_compat,
auth,
browser_context,
business_mapping,
channels,
checkpoint_migration,
feedback,
html_page_favorites,
intent,
knowledge,
llm_metrics,
mcp,
memory,
@@ -45,6 +49,7 @@ from app.gateway.routers import (
parallel_agent_tasks,
parallel_agents,
public_embed,
public_leaderboard,
public_scheduled_tasks,
recommend,
recommended_questions,
@@ -469,6 +474,91 @@ _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
"ADMIN_LEADERBOARD_BACKGROUND_STARTUP_DELAY_SECONDS",
60.0,
)
# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
# *previous* day's snapshot once, so records that landed near midnight are fully
# captured. Default 1 → "每天凌晨 1 点重新统计前一天".
_LEADERBOARD_FINALIZE_HOUR = min(
23,
_env_int("ADMIN_LEADERBOARD_FINALIZE_HOUR", 1, minimum=0),
)
def _leaderboard_scheduler_sql_for_log(stmt) -> str:
try:
return str(stmt.compile(compile_kwargs={"literal_binds": True}))
except Exception:
return str(stmt)
async def _execute_leaderboard_scheduler_query(session, step: str, stmt, **context):
sql = _leaderboard_scheduler_sql_for_log(stmt)
logger.info(
"Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
step,
context.get("stat_date") or context.get("stat_dates") or "",
context.get("job_key") or "",
context.get("settings_hash") or "",
datetime.now().isoformat(),
sql,
)
start = perf_counter()
try:
result = await session.execute(stmt)
except Exception:
logger.exception(
"Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
step,
context.get("stat_date") or context.get("stat_dates") or "",
context.get("job_key") or "",
context.get("settings_hash") or "",
(perf_counter() - start) * 1000,
)
raise
logger.info(
"Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
step,
context.get("stat_date") or context.get("stat_dates") or "",
context.get("job_key") or "",
context.get("settings_hash") or "",
datetime.now().isoformat(),
(perf_counter() - start) * 1000,
)
return result
async def _time_leaderboard_scheduler_db_operation(step: str, operation, *, sql: str = "", **context):
logger.info(
"Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
step,
context.get("stat_date") or context.get("stat_dates") or "",
context.get("job_key") or "",
context.get("settings_hash") or "",
datetime.now().isoformat(),
sql,
)
start = perf_counter()
try:
result = await operation()
except Exception:
logger.exception(
"Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
step,
context.get("stat_date") or context.get("stat_dates") or "",
context.get("job_key") or "",
context.get("settings_hash") or "",
(perf_counter() - start) * 1000,
)
raise
logger.info(
"Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
step,
context.get("stat_date") or context.get("stat_dates") or "",
context.get("job_key") or "",
context.get("settings_hash") or "",
datetime.now().isoformat(),
(perf_counter() - start) * 1000,
)
return result
def _recent_leaderboard_stat_dates(days: int) -> list[str]:
@@ -496,21 +586,27 @@ async def _leaderboard_snapshot_due_dates(
cutoff = datetime.now(UTC) - timedelta(seconds=running_ttl_seconds)
async with session_factory() as session:
rows = (
await session.execute(
select(AdminLeaderboardDailyStatRow.stat_date)
.where(
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
or_(
AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
and_(
AdminLeaderboardDailyStatRow.status == "running",
AdminLeaderboardDailyStatRow.updated_at < cutoff,
),
stmt = (
select(AdminLeaderboardDailyStatRow.stat_date)
.where(
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
or_(
AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
and_(
AdminLeaderboardDailyStatRow.status == "running",
AdminLeaderboardDailyStatRow.updated_at < cutoff,
),
)
.order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
.limit(limit)
),
)
.order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
.limit(limit)
)
rows = (
await _execute_leaderboard_scheduler_query(
session,
"scheduler.due_dates",
stmt,
settings_hash=settings_hash,
)
).all()
return [str(row[0]) for row in rows]
@@ -530,9 +626,15 @@ async def _claim_leaderboard_snapshot(
from deerflow.persistence.types import BEIJING_TZ
async with session_factory() as session:
row = await session.get(
AdminLeaderboardDailyStatRow,
{"stat_date": stat_date, "settings_hash": settings_hash},
row = await _time_leaderboard_scheduler_db_operation(
"scheduler.claim.get",
lambda: session.get(
AdminLeaderboardDailyStatRow,
{"stat_date": stat_date, "settings_hash": settings_hash},
),
sql="SELECT admin_leaderboard_daily_stats by primary key",
stat_date=stat_date,
settings_hash=settings_hash,
)
if row is None or _snapshot_is_fresh(row, stat_date):
return False
@@ -548,10 +650,68 @@ async def _claim_leaderboard_snapshot(
row.message = "排行榜后台统计任务已认领。"
row.error = None
row.updated_at = datetime.now(UTC)
await session.commit()
await _time_leaderboard_scheduler_db_operation(
"scheduler.claim.commit",
lambda: session.commit(),
sql="COMMIT",
stat_date=stat_date,
settings_hash=settings_hash,
)
return True
async def _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash: str) -> bool:
"""Once per day after ``_LEADERBOARD_FINALIZE_HOUR``, force-recompute yesterday.
A ``ready`` past-day snapshot is otherwise treated as fresh forever and never
rebuilt. But the previous day's snapshot is usually generated before midnight,
so it can miss runs/tool-calls that only land in the DB around the day
boundary. This re-queues yesterday (``force=True``) a single time; the rebuild
stamps ``generated_at`` to today, which stops it from re-firing for the rest of
the day.
"""
from datetime import timedelta
from app.gateway.routers.admin_users import (
_as_beijing,
_queue_daily_leaderboard_snapshots,
)
from deerflow.persistence.admin_stats.finalize import should_finalize_previous_day
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.persistence.types import BEIJING_TZ
now = datetime.now(BEIJING_TZ)
yesterday = (now.date() - timedelta(days=1)).isoformat()
async with session_factory() as session:
row = await session.get(
AdminLeaderboardDailyStatRow,
{"stat_date": yesterday, "settings_hash": settings_hash},
)
generated_at = _as_beijing(row.generated_at) if (row is not None and row.generated_at is not None) else None
if not should_finalize_previous_day(
status=(row.status if row is not None else None),
generated_at=generated_at,
now=now,
finalize_hour=_LEADERBOARD_FINALIZE_HOUR,
):
return False
queued = await _queue_daily_leaderboard_snapshots(
session_factory,
[yesterday],
settings_hash,
force=True,
message="每日凌晨重新统计前一天完整数据,确保全部记录纳入统计。",
)
if queued:
logger.info(
"Leaderboard snapshot finalize: queued previous day %s for full recompute (hash=%s)",
yesterday,
settings_hash[:10],
)
return True
return False
async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
from app.gateway.routers.admin_users import (
_daily_job_key,
@@ -576,6 +736,12 @@ async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
message="排行榜后台统计任务定期预热/补算队列。",
)
# 每天凌晨(默认 1 点)后强制重算"前一天"的完整快照一次,覆盖跨午夜才落库的记录。
try:
await _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash)
except Exception:
logger.exception("Leaderboard snapshot finalize check failed (non-fatal)")
due_dates = await _leaderboard_snapshot_due_dates(
session_factory,
settings_hash=settings_hash,
@@ -1115,6 +1281,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Uploads API is mounted at /api/threads/{thread_id}/uploads
app.include_router(uploads.router)
# Optional Chrome-extension page capture API is mounted at /api/browser-context
app.include_router(browser_context.router)
# Thread cleanup API is mounted at /api/threads/{thread_id}
app.include_router(threads.router)
@@ -1162,10 +1331,16 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# surface results without a DeerFlow login.
app.include_router(public_scheduled_tasks.router)
# Public read-only user leaderboard publish page (/api/public/leaderboard)
app.include_router(public_leaderboard.router)
# Public embed Q&A status polling (/api/public/embed/*)
app.include_router(public_embed.router)
app.include_router(html_page_favorites.router)
# Product knowledge base API is mounted at /api/knowledge
app.include_router(knowledge.router)
# System settings API is mounted at /api/system-settings
app.include_router(system_settings.router)
@@ -1174,6 +1349,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Admin window into other users' threads / messages / memory
app.include_router(admin_users.router)
app.include_router(admin_users.self_router)
# Admin tool / skill call metrics audit + export
app.include_router(tool_metrics.router)
@@ -20,6 +20,7 @@ from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, i
from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse
from app.gateway.authz import _ALL_PERMISSIONS, AuthContext
from app.gateway.internal_auth import INTERNAL_AUTH_HEADER_NAME, get_internal_user, is_valid_internal_auth_token
from app.gateway.proxy_path import app_relative_path
from deerflow.runtime.user_context import reset_current_user, set_current_user
# Paths that never require authentication.
@@ -84,7 +85,9 @@ class AuthMiddleware(BaseHTTPMiddleware):
super().__init__(app)
async def dispatch(self, request: Request, call_next: Callable) -> Response:
if _is_public(request.url.path):
# Match the application-relative path so a reverse-proxy prefix
# (e.g. /xxx/api/public/...) does not defeat the public whitelist.
if _is_public(app_relative_path(request)):
return await call_next(request)
if is_auth_disabled():
@@ -8,6 +8,7 @@ import secrets
from collections.abc import Callable
from app.gateway.auth.disabled_mode import is_auth_disabled
from app.gateway.proxy_path import app_relative_path
from fastapi import Request, Response
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.responses import JSONResponse
@@ -37,7 +38,9 @@ def should_check_csrf(request: Request) -> bool:
if request.method not in ("POST", "PUT", "DELETE", "PATCH"):
return False
path = request.url.path
# Match the application-relative path so a reverse-proxy prefix
# (e.g. /xxx/api/v1/auth/me) does not defeat these whitelists.
path = app_relative_path(request)
stripped = path.rstrip("/")
# Exempt /api/v1/auth/me endpoint
if stripped == "/api/v1/auth/me":
@@ -56,6 +59,10 @@ _AUTH_EXEMPT_PATHS: frozenset[str] = frozenset(
{
"/api/v1/auth/login/local",
"/api/v1/auth/login/username",
# Token-exchange login is sessionless on first call (no CSRF cookie yet),
# exactly like the other login endpoints — mirror auth_middleware's
# _PUBLIC_EXACT_PATHS, which already whitelists it.
"/api/v1/auth/login/token",
"/api/v1/auth/logout",
"/api/v1/auth/register",
"/api/v1/auth/initialize",
@@ -69,8 +76,10 @@ def is_auth_endpoint(request: Request) -> bool:
"""Check if the request is to an auth endpoint.
Auth endpoints don't need CSRF validation on first call (no token).
Matches the application-relative path so a reverse-proxy prefix does not
defeat the whitelist.
"""
return request.url.path.rstrip("/") in _AUTH_EXEMPT_PATHS
return app_relative_path(request).rstrip("/") in _AUTH_EXEMPT_PATHS
def _has_bearer_auth(request: Request) -> bool:
@@ -82,6 +91,22 @@ def _has_bearer_auth(request: Request) -> bool:
return scheme.lower() == "bearer" and bool(token.strip())
def _has_cookie_session(request: Request) -> bool:
"""Return True when the request carries a cookie-based session.
Double-Submit-Cookie CSRF only protects requests the browser authenticates
**automatically via cookies** — that is the exact surface a cross-site
forgery can ride. A request with no ``access_token`` session cookie has no
session for an attacker to exploit, so enforcing CSRF on it is both
pointless and harmful: a freshly-loaded browser (e.g. one that just cleared
its cache/cookies) hitting the login endpoint would otherwise be rejected
with "CSRF token missing" before it can even sign in. Such an
unauthenticated request, if forged, is rejected by the auth layer (401)
anyway.
"""
return bool(request.cookies.get("access_token"))
class CSRFMiddleware(BaseHTTPMiddleware):
"""Middleware that implements CSRF protection using Double Submit Cookie pattern."""
@@ -95,8 +120,11 @@ class CSRFMiddleware(BaseHTTPMiddleware):
return await call_next(request)
# Double-submit-cookie CSRF applies to cookie-based browser sessions.
# API clients using explicit Bearer tokens are not CSRF-prone.
if should_check_csrf(request) and not _is_auth and not _has_bearer_auth(request):
# API clients using explicit Bearer tokens are not CSRF-prone, and a
# request with no session cookie at all (e.g. a fresh browser signing
# in) has no session to forge — only enforce when an ``access_token``
# cookie is actually present.
if should_check_csrf(request) and not _is_auth and not _has_bearer_auth(request) and _has_cookie_session(request):
cookie_token = request.cookies.get(CSRF_COOKIE_NAME)
header_token = request.headers.get(CSRF_HEADER_NAME)
@@ -143,6 +143,8 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.persistence.ai_writing_sessions import (
make_ai_writing_session_store,
)
from deerflow.persistence.ai_writing_sessions import (
register_default_store as _register_ai_writing_default_store,
)
@@ -159,6 +161,25 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.memory_config_store = make_user_memory_config_store(sf)
from deerflow.knowledge import make_knowledge_service
app.state.knowledge_service = make_knowledge_service(sf, config.knowledge)
# Background auto-sedimentation queue (phase 3) — drains finished
# conversations off the chat path. Worker only does work when
# ``knowledge.auto_ingest_enabled`` is set; the queue is always wired so
# the middleware can enqueue without a hard dependency.
from deerflow.knowledge.auto_ingest import AutoIngestQueue, set_auto_ingest_queue
if app.state.knowledge_service is not None:
_auto_ingest_queue = AutoIngestQueue(
lambda: getattr(app.state, "knowledge_service", None),
min_chars=config.knowledge.auto_ingest_min_chars,
)
_auto_ingest_queue.start()
set_auto_ingest_queue(_auto_ingest_queue)
app.state.knowledge_auto_ingest_queue = _auto_ingest_queue
from deerflow.persistence.user_prompts import make_user_prompt_store
app.state.user_prompt_store = make_user_prompt_store(sf)
@@ -189,6 +210,8 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.persistence.roundtable_jobs import (
make_roundtable_job_store,
)
from deerflow.persistence.roundtable_jobs import (
register_default_store as _register_roundtable_job_default_store,
)
@@ -246,6 +269,12 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.persistence.notifications import set_notification_store
set_notification_store(None)
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
if queue is not None:
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
await queue.stop()
set_auto_ingest_queue(None)
await close_engine()
@@ -380,6 +409,14 @@ def get_memory_config_store(request: Request):
return val
def get_knowledge_service(request: Request):
"""Return the knowledge base service (503 when disabled / memory backend)."""
val = getattr(request.app.state, "knowledge_service", None)
if val is None:
raise HTTPException(status_code=503, detail="Knowledge base not available")
return val
def get_run_context(request: Request) -> RunContext:
"""Build a :class:`RunContext` from ``app.state`` singletons.
@@ -40,6 +40,13 @@ def _check_csrf(request) -> None:
# CSRF protection is for cookie-auth browser flows.
return
# No cookie-based session → nothing for a cross-site forgery to ride, so
# CSRF is moot. Enforcing it here would reject a freshly-loaded browser
# (just cleared its cookies) before it can authenticate; such a request is
# rejected as unauthenticated (401) below anyway.
if not request.cookies.get("access_token"):
return
cookie_token = request.cookies.get("csrf_token")
header_token = request.headers.get("x-csrf-token")
@@ -0,0 +1,39 @@
"""Resolve the application-relative request path behind a reverse proxy.
A reverse proxy may forward requests under a path prefix, e.g.
``/xxx/api/v1/auth/login/token``. FastAPI strips that prefix for **routing**
when it is told about it via the ASGI ``root_path`` (uvicorn ``--root-path`` /
``--proxy-headers``, or ``X-Forwarded-Prefix`` from the proxy) — which is why
the route still matches and the request reaches a handler.
Starlette middleware, however, runs *before* routing and sees the full,
prefixed ``request.url.path``. Middlewares that match that raw path against
fixed whitelists — the CSRF-exempt auth paths and the AuthMiddleware public
paths — then mis-match, so login and ``/api/public/*`` endpoints get wrongly
rejected (``CSRF token missing`` / ``401``) on a prefixed deployment.
``app_relative_path`` returns the path with the proxy prefix removed so that
whitelist matching is prefix-agnostic. It is a no-op when no prefix is present
(direct deployment), and tolerates the prefix already being stripped from the
path (it only strips when the path actually starts with the prefix).
"""
from fastapi import Request
def app_relative_path(request: Request) -> str:
"""Return ``request.url.path`` with any reverse-proxy prefix removed.
Prefix source order: ASGI ``root_path`` (set by the server when the proxy
forwards it), then the ``X-Forwarded-Prefix`` header (proxies that add a
prefix without informing the ASGI server). Returns ``"/"`` for an empty
result so downstream ``rstrip("/")`` / ``startswith`` checks stay sane.
"""
path = request.url.path or "/"
prefix = (request.scope.get("root_path") or request.headers.get("x-forwarded-prefix") or "").rstrip("/")
if prefix and path.startswith(prefix):
stripped = path[len(prefix):]
if not stripped:
return "/"
return stripped if stripped.startswith("/") else "/" + stripped
return path
@@ -19,8 +19,11 @@ import logging
import os
import re
from collections.abc import AsyncIterator
from contextvars import ContextVar
from difflib import SequenceMatcher
from hashlib import sha256
from time import perf_counter
from typing import Any
from datetime import UTC, datetime, timedelta, time as datetime_time
@@ -56,6 +59,7 @@ from deerflow.persistence.user.model import UserRow
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/admin/users", tags=["admin-users"])
self_router = APIRouter(prefix="/api/users/me", tags=["user-qa"])
_LEADERBOARD_MAX_CONCURRENT = max(
1,
@@ -99,6 +103,119 @@ _LEADERBOARD_QUEUE_ON_REQUEST_ENABLED = os.getenv(
"off",
}
_leaderboard_semaphore = asyncio.Semaphore(_LEADERBOARD_MAX_CONCURRENT)
_LEADERBOARD_TRACE_CONTEXT: ContextVar[dict[str, object] | None] = ContextVar(
"leaderboard_trace_context",
default=None,
)
def _leaderboard_log_context(**context: object) -> dict[str, object] | None:
current = _LEADERBOARD_TRACE_CONTEXT.get()
merged: dict[str, object] = dict(current or {})
for key, value in context.items():
if value is None or value == "":
continue
merged[key] = value
return merged or None
def _leaderboard_sql_for_log(stmt) -> str:
try:
return str(stmt.compile(compile_kwargs={"literal_binds": True}))
except Exception:
return str(stmt)
async def _execute_leaderboard_query(session, step: str, stmt, **context: object):
log_context = _leaderboard_log_context(**context)
if not log_context:
return await session.execute(stmt)
started_at = datetime.now(BEIJING_TZ).isoformat()
sql = _leaderboard_sql_for_log(stmt)
logger.info(
"Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
step,
log_context.get("stat_date") or log_context.get("stat_dates") or "",
log_context.get("job_key") or "",
log_context.get("settings_hash") or "",
started_at,
sql,
)
start = perf_counter()
try:
result = await session.execute(stmt)
except Exception:
elapsed_ms = (perf_counter() - start) * 1000
logger.exception(
"Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
step,
log_context.get("stat_date") or log_context.get("stat_dates") or "",
log_context.get("job_key") or "",
log_context.get("settings_hash") or "",
elapsed_ms,
)
raise
elapsed_ms = (perf_counter() - start) * 1000
logger.info(
"Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
step,
log_context.get("stat_date") or log_context.get("stat_dates") or "",
log_context.get("job_key") or "",
log_context.get("settings_hash") or "",
datetime.now(BEIJING_TZ).isoformat(),
elapsed_ms,
)
return result
async def _time_leaderboard_db_operation(
step: str,
operation,
*,
sql: str = "",
**context: object,
):
log_context = _leaderboard_log_context(**context)
if not log_context:
return await operation()
logger.info(
"Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
step,
log_context.get("stat_date") or log_context.get("stat_dates") or "",
log_context.get("job_key") or "",
log_context.get("settings_hash") or "",
datetime.now(BEIJING_TZ).isoformat(),
sql,
)
start = perf_counter()
try:
result = await operation()
except Exception:
elapsed_ms = (perf_counter() - start) * 1000
logger.exception(
"Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
step,
log_context.get("stat_date") or log_context.get("stat_dates") or "",
log_context.get("job_key") or "",
log_context.get("settings_hash") or "",
elapsed_ms,
)
raise
elapsed_ms = (perf_counter() - start) * 1000
logger.info(
"Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
step,
log_context.get("stat_date") or log_context.get("stat_dates") or "",
log_context.get("job_key") or "",
log_context.get("settings_hash") or "",
datetime.now(BEIJING_TZ).isoformat(),
elapsed_ms,
)
return result
async def _require_admin(request: Request) -> None:
@@ -285,6 +402,13 @@ def _parse_datetime(value: str | None, fallback: datetime) -> datetime:
raise HTTPException(status_code=400, detail=f"Invalid datetime: {value}") from exc
def _parse_optional_datetime(value: str | None) -> datetime | None:
if not value:
return None
parsed = _parse_datetime(value, _now())
return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=BEIJING_TZ)
def _resolve_range(days: int, since: str | None, until: str | None) -> tuple[datetime, datetime, int]:
end = _parse_datetime(until, _now())
start = _parse_datetime(since, end - timedelta(days=max(1, min(days, 365))))
@@ -325,7 +449,7 @@ def _bounds_for_stat_date(stat_date: str) -> tuple[datetime, datetime]:
async def _try_set_session_value(session, sql: str) -> None:
try:
await session.execute(text(sql))
await _execute_leaderboard_query(session, "session_setting", text(sql))
except Exception:
logger.debug("Failed to apply analytics session setting: %s", sql, exc_info=True)
try:
@@ -334,13 +458,36 @@ async def _try_set_session_value(session, sql: str) -> None:
logger.debug("Failed to rollback after analytics session setting failure", exc_info=True)
def _is_mariadb(dialect) -> bool:
"""SQLAlchemy reports ``dialect.name == 'mysql'`` for MariaDB too."""
if getattr(dialect, "_is_mariadb", False) or getattr(dialect, "is_mariadb", False):
return True
version_info = getattr(dialect, "server_version_info", None) or ()
return any("mariadb" in str(part).lower() for part in version_info)
async def _set_analytics_read_timeout(session) -> None:
"""Best-effort per-statement read timeout for the analytics queries.
The right knob differs by server, and some servers have none at all:
- MySQL >= 5.7.4: ``max_execution_time`` (milliseconds)
- MariaDB: ``max_statement_time`` (seconds)
- MySQL <= 5.6: no per-statement timeout variable — skip it entirely
(5.6 raises "Unknown system variable" otherwise)
- PostgreSQL: ``statement_timeout`` (milliseconds)
"""
conn = await session.connection()
dialect = conn.dialect.name
if dialect == "mysql":
await _try_set_session_value(session, f"SET SESSION max_execution_time = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
dialect = conn.dialect
name = dialect.name
if name == "mysql":
if _is_mariadb(dialect):
# MariaDB: max_statement_time is in SECONDS (fractional ok).
await _try_set_session_value(session, f"SET SESSION max_statement_time = {_LEADERBOARD_QUERY_TIMEOUT_MS / 1000}")
elif (getattr(dialect, "server_version_info", None) or ()) >= (5, 7, 4):
await _try_set_session_value(session, f"SET SESSION max_execution_time = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
# else: MySQL <= 5.6 has no statement-timeout variable — skip silently.
await _try_set_session_value(session, f"SET SESSION innodb_lock_wait_timeout = {_LEADERBOARD_LOCK_WAIT_SECONDS}")
elif dialect == "postgresql":
elif name == "postgresql":
await _try_set_session_value(session, f"SET LOCAL statement_timeout = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
@@ -661,11 +808,15 @@ async def _query_user_leaderboard(
if user_id:
stmt = stmt.where(UserRow.id == user_id)
stmt = _apply_user_scope(stmt, UserRow.id, exclude_user_ids=exclude_user_ids, include_admins=include_admins)
# 排行榜只统计真正产生过问答的用户;从未问答的账号(仅注册、零活跃)不计入榜单与人数。
# 个人分析(指定 user_id)不受此限制,仍可查看任意单个用户。
if user_id is None:
stmt = stmt.where(func.coalesce(run_agg.c.qa_count, 0) > 0)
stmt = stmt.order_by(
desc(func.coalesce(run_agg.c.run_count, 0) + func.coalesce(tool_agg.c.tool_call_count, 0)),
desc(func.coalesce(run_agg.c.total_tokens, 0)),
)
rows = (await session.execute(stmt)).all()
rows = (await _execute_leaderboard_query(session, "users", stmt)).all()
return [
AdminLeaderboardUser(
user_id=r[0],
@@ -725,7 +876,7 @@ async def _query_call_rankings(
include_admins=include_admins,
include_scheduled=include_scheduled,
)
rows = (await session.execute(stmt)).all()
rows = (await _execute_leaderboard_query(session, f"{field}_rankings", stmt)).all()
return [
AdminLeaderboardSkill(
name=r[0] or "",
@@ -791,7 +942,7 @@ async def _query_run_dimensions(
include_failed=include_failed,
include_scheduled=include_scheduled,
)
rows = (await session.execute(stmt)).all()
rows = (await _execute_leaderboard_query(session, f"{field}_dimensions", stmt)).all()
return [
AdminLeaderboardDimension(
name=r[0] or "",
@@ -848,7 +999,7 @@ async def _query_model_rankings(
stmt = stmt.where(LlmCallMetricRow.status != "error")
if not include_scheduled:
stmt = stmt.where(or_(LlmCallMetricRow.run_id.is_(None), LlmCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
rows = (await session.execute(stmt)).all()
rows = (await _execute_leaderboard_query(session, "model_rankings", stmt)).all()
return [
AdminLeaderboardDimension(
name=r[0] or "",
@@ -895,7 +1046,7 @@ async def _query_recent_questions(
include_failed=include_failed,
include_scheduled=include_scheduled,
)
rows = (await session.execute(stmt)).all()
rows = (await _execute_leaderboard_query(session, "recent_questions", stmt)).all()
questions: list[AdminLeaderboardQuestion] = []
for r in rows:
text = _clean_question_text(r[0]) if cleanup_system_questions else str(r[0] or "").strip()
@@ -947,7 +1098,7 @@ async def _query_frequent_questions(
include_failed=include_failed,
include_scheduled=include_scheduled,
)
rows = (await session.execute(stmt)).all()
rows = (await _execute_leaderboard_query(session, "frequent_questions", stmt)).all()
groups: list[dict] = []
for r in rows:
text = _clean_question_text(r[0]) if cleanup_system_questions else str(r[0] or "").strip()
@@ -1022,7 +1173,7 @@ async def _query_trends(
include_failed=include_failed,
include_scheduled=include_scheduled,
)
rows = (await session.execute(stmt)).all()
rows = (await _execute_leaderboard_query(session, "trends", stmt)).all()
return [
AdminLeaderboardTrendPoint(
date=str(r[0]),
@@ -1071,8 +1222,8 @@ async def _query_overview(
total_users_stmt = select(func.count(UserRow.id))
total_users_stmt = _apply_user_scope(total_users_stmt, UserRow.id, exclude_user_ids=exclude_user_ids, include_admins=include_admins)
total_users = 1 if user_id else (await session.execute(total_users_stmt)).scalar_one()
run_row = (await session.execute(
total_users = 1 if user_id else (await _execute_leaderboard_query(session, "overview.total_users", total_users_stmt)).scalar_one()
run_summary_stmt = (
select(
func.count(RunRow.run_id),
_question_count_expr(),
@@ -1080,16 +1231,19 @@ async def _query_overview(
func.count(func.distinct(RunRow.user_id)),
func.sum(case((RunRow.status == "error", 1), else_=0)),
).where(*run_where)
)).one()
thread_count = (await session.execute(select(func.count(ThreadMetaRow.thread_id)).where(*thread_where))).scalar_one()
tool_row = (await session.execute(
)
run_row = (await _execute_leaderboard_query(session, "overview.run_summary", run_summary_stmt)).one()
thread_count_stmt = select(func.count(ThreadMetaRow.thread_id)).where(*thread_where)
thread_count = (await _execute_leaderboard_query(session, "overview.thread_count", thread_count_stmt)).scalar_one()
tool_summary_stmt = (
select(
func.count(ToolCallMetricRow.id),
func.sum(case((func.nullif(ToolCallMetricRow.skill_name, "").is_not(None), 1), else_=0)),
)
.outerjoin(RunRow, RunRow.run_id == ToolCallMetricRow.run_id)
.where(*tool_where)
)).one()
)
tool_row = (await _execute_leaderboard_query(session, "overview.tool_summary", tool_summary_stmt)).one()
return AdminLeaderboardOverview(
total_users=int(total_users or 0),
@@ -1579,13 +1733,12 @@ async def _load_daily_snapshot_rows(
if not stat_dates:
return {}
async with session_factory() as session:
stmt = select(AdminLeaderboardDailyStatRow).where(
AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
)
rows = (
await session.execute(
select(AdminLeaderboardDailyStatRow).where(
AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
)
)
await _execute_leaderboard_query(session, "snapshot_rows.load", stmt)
).scalars().all()
return {row.stat_date: row for row in rows}
@@ -1603,9 +1756,15 @@ async def _store_daily_snapshot(
) -> None:
async with session_factory() as session:
now = datetime.now(UTC)
row = await session.get(
AdminLeaderboardDailyStatRow,
{"stat_date": stat_date, "settings_hash": settings_hash},
row = await _time_leaderboard_db_operation(
"snapshot.store.get",
lambda: session.get(
AdminLeaderboardDailyStatRow,
{"stat_date": stat_date, "settings_hash": settings_hash},
),
sql="SELECT admin_leaderboard_daily_stats by primary key",
stat_date=stat_date,
settings_hash=settings_hash,
)
if row is None:
row = AdminLeaderboardDailyStatRow(
@@ -1630,7 +1789,13 @@ async def _store_daily_snapshot(
if generated_at is not None:
row.generated_at = generated_at
row.updated_at = now
await session.commit()
await _time_leaderboard_db_operation(
"snapshot.store.commit",
lambda: session.commit(),
sql="COMMIT",
stat_date=stat_date,
settings_hash=settings_hash,
)
async def _queue_daily_leaderboard_snapshots(
@@ -1646,12 +1811,17 @@ async def _queue_daily_leaderboard_snapshots(
return queued
async with session_factory() as session:
now = datetime.now(UTC)
stmt = select(AdminLeaderboardDailyStatRow).where(
AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
)
rows = (
await session.execute(
select(AdminLeaderboardDailyStatRow).where(
AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
)
await _execute_leaderboard_query(
session,
"snapshot_queue.load_existing",
stmt,
stat_dates=stat_dates,
settings_hash=settings_hash,
)
).scalars().all()
rows_by_date = {row.stat_date: row for row in rows}
@@ -1680,7 +1850,13 @@ async def _queue_daily_leaderboard_snapshots(
row.updated_at = now
queued.append(stat_date)
if queued:
await session.commit()
await _time_leaderboard_db_operation(
"snapshot_queue.commit",
lambda: session.commit(),
sql="COMMIT",
stat_dates=queued,
settings_hash=settings_hash,
)
return queued
@@ -1722,7 +1898,22 @@ async def _refresh_daily_leaderboard_snapshot(
include_failed: bool,
) -> None:
acquired = False
trace_token = _LEADERBOARD_TRACE_CONTEXT.set(
{
"job_key": job_key,
"stat_date": stat_date,
"settings_hash": settings_hash,
}
)
snapshot_start = perf_counter()
try:
logger.info(
"Leaderboard snapshot build start: job=%s stat_date=%s settings_hash=%s started_at=%s",
job_key,
stat_date,
settings_hash,
datetime.now(BEIJING_TZ).isoformat(),
)
if _LEADERBOARD_START_DELAY_SECONDS > 0:
await asyncio.sleep(_LEADERBOARD_START_DELAY_SECONDS)
await _store_daily_snapshot(
@@ -1758,6 +1949,14 @@ async def _refresh_daily_leaderboard_snapshot(
error=None,
generated_at=generated_at,
)
logger.info(
"Leaderboard snapshot build done: job=%s stat_date=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
job_key,
stat_date,
settings_hash,
datetime.now(BEIJING_TZ).isoformat(),
(perf_counter() - snapshot_start) * 1000,
)
except asyncio.TimeoutError:
logger.warning("Daily leaderboard snapshot timed out: %s", job_key)
await _store_daily_snapshot(
@@ -1781,6 +1980,7 @@ async def _refresh_daily_leaderboard_snapshot(
finally:
if acquired:
_leaderboard_semaphore.release()
_LEADERBOARD_TRACE_CONTEXT.reset(trace_token)
async def _load_or_schedule_daily_leaderboard(
@@ -1978,6 +2178,72 @@ async def queue_leaderboard_snapshot_backfill(
)
@router.post("/leaderboard/refresh-today", response_model=AdminLeaderboardResponse)
async def refresh_today_leaderboard(
request: Request,
days: int = Query(default=30, ge=1, le=365),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
exclude_user_ids: list[str] | None = Query(default=None, description="User ids excluded from this leaderboard aggregation"),
include_scheduled: bool | None = Query(default=None, description="Include runs created by scheduled tasks"),
include_admins: bool | None = Query(default=None, description="Include admin accounts in leaderboard aggregation"),
include_failed: bool | None = Query(default=None, description="Include failed runs and LLM calls"),
limit: int = Query(default=20, ge=5, le=100),
) -> AdminLeaderboardResponse:
"""Force a synchronous rebuild of *today's* snapshot, then return the range.
Today's snapshot otherwise refreshes only every ``_LEADERBOARD_TODAY_TTL_SECONDS``
(~5 min) via the background scheduler. This endpoint lets an admin pull the
latest real-time records into the stats on demand: it rebuilds today's daily
snapshot immediately (bounded by the same semaphore/timeout the scheduler uses)
and then returns the merged leaderboard for the requested range.
"""
await _require_admin(request)
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
(
excluded,
resolved_include_scheduled,
resolved_include_admins,
resolved_include_failed,
settings_hash,
) = _resolve_leaderboard_snapshot_scope(
settings,
exclude_user_ids=exclude_user_ids,
include_scheduled=include_scheduled,
include_admins=include_admins,
include_failed=include_failed,
)
today = _today_stat_date()
await _refresh_daily_leaderboard_snapshot(
_daily_job_key(today, settings_hash),
session_factory=session_factory,
settings=settings,
stat_date=today,
settings_hash=settings_hash,
excluded=excluded,
include_scheduled=resolved_include_scheduled,
include_admins=resolved_include_admins,
include_failed=resolved_include_failed,
)
since_dt, until_dt, _range_days = _resolve_range(days, since, until)
return await _load_or_schedule_daily_leaderboard(
session_factory,
settings,
since_dt,
until_dt,
limit=limit,
exclude_user_ids=exclude_user_ids,
include_scheduled=include_scheduled,
include_admins=include_admins,
include_failed=include_failed,
)
async def _build_leaderboard_report_prompt(
session_factory,
settings,
@@ -2248,22 +2514,182 @@ class AdminUserQaPageResponse(BaseModel):
items: list[AdminUserQaItem] = Field(default_factory=list)
@router.get("/{user_id}/qa", response_model=AdminUserQaPageResponse)
async def list_user_qa(
user_id: str,
request: Request,
page: int = Query(default=1, ge=1),
page_size: int = Query(default=20, ge=1, le=100),
q: str | None = Query(default=None, description="Search question, answer, run id, or thread id"),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
) -> AdminUserQaPageResponse:
"""Return one user's Q&A runs from lightweight run summary fields."""
await _require_admin(request)
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="Q&A audit requires database persistence")
def _qa_sort_datetime(value: str | None) -> datetime:
if not value:
return datetime.min.replace(tzinfo=BEIJING_TZ)
text_value = str(value).strip()
if text_value.endswith("Z"):
text_value = text_value[:-1] + "+00:00"
try:
parsed = datetime.fromisoformat(text_value)
except ValueError:
return datetime.min.replace(tzinfo=BEIJING_TZ)
if parsed.tzinfo is None:
return parsed.replace(tzinfo=BEIJING_TZ)
return parsed.astimezone(BEIJING_TZ)
def _qa_dedupe_key(item: AdminUserQaItem) -> tuple[str, str, str]:
question_key = _question_similarity_key(item.question)[:160]
answer_key = _question_similarity_key(item.answer)[:160]
return (item.thread_id, question_key, answer_key)
def _message_value(message: Any, key: str, default: Any = None) -> Any:
if isinstance(message, dict):
return message.get(key, default)
return getattr(message, key, default)
def _qa_message_type(message: Any) -> str:
value = _message_value(message, "type") or _message_value(message, "role") or ""
if value == "user":
return "human"
if value == "assistant":
return "ai"
return str(value)
def _qa_message_id(message: Any, fallback: str) -> str:
return str(_message_value(message, "id") or fallback)
def _qa_message_text(message: Any) -> str:
return _message_content_to_text(_message_value(message, "content")).strip()
async def _read_thread_checkpoint_messages(checkpointer, thread_id: str) -> list[dict]:
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
except Exception:
logger.warning("Failed to read checkpoint while listing Q&A: thread=%s", thread_id, exc_info=True)
return []
if checkpoint_tuple is None:
return []
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
channel_values = checkpoint.get("channel_values", {}) if isinstance(checkpoint, dict) else {}
values = serialize_channel_values(channel_values)
messages = values.get("messages", [])
return messages if isinstance(messages, list) else []
def _qa_items_from_checkpoint_messages(
*,
user_id: str,
thread_id: str,
thread_title: str,
assistant_id: str,
thread_time: str,
messages: list[dict],
) -> list[AdminUserQaItem]:
items: list[AdminUserQaItem] = []
current_question = ""
current_answer = ""
current_id = ""
current_index = 0
def flush() -> None:
nonlocal current_question, current_answer, current_id, current_index
question = _clean_question_text(current_question)
if not _is_displayable_question(question):
current_question = ""
current_answer = ""
current_id = ""
return
items.append(
AdminUserQaItem(
run_id=current_id or f"{thread_id}:checkpoint:{current_index}",
thread_id=thread_id,
thread_title=thread_title,
assistant_id=assistant_id,
status="checkpoint",
question=question[:2000],
answer=current_answer.strip()[:6000],
created_at=thread_time,
updated_at=thread_time,
)
)
current_question = ""
current_answer = ""
current_id = ""
for index, message in enumerate(messages):
msg_type = _qa_message_type(message)
if msg_type == "human":
if current_question:
flush()
current_question = _qa_message_text(message)
current_answer = ""
current_id = _qa_message_id(message, f"{thread_id}:checkpoint:{index}")
current_index = index
continue
if msg_type == "ai" and current_question:
text = _qa_message_text(message)
if text:
current_answer = text
if current_question:
flush()
return items
async def _list_thread_checkpoint_qa_items(
*,
request: Request,
user_id: str,
) -> list[AdminUserQaItem]:
repo = get_thread_store(request)
checkpointer = get_checkpointer(request)
items: list[AdminUserQaItem] = []
offset = 0
page_size = 200
while True:
rows = await repo.search(limit=page_size, offset=offset, user_id=user_id)
if not rows:
break
for row in rows:
thread_id = str(row.get("thread_id") or "")
if not thread_id:
continue
messages = await _read_thread_checkpoint_messages(checkpointer, thread_id)
if not messages:
continue
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
context = row.get("context") if isinstance(row.get("context"), dict) else {}
assistant_id = str(
row.get("assistant_id")
or (context or {}).get("agent_id")
or (metadata or {}).get("agent_id")
or (context or {}).get("agent_name")
or (metadata or {}).get("agent_name")
or ""
)
thread_time = str(row.get("updated_at") or row.get("created_at") or "")
items.extend(
_qa_items_from_checkpoint_messages(
user_id=user_id,
thread_id=thread_id,
thread_title=str(row.get("display_name") or ""),
assistant_id=assistant_id,
thread_time=thread_time,
messages=messages,
)
)
if len(rows) < page_size:
break
offset += len(rows)
return items
async def _list_run_summary_qa_items(
*,
session_factory,
user_id: str,
since: str | None,
until: str | None,
) -> list[AdminUserQaItem]:
filters = [
RunRow.user_id == user_id,
RunRow.first_human_message.is_not(None),
@@ -2272,26 +2698,9 @@ async def list_user_qa(
if since or until:
since_dt, until_dt, _range_days = _resolve_range(365, since, until)
filters.append(_run_time_filter(since_dt, until_dt))
needle = (q or "").strip()
if needle:
like = f"%{needle}%"
filters.append(
or_(
RunRow.first_human_message.like(like),
RunRow.last_ai_message.like(like),
RunRow.run_id.like(like),
RunRow.thread_id.like(like),
)
)
offset = (page - 1) * page_size
async with session_factory() as session:
await _set_analytics_read_timeout(session)
total = (
await session.execute(
select(func.count(RunRow.run_id)).where(*filters)
)
).scalar_one()
rows = (
await session.execute(
select(
@@ -2316,32 +2725,152 @@ async def list_user_qa(
)
.where(*filters)
.order_by(RunRow.created_at.desc(), RunRow.run_id.desc())
.limit(page_size)
.offset(offset)
)
).all()
return [
AdminUserQaItem(
run_id=r[0] or "",
thread_id=r[1] or "",
thread_title=r[2] or "",
assistant_id=r[3] or "",
status=r[4] or "",
model_name=r[5] or "",
question=_clean_question_text(r[6])[:2000],
answer=str(r[7] or "").strip()[:6000],
total_tokens=int(r[8] or 0),
created_at=_iso(r[9]),
updated_at=_iso(r[10]),
)
for r in rows
if _is_displayable_question(r[6])
]
def _filter_qa_items(
items: list[AdminUserQaItem],
*,
q: str | None,
since: str | None,
until: str | None,
) -> list[AdminUserQaItem]:
needle = (q or "").strip().lower()
since_dt = _parse_optional_datetime(since) if since else None
until_dt = _parse_optional_datetime(until) if until else None
filtered: list[AdminUserQaItem] = []
for item in items:
item_dt = _qa_sort_datetime(item.created_at or item.updated_at)
if since_dt and item_dt < since_dt:
continue
if until_dt and item_dt > until_dt:
continue
if needle:
haystack = "\n".join(
[
item.question,
item.answer,
item.run_id,
item.thread_id,
item.thread_title,
item.assistant_id,
]
).lower()
if needle not in haystack:
continue
filtered.append(item)
return filtered
async def _list_user_qa_page(
*,
request: Request,
user_id: str,
page: int,
page_size: int,
q: str | None,
since: str | None,
until: str | None,
include_checkpoints: bool = True,
) -> AdminUserQaPageResponse:
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="Q&A audit requires database persistence")
items = await _list_run_summary_qa_items(
session_factory=session_factory,
user_id=user_id,
since=since,
until=until,
)
if include_checkpoints:
items.extend(await _list_thread_checkpoint_qa_items(request=request, user_id=user_id))
by_key: dict[tuple[str, str, str], AdminUserQaItem] = {}
for item in items:
key = _qa_dedupe_key(item)
existing = by_key.get(key)
if existing is None or (existing.status == "checkpoint" and item.status != "checkpoint"):
by_key[key] = item
filtered = _filter_qa_items(list(by_key.values()), q=q, since=since, until=until)
filtered.sort(
key=lambda item: (_qa_sort_datetime(item.created_at or item.updated_at), item.thread_id, item.run_id),
reverse=True,
)
offset = (page - 1) * page_size
return AdminUserQaPageResponse(
user_id=user_id,
page=page,
page_size=page_size,
total=int(total or 0),
items=[
AdminUserQaItem(
run_id=r[0] or "",
thread_id=r[1] or "",
thread_title=r[2] or "",
assistant_id=r[3] or "",
status=r[4] or "",
model_name=r[5] or "",
question=_clean_question_text(r[6])[:2000],
answer=str(r[7] or "").strip()[:6000],
total_tokens=int(r[8] or 0),
created_at=_iso(r[9]),
updated_at=_iso(r[10]),
)
for r in rows
],
total=len(filtered),
items=filtered[offset : offset + page_size],
)
@router.get("/{user_id}/qa", response_model=AdminUserQaPageResponse)
async def list_user_qa(
user_id: str,
request: Request,
page: int = Query(default=1, ge=1),
page_size: int = Query(default=20, ge=1, le=100),
q: str | None = Query(default=None, description="Search question, answer, run id, or thread id"),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
) -> AdminUserQaPageResponse:
"""Return one user's Q&A runs, including older checkpoint-only threads."""
await _require_admin(request)
return await _list_user_qa_page(
request=request,
user_id=user_id,
page=page,
page_size=page_size,
q=q,
since=since,
until=until,
)
@self_router.get("/qa", response_model=AdminUserQaPageResponse)
async def list_my_qa(
request: Request,
page: int = Query(default=1, ge=1),
page_size: int = Query(default=20, ge=1, le=100),
q: str | None = Query(default=None, description="Search question, answer, run id, or thread id"),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
) -> AdminUserQaPageResponse:
"""Return the current user's own Q&A history."""
user = getattr(request.state, "user", None)
if user is None:
user = await get_current_user_from_request(request)
return await _list_user_qa_page(
request=request,
user_id=str(user.id),
page=page,
page_size=page_size,
q=q,
since=since,
until=until,
)
@@ -15,7 +15,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.auth.password import verify_password_async
from app.gateway.deps import get_agent_store, get_config, get_tag_store
from app.gateway.deps import get_agent_store, get_config, get_tag_store
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
from deerflow.config.agents_api_config import get_agents_api_config
from deerflow.config.agents_config import list_custom_agents, load_agent_config, load_agent_soul, validate_agent_id
@@ -39,6 +39,7 @@ class AgentResponse(BaseModel):
name: str = Field(..., description="Display name. Chinese and duplicate names are allowed.")
description: str = Field(default="", description="Agent description")
user_id: str | None = Field(default=None, description="Owner user id. Null means built-in.")
owner_username: str | None = Field(default=None, description="Publisher username (email local part). Null for built-in agents.")
published: bool = Field(default=False, description="Whether a user-created agent is public")
is_private: bool = Field(default=False, description="Whether the agent lives in the password-gated private square instead of the public one")
builtin: bool = Field(default=False, description="True when user_id is null")
@@ -62,15 +63,15 @@ class AgentResponse(BaseModel):
is_favorite: bool = Field(
default=False,
description="Whether the requesting user has favorited (收藏) this agent.",
)
is_pinned: bool = Field(
default=False,
description="Whether the requesting user has pinned this agent.",
)
class AgentsListResponse(BaseModel):
)
is_pinned: bool = Field(
default=False,
description="Whether the requesting user has pinned this agent.",
)
class AgentsListResponse(BaseModel):
"""Response model for listing agents.
``total`` / ``page`` / ``page_size`` describe server-side pagination. When
@@ -115,7 +116,7 @@ class AgentUpdateRequest(BaseModel):
tool_groups: list[str] | None = Field(default=None, description="Updated tool group whitelist (config.yaml)")
class AgentSelectorResponse(BaseModel):
class AgentSelectorResponse(BaseModel):
"""Response for the homepage agent selector dropdown.
The list is fully curated by admins (``featured_order`` on each row).
@@ -125,10 +126,10 @@ class AgentSelectorResponse(BaseModel):
agents: list[AgentResponse] = Field(
default_factory=list,
description="Admin-curated agents, sorted by featured_order ascending.",
)
class AgentFeaturedRequest(BaseModel):
)
class AgentFeaturedRequest(BaseModel):
"""Request body for setting/clearing an agent's homepage featured rank."""
order: int | None = Field(
@@ -159,6 +160,41 @@ def _is_admin_user(request: Request) -> bool:
return getattr(user, "system_role", None) == "admin"
def _email_local_part(email: str | None) -> str | None:
"""Return the username portion (before ``@``) of an email address."""
if not email:
return None
at = email.find("@")
return email[:at] if at > 0 else email
async def _resolve_usernames(user_ids: set[str]) -> dict[str, str]:
"""Batch-resolve owner user ids to friendly usernames (email local part).
One lookup per distinct id; tolerates missing accounts (deleted users) and
an uninitialised auth provider by simply omitting those ids from the map.
"""
user_ids = {uid for uid in user_ids if uid}
if not user_ids:
return {}
try:
from app.gateway.deps import get_local_provider
provider = get_local_provider()
except Exception:
return {}
names: dict[str, str] = {}
for uid in user_ids:
try:
user = await provider.get_user(uid)
except Exception:
user = None
local = _email_local_part(getattr(user, "email", None)) if user is not None else None
if local:
names[uid] = local
return names
def _validate_display_name(name: str) -> str:
normalized = name.strip()
if not normalized:
@@ -348,7 +384,7 @@ def _ensure_agent_dir_from_record(record: dict[str, Any]) -> None:
logger.exception("Failed to self-heal agent dir for '%s'", agent_id)
def _record_to_response(
def _record_to_response(
record: dict[str, Any],
*,
include_soul: bool = False,
@@ -390,21 +426,21 @@ def _record_to_response(
tool_groups=extras["tool_groups"],
skills=extras["skills"],
featured_order=record.get("featured_order"),
)
def _sort_agent_responses(responses: list[AgentResponse], sort: str) -> list[AgentResponse]:
reverse = sort not in {"time_asc", "updated_asc", "name_asc"}
if sort in {"name_asc", "name_desc"}:
ordered = sorted(responses, key=lambda r: (r.name or "").lower(), reverse=reverse)
elif sort in {"updated_desc", "updated_asc"}:
ordered = sorted(responses, key=lambda r: str(r.updated_at or ""), reverse=reverse)
else:
ordered = sorted(responses, key=lambda r: str(r.created_at or ""), reverse=reverse)
return sorted(ordered, key=lambda r: 0 if r.is_pinned else 1)
async def _record_to_response_async(
)
def _sort_agent_responses(responses: list[AgentResponse], sort: str) -> list[AgentResponse]:
reverse = sort not in {"time_asc", "updated_asc", "name_asc"}
if sort in {"name_asc", "name_desc"}:
ordered = sorted(responses, key=lambda r: (r.name or "").lower(), reverse=reverse)
elif sort in {"updated_desc", "updated_asc"}:
ordered = sorted(responses, key=lambda r: str(r.updated_at or ""), reverse=reverse)
else:
ordered = sorted(responses, key=lambda r: str(r.created_at or ""), reverse=reverse)
return sorted(ordered, key=lambda r: 0 if r.is_pinned else 1)
async def _record_to_response_async(
record: dict[str, Any],
*,
include_soul: bool = False,
@@ -478,20 +514,23 @@ async def _build_list_responses(
_record_to_response(a, include_soul=False, include_extras=False) for a in records
]
ids = [r.id for r in responses]
extras = await store.get_extras_for(ids)
assignments = await tag_store.list_assignments("agent", ids)
favorite_ids = await store.list_favorite_ids(user_id)
pin_ids = await store.list_pin_ids(user_id)
for resp in responses:
extras = await store.get_extras_for(ids)
assignments = await tag_store.list_assignments("agent", ids)
favorite_ids = await store.list_favorite_ids(user_id)
pin_ids = await store.list_pin_ids(user_id)
usernames = await _resolve_usernames({r.user_id for r in responses if r.user_id})
for resp in responses:
cached = extras.get(resp.id)
if cached is not None:
resp.skills = cached.get("skills")
resp.tool_groups = cached.get("tool_groups")
resp.model = cached.get("model")
resp.tags = assignments.get(resp.id, [])
resp.is_favorite = resp.id in favorite_ids
resp.is_pinned = resp.id in pin_ids
return responses
resp.tags = assignments.get(resp.id, [])
resp.is_favorite = resp.id in favorite_ids
resp.is_pinned = resp.id in pin_ids
if resp.user_id:
resp.owner_username = usernames.get(resp.user_id)
return responses
@router.get(
@@ -509,14 +548,18 @@ async def list_agents(
pattern="^(mine|square)$",
description="Pagination scope. 'mine' = owned + favorited; 'square' = built-in/published public. Omit for the default combined visibility list.",
),
page: int | None = Query(default=None, ge=1, description="1-based page index. Providing this (or 'scope') enables server-side pagination."),
page_size: int = Query(default=20, ge=1, le=200, description="Items per page when paginating."),
sort: str = Query(
default="time_desc",
pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
description="Sort order. Default is newest created first.",
),
admin: bool = Query(
page: int | None = Query(default=None, ge=1, description="1-based page index. Providing this (or 'scope') enables server-side pagination."),
page_size: int = Query(default=20, ge=1, le=200, description="Items per page when paginating."),
sort: str = Query(
default="time_desc",
pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
description="Sort order. Default is newest created first.",
),
owner: str | None = Query(
default=None,
description="Keep only agents published by this user id (the card's publisher). Built-in agents never match.",
),
admin: bool = Query(
default=False,
description=(
"Admin-only override. When ``true`` and the caller is a system admin, "
@@ -543,11 +586,12 @@ async def list_agents(
scope=scope,
search=search,
tag_ids=tag_ids,
page=eff_page,
page_size=page_size,
admin=admin,
sort=sort,
)
page=eff_page,
page_size=page_size,
admin=admin,
sort=sort,
owner=owner,
)
responses = await _build_list_responses(store, tag_store, user_id, rows)
return AgentsListResponse(agents=responses, total=total, page=eff_page, page_size=page_size)
@@ -559,11 +603,13 @@ async def list_agents(
if search:
needle = search.strip().lower()
responses = [r for r in responses if needle in r.name.lower()]
if tag_ids:
wanted = set(tag_ids)
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
responses = _sort_agent_responses(responses, sort)
return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
if tag_ids:
wanted = set(tag_ids)
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
if owner and owner.strip():
responses = [r for r in responses if r.user_id == owner.strip()]
responses = _sort_agent_responses(responses, sort)
return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
except HTTPException:
raise
except Exception as e:
@@ -576,14 +622,14 @@ async def list_agents(
summary="Check Agent Name",
description="Validate an agent display name. Duplicate names are allowed.",
)
async def check_agent_name(name: str) -> dict[str, Any]:
_require_agents_api_enabled()
normalized = _validate_display_name(name)
return {"available": True, "name": normalized}
@router.get(
"/agents/selector",
async def check_agent_name(name: str) -> dict[str, Any]:
_require_agents_api_enabled()
normalized = _validate_display_name(name)
return {"available": True, "name": normalized}
@router.get(
"/agents/selector",
response_model=AgentSelectorResponse,
summary="Agent Homepage Selector",
description=(
@@ -635,12 +681,12 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
# Detail view legitimately reads SOUL.md + config.yaml — hop to a
# worker thread so the asyncio loop stays responsive under concurrent
# detail fetches.
response = await _record_to_response_async(record, include_soul=True, include_extras=True)
favorite_ids = await store.list_favorite_ids(user_id)
pin_ids = await store.list_pin_ids(user_id)
response.is_favorite = agent_id in favorite_ids
response.is_pinned = agent_id in pin_ids
return response
response = await _record_to_response_async(record, include_soul=True, include_extras=True)
favorite_ids = await store.list_favorite_ids(user_id)
pin_ids = await store.list_pin_ids(user_id)
response.is_favorite = agent_id in favorite_ids
response.is_pinned = agent_id in pin_ids
return response
except HTTPException:
raise
except Exception as e:
@@ -874,16 +920,16 @@ async def delete_agent(agent_id: str, request: Request) -> None:
await get_tag_store(request).unassign_all("agent", agent_id)
except Exception: # noqa: BLE001 — tag cleanup must not block deletion
pass
try:
await store.delete_favorites_for_agent(agent_id)
except Exception: # noqa: BLE001 — favorite cleanup must not block deletion
pass
try:
await store.delete_pins_for_agent(agent_id)
except Exception: # noqa: BLE001 - pin cleanup must not block deletion
pass
try:
await store.delete_extras_for_agent(agent_id)
try:
await store.delete_favorites_for_agent(agent_id)
except Exception: # noqa: BLE001 — favorite cleanup must not block deletion
pass
try:
await store.delete_pins_for_agent(agent_id)
except Exception: # noqa: BLE001 - pin cleanup must not block deletion
pass
try:
await store.delete_extras_for_agent(agent_id)
except Exception: # noqa: BLE001 — cache cleanup must not block deletion
pass
logger.info("Deleted agent '%s' from %s", agent_id, agent_dir)
@@ -925,7 +971,7 @@ async def favorite_agent(agent_id: str, request: Request) -> None:
summary="Unfavorite Agent",
description="Remove an agent from the current user's favorites. No-op when not favorited.",
)
async def unfavorite_agent(agent_id: str, request: Request) -> None:
async def unfavorite_agent(agent_id: str, request: Request) -> None:
_require_agents_api_enabled()
agent_id = _validate_storage_id(agent_id)
@@ -936,60 +982,60 @@ async def unfavorite_agent(agent_id: str, request: Request) -> None:
logger.info("User '%s' unfavorited agent '%s'", user_id, agent_id)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to unfavorite agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to unfavorite agent: {str(e)}") from e
@router.post(
"/agents/{agent_id}/pin",
status_code=204,
summary="Pin Agent",
description="Pin a visible agent to the top of the current user's lists. Idempotent.",
)
async def pin_agent(agent_id: str, request: Request) -> None:
_require_agents_api_enabled()
agent_id = _validate_storage_id(agent_id)
try:
store = get_agent_store(request)
user_id = _current_user_id(request)
record = await store.get_visible(agent_id, user_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
await store.add_pin(user_id, agent_id)
logger.info("User '%s' pinned agent '%s'", user_id, agent_id)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to pin agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to pin agent: {str(e)}") from e
@router.delete(
"/agents/{agent_id}/pin",
status_code=204,
summary="Unpin Agent",
description="Remove the current user's pin from an agent. No-op when not pinned.",
)
async def unpin_agent(agent_id: str, request: Request) -> None:
_require_agents_api_enabled()
agent_id = _validate_storage_id(agent_id)
try:
store = get_agent_store(request)
user_id = _current_user_id(request)
await store.remove_pin(user_id, agent_id)
logger.info("User '%s' unpinned agent '%s'", user_id, agent_id)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to unpin agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to unpin agent: {str(e)}") from e
@router.put(
"/agents/{agent_id}/featured",
except Exception as e:
logger.error("Failed to unfavorite agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to unfavorite agent: {str(e)}") from e
@router.post(
"/agents/{agent_id}/pin",
status_code=204,
summary="Pin Agent",
description="Pin a visible agent to the top of the current user's lists. Idempotent.",
)
async def pin_agent(agent_id: str, request: Request) -> None:
_require_agents_api_enabled()
agent_id = _validate_storage_id(agent_id)
try:
store = get_agent_store(request)
user_id = _current_user_id(request)
record = await store.get_visible(agent_id, user_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
await store.add_pin(user_id, agent_id)
logger.info("User '%s' pinned agent '%s'", user_id, agent_id)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to pin agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to pin agent: {str(e)}") from e
@router.delete(
"/agents/{agent_id}/pin",
status_code=204,
summary="Unpin Agent",
description="Remove the current user's pin from an agent. No-op when not pinned.",
)
async def unpin_agent(agent_id: str, request: Request) -> None:
_require_agents_api_enabled()
agent_id = _validate_storage_id(agent_id)
try:
store = get_agent_store(request)
user_id = _current_user_id(request)
await store.remove_pin(user_id, agent_id)
logger.info("User '%s' unpinned agent '%s'", user_id, agent_id)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to unpin agent '%s': %s", agent_id, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to unpin agent: {str(e)}") from e
@router.put(
"/agents/{agent_id}/featured",
response_model=AgentResponse,
summary="Set Agent Featured Rank (admin)",
description=(
@@ -1031,19 +1077,20 @@ async def set_agent_featured(
raise HTTPException(status_code=500, detail=f"Failed to set featured: {str(e)}") from e
class PrivateSquareListRequest(BaseModel):
class PrivateSquareListRequest(BaseModel):
"""Request body for listing all private-square agents."""
password: str = Field(..., description="Plain-text private-square password. Verified against the admin-configured hash before the listing is returned.")
search: str | None = Field(default=None, description="Case-insensitive fuzzy match on agent name (raw or desensitized).")
tag_ids: list[str] | None = Field(default=None, description="Keep agents carrying any of these tags (OR).")
page: int | None = Field(default=None, ge=1, description="1-based page index. Providing this enables server-side pagination.")
page_size: int = Field(default=20, ge=1, le=200, description="Items per page when paginating.")
sort: str = Field(
default="time_desc",
pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
description="Sort order. Default is newest created first.",
)
page: int | None = Field(default=None, ge=1, description="1-based page index. Providing this enables server-side pagination.")
page_size: int = Field(default=20, ge=1, le=200, description="Items per page when paginating.")
sort: str = Field(
default="time_desc",
pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
description="Sort order. Default is newest created first.",
)
owner: str | None = Field(default=None, description="Keep only agents published by this user id (the card's publisher).")
class AgentPrivateUpdateRequest(BaseModel):
@@ -1086,17 +1133,18 @@ async def list_private_square(body: PrivateSquareListRequest, request: Request)
scope="private",
search=body.search,
tag_ids=body.tag_ids,
page=body.page,
page_size=body.page_size,
sort=body.sort,
)
page=body.page,
page_size=body.page_size,
sort=body.sort,
owner=body.owner,
)
responses = await _build_list_responses(store, tag_store, user_id, rows)
return AgentsListResponse(agents=responses, total=total, page=body.page, page_size=body.page_size)
records = await store.list_private()
responses = await _build_list_responses(store, tag_store, user_id, records)
responses = _sort_agent_responses(responses, body.sort)
return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
responses = await _build_list_responses(store, tag_store, user_id, records)
responses = _sort_agent_responses(responses, body.sort)
return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
except HTTPException:
raise
except Exception as e:
@@ -0,0 +1,590 @@
"""Browser page context capture endpoints.
This router is intentionally isolated behind ``config.yaml``:
browser_context:
enabled: true
When disabled, mutating endpoints behave as if the integration is unavailable.
"""
from __future__ import annotations
import asyncio
import logging
import re
from datetime import UTC, datetime
from typing import Any, Literal
from urllib.parse import urlencode
from uuid import uuid4
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer, get_config, get_thread_store
from app.gateway.routers.thread_runs import RunCreateRequest
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
from app.gateway.routers.uploads import _make_file_sandbox_writable
from app.gateway.services import start_run
from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks
from deerflow.config.app_config import AppConfig
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.sandbox.sandbox_provider import get_sandbox_provider
from deerflow.uploads.manager import (
UnsafeUploadPathError,
ensure_uploads_dir,
normalize_filename,
open_upload_file_no_symlink,
upload_artifact_url,
upload_virtual_path,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/browser-context", tags=["browser-context"])
DEFAULT_BROWSER_CONTEXT_PROMPT = "\u8bf7\u57fa\u4e8e\u521a\u521a\u53d1\u9001\u7684\u7f51\u9875\u5185\u5bb9\uff0c\u5148\u7528\u4e2d\u6587\u603b\u7ed3\u8981\u70b9\uff0c\u5e76\u7b49\u5f85\u6211\u7ee7\u7eed\u63d0\u95ee\u3002"
class BrowserContextStatusResponse(BaseModel):
enabled: bool
max_content_chars: int
max_title_chars: int
max_url_chars: int
task_result_timeout_seconds: int
task_poll_interval_seconds: float
task_lease_seconds: int
tool_result_max_chars: int
class BrowserContextCaptureRequest(BaseModel):
thread_id: str | None = Field(default=None, description="Existing or caller-generated zncm thread id")
title: str | None = Field(default=None, description="Current browser page title")
url: str | None = Field(default=None, description="Current browser page URL")
content: str = Field(description="Captured page content, preferably Markdown")
content_type: Literal["markdown", "text"] = Field(default="markdown")
prompt: str | None = Field(default=None, description="Optional first question to ask in zncm after capture")
class BrowserContextFileInfo(BaseModel):
filename: str
size: int
path: str
virtual_path: str
artifact_url: str
class BrowserContextCaptureResponse(BaseModel):
success: bool
thread_id: str
file: BrowserContextFileInfo
chat_path: str
chat_url: str
browser_context_token: str
prompt: str
truncated: bool = False
message: str
class BrowserContextAskRequest(BaseModel):
thread_id: str | None = Field(default=None, description="Existing zncm thread id for follow-up questions")
title: str | None = Field(default=None, description="Current browser page title")
url: str | None = Field(default=None, description="Current browser page URL")
content: str = Field(description="Captured page content, preferably Markdown")
content_type: Literal["markdown", "text"] = Field(default="markdown")
question: str = Field(description="Question asked from the in-page assistant")
class BrowserContextAskResponse(BaseModel):
success: bool
thread_id: str
run_id: str
answer: str
truncated: bool = False
message: str
class BrowserContextTaskPayload(BaseModel):
id: str
kind: str = "fetch"
url: str
instruction: str = ""
session_id: str = ""
action: dict[str, Any] | None = None
created_at: str
updated_at: str
max_content_chars: int
class BrowserContextTaskNextResponse(BaseModel):
task: BrowserContextTaskPayload | None = None
class BrowserContextTaskResultRequest(BaseModel):
kind: str = Field(default="fetch", description="Task kind the extension processed: fetch or act")
title: str | None = Field(default=None, description="Captured page title")
url: str | None = Field(default=None, description="Final page URL after redirects")
content: str | None = Field(default=None, description="Captured page content")
content_type: Literal["markdown", "text"] = Field(default="markdown")
observation: dict[str, Any] | None = Field(default=None, description="Structured observation for an act step (elements, text, status)")
error: str | None = Field(default=None, description="Extension-side task error")
metadata: dict[str, Any] = Field(default_factory=dict)
class BrowserContextTaskResultResponse(BaseModel):
success: bool
task_id: str
status: str
truncated: bool = False
def _settings(config: AppConfig) -> BrowserContextConfig:
value = getattr(config, "browser_context", None)
if isinstance(value, BrowserContextConfig):
return value
if isinstance(value, dict):
return BrowserContextConfig.model_validate(value)
return BrowserContextConfig()
def _require_enabled(config: AppConfig) -> BrowserContextConfig:
settings = _settings(config)
if not settings.enabled:
raise HTTPException(status_code=404, detail="Browser context integration is disabled")
return settings
def _trim(value: str | None, limit: int) -> str:
if not value:
return ""
return value.strip()[:limit]
def _limit_utf8_bytes(value: str, max_bytes: int) -> str:
text = value
while len(text.encode("utf-8")) > max_bytes:
text = text[:-1]
return text
def _filename_part(value: str, fallback: str, *, max_bytes: int) -> str:
text = value.strip() or fallback
# Keep unicode word chars so Chinese titles remain readable, while removing
# path separators and shell-hostile punctuation.
text = re.sub(r"[^\w.-]+", "-", text, flags=re.UNICODE).strip("-._")
text = _limit_utf8_bytes(text, max_bytes)
return text or fallback
def _build_markdown(
*,
title: str,
url: str,
content: str,
content_type: str,
captured_at: str,
truncated: bool,
) -> str:
lines = [
f"# {title or 'Captured Web Page'}",
"",
f"- Source URL: {url or '(unknown)'}",
f"- Captured at: {captured_at}",
]
if truncated:
lines.append("- Note: Content was truncated by the browser_context.max_content_chars limit.")
lines.extend(["", "---", ""])
if content_type == "markdown":
lines.append(content.strip())
else:
lines.append("```text")
lines.append(content.strip())
lines.append("```")
lines.append("")
return "\n".join(lines)
def _build_ask_prompt(
*,
title: str,
url: str,
content: str,
content_type: str,
captured_at: str,
truncated: bool,
) -> str:
note = "页面内容已按配置截断,请只基于可见内容回答。" if truncated else "页面内容如下。"
body = content.strip()
if content_type == "text":
body = f"```text\n{body}\n```"
return "\n\n".join(
[
"你是 zncm 页面问答助手。请直接基于当前网页内容回答用户问题。",
"如果网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。",
f"页面标题:{title or '未知标题'}",
f"页面 URL:{url or '未知 URL'}",
f"读取时间:{captured_at}",
f"说明:{note}",
"当前网页内容:",
body,
]
)
def _message_type(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("type") or message.get("role") or "").lower()
return str(getattr(message, "type", "") or getattr(message, "role", "") or "").lower()
def _content_to_text(content: Any) -> str:
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: list[str] = []
for item in content:
if isinstance(item, str):
parts.append(item)
elif isinstance(item, dict):
if isinstance(item.get("text"), str):
parts.append(item["text"])
elif item.get("type") == "text" and isinstance(item.get("content"), str):
parts.append(item["content"])
return "\n".join(part.strip() for part in parts if part and part.strip()).strip()
return ""
def _latest_ai_answer(values: dict[str, Any]) -> str:
messages = values.get("messages")
if not isinstance(messages, list):
return ""
for message in reversed(messages):
if _message_type(message) not in {"ai", "assistant"}:
continue
answer = _content_to_text(message.get("content") if isinstance(message, dict) else getattr(message, "content", None))
if answer:
return answer
return ""
async def _sync_to_sandbox_if_needed(thread_id: str, virtual_path: str, file_path, data: bytes) -> None:
sandbox_provider = get_sandbox_provider()
if bool(getattr(sandbox_provider, "uses_thread_data_mounts", False)):
return
sandbox_id = sandbox_provider.acquire(thread_id)
sandbox = sandbox_provider.get(sandbox_id)
if sandbox is None:
raise HTTPException(status_code=500, detail="Failed to acquire sandbox")
_make_file_sandbox_writable(file_path)
sandbox.update_file(virtual_path, data)
@router.get("/status", response_model=BrowserContextStatusResponse)
async def get_browser_context_status(
config: AppConfig = Depends(get_config),
) -> BrowserContextStatusResponse:
settings = _settings(config)
return BrowserContextStatusResponse(
enabled=settings.enabled,
max_content_chars=settings.max_content_chars,
max_title_chars=settings.max_title_chars,
max_url_chars=settings.max_url_chars,
task_result_timeout_seconds=settings.task_result_timeout_seconds,
task_poll_interval_seconds=settings.task_poll_interval_seconds,
task_lease_seconds=settings.task_lease_seconds,
tool_result_max_chars=settings.tool_result_max_chars,
)
@router.get("/tasks/next", response_model=BrowserContextTaskNextResponse)
@require_permission("runs", "create")
async def claim_browser_context_task(
request: Request,
worker_id: str | None = None,
config: AppConfig = Depends(get_config),
) -> BrowserContextTaskNextResponse:
settings = _require_enabled(config)
prune_browser_tasks()
task = claim_next_browser_task(
user_id=get_effective_user_id(),
worker_id=worker_id or "chrome-extension",
lease_seconds=settings.task_lease_seconds,
)
if task is None:
return BrowserContextTaskNextResponse(task=None)
return BrowserContextTaskNextResponse(task=BrowserContextTaskPayload.model_validate(task))
@router.post("/tasks/{task_id}/result", response_model=BrowserContextTaskResultResponse)
@require_permission("runs", "create")
async def submit_browser_context_task_result(
task_id: str,
body: BrowserContextTaskResultRequest,
request: Request,
config: AppConfig = Depends(get_config),
) -> BrowserContextTaskResultResponse:
settings = _require_enabled(config)
error = _trim(body.error, 4000)
content = body.content or ""
is_act = (body.kind or "fetch") == "act"
if not error and not is_act and not content.strip():
raise HTTPException(status_code=400, detail="Captured page content is empty")
if not error and is_act and not isinstance(body.observation, dict):
raise HTTPException(status_code=400, detail="Act task result requires an observation")
task = complete_browser_task(
task_id=task_id,
user_id=get_effective_user_id(),
title=_trim(body.title, settings.max_title_chars),
url=_trim(body.url, settings.max_url_chars),
content=content,
content_type=body.content_type,
error=error,
metadata=body.metadata,
observation=body.observation,
)
if task is None:
raise HTTPException(status_code=404, detail=f"Browser task {task_id} not found")
result = task.get("result") or {}
return BrowserContextTaskResultResponse(
success=True,
task_id=task_id,
status=str(task.get("status") or "unknown"),
truncated=bool(result.get("truncated")),
)
@router.post("/capture", response_model=BrowserContextCaptureResponse)
@require_permission("threads", "write")
async def capture_browser_context(
body: BrowserContextCaptureRequest,
request: Request,
config: AppConfig = Depends(get_config),
) -> BrowserContextCaptureResponse:
settings = _require_enabled(config)
raw_content = (body.content or "").strip()
if not raw_content:
raise HTTPException(status_code=400, detail="Captured page content is empty")
title = _trim(body.title, settings.max_title_chars)
url = _trim(body.url, settings.max_url_chars)
truncated = len(raw_content) > settings.max_content_chars
content = raw_content[: settings.max_content_chars] if truncated else raw_content
requested_thread_id = (body.thread_id or "").strip() or None
if requested_thread_id:
thread_store = get_thread_store(request)
allowed = await thread_store.check_access(
requested_thread_id,
get_effective_user_id(),
require_existing=False,
)
if not allowed:
raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found")
thread_response = await create_thread(
ThreadCreateRequest(
thread_id=requested_thread_id,
metadata={
"source": "browser_context",
"browser_context_url": url,
"browser_context_title": title,
},
),
request,
)
thread_id = thread_response.thread_id
captured_at = datetime.now(UTC).isoformat()
markdown = _build_markdown(
title=title,
url=url,
content=content,
content_type=body.content_type,
captured_at=captured_at,
truncated=truncated,
)
data = markdown.encode("utf-8")
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
prefix = _filename_part(settings.filename_prefix, "webpage", max_bytes=40)
title_part = _filename_part(title, "page", max_bytes=120)
filename = normalize_filename(f"{prefix}-{timestamp}-{title_part}.md")
try:
uploads_dir = ensure_uploads_dir(thread_id)
file_path, fh = open_upload_file_no_symlink(uploads_dir, filename)
with fh:
fh.write(data)
except UnsafeUploadPathError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc:
logger.exception("Failed to save browser context capture for thread %s", thread_id)
raise HTTPException(status_code=500, detail="Failed to save captured page") from exc
virtual_path = upload_virtual_path(filename)
await _sync_to_sandbox_if_needed(thread_id, virtual_path, file_path, data)
prompt = (body.prompt or "").strip() or DEFAULT_BROWSER_CONTEXT_PROMPT
token = uuid4().hex
route_base = settings.route_base.rstrip("/") or "/page/workspace"
chat_path = f"{route_base}/chats/{thread_id}"
query = urlencode(
{
"browser_context": token,
"bc_filename": filename,
"bc_size": str(len(data)),
"bc_path": virtual_path,
"bc_prompt": prompt,
}
)
chat_url = f"#{chat_path}?{query}"
return BrowserContextCaptureResponse(
success=True,
thread_id=thread_id,
file=BrowserContextFileInfo(
filename=filename,
size=len(data),
path=virtual_path,
virtual_path=virtual_path,
artifact_url=upload_artifact_url(thread_id, filename),
),
chat_path=chat_path,
chat_url=chat_url,
browser_context_token=token,
prompt=prompt,
truncated=truncated,
message="Captured page saved as a zncm upload",
)
@router.post("/ask", response_model=BrowserContextAskResponse)
@require_permission("runs", "create")
async def ask_browser_context(
body: BrowserContextAskRequest,
request: Request,
config: AppConfig = Depends(get_config),
) -> BrowserContextAskResponse:
settings = _require_enabled(config)
raw_content = (body.content or "").strip()
if not raw_content:
raise HTTPException(status_code=400, detail="Captured page content is empty")
question = (body.question or "").strip()
if not question:
raise HTTPException(status_code=400, detail="Question is empty")
title = _trim(body.title, settings.max_title_chars)
url = _trim(body.url, settings.max_url_chars)
content_limit = max(1_000, min(settings.max_content_chars, settings.tool_result_max_chars))
truncated = len(raw_content) > content_limit
content = raw_content[:content_limit] if truncated else raw_content
requested_thread_id = (body.thread_id or "").strip() or None
if requested_thread_id:
thread_store = get_thread_store(request)
allowed = await thread_store.check_access(
requested_thread_id,
get_effective_user_id(),
require_existing=False,
)
if not allowed:
raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found")
thread_response = await create_thread(
ThreadCreateRequest(
thread_id=requested_thread_id,
assistant_id="lead_agent",
metadata={
"source": "browser_context_page_assistant",
"browser_context_url": url,
"browser_context_title": title,
},
),
request,
)
thread_id = thread_response.thread_id
captured_at = datetime.now(UTC).isoformat()
is_follow_up = requested_thread_id is not None
if is_follow_up:
prompt_prefix = "\n\n".join(
[
"你是 zncm 页面问答助手。请继续基于本会话里已经读取的网页内容回答用户问题。",
"如果本会话里的网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。",
f"当前页面标题:{title or '未知标题'}",
f"当前页面 URL:{url or '未知 URL'}",
f"本轮提问时间:{captured_at}",
]
)
else:
prompt_prefix = _build_ask_prompt(
title=title,
url=url,
content=content,
content_type=body.content_type,
captured_at=captured_at,
truncated=truncated,
)
run_body = RunCreateRequest(
assistant_id="lead_agent",
input={
"messages": [
{
"type": "human",
"content": question,
"additional_kwargs": {
"prompt_prefix": prompt_prefix,
},
}
]
},
metadata={
"source": "browser_context_page_assistant",
"browser_context_url": url,
"browser_context_title": title,
"browser_context_truncated": truncated,
},
stream_mode=["values"],
on_disconnect="continue",
multitask_strategy="reject",
)
record = await start_run(run_body, thread_id, request)
if record.task is not None:
try:
await record.task
except asyncio.CancelledError:
pass
checkpointer = get_checkpointer(request)
values: dict[str, Any] = {}
try:
checkpoint_tuple = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
if checkpoint_tuple is not None:
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
except Exception:
logger.exception("Failed to fetch browser context answer for run %s", record.run_id)
answer = _latest_ai_answer(values)
if not answer:
answer = record.error or "zncm did not return an answer."
return BrowserContextAskResponse(
success=bool(answer),
thread_id=thread_id,
run_id=record.run_id,
answer=answer,
truncated=truncated,
message="zncm page assistant answer completed",
)
@@ -0,0 +1,521 @@
"""Knowledge base API (``/api/knowledge``).
Sediments threads / searches into an obsidian-wiki compatible vault, mirrors
metadata into the database, and serves listing, detail, edit, delete, search
and graph export. The knowledge base is **global / shared** — no per-user
isolation; writes record ``created_by`` / ``updated_by`` for audit.
All obsidian-wiki specifics live in :mod:`deerflow.knowledge`; this router only
adapts HTTP ↔ the :class:`~deerflow.knowledge.service.KnowledgeService`.
"""
from __future__ import annotations
import logging
import mimetypes
from typing import Any
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from app.gateway.authz import require_auth, require_permission
from app.gateway.deps import get_checkpointer, get_knowledge_service, get_local_provider, get_thread_store
from app.gateway.utils import sanitize_log_param
from deerflow.knowledge.service import EmptyThreadError
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.references import build_reference_batches
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/knowledge", tags=["knowledge"])
# --------------------------------------------------------------------------- #
# Request / response models
# --------------------------------------------------------------------------- #
class IngestThreadRequest(BaseModel):
mode: str = Field(default="summary", description="summary | full")
title: str | None = None
status: str = Field(default="approved", description="draft | approved")
tags: list[str] = Field(default_factory=list)
include_sources: bool = True
background: bool = Field(
default=False,
description="When true, sediment the thread asynchronously in a background task and return immediately.",
)
model_name: str | None = Field(default=None, description="可选:指定用于知识提炼的大模型名称;留空使用默认配置")
class BatchIngestRequest(BaseModel):
limit: int = Field(default=100, ge=1, le=500)
offset: int = Field(default=0, ge=0)
status: str = "approved"
include_sources: bool = True
skip_existing: bool = True
mode: str = "summary"
model_name: str | None = Field(default=None, description="可选:指定用于知识提炼的大模型名称;留空使用默认配置")
class SearchResultItem(BaseModel):
title: str | None = None
url: str | None = None
snippet: str | None = None
tool_name: str | None = None
score: float | None = None
class IngestSearchRequest(BaseModel):
query: str
results: list[SearchResultItem] = Field(default_factory=list)
thread_id: str | None = None
message_id: str | None = None
title: str | None = None
tags: list[str] = Field(default_factory=list)
status: str = "approved"
class CreateNoteRequest(BaseModel):
title: str
content_md: str
summary: str | None = None
tags: list[str] = Field(default_factory=list)
status: str = "approved"
class UpdateNoteRequest(BaseModel):
title: str | None = None
content_md: str | None = None
summary: str | None = None
tags: list[str] | None = None
status: str | None = None
class SearchRequest(BaseModel):
query: str
mode: str = Field(default="keyword", description="keyword | vector | hybrid (vector/hybrid fall back to keyword in v1)")
limit: int = Field(default=8, ge=1, le=50)
class ResolveWikilinksRequest(BaseModel):
targets: list[str] = Field(default_factory=list, description="`[[wikilink]]` targets to resolve to note ids / vault pages.")
# --------------------------------------------------------------------------- #
# Helpers
# --------------------------------------------------------------------------- #
async def _read_thread_values(checkpointer, thread_id: str) -> dict[str, Any]:
"""Return serialized channel values for a thread, or raise 404/500."""
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
except Exception:
logger.exception("Failed to read thread %s for knowledge ingest", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to read thread state")
if checkpoint_tuple is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
return serialize_channel_values(checkpoint.get("channel_values", {}) or {})
async def _enrich_actor_names(note: dict[str, Any]) -> None:
"""Resolve created_by/updated_by user ids to a human label (email).
Adds ``created_by_name`` / ``updated_by_name`` to the note dict so the UI can
show a person instead of a raw UUID. Best-effort: unresolved ids fall back to
``None`` and the frontend then shows a shortened id.
"""
provider = get_local_provider()
cache: dict[str, str | None] = {}
async def name_of(uid: str | None) -> str | None:
if not uid:
return None
if uid not in cache:
try:
user = await provider.get_user(uid)
cache[uid] = user.email if user else None
except Exception:
cache[uid] = None
return cache[uid]
note["created_by_name"] = await name_of(note.get("created_by"))
note["updated_by_name"] = await name_of(note.get("updated_by"))
def _messages_and_meta(values: dict[str, Any]) -> tuple[list[dict[str, Any]], str | None, list[dict[str, Any]]]:
messages = values.get("messages")
if not isinstance(messages, list):
messages = []
title = values.get("title") if isinstance(values.get("title"), str) else None
batches = build_reference_batches(messages) if messages else []
return messages, title, batches
async def _ingest_thread_background(service, checkpointer, thread_id: str, body: IngestThreadRequest, created_by: str | None) -> None:
"""Sediment a thread into a note without an HTTP context.
Runs after the response is sent (FastAPI ``BackgroundTasks``); ``created_by``
must be captured by the request handler since the user contextvar is gone
by the time this executes. All failures are swallowed (logged) — the user
has already been told the save was queued.
"""
try:
values = await _read_thread_values(checkpointer, thread_id)
messages, title, batches = _messages_and_meta(values)
if not messages:
logger.warning("Background knowledge ingest skipped: thread %s has no messages", sanitize_log_param(thread_id))
return
await service.capture_thread(
thread_id=thread_id,
messages=messages,
thread_title=title,
mode=body.mode,
title=body.title,
status=body.status,
tags=body.tags,
include_sources=body.include_sources,
reference_batches=batches,
created_by=created_by,
model_name=body.model_name,
)
except EmptyThreadError:
logger.warning("Background knowledge ingest skipped: thread %s has no sediment-able content", sanitize_log_param(thread_id))
except Exception:
logger.exception("Background knowledge ingest failed for thread %s", sanitize_log_param(thread_id))
# --------------------------------------------------------------------------- #
# Ingest
# --------------------------------------------------------------------------- #
@router.post("/ingest/thread/{thread_id}")
@require_auth
@require_permission("threads", "read", owner_check=True)
async def ingest_thread(thread_id: str, body: IngestThreadRequest, request: Request, background_tasks: BackgroundTasks) -> dict[str, Any]:
"""Sediment a single thread into a knowledge note.
With ``background=true`` the heavy capture (read checkpoint → extract →
write vault + DB) is deferred to a background task and the endpoint returns
``{"queued": true}`` immediately, so the user's "保存" click never blocks.
"""
service = get_knowledge_service(request)
checkpointer = get_checkpointer(request)
if body.background:
# Capture the user id now — the contextvar is gone once the task runs.
background_tasks.add_task(
_ingest_thread_background,
service,
checkpointer,
thread_id,
body,
get_effective_user_id(),
)
return {"queued": True}
values = await _read_thread_values(checkpointer, thread_id)
messages, title, batches = _messages_and_meta(values)
if not messages:
raise HTTPException(status_code=400, detail="Thread has no messages to sediment")
try:
result = await service.capture_thread(
thread_id=thread_id,
messages=messages,
thread_title=title,
mode=body.mode,
title=body.title,
status=body.status,
tags=body.tags,
include_sources=body.include_sources,
reference_batches=batches,
created_by=get_effective_user_id(),
model_name=body.model_name,
)
except EmptyThreadError:
raise HTTPException(status_code=400, detail="Thread has no sediment-able content")
return {
"note": result["note"],
"created": result["created"],
"notes": result.get("notes", [result["note"]]),
"llm_fallback": result.get("llm_fallback", False),
}
@router.post("/ingest/threads/batch")
@require_auth
async def ingest_threads_batch(body: BatchIngestRequest, request: Request) -> dict[str, Any]:
"""Batch-sediment the caller's recent threads into knowledge notes."""
service = get_knowledge_service(request)
checkpointer = get_checkpointer(request)
thread_store = get_thread_store(request)
threads = await thread_store.search(limit=body.limit, offset=body.offset)
created = skipped = failed = 0
# Notes that landed via the rule-based fallback because the chosen model
# produced nothing usable (timeout / API error / non-JSON). Surfaced so the
# caller can tell the user "X 条因大模型异常改用了规则抽取".
llm_fallback = 0
items: list[dict[str, Any]] = []
for meta in threads:
thread_id = meta.get("thread_id")
if not thread_id:
continue
try:
values = await _read_thread_values(checkpointer, thread_id)
messages, title, batches = _messages_and_meta(values)
if not messages:
skipped += 1
items.append({"thread_id": thread_id, "note_id": None, "status": "skipped"})
continue
result = await service.capture_thread(
thread_id=thread_id,
messages=messages,
thread_title=title or meta.get("display_name"),
mode=body.mode,
status=body.status,
include_sources=body.include_sources,
reference_batches=batches,
created_by=get_effective_user_id(),
skip_existing=body.skip_existing,
model_name=body.model_name,
)
if result.get("llm_fallback"):
llm_fallback += 1
if result["created"]:
created += 1
items.append({"thread_id": thread_id, "note_id": result["note"]["id"], "status": "created", "llm_fallback": bool(result.get("llm_fallback"))})
else:
skipped += 1
items.append({"thread_id": thread_id, "note_id": result["note"]["id"], "status": "skipped"})
except EmptyThreadError:
skipped += 1
items.append({"thread_id": thread_id, "note_id": None, "status": "skipped"})
except HTTPException:
failed += 1
items.append({"thread_id": thread_id, "note_id": None, "status": "failed"})
except Exception:
logger.exception("Batch ingest failed for thread %s", sanitize_log_param(str(thread_id)))
failed += 1
items.append({"thread_id": thread_id, "note_id": None, "status": "failed"})
return {"ok": True, "processed": len(items), "created": created, "skipped": skipped, "failed": failed, "llm_fallback": llm_fallback, "items": items}
@router.post("/ingest/search")
@require_auth
async def ingest_search(body: IngestSearchRequest, request: Request) -> dict[str, Any]:
"""Sediment search/tool results as a reference note."""
service = get_knowledge_service(request)
note = await service.capture_search(
query=body.query,
results=[r.model_dump() for r in body.results],
thread_id=body.thread_id,
message_id=body.message_id,
title=body.title,
tags=body.tags,
status=body.status,
created_by=get_effective_user_id(),
)
return {"note": note}
# --------------------------------------------------------------------------- #
# CRUD
# --------------------------------------------------------------------------- #
@router.post("/notes")
@require_auth
async def create_note(body: CreateNoteRequest, request: Request) -> dict[str, Any]:
service = get_knowledge_service(request)
note = await service.create_manual_note(
title=body.title,
content_md=body.content_md,
summary=body.summary,
tags=body.tags,
status=body.status,
created_by=get_effective_user_id(),
)
return {"note": note}
@router.get("/notes")
@require_auth
async def list_notes(
request: Request,
keyword: str | None = None,
tag: str | None = None,
source_type: str | None = None,
status: str | None = None,
limit: int = 20,
offset: int = 0,
) -> dict[str, Any]:
service = get_knowledge_service(request)
limit = max(1, min(limit, 100))
offset = max(0, offset)
return await service.list_notes(keyword=keyword, tag=tag, source_type=source_type, status=status, limit=limit, offset=offset)
@router.get("/notes/{note_id}")
@require_auth
async def get_note(note_id: str, request: Request) -> dict[str, Any]:
service = get_knowledge_service(request)
note = await service.get_note(note_id)
if note is None:
raise HTTPException(status_code=404, detail="Knowledge note not found")
await _enrich_actor_names(note)
return note
@router.put("/notes/{note_id}")
@require_auth
async def update_note(note_id: str, body: UpdateNoteRequest, request: Request) -> dict[str, Any]:
service = get_knowledge_service(request)
note = await service.update_note(
note_id,
title=body.title,
content_md=body.content_md,
summary=body.summary,
tags=body.tags,
status=body.status,
updated_by=get_effective_user_id(),
)
if note is None:
raise HTTPException(status_code=404, detail="Knowledge note not found")
return note
@router.delete("/notes/{note_id}")
@require_auth
async def delete_note(note_id: str, request: Request, hard: bool = False) -> dict[str, Any]:
service = get_knowledge_service(request)
ok = await service.delete_note(note_id, updated_by=get_effective_user_id(), hard=hard)
if not ok:
raise HTTPException(status_code=404, detail="Knowledge note not found")
return {"ok": True, "note_id": note_id}
# --------------------------------------------------------------------------- #
# Search & graph
# --------------------------------------------------------------------------- #
@router.post("/search")
@require_auth
async def search(body: SearchRequest, request: Request) -> dict[str, Any]:
service = get_knowledge_service(request)
items = await service.search(body.query, mode=body.mode, limit=body.limit)
return {"items": items}
# --------------------------------------------------------------------------- #
# Wikilinks ([[...]] resolution + vault meta-page viewer)
# --------------------------------------------------------------------------- #
@router.post("/resolve")
@require_auth
async def resolve_wikilinks(body: ResolveWikilinksRequest, request: Request) -> dict[str, Any]:
"""Resolve ``[[wikilink]]`` targets to navigable destinations.
Each target resolves to a knowledge note (``kind=note``), a vault meta/hub
page (``kind=vault``), or nothing (``kind=missing``). Powers clickable
wikilinks in the knowledge UI.
"""
service = get_knowledge_service(request)
return {"resolved": await service.resolve_wikilinks(body.targets)}
@router.get("/notes/{note_id}/wikilinks")
@require_auth
async def resolve_note_wikilinks(note_id: str, request: Request) -> dict[str, Any]:
"""Resolve every ``[[wikilink]]`` found in a note's body in one call."""
service = get_knowledge_service(request)
resolved = await service.resolve_note_wikilinks(note_id)
if resolved is None:
raise HTTPException(status_code=404, detail="Knowledge note not found")
return {"resolved": resolved}
@router.get("/vault-page")
@require_auth
async def get_vault_page(request: Request, path: str) -> dict[str, Any]:
"""Return a raw vault Markdown meta/hub page (no DB note) by its path."""
service = get_knowledge_service(request)
page = service.read_vault_page(path)
if page is None:
raise HTTPException(status_code=404, detail="Vault page not found")
return page
class MergeRequest(BaseModel):
primary_id: str
duplicate_ids: list[str] = Field(default_factory=list)
@router.get("/duplicates")
@require_auth
async def list_duplicates(request: Request) -> dict[str, Any]:
"""Return suggested duplicate groups (phase 3, heuristic)."""
service = get_knowledge_service(request)
return {"groups": await service.find_duplicates()}
@router.post("/merge")
@require_auth
async def merge_notes(body: MergeRequest, request: Request) -> dict[str, Any]:
"""Merge duplicate notes into a primary note (sources moved, dups archived)."""
service = get_knowledge_service(request)
note = await service.merge_notes(body.primary_id, body.duplicate_ids, updated_by=get_effective_user_id())
if note is None:
raise HTTPException(status_code=404, detail="Primary note not found")
return {"note": note}
@router.post("/reindex")
@require_auth
async def reindex_embeddings(request: Request) -> dict[str, Any]:
"""Rebuild the vector index for all notes (maintenance; no-op if embeddings off)."""
service = get_knowledge_service(request)
return await service.reindex_all()
@router.get("/notes/{note_id}/versions")
@require_auth
async def list_note_versions(note_id: str, request: Request) -> dict[str, Any]:
"""Return the edit history (pre-update snapshots) for a note."""
service = get_knowledge_service(request)
return {"items": await service.list_versions(note_id)}
@router.get("/graph")
@require_auth
async def get_graph(request: Request) -> dict[str, Any]:
"""Return product knowledge-graph data (notes + entities + relations)."""
service = get_knowledge_service(request)
return await service.get_graph()
@router.post("/export/graph")
@require_auth
async def export_graph(request: Request) -> dict[str, Any]:
service = get_knowledge_service(request)
result = await service.export_graph()
base = "/api/knowledge/export/files"
return {
"ok": True,
"files": {
"graph_json": f"{base}/graph.json",
"graph_html": f"{base}/graph.html",
"graph_graphml": f"{base}/graph.graphml",
"cypher": f"{base}/cypher.txt",
},
"stats": result["stats"],
}
@router.get("/export/files/{filename}")
@require_auth
async def export_file(filename: str, request: Request) -> FileResponse:
service = get_knowledge_service(request)
path = service.export_file_path(filename)
if path is None:
# Lazily build the export on first access so the iframe never 404s.
await service.export_graph()
path = service.export_file_path(filename)
if path is None:
raise HTTPException(status_code=404, detail="Export file not found")
media_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
return FileResponse(path, media_type=media_type)
@@ -0,0 +1,97 @@
"""Public (unauthenticated) read-only user leaderboard.
Mounted under the ``/api/public/`` prefix so the auth middleware lets this route
through without a session cookie. Powers the public "publish" page for the user
leaderboard — a view-only snapshot with no admin controls and no mutations.
The snapshot honors the same system-wide leaderboard settings (excluded users,
scheduled/admin/failed inclusion) as the authenticated admin endpoint, so any
accounts an admin chose to hide stay hidden on the public page as well.
"""
from __future__ import annotations
import hashlib
import logging
from fastapi import APIRouter, HTTPException, Query
from app.gateway.routers.admin_users import (
AdminLeaderboardResponse,
_load_or_schedule_daily_leaderboard,
_resolve_range,
)
from deerflow.config.system_settings import load_system_settings
from deerflow.persistence.engine import get_session_factory
logger = logging.getLogger(__name__)
# Whitelisted under ``auth_middleware._PUBLIC_PATH_PREFIXES`` (``/api/public/``).
router = APIRouter(prefix="/api/public/leaderboard", tags=["public-leaderboard"])
def _mask_email(email: str | None) -> str:
"""Reduce an email to its local part so the public page can show a
recognizable handle without leaking the full address (domain).
The leaderboard only makes sense if it names its top users, so the local
part is kept intentionally; the privacy gain is dropping the domain and the
raw account id from the unauthenticated payload.
"""
if not email:
return ""
return email.split("@", 1)[0].strip() or ""
def _anonymize(resp: AdminLeaderboardResponse) -> AdminLeaderboardResponse:
"""Strip PII from a leaderboard snapshot before it leaves over the public,
unauthenticated endpoint: email → local part, account/thread ids → opaque
short hashes (kept stable so they still work as React keys)."""
def hashed(value: str | None) -> str:
if not value:
return ""
return hashlib.sha1(value.encode("utf-8")).hexdigest()[:12]
seen_users: set[int] = set()
for bucket in (resp.users_by_activity, resp.users_by_questions, resp.users_by_tokens):
for user in bucket:
if id(user) in seen_users:
continue
seen_users.add(id(user))
user.email = _mask_email(user.email)
user.user_id = hashed(user.user_id)
for bucket in (resp.recent_questions, resp.frequent_questions):
for question in bucket:
question.email = _mask_email(question.email)
question.user_id = hashed(question.user_id)
question.thread_id = ""
question.run_id = ""
return resp
@router.get("", response_model=AdminLeaderboardResponse)
@router.get("/", response_model=AdminLeaderboardResponse)
async def get_public_leaderboard(
days: int = Query(default=30, ge=1, le=365),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
limit: int = Query(default=20, ge=5, le=100),
) -> AdminLeaderboardResponse:
"""Return a read-only leaderboard snapshot for the public publish page."""
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
since_dt, until_dt, _range_days = _resolve_range(days, since, until)
resp = await _load_or_schedule_daily_leaderboard(
session_factory,
settings,
since_dt,
until_dt,
limit=limit,
)
return _anonymize(resp)
@@ -56,6 +56,44 @@ def _is_admin_user(request: Request) -> bool:
return getattr(user, "system_role", None) == "admin"
def _email_local_part(email: str | None) -> str | None:
"""Return the username portion (before ``@``) of an email address."""
if not email:
return None
at = email.find("@")
return email[:at] if at > 0 else email
async def _attach_owner_usernames(responses: list["SkillResponse"]) -> list["SkillResponse"]:
"""Resolve each custom skill's ``owner_user_id`` to a friendly username.
One lookup per distinct owner id; tolerates deleted accounts and an
uninitialised auth provider by leaving ``owner_username`` as None.
"""
owner_ids = {r.owner_user_id for r in responses if r.owner_user_id}
if not owner_ids:
return responses
try:
from app.gateway.deps import get_local_provider
provider = get_local_provider()
except Exception:
return responses
names: dict[str, str] = {}
for uid in owner_ids:
try:
user = await provider.get_user(uid)
except Exception:
user = None
local = _email_local_part(getattr(user, "email", None)) if user is not None else None
if local:
names[uid] = local
for resp in responses:
if resp.owner_user_id:
resp.owner_username = names.get(resp.owner_user_id)
return responses
def _can_view_skill_source(
category: SkillCategory,
owner_user_id: str | None,
@@ -111,6 +149,7 @@ class SkillResponse(BaseModel):
"""Response model for skill information."""
name: str = Field(..., description="Name of the skill")
name_zh: str | None = Field(default=None, description="用户配置或大模型生成的中文名称")
description: str = Field(..., description="Description of what the skill does")
license: str | None = Field(None, description="License information")
category: SkillCategory = Field(..., description="Category of the skill (public or custom)")
@@ -125,6 +164,7 @@ class SkillResponse(BaseModel):
featured: bool = Field(default=False, description="Whether this skill is pinned to the top of the list (sort only)")
source: str = Field(default="upload", description="Origin: 'agent' for agent-created, 'upload' for user-uploaded")
owner_user_id: str | None = Field(default=None, description="Custom skill owner (null = built-in or legacy)")
owner_username: str | None = Field(default=None, description="Publisher username (email local part). Null for built-in skills.")
published: bool = Field(default=False, description="Whether the owner has made this skill visible to other users")
builtin: bool = Field(default=False, description="True when owner_user_id is null (no human owner)")
detail: str | None = Field(default=None, description="Human-authored long-form detail text (Markdown), editable by owner/admin")
@@ -267,11 +307,13 @@ def _skill_to_response(
published = False
builtin = skill.category != SkillCategory.CUSTOM
detail: str | None = None
name_zh: str | None = None
if ownership is not None:
owner_user_id = ownership.get("owner_user_id")
published = bool(ownership.get("published", False))
builtin = bool(ownership.get("builtin", owner_user_id is None))
detail = ownership.get("detail")
name_zh = ownership.get("name_zh")
display = None if display_override is _UNSET else display_override
if display_override is _UNSET:
@@ -284,6 +326,7 @@ def _skill_to_response(
return SkillResponse(
name=skill.name,
name_zh=name_zh,
description=skill.description,
license=skill.license,
category=skill.category,
@@ -697,7 +740,8 @@ async def _attach_and_filter_tags(
resp.tags = assignments.get(resp.name, [])
if search:
needle = search.strip().lower()
responses = [r for r in responses if needle in r.name.lower()]
# Match the English name OR the Chinese display name so 中文 search works.
responses = [r for r in responses if needle in r.name.lower() or (r.name_zh or "").lower().find(needle) >= 0]
if tag_ids:
wanted = set(tag_ids)
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
@@ -726,7 +770,7 @@ async def list_skills(
config: AppConfig = Depends(get_config),
skill_store: SkillStore = Depends(get_skill_store),
tag_store: TagStore = Depends(get_tag_store),
search: str | None = Query(default=None, description="Case-insensitive fuzzy match on skill name"),
search: str | None = Query(default=None, description="Case-insensitive fuzzy match on skill name or Chinese name (name_zh)"),
tag_ids: list[str] | None = Query(default=None, description="Keep skills carrying any of these tags (OR)"),
sort: str = Query(
default="time_desc",
@@ -786,6 +830,7 @@ async def list_skills(
)
)
responses = await _attach_and_filter_tags(tag_store, responses, search, tag_ids)
responses = await _attach_owner_usernames(responses)
responses = _sort_skill_responses(responses, sort)
return SkillsListResponse(skills=responses)
except HTTPException:
@@ -1176,6 +1221,7 @@ async def list_custom_skills(
display_sample=adapter.get("sample") if adapter else None,
)
)
responses = await _attach_owner_usernames(responses)
return SkillsListResponse(skills=responses)
except Exception as e:
logger.error("Failed to list custom skills: %s", e, exc_info=True)
@@ -2309,6 +2355,324 @@ async def update_skill_detail(
raise HTTPException(status_code=500, detail=f"Failed to update skill detail: {str(e)}")
# ---------------------------------------------------------------------------
# 技能中文名称(用户可配置 + 大模型自动生成 + 一键补充)
# ---------------------------------------------------------------------------
_SKILL_NAME_ZH_SYSTEM_PROMPT = (
"你是一名中文产品命名助手。根据给定技能的英文名与功能描述,生成一个简洁、自然、"
"易懂的中文名称。要求:2 到 10 个汉字;准确概括技能用途;不要包含标点、引号、英文、"
"解释或多余文字;只输出中文名称本身。"
)
class SkillNameZhModelError(Exception):
"""The LLM call backing skill-name generation failed (a *model-side* problem).
Distinguishes a genuine model failure (timeout / API error / unbuildable
model) — which must be surfaced to the user as "大模型报错导致无法补全" — from
the model merely returning an unusable name (callers get ``None`` instead).
"""
def _clean_generated_name_zh(raw: str) -> str | None:
"""Normalise an LLM-produced Chinese name: first line, strip quotes/punct, cap length."""
if not raw:
return None
text = str(raw).strip()
# Take the first non-empty line only.
for line in text.splitlines():
line = line.strip()
if line:
text = line
break
# Strip wrapping quotes / common trailing punctuation.
text = text.strip().strip("\"'“”‘’`「」《》【】()()[]").strip()
text = text.rstrip("。.,,、!!??;;::")
if not text:
return None
# Keep it short — guard against the model returning a sentence.
if len(text) > 20:
text = text[:20]
return text or None
def _resolve_skill_name_zh_model_name(app_config: AppConfig, model_name: str | None) -> str | None:
"""Pick the model name for skill-name generation.
An explicit ``model_name`` (user-selected in the UI) wins; otherwise fall
back to the curator/security-scan model setting (itself falling back to the
main model when unset).
"""
if model_name and model_name.strip():
return model_name.strip()
if app_config.skill_evolution and app_config.skill_evolution.curator:
return app_config.skill_evolution.curator.model_name
return None
async def _generate_skill_name_zh(skill: Skill, app_config: AppConfig, model: Any = None, *, model_name: str | None = None) -> str | None:
"""Generate a Chinese display name for ``skill`` via the LLM.
An explicit ``model_name`` lets the caller pick the model; otherwise it
reuses the curator/security-scan model setting (falls back to the main
model). Returns ``None`` only when the model replies but the output cannot be
normalised into a short Chinese name. Raises :class:`SkillNameZhModelError`
when the model itself fails (cannot be built / times out / API error), so the
caller can tell the user it was a *model* problem.
"""
user_prompt = (
f"技能英文名:{skill.name}\n"
f"功能描述:{skill.description or '(无描述)'}\n\n"
"请为该技能起一个中文名称。"
)
if model is None:
try:
from deerflow.models import create_chat_model
resolved = _resolve_skill_name_zh_model_name(app_config, model_name)
model = create_chat_model(name=resolved, thinking_enabled=False, app_config=app_config)
except Exception as e:
logger.warning("Skill name_zh model build failed for %s", skill.name, exc_info=True)
raise SkillNameZhModelError(str(e) or "无法初始化所选大模型") from e
try:
response = await asyncio.wait_for(
model.ainvoke(
[
{"role": "system", "content": _SKILL_NAME_ZH_SYSTEM_PROMPT},
{"role": "user", "content": user_prompt},
],
config={"run_name": "skill_name_zh"},
),
timeout=60,
)
except asyncio.TimeoutError as e:
logger.warning("Skill name_zh LLM call timed out for %s", skill.name)
raise SkillNameZhModelError("大模型响应超时(60 秒)") from e
except Exception as e:
logger.warning("Skill name_zh LLM call failed for %s", skill.name, exc_info=True)
raise SkillNameZhModelError(str(e) or "大模型调用失败") from e
return _clean_generated_name_zh(str(getattr(response, "content", "") or ""))
async def _persist_skill_name_zh(
skill: Skill,
name_zh: str | None,
skill_store: SkillStore,
user_id: str,
*,
is_admin: bool,
) -> str | None:
"""Persist ``name_zh`` into the ``skills`` DB table (never the on-disk SKILL.md).
Authorization mirrors the ``detail`` annotation: a custom skill is editable by
its owner or an admin; a built-in / public skill only by an admin (its DB row
is materialized on demand). Returns the stored value.
"""
if skill.category == SkillCategory.CUSTOM:
_record, mode = await _get_editable_skill_or_raise(skill_store, skill.name, user_id, is_admin=is_admin)
if mode == "admin":
updated = await skill_store.update_admin(skill.name, {"name_zh": name_zh})
else:
updated = await skill_store.update(skill.name, user_id, {"name_zh": name_zh})
else:
if not is_admin:
raise HTTPException(status_code=403, detail="Only admin users can edit built-in skill Chinese names")
if await skill_store.get_any(skill.name) is None:
await skill_store.ensure_legacy({"name": skill.name, "owner_user_id": None, "published": True})
updated = await skill_store.update_admin(skill.name, {"name_zh": name_zh})
if updated is None:
raise HTTPException(status_code=500, detail="Failed to persist skill Chinese name")
return updated.get("name_zh")
class SkillNameZhRequest(BaseModel):
"""Request body for setting a skill's Chinese display name."""
name_zh: str = Field(default="", description="中文名称;传空字符串表示清空")
class SkillNameZhResponse(BaseModel):
"""Response model for a single skill's Chinese-name update."""
name: str = Field(..., description="Skill name")
name_zh: str | None = Field(default=None, description="中文名称(为空表示已清空)")
class SkillNameZhBulkResponse(BaseModel):
"""Response model for the one-click bulk fill of missing Chinese names."""
generated: list[SkillNameZhResponse] = Field(default_factory=list, description="本次成功生成中文名的技能")
failed: list[str] = Field(default_factory=list, description="生成失败的技能英文名")
skipped: int = Field(default=0, description="已有中文名而被跳过的技能数")
model_error: str | None = Field(default=None, description="若有技能因大模型报错而失败,这里给出代表性的报错信息")
class GenerateSkillNameZhRequest(BaseModel):
"""Optional body for skill-name generation: pick the model to use."""
model_name: str | None = Field(default=None, description="可选:指定用于生成的大模型名称;留空使用默认配置")
def _can_edit_skill_name_zh(skill: Skill, record: dict[str, Any] | None, user_id: str, *, is_admin: bool) -> bool:
"""Whether ``user_id`` may set this skill's Chinese name (mirrors detail auth)."""
if is_admin:
return True
if skill.category != SkillCategory.CUSTOM:
return False
return record is not None and record.get("owner_user_id") == user_id
@router.put(
"/skills/{skill_name}/name-zh",
response_model=SkillNameZhResponse,
summary="Set Skill Chinese Name",
description="设置技能的中文名称(存数据库,不改源文件)。自定义技能由其所有者或管理员编辑;内置/公开技能仅管理员可编辑。传空字符串表示清空,回退到使用英文名。",
)
async def set_skill_name_zh(
skill_name: str,
body: SkillNameZhRequest,
request: Request,
config: AppConfig = Depends(get_config),
skill_store: SkillStore = Depends(get_skill_store),
) -> SkillNameZhResponse:
try:
skill_name = skill_name.replace("\r\n", "").replace("\n", "")
user_id = _current_user_id(request)
is_admin = _is_admin_user(request)
skills = get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False)
skill = next((s for s in skills if s.name == skill_name), None)
if skill is None:
raise HTTPException(status_code=404, detail=f"Skill '{skill_name}' not found")
name_zh = body.name_zh.strip() or None
stored = await _persist_skill_name_zh(skill, name_zh, skill_store, user_id, is_admin=is_admin)
return SkillNameZhResponse(name=skill_name, name_zh=stored)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to set name_zh for %s: %s", skill_name, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to set skill Chinese name: {str(e)}")
@router.post(
"/skills/{skill_name}/name-zh/generate",
response_model=SkillNameZhResponse,
summary="Generate Skill Chinese Name",
description="用大模型为该技能生成一个中文名称并保存到数据库。可在请求体里通过 model_name 指定使用的大模型。",
)
async def generate_skill_name_zh(
skill_name: str,
request: Request,
body: GenerateSkillNameZhRequest | None = None,
config: AppConfig = Depends(get_config),
skill_store: SkillStore = Depends(get_skill_store),
) -> SkillNameZhResponse:
try:
body = body or GenerateSkillNameZhRequest()
skill_name = skill_name.replace("\r\n", "").replace("\n", "")
user_id = _current_user_id(request)
is_admin = _is_admin_user(request)
skills = get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False)
skill = next((s for s in skills if s.name == skill_name), None)
if skill is None:
raise HTTPException(status_code=404, detail=f"Skill '{skill_name}' not found")
try:
name_zh = await _generate_skill_name_zh(skill, config, model_name=body.model_name)
except SkillNameZhModelError as e:
raise HTTPException(status_code=502, detail=f"大模型报错,无法补全技能中文名:{e}")
if not name_zh:
raise HTTPException(status_code=502, detail="大模型未能生成有效的中文名称,请稍后重试或手动填写")
stored = await _persist_skill_name_zh(skill, name_zh, skill_store, user_id, is_admin=is_admin)
return SkillNameZhResponse(name=skill_name, name_zh=stored)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to generate name_zh for %s: %s", skill_name, e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to generate skill Chinese name: {str(e)}")
@router.post(
"/skills/name-zh/generate-missing",
response_model=SkillNameZhBulkResponse,
summary="Bulk Generate Missing Skill Chinese Names",
description="管理员一键为库中所有尚未配置中文名称的技能生成中文名(存数据库)。可在请求体里通过 model_name 指定使用的大模型。",
)
async def generate_missing_skill_names_zh(
request: Request,
body: GenerateSkillNameZhRequest | None = None,
config: AppConfig = Depends(get_config),
skill_store: SkillStore = Depends(get_skill_store),
) -> SkillNameZhBulkResponse:
try:
from deerflow.models import create_chat_model
body = body or GenerateSkillNameZhRequest()
user_id = _current_user_id(request)
is_admin = _is_admin_user(request)
# 一键补充技能中文名仅对管理员开放。
if not is_admin:
raise HTTPException(status_code=403, detail="仅管理员可一键补充技能中文名")
skills = get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False)
# Admin sees every row, so existing name_zh values come from the full set.
records = {r["name"]: r for r in await skill_store.list_all()}
pending: list[Skill] = []
skipped = 0
for skill in skills:
record = records.get(skill.name)
if record and record.get("name_zh"):
skipped += 1
continue
pending.append(skill)
generated: list[SkillNameZhResponse] = []
failed: list[str] = []
model_error: str | None = None
if pending:
# Build the model once and reuse it for every skill. A build failure
# is a model problem — fail fast with a clear message rather than
# silently producing zero results.
try:
resolved_name = _resolve_skill_name_zh_model_name(config, body.model_name)
model = create_chat_model(name=resolved_name, thinking_enabled=False, app_config=config)
except Exception as e:
logger.warning("Failed to build model for bulk name_zh generation", exc_info=True)
raise HTTPException(status_code=502, detail=f"大模型报错,无法补全技能中文名:{str(e) or '无法初始化所选大模型'}")
# Generate concurrently with a small bound to avoid overloading the model.
semaphore = asyncio.Semaphore(4)
async def _one(skill: Skill) -> tuple[Skill, str | None, str | None]:
async with semaphore:
try:
return skill, await _generate_skill_name_zh(skill, config, model=model), None
except SkillNameZhModelError as e:
return skill, None, str(e)
results = await asyncio.gather(*[_one(s) for s in pending])
for skill, name_zh, err in results:
if err and model_error is None:
model_error = err
if not name_zh:
failed.append(skill.name)
continue
try:
stored = await _persist_skill_name_zh(skill, name_zh, skill_store, user_id, is_admin=is_admin)
generated.append(SkillNameZhResponse(name=skill.name, name_zh=stored))
except HTTPException:
# Permission / persistence issue for this one skill — skip it.
failed.append(skill.name)
return SkillNameZhBulkResponse(generated=generated, failed=failed, skipped=skipped, model_error=model_error)
except HTTPException:
raise
except Exception as e:
logger.error("Failed to bulk-generate skill Chinese names: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to bulk-generate skill Chinese names: {str(e)}")
class SkillPinRequest(BaseModel):
pinned: bool = Field(..., description="Whether to pin or unpin the skill")