Merge remote-tracking branch 'origin/master' into xuhd
# Conflicts: # frontend-web/.env.production # frontend-web/.env.sit # frontend-web/package.json # offline-backend-20260512/backend/config.yaml
This commit is contained in:
commit
1668c1da87
177 files changed
+21030
-2738
No files matched your search
@@ -6,6 +6,8 @@ import sys
|
||||
import threading
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from time import perf_counter
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import FastAPI
|
||||
@@ -30,12 +32,14 @@ from app.gateway.routers import (
|
||||
artifacts,
|
||||
assistants_compat,
|
||||
auth,
|
||||
browser_context,
|
||||
business_mapping,
|
||||
channels,
|
||||
checkpoint_migration,
|
||||
feedback,
|
||||
html_page_favorites,
|
||||
intent,
|
||||
knowledge,
|
||||
llm_metrics,
|
||||
mcp,
|
||||
memory,
|
||||
@@ -45,6 +49,7 @@ from app.gateway.routers import (
|
||||
parallel_agent_tasks,
|
||||
parallel_agents,
|
||||
public_embed,
|
||||
public_leaderboard,
|
||||
public_scheduled_tasks,
|
||||
recommend,
|
||||
recommended_questions,
|
||||
@@ -469,6 +474,91 @@ _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_STARTUP_DELAY_SECONDS",
|
||||
60.0,
|
||||
)
|
||||
# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
|
||||
# *previous* day's snapshot once, so records that landed near midnight are fully
|
||||
# captured. Default 1 → "每天凌晨 1 点重新统计前一天".
|
||||
_LEADERBOARD_FINALIZE_HOUR = min(
|
||||
23,
|
||||
_env_int("ADMIN_LEADERBOARD_FINALIZE_HOUR", 1, minimum=0),
|
||||
)
|
||||
|
||||
|
||||
def _leaderboard_scheduler_sql_for_log(stmt) -> str:
|
||||
try:
|
||||
return str(stmt.compile(compile_kwargs={"literal_binds": True}))
|
||||
except Exception:
|
||||
return str(stmt)
|
||||
|
||||
|
||||
async def _execute_leaderboard_scheduler_query(session, step: str, stmt, **context):
|
||||
sql = _leaderboard_scheduler_sql_for_log(stmt)
|
||||
logger.info(
|
||||
"Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
|
||||
step,
|
||||
context.get("stat_date") or context.get("stat_dates") or "",
|
||||
context.get("job_key") or "",
|
||||
context.get("settings_hash") or "",
|
||||
datetime.now().isoformat(),
|
||||
sql,
|
||||
)
|
||||
start = perf_counter()
|
||||
try:
|
||||
result = await session.execute(stmt)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
|
||||
step,
|
||||
context.get("stat_date") or context.get("stat_dates") or "",
|
||||
context.get("job_key") or "",
|
||||
context.get("settings_hash") or "",
|
||||
(perf_counter() - start) * 1000,
|
||||
)
|
||||
raise
|
||||
logger.info(
|
||||
"Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
|
||||
step,
|
||||
context.get("stat_date") or context.get("stat_dates") or "",
|
||||
context.get("job_key") or "",
|
||||
context.get("settings_hash") or "",
|
||||
datetime.now().isoformat(),
|
||||
(perf_counter() - start) * 1000,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def _time_leaderboard_scheduler_db_operation(step: str, operation, *, sql: str = "", **context):
|
||||
logger.info(
|
||||
"Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
|
||||
step,
|
||||
context.get("stat_date") or context.get("stat_dates") or "",
|
||||
context.get("job_key") or "",
|
||||
context.get("settings_hash") or "",
|
||||
datetime.now().isoformat(),
|
||||
sql,
|
||||
)
|
||||
start = perf_counter()
|
||||
try:
|
||||
result = await operation()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
|
||||
step,
|
||||
context.get("stat_date") or context.get("stat_dates") or "",
|
||||
context.get("job_key") or "",
|
||||
context.get("settings_hash") or "",
|
||||
(perf_counter() - start) * 1000,
|
||||
)
|
||||
raise
|
||||
logger.info(
|
||||
"Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
|
||||
step,
|
||||
context.get("stat_date") or context.get("stat_dates") or "",
|
||||
context.get("job_key") or "",
|
||||
context.get("settings_hash") or "",
|
||||
datetime.now().isoformat(),
|
||||
(perf_counter() - start) * 1000,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _recent_leaderboard_stat_dates(days: int) -> list[str]:
|
||||
@@ -496,21 +586,27 @@ async def _leaderboard_snapshot_due_dates(
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(seconds=running_ttl_seconds)
|
||||
async with session_factory() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(AdminLeaderboardDailyStatRow.stat_date)
|
||||
.where(
|
||||
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
|
||||
or_(
|
||||
AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
|
||||
and_(
|
||||
AdminLeaderboardDailyStatRow.status == "running",
|
||||
AdminLeaderboardDailyStatRow.updated_at < cutoff,
|
||||
),
|
||||
stmt = (
|
||||
select(AdminLeaderboardDailyStatRow.stat_date)
|
||||
.where(
|
||||
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
|
||||
or_(
|
||||
AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
|
||||
and_(
|
||||
AdminLeaderboardDailyStatRow.status == "running",
|
||||
AdminLeaderboardDailyStatRow.updated_at < cutoff,
|
||||
),
|
||||
)
|
||||
.order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
|
||||
.limit(limit)
|
||||
),
|
||||
)
|
||||
.order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
rows = (
|
||||
await _execute_leaderboard_scheduler_query(
|
||||
session,
|
||||
"scheduler.due_dates",
|
||||
stmt,
|
||||
settings_hash=settings_hash,
|
||||
)
|
||||
).all()
|
||||
return [str(row[0]) for row in rows]
|
||||
@@ -530,9 +626,15 @@ async def _claim_leaderboard_snapshot(
|
||||
from deerflow.persistence.types import BEIJING_TZ
|
||||
|
||||
async with session_factory() as session:
|
||||
row = await session.get(
|
||||
AdminLeaderboardDailyStatRow,
|
||||
{"stat_date": stat_date, "settings_hash": settings_hash},
|
||||
row = await _time_leaderboard_scheduler_db_operation(
|
||||
"scheduler.claim.get",
|
||||
lambda: session.get(
|
||||
AdminLeaderboardDailyStatRow,
|
||||
{"stat_date": stat_date, "settings_hash": settings_hash},
|
||||
),
|
||||
sql="SELECT admin_leaderboard_daily_stats by primary key",
|
||||
stat_date=stat_date,
|
||||
settings_hash=settings_hash,
|
||||
)
|
||||
if row is None or _snapshot_is_fresh(row, stat_date):
|
||||
return False
|
||||
@@ -548,10 +650,68 @@ async def _claim_leaderboard_snapshot(
|
||||
row.message = "排行榜后台统计任务已认领。"
|
||||
row.error = None
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await _time_leaderboard_scheduler_db_operation(
|
||||
"scheduler.claim.commit",
|
||||
lambda: session.commit(),
|
||||
sql="COMMIT",
|
||||
stat_date=stat_date,
|
||||
settings_hash=settings_hash,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash: str) -> bool:
|
||||
"""Once per day after ``_LEADERBOARD_FINALIZE_HOUR``, force-recompute yesterday.
|
||||
|
||||
A ``ready`` past-day snapshot is otherwise treated as fresh forever and never
|
||||
rebuilt. But the previous day's snapshot is usually generated before midnight,
|
||||
so it can miss runs/tool-calls that only land in the DB around the day
|
||||
boundary. This re-queues yesterday (``force=True``) a single time; the rebuild
|
||||
stamps ``generated_at`` to today, which stops it from re-firing for the rest of
|
||||
the day.
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
from app.gateway.routers.admin_users import (
|
||||
_as_beijing,
|
||||
_queue_daily_leaderboard_snapshots,
|
||||
)
|
||||
from deerflow.persistence.admin_stats.finalize import should_finalize_previous_day
|
||||
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
|
||||
from deerflow.persistence.types import BEIJING_TZ
|
||||
|
||||
now = datetime.now(BEIJING_TZ)
|
||||
yesterday = (now.date() - timedelta(days=1)).isoformat()
|
||||
async with session_factory() as session:
|
||||
row = await session.get(
|
||||
AdminLeaderboardDailyStatRow,
|
||||
{"stat_date": yesterday, "settings_hash": settings_hash},
|
||||
)
|
||||
generated_at = _as_beijing(row.generated_at) if (row is not None and row.generated_at is not None) else None
|
||||
if not should_finalize_previous_day(
|
||||
status=(row.status if row is not None else None),
|
||||
generated_at=generated_at,
|
||||
now=now,
|
||||
finalize_hour=_LEADERBOARD_FINALIZE_HOUR,
|
||||
):
|
||||
return False
|
||||
queued = await _queue_daily_leaderboard_snapshots(
|
||||
session_factory,
|
||||
[yesterday],
|
||||
settings_hash,
|
||||
force=True,
|
||||
message="每日凌晨重新统计前一天完整数据,确保全部记录纳入统计。",
|
||||
)
|
||||
if queued:
|
||||
logger.info(
|
||||
"Leaderboard snapshot finalize: queued previous day %s for full recompute (hash=%s)",
|
||||
yesterday,
|
||||
settings_hash[:10],
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
|
||||
from app.gateway.routers.admin_users import (
|
||||
_daily_job_key,
|
||||
@@ -576,6 +736,12 @@ async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
|
||||
message="排行榜后台统计任务定期预热/补算队列。",
|
||||
)
|
||||
|
||||
# 每天凌晨(默认 1 点)后强制重算"前一天"的完整快照一次,覆盖跨午夜才落库的记录。
|
||||
try:
|
||||
await _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash)
|
||||
except Exception:
|
||||
logger.exception("Leaderboard snapshot finalize check failed (non-fatal)")
|
||||
|
||||
due_dates = await _leaderboard_snapshot_due_dates(
|
||||
session_factory,
|
||||
settings_hash=settings_hash,
|
||||
@@ -1115,6 +1281,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Uploads API is mounted at /api/threads/{thread_id}/uploads
|
||||
app.include_router(uploads.router)
|
||||
|
||||
# Optional Chrome-extension page capture API is mounted at /api/browser-context
|
||||
app.include_router(browser_context.router)
|
||||
|
||||
# Thread cleanup API is mounted at /api/threads/{thread_id}
|
||||
app.include_router(threads.router)
|
||||
|
||||
@@ -1162,10 +1331,16 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# surface results without a DeerFlow login.
|
||||
app.include_router(public_scheduled_tasks.router)
|
||||
|
||||
# Public read-only user leaderboard publish page (/api/public/leaderboard)
|
||||
app.include_router(public_leaderboard.router)
|
||||
|
||||
# Public embed Q&A status polling (/api/public/embed/*)
|
||||
app.include_router(public_embed.router)
|
||||
app.include_router(html_page_favorites.router)
|
||||
|
||||
# Product knowledge base API is mounted at /api/knowledge
|
||||
app.include_router(knowledge.router)
|
||||
|
||||
# System settings API is mounted at /api/system-settings
|
||||
app.include_router(system_settings.router)
|
||||
|
||||
@@ -1174,6 +1349,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
# Admin window into other users' threads / messages / memory
|
||||
app.include_router(admin_users.router)
|
||||
app.include_router(admin_users.self_router)
|
||||
|
||||
# Admin tool / skill call metrics audit + export
|
||||
app.include_router(tool_metrics.router)
|
||||
|
||||
@@ -20,6 +20,7 @@ from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, i
|
||||
from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse
|
||||
from app.gateway.authz import _ALL_PERMISSIONS, AuthContext
|
||||
from app.gateway.internal_auth import INTERNAL_AUTH_HEADER_NAME, get_internal_user, is_valid_internal_auth_token
|
||||
from app.gateway.proxy_path import app_relative_path
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
|
||||
# Paths that never require authentication.
|
||||
@@ -84,7 +85,9 @@ class AuthMiddleware(BaseHTTPMiddleware):
|
||||
super().__init__(app)
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
if _is_public(request.url.path):
|
||||
# Match the application-relative path so a reverse-proxy prefix
|
||||
# (e.g. /xxx/api/public/...) does not defeat the public whitelist.
|
||||
if _is_public(app_relative_path(request)):
|
||||
return await call_next(request)
|
||||
|
||||
if is_auth_disabled():
|
||||
|
||||
@@ -8,6 +8,7 @@ import secrets
|
||||
from collections.abc import Callable
|
||||
|
||||
from app.gateway.auth.disabled_mode import is_auth_disabled
|
||||
from app.gateway.proxy_path import app_relative_path
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.responses import JSONResponse
|
||||
@@ -37,7 +38,9 @@ def should_check_csrf(request: Request) -> bool:
|
||||
if request.method not in ("POST", "PUT", "DELETE", "PATCH"):
|
||||
return False
|
||||
|
||||
path = request.url.path
|
||||
# Match the application-relative path so a reverse-proxy prefix
|
||||
# (e.g. /xxx/api/v1/auth/me) does not defeat these whitelists.
|
||||
path = app_relative_path(request)
|
||||
stripped = path.rstrip("/")
|
||||
# Exempt /api/v1/auth/me endpoint
|
||||
if stripped == "/api/v1/auth/me":
|
||||
@@ -56,6 +59,10 @@ _AUTH_EXEMPT_PATHS: frozenset[str] = frozenset(
|
||||
{
|
||||
"/api/v1/auth/login/local",
|
||||
"/api/v1/auth/login/username",
|
||||
# Token-exchange login is sessionless on first call (no CSRF cookie yet),
|
||||
# exactly like the other login endpoints — mirror auth_middleware's
|
||||
# _PUBLIC_EXACT_PATHS, which already whitelists it.
|
||||
"/api/v1/auth/login/token",
|
||||
"/api/v1/auth/logout",
|
||||
"/api/v1/auth/register",
|
||||
"/api/v1/auth/initialize",
|
||||
@@ -69,8 +76,10 @@ def is_auth_endpoint(request: Request) -> bool:
|
||||
"""Check if the request is to an auth endpoint.
|
||||
|
||||
Auth endpoints don't need CSRF validation on first call (no token).
|
||||
Matches the application-relative path so a reverse-proxy prefix does not
|
||||
defeat the whitelist.
|
||||
"""
|
||||
return request.url.path.rstrip("/") in _AUTH_EXEMPT_PATHS
|
||||
return app_relative_path(request).rstrip("/") in _AUTH_EXEMPT_PATHS
|
||||
|
||||
|
||||
def _has_bearer_auth(request: Request) -> bool:
|
||||
@@ -82,6 +91,22 @@ def _has_bearer_auth(request: Request) -> bool:
|
||||
return scheme.lower() == "bearer" and bool(token.strip())
|
||||
|
||||
|
||||
def _has_cookie_session(request: Request) -> bool:
|
||||
"""Return True when the request carries a cookie-based session.
|
||||
|
||||
Double-Submit-Cookie CSRF only protects requests the browser authenticates
|
||||
**automatically via cookies** — that is the exact surface a cross-site
|
||||
forgery can ride. A request with no ``access_token`` session cookie has no
|
||||
session for an attacker to exploit, so enforcing CSRF on it is both
|
||||
pointless and harmful: a freshly-loaded browser (e.g. one that just cleared
|
||||
its cache/cookies) hitting the login endpoint would otherwise be rejected
|
||||
with "CSRF token missing" before it can even sign in. Such an
|
||||
unauthenticated request, if forged, is rejected by the auth layer (401)
|
||||
anyway.
|
||||
"""
|
||||
return bool(request.cookies.get("access_token"))
|
||||
|
||||
|
||||
class CSRFMiddleware(BaseHTTPMiddleware):
|
||||
"""Middleware that implements CSRF protection using Double Submit Cookie pattern."""
|
||||
|
||||
@@ -95,8 +120,11 @@ class CSRFMiddleware(BaseHTTPMiddleware):
|
||||
return await call_next(request)
|
||||
|
||||
# Double-submit-cookie CSRF applies to cookie-based browser sessions.
|
||||
# API clients using explicit Bearer tokens are not CSRF-prone.
|
||||
if should_check_csrf(request) and not _is_auth and not _has_bearer_auth(request):
|
||||
# API clients using explicit Bearer tokens are not CSRF-prone, and a
|
||||
# request with no session cookie at all (e.g. a fresh browser signing
|
||||
# in) has no session to forge — only enforce when an ``access_token``
|
||||
# cookie is actually present.
|
||||
if should_check_csrf(request) and not _is_auth and not _has_bearer_auth(request) and _has_cookie_session(request):
|
||||
cookie_token = request.cookies.get(CSRF_COOKIE_NAME)
|
||||
header_token = request.headers.get(CSRF_HEADER_NAME)
|
||||
|
||||
|
||||
@@ -143,6 +143,8 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
from deerflow.persistence.ai_writing_sessions import (
|
||||
make_ai_writing_session_store,
|
||||
)
|
||||
from deerflow.persistence.ai_writing_sessions import (
|
||||
register_default_store as _register_ai_writing_default_store,
|
||||
)
|
||||
|
||||
@@ -159,6 +161,25 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
app.state.memory_config_store = make_user_memory_config_store(sf)
|
||||
|
||||
from deerflow.knowledge import make_knowledge_service
|
||||
|
||||
app.state.knowledge_service = make_knowledge_service(sf, config.knowledge)
|
||||
|
||||
# Background auto-sedimentation queue (phase 3) — drains finished
|
||||
# conversations off the chat path. Worker only does work when
|
||||
# ``knowledge.auto_ingest_enabled`` is set; the queue is always wired so
|
||||
# the middleware can enqueue without a hard dependency.
|
||||
from deerflow.knowledge.auto_ingest import AutoIngestQueue, set_auto_ingest_queue
|
||||
|
||||
if app.state.knowledge_service is not None:
|
||||
_auto_ingest_queue = AutoIngestQueue(
|
||||
lambda: getattr(app.state, "knowledge_service", None),
|
||||
min_chars=config.knowledge.auto_ingest_min_chars,
|
||||
)
|
||||
_auto_ingest_queue.start()
|
||||
set_auto_ingest_queue(_auto_ingest_queue)
|
||||
app.state.knowledge_auto_ingest_queue = _auto_ingest_queue
|
||||
|
||||
from deerflow.persistence.user_prompts import make_user_prompt_store
|
||||
|
||||
app.state.user_prompt_store = make_user_prompt_store(sf)
|
||||
@@ -189,6 +210,8 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
from deerflow.persistence.roundtable_jobs import (
|
||||
make_roundtable_job_store,
|
||||
)
|
||||
from deerflow.persistence.roundtable_jobs import (
|
||||
register_default_store as _register_roundtable_job_default_store,
|
||||
)
|
||||
|
||||
@@ -246,6 +269,12 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
from deerflow.persistence.notifications import set_notification_store
|
||||
|
||||
set_notification_store(None)
|
||||
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
|
||||
if queue is not None:
|
||||
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
|
||||
|
||||
await queue.stop()
|
||||
set_auto_ingest_queue(None)
|
||||
await close_engine()
|
||||
|
||||
|
||||
@@ -380,6 +409,14 @@ def get_memory_config_store(request: Request):
|
||||
return val
|
||||
|
||||
|
||||
def get_knowledge_service(request: Request):
|
||||
"""Return the knowledge base service (503 when disabled / memory backend)."""
|
||||
val = getattr(request.app.state, "knowledge_service", None)
|
||||
if val is None:
|
||||
raise HTTPException(status_code=503, detail="Knowledge base not available")
|
||||
return val
|
||||
|
||||
|
||||
def get_run_context(request: Request) -> RunContext:
|
||||
"""Build a :class:`RunContext` from ``app.state`` singletons.
|
||||
|
||||
|
||||
@@ -40,6 +40,13 @@ def _check_csrf(request) -> None:
|
||||
# CSRF protection is for cookie-auth browser flows.
|
||||
return
|
||||
|
||||
# No cookie-based session → nothing for a cross-site forgery to ride, so
|
||||
# CSRF is moot. Enforcing it here would reject a freshly-loaded browser
|
||||
# (just cleared its cookies) before it can authenticate; such a request is
|
||||
# rejected as unauthenticated (401) below anyway.
|
||||
if not request.cookies.get("access_token"):
|
||||
return
|
||||
|
||||
cookie_token = request.cookies.get("csrf_token")
|
||||
header_token = request.headers.get("x-csrf-token")
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Resolve the application-relative request path behind a reverse proxy.
|
||||
|
||||
A reverse proxy may forward requests under a path prefix, e.g.
|
||||
``/xxx/api/v1/auth/login/token``. FastAPI strips that prefix for **routing**
|
||||
when it is told about it via the ASGI ``root_path`` (uvicorn ``--root-path`` /
|
||||
``--proxy-headers``, or ``X-Forwarded-Prefix`` from the proxy) — which is why
|
||||
the route still matches and the request reaches a handler.
|
||||
|
||||
Starlette middleware, however, runs *before* routing and sees the full,
|
||||
prefixed ``request.url.path``. Middlewares that match that raw path against
|
||||
fixed whitelists — the CSRF-exempt auth paths and the AuthMiddleware public
|
||||
paths — then mis-match, so login and ``/api/public/*`` endpoints get wrongly
|
||||
rejected (``CSRF token missing`` / ``401``) on a prefixed deployment.
|
||||
|
||||
``app_relative_path`` returns the path with the proxy prefix removed so that
|
||||
whitelist matching is prefix-agnostic. It is a no-op when no prefix is present
|
||||
(direct deployment), and tolerates the prefix already being stripped from the
|
||||
path (it only strips when the path actually starts with the prefix).
|
||||
"""
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
|
||||
def app_relative_path(request: Request) -> str:
|
||||
"""Return ``request.url.path`` with any reverse-proxy prefix removed.
|
||||
|
||||
Prefix source order: ASGI ``root_path`` (set by the server when the proxy
|
||||
forwards it), then the ``X-Forwarded-Prefix`` header (proxies that add a
|
||||
prefix without informing the ASGI server). Returns ``"/"`` for an empty
|
||||
result so downstream ``rstrip("/")`` / ``startswith`` checks stay sane.
|
||||
"""
|
||||
path = request.url.path or "/"
|
||||
prefix = (request.scope.get("root_path") or request.headers.get("x-forwarded-prefix") or "").rstrip("/")
|
||||
if prefix and path.startswith(prefix):
|
||||
stripped = path[len(prefix):]
|
||||
if not stripped:
|
||||
return "/"
|
||||
return stripped if stripped.startswith("/") else "/" + stripped
|
||||
return path
|
||||
@@ -19,24 +19,30 @@
|
||||
并配合 ``SkillStopMiddleware`` 在 ``agent_orchestration`` 调用后停图,便于网关从
|
||||
AIMessage.tool_calls 解析派活列表。
|
||||
- ``seat`` —— 子智能体交付:开 thinking + medium reasoning;**可写文件**
|
||||
(``write_file`` / ``str_replace`` 产出 md 等成品文档),但禁派活 / 澄清 /
|
||||
展示文件 / 看图 / 联网搜索 / **``bash``**。禁 ``bash`` 的原因:圆桌席位是
|
||||
「文档产出者」,弱模型爱写 ``.py`` 脚本再跑出产物(乱且抓不到预览),与
|
||||
``report`` 角色禁 ``bash`` 同理;让它**直接 write_file 写成品文档**。
|
||||
(``write_file`` / ``str_replace`` 产出 md 等成品文档)+ **可跑 ``bash``**
|
||||
(2026-06 放开:业务需要跑知识库检索 / Python 辅助分析)。禁派活 / 澄清 /
|
||||
展示文件 / 看图 / 联网搜索。``report`` 通过 ``[*_SEAT_EXCLUDED, ...]`` 继承本角色,
|
||||
所以 Step3 报告也跟着拿到 bash;只有 ``leader`` 和 ``seat_parallel`` 仍各自显式禁。
|
||||
历史警示(保留):弱模型用 ``bash`` 跑 ``python xxx.py > out.md`` 生成产物会
|
||||
**绕过前端虚拟预览**(预览只拦截 ``write_file.args.content``)→ 沙箱里有文件但
|
||||
Step2 右侧预览空白。席位 SOUL 必须明确「bash 只用于辅助查询拿数据,**最终交付
|
||||
必须用 write_file 写 .md 落盘**,不要让脚本生成最终产物」。
|
||||
- ``seat_parallel`` —— **DAG 快速(fast)模式的并行席位(方案 A,2026-06)**:**禁写文件**
|
||||
(``write_file`` / ``str_replace``,产物走正文)+ 禁 ``bash`` + 禁「打断 / 越权」类,
|
||||
(``write_file`` / ``str_replace``,产物走正文)+ **禁 ``bash``** + 禁「打断 / 越权」类,
|
||||
**放开 ``web_search`` 与技能**(搜索不碰 sandbox;技能靠 ``read_file`` SKILL.md)——
|
||||
否则并行席位几乎调不动工具、产生不了步骤(步骤条不显示)。
|
||||
并行席位同时跑 bash 会在共享 sandbox 里抢路径 / 占进程数;且 fast 模式整体不
|
||||
写文件,bash 跑出来的产物既不能预览也没意义。
|
||||
「写文件产出 md」归 **沙箱(sandbox)模式**(走 ``seat`` policy,单列 + 沙箱跟随预览);
|
||||
fast 模式快、双列、产物走正文 —— 用户明确要求 fast 不生成文件。详见
|
||||
``docs/roundtable-dag-orchestration.md`` §4 / §12.1 / §12.8 / §12.10。
|
||||
与 ``seat`` 的差异:``seat`` 禁 ``web_search``(旧「圆桌一律禁搜索」),
|
||||
``seat_parallel`` **放开搜索**(本期诉求)。
|
||||
- ``report`` —— Step3 报告绘制:在 ``seat``(已禁 ``bash``)基础上**额外禁
|
||||
``str_replace``**:
|
||||
* ``bash`` 已在 ``seat`` 禁 → 杜绝写 Python 脚本再跑出 HTML(脚本产物落在
|
||||
outputs,但前端虚拟预览只抓 ``write_file`` 的 ``args.content``,抓不到脚本
|
||||
产物 → 预览停在半截 ``write_file``,正是「没写完就去生成 py 文件」的现象);
|
||||
``seat_parallel`` **放开搜索**(本期诉求);``seat`` 已放开 ``bash``,本角色仍禁。
|
||||
- ``report`` —— Step3 报告绘制:在 ``seat`` 基础上**额外禁 ``str_replace``**:
|
||||
* 继承 ``seat`` 的 bash 放开(2026-06)—— Step3 也可跑 bash 做辅助查询,同
|
||||
``seat`` 警示:**结果必须 write_file 落盘**,否则前端虚拟预览会空白
|
||||
(预览只抓 ``write_file.args.content``,抓不到脚本产物 → 预览停在半截
|
||||
``write_file``,正是「没写完就去生成 py 文件」的现象)。
|
||||
* 禁 ``str_replace`` → 强制返工用整文件 ``write_file`` 重写
|
||||
(``str_replace`` 没有 ``content`` 字段,会让前端虚拟预览读到 undefined
|
||||
而清空沙箱)。
|
||||
@@ -72,16 +78,21 @@ _LEADER_EXCLUDED: list[str] = [
|
||||
"hindsight_retain",
|
||||
"read_peer_delivery", # 总控靠 _collect_delivery_status 直读各席位,不需要这个席位侧工具
|
||||
]
|
||||
# seat:席位**可以写文件**(write_file / str_replace)产出 md 等成品文档,但**禁 bash** ——
|
||||
# 圆桌席位是「文档产出者」,不该跑 shell/python 脚本(弱模型爱写 .py 脚本再跑生成产物,既乱又
|
||||
# 抓不到产物预览,与 report 角色禁 bash 同理)。禁 web_search(圆桌串行一律禁搜索,旧决策)。
|
||||
# seat:席位**可以写文件**(write_file / str_replace)产出 md 等成品文档,**bash 放开**
|
||||
# (2026-06 决策,业务需要 bash 跑知识库检索 / Python 辅助分析)。禁派活 / 澄清 /
|
||||
# 展示文件 / 看图 / 联网搜索。``report`` 通过 ``[*_SEAT_EXCLUDED, ...]`` 继承本角色,
|
||||
# 所以 Step3 报告跟着拿到 bash;只有 leader / seat_parallel 仍各自显式禁。
|
||||
#
|
||||
# **历史警示(保留)**:弱模型用 bash 跑 ``python xxx.py > out.md`` 生成最终产物会
|
||||
# 绕过前端虚拟预览(只拦 write_file.args.content) → 沙箱里有文件但 Step2 右侧预览
|
||||
# 空白。必须靠席位 SOUL 强约束「bash 只用于辅助查询拿数据,最终交付必须用 write_file
|
||||
# 写 .md 落盘」。
|
||||
_SEAT_EXCLUDED: list[str] = [
|
||||
"web_search",
|
||||
"ask_clarification",
|
||||
"present_files",
|
||||
"view_image",
|
||||
"agent_orchestration",
|
||||
"bash",
|
||||
]
|
||||
# 并行席位 = **快速(fast)模式**(双列):**禁写文件**(产物走正文)+ 禁跑命令 +「打断/越权」类。
|
||||
# **禁 web_search** —— 离线内网部署无外网,席位检索一律走「技能(skill)」(靠 read_file 读
|
||||
|
||||
@@ -19,8 +19,11 @@ import logging
|
||||
import os
|
||||
import re
|
||||
from collections.abc import AsyncIterator
|
||||
from contextvars import ContextVar
|
||||
from difflib import SequenceMatcher
|
||||
from hashlib import sha256
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
from datetime import UTC, datetime, timedelta, time as datetime_time
|
||||
|
||||
@@ -56,6 +59,7 @@ from deerflow.persistence.user.model import UserRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/admin/users", tags=["admin-users"])
|
||||
self_router = APIRouter(prefix="/api/users/me", tags=["user-qa"])
|
||||
|
||||
_LEADERBOARD_MAX_CONCURRENT = max(
|
||||
1,
|
||||
@@ -99,6 +103,119 @@ _LEADERBOARD_QUEUE_ON_REQUEST_ENABLED = os.getenv(
|
||||
"off",
|
||||
}
|
||||
_leaderboard_semaphore = asyncio.Semaphore(_LEADERBOARD_MAX_CONCURRENT)
|
||||
_LEADERBOARD_TRACE_CONTEXT: ContextVar[dict[str, object] | None] = ContextVar(
|
||||
"leaderboard_trace_context",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
def _leaderboard_log_context(**context: object) -> dict[str, object] | None:
|
||||
current = _LEADERBOARD_TRACE_CONTEXT.get()
|
||||
merged: dict[str, object] = dict(current or {})
|
||||
for key, value in context.items():
|
||||
if value is None or value == "":
|
||||
continue
|
||||
merged[key] = value
|
||||
return merged or None
|
||||
|
||||
|
||||
def _leaderboard_sql_for_log(stmt) -> str:
|
||||
try:
|
||||
return str(stmt.compile(compile_kwargs={"literal_binds": True}))
|
||||
except Exception:
|
||||
return str(stmt)
|
||||
|
||||
|
||||
async def _execute_leaderboard_query(session, step: str, stmt, **context: object):
|
||||
log_context = _leaderboard_log_context(**context)
|
||||
if not log_context:
|
||||
return await session.execute(stmt)
|
||||
|
||||
started_at = datetime.now(BEIJING_TZ).isoformat()
|
||||
sql = _leaderboard_sql_for_log(stmt)
|
||||
logger.info(
|
||||
"Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
|
||||
step,
|
||||
log_context.get("stat_date") or log_context.get("stat_dates") or "",
|
||||
log_context.get("job_key") or "",
|
||||
log_context.get("settings_hash") or "",
|
||||
started_at,
|
||||
sql,
|
||||
)
|
||||
start = perf_counter()
|
||||
try:
|
||||
result = await session.execute(stmt)
|
||||
except Exception:
|
||||
elapsed_ms = (perf_counter() - start) * 1000
|
||||
logger.exception(
|
||||
"Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
|
||||
step,
|
||||
log_context.get("stat_date") or log_context.get("stat_dates") or "",
|
||||
log_context.get("job_key") or "",
|
||||
log_context.get("settings_hash") or "",
|
||||
elapsed_ms,
|
||||
)
|
||||
raise
|
||||
|
||||
elapsed_ms = (perf_counter() - start) * 1000
|
||||
logger.info(
|
||||
"Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
|
||||
step,
|
||||
log_context.get("stat_date") or log_context.get("stat_dates") or "",
|
||||
log_context.get("job_key") or "",
|
||||
log_context.get("settings_hash") or "",
|
||||
datetime.now(BEIJING_TZ).isoformat(),
|
||||
elapsed_ms,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def _time_leaderboard_db_operation(
|
||||
step: str,
|
||||
operation,
|
||||
*,
|
||||
sql: str = "",
|
||||
**context: object,
|
||||
):
|
||||
log_context = _leaderboard_log_context(**context)
|
||||
if not log_context:
|
||||
return await operation()
|
||||
|
||||
logger.info(
|
||||
"Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
|
||||
step,
|
||||
log_context.get("stat_date") or log_context.get("stat_dates") or "",
|
||||
log_context.get("job_key") or "",
|
||||
log_context.get("settings_hash") or "",
|
||||
datetime.now(BEIJING_TZ).isoformat(),
|
||||
sql,
|
||||
)
|
||||
start = perf_counter()
|
||||
try:
|
||||
result = await operation()
|
||||
except Exception:
|
||||
elapsed_ms = (perf_counter() - start) * 1000
|
||||
logger.exception(
|
||||
"Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
|
||||
step,
|
||||
log_context.get("stat_date") or log_context.get("stat_dates") or "",
|
||||
log_context.get("job_key") or "",
|
||||
log_context.get("settings_hash") or "",
|
||||
elapsed_ms,
|
||||
)
|
||||
raise
|
||||
|
||||
elapsed_ms = (perf_counter() - start) * 1000
|
||||
logger.info(
|
||||
"Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
|
||||
step,
|
||||
log_context.get("stat_date") or log_context.get("stat_dates") or "",
|
||||
log_context.get("job_key") or "",
|
||||
log_context.get("settings_hash") or "",
|
||||
datetime.now(BEIJING_TZ).isoformat(),
|
||||
elapsed_ms,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def _require_admin(request: Request) -> None:
|
||||
@@ -285,6 +402,13 @@ def _parse_datetime(value: str | None, fallback: datetime) -> datetime:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid datetime: {value}") from exc
|
||||
|
||||
|
||||
def _parse_optional_datetime(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
parsed = _parse_datetime(value, _now())
|
||||
return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=BEIJING_TZ)
|
||||
|
||||
|
||||
def _resolve_range(days: int, since: str | None, until: str | None) -> tuple[datetime, datetime, int]:
|
||||
end = _parse_datetime(until, _now())
|
||||
start = _parse_datetime(since, end - timedelta(days=max(1, min(days, 365))))
|
||||
@@ -325,7 +449,7 @@ def _bounds_for_stat_date(stat_date: str) -> tuple[datetime, datetime]:
|
||||
|
||||
async def _try_set_session_value(session, sql: str) -> None:
|
||||
try:
|
||||
await session.execute(text(sql))
|
||||
await _execute_leaderboard_query(session, "session_setting", text(sql))
|
||||
except Exception:
|
||||
logger.debug("Failed to apply analytics session setting: %s", sql, exc_info=True)
|
||||
try:
|
||||
@@ -334,13 +458,36 @@ async def _try_set_session_value(session, sql: str) -> None:
|
||||
logger.debug("Failed to rollback after analytics session setting failure", exc_info=True)
|
||||
|
||||
|
||||
def _is_mariadb(dialect) -> bool:
|
||||
"""SQLAlchemy reports ``dialect.name == 'mysql'`` for MariaDB too."""
|
||||
if getattr(dialect, "_is_mariadb", False) or getattr(dialect, "is_mariadb", False):
|
||||
return True
|
||||
version_info = getattr(dialect, "server_version_info", None) or ()
|
||||
return any("mariadb" in str(part).lower() for part in version_info)
|
||||
|
||||
|
||||
async def _set_analytics_read_timeout(session) -> None:
|
||||
"""Best-effort per-statement read timeout for the analytics queries.
|
||||
|
||||
The right knob differs by server, and some servers have none at all:
|
||||
- MySQL >= 5.7.4: ``max_execution_time`` (milliseconds)
|
||||
- MariaDB: ``max_statement_time`` (seconds)
|
||||
- MySQL <= 5.6: no per-statement timeout variable — skip it entirely
|
||||
(5.6 raises "Unknown system variable" otherwise)
|
||||
- PostgreSQL: ``statement_timeout`` (milliseconds)
|
||||
"""
|
||||
conn = await session.connection()
|
||||
dialect = conn.dialect.name
|
||||
if dialect == "mysql":
|
||||
await _try_set_session_value(session, f"SET SESSION max_execution_time = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
|
||||
dialect = conn.dialect
|
||||
name = dialect.name
|
||||
if name == "mysql":
|
||||
if _is_mariadb(dialect):
|
||||
# MariaDB: max_statement_time is in SECONDS (fractional ok).
|
||||
await _try_set_session_value(session, f"SET SESSION max_statement_time = {_LEADERBOARD_QUERY_TIMEOUT_MS / 1000}")
|
||||
elif (getattr(dialect, "server_version_info", None) or ()) >= (5, 7, 4):
|
||||
await _try_set_session_value(session, f"SET SESSION max_execution_time = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
|
||||
# else: MySQL <= 5.6 has no statement-timeout variable — skip silently.
|
||||
await _try_set_session_value(session, f"SET SESSION innodb_lock_wait_timeout = {_LEADERBOARD_LOCK_WAIT_SECONDS}")
|
||||
elif dialect == "postgresql":
|
||||
elif name == "postgresql":
|
||||
await _try_set_session_value(session, f"SET LOCAL statement_timeout = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
|
||||
|
||||
|
||||
@@ -661,11 +808,15 @@ async def _query_user_leaderboard(
|
||||
if user_id:
|
||||
stmt = stmt.where(UserRow.id == user_id)
|
||||
stmt = _apply_user_scope(stmt, UserRow.id, exclude_user_ids=exclude_user_ids, include_admins=include_admins)
|
||||
# 排行榜只统计真正产生过问答的用户;从未问答的账号(仅注册、零活跃)不计入榜单与人数。
|
||||
# 个人分析(指定 user_id)不受此限制,仍可查看任意单个用户。
|
||||
if user_id is None:
|
||||
stmt = stmt.where(func.coalesce(run_agg.c.qa_count, 0) > 0)
|
||||
stmt = stmt.order_by(
|
||||
desc(func.coalesce(run_agg.c.run_count, 0) + func.coalesce(tool_agg.c.tool_call_count, 0)),
|
||||
desc(func.coalesce(run_agg.c.total_tokens, 0)),
|
||||
)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
rows = (await _execute_leaderboard_query(session, "users", stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardUser(
|
||||
user_id=r[0],
|
||||
@@ -725,7 +876,7 @@ async def _query_call_rankings(
|
||||
include_admins=include_admins,
|
||||
include_scheduled=include_scheduled,
|
||||
)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
rows = (await _execute_leaderboard_query(session, f"{field}_rankings", stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardSkill(
|
||||
name=r[0] or "",
|
||||
@@ -791,7 +942,7 @@ async def _query_run_dimensions(
|
||||
include_failed=include_failed,
|
||||
include_scheduled=include_scheduled,
|
||||
)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
rows = (await _execute_leaderboard_query(session, f"{field}_dimensions", stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardDimension(
|
||||
name=r[0] or "",
|
||||
@@ -848,7 +999,7 @@ async def _query_model_rankings(
|
||||
stmt = stmt.where(LlmCallMetricRow.status != "error")
|
||||
if not include_scheduled:
|
||||
stmt = stmt.where(or_(LlmCallMetricRow.run_id.is_(None), LlmCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
|
||||
rows = (await session.execute(stmt)).all()
|
||||
rows = (await _execute_leaderboard_query(session, "model_rankings", stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardDimension(
|
||||
name=r[0] or "",
|
||||
@@ -895,7 +1046,7 @@ async def _query_recent_questions(
|
||||
include_failed=include_failed,
|
||||
include_scheduled=include_scheduled,
|
||||
)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
rows = (await _execute_leaderboard_query(session, "recent_questions", stmt)).all()
|
||||
questions: list[AdminLeaderboardQuestion] = []
|
||||
for r in rows:
|
||||
text = _clean_question_text(r[0]) if cleanup_system_questions else str(r[0] or "").strip()
|
||||
@@ -947,7 +1098,7 @@ async def _query_frequent_questions(
|
||||
include_failed=include_failed,
|
||||
include_scheduled=include_scheduled,
|
||||
)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
rows = (await _execute_leaderboard_query(session, "frequent_questions", stmt)).all()
|
||||
groups: list[dict] = []
|
||||
for r in rows:
|
||||
text = _clean_question_text(r[0]) if cleanup_system_questions else str(r[0] or "").strip()
|
||||
@@ -1022,7 +1173,7 @@ async def _query_trends(
|
||||
include_failed=include_failed,
|
||||
include_scheduled=include_scheduled,
|
||||
)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
rows = (await _execute_leaderboard_query(session, "trends", stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardTrendPoint(
|
||||
date=str(r[0]),
|
||||
@@ -1071,8 +1222,8 @@ async def _query_overview(
|
||||
|
||||
total_users_stmt = select(func.count(UserRow.id))
|
||||
total_users_stmt = _apply_user_scope(total_users_stmt, UserRow.id, exclude_user_ids=exclude_user_ids, include_admins=include_admins)
|
||||
total_users = 1 if user_id else (await session.execute(total_users_stmt)).scalar_one()
|
||||
run_row = (await session.execute(
|
||||
total_users = 1 if user_id else (await _execute_leaderboard_query(session, "overview.total_users", total_users_stmt)).scalar_one()
|
||||
run_summary_stmt = (
|
||||
select(
|
||||
func.count(RunRow.run_id),
|
||||
_question_count_expr(),
|
||||
@@ -1080,16 +1231,19 @@ async def _query_overview(
|
||||
func.count(func.distinct(RunRow.user_id)),
|
||||
func.sum(case((RunRow.status == "error", 1), else_=0)),
|
||||
).where(*run_where)
|
||||
)).one()
|
||||
thread_count = (await session.execute(select(func.count(ThreadMetaRow.thread_id)).where(*thread_where))).scalar_one()
|
||||
tool_row = (await session.execute(
|
||||
)
|
||||
run_row = (await _execute_leaderboard_query(session, "overview.run_summary", run_summary_stmt)).one()
|
||||
thread_count_stmt = select(func.count(ThreadMetaRow.thread_id)).where(*thread_where)
|
||||
thread_count = (await _execute_leaderboard_query(session, "overview.thread_count", thread_count_stmt)).scalar_one()
|
||||
tool_summary_stmt = (
|
||||
select(
|
||||
func.count(ToolCallMetricRow.id),
|
||||
func.sum(case((func.nullif(ToolCallMetricRow.skill_name, "").is_not(None), 1), else_=0)),
|
||||
)
|
||||
.outerjoin(RunRow, RunRow.run_id == ToolCallMetricRow.run_id)
|
||||
.where(*tool_where)
|
||||
)).one()
|
||||
)
|
||||
tool_row = (await _execute_leaderboard_query(session, "overview.tool_summary", tool_summary_stmt)).one()
|
||||
|
||||
return AdminLeaderboardOverview(
|
||||
total_users=int(total_users or 0),
|
||||
@@ -1579,13 +1733,12 @@ async def _load_daily_snapshot_rows(
|
||||
if not stat_dates:
|
||||
return {}
|
||||
async with session_factory() as session:
|
||||
stmt = select(AdminLeaderboardDailyStatRow).where(
|
||||
AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
|
||||
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
|
||||
)
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(AdminLeaderboardDailyStatRow).where(
|
||||
AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
|
||||
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
|
||||
)
|
||||
)
|
||||
await _execute_leaderboard_query(session, "snapshot_rows.load", stmt)
|
||||
).scalars().all()
|
||||
return {row.stat_date: row for row in rows}
|
||||
|
||||
@@ -1603,9 +1756,15 @@ async def _store_daily_snapshot(
|
||||
) -> None:
|
||||
async with session_factory() as session:
|
||||
now = datetime.now(UTC)
|
||||
row = await session.get(
|
||||
AdminLeaderboardDailyStatRow,
|
||||
{"stat_date": stat_date, "settings_hash": settings_hash},
|
||||
row = await _time_leaderboard_db_operation(
|
||||
"snapshot.store.get",
|
||||
lambda: session.get(
|
||||
AdminLeaderboardDailyStatRow,
|
||||
{"stat_date": stat_date, "settings_hash": settings_hash},
|
||||
),
|
||||
sql="SELECT admin_leaderboard_daily_stats by primary key",
|
||||
stat_date=stat_date,
|
||||
settings_hash=settings_hash,
|
||||
)
|
||||
if row is None:
|
||||
row = AdminLeaderboardDailyStatRow(
|
||||
@@ -1630,7 +1789,13 @@ async def _store_daily_snapshot(
|
||||
if generated_at is not None:
|
||||
row.generated_at = generated_at
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
await _time_leaderboard_db_operation(
|
||||
"snapshot.store.commit",
|
||||
lambda: session.commit(),
|
||||
sql="COMMIT",
|
||||
stat_date=stat_date,
|
||||
settings_hash=settings_hash,
|
||||
)
|
||||
|
||||
|
||||
async def _queue_daily_leaderboard_snapshots(
|
||||
@@ -1646,12 +1811,17 @@ async def _queue_daily_leaderboard_snapshots(
|
||||
return queued
|
||||
async with session_factory() as session:
|
||||
now = datetime.now(UTC)
|
||||
stmt = select(AdminLeaderboardDailyStatRow).where(
|
||||
AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
|
||||
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
|
||||
)
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(AdminLeaderboardDailyStatRow).where(
|
||||
AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
|
||||
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
|
||||
)
|
||||
await _execute_leaderboard_query(
|
||||
session,
|
||||
"snapshot_queue.load_existing",
|
||||
stmt,
|
||||
stat_dates=stat_dates,
|
||||
settings_hash=settings_hash,
|
||||
)
|
||||
).scalars().all()
|
||||
rows_by_date = {row.stat_date: row for row in rows}
|
||||
@@ -1680,7 +1850,13 @@ async def _queue_daily_leaderboard_snapshots(
|
||||
row.updated_at = now
|
||||
queued.append(stat_date)
|
||||
if queued:
|
||||
await session.commit()
|
||||
await _time_leaderboard_db_operation(
|
||||
"snapshot_queue.commit",
|
||||
lambda: session.commit(),
|
||||
sql="COMMIT",
|
||||
stat_dates=queued,
|
||||
settings_hash=settings_hash,
|
||||
)
|
||||
return queued
|
||||
|
||||
|
||||
@@ -1722,7 +1898,22 @@ async def _refresh_daily_leaderboard_snapshot(
|
||||
include_failed: bool,
|
||||
) -> None:
|
||||
acquired = False
|
||||
trace_token = _LEADERBOARD_TRACE_CONTEXT.set(
|
||||
{
|
||||
"job_key": job_key,
|
||||
"stat_date": stat_date,
|
||||
"settings_hash": settings_hash,
|
||||
}
|
||||
)
|
||||
snapshot_start = perf_counter()
|
||||
try:
|
||||
logger.info(
|
||||
"Leaderboard snapshot build start: job=%s stat_date=%s settings_hash=%s started_at=%s",
|
||||
job_key,
|
||||
stat_date,
|
||||
settings_hash,
|
||||
datetime.now(BEIJING_TZ).isoformat(),
|
||||
)
|
||||
if _LEADERBOARD_START_DELAY_SECONDS > 0:
|
||||
await asyncio.sleep(_LEADERBOARD_START_DELAY_SECONDS)
|
||||
await _store_daily_snapshot(
|
||||
@@ -1758,6 +1949,14 @@ async def _refresh_daily_leaderboard_snapshot(
|
||||
error=None,
|
||||
generated_at=generated_at,
|
||||
)
|
||||
logger.info(
|
||||
"Leaderboard snapshot build done: job=%s stat_date=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
|
||||
job_key,
|
||||
stat_date,
|
||||
settings_hash,
|
||||
datetime.now(BEIJING_TZ).isoformat(),
|
||||
(perf_counter() - snapshot_start) * 1000,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("Daily leaderboard snapshot timed out: %s", job_key)
|
||||
await _store_daily_snapshot(
|
||||
@@ -1781,6 +1980,7 @@ async def _refresh_daily_leaderboard_snapshot(
|
||||
finally:
|
||||
if acquired:
|
||||
_leaderboard_semaphore.release()
|
||||
_LEADERBOARD_TRACE_CONTEXT.reset(trace_token)
|
||||
|
||||
|
||||
async def _load_or_schedule_daily_leaderboard(
|
||||
@@ -1978,6 +2178,72 @@ async def queue_leaderboard_snapshot_backfill(
|
||||
)
|
||||
|
||||
|
||||
@router.post("/leaderboard/refresh-today", response_model=AdminLeaderboardResponse)
|
||||
async def refresh_today_leaderboard(
|
||||
request: Request,
|
||||
days: int = Query(default=30, ge=1, le=365),
|
||||
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
|
||||
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
|
||||
exclude_user_ids: list[str] | None = Query(default=None, description="User ids excluded from this leaderboard aggregation"),
|
||||
include_scheduled: bool | None = Query(default=None, description="Include runs created by scheduled tasks"),
|
||||
include_admins: bool | None = Query(default=None, description="Include admin accounts in leaderboard aggregation"),
|
||||
include_failed: bool | None = Query(default=None, description="Include failed runs and LLM calls"),
|
||||
limit: int = Query(default=20, ge=5, le=100),
|
||||
) -> AdminLeaderboardResponse:
|
||||
"""Force a synchronous rebuild of *today's* snapshot, then return the range.
|
||||
|
||||
Today's snapshot otherwise refreshes only every ``_LEADERBOARD_TODAY_TTL_SECONDS``
|
||||
(~5 min) via the background scheduler. This endpoint lets an admin pull the
|
||||
latest real-time records into the stats on demand: it rebuilds today's daily
|
||||
snapshot immediately (bounded by the same semaphore/timeout the scheduler uses)
|
||||
and then returns the merged leaderboard for the requested range.
|
||||
"""
|
||||
await _require_admin(request)
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
|
||||
|
||||
settings = load_system_settings().leaderboard
|
||||
(
|
||||
excluded,
|
||||
resolved_include_scheduled,
|
||||
resolved_include_admins,
|
||||
resolved_include_failed,
|
||||
settings_hash,
|
||||
) = _resolve_leaderboard_snapshot_scope(
|
||||
settings,
|
||||
exclude_user_ids=exclude_user_ids,
|
||||
include_scheduled=include_scheduled,
|
||||
include_admins=include_admins,
|
||||
include_failed=include_failed,
|
||||
)
|
||||
today = _today_stat_date()
|
||||
await _refresh_daily_leaderboard_snapshot(
|
||||
_daily_job_key(today, settings_hash),
|
||||
session_factory=session_factory,
|
||||
settings=settings,
|
||||
stat_date=today,
|
||||
settings_hash=settings_hash,
|
||||
excluded=excluded,
|
||||
include_scheduled=resolved_include_scheduled,
|
||||
include_admins=resolved_include_admins,
|
||||
include_failed=resolved_include_failed,
|
||||
)
|
||||
|
||||
since_dt, until_dt, _range_days = _resolve_range(days, since, until)
|
||||
return await _load_or_schedule_daily_leaderboard(
|
||||
session_factory,
|
||||
settings,
|
||||
since_dt,
|
||||
until_dt,
|
||||
limit=limit,
|
||||
exclude_user_ids=exclude_user_ids,
|
||||
include_scheduled=include_scheduled,
|
||||
include_admins=include_admins,
|
||||
include_failed=include_failed,
|
||||
)
|
||||
|
||||
|
||||
async def _build_leaderboard_report_prompt(
|
||||
session_factory,
|
||||
settings,
|
||||
@@ -2248,22 +2514,182 @@ class AdminUserQaPageResponse(BaseModel):
|
||||
items: list[AdminUserQaItem] = Field(default_factory=list)
|
||||
|
||||
|
||||
@router.get("/{user_id}/qa", response_model=AdminUserQaPageResponse)
|
||||
async def list_user_qa(
|
||||
user_id: str,
|
||||
request: Request,
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=20, ge=1, le=100),
|
||||
q: str | None = Query(default=None, description="Search question, answer, run id, or thread id"),
|
||||
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
|
||||
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
|
||||
) -> AdminUserQaPageResponse:
|
||||
"""Return one user's Q&A runs from lightweight run summary fields."""
|
||||
await _require_admin(request)
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
raise HTTPException(status_code=503, detail="Q&A audit requires database persistence")
|
||||
def _qa_sort_datetime(value: str | None) -> datetime:
|
||||
if not value:
|
||||
return datetime.min.replace(tzinfo=BEIJING_TZ)
|
||||
text_value = str(value).strip()
|
||||
if text_value.endswith("Z"):
|
||||
text_value = text_value[:-1] + "+00:00"
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text_value)
|
||||
except ValueError:
|
||||
return datetime.min.replace(tzinfo=BEIJING_TZ)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=BEIJING_TZ)
|
||||
return parsed.astimezone(BEIJING_TZ)
|
||||
|
||||
|
||||
def _qa_dedupe_key(item: AdminUserQaItem) -> tuple[str, str, str]:
|
||||
question_key = _question_similarity_key(item.question)[:160]
|
||||
answer_key = _question_similarity_key(item.answer)[:160]
|
||||
return (item.thread_id, question_key, answer_key)
|
||||
|
||||
|
||||
def _message_value(message: Any, key: str, default: Any = None) -> Any:
|
||||
if isinstance(message, dict):
|
||||
return message.get(key, default)
|
||||
return getattr(message, key, default)
|
||||
|
||||
|
||||
def _qa_message_type(message: Any) -> str:
|
||||
value = _message_value(message, "type") or _message_value(message, "role") or ""
|
||||
if value == "user":
|
||||
return "human"
|
||||
if value == "assistant":
|
||||
return "ai"
|
||||
return str(value)
|
||||
|
||||
|
||||
def _qa_message_id(message: Any, fallback: str) -> str:
|
||||
return str(_message_value(message, "id") or fallback)
|
||||
|
||||
|
||||
def _qa_message_text(message: Any) -> str:
|
||||
return _message_content_to_text(_message_value(message, "content")).strip()
|
||||
|
||||
|
||||
async def _read_thread_checkpoint_messages(checkpointer, thread_id: str) -> list[dict]:
|
||||
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
try:
|
||||
checkpoint_tuple = await checkpointer.aget_tuple(config)
|
||||
except Exception:
|
||||
logger.warning("Failed to read checkpoint while listing Q&A: thread=%s", thread_id, exc_info=True)
|
||||
return []
|
||||
if checkpoint_tuple is None:
|
||||
return []
|
||||
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
|
||||
channel_values = checkpoint.get("channel_values", {}) if isinstance(checkpoint, dict) else {}
|
||||
values = serialize_channel_values(channel_values)
|
||||
messages = values.get("messages", [])
|
||||
return messages if isinstance(messages, list) else []
|
||||
|
||||
|
||||
def _qa_items_from_checkpoint_messages(
|
||||
*,
|
||||
user_id: str,
|
||||
thread_id: str,
|
||||
thread_title: str,
|
||||
assistant_id: str,
|
||||
thread_time: str,
|
||||
messages: list[dict],
|
||||
) -> list[AdminUserQaItem]:
|
||||
items: list[AdminUserQaItem] = []
|
||||
current_question = ""
|
||||
current_answer = ""
|
||||
current_id = ""
|
||||
current_index = 0
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal current_question, current_answer, current_id, current_index
|
||||
question = _clean_question_text(current_question)
|
||||
if not _is_displayable_question(question):
|
||||
current_question = ""
|
||||
current_answer = ""
|
||||
current_id = ""
|
||||
return
|
||||
items.append(
|
||||
AdminUserQaItem(
|
||||
run_id=current_id or f"{thread_id}:checkpoint:{current_index}",
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
assistant_id=assistant_id,
|
||||
status="checkpoint",
|
||||
question=question[:2000],
|
||||
answer=current_answer.strip()[:6000],
|
||||
created_at=thread_time,
|
||||
updated_at=thread_time,
|
||||
)
|
||||
)
|
||||
current_question = ""
|
||||
current_answer = ""
|
||||
current_id = ""
|
||||
|
||||
for index, message in enumerate(messages):
|
||||
msg_type = _qa_message_type(message)
|
||||
if msg_type == "human":
|
||||
if current_question:
|
||||
flush()
|
||||
current_question = _qa_message_text(message)
|
||||
current_answer = ""
|
||||
current_id = _qa_message_id(message, f"{thread_id}:checkpoint:{index}")
|
||||
current_index = index
|
||||
continue
|
||||
if msg_type == "ai" and current_question:
|
||||
text = _qa_message_text(message)
|
||||
if text:
|
||||
current_answer = text
|
||||
|
||||
if current_question:
|
||||
flush()
|
||||
|
||||
return items
|
||||
|
||||
|
||||
async def _list_thread_checkpoint_qa_items(
|
||||
*,
|
||||
request: Request,
|
||||
user_id: str,
|
||||
) -> list[AdminUserQaItem]:
|
||||
repo = get_thread_store(request)
|
||||
checkpointer = get_checkpointer(request)
|
||||
items: list[AdminUserQaItem] = []
|
||||
offset = 0
|
||||
page_size = 200
|
||||
while True:
|
||||
rows = await repo.search(limit=page_size, offset=offset, user_id=user_id)
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
thread_id = str(row.get("thread_id") or "")
|
||||
if not thread_id:
|
||||
continue
|
||||
messages = await _read_thread_checkpoint_messages(checkpointer, thread_id)
|
||||
if not messages:
|
||||
continue
|
||||
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
|
||||
context = row.get("context") if isinstance(row.get("context"), dict) else {}
|
||||
assistant_id = str(
|
||||
row.get("assistant_id")
|
||||
or (context or {}).get("agent_id")
|
||||
or (metadata or {}).get("agent_id")
|
||||
or (context or {}).get("agent_name")
|
||||
or (metadata or {}).get("agent_name")
|
||||
or ""
|
||||
)
|
||||
thread_time = str(row.get("updated_at") or row.get("created_at") or "")
|
||||
items.extend(
|
||||
_qa_items_from_checkpoint_messages(
|
||||
user_id=user_id,
|
||||
thread_id=thread_id,
|
||||
thread_title=str(row.get("display_name") or ""),
|
||||
assistant_id=assistant_id,
|
||||
thread_time=thread_time,
|
||||
messages=messages,
|
||||
)
|
||||
)
|
||||
if len(rows) < page_size:
|
||||
break
|
||||
offset += len(rows)
|
||||
return items
|
||||
|
||||
|
||||
async def _list_run_summary_qa_items(
|
||||
*,
|
||||
session_factory,
|
||||
user_id: str,
|
||||
since: str | None,
|
||||
until: str | None,
|
||||
) -> list[AdminUserQaItem]:
|
||||
filters = [
|
||||
RunRow.user_id == user_id,
|
||||
RunRow.first_human_message.is_not(None),
|
||||
@@ -2272,26 +2698,9 @@ async def list_user_qa(
|
||||
if since or until:
|
||||
since_dt, until_dt, _range_days = _resolve_range(365, since, until)
|
||||
filters.append(_run_time_filter(since_dt, until_dt))
|
||||
needle = (q or "").strip()
|
||||
if needle:
|
||||
like = f"%{needle}%"
|
||||
filters.append(
|
||||
or_(
|
||||
RunRow.first_human_message.like(like),
|
||||
RunRow.last_ai_message.like(like),
|
||||
RunRow.run_id.like(like),
|
||||
RunRow.thread_id.like(like),
|
||||
)
|
||||
)
|
||||
|
||||
offset = (page - 1) * page_size
|
||||
async with session_factory() as session:
|
||||
await _set_analytics_read_timeout(session)
|
||||
total = (
|
||||
await session.execute(
|
||||
select(func.count(RunRow.run_id)).where(*filters)
|
||||
)
|
||||
).scalar_one()
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
@@ -2316,32 +2725,152 @@ async def list_user_qa(
|
||||
)
|
||||
.where(*filters)
|
||||
.order_by(RunRow.created_at.desc(), RunRow.run_id.desc())
|
||||
.limit(page_size)
|
||||
.offset(offset)
|
||||
)
|
||||
).all()
|
||||
|
||||
return [
|
||||
AdminUserQaItem(
|
||||
run_id=r[0] or "",
|
||||
thread_id=r[1] or "",
|
||||
thread_title=r[2] or "",
|
||||
assistant_id=r[3] or "",
|
||||
status=r[4] or "",
|
||||
model_name=r[5] or "",
|
||||
question=_clean_question_text(r[6])[:2000],
|
||||
answer=str(r[7] or "").strip()[:6000],
|
||||
total_tokens=int(r[8] or 0),
|
||||
created_at=_iso(r[9]),
|
||||
updated_at=_iso(r[10]),
|
||||
)
|
||||
for r in rows
|
||||
if _is_displayable_question(r[6])
|
||||
]
|
||||
|
||||
|
||||
def _filter_qa_items(
|
||||
items: list[AdminUserQaItem],
|
||||
*,
|
||||
q: str | None,
|
||||
since: str | None,
|
||||
until: str | None,
|
||||
) -> list[AdminUserQaItem]:
|
||||
needle = (q or "").strip().lower()
|
||||
since_dt = _parse_optional_datetime(since) if since else None
|
||||
until_dt = _parse_optional_datetime(until) if until else None
|
||||
filtered: list[AdminUserQaItem] = []
|
||||
for item in items:
|
||||
item_dt = _qa_sort_datetime(item.created_at or item.updated_at)
|
||||
if since_dt and item_dt < since_dt:
|
||||
continue
|
||||
if until_dt and item_dt > until_dt:
|
||||
continue
|
||||
if needle:
|
||||
haystack = "\n".join(
|
||||
[
|
||||
item.question,
|
||||
item.answer,
|
||||
item.run_id,
|
||||
item.thread_id,
|
||||
item.thread_title,
|
||||
item.assistant_id,
|
||||
]
|
||||
).lower()
|
||||
if needle not in haystack:
|
||||
continue
|
||||
filtered.append(item)
|
||||
return filtered
|
||||
|
||||
|
||||
async def _list_user_qa_page(
|
||||
*,
|
||||
request: Request,
|
||||
user_id: str,
|
||||
page: int,
|
||||
page_size: int,
|
||||
q: str | None,
|
||||
since: str | None,
|
||||
until: str | None,
|
||||
include_checkpoints: bool = True,
|
||||
) -> AdminUserQaPageResponse:
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
raise HTTPException(status_code=503, detail="Q&A audit requires database persistence")
|
||||
|
||||
items = await _list_run_summary_qa_items(
|
||||
session_factory=session_factory,
|
||||
user_id=user_id,
|
||||
since=since,
|
||||
until=until,
|
||||
)
|
||||
if include_checkpoints:
|
||||
items.extend(await _list_thread_checkpoint_qa_items(request=request, user_id=user_id))
|
||||
|
||||
by_key: dict[tuple[str, str, str], AdminUserQaItem] = {}
|
||||
for item in items:
|
||||
key = _qa_dedupe_key(item)
|
||||
existing = by_key.get(key)
|
||||
if existing is None or (existing.status == "checkpoint" and item.status != "checkpoint"):
|
||||
by_key[key] = item
|
||||
|
||||
filtered = _filter_qa_items(list(by_key.values()), q=q, since=since, until=until)
|
||||
filtered.sort(
|
||||
key=lambda item: (_qa_sort_datetime(item.created_at or item.updated_at), item.thread_id, item.run_id),
|
||||
reverse=True,
|
||||
)
|
||||
offset = (page - 1) * page_size
|
||||
return AdminUserQaPageResponse(
|
||||
user_id=user_id,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
total=int(total or 0),
|
||||
items=[
|
||||
AdminUserQaItem(
|
||||
run_id=r[0] or "",
|
||||
thread_id=r[1] or "",
|
||||
thread_title=r[2] or "",
|
||||
assistant_id=r[3] or "",
|
||||
status=r[4] or "",
|
||||
model_name=r[5] or "",
|
||||
question=_clean_question_text(r[6])[:2000],
|
||||
answer=str(r[7] or "").strip()[:6000],
|
||||
total_tokens=int(r[8] or 0),
|
||||
created_at=_iso(r[9]),
|
||||
updated_at=_iso(r[10]),
|
||||
)
|
||||
for r in rows
|
||||
],
|
||||
total=len(filtered),
|
||||
items=filtered[offset : offset + page_size],
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{user_id}/qa", response_model=AdminUserQaPageResponse)
|
||||
async def list_user_qa(
|
||||
user_id: str,
|
||||
request: Request,
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=20, ge=1, le=100),
|
||||
q: str | None = Query(default=None, description="Search question, answer, run id, or thread id"),
|
||||
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
|
||||
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
|
||||
) -> AdminUserQaPageResponse:
|
||||
"""Return one user's Q&A runs, including older checkpoint-only threads."""
|
||||
await _require_admin(request)
|
||||
return await _list_user_qa_page(
|
||||
request=request,
|
||||
user_id=user_id,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
q=q,
|
||||
since=since,
|
||||
until=until,
|
||||
)
|
||||
|
||||
|
||||
@self_router.get("/qa", response_model=AdminUserQaPageResponse)
|
||||
async def list_my_qa(
|
||||
request: Request,
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=20, ge=1, le=100),
|
||||
q: str | None = Query(default=None, description="Search question, answer, run id, or thread id"),
|
||||
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
|
||||
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
|
||||
) -> AdminUserQaPageResponse:
|
||||
"""Return the current user's own Q&A history."""
|
||||
user = getattr(request.state, "user", None)
|
||||
if user is None:
|
||||
user = await get_current_user_from_request(request)
|
||||
return await _list_user_qa_page(
|
||||
request=request,
|
||||
user_id=str(user.id),
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
q=q,
|
||||
since=since,
|
||||
until=until,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.auth.password import verify_password_async
|
||||
from app.gateway.deps import get_agent_store, get_config, get_tag_store
|
||||
from app.gateway.deps import get_agent_store, get_config, get_tag_store
|
||||
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
|
||||
from deerflow.config.agents_api_config import get_agents_api_config
|
||||
from deerflow.config.agents_config import list_custom_agents, load_agent_config, load_agent_soul, validate_agent_id
|
||||
@@ -39,6 +39,7 @@ class AgentResponse(BaseModel):
|
||||
name: str = Field(..., description="Display name. Chinese and duplicate names are allowed.")
|
||||
description: str = Field(default="", description="Agent description")
|
||||
user_id: str | None = Field(default=None, description="Owner user id. Null means built-in.")
|
||||
owner_username: str | None = Field(default=None, description="Publisher username (email local part). Null for built-in agents.")
|
||||
published: bool = Field(default=False, description="Whether a user-created agent is public")
|
||||
is_private: bool = Field(default=False, description="Whether the agent lives in the password-gated private square instead of the public one")
|
||||
builtin: bool = Field(default=False, description="True when user_id is null")
|
||||
@@ -62,15 +63,15 @@ class AgentResponse(BaseModel):
|
||||
is_favorite: bool = Field(
|
||||
default=False,
|
||||
description="Whether the requesting user has favorited (收藏) this agent.",
|
||||
)
|
||||
|
||||
is_pinned: bool = Field(
|
||||
default=False,
|
||||
description="Whether the requesting user has pinned this agent.",
|
||||
)
|
||||
|
||||
|
||||
class AgentsListResponse(BaseModel):
|
||||
)
|
||||
|
||||
is_pinned: bool = Field(
|
||||
default=False,
|
||||
description="Whether the requesting user has pinned this agent.",
|
||||
)
|
||||
|
||||
|
||||
class AgentsListResponse(BaseModel):
|
||||
"""Response model for listing agents.
|
||||
|
||||
``total`` / ``page`` / ``page_size`` describe server-side pagination. When
|
||||
@@ -115,7 +116,7 @@ class AgentUpdateRequest(BaseModel):
|
||||
tool_groups: list[str] | None = Field(default=None, description="Updated tool group whitelist (config.yaml)")
|
||||
|
||||
|
||||
class AgentSelectorResponse(BaseModel):
|
||||
class AgentSelectorResponse(BaseModel):
|
||||
"""Response for the homepage agent selector dropdown.
|
||||
|
||||
The list is fully curated by admins (``featured_order`` on each row).
|
||||
@@ -125,10 +126,10 @@ class AgentSelectorResponse(BaseModel):
|
||||
agents: list[AgentResponse] = Field(
|
||||
default_factory=list,
|
||||
description="Admin-curated agents, sorted by featured_order ascending.",
|
||||
)
|
||||
|
||||
|
||||
class AgentFeaturedRequest(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class AgentFeaturedRequest(BaseModel):
|
||||
"""Request body for setting/clearing an agent's homepage featured rank."""
|
||||
|
||||
order: int | None = Field(
|
||||
@@ -159,6 +160,41 @@ def _is_admin_user(request: Request) -> bool:
|
||||
return getattr(user, "system_role", None) == "admin"
|
||||
|
||||
|
||||
def _email_local_part(email: str | None) -> str | None:
|
||||
"""Return the username portion (before ``@``) of an email address."""
|
||||
if not email:
|
||||
return None
|
||||
at = email.find("@")
|
||||
return email[:at] if at > 0 else email
|
||||
|
||||
|
||||
async def _resolve_usernames(user_ids: set[str]) -> dict[str, str]:
|
||||
"""Batch-resolve owner user ids to friendly usernames (email local part).
|
||||
|
||||
One lookup per distinct id; tolerates missing accounts (deleted users) and
|
||||
an uninitialised auth provider by simply omitting those ids from the map.
|
||||
"""
|
||||
user_ids = {uid for uid in user_ids if uid}
|
||||
if not user_ids:
|
||||
return {}
|
||||
try:
|
||||
from app.gateway.deps import get_local_provider
|
||||
|
||||
provider = get_local_provider()
|
||||
except Exception:
|
||||
return {}
|
||||
names: dict[str, str] = {}
|
||||
for uid in user_ids:
|
||||
try:
|
||||
user = await provider.get_user(uid)
|
||||
except Exception:
|
||||
user = None
|
||||
local = _email_local_part(getattr(user, "email", None)) if user is not None else None
|
||||
if local:
|
||||
names[uid] = local
|
||||
return names
|
||||
|
||||
|
||||
def _validate_display_name(name: str) -> str:
|
||||
normalized = name.strip()
|
||||
if not normalized:
|
||||
@@ -348,7 +384,7 @@ def _ensure_agent_dir_from_record(record: dict[str, Any]) -> None:
|
||||
logger.exception("Failed to self-heal agent dir for '%s'", agent_id)
|
||||
|
||||
|
||||
def _record_to_response(
|
||||
def _record_to_response(
|
||||
record: dict[str, Any],
|
||||
*,
|
||||
include_soul: bool = False,
|
||||
@@ -390,21 +426,21 @@ def _record_to_response(
|
||||
tool_groups=extras["tool_groups"],
|
||||
skills=extras["skills"],
|
||||
featured_order=record.get("featured_order"),
|
||||
)
|
||||
|
||||
|
||||
def _sort_agent_responses(responses: list[AgentResponse], sort: str) -> list[AgentResponse]:
|
||||
reverse = sort not in {"time_asc", "updated_asc", "name_asc"}
|
||||
if sort in {"name_asc", "name_desc"}:
|
||||
ordered = sorted(responses, key=lambda r: (r.name or "").lower(), reverse=reverse)
|
||||
elif sort in {"updated_desc", "updated_asc"}:
|
||||
ordered = sorted(responses, key=lambda r: str(r.updated_at or ""), reverse=reverse)
|
||||
else:
|
||||
ordered = sorted(responses, key=lambda r: str(r.created_at or ""), reverse=reverse)
|
||||
return sorted(ordered, key=lambda r: 0 if r.is_pinned else 1)
|
||||
|
||||
|
||||
async def _record_to_response_async(
|
||||
)
|
||||
|
||||
|
||||
def _sort_agent_responses(responses: list[AgentResponse], sort: str) -> list[AgentResponse]:
|
||||
reverse = sort not in {"time_asc", "updated_asc", "name_asc"}
|
||||
if sort in {"name_asc", "name_desc"}:
|
||||
ordered = sorted(responses, key=lambda r: (r.name or "").lower(), reverse=reverse)
|
||||
elif sort in {"updated_desc", "updated_asc"}:
|
||||
ordered = sorted(responses, key=lambda r: str(r.updated_at or ""), reverse=reverse)
|
||||
else:
|
||||
ordered = sorted(responses, key=lambda r: str(r.created_at or ""), reverse=reverse)
|
||||
return sorted(ordered, key=lambda r: 0 if r.is_pinned else 1)
|
||||
|
||||
|
||||
async def _record_to_response_async(
|
||||
record: dict[str, Any],
|
||||
*,
|
||||
include_soul: bool = False,
|
||||
@@ -478,20 +514,23 @@ async def _build_list_responses(
|
||||
_record_to_response(a, include_soul=False, include_extras=False) for a in records
|
||||
]
|
||||
ids = [r.id for r in responses]
|
||||
extras = await store.get_extras_for(ids)
|
||||
assignments = await tag_store.list_assignments("agent", ids)
|
||||
favorite_ids = await store.list_favorite_ids(user_id)
|
||||
pin_ids = await store.list_pin_ids(user_id)
|
||||
for resp in responses:
|
||||
extras = await store.get_extras_for(ids)
|
||||
assignments = await tag_store.list_assignments("agent", ids)
|
||||
favorite_ids = await store.list_favorite_ids(user_id)
|
||||
pin_ids = await store.list_pin_ids(user_id)
|
||||
usernames = await _resolve_usernames({r.user_id for r in responses if r.user_id})
|
||||
for resp in responses:
|
||||
cached = extras.get(resp.id)
|
||||
if cached is not None:
|
||||
resp.skills = cached.get("skills")
|
||||
resp.tool_groups = cached.get("tool_groups")
|
||||
resp.model = cached.get("model")
|
||||
resp.tags = assignments.get(resp.id, [])
|
||||
resp.is_favorite = resp.id in favorite_ids
|
||||
resp.is_pinned = resp.id in pin_ids
|
||||
return responses
|
||||
resp.tags = assignments.get(resp.id, [])
|
||||
resp.is_favorite = resp.id in favorite_ids
|
||||
resp.is_pinned = resp.id in pin_ids
|
||||
if resp.user_id:
|
||||
resp.owner_username = usernames.get(resp.user_id)
|
||||
return responses
|
||||
|
||||
|
||||
@router.get(
|
||||
@@ -509,14 +548,18 @@ async def list_agents(
|
||||
pattern="^(mine|square)$",
|
||||
description="Pagination scope. 'mine' = owned + favorited; 'square' = built-in/published public. Omit for the default combined visibility list.",
|
||||
),
|
||||
page: int | None = Query(default=None, ge=1, description="1-based page index. Providing this (or 'scope') enables server-side pagination."),
|
||||
page_size: int = Query(default=20, ge=1, le=200, description="Items per page when paginating."),
|
||||
sort: str = Query(
|
||||
default="time_desc",
|
||||
pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
|
||||
description="Sort order. Default is newest created first.",
|
||||
),
|
||||
admin: bool = Query(
|
||||
page: int | None = Query(default=None, ge=1, description="1-based page index. Providing this (or 'scope') enables server-side pagination."),
|
||||
page_size: int = Query(default=20, ge=1, le=200, description="Items per page when paginating."),
|
||||
sort: str = Query(
|
||||
default="time_desc",
|
||||
pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
|
||||
description="Sort order. Default is newest created first.",
|
||||
),
|
||||
owner: str | None = Query(
|
||||
default=None,
|
||||
description="Keep only agents published by this user id (the card's publisher). Built-in agents never match.",
|
||||
),
|
||||
admin: bool = Query(
|
||||
default=False,
|
||||
description=(
|
||||
"Admin-only override. When ``true`` and the caller is a system admin, "
|
||||
@@ -543,11 +586,12 @@ async def list_agents(
|
||||
scope=scope,
|
||||
search=search,
|
||||
tag_ids=tag_ids,
|
||||
page=eff_page,
|
||||
page_size=page_size,
|
||||
admin=admin,
|
||||
sort=sort,
|
||||
)
|
||||
page=eff_page,
|
||||
page_size=page_size,
|
||||
admin=admin,
|
||||
sort=sort,
|
||||
owner=owner,
|
||||
)
|
||||
responses = await _build_list_responses(store, tag_store, user_id, rows)
|
||||
return AgentsListResponse(agents=responses, total=total, page=eff_page, page_size=page_size)
|
||||
|
||||
@@ -559,11 +603,13 @@ async def list_agents(
|
||||
if search:
|
||||
needle = search.strip().lower()
|
||||
responses = [r for r in responses if needle in r.name.lower()]
|
||||
if tag_ids:
|
||||
wanted = set(tag_ids)
|
||||
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
|
||||
responses = _sort_agent_responses(responses, sort)
|
||||
return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
|
||||
if tag_ids:
|
||||
wanted = set(tag_ids)
|
||||
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
|
||||
if owner and owner.strip():
|
||||
responses = [r for r in responses if r.user_id == owner.strip()]
|
||||
responses = _sort_agent_responses(responses, sort)
|
||||
return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
@@ -576,14 +622,14 @@ async def list_agents(
|
||||
summary="Check Agent Name",
|
||||
description="Validate an agent display name. Duplicate names are allowed.",
|
||||
)
|
||||
async def check_agent_name(name: str) -> dict[str, Any]:
|
||||
_require_agents_api_enabled()
|
||||
normalized = _validate_display_name(name)
|
||||
return {"available": True, "name": normalized}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/agents/selector",
|
||||
async def check_agent_name(name: str) -> dict[str, Any]:
|
||||
_require_agents_api_enabled()
|
||||
normalized = _validate_display_name(name)
|
||||
return {"available": True, "name": normalized}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/agents/selector",
|
||||
response_model=AgentSelectorResponse,
|
||||
summary="Agent Homepage Selector",
|
||||
description=(
|
||||
@@ -635,12 +681,12 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
|
||||
# Detail view legitimately reads SOUL.md + config.yaml — hop to a
|
||||
# worker thread so the asyncio loop stays responsive under concurrent
|
||||
# detail fetches.
|
||||
response = await _record_to_response_async(record, include_soul=True, include_extras=True)
|
||||
favorite_ids = await store.list_favorite_ids(user_id)
|
||||
pin_ids = await store.list_pin_ids(user_id)
|
||||
response.is_favorite = agent_id in favorite_ids
|
||||
response.is_pinned = agent_id in pin_ids
|
||||
return response
|
||||
response = await _record_to_response_async(record, include_soul=True, include_extras=True)
|
||||
favorite_ids = await store.list_favorite_ids(user_id)
|
||||
pin_ids = await store.list_pin_ids(user_id)
|
||||
response.is_favorite = agent_id in favorite_ids
|
||||
response.is_pinned = agent_id in pin_ids
|
||||
return response
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
@@ -874,16 +920,16 @@ async def delete_agent(agent_id: str, request: Request) -> None:
|
||||
await get_tag_store(request).unassign_all("agent", agent_id)
|
||||
except Exception: # noqa: BLE001 — tag cleanup must not block deletion
|
||||
pass
|
||||
try:
|
||||
await store.delete_favorites_for_agent(agent_id)
|
||||
except Exception: # noqa: BLE001 — favorite cleanup must not block deletion
|
||||
pass
|
||||
try:
|
||||
await store.delete_pins_for_agent(agent_id)
|
||||
except Exception: # noqa: BLE001 - pin cleanup must not block deletion
|
||||
pass
|
||||
try:
|
||||
await store.delete_extras_for_agent(agent_id)
|
||||
try:
|
||||
await store.delete_favorites_for_agent(agent_id)
|
||||
except Exception: # noqa: BLE001 — favorite cleanup must not block deletion
|
||||
pass
|
||||
try:
|
||||
await store.delete_pins_for_agent(agent_id)
|
||||
except Exception: # noqa: BLE001 - pin cleanup must not block deletion
|
||||
pass
|
||||
try:
|
||||
await store.delete_extras_for_agent(agent_id)
|
||||
except Exception: # noqa: BLE001 — cache cleanup must not block deletion
|
||||
pass
|
||||
logger.info("Deleted agent '%s' from %s", agent_id, agent_dir)
|
||||
@@ -925,7 +971,7 @@ async def favorite_agent(agent_id: str, request: Request) -> None:
|
||||
summary="Unfavorite Agent",
|
||||
description="Remove an agent from the current user's favorites. No-op when not favorited.",
|
||||
)
|
||||
async def unfavorite_agent(agent_id: str, request: Request) -> None:
|
||||
async def unfavorite_agent(agent_id: str, request: Request) -> None:
|
||||
_require_agents_api_enabled()
|
||||
agent_id = _validate_storage_id(agent_id)
|
||||
|
||||
@@ -936,60 +982,60 @@ async def unfavorite_agent(agent_id: str, request: Request) -> None:
|
||||
logger.info("User '%s' unfavorited agent '%s'", user_id, agent_id)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to unfavorite agent '%s': %s", agent_id, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to unfavorite agent: {str(e)}") from e
|
||||
|
||||
|
||||
@router.post(
|
||||
"/agents/{agent_id}/pin",
|
||||
status_code=204,
|
||||
summary="Pin Agent",
|
||||
description="Pin a visible agent to the top of the current user's lists. Idempotent.",
|
||||
)
|
||||
async def pin_agent(agent_id: str, request: Request) -> None:
|
||||
_require_agents_api_enabled()
|
||||
agent_id = _validate_storage_id(agent_id)
|
||||
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
user_id = _current_user_id(request)
|
||||
record = await store.get_visible(agent_id, user_id)
|
||||
if record is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
await store.add_pin(user_id, agent_id)
|
||||
logger.info("User '%s' pinned agent '%s'", user_id, agent_id)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to pin agent '%s': %s", agent_id, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to pin agent: {str(e)}") from e
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/agents/{agent_id}/pin",
|
||||
status_code=204,
|
||||
summary="Unpin Agent",
|
||||
description="Remove the current user's pin from an agent. No-op when not pinned.",
|
||||
)
|
||||
async def unpin_agent(agent_id: str, request: Request) -> None:
|
||||
_require_agents_api_enabled()
|
||||
agent_id = _validate_storage_id(agent_id)
|
||||
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
user_id = _current_user_id(request)
|
||||
await store.remove_pin(user_id, agent_id)
|
||||
logger.info("User '%s' unpinned agent '%s'", user_id, agent_id)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to unpin agent '%s': %s", agent_id, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to unpin agent: {str(e)}") from e
|
||||
|
||||
|
||||
@router.put(
|
||||
"/agents/{agent_id}/featured",
|
||||
except Exception as e:
|
||||
logger.error("Failed to unfavorite agent '%s': %s", agent_id, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to unfavorite agent: {str(e)}") from e
|
||||
|
||||
|
||||
@router.post(
|
||||
"/agents/{agent_id}/pin",
|
||||
status_code=204,
|
||||
summary="Pin Agent",
|
||||
description="Pin a visible agent to the top of the current user's lists. Idempotent.",
|
||||
)
|
||||
async def pin_agent(agent_id: str, request: Request) -> None:
|
||||
_require_agents_api_enabled()
|
||||
agent_id = _validate_storage_id(agent_id)
|
||||
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
user_id = _current_user_id(request)
|
||||
record = await store.get_visible(agent_id, user_id)
|
||||
if record is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
await store.add_pin(user_id, agent_id)
|
||||
logger.info("User '%s' pinned agent '%s'", user_id, agent_id)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to pin agent '%s': %s", agent_id, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to pin agent: {str(e)}") from e
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/agents/{agent_id}/pin",
|
||||
status_code=204,
|
||||
summary="Unpin Agent",
|
||||
description="Remove the current user's pin from an agent. No-op when not pinned.",
|
||||
)
|
||||
async def unpin_agent(agent_id: str, request: Request) -> None:
|
||||
_require_agents_api_enabled()
|
||||
agent_id = _validate_storage_id(agent_id)
|
||||
|
||||
try:
|
||||
store = get_agent_store(request)
|
||||
user_id = _current_user_id(request)
|
||||
await store.remove_pin(user_id, agent_id)
|
||||
logger.info("User '%s' unpinned agent '%s'", user_id, agent_id)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to unpin agent '%s': %s", agent_id, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to unpin agent: {str(e)}") from e
|
||||
|
||||
|
||||
@router.put(
|
||||
"/agents/{agent_id}/featured",
|
||||
response_model=AgentResponse,
|
||||
summary="Set Agent Featured Rank (admin)",
|
||||
description=(
|
||||
@@ -1031,19 +1077,20 @@ async def set_agent_featured(
|
||||
raise HTTPException(status_code=500, detail=f"Failed to set featured: {str(e)}") from e
|
||||
|
||||
|
||||
class PrivateSquareListRequest(BaseModel):
|
||||
class PrivateSquareListRequest(BaseModel):
|
||||
"""Request body for listing all private-square agents."""
|
||||
|
||||
password: str = Field(..., description="Plain-text private-square password. Verified against the admin-configured hash before the listing is returned.")
|
||||
search: str | None = Field(default=None, description="Case-insensitive fuzzy match on agent name (raw or desensitized).")
|
||||
tag_ids: list[str] | None = Field(default=None, description="Keep agents carrying any of these tags (OR).")
|
||||
page: int | None = Field(default=None, ge=1, description="1-based page index. Providing this enables server-side pagination.")
|
||||
page_size: int = Field(default=20, ge=1, le=200, description="Items per page when paginating.")
|
||||
sort: str = Field(
|
||||
default="time_desc",
|
||||
pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
|
||||
description="Sort order. Default is newest created first.",
|
||||
)
|
||||
page: int | None = Field(default=None, ge=1, description="1-based page index. Providing this enables server-side pagination.")
|
||||
page_size: int = Field(default=20, ge=1, le=200, description="Items per page when paginating.")
|
||||
sort: str = Field(
|
||||
default="time_desc",
|
||||
pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
|
||||
description="Sort order. Default is newest created first.",
|
||||
)
|
||||
owner: str | None = Field(default=None, description="Keep only agents published by this user id (the card's publisher).")
|
||||
|
||||
|
||||
class AgentPrivateUpdateRequest(BaseModel):
|
||||
@@ -1086,17 +1133,18 @@ async def list_private_square(body: PrivateSquareListRequest, request: Request)
|
||||
scope="private",
|
||||
search=body.search,
|
||||
tag_ids=body.tag_ids,
|
||||
page=body.page,
|
||||
page_size=body.page_size,
|
||||
sort=body.sort,
|
||||
)
|
||||
page=body.page,
|
||||
page_size=body.page_size,
|
||||
sort=body.sort,
|
||||
owner=body.owner,
|
||||
)
|
||||
responses = await _build_list_responses(store, tag_store, user_id, rows)
|
||||
return AgentsListResponse(agents=responses, total=total, page=body.page, page_size=body.page_size)
|
||||
|
||||
records = await store.list_private()
|
||||
responses = await _build_list_responses(store, tag_store, user_id, records)
|
||||
responses = _sort_agent_responses(responses, body.sort)
|
||||
return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
|
||||
responses = await _build_list_responses(store, tag_store, user_id, records)
|
||||
responses = _sort_agent_responses(responses, body.sort)
|
||||
return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
|
||||
@@ -0,0 +1,590 @@
|
||||
"""Browser page context capture endpoints.
|
||||
|
||||
This router is intentionally isolated behind ``config.yaml``:
|
||||
|
||||
browser_context:
|
||||
enabled: true
|
||||
|
||||
When disabled, mutating endpoints behave as if the integration is unavailable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import urlencode
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.deps import get_checkpointer, get_config, get_thread_store
|
||||
from app.gateway.routers.thread_runs import RunCreateRequest
|
||||
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
|
||||
from app.gateway.routers.uploads import _make_file_sandbox_writable
|
||||
from app.gateway.services import start_run
|
||||
from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.sandbox.sandbox_provider import get_sandbox_provider
|
||||
from deerflow.uploads.manager import (
|
||||
UnsafeUploadPathError,
|
||||
ensure_uploads_dir,
|
||||
normalize_filename,
|
||||
open_upload_file_no_symlink,
|
||||
upload_artifact_url,
|
||||
upload_virtual_path,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/browser-context", tags=["browser-context"])
|
||||
DEFAULT_BROWSER_CONTEXT_PROMPT = "\u8bf7\u57fa\u4e8e\u521a\u521a\u53d1\u9001\u7684\u7f51\u9875\u5185\u5bb9\uff0c\u5148\u7528\u4e2d\u6587\u603b\u7ed3\u8981\u70b9\uff0c\u5e76\u7b49\u5f85\u6211\u7ee7\u7eed\u63d0\u95ee\u3002"
|
||||
|
||||
|
||||
class BrowserContextStatusResponse(BaseModel):
|
||||
enabled: bool
|
||||
max_content_chars: int
|
||||
max_title_chars: int
|
||||
max_url_chars: int
|
||||
task_result_timeout_seconds: int
|
||||
task_poll_interval_seconds: float
|
||||
task_lease_seconds: int
|
||||
tool_result_max_chars: int
|
||||
|
||||
|
||||
class BrowserContextCaptureRequest(BaseModel):
|
||||
thread_id: str | None = Field(default=None, description="Existing or caller-generated zncm thread id")
|
||||
title: str | None = Field(default=None, description="Current browser page title")
|
||||
url: str | None = Field(default=None, description="Current browser page URL")
|
||||
content: str = Field(description="Captured page content, preferably Markdown")
|
||||
content_type: Literal["markdown", "text"] = Field(default="markdown")
|
||||
prompt: str | None = Field(default=None, description="Optional first question to ask in zncm after capture")
|
||||
|
||||
|
||||
class BrowserContextFileInfo(BaseModel):
|
||||
filename: str
|
||||
size: int
|
||||
path: str
|
||||
virtual_path: str
|
||||
artifact_url: str
|
||||
|
||||
|
||||
class BrowserContextCaptureResponse(BaseModel):
|
||||
success: bool
|
||||
thread_id: str
|
||||
file: BrowserContextFileInfo
|
||||
chat_path: str
|
||||
chat_url: str
|
||||
browser_context_token: str
|
||||
prompt: str
|
||||
truncated: bool = False
|
||||
message: str
|
||||
|
||||
|
||||
class BrowserContextAskRequest(BaseModel):
|
||||
thread_id: str | None = Field(default=None, description="Existing zncm thread id for follow-up questions")
|
||||
title: str | None = Field(default=None, description="Current browser page title")
|
||||
url: str | None = Field(default=None, description="Current browser page URL")
|
||||
content: str = Field(description="Captured page content, preferably Markdown")
|
||||
content_type: Literal["markdown", "text"] = Field(default="markdown")
|
||||
question: str = Field(description="Question asked from the in-page assistant")
|
||||
|
||||
|
||||
class BrowserContextAskResponse(BaseModel):
|
||||
success: bool
|
||||
thread_id: str
|
||||
run_id: str
|
||||
answer: str
|
||||
truncated: bool = False
|
||||
message: str
|
||||
|
||||
|
||||
class BrowserContextTaskPayload(BaseModel):
|
||||
id: str
|
||||
kind: str = "fetch"
|
||||
url: str
|
||||
instruction: str = ""
|
||||
session_id: str = ""
|
||||
action: dict[str, Any] | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
max_content_chars: int
|
||||
|
||||
|
||||
class BrowserContextTaskNextResponse(BaseModel):
|
||||
task: BrowserContextTaskPayload | None = None
|
||||
|
||||
|
||||
class BrowserContextTaskResultRequest(BaseModel):
|
||||
kind: str = Field(default="fetch", description="Task kind the extension processed: fetch or act")
|
||||
title: str | None = Field(default=None, description="Captured page title")
|
||||
url: str | None = Field(default=None, description="Final page URL after redirects")
|
||||
content: str | None = Field(default=None, description="Captured page content")
|
||||
content_type: Literal["markdown", "text"] = Field(default="markdown")
|
||||
observation: dict[str, Any] | None = Field(default=None, description="Structured observation for an act step (elements, text, status)")
|
||||
error: str | None = Field(default=None, description="Extension-side task error")
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class BrowserContextTaskResultResponse(BaseModel):
|
||||
success: bool
|
||||
task_id: str
|
||||
status: str
|
||||
truncated: bool = False
|
||||
|
||||
|
||||
def _settings(config: AppConfig) -> BrowserContextConfig:
|
||||
value = getattr(config, "browser_context", None)
|
||||
if isinstance(value, BrowserContextConfig):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return BrowserContextConfig.model_validate(value)
|
||||
return BrowserContextConfig()
|
||||
|
||||
|
||||
def _require_enabled(config: AppConfig) -> BrowserContextConfig:
|
||||
settings = _settings(config)
|
||||
if not settings.enabled:
|
||||
raise HTTPException(status_code=404, detail="Browser context integration is disabled")
|
||||
return settings
|
||||
|
||||
|
||||
def _trim(value: str | None, limit: int) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
return value.strip()[:limit]
|
||||
|
||||
|
||||
def _limit_utf8_bytes(value: str, max_bytes: int) -> str:
|
||||
text = value
|
||||
while len(text.encode("utf-8")) > max_bytes:
|
||||
text = text[:-1]
|
||||
return text
|
||||
|
||||
|
||||
def _filename_part(value: str, fallback: str, *, max_bytes: int) -> str:
|
||||
text = value.strip() or fallback
|
||||
# Keep unicode word chars so Chinese titles remain readable, while removing
|
||||
# path separators and shell-hostile punctuation.
|
||||
text = re.sub(r"[^\w.-]+", "-", text, flags=re.UNICODE).strip("-._")
|
||||
text = _limit_utf8_bytes(text, max_bytes)
|
||||
return text or fallback
|
||||
|
||||
|
||||
def _build_markdown(
|
||||
*,
|
||||
title: str,
|
||||
url: str,
|
||||
content: str,
|
||||
content_type: str,
|
||||
captured_at: str,
|
||||
truncated: bool,
|
||||
) -> str:
|
||||
lines = [
|
||||
f"# {title or 'Captured Web Page'}",
|
||||
"",
|
||||
f"- Source URL: {url or '(unknown)'}",
|
||||
f"- Captured at: {captured_at}",
|
||||
]
|
||||
if truncated:
|
||||
lines.append("- Note: Content was truncated by the browser_context.max_content_chars limit.")
|
||||
lines.extend(["", "---", ""])
|
||||
|
||||
if content_type == "markdown":
|
||||
lines.append(content.strip())
|
||||
else:
|
||||
lines.append("```text")
|
||||
lines.append(content.strip())
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _build_ask_prompt(
|
||||
*,
|
||||
title: str,
|
||||
url: str,
|
||||
content: str,
|
||||
content_type: str,
|
||||
captured_at: str,
|
||||
truncated: bool,
|
||||
) -> str:
|
||||
note = "页面内容已按配置截断,请只基于可见内容回答。" if truncated else "页面内容如下。"
|
||||
body = content.strip()
|
||||
if content_type == "text":
|
||||
body = f"```text\n{body}\n```"
|
||||
return "\n\n".join(
|
||||
[
|
||||
"你是 zncm 页面问答助手。请直接基于当前网页内容回答用户问题。",
|
||||
"如果网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。",
|
||||
f"页面标题:{title or '未知标题'}",
|
||||
f"页面 URL:{url or '未知 URL'}",
|
||||
f"读取时间:{captured_at}",
|
||||
f"说明:{note}",
|
||||
"当前网页内容:",
|
||||
body,
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _message_type(message: Any) -> str:
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("type") or message.get("role") or "").lower()
|
||||
return str(getattr(message, "type", "") or getattr(message, "role", "") or "").lower()
|
||||
|
||||
|
||||
def _content_to_text(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content.strip()
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
parts.append(item)
|
||||
elif isinstance(item, dict):
|
||||
if isinstance(item.get("text"), str):
|
||||
parts.append(item["text"])
|
||||
elif item.get("type") == "text" and isinstance(item.get("content"), str):
|
||||
parts.append(item["content"])
|
||||
return "\n".join(part.strip() for part in parts if part and part.strip()).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _latest_ai_answer(values: dict[str, Any]) -> str:
|
||||
messages = values.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
return ""
|
||||
for message in reversed(messages):
|
||||
if _message_type(message) not in {"ai", "assistant"}:
|
||||
continue
|
||||
answer = _content_to_text(message.get("content") if isinstance(message, dict) else getattr(message, "content", None))
|
||||
if answer:
|
||||
return answer
|
||||
return ""
|
||||
|
||||
|
||||
async def _sync_to_sandbox_if_needed(thread_id: str, virtual_path: str, file_path, data: bytes) -> None:
|
||||
sandbox_provider = get_sandbox_provider()
|
||||
if bool(getattr(sandbox_provider, "uses_thread_data_mounts", False)):
|
||||
return
|
||||
sandbox_id = sandbox_provider.acquire(thread_id)
|
||||
sandbox = sandbox_provider.get(sandbox_id)
|
||||
if sandbox is None:
|
||||
raise HTTPException(status_code=500, detail="Failed to acquire sandbox")
|
||||
_make_file_sandbox_writable(file_path)
|
||||
sandbox.update_file(virtual_path, data)
|
||||
|
||||
|
||||
@router.get("/status", response_model=BrowserContextStatusResponse)
|
||||
async def get_browser_context_status(
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextStatusResponse:
|
||||
settings = _settings(config)
|
||||
return BrowserContextStatusResponse(
|
||||
enabled=settings.enabled,
|
||||
max_content_chars=settings.max_content_chars,
|
||||
max_title_chars=settings.max_title_chars,
|
||||
max_url_chars=settings.max_url_chars,
|
||||
task_result_timeout_seconds=settings.task_result_timeout_seconds,
|
||||
task_poll_interval_seconds=settings.task_poll_interval_seconds,
|
||||
task_lease_seconds=settings.task_lease_seconds,
|
||||
tool_result_max_chars=settings.tool_result_max_chars,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/tasks/next", response_model=BrowserContextTaskNextResponse)
|
||||
@require_permission("runs", "create")
|
||||
async def claim_browser_context_task(
|
||||
request: Request,
|
||||
worker_id: str | None = None,
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextTaskNextResponse:
|
||||
settings = _require_enabled(config)
|
||||
prune_browser_tasks()
|
||||
task = claim_next_browser_task(
|
||||
user_id=get_effective_user_id(),
|
||||
worker_id=worker_id or "chrome-extension",
|
||||
lease_seconds=settings.task_lease_seconds,
|
||||
)
|
||||
if task is None:
|
||||
return BrowserContextTaskNextResponse(task=None)
|
||||
return BrowserContextTaskNextResponse(task=BrowserContextTaskPayload.model_validate(task))
|
||||
|
||||
|
||||
@router.post("/tasks/{task_id}/result", response_model=BrowserContextTaskResultResponse)
|
||||
@require_permission("runs", "create")
|
||||
async def submit_browser_context_task_result(
|
||||
task_id: str,
|
||||
body: BrowserContextTaskResultRequest,
|
||||
request: Request,
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextTaskResultResponse:
|
||||
settings = _require_enabled(config)
|
||||
error = _trim(body.error, 4000)
|
||||
content = body.content or ""
|
||||
is_act = (body.kind or "fetch") == "act"
|
||||
if not error and not is_act and not content.strip():
|
||||
raise HTTPException(status_code=400, detail="Captured page content is empty")
|
||||
if not error and is_act and not isinstance(body.observation, dict):
|
||||
raise HTTPException(status_code=400, detail="Act task result requires an observation")
|
||||
|
||||
task = complete_browser_task(
|
||||
task_id=task_id,
|
||||
user_id=get_effective_user_id(),
|
||||
title=_trim(body.title, settings.max_title_chars),
|
||||
url=_trim(body.url, settings.max_url_chars),
|
||||
content=content,
|
||||
content_type=body.content_type,
|
||||
error=error,
|
||||
metadata=body.metadata,
|
||||
observation=body.observation,
|
||||
)
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail=f"Browser task {task_id} not found")
|
||||
|
||||
result = task.get("result") or {}
|
||||
return BrowserContextTaskResultResponse(
|
||||
success=True,
|
||||
task_id=task_id,
|
||||
status=str(task.get("status") or "unknown"),
|
||||
truncated=bool(result.get("truncated")),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/capture", response_model=BrowserContextCaptureResponse)
|
||||
@require_permission("threads", "write")
|
||||
async def capture_browser_context(
|
||||
body: BrowserContextCaptureRequest,
|
||||
request: Request,
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextCaptureResponse:
|
||||
settings = _require_enabled(config)
|
||||
|
||||
raw_content = (body.content or "").strip()
|
||||
if not raw_content:
|
||||
raise HTTPException(status_code=400, detail="Captured page content is empty")
|
||||
|
||||
title = _trim(body.title, settings.max_title_chars)
|
||||
url = _trim(body.url, settings.max_url_chars)
|
||||
truncated = len(raw_content) > settings.max_content_chars
|
||||
content = raw_content[: settings.max_content_chars] if truncated else raw_content
|
||||
|
||||
requested_thread_id = (body.thread_id or "").strip() or None
|
||||
if requested_thread_id:
|
||||
thread_store = get_thread_store(request)
|
||||
allowed = await thread_store.check_access(
|
||||
requested_thread_id,
|
||||
get_effective_user_id(),
|
||||
require_existing=False,
|
||||
)
|
||||
if not allowed:
|
||||
raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found")
|
||||
|
||||
thread_response = await create_thread(
|
||||
ThreadCreateRequest(
|
||||
thread_id=requested_thread_id,
|
||||
metadata={
|
||||
"source": "browser_context",
|
||||
"browser_context_url": url,
|
||||
"browser_context_title": title,
|
||||
},
|
||||
),
|
||||
request,
|
||||
)
|
||||
thread_id = thread_response.thread_id
|
||||
|
||||
captured_at = datetime.now(UTC).isoformat()
|
||||
markdown = _build_markdown(
|
||||
title=title,
|
||||
url=url,
|
||||
content=content,
|
||||
content_type=body.content_type,
|
||||
captured_at=captured_at,
|
||||
truncated=truncated,
|
||||
)
|
||||
data = markdown.encode("utf-8")
|
||||
|
||||
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
prefix = _filename_part(settings.filename_prefix, "webpage", max_bytes=40)
|
||||
title_part = _filename_part(title, "page", max_bytes=120)
|
||||
filename = normalize_filename(f"{prefix}-{timestamp}-{title_part}.md")
|
||||
|
||||
try:
|
||||
uploads_dir = ensure_uploads_dir(thread_id)
|
||||
file_path, fh = open_upload_file_no_symlink(uploads_dir, filename)
|
||||
with fh:
|
||||
fh.write(data)
|
||||
except UnsafeUploadPathError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to save browser context capture for thread %s", thread_id)
|
||||
raise HTTPException(status_code=500, detail="Failed to save captured page") from exc
|
||||
|
||||
virtual_path = upload_virtual_path(filename)
|
||||
await _sync_to_sandbox_if_needed(thread_id, virtual_path, file_path, data)
|
||||
|
||||
prompt = (body.prompt or "").strip() or DEFAULT_BROWSER_CONTEXT_PROMPT
|
||||
token = uuid4().hex
|
||||
route_base = settings.route_base.rstrip("/") or "/page/workspace"
|
||||
chat_path = f"{route_base}/chats/{thread_id}"
|
||||
query = urlencode(
|
||||
{
|
||||
"browser_context": token,
|
||||
"bc_filename": filename,
|
||||
"bc_size": str(len(data)),
|
||||
"bc_path": virtual_path,
|
||||
"bc_prompt": prompt,
|
||||
}
|
||||
)
|
||||
chat_url = f"#{chat_path}?{query}"
|
||||
|
||||
return BrowserContextCaptureResponse(
|
||||
success=True,
|
||||
thread_id=thread_id,
|
||||
file=BrowserContextFileInfo(
|
||||
filename=filename,
|
||||
size=len(data),
|
||||
path=virtual_path,
|
||||
virtual_path=virtual_path,
|
||||
artifact_url=upload_artifact_url(thread_id, filename),
|
||||
),
|
||||
chat_path=chat_path,
|
||||
chat_url=chat_url,
|
||||
browser_context_token=token,
|
||||
prompt=prompt,
|
||||
truncated=truncated,
|
||||
message="Captured page saved as a zncm upload",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/ask", response_model=BrowserContextAskResponse)
|
||||
@require_permission("runs", "create")
|
||||
async def ask_browser_context(
|
||||
body: BrowserContextAskRequest,
|
||||
request: Request,
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextAskResponse:
|
||||
settings = _require_enabled(config)
|
||||
|
||||
raw_content = (body.content or "").strip()
|
||||
if not raw_content:
|
||||
raise HTTPException(status_code=400, detail="Captured page content is empty")
|
||||
|
||||
question = (body.question or "").strip()
|
||||
if not question:
|
||||
raise HTTPException(status_code=400, detail="Question is empty")
|
||||
|
||||
title = _trim(body.title, settings.max_title_chars)
|
||||
url = _trim(body.url, settings.max_url_chars)
|
||||
content_limit = max(1_000, min(settings.max_content_chars, settings.tool_result_max_chars))
|
||||
truncated = len(raw_content) > content_limit
|
||||
content = raw_content[:content_limit] if truncated else raw_content
|
||||
|
||||
requested_thread_id = (body.thread_id or "").strip() or None
|
||||
if requested_thread_id:
|
||||
thread_store = get_thread_store(request)
|
||||
allowed = await thread_store.check_access(
|
||||
requested_thread_id,
|
||||
get_effective_user_id(),
|
||||
require_existing=False,
|
||||
)
|
||||
if not allowed:
|
||||
raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found")
|
||||
|
||||
thread_response = await create_thread(
|
||||
ThreadCreateRequest(
|
||||
thread_id=requested_thread_id,
|
||||
assistant_id="lead_agent",
|
||||
metadata={
|
||||
"source": "browser_context_page_assistant",
|
||||
"browser_context_url": url,
|
||||
"browser_context_title": title,
|
||||
},
|
||||
),
|
||||
request,
|
||||
)
|
||||
thread_id = thread_response.thread_id
|
||||
captured_at = datetime.now(UTC).isoformat()
|
||||
is_follow_up = requested_thread_id is not None
|
||||
if is_follow_up:
|
||||
prompt_prefix = "\n\n".join(
|
||||
[
|
||||
"你是 zncm 页面问答助手。请继续基于本会话里已经读取的网页内容回答用户问题。",
|
||||
"如果本会话里的网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。",
|
||||
f"当前页面标题:{title or '未知标题'}",
|
||||
f"当前页面 URL:{url or '未知 URL'}",
|
||||
f"本轮提问时间:{captured_at}",
|
||||
]
|
||||
)
|
||||
else:
|
||||
prompt_prefix = _build_ask_prompt(
|
||||
title=title,
|
||||
url=url,
|
||||
content=content,
|
||||
content_type=body.content_type,
|
||||
captured_at=captured_at,
|
||||
truncated=truncated,
|
||||
)
|
||||
|
||||
run_body = RunCreateRequest(
|
||||
assistant_id="lead_agent",
|
||||
input={
|
||||
"messages": [
|
||||
{
|
||||
"type": "human",
|
||||
"content": question,
|
||||
"additional_kwargs": {
|
||||
"prompt_prefix": prompt_prefix,
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
metadata={
|
||||
"source": "browser_context_page_assistant",
|
||||
"browser_context_url": url,
|
||||
"browser_context_title": title,
|
||||
"browser_context_truncated": truncated,
|
||||
},
|
||||
stream_mode=["values"],
|
||||
on_disconnect="continue",
|
||||
multitask_strategy="reject",
|
||||
)
|
||||
|
||||
record = await start_run(run_body, thread_id, request)
|
||||
if record.task is not None:
|
||||
try:
|
||||
await record.task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
checkpointer = get_checkpointer(request)
|
||||
values: dict[str, Any] = {}
|
||||
try:
|
||||
checkpoint_tuple = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
|
||||
if checkpoint_tuple is not None:
|
||||
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
|
||||
values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
|
||||
except Exception:
|
||||
logger.exception("Failed to fetch browser context answer for run %s", record.run_id)
|
||||
|
||||
answer = _latest_ai_answer(values)
|
||||
if not answer:
|
||||
answer = record.error or "zncm did not return an answer."
|
||||
|
||||
return BrowserContextAskResponse(
|
||||
success=bool(answer),
|
||||
thread_id=thread_id,
|
||||
run_id=record.run_id,
|
||||
answer=answer,
|
||||
truncated=truncated,
|
||||
message="zncm page assistant answer completed",
|
||||
)
|
||||
@@ -328,9 +328,15 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
|
||||
# config.yaml 的 models[0],与 /api/ai-writing 等其它路由一致,内网部署只配
|
||||
# 自己的本地模型即可工作。
|
||||
model_name = str(payload.get("model") or "")
|
||||
# 用户在 Step 1 上传的参考文件(已经过 /api/threads/{thread_id}/uploads)。
|
||||
# 前端传 [{filename, size, path, status}, ...],这里原样透传给 _run_payload →
|
||||
# additional_kwargs.files → UploadsMiddleware 自动注入清单 + 解锁 read_file 等工具。
|
||||
raw_files = payload.get("files")
|
||||
files = raw_files if isinstance(raw_files, list) else []
|
||||
if not thread_id:
|
||||
raise HTTPException(status_code=400, detail="thread_id is required")
|
||||
if not message:
|
||||
# 允许「仅文件、无文本」的回合:带了文件就不强制要求 message。
|
||||
if not message and not files:
|
||||
raise HTTPException(status_code=400, detail="message is required")
|
||||
|
||||
# 调用前自检:防御性二次检查——若管理员在 init 之后手动删了目录,这里也能自愈。
|
||||
@@ -381,8 +387,17 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
|
||||
_MAX_CLARIFICATION_ROUNDS,
|
||||
)
|
||||
|
||||
# 带了上传文件时,解锁「读」类工具(read_file / ls / grep / view_image),
|
||||
# 否则 UploadsMiddleware 注入了文件清单、模型却没工具读取。仍保留对
|
||||
# write_file / str_replace / present_files / bash / agent_orchestration 的封锁,
|
||||
# 防止意图阶段越界做方案/派活。
|
||||
if files:
|
||||
read_tools = {"read_file", "ls", "grep", "view_image"}
|
||||
base_excluded_tools = [t for t in base_excluded_tools if t not in read_tools]
|
||||
|
||||
# 2. message 前缀——给模型一个明确的"本轮必须收敛"信号。
|
||||
effective_message = message
|
||||
# 仅文件、无文本时给一句兜底引导,让模型先读文件再澄清意图。
|
||||
effective_message = message or "我上传了文件,请先阅读文件内容,再据此理解并澄清我的任务意图。"
|
||||
if past_clarifications >= _MAX_CLARIFICATION_ROUNDS:
|
||||
effective_message = (
|
||||
f"[系统强制] 已澄清 {past_clarifications} 轮,触发硬上限。"
|
||||
@@ -409,6 +424,7 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
|
||||
# not reliable on its own, hard exclusion is.
|
||||
excluded_tools=base_excluded_tools,
|
||||
skill_stop_names=[],
|
||||
files=files,
|
||||
)
|
||||
|
||||
# captured:上游 SSE 全部 data 行,用于流结束后从 values 快照取 messages。
|
||||
|
||||
@@ -0,0 +1,521 @@
|
||||
"""Knowledge base API (``/api/knowledge``).
|
||||
|
||||
Sediments threads / searches into an obsidian-wiki compatible vault, mirrors
|
||||
metadata into the database, and serves listing, detail, edit, delete, search
|
||||
and graph export. The knowledge base is **global / shared** — no per-user
|
||||
isolation; writes record ``created_by`` / ``updated_by`` for audit.
|
||||
|
||||
All obsidian-wiki specifics live in :mod:`deerflow.knowledge`; this router only
|
||||
adapts HTTP ↔ the :class:`~deerflow.knowledge.service.KnowledgeService`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import mimetypes
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.authz import require_auth, require_permission
|
||||
from app.gateway.deps import get_checkpointer, get_knowledge_service, get_local_provider, get_thread_store
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.knowledge.service import EmptyThreadError
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.references import build_reference_batches
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/knowledge", tags=["knowledge"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Request / response models
|
||||
# --------------------------------------------------------------------------- #
|
||||
class IngestThreadRequest(BaseModel):
|
||||
mode: str = Field(default="summary", description="summary | full")
|
||||
title: str | None = None
|
||||
status: str = Field(default="approved", description="draft | approved")
|
||||
tags: list[str] = Field(default_factory=list)
|
||||
include_sources: bool = True
|
||||
background: bool = Field(
|
||||
default=False,
|
||||
description="When true, sediment the thread asynchronously in a background task and return immediately.",
|
||||
)
|
||||
model_name: str | None = Field(default=None, description="可选:指定用于知识提炼的大模型名称;留空使用默认配置")
|
||||
|
||||
|
||||
class BatchIngestRequest(BaseModel):
|
||||
limit: int = Field(default=100, ge=1, le=500)
|
||||
offset: int = Field(default=0, ge=0)
|
||||
status: str = "approved"
|
||||
include_sources: bool = True
|
||||
skip_existing: bool = True
|
||||
mode: str = "summary"
|
||||
model_name: str | None = Field(default=None, description="可选:指定用于知识提炼的大模型名称;留空使用默认配置")
|
||||
|
||||
|
||||
class SearchResultItem(BaseModel):
|
||||
title: str | None = None
|
||||
url: str | None = None
|
||||
snippet: str | None = None
|
||||
tool_name: str | None = None
|
||||
score: float | None = None
|
||||
|
||||
|
||||
class IngestSearchRequest(BaseModel):
|
||||
query: str
|
||||
results: list[SearchResultItem] = Field(default_factory=list)
|
||||
thread_id: str | None = None
|
||||
message_id: str | None = None
|
||||
title: str | None = None
|
||||
tags: list[str] = Field(default_factory=list)
|
||||
status: str = "approved"
|
||||
|
||||
|
||||
class CreateNoteRequest(BaseModel):
|
||||
title: str
|
||||
content_md: str
|
||||
summary: str | None = None
|
||||
tags: list[str] = Field(default_factory=list)
|
||||
status: str = "approved"
|
||||
|
||||
|
||||
class UpdateNoteRequest(BaseModel):
|
||||
title: str | None = None
|
||||
content_md: str | None = None
|
||||
summary: str | None = None
|
||||
tags: list[str] | None = None
|
||||
status: str | None = None
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
query: str
|
||||
mode: str = Field(default="keyword", description="keyword | vector | hybrid (vector/hybrid fall back to keyword in v1)")
|
||||
limit: int = Field(default=8, ge=1, le=50)
|
||||
|
||||
|
||||
class ResolveWikilinksRequest(BaseModel):
|
||||
targets: list[str] = Field(default_factory=list, description="`[[wikilink]]` targets to resolve to note ids / vault pages.")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Helpers
|
||||
# --------------------------------------------------------------------------- #
|
||||
async def _read_thread_values(checkpointer, thread_id: str) -> dict[str, Any]:
|
||||
"""Return serialized channel values for a thread, or raise 404/500."""
|
||||
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
try:
|
||||
checkpoint_tuple = await checkpointer.aget_tuple(config)
|
||||
except Exception:
|
||||
logger.exception("Failed to read thread %s for knowledge ingest", sanitize_log_param(thread_id))
|
||||
raise HTTPException(status_code=500, detail="Failed to read thread state")
|
||||
if checkpoint_tuple is None:
|
||||
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
|
||||
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
|
||||
return serialize_channel_values(checkpoint.get("channel_values", {}) or {})
|
||||
|
||||
|
||||
async def _enrich_actor_names(note: dict[str, Any]) -> None:
|
||||
"""Resolve created_by/updated_by user ids to a human label (email).
|
||||
|
||||
Adds ``created_by_name`` / ``updated_by_name`` to the note dict so the UI can
|
||||
show a person instead of a raw UUID. Best-effort: unresolved ids fall back to
|
||||
``None`` and the frontend then shows a shortened id.
|
||||
"""
|
||||
provider = get_local_provider()
|
||||
cache: dict[str, str | None] = {}
|
||||
|
||||
async def name_of(uid: str | None) -> str | None:
|
||||
if not uid:
|
||||
return None
|
||||
if uid not in cache:
|
||||
try:
|
||||
user = await provider.get_user(uid)
|
||||
cache[uid] = user.email if user else None
|
||||
except Exception:
|
||||
cache[uid] = None
|
||||
return cache[uid]
|
||||
|
||||
note["created_by_name"] = await name_of(note.get("created_by"))
|
||||
note["updated_by_name"] = await name_of(note.get("updated_by"))
|
||||
|
||||
|
||||
def _messages_and_meta(values: dict[str, Any]) -> tuple[list[dict[str, Any]], str | None, list[dict[str, Any]]]:
|
||||
messages = values.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
messages = []
|
||||
title = values.get("title") if isinstance(values.get("title"), str) else None
|
||||
batches = build_reference_batches(messages) if messages else []
|
||||
return messages, title, batches
|
||||
|
||||
|
||||
async def _ingest_thread_background(service, checkpointer, thread_id: str, body: IngestThreadRequest, created_by: str | None) -> None:
|
||||
"""Sediment a thread into a note without an HTTP context.
|
||||
|
||||
Runs after the response is sent (FastAPI ``BackgroundTasks``); ``created_by``
|
||||
must be captured by the request handler since the user contextvar is gone
|
||||
by the time this executes. All failures are swallowed (logged) — the user
|
||||
has already been told the save was queued.
|
||||
"""
|
||||
try:
|
||||
values = await _read_thread_values(checkpointer, thread_id)
|
||||
messages, title, batches = _messages_and_meta(values)
|
||||
if not messages:
|
||||
logger.warning("Background knowledge ingest skipped: thread %s has no messages", sanitize_log_param(thread_id))
|
||||
return
|
||||
await service.capture_thread(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
thread_title=title,
|
||||
mode=body.mode,
|
||||
title=body.title,
|
||||
status=body.status,
|
||||
tags=body.tags,
|
||||
include_sources=body.include_sources,
|
||||
reference_batches=batches,
|
||||
created_by=created_by,
|
||||
model_name=body.model_name,
|
||||
)
|
||||
except EmptyThreadError:
|
||||
logger.warning("Background knowledge ingest skipped: thread %s has no sediment-able content", sanitize_log_param(thread_id))
|
||||
except Exception:
|
||||
logger.exception("Background knowledge ingest failed for thread %s", sanitize_log_param(thread_id))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Ingest
|
||||
# --------------------------------------------------------------------------- #
|
||||
@router.post("/ingest/thread/{thread_id}")
|
||||
@require_auth
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def ingest_thread(thread_id: str, body: IngestThreadRequest, request: Request, background_tasks: BackgroundTasks) -> dict[str, Any]:
|
||||
"""Sediment a single thread into a knowledge note.
|
||||
|
||||
With ``background=true`` the heavy capture (read checkpoint → extract →
|
||||
write vault + DB) is deferred to a background task and the endpoint returns
|
||||
``{"queued": true}`` immediately, so the user's "保存" click never blocks.
|
||||
"""
|
||||
service = get_knowledge_service(request)
|
||||
checkpointer = get_checkpointer(request)
|
||||
if body.background:
|
||||
# Capture the user id now — the contextvar is gone once the task runs.
|
||||
background_tasks.add_task(
|
||||
_ingest_thread_background,
|
||||
service,
|
||||
checkpointer,
|
||||
thread_id,
|
||||
body,
|
||||
get_effective_user_id(),
|
||||
)
|
||||
return {"queued": True}
|
||||
values = await _read_thread_values(checkpointer, thread_id)
|
||||
messages, title, batches = _messages_and_meta(values)
|
||||
if not messages:
|
||||
raise HTTPException(status_code=400, detail="Thread has no messages to sediment")
|
||||
try:
|
||||
result = await service.capture_thread(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
thread_title=title,
|
||||
mode=body.mode,
|
||||
title=body.title,
|
||||
status=body.status,
|
||||
tags=body.tags,
|
||||
include_sources=body.include_sources,
|
||||
reference_batches=batches,
|
||||
created_by=get_effective_user_id(),
|
||||
model_name=body.model_name,
|
||||
)
|
||||
except EmptyThreadError:
|
||||
raise HTTPException(status_code=400, detail="Thread has no sediment-able content")
|
||||
return {
|
||||
"note": result["note"],
|
||||
"created": result["created"],
|
||||
"notes": result.get("notes", [result["note"]]),
|
||||
"llm_fallback": result.get("llm_fallback", False),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/ingest/threads/batch")
|
||||
@require_auth
|
||||
async def ingest_threads_batch(body: BatchIngestRequest, request: Request) -> dict[str, Any]:
|
||||
"""Batch-sediment the caller's recent threads into knowledge notes."""
|
||||
service = get_knowledge_service(request)
|
||||
checkpointer = get_checkpointer(request)
|
||||
thread_store = get_thread_store(request)
|
||||
|
||||
threads = await thread_store.search(limit=body.limit, offset=body.offset)
|
||||
created = skipped = failed = 0
|
||||
# Notes that landed via the rule-based fallback because the chosen model
|
||||
# produced nothing usable (timeout / API error / non-JSON). Surfaced so the
|
||||
# caller can tell the user "X 条因大模型异常改用了规则抽取".
|
||||
llm_fallback = 0
|
||||
items: list[dict[str, Any]] = []
|
||||
for meta in threads:
|
||||
thread_id = meta.get("thread_id")
|
||||
if not thread_id:
|
||||
continue
|
||||
try:
|
||||
values = await _read_thread_values(checkpointer, thread_id)
|
||||
messages, title, batches = _messages_and_meta(values)
|
||||
if not messages:
|
||||
skipped += 1
|
||||
items.append({"thread_id": thread_id, "note_id": None, "status": "skipped"})
|
||||
continue
|
||||
result = await service.capture_thread(
|
||||
thread_id=thread_id,
|
||||
messages=messages,
|
||||
thread_title=title or meta.get("display_name"),
|
||||
mode=body.mode,
|
||||
status=body.status,
|
||||
include_sources=body.include_sources,
|
||||
reference_batches=batches,
|
||||
created_by=get_effective_user_id(),
|
||||
skip_existing=body.skip_existing,
|
||||
model_name=body.model_name,
|
||||
)
|
||||
if result.get("llm_fallback"):
|
||||
llm_fallback += 1
|
||||
if result["created"]:
|
||||
created += 1
|
||||
items.append({"thread_id": thread_id, "note_id": result["note"]["id"], "status": "created", "llm_fallback": bool(result.get("llm_fallback"))})
|
||||
else:
|
||||
skipped += 1
|
||||
items.append({"thread_id": thread_id, "note_id": result["note"]["id"], "status": "skipped"})
|
||||
except EmptyThreadError:
|
||||
skipped += 1
|
||||
items.append({"thread_id": thread_id, "note_id": None, "status": "skipped"})
|
||||
except HTTPException:
|
||||
failed += 1
|
||||
items.append({"thread_id": thread_id, "note_id": None, "status": "failed"})
|
||||
except Exception:
|
||||
logger.exception("Batch ingest failed for thread %s", sanitize_log_param(str(thread_id)))
|
||||
failed += 1
|
||||
items.append({"thread_id": thread_id, "note_id": None, "status": "failed"})
|
||||
|
||||
return {"ok": True, "processed": len(items), "created": created, "skipped": skipped, "failed": failed, "llm_fallback": llm_fallback, "items": items}
|
||||
|
||||
|
||||
@router.post("/ingest/search")
|
||||
@require_auth
|
||||
async def ingest_search(body: IngestSearchRequest, request: Request) -> dict[str, Any]:
|
||||
"""Sediment search/tool results as a reference note."""
|
||||
service = get_knowledge_service(request)
|
||||
note = await service.capture_search(
|
||||
query=body.query,
|
||||
results=[r.model_dump() for r in body.results],
|
||||
thread_id=body.thread_id,
|
||||
message_id=body.message_id,
|
||||
title=body.title,
|
||||
tags=body.tags,
|
||||
status=body.status,
|
||||
created_by=get_effective_user_id(),
|
||||
)
|
||||
return {"note": note}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# CRUD
|
||||
# --------------------------------------------------------------------------- #
|
||||
@router.post("/notes")
|
||||
@require_auth
|
||||
async def create_note(body: CreateNoteRequest, request: Request) -> dict[str, Any]:
|
||||
service = get_knowledge_service(request)
|
||||
note = await service.create_manual_note(
|
||||
title=body.title,
|
||||
content_md=body.content_md,
|
||||
summary=body.summary,
|
||||
tags=body.tags,
|
||||
status=body.status,
|
||||
created_by=get_effective_user_id(),
|
||||
)
|
||||
return {"note": note}
|
||||
|
||||
|
||||
@router.get("/notes")
|
||||
@require_auth
|
||||
async def list_notes(
|
||||
request: Request,
|
||||
keyword: str | None = None,
|
||||
tag: str | None = None,
|
||||
source_type: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
service = get_knowledge_service(request)
|
||||
limit = max(1, min(limit, 100))
|
||||
offset = max(0, offset)
|
||||
return await service.list_notes(keyword=keyword, tag=tag, source_type=source_type, status=status, limit=limit, offset=offset)
|
||||
|
||||
|
||||
@router.get("/notes/{note_id}")
|
||||
@require_auth
|
||||
async def get_note(note_id: str, request: Request) -> dict[str, Any]:
|
||||
service = get_knowledge_service(request)
|
||||
note = await service.get_note(note_id)
|
||||
if note is None:
|
||||
raise HTTPException(status_code=404, detail="Knowledge note not found")
|
||||
await _enrich_actor_names(note)
|
||||
return note
|
||||
|
||||
|
||||
@router.put("/notes/{note_id}")
|
||||
@require_auth
|
||||
async def update_note(note_id: str, body: UpdateNoteRequest, request: Request) -> dict[str, Any]:
|
||||
service = get_knowledge_service(request)
|
||||
note = await service.update_note(
|
||||
note_id,
|
||||
title=body.title,
|
||||
content_md=body.content_md,
|
||||
summary=body.summary,
|
||||
tags=body.tags,
|
||||
status=body.status,
|
||||
updated_by=get_effective_user_id(),
|
||||
)
|
||||
if note is None:
|
||||
raise HTTPException(status_code=404, detail="Knowledge note not found")
|
||||
return note
|
||||
|
||||
|
||||
@router.delete("/notes/{note_id}")
|
||||
@require_auth
|
||||
async def delete_note(note_id: str, request: Request, hard: bool = False) -> dict[str, Any]:
|
||||
service = get_knowledge_service(request)
|
||||
ok = await service.delete_note(note_id, updated_by=get_effective_user_id(), hard=hard)
|
||||
if not ok:
|
||||
raise HTTPException(status_code=404, detail="Knowledge note not found")
|
||||
return {"ok": True, "note_id": note_id}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Search & graph
|
||||
# --------------------------------------------------------------------------- #
|
||||
@router.post("/search")
|
||||
@require_auth
|
||||
async def search(body: SearchRequest, request: Request) -> dict[str, Any]:
|
||||
service = get_knowledge_service(request)
|
||||
items = await service.search(body.query, mode=body.mode, limit=body.limit)
|
||||
return {"items": items}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Wikilinks ([[...]] resolution + vault meta-page viewer)
|
||||
# --------------------------------------------------------------------------- #
|
||||
@router.post("/resolve")
|
||||
@require_auth
|
||||
async def resolve_wikilinks(body: ResolveWikilinksRequest, request: Request) -> dict[str, Any]:
|
||||
"""Resolve ``[[wikilink]]`` targets to navigable destinations.
|
||||
|
||||
Each target resolves to a knowledge note (``kind=note``), a vault meta/hub
|
||||
page (``kind=vault``), or nothing (``kind=missing``). Powers clickable
|
||||
wikilinks in the knowledge UI.
|
||||
"""
|
||||
service = get_knowledge_service(request)
|
||||
return {"resolved": await service.resolve_wikilinks(body.targets)}
|
||||
|
||||
|
||||
@router.get("/notes/{note_id}/wikilinks")
|
||||
@require_auth
|
||||
async def resolve_note_wikilinks(note_id: str, request: Request) -> dict[str, Any]:
|
||||
"""Resolve every ``[[wikilink]]`` found in a note's body in one call."""
|
||||
service = get_knowledge_service(request)
|
||||
resolved = await service.resolve_note_wikilinks(note_id)
|
||||
if resolved is None:
|
||||
raise HTTPException(status_code=404, detail="Knowledge note not found")
|
||||
return {"resolved": resolved}
|
||||
|
||||
|
||||
@router.get("/vault-page")
|
||||
@require_auth
|
||||
async def get_vault_page(request: Request, path: str) -> dict[str, Any]:
|
||||
"""Return a raw vault Markdown meta/hub page (no DB note) by its path."""
|
||||
service = get_knowledge_service(request)
|
||||
page = service.read_vault_page(path)
|
||||
if page is None:
|
||||
raise HTTPException(status_code=404, detail="Vault page not found")
|
||||
return page
|
||||
|
||||
|
||||
class MergeRequest(BaseModel):
|
||||
primary_id: str
|
||||
duplicate_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
@router.get("/duplicates")
|
||||
@require_auth
|
||||
async def list_duplicates(request: Request) -> dict[str, Any]:
|
||||
"""Return suggested duplicate groups (phase 3, heuristic)."""
|
||||
service = get_knowledge_service(request)
|
||||
return {"groups": await service.find_duplicates()}
|
||||
|
||||
|
||||
@router.post("/merge")
|
||||
@require_auth
|
||||
async def merge_notes(body: MergeRequest, request: Request) -> dict[str, Any]:
|
||||
"""Merge duplicate notes into a primary note (sources moved, dups archived)."""
|
||||
service = get_knowledge_service(request)
|
||||
note = await service.merge_notes(body.primary_id, body.duplicate_ids, updated_by=get_effective_user_id())
|
||||
if note is None:
|
||||
raise HTTPException(status_code=404, detail="Primary note not found")
|
||||
return {"note": note}
|
||||
|
||||
|
||||
@router.post("/reindex")
|
||||
@require_auth
|
||||
async def reindex_embeddings(request: Request) -> dict[str, Any]:
|
||||
"""Rebuild the vector index for all notes (maintenance; no-op if embeddings off)."""
|
||||
service = get_knowledge_service(request)
|
||||
return await service.reindex_all()
|
||||
|
||||
|
||||
@router.get("/notes/{note_id}/versions")
|
||||
@require_auth
|
||||
async def list_note_versions(note_id: str, request: Request) -> dict[str, Any]:
|
||||
"""Return the edit history (pre-update snapshots) for a note."""
|
||||
service = get_knowledge_service(request)
|
||||
return {"items": await service.list_versions(note_id)}
|
||||
|
||||
|
||||
@router.get("/graph")
|
||||
@require_auth
|
||||
async def get_graph(request: Request) -> dict[str, Any]:
|
||||
"""Return product knowledge-graph data (notes + entities + relations)."""
|
||||
service = get_knowledge_service(request)
|
||||
return await service.get_graph()
|
||||
|
||||
|
||||
@router.post("/export/graph")
|
||||
@require_auth
|
||||
async def export_graph(request: Request) -> dict[str, Any]:
|
||||
service = get_knowledge_service(request)
|
||||
result = await service.export_graph()
|
||||
base = "/api/knowledge/export/files"
|
||||
return {
|
||||
"ok": True,
|
||||
"files": {
|
||||
"graph_json": f"{base}/graph.json",
|
||||
"graph_html": f"{base}/graph.html",
|
||||
"graph_graphml": f"{base}/graph.graphml",
|
||||
"cypher": f"{base}/cypher.txt",
|
||||
},
|
||||
"stats": result["stats"],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/export/files/{filename}")
|
||||
@require_auth
|
||||
async def export_file(filename: str, request: Request) -> FileResponse:
|
||||
service = get_knowledge_service(request)
|
||||
path = service.export_file_path(filename)
|
||||
if path is None:
|
||||
# Lazily build the export on first access so the iframe never 404s.
|
||||
await service.export_graph()
|
||||
path = service.export_file_path(filename)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="Export file not found")
|
||||
media_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
|
||||
return FileResponse(path, media_type=media_type)
|
||||
@@ -638,6 +638,7 @@ def _run_payload(
|
||||
reasoning_effort: str = "medium",
|
||||
multitask_strategy: str = "reject",
|
||||
extra_context: dict[str, Any] | None = None,
|
||||
files: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""构造 ``POST /api/threads/{thread_id}/runs/stream`` 的请求体。
|
||||
|
||||
@@ -650,13 +651,19 @@ def _run_payload(
|
||||
应传 ``thinking_enabled=False`` + ``reasoning_effort="low"``,省掉模型先吐
|
||||
reasoning tokens 才开口的时间,显著降低前端"调度中"首字节延迟。
|
||||
"""
|
||||
# 用户上传的文件元数据透传给 UploadsMiddleware(与普通聊天同链路):
|
||||
# before_agent 钩子读 additional_kwargs.files,生成 <uploaded_files> 清单注入消息,
|
||||
# 之后席位/总控用内置 read_file / grep / view_image 工具按需读取。空时落空 dict,行为不变。
|
||||
additional_kwargs: dict[str, Any] = {}
|
||||
if files:
|
||||
additional_kwargs["files"] = files
|
||||
payload: dict[str, Any] = {
|
||||
"input": {
|
||||
"messages": [
|
||||
{
|
||||
"type": "human",
|
||||
"content": [{"type": "text", "text": new_message}],
|
||||
"additional_kwargs": {},
|
||||
"additional_kwargs": additional_kwargs,
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1258,6 +1265,7 @@ async def _leader_run(
|
||||
seat_meta: dict[str, dict[str, Any]] | None = None,
|
||||
synthesis_mode: bool = False,
|
||||
suppress_pre_broadcast: bool = False,
|
||||
files: list[dict[str, Any]] | None = None,
|
||||
) -> AsyncIterator[str | dict[str, Any]]:
|
||||
"""总控(leader)单轮:透传上游 SSE + 末尾 yield 一个 dict 状态帧(由路由层 ``json.dumps``)。
|
||||
|
||||
@@ -1346,18 +1354,26 @@ async def _leader_run(
|
||||
# 调用方可再叠加自己的 skill_stop。后台进程内网关用同一份 policy,保证传参一致。
|
||||
policy = roundtable_run_policy("leader")
|
||||
effective_stop_names = merge_skill_stop_names(policy.skill_stop_names, skill_stop_names)
|
||||
# 带了上传文件时,给总控解锁「读」类工具(leader policy 默认禁 read_file/ls/view_image),
|
||||
# 否则文件清单注入了却无工具读取。写/派活类仍按 policy 封锁,不放开。
|
||||
leader_excluded = policy.excluded_tools
|
||||
if files:
|
||||
_read_tools = {"read_file", "ls", "grep", "view_image"}
|
||||
leader_excluded = [t for t in leader_excluded if t not in _read_tools]
|
||||
body = _run_payload(
|
||||
agent_name=agent_name,
|
||||
model_name=model_name,
|
||||
thread_id=thread_id,
|
||||
# 注入了席位清单的输入(leader_input);seat_roster 为空时等同 new_message。
|
||||
new_message=leader_input,
|
||||
excluded_tools=policy.excluded_tools,
|
||||
excluded_tools=leader_excluded,
|
||||
skill_stop_names=effective_stop_names,
|
||||
thinking_enabled=policy.thinking_enabled,
|
||||
reasoning_effort=policy.reasoning_effort,
|
||||
# 人工干预/恢复会重发 leader run 到同一总控 thread;抢占掉没结束的旧 run,避免 409。
|
||||
multitask_strategy="rollback",
|
||||
# 用户随干预上传的参考文件 → 总控可用 read_file 等工具读取后再派活。
|
||||
files=files,
|
||||
)
|
||||
t_payload_built = time.perf_counter()
|
||||
logger.info(
|
||||
@@ -1532,6 +1548,7 @@ async def _special_run(
|
||||
display_name: str = "",
|
||||
no_file: bool = False,
|
||||
peer_deliveries: list[dict[str, Any]] | None = None,
|
||||
files: list[dict[str, Any]] | None = None,
|
||||
) -> AsyncIterator[str | dict[str, Any]]:
|
||||
"""子智能体(special)单轮:执行总控下发的 task 文本,交付 content。
|
||||
|
||||
@@ -1560,12 +1577,18 @@ async def _special_run(
|
||||
else:
|
||||
role = "seat"
|
||||
policy = roundtable_run_policy(role)
|
||||
# 带了上传文件时,给席位解锁「读」类工具(view_image 默认被禁;read_file 本就可用),
|
||||
# 让席位能读取/识别用户上传的文件。写/派活类仍按 policy 封锁。
|
||||
seat_excluded = policy.excluded_tools
|
||||
if files:
|
||||
_read_tools = {"read_file", "ls", "grep", "view_image"}
|
||||
seat_excluded = [t for t in seat_excluded if t not in _read_tools]
|
||||
body = _run_payload(
|
||||
agent_name=agent_name,
|
||||
model_name=model_name,
|
||||
thread_id=thread_id,
|
||||
new_message=new_message,
|
||||
excluded_tools=policy.excluded_tools,
|
||||
excluded_tools=seat_excluded,
|
||||
skill_stop_names=merge_skill_stop_names(policy.skill_stop_names, skill_stop_names),
|
||||
thinking_enabled=policy.thinking_enabled,
|
||||
reasoning_effort=policy.reasoning_effort,
|
||||
@@ -1575,6 +1598,8 @@ async def _special_run(
|
||||
extra_context=(
|
||||
{"roundtable_peer_deliveries": peer_deliveries} if peer_deliveries else None
|
||||
),
|
||||
# 用户随干预上传的参考文件 → 席位可用 read_file 等工具读取。
|
||||
files=files,
|
||||
)
|
||||
t_payload_built = time.perf_counter()
|
||||
logger.info(
|
||||
@@ -1677,6 +1702,11 @@ async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) ->
|
||||
display_name: str = str(payload.get("display_name") or "").strip()
|
||||
new_message: str = str(payload.get("new_message") or "")
|
||||
skill_stop_names: list[str] = payload.get("skill_stop_names") or []
|
||||
# 用户随干预消息上传的参考文件(已经过 /api/threads/{thread_id}/uploads)。
|
||||
# 前端传 [{filename, size, path, status}, ...],透传给 _run_payload →
|
||||
# additional_kwargs.files → UploadsMiddleware 注入清单 + 解锁 read_file 等工具。
|
||||
_files_raw = payload.get("files")
|
||||
files: list[dict[str, Any]] = _files_raw if isinstance(_files_raw, list) else []
|
||||
# 综合轮标记(仅 leader 用):前端在「派活后让总控判断继续/综合」的轮次置真,网关据此
|
||||
# 额外把各已交付席位的完整交付正文注入总控输入,供其写最终方案。缺省 False(派活决策轮
|
||||
# 只给摘要,保持上下文精简)。旧前端不传 → False,行为安全退化。
|
||||
@@ -1741,6 +1771,7 @@ async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) ->
|
||||
seat_meta=seat_meta,
|
||||
synthesis_mode=synthesis_mode,
|
||||
suppress_pre_broadcast=suppress_pre_broadcast,
|
||||
files=files,
|
||||
)
|
||||
else:
|
||||
gen = _special_run(
|
||||
@@ -1748,6 +1779,7 @@ async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) ->
|
||||
display_name=display_name,
|
||||
no_file=no_file,
|
||||
peer_deliveries=peer_deliveries,
|
||||
files=files,
|
||||
)
|
||||
async for chunk in gen:
|
||||
if isinstance(chunk, dict):
|
||||
|
||||
@@ -70,6 +70,14 @@ class TaskUpdateRequest(BaseModel):
|
||||
messages: list[dict[str, Any]] | None = None
|
||||
|
||||
|
||||
class TaskEnsureRequest(BaseModel):
|
||||
"""get-or-create:宿主传的 taskId 不固定,新 id 直接**初始化**一条空记录返回(不 404)。"""
|
||||
id: str = Field(min_length=1, max_length=64)
|
||||
title: str = Field(default="", max_length=512)
|
||||
brief: str | None = None
|
||||
agents: list[TaskAgentSchema] = []
|
||||
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -115,6 +123,30 @@ async def create_task(request: Request, body: TaskCreateRequest) -> TaskResponse
|
||||
return TaskResponse(**row)
|
||||
|
||||
|
||||
@router.post("/ensure", response_model=TaskResponse)
|
||||
async def ensure_task(request: Request, body: TaskEnsureRequest) -> TaskResponse:
|
||||
"""存在则返回(含 messages 供回显);不存在则用该 id **初始化**一条空记录返回。
|
||||
|
||||
宿主传的 taskId 不固定:新 id 不再 404,而是即时建好、可直接回显/保存。幂等。
|
||||
"""
|
||||
store = _get_store(request)
|
||||
user_id = _current_user_id(request)
|
||||
existing = await store.get_task(body.id, user_id)
|
||||
if existing is not None:
|
||||
return TaskResponse(**existing)
|
||||
row = await store.create_task(
|
||||
user_id,
|
||||
{
|
||||
"id": body.id,
|
||||
"title": body.title.strip(),
|
||||
"brief": body.brief,
|
||||
"agents": [a.model_dump() for a in body.agents],
|
||||
"messages": [],
|
||||
},
|
||||
)
|
||||
return TaskResponse(**row)
|
||||
|
||||
|
||||
@router.get("/{task_id}", response_model=TaskResponse)
|
||||
async def get_task(request: Request, task_id: str) -> TaskResponse:
|
||||
store = _get_store(request)
|
||||
|
||||
@@ -276,6 +276,13 @@ def _build_task_message(task_name: str, brief: str) -> str:
|
||||
"调度对象 / 分配给『…』的任务 / agent_name」之类调度内容,严禁罗列或转派其他席位的任务,严禁调用"
|
||||
"任何派活 / 调度类工具,也不要用脚本(bash / python)去调用内部接口或翻找派活脚本。"
|
||||
)
|
||||
# 澄清护栏:要向用户确认时,必须用 ask_clarification 工具发起(前端据此渲染可点击的协作
|
||||
# 卡片),不能只把问题写在正文里 —— 否则用户看不到卡片、无法回应。能合理假设就先假设并标注。
|
||||
parts.append(
|
||||
"【需要向用户确认时】如果确实缺少完成分析所必需的关键信息,请**调用 `ask_clarification` 工具**"
|
||||
"发起确认(这样用户会看到一张可点击的协作卡片来回答);**不要只把问题写在正文里**,否则用户无法回应。"
|
||||
"能基于已有信息合理假设的,就先假设并在正文中标注你的假设;只有当不同假设会显著改变结论时,才发起确认。"
|
||||
)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Public (unauthenticated) read-only user leaderboard.
|
||||
|
||||
Mounted under the ``/api/public/`` prefix so the auth middleware lets this route
|
||||
through without a session cookie. Powers the public "publish" page for the user
|
||||
leaderboard — a view-only snapshot with no admin controls and no mutations.
|
||||
|
||||
The snapshot honors the same system-wide leaderboard settings (excluded users,
|
||||
scheduled/admin/failed inclusion) as the authenticated admin endpoint, so any
|
||||
accounts an admin chose to hide stay hidden on the public page as well.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
|
||||
from app.gateway.routers.admin_users import (
|
||||
AdminLeaderboardResponse,
|
||||
_load_or_schedule_daily_leaderboard,
|
||||
_resolve_range,
|
||||
)
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Whitelisted under ``auth_middleware._PUBLIC_PATH_PREFIXES`` (``/api/public/``).
|
||||
router = APIRouter(prefix="/api/public/leaderboard", tags=["public-leaderboard"])
|
||||
|
||||
|
||||
def _mask_email(email: str | None) -> str:
|
||||
"""Reduce an email to its local part so the public page can show a
|
||||
recognizable handle without leaking the full address (domain).
|
||||
|
||||
The leaderboard only makes sense if it names its top users, so the local
|
||||
part is kept intentionally; the privacy gain is dropping the domain and the
|
||||
raw account id from the unauthenticated payload.
|
||||
"""
|
||||
if not email:
|
||||
return ""
|
||||
return email.split("@", 1)[0].strip() or ""
|
||||
|
||||
|
||||
def _anonymize(resp: AdminLeaderboardResponse) -> AdminLeaderboardResponse:
|
||||
"""Strip PII from a leaderboard snapshot before it leaves over the public,
|
||||
unauthenticated endpoint: email → local part, account/thread ids → opaque
|
||||
short hashes (kept stable so they still work as React keys)."""
|
||||
|
||||
def hashed(value: str | None) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
return hashlib.sha1(value.encode("utf-8")).hexdigest()[:12]
|
||||
|
||||
seen_users: set[int] = set()
|
||||
for bucket in (resp.users_by_activity, resp.users_by_questions, resp.users_by_tokens):
|
||||
for user in bucket:
|
||||
if id(user) in seen_users:
|
||||
continue
|
||||
seen_users.add(id(user))
|
||||
user.email = _mask_email(user.email)
|
||||
user.user_id = hashed(user.user_id)
|
||||
|
||||
for bucket in (resp.recent_questions, resp.frequent_questions):
|
||||
for question in bucket:
|
||||
question.email = _mask_email(question.email)
|
||||
question.user_id = hashed(question.user_id)
|
||||
question.thread_id = ""
|
||||
question.run_id = ""
|
||||
|
||||
return resp
|
||||
|
||||
|
||||
@router.get("", response_model=AdminLeaderboardResponse)
|
||||
@router.get("/", response_model=AdminLeaderboardResponse)
|
||||
async def get_public_leaderboard(
|
||||
days: int = Query(default=30, ge=1, le=365),
|
||||
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
|
||||
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
|
||||
limit: int = Query(default=20, ge=5, le=100),
|
||||
) -> AdminLeaderboardResponse:
|
||||
"""Return a read-only leaderboard snapshot for the public publish page."""
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
|
||||
|
||||
settings = load_system_settings().leaderboard
|
||||
since_dt, until_dt, _range_days = _resolve_range(days, since, until)
|
||||
resp = await _load_or_schedule_daily_leaderboard(
|
||||
session_factory,
|
||||
settings,
|
||||
since_dt,
|
||||
until_dt,
|
||||
limit=limit,
|
||||
)
|
||||
return _anonymize(resp)
|
||||
@@ -56,6 +56,44 @@ def _is_admin_user(request: Request) -> bool:
|
||||
return getattr(user, "system_role", None) == "admin"
|
||||
|
||||
|
||||
def _email_local_part(email: str | None) -> str | None:
|
||||
"""Return the username portion (before ``@``) of an email address."""
|
||||
if not email:
|
||||
return None
|
||||
at = email.find("@")
|
||||
return email[:at] if at > 0 else email
|
||||
|
||||
|
||||
async def _attach_owner_usernames(responses: list["SkillResponse"]) -> list["SkillResponse"]:
|
||||
"""Resolve each custom skill's ``owner_user_id`` to a friendly username.
|
||||
|
||||
One lookup per distinct owner id; tolerates deleted accounts and an
|
||||
uninitialised auth provider by leaving ``owner_username`` as None.
|
||||
"""
|
||||
owner_ids = {r.owner_user_id for r in responses if r.owner_user_id}
|
||||
if not owner_ids:
|
||||
return responses
|
||||
try:
|
||||
from app.gateway.deps import get_local_provider
|
||||
|
||||
provider = get_local_provider()
|
||||
except Exception:
|
||||
return responses
|
||||
names: dict[str, str] = {}
|
||||
for uid in owner_ids:
|
||||
try:
|
||||
user = await provider.get_user(uid)
|
||||
except Exception:
|
||||
user = None
|
||||
local = _email_local_part(getattr(user, "email", None)) if user is not None else None
|
||||
if local:
|
||||
names[uid] = local
|
||||
for resp in responses:
|
||||
if resp.owner_user_id:
|
||||
resp.owner_username = names.get(resp.owner_user_id)
|
||||
return responses
|
||||
|
||||
|
||||
def _can_view_skill_source(
|
||||
category: SkillCategory,
|
||||
owner_user_id: str | None,
|
||||
@@ -111,6 +149,7 @@ class SkillResponse(BaseModel):
|
||||
"""Response model for skill information."""
|
||||
|
||||
name: str = Field(..., description="Name of the skill")
|
||||
name_zh: str | None = Field(default=None, description="用户配置或大模型生成的中文名称")
|
||||
description: str = Field(..., description="Description of what the skill does")
|
||||
license: str | None = Field(None, description="License information")
|
||||
category: SkillCategory = Field(..., description="Category of the skill (public or custom)")
|
||||
@@ -125,6 +164,7 @@ class SkillResponse(BaseModel):
|
||||
featured: bool = Field(default=False, description="Whether this skill is pinned to the top of the list (sort only)")
|
||||
source: str = Field(default="upload", description="Origin: 'agent' for agent-created, 'upload' for user-uploaded")
|
||||
owner_user_id: str | None = Field(default=None, description="Custom skill owner (null = built-in or legacy)")
|
||||
owner_username: str | None = Field(default=None, description="Publisher username (email local part). Null for built-in skills.")
|
||||
published: bool = Field(default=False, description="Whether the owner has made this skill visible to other users")
|
||||
builtin: bool = Field(default=False, description="True when owner_user_id is null (no human owner)")
|
||||
detail: str | None = Field(default=None, description="Human-authored long-form detail text (Markdown), editable by owner/admin")
|
||||
@@ -267,11 +307,13 @@ def _skill_to_response(
|
||||
published = False
|
||||
builtin = skill.category != SkillCategory.CUSTOM
|
||||
detail: str | None = None
|
||||
name_zh: str | None = None
|
||||
if ownership is not None:
|
||||
owner_user_id = ownership.get("owner_user_id")
|
||||
published = bool(ownership.get("published", False))
|
||||
builtin = bool(ownership.get("builtin", owner_user_id is None))
|
||||
detail = ownership.get("detail")
|
||||
name_zh = ownership.get("name_zh")
|
||||
|
||||
display = None if display_override is _UNSET else display_override
|
||||
if display_override is _UNSET:
|
||||
@@ -284,6 +326,7 @@ def _skill_to_response(
|
||||
|
||||
return SkillResponse(
|
||||
name=skill.name,
|
||||
name_zh=name_zh,
|
||||
description=skill.description,
|
||||
license=skill.license,
|
||||
category=skill.category,
|
||||
@@ -697,7 +740,8 @@ async def _attach_and_filter_tags(
|
||||
resp.tags = assignments.get(resp.name, [])
|
||||
if search:
|
||||
needle = search.strip().lower()
|
||||
responses = [r for r in responses if needle in r.name.lower()]
|
||||
# Match the English name OR the Chinese display name so 中文 search works.
|
||||
responses = [r for r in responses if needle in r.name.lower() or (r.name_zh or "").lower().find(needle) >= 0]
|
||||
if tag_ids:
|
||||
wanted = set(tag_ids)
|
||||
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
|
||||
@@ -726,7 +770,7 @@ async def list_skills(
|
||||
config: AppConfig = Depends(get_config),
|
||||
skill_store: SkillStore = Depends(get_skill_store),
|
||||
tag_store: TagStore = Depends(get_tag_store),
|
||||
search: str | None = Query(default=None, description="Case-insensitive fuzzy match on skill name"),
|
||||
search: str | None = Query(default=None, description="Case-insensitive fuzzy match on skill name or Chinese name (name_zh)"),
|
||||
tag_ids: list[str] | None = Query(default=None, description="Keep skills carrying any of these tags (OR)"),
|
||||
sort: str = Query(
|
||||
default="time_desc",
|
||||
@@ -786,6 +830,7 @@ async def list_skills(
|
||||
)
|
||||
)
|
||||
responses = await _attach_and_filter_tags(tag_store, responses, search, tag_ids)
|
||||
responses = await _attach_owner_usernames(responses)
|
||||
responses = _sort_skill_responses(responses, sort)
|
||||
return SkillsListResponse(skills=responses)
|
||||
except HTTPException:
|
||||
@@ -1176,6 +1221,7 @@ async def list_custom_skills(
|
||||
display_sample=adapter.get("sample") if adapter else None,
|
||||
)
|
||||
)
|
||||
responses = await _attach_owner_usernames(responses)
|
||||
return SkillsListResponse(skills=responses)
|
||||
except Exception as e:
|
||||
logger.error("Failed to list custom skills: %s", e, exc_info=True)
|
||||
@@ -2309,6 +2355,324 @@ async def update_skill_detail(
|
||||
raise HTTPException(status_code=500, detail=f"Failed to update skill detail: {str(e)}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 技能中文名称(用户可配置 + 大模型自动生成 + 一键补充)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SKILL_NAME_ZH_SYSTEM_PROMPT = (
|
||||
"你是一名中文产品命名助手。根据给定技能的英文名与功能描述,生成一个简洁、自然、"
|
||||
"易懂的中文名称。要求:2 到 10 个汉字;准确概括技能用途;不要包含标点、引号、英文、"
|
||||
"解释或多余文字;只输出中文名称本身。"
|
||||
)
|
||||
|
||||
|
||||
class SkillNameZhModelError(Exception):
|
||||
"""The LLM call backing skill-name generation failed (a *model-side* problem).
|
||||
|
||||
Distinguishes a genuine model failure (timeout / API error / unbuildable
|
||||
model) — which must be surfaced to the user as "大模型报错导致无法补全" — from
|
||||
the model merely returning an unusable name (callers get ``None`` instead).
|
||||
"""
|
||||
|
||||
|
||||
def _clean_generated_name_zh(raw: str) -> str | None:
|
||||
"""Normalise an LLM-produced Chinese name: first line, strip quotes/punct, cap length."""
|
||||
if not raw:
|
||||
return None
|
||||
text = str(raw).strip()
|
||||
# Take the first non-empty line only.
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
text = line
|
||||
break
|
||||
# Strip wrapping quotes / common trailing punctuation.
|
||||
text = text.strip().strip("\"'“”‘’`「」《》【】()()[]").strip()
|
||||
text = text.rstrip("。.,,、!!??;;::")
|
||||
if not text:
|
||||
return None
|
||||
# Keep it short — guard against the model returning a sentence.
|
||||
if len(text) > 20:
|
||||
text = text[:20]
|
||||
return text or None
|
||||
|
||||
|
||||
def _resolve_skill_name_zh_model_name(app_config: AppConfig, model_name: str | None) -> str | None:
|
||||
"""Pick the model name for skill-name generation.
|
||||
|
||||
An explicit ``model_name`` (user-selected in the UI) wins; otherwise fall
|
||||
back to the curator/security-scan model setting (itself falling back to the
|
||||
main model when unset).
|
||||
"""
|
||||
if model_name and model_name.strip():
|
||||
return model_name.strip()
|
||||
if app_config.skill_evolution and app_config.skill_evolution.curator:
|
||||
return app_config.skill_evolution.curator.model_name
|
||||
return None
|
||||
|
||||
|
||||
async def _generate_skill_name_zh(skill: Skill, app_config: AppConfig, model: Any = None, *, model_name: str | None = None) -> str | None:
|
||||
"""Generate a Chinese display name for ``skill`` via the LLM.
|
||||
|
||||
An explicit ``model_name`` lets the caller pick the model; otherwise it
|
||||
reuses the curator/security-scan model setting (falls back to the main
|
||||
model). Returns ``None`` only when the model replies but the output cannot be
|
||||
normalised into a short Chinese name. Raises :class:`SkillNameZhModelError`
|
||||
when the model itself fails (cannot be built / times out / API error), so the
|
||||
caller can tell the user it was a *model* problem.
|
||||
"""
|
||||
user_prompt = (
|
||||
f"技能英文名:{skill.name}\n"
|
||||
f"功能描述:{skill.description or '(无描述)'}\n\n"
|
||||
"请为该技能起一个中文名称。"
|
||||
)
|
||||
if model is None:
|
||||
try:
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
resolved = _resolve_skill_name_zh_model_name(app_config, model_name)
|
||||
model = create_chat_model(name=resolved, thinking_enabled=False, app_config=app_config)
|
||||
except Exception as e:
|
||||
logger.warning("Skill name_zh model build failed for %s", skill.name, exc_info=True)
|
||||
raise SkillNameZhModelError(str(e) or "无法初始化所选大模型") from e
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
model.ainvoke(
|
||||
[
|
||||
{"role": "system", "content": _SKILL_NAME_ZH_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
config={"run_name": "skill_name_zh"},
|
||||
),
|
||||
timeout=60,
|
||||
)
|
||||
except asyncio.TimeoutError as e:
|
||||
logger.warning("Skill name_zh LLM call timed out for %s", skill.name)
|
||||
raise SkillNameZhModelError("大模型响应超时(60 秒)") from e
|
||||
except Exception as e:
|
||||
logger.warning("Skill name_zh LLM call failed for %s", skill.name, exc_info=True)
|
||||
raise SkillNameZhModelError(str(e) or "大模型调用失败") from e
|
||||
return _clean_generated_name_zh(str(getattr(response, "content", "") or ""))
|
||||
|
||||
|
||||
async def _persist_skill_name_zh(
|
||||
skill: Skill,
|
||||
name_zh: str | None,
|
||||
skill_store: SkillStore,
|
||||
user_id: str,
|
||||
*,
|
||||
is_admin: bool,
|
||||
) -> str | None:
|
||||
"""Persist ``name_zh`` into the ``skills`` DB table (never the on-disk SKILL.md).
|
||||
|
||||
Authorization mirrors the ``detail`` annotation: a custom skill is editable by
|
||||
its owner or an admin; a built-in / public skill only by an admin (its DB row
|
||||
is materialized on demand). Returns the stored value.
|
||||
"""
|
||||
if skill.category == SkillCategory.CUSTOM:
|
||||
_record, mode = await _get_editable_skill_or_raise(skill_store, skill.name, user_id, is_admin=is_admin)
|
||||
if mode == "admin":
|
||||
updated = await skill_store.update_admin(skill.name, {"name_zh": name_zh})
|
||||
else:
|
||||
updated = await skill_store.update(skill.name, user_id, {"name_zh": name_zh})
|
||||
else:
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Only admin users can edit built-in skill Chinese names")
|
||||
if await skill_store.get_any(skill.name) is None:
|
||||
await skill_store.ensure_legacy({"name": skill.name, "owner_user_id": None, "published": True})
|
||||
updated = await skill_store.update_admin(skill.name, {"name_zh": name_zh})
|
||||
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=500, detail="Failed to persist skill Chinese name")
|
||||
return updated.get("name_zh")
|
||||
|
||||
|
||||
class SkillNameZhRequest(BaseModel):
|
||||
"""Request body for setting a skill's Chinese display name."""
|
||||
|
||||
name_zh: str = Field(default="", description="中文名称;传空字符串表示清空")
|
||||
|
||||
|
||||
class SkillNameZhResponse(BaseModel):
|
||||
"""Response model for a single skill's Chinese-name update."""
|
||||
|
||||
name: str = Field(..., description="Skill name")
|
||||
name_zh: str | None = Field(default=None, description="中文名称(为空表示已清空)")
|
||||
|
||||
|
||||
class SkillNameZhBulkResponse(BaseModel):
|
||||
"""Response model for the one-click bulk fill of missing Chinese names."""
|
||||
|
||||
generated: list[SkillNameZhResponse] = Field(default_factory=list, description="本次成功生成中文名的技能")
|
||||
failed: list[str] = Field(default_factory=list, description="生成失败的技能英文名")
|
||||
skipped: int = Field(default=0, description="已有中文名而被跳过的技能数")
|
||||
model_error: str | None = Field(default=None, description="若有技能因大模型报错而失败,这里给出代表性的报错信息")
|
||||
|
||||
|
||||
class GenerateSkillNameZhRequest(BaseModel):
|
||||
"""Optional body for skill-name generation: pick the model to use."""
|
||||
|
||||
model_name: str | None = Field(default=None, description="可选:指定用于生成的大模型名称;留空使用默认配置")
|
||||
|
||||
|
||||
def _can_edit_skill_name_zh(skill: Skill, record: dict[str, Any] | None, user_id: str, *, is_admin: bool) -> bool:
|
||||
"""Whether ``user_id`` may set this skill's Chinese name (mirrors detail auth)."""
|
||||
if is_admin:
|
||||
return True
|
||||
if skill.category != SkillCategory.CUSTOM:
|
||||
return False
|
||||
return record is not None and record.get("owner_user_id") == user_id
|
||||
|
||||
|
||||
@router.put(
|
||||
"/skills/{skill_name}/name-zh",
|
||||
response_model=SkillNameZhResponse,
|
||||
summary="Set Skill Chinese Name",
|
||||
description="设置技能的中文名称(存数据库,不改源文件)。自定义技能由其所有者或管理员编辑;内置/公开技能仅管理员可编辑。传空字符串表示清空,回退到使用英文名。",
|
||||
)
|
||||
async def set_skill_name_zh(
|
||||
skill_name: str,
|
||||
body: SkillNameZhRequest,
|
||||
request: Request,
|
||||
config: AppConfig = Depends(get_config),
|
||||
skill_store: SkillStore = Depends(get_skill_store),
|
||||
) -> SkillNameZhResponse:
|
||||
try:
|
||||
skill_name = skill_name.replace("\r\n", "").replace("\n", "")
|
||||
user_id = _current_user_id(request)
|
||||
is_admin = _is_admin_user(request)
|
||||
skills = get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False)
|
||||
skill = next((s for s in skills if s.name == skill_name), None)
|
||||
if skill is None:
|
||||
raise HTTPException(status_code=404, detail=f"Skill '{skill_name}' not found")
|
||||
name_zh = body.name_zh.strip() or None
|
||||
stored = await _persist_skill_name_zh(skill, name_zh, skill_store, user_id, is_admin=is_admin)
|
||||
return SkillNameZhResponse(name=skill_name, name_zh=stored)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to set name_zh for %s: %s", skill_name, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to set skill Chinese name: {str(e)}")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/skills/{skill_name}/name-zh/generate",
|
||||
response_model=SkillNameZhResponse,
|
||||
summary="Generate Skill Chinese Name",
|
||||
description="用大模型为该技能生成一个中文名称并保存到数据库。可在请求体里通过 model_name 指定使用的大模型。",
|
||||
)
|
||||
async def generate_skill_name_zh(
|
||||
skill_name: str,
|
||||
request: Request,
|
||||
body: GenerateSkillNameZhRequest | None = None,
|
||||
config: AppConfig = Depends(get_config),
|
||||
skill_store: SkillStore = Depends(get_skill_store),
|
||||
) -> SkillNameZhResponse:
|
||||
try:
|
||||
body = body or GenerateSkillNameZhRequest()
|
||||
skill_name = skill_name.replace("\r\n", "").replace("\n", "")
|
||||
user_id = _current_user_id(request)
|
||||
is_admin = _is_admin_user(request)
|
||||
skills = get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False)
|
||||
skill = next((s for s in skills if s.name == skill_name), None)
|
||||
if skill is None:
|
||||
raise HTTPException(status_code=404, detail=f"Skill '{skill_name}' not found")
|
||||
try:
|
||||
name_zh = await _generate_skill_name_zh(skill, config, model_name=body.model_name)
|
||||
except SkillNameZhModelError as e:
|
||||
raise HTTPException(status_code=502, detail=f"大模型报错,无法补全技能中文名:{e}")
|
||||
if not name_zh:
|
||||
raise HTTPException(status_code=502, detail="大模型未能生成有效的中文名称,请稍后重试或手动填写")
|
||||
stored = await _persist_skill_name_zh(skill, name_zh, skill_store, user_id, is_admin=is_admin)
|
||||
return SkillNameZhResponse(name=skill_name, name_zh=stored)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to generate name_zh for %s: %s", skill_name, e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to generate skill Chinese name: {str(e)}")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/skills/name-zh/generate-missing",
|
||||
response_model=SkillNameZhBulkResponse,
|
||||
summary="Bulk Generate Missing Skill Chinese Names",
|
||||
description="管理员一键为库中所有尚未配置中文名称的技能生成中文名(存数据库)。可在请求体里通过 model_name 指定使用的大模型。",
|
||||
)
|
||||
async def generate_missing_skill_names_zh(
|
||||
request: Request,
|
||||
body: GenerateSkillNameZhRequest | None = None,
|
||||
config: AppConfig = Depends(get_config),
|
||||
skill_store: SkillStore = Depends(get_skill_store),
|
||||
) -> SkillNameZhBulkResponse:
|
||||
try:
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
body = body or GenerateSkillNameZhRequest()
|
||||
user_id = _current_user_id(request)
|
||||
is_admin = _is_admin_user(request)
|
||||
# 一键补充技能中文名仅对管理员开放。
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="仅管理员可一键补充技能中文名")
|
||||
skills = get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False)
|
||||
|
||||
# Admin sees every row, so existing name_zh values come from the full set.
|
||||
records = {r["name"]: r for r in await skill_store.list_all()}
|
||||
|
||||
pending: list[Skill] = []
|
||||
skipped = 0
|
||||
for skill in skills:
|
||||
record = records.get(skill.name)
|
||||
if record and record.get("name_zh"):
|
||||
skipped += 1
|
||||
continue
|
||||
pending.append(skill)
|
||||
|
||||
generated: list[SkillNameZhResponse] = []
|
||||
failed: list[str] = []
|
||||
model_error: str | None = None
|
||||
if pending:
|
||||
# Build the model once and reuse it for every skill. A build failure
|
||||
# is a model problem — fail fast with a clear message rather than
|
||||
# silently producing zero results.
|
||||
try:
|
||||
resolved_name = _resolve_skill_name_zh_model_name(config, body.model_name)
|
||||
model = create_chat_model(name=resolved_name, thinking_enabled=False, app_config=config)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to build model for bulk name_zh generation", exc_info=True)
|
||||
raise HTTPException(status_code=502, detail=f"大模型报错,无法补全技能中文名:{str(e) or '无法初始化所选大模型'}")
|
||||
|
||||
# Generate concurrently with a small bound to avoid overloading the model.
|
||||
semaphore = asyncio.Semaphore(4)
|
||||
|
||||
async def _one(skill: Skill) -> tuple[Skill, str | None, str | None]:
|
||||
async with semaphore:
|
||||
try:
|
||||
return skill, await _generate_skill_name_zh(skill, config, model=model), None
|
||||
except SkillNameZhModelError as e:
|
||||
return skill, None, str(e)
|
||||
|
||||
results = await asyncio.gather(*[_one(s) for s in pending])
|
||||
for skill, name_zh, err in results:
|
||||
if err and model_error is None:
|
||||
model_error = err
|
||||
if not name_zh:
|
||||
failed.append(skill.name)
|
||||
continue
|
||||
try:
|
||||
stored = await _persist_skill_name_zh(skill, name_zh, skill_store, user_id, is_admin=is_admin)
|
||||
generated.append(SkillNameZhResponse(name=skill.name, name_zh=stored))
|
||||
except HTTPException:
|
||||
# Permission / persistence issue for this one skill — skip it.
|
||||
failed.append(skill.name)
|
||||
|
||||
return SkillNameZhBulkResponse(generated=generated, failed=failed, skipped=skipped, model_error=model_error)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Failed to bulk-generate skill Chinese names: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to bulk-generate skill Chinese names: {str(e)}")
|
||||
|
||||
|
||||
class SkillPinRequest(BaseModel):
|
||||
pinned: bool = Field(..., description="Whether to pin or unpin the skill")
|
||||
|
||||
|
||||
Reference in new issue
Block a user