This commit is contained in:
董宏凯 committed 2026-06-07 21:25:54 +08:00
commit 05d14bb364
32 files changed
+2290 -507

No files matched your search

@@ -378,7 +378,7 @@ def _release_singleton_lock(handle) -> None:
pass
async def _curator_scheduler_loop() -> None:
async def _curator_scheduler_loop() -> None:
"""常驻后台任务:每隔 _CURATOR_CHECK_INTERVAL_SECONDS 检查一次需不需要进化。
多 worker 部署下用文件锁做 leader 选举 —— 只有抢到锁的那个 worker 进程真正
@@ -404,11 +404,248 @@ async def _curator_scheduler_loop() -> None:
except Exception:
logger.exception("Curator scheduler tick failed (non-fatal)")
await asyncio.sleep(_CURATOR_CHECK_INTERVAL_SECONDS)
finally:
_release_singleton_lock(lock_handle)
def _warn_if_multi_worker() -> None:
finally:
_release_singleton_lock(lock_handle)
def _env_bool(name: str, default: bool) -> bool:
raw = os.getenv(name)
if raw is None:
return default
return raw.strip().lower() not in {"0", "false", "no", "off"}
def _env_int(name: str, default: int, *, alias: str | None = None, minimum: int = 1) -> int:
raw = os.getenv(name)
if raw is None and alias:
raw = os.getenv(alias)
if raw is None:
raw = str(default)
try:
value = int(raw)
except ValueError:
return default
return max(minimum, value)
def _env_float(name: str, default: float, *, alias: str | None = None, minimum: float = 0.0) -> float:
raw = os.getenv(name)
if raw is None and alias:
raw = os.getenv(alias)
if raw is None:
raw = str(default)
try:
value = float(raw)
except ValueError:
return default
return max(minimum, value)
_LEADERBOARD_SNAPSHOT_BACKGROUND_ENABLED = _env_bool("ADMIN_LEADERBOARD_BACKGROUND_ENABLED", True)
_LEADERBOARD_SNAPSHOT_PREWARM_DAYS = _env_int(
"ADMIN_LEADERBOARD_BACKGROUND_PREWARM_DAYS",
30,
)
_LEADERBOARD_SNAPSHOT_BATCH_SIZE = _env_int(
"ADMIN_LEADERBOARD_BACKGROUND_BATCH_SIZE",
1,
)
_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS = _env_float(
"ADMIN_LEADERBOARD_BACKGROUND_INTERVAL_SECONDS",
300.0,
)
_LEADERBOARD_SNAPSHOT_SLEEP_SECONDS = _env_float(
"ADMIN_LEADERBOARD_BACKGROUND_SLEEP_SECONDS",
2.0,
)
_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS = _env_int(
"ADMIN_LEADERBOARD_BACKGROUND_RUNNING_TTL_SECONDS",
1800,
)
_LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
"ADMIN_LEADERBOARD_BACKGROUND_STARTUP_DELAY_SECONDS",
60.0,
)
def _recent_leaderboard_stat_dates(days: int) -> list[str]:
from datetime import datetime, timedelta
from deerflow.persistence.types import BEIJING_TZ
today = datetime.now(BEIJING_TZ).date()
start = today - timedelta(days=max(1, days) - 1)
return [(start + timedelta(days=i)).isoformat() for i in range((today - start).days + 1)]
async def _leaderboard_snapshot_due_dates(
session_factory,
*,
settings_hash: str,
limit: int,
running_ttl_seconds: int,
) -> list[str]:
from datetime import UTC, datetime, timedelta
from sqlalchemy import and_, or_, select
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
cutoff = datetime.now(UTC) - timedelta(seconds=running_ttl_seconds)
async with session_factory() as session:
rows = (
await session.execute(
select(AdminLeaderboardDailyStatRow.stat_date)
.where(
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
or_(
AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
and_(
AdminLeaderboardDailyStatRow.status == "running",
AdminLeaderboardDailyStatRow.updated_at < cutoff,
),
),
)
.order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
.limit(limit)
)
).all()
return [str(row[0]) for row in rows]
async def _claim_leaderboard_snapshot(
session_factory,
*,
stat_date: str,
settings_hash: str,
running_ttl_seconds: int,
) -> bool:
from datetime import UTC, datetime
from app.gateway.routers.admin_users import _snapshot_is_fresh
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.persistence.types import BEIJING_TZ
async with session_factory() as session:
row = await session.get(
AdminLeaderboardDailyStatRow,
{"stat_date": stat_date, "settings_hash": settings_hash},
)
if row is None or _snapshot_is_fresh(row, stat_date):
return False
if row.status == "running":
updated_at = row.updated_at
if isinstance(updated_at, datetime):
if updated_at.tzinfo is None:
updated_at = updated_at.replace(tzinfo=BEIJING_TZ)
age = (datetime.now(BEIJING_TZ) - updated_at.astimezone(BEIJING_TZ)).total_seconds()
if age < running_ttl_seconds:
return False
row.status = "running"
row.message = "排行榜后台统计任务已认领。"
row.error = None
row.updated_at = datetime.now(UTC)
await session.commit()
return True
async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
from app.gateway.routers.admin_users import (
_daily_job_key,
_queue_daily_leaderboard_snapshots,
_refresh_daily_leaderboard_snapshot,
_resolve_leaderboard_snapshot_scope,
)
from deerflow.config.system_settings import load_system_settings
from deerflow.persistence.engine import get_session_factory
session_factory = get_session_factory()
if session_factory is None:
return 0, 0
settings = load_system_settings().leaderboard
excluded, include_scheduled, include_admins, include_failed, settings_hash = _resolve_leaderboard_snapshot_scope(settings)
stat_dates = _recent_leaderboard_stat_dates(_LEADERBOARD_SNAPSHOT_PREWARM_DAYS)
queued_dates = await _queue_daily_leaderboard_snapshots(
session_factory,
stat_dates,
settings_hash,
message="排行榜后台统计任务定期预热/补算队列。",
)
due_dates = await _leaderboard_snapshot_due_dates(
session_factory,
settings_hash=settings_hash,
limit=_LEADERBOARD_SNAPSHOT_BATCH_SIZE,
running_ttl_seconds=_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS,
)
processed = 0
for stat_date in due_dates:
claimed = await _claim_leaderboard_snapshot(
session_factory,
stat_date=stat_date,
settings_hash=settings_hash,
running_ttl_seconds=_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS,
)
if not claimed:
continue
logger.info("Leaderboard snapshot scheduler: building date=%s hash=%s", stat_date, settings_hash[:10])
await _refresh_daily_leaderboard_snapshot(
_daily_job_key(stat_date, settings_hash),
session_factory=session_factory,
settings=settings,
stat_date=stat_date,
settings_hash=settings_hash,
excluded=excluded,
include_scheduled=include_scheduled,
include_admins=include_admins,
include_failed=include_failed,
)
processed += 1
if _LEADERBOARD_SNAPSHOT_SLEEP_SECONDS > 0 and processed < len(due_dates):
await asyncio.sleep(_LEADERBOARD_SNAPSHOT_SLEEP_SECONDS)
return processed, len(queued_dates)
async def _leaderboard_snapshot_scheduler_loop() -> None:
"""Background task inside the existing gateway process for daily snapshots."""
if not _LEADERBOARD_SNAPSHOT_BACKGROUND_ENABLED:
logger.info("Leaderboard snapshot scheduler disabled by ADMIN_LEADERBOARD_BACKGROUND_ENABLED")
return
from deerflow.config.runtime_paths import runtime_home
lock_handle = _acquire_singleton_lock(runtime_home() / ".leaderboard_snapshot_scheduler.lock")
if lock_handle is None:
logger.info("Leaderboard snapshot scheduler: another gateway worker holds the lock; this worker skips")
return
logger.info(
"Leaderboard snapshot scheduler started: prewarm_days=%d batch_size=%d interval=%.1fs",
_LEADERBOARD_SNAPSHOT_PREWARM_DAYS,
_LEADERBOARD_SNAPSHOT_BATCH_SIZE,
_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS,
)
try:
if _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS > 0:
await asyncio.sleep(_LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS)
while True:
try:
processed, queued = await _run_leaderboard_snapshot_tick()
if processed or queued:
logger.info(
"Leaderboard snapshot scheduler tick: queued=%d processed=%d",
queued,
processed,
)
except Exception:
logger.exception("Leaderboard snapshot scheduler tick failed (non-fatal)")
await asyncio.sleep(_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS)
finally:
_release_singleton_lock(lock_handle)
def _warn_if_multi_worker() -> None:
"""检测 uvicorn / gunicorn 多 worker 部署并打 ERROR 日志。
主 agent / canvas / AI 写作流式响应在 Gateway 进程内通过 ``StreamBridge``
@@ -562,16 +799,21 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
# 默认每天凌晨 3 点扫一次,删 retention_days(默认 7 天)以前的会话,联动
# 清理 MySQL 业务表 + SQLite checkpointer 里的 thread 状态。多 worker
# 下用文件锁选 leader,只有抢到锁的 worker 真正跑清理。
try:
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop(app))
except Exception:
logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
# Start IM channel service if any channels are configured
try:
from app.channels.service import start_channel_service
try:
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop(app))
except Exception:
logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
try:
app.state.leaderboard_snapshot_scheduler_task = asyncio.create_task(_leaderboard_snapshot_scheduler_loop())
except Exception:
logger.exception("Leaderboard snapshot scheduler failed to start (non-fatal)")
# Start IM channel service if any channels are configured
try:
from app.channels.service import start_channel_service
channel_service = await start_channel_service(app.state.config)
logger.info("Channel service started: %s", channel_service.get_status())
@@ -615,7 +857,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except asyncio.CancelledError:
pass
except Exception:
logger.exception("AI writing cleanup scheduler shutdown error")
logger.exception("AI writing cleanup scheduler shutdown error")
leaderboard_snapshot_task = getattr(app.state, "leaderboard_snapshot_scheduler_task", None)
if leaderboard_snapshot_task is not None:
leaderboard_snapshot_task.cancel()
try:
await leaderboard_snapshot_task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("Leaderboard snapshot scheduler shutdown error")
# 阶段 7 删冗余:AI 写作 graph 不再自管 checkpointer / in-flight task;
# thread 状态由 LangGraph Server 主 checkpointer 接管,lifespan 出口由
File diff suppressed because it is too large. Load diff
@@ -1,9 +1,9 @@
"""Tag management API.
Tags are split into three types (``agent`` / ``skill`` / ``scheduled_task``).
A tag may only be applied to a resource of its own type. Tag CRUD and tag
assignment are admin-only; listing tags and reading a resource's tags is open
to everyone (the frontend needs them to render chips and search filters).
A tag may only be applied to a resource of its own type. Tag CRUD, listing,
and assignment are available to ordinary users so teams can organize resources
without asking an administrator to tag every item.
"""
from __future__ import annotations
@@ -11,10 +11,10 @@ from __future__ import annotations
import logging
import uuid
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel, Field
from app.gateway.deps import get_optional_user_from_request, get_tag_store
from app.gateway.deps import get_tag_store
from deerflow.persistence.tags import TAG_TYPES, TagStore
logger = logging.getLogger(__name__)
@@ -22,18 +22,6 @@ logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/tags", tags=["tags"])
# --------------------------------------------------------------------------- auth
async def _require_admin(request: Request) -> None:
"""Reject non-admins. Lets the call through when auth is disabled."""
user = await get_optional_user_from_request(request)
if user is None:
return
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="Tag management is admin-only")
def _validate_tag_type(tag_type: str) -> str:
if tag_type not in TAG_TYPES:
raise HTTPException(status_code=422, detail=f"Invalid tag type '{tag_type}'. Allowed: {', '.join(TAG_TYPES)}")
@@ -96,10 +84,8 @@ async def list_tags(
@router.post("", response_model=TagResponse, status_code=201, summary="Create Tag")
async def create_tag(
body: TagCreateRequest,
request: Request,
store: TagStore = Depends(get_tag_store),
) -> TagResponse:
await _require_admin(request)
_validate_tag_type(body.tag_type)
try:
record = await store.create_tag(
@@ -125,10 +111,8 @@ async def create_tag(
async def update_tag(
tag_id: str,
body: TagUpdateRequest,
request: Request,
store: TagStore = Depends(get_tag_store),
) -> TagResponse:
await _require_admin(request)
update_data = body.model_dump(exclude_unset=True)
if "name" in update_data and update_data["name"] is not None:
update_data["name"] = update_data["name"].strip()
@@ -149,10 +133,8 @@ async def update_tag(
@router.delete("/{tag_id}", status_code=204, summary="Delete Tag")
async def delete_tag(
tag_id: str,
request: Request,
store: TagStore = Depends(get_tag_store),
) -> None:
await _require_admin(request)
try:
deleted = await store.delete_tag(tag_id)
if not deleted:
@@ -195,10 +177,8 @@ async def set_resource_tags(
target_type: str,
target_id: str,
body: TagAssignmentRequest,
request: Request,
store: TagStore = Depends(get_tag_store),
) -> TagListResponse:
await _require_admin(request)
_validate_tag_type(target_type)
try:
records = await store.set_target_tags(target_type, target_id, body.tag_ids)