feat: 定时任务管理 + 会话只读分享

定时任务:
- 管理员可查看/编辑/删除/触发全部用户的定时任务(新增 /api/scheduled-tasks/admin/* 接口与管理页)
- 卡死任务恢复:单次执行超 30 分钟超时重试(最多 3 次),耗尽后标记 failed;看门狗回收服务重启遗留的「执行中」任务

会话分享:
- 新增「只读链接」分享模式,生成公开免登录的只读对话快照页(/share/:code)
- 公开接口 /api/public/shares/* 提供对话快照与图片附件

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
吉鑫升andClaude Opus 4.7 committed 2026-05-18 15:39:40 +08:00
1 parent 4761677676
commit e0c1cb6581
45 files changed
+3235 -434

No files matched your search

@@ -9,7 +9,7 @@ import re
import shutil
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any
@@ -27,6 +27,14 @@ from deerflow.utils.time import now_iso
logger = logging.getLogger(__name__)
# A single execution attempt may not exceed this wall-clock budget. When it
# does, the run is treated as stuck (the agent loop hung, a tool blocked, etc.)
# and retried. The watchdog uses the same window to recover runs orphaned by a
# crash or restart — those stay "running" in the DB with no in-process owner.
_RUN_ATTEMPT_TIMEOUT_SECONDS = 30 * 60 # 30 minutes
# Retries allowed after the first attempt before the run is marked "failed".
_MAX_RUN_RETRIES = 3
@dataclass
class _RuntimeUser:
@@ -398,7 +406,13 @@ class ScheduledTaskService:
)
return task
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
@staticmethod
def _build_task_update(data: dict[str, Any], existing: dict[str, Any] | None) -> dict[str, Any]:
"""Translate an API update payload into a storage update dict.
Shared by owner-scoped :meth:`update_task` and admin :meth:`admin_update_task`
so both apply identical schedule/timezone normalization.
"""
update_data: dict[str, Any] = {}
for key in ("name", "prompt", "enabled", "execution_agent_name"):
if key in data and data[key] is not None:
@@ -409,20 +423,74 @@ class ScheduledTaskService:
if data.get("schedule") is not None:
schedule = str(data["schedule"])
cron_expr = _schedule_to_cron(schedule)
tz = update_data.get("timezone")
if tz is None:
existing = await self.store.get_task(task_id, user_id)
tz = (existing or {}).get("timezone", "UTC")
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
update_data["schedule_text"] = schedule
update_data["cron_expr"] = cron_expr
update_data["next_run_at"] = _next_run_at(cron_expr, tz)
if data.get("execution_context") is not None:
update_data["execution_context"] = data["execution_context"]
# Re-enabling a task must also schedule it forward from now, otherwise a
# paused task with a stale/empty next_run_at would never fire again.
if update_data.get("enabled") is True and "next_run_at" not in update_data:
cron = (existing or {}).get("cron_expr")
if cron:
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
update_data["next_run_at"] = _next_run_at(cron, tz)
return update_data
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
existing = await self.store.get_task(task_id, user_id)
if existing is None:
return None
update_data = self._build_task_update(data, existing)
return await self.store.update_task(task_id, user_id, update_data)
async def delete_task(self, user_id: str, task_id: str) -> bool:
return await self.store.delete_task(task_id, user_id)
# --- Admin (cross-user management) ---
async def admin_list_tasks(self) -> list[dict[str, Any]]:
"""List every scheduled task across all users. Admin-only callers."""
tasks = await self.store.list_all_tasks()
for task in tasks:
task["role"] = "owner"
return tasks
async def admin_update_task(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update any task regardless of owner. Admin-only callers."""
existing = await self.store.get_task_any(task_id)
if existing is None:
return None
update_data = self._build_task_update(data, existing)
return await self.store.update_task_any(task_id, update_data)
async def admin_delete_task(self, task_id: str) -> bool:
"""Delete any task regardless of owner. Admin-only callers."""
return await self.store.delete_task_any(task_id)
async def admin_list_runs(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
"""List runs for any task regardless of owner. Admin-only callers."""
runs = await self.store.list_runs_any(task_id, limit=limit)
for run in runs:
run["can_edit"] = True
return runs
async def admin_get_run(self, run_id: str) -> dict[str, Any] | None:
"""Fetch a single run regardless of owner. Admin-only callers."""
run = await self.store.get_run_any(run_id)
if run is not None:
run["can_edit"] = True
return run
async def admin_trigger_task(self, task_id: str) -> dict[str, Any] | None:
"""Trigger an immediate run of any task regardless of owner."""
task = await self.store.get_task_any(task_id)
if task is None:
return None
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
return task
async def pause_task(self, user_id: str, task_id: str) -> dict[str, Any] | None:
return await self.store.update_task(task_id, user_id, {"enabled": False})
@@ -437,11 +505,19 @@ class ScheduledTaskService:
task = await self.store.get_task(task_id, user_id)
if task is None:
return None
asyncio.create_task(self._execute_task(task, scheduled_for=datetime.now(UTC), manual=True))
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
return task
async def _run_loop(self) -> None:
while not self._stopping.is_set():
# Recover hung / orphaned runs first so a single stuck task cannot
# silently wedge the scheduler indefinitely.
try:
await self._reconcile_stuck_runs()
except asyncio.CancelledError:
raise
except Exception:
logger.exception("Scheduled task watchdog scan failed")
try:
due = await self.store.due_tasks(datetime.now(UTC), limit=20)
for task in due:
@@ -466,64 +542,195 @@ class ScheduledTaskService:
finally:
self._running_task_ids.discard(task["task_id"])
async def _execute_task(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
async def _execute_tracked(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
"""Run a task while marking it busy so the watchdog won't double-drive it."""
self._running_task_ids.add(task["task_id"])
try:
await self._execute_task(task, scheduled_for=scheduled_for, manual=manual)
finally:
self._running_task_ids.discard(task["task_id"])
async def _reconcile_stuck_runs(self) -> None:
"""Recover runs left in the ``running`` state with no in-process owner.
A run becomes stuck when the agent loop hangs past the per-attempt
timeout, or when the process crashes / restarts mid-run (the original
asyncio task is gone but the DB row still says ``running``). For each
such run we retry up to ``_MAX_RUN_RETRIES`` times, then mark it
``failed`` so it no longer shows as perpetually executing.
"""
cutoff = datetime.now(UTC) - timedelta(seconds=_RUN_ATTEMPT_TIMEOUT_SECONDS)
stuck = await self.store.list_stuck_runs(cutoff, limit=20)
for run in stuck:
task_id = run["task_id"]
# A run still owned by a live in-process execution is handled by
# that execution's own timeout/retry loop — don't touch it here.
if task_id in self._running_task_ids:
continue
self._running_task_ids.add(task_id)
asyncio.create_task(self._recover_and_untrack(run))
async def _recover_and_untrack(self, run: dict[str, Any]) -> None:
try:
await self._recover_stuck_run(run)
except Exception:
logger.exception("Failed to recover stuck scheduled task run %s", run.get("id"))
finally:
self._running_task_ids.discard(run["task_id"])
async def _recover_stuck_run(self, run: dict[str, Any]) -> None:
run_id = run["id"]
retry_count = int(run.get("retry_count") or 0)
task = await self.store.get_task_any(run["task_id"])
scheduled_for = _coerce_datetime(run.get("scheduled_for")) or datetime.now(UTC)
if task is None:
# Parent task was deleted — just close out the orphaned run.
await self.store.update_run(
run_id,
{"status": "failed", "finished_at": datetime.now(UTC), "error": "执行超时:所属任务已被删除"},
)
return
if retry_count >= _MAX_RUN_RETRIES:
await self._fail_run(
task,
run_id,
scheduled_for,
error=f"执行超时:超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟未完成,重试 {_MAX_RUN_RETRIES} 次后仍失败",
retry_count=retry_count,
)
return
logger.warning("Recovering stuck scheduled task run %s (retry budget %s/%s)", run_id, retry_count, _MAX_RUN_RETRIES)
await self._execute_task(task, scheduled_for=scheduled_for, manual=False, run_row=run)
async def _fail_run(
self,
task: dict[str, Any],
run_id: str,
scheduled_for: datetime,
*,
error: str,
retry_count: int = 0,
) -> None:
"""Mark a run failed and reflect the outcome on the parent task."""
result = _build_result(task, scheduled_for=scheduled_for, content=f"Scheduled task failed: {error}")
await self.store.update_run(
run_id,
{"status": "failed", "finished_at": datetime.now(UTC), "error": error, "result": result, "retry_count": retry_count},
)
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
async def _execute_task(
self,
task: dict[str, Any],
*,
scheduled_for: datetime,
manual: bool,
run_row: dict[str, Any] | None = None,
) -> None:
# Each run uses its own throwaway thread so message history cannot accumulate
# across runs and blow past the model's input window. The per-user
# task["scheduler_thread_id"] is no longer used to drive the LLM call —
# we keep it on the task row only for backwards compatibility.
runtime_thread_id = str(uuid.uuid4())
run_row = await self.store.create_run_once(
{
"id": str(uuid.uuid4()),
"task_id": task["task_id"],
"user_id": task["user_id"],
"scheduler_thread_id": runtime_thread_id,
"scheduled_for": scheduled_for,
"status": "running",
"started_at": datetime.now(UTC),
}
)
is_recovery = run_row is not None
if run_row is None:
return
run_row = await self.store.create_run_once(
{
"id": str(uuid.uuid4()),
"task_id": task["task_id"],
"user_id": task["user_id"],
"scheduler_thread_id": str(uuid.uuid4()),
"scheduled_for": scheduled_for,
"status": "running",
"started_at": datetime.now(UTC),
}
)
if run_row is None:
return
else:
# Recovery path: re-claim an existing (orphaned) run row.
await self.store.update_run(run_row["id"], {"status": "running", "started_at": datetime.now(UTC)})
if not manual:
# Advancing the schedule only happens once, when the run is first
# created — recovery retries reuse the original scheduled_for.
if not manual and not is_recovery:
next_run_at = _next_run_at(task["cron_expr"], task.get("timezone") or "UTC", base=scheduled_for)
await self.store.update_task(task["task_id"], task["user_id"], {"next_run_at": next_run_at, "last_status": "running"})
try:
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
async with self._agent_run_lock:
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
await self.store.update_run(run_row["id"], {"agent_run_id": record.run_id})
if record.task is not None:
await record.task
result_text = await self._extract_latest_ai_text(runtime_thread_id)
result_files = await self._extract_result_files(
runtime_thread_id,
task["user_id"],
run_id=run_row["id"],
output_snapshot=output_snapshot,
run_id = run_row["id"]
retry_count = int(run_row.get("retry_count") or 0)
while True:
runtime_thread_id = str(uuid.uuid4())
try:
result = await asyncio.wait_for(
self._run_agent_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
timeout=_RUN_ATTEMPT_TIMEOUT_SECONDS,
)
result = _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
except TimeoutError:
# The attempt overran the 30-minute cap. Retry a bounded number
# of times, then give up and mark the run failed.
if retry_count >= _MAX_RUN_RETRIES:
await self._fail_run(
task,
run_id,
scheduled_for,
error=f"执行超时:单次执行超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟,重试 {_MAX_RUN_RETRIES} 次后仍失败",
retry_count=retry_count,
)
return
retry_count += 1
logger.warning("Scheduled task %s timed out; retry %s/%s", task["task_id"], retry_count, _MAX_RUN_RETRIES)
await self.store.update_run(
run_id,
{
"retry_count": retry_count,
"status": "running",
"started_at": datetime.now(UTC),
"error": f"执行超时,正在重试(第 {retry_count}/{_MAX_RUN_RETRIES} 次)",
},
)
continue
except Exception as exc:
logger.exception("Scheduled task %s failed", task["task_id"])
await self._fail_run(task, run_id, scheduled_for, error=str(exc), retry_count=retry_count)
return
await self.store.update_run(
run_row["id"],
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result},
run_id,
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result, "retry_count": retry_count, "error": None},
)
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "succeeded"})
await self._deliver_element_result(task, result["content"])
except Exception as exc:
logger.exception("Scheduled task %s failed", task["task_id"])
result = _build_result(
task,
scheduled_for=scheduled_for,
content=f"Scheduled task failed: {exc}",
return
async def _run_agent_attempt(
self,
task: dict[str, Any],
*,
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> dict[str, Any]:
"""Run the agent once and build the result payload.
Wrapped in :func:`asyncio.wait_for` by the caller so a hung agent loop
cannot hold the shared run lock — and the whole scheduler — forever.
"""
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
async with self._agent_run_lock:
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
if record.task is not None:
await record.task
result_text = await self._extract_latest_ai_text(runtime_thread_id)
result_files = await self._extract_result_files(
runtime_thread_id,
task["user_id"],
run_id=run_id,
output_snapshot=output_snapshot,
)
await self.store.update_run(
run_row["id"],
{"status": "failed", "finished_at": datetime.now(UTC), "error": str(exc), "result": result},
)
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
return _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
async def _ensure_thread_meta(self, user_id: str, thread_id: str) -> None:
token = set_current_user(_RuntimeUser(id=user_id))