feat: 定时任务管理 + 会话只读分享
定时任务: - 管理员可查看/编辑/删除/触发全部用户的定时任务(新增 /api/scheduled-tasks/admin/* 接口与管理页) - 卡死任务恢复:单次执行超 30 分钟超时重试(最多 3 次),耗尽后标记 failed;看门狗回收服务重启遗留的「执行中」任务 会话分享: - 新增「只读链接」分享模式,生成公开免登录的只读对话快照页(/share/:code) - 公开接口 /api/public/shares/* 提供对话快照与图片附件 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4761677676
commit
e0c1cb6581
45 files changed
+3235
-434
No files matched your search
+257
-50
@@ -9,7 +9,7 @@ import re
|
||||
import shutil
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
@@ -27,6 +27,14 @@ from deerflow.utils.time import now_iso
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# A single execution attempt may not exceed this wall-clock budget. When it
|
||||
# does, the run is treated as stuck (the agent loop hung, a tool blocked, etc.)
|
||||
# and retried. The watchdog uses the same window to recover runs orphaned by a
|
||||
# crash or restart — those stay "running" in the DB with no in-process owner.
|
||||
_RUN_ATTEMPT_TIMEOUT_SECONDS = 30 * 60 # 30 minutes
|
||||
# Retries allowed after the first attempt before the run is marked "failed".
|
||||
_MAX_RUN_RETRIES = 3
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RuntimeUser:
|
||||
@@ -398,7 +406,13 @@ class ScheduledTaskService:
|
||||
)
|
||||
return task
|
||||
|
||||
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
@staticmethod
|
||||
def _build_task_update(data: dict[str, Any], existing: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Translate an API update payload into a storage update dict.
|
||||
|
||||
Shared by owner-scoped :meth:`update_task` and admin :meth:`admin_update_task`
|
||||
so both apply identical schedule/timezone normalization.
|
||||
"""
|
||||
update_data: dict[str, Any] = {}
|
||||
for key in ("name", "prompt", "enabled", "execution_agent_name"):
|
||||
if key in data and data[key] is not None:
|
||||
@@ -409,20 +423,74 @@ class ScheduledTaskService:
|
||||
if data.get("schedule") is not None:
|
||||
schedule = str(data["schedule"])
|
||||
cron_expr = _schedule_to_cron(schedule)
|
||||
tz = update_data.get("timezone")
|
||||
if tz is None:
|
||||
existing = await self.store.get_task(task_id, user_id)
|
||||
tz = (existing or {}).get("timezone", "UTC")
|
||||
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
|
||||
update_data["schedule_text"] = schedule
|
||||
update_data["cron_expr"] = cron_expr
|
||||
update_data["next_run_at"] = _next_run_at(cron_expr, tz)
|
||||
if data.get("execution_context") is not None:
|
||||
update_data["execution_context"] = data["execution_context"]
|
||||
# Re-enabling a task must also schedule it forward from now, otherwise a
|
||||
# paused task with a stale/empty next_run_at would never fire again.
|
||||
if update_data.get("enabled") is True and "next_run_at" not in update_data:
|
||||
cron = (existing or {}).get("cron_expr")
|
||||
if cron:
|
||||
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
|
||||
update_data["next_run_at"] = _next_run_at(cron, tz)
|
||||
return update_data
|
||||
|
||||
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
existing = await self.store.get_task(task_id, user_id)
|
||||
if existing is None:
|
||||
return None
|
||||
update_data = self._build_task_update(data, existing)
|
||||
return await self.store.update_task(task_id, user_id, update_data)
|
||||
|
||||
async def delete_task(self, user_id: str, task_id: str) -> bool:
|
||||
return await self.store.delete_task(task_id, user_id)
|
||||
|
||||
# --- Admin (cross-user management) ---
|
||||
|
||||
async def admin_list_tasks(self) -> list[dict[str, Any]]:
|
||||
"""List every scheduled task across all users. Admin-only callers."""
|
||||
tasks = await self.store.list_all_tasks()
|
||||
for task in tasks:
|
||||
task["role"] = "owner"
|
||||
return tasks
|
||||
|
||||
async def admin_update_task(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update any task regardless of owner. Admin-only callers."""
|
||||
existing = await self.store.get_task_any(task_id)
|
||||
if existing is None:
|
||||
return None
|
||||
update_data = self._build_task_update(data, existing)
|
||||
return await self.store.update_task_any(task_id, update_data)
|
||||
|
||||
async def admin_delete_task(self, task_id: str) -> bool:
|
||||
"""Delete any task regardless of owner. Admin-only callers."""
|
||||
return await self.store.delete_task_any(task_id)
|
||||
|
||||
async def admin_list_runs(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""List runs for any task regardless of owner. Admin-only callers."""
|
||||
runs = await self.store.list_runs_any(task_id, limit=limit)
|
||||
for run in runs:
|
||||
run["can_edit"] = True
|
||||
return runs
|
||||
|
||||
async def admin_get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""Fetch a single run regardless of owner. Admin-only callers."""
|
||||
run = await self.store.get_run_any(run_id)
|
||||
if run is not None:
|
||||
run["can_edit"] = True
|
||||
return run
|
||||
|
||||
async def admin_trigger_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""Trigger an immediate run of any task regardless of owner."""
|
||||
task = await self.store.get_task_any(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
|
||||
return task
|
||||
|
||||
async def pause_task(self, user_id: str, task_id: str) -> dict[str, Any] | None:
|
||||
return await self.store.update_task(task_id, user_id, {"enabled": False})
|
||||
|
||||
@@ -437,11 +505,19 @@ class ScheduledTaskService:
|
||||
task = await self.store.get_task(task_id, user_id)
|
||||
if task is None:
|
||||
return None
|
||||
asyncio.create_task(self._execute_task(task, scheduled_for=datetime.now(UTC), manual=True))
|
||||
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
|
||||
return task
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
while not self._stopping.is_set():
|
||||
# Recover hung / orphaned runs first so a single stuck task cannot
|
||||
# silently wedge the scheduler indefinitely.
|
||||
try:
|
||||
await self._reconcile_stuck_runs()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Scheduled task watchdog scan failed")
|
||||
try:
|
||||
due = await self.store.due_tasks(datetime.now(UTC), limit=20)
|
||||
for task in due:
|
||||
@@ -466,64 +542,195 @@ class ScheduledTaskService:
|
||||
finally:
|
||||
self._running_task_ids.discard(task["task_id"])
|
||||
|
||||
async def _execute_task(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
|
||||
async def _execute_tracked(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
|
||||
"""Run a task while marking it busy so the watchdog won't double-drive it."""
|
||||
self._running_task_ids.add(task["task_id"])
|
||||
try:
|
||||
await self._execute_task(task, scheduled_for=scheduled_for, manual=manual)
|
||||
finally:
|
||||
self._running_task_ids.discard(task["task_id"])
|
||||
|
||||
async def _reconcile_stuck_runs(self) -> None:
|
||||
"""Recover runs left in the ``running`` state with no in-process owner.
|
||||
|
||||
A run becomes stuck when the agent loop hangs past the per-attempt
|
||||
timeout, or when the process crashes / restarts mid-run (the original
|
||||
asyncio task is gone but the DB row still says ``running``). For each
|
||||
such run we retry up to ``_MAX_RUN_RETRIES`` times, then mark it
|
||||
``failed`` so it no longer shows as perpetually executing.
|
||||
"""
|
||||
cutoff = datetime.now(UTC) - timedelta(seconds=_RUN_ATTEMPT_TIMEOUT_SECONDS)
|
||||
stuck = await self.store.list_stuck_runs(cutoff, limit=20)
|
||||
for run in stuck:
|
||||
task_id = run["task_id"]
|
||||
# A run still owned by a live in-process execution is handled by
|
||||
# that execution's own timeout/retry loop — don't touch it here.
|
||||
if task_id in self._running_task_ids:
|
||||
continue
|
||||
self._running_task_ids.add(task_id)
|
||||
asyncio.create_task(self._recover_and_untrack(run))
|
||||
|
||||
async def _recover_and_untrack(self, run: dict[str, Any]) -> None:
|
||||
try:
|
||||
await self._recover_stuck_run(run)
|
||||
except Exception:
|
||||
logger.exception("Failed to recover stuck scheduled task run %s", run.get("id"))
|
||||
finally:
|
||||
self._running_task_ids.discard(run["task_id"])
|
||||
|
||||
async def _recover_stuck_run(self, run: dict[str, Any]) -> None:
|
||||
run_id = run["id"]
|
||||
retry_count = int(run.get("retry_count") or 0)
|
||||
task = await self.store.get_task_any(run["task_id"])
|
||||
scheduled_for = _coerce_datetime(run.get("scheduled_for")) or datetime.now(UTC)
|
||||
if task is None:
|
||||
# Parent task was deleted — just close out the orphaned run.
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": "执行超时:所属任务已被删除"},
|
||||
)
|
||||
return
|
||||
if retry_count >= _MAX_RUN_RETRIES:
|
||||
await self._fail_run(
|
||||
task,
|
||||
run_id,
|
||||
scheduled_for,
|
||||
error=f"执行超时:超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟未完成,重试 {_MAX_RUN_RETRIES} 次后仍失败",
|
||||
retry_count=retry_count,
|
||||
)
|
||||
return
|
||||
logger.warning("Recovering stuck scheduled task run %s (retry budget %s/%s)", run_id, retry_count, _MAX_RUN_RETRIES)
|
||||
await self._execute_task(task, scheduled_for=scheduled_for, manual=False, run_row=run)
|
||||
|
||||
async def _fail_run(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
run_id: str,
|
||||
scheduled_for: datetime,
|
||||
*,
|
||||
error: str,
|
||||
retry_count: int = 0,
|
||||
) -> None:
|
||||
"""Mark a run failed and reflect the outcome on the parent task."""
|
||||
result = _build_result(task, scheduled_for=scheduled_for, content=f"Scheduled task failed: {error}")
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": error, "result": result, "retry_count": retry_count},
|
||||
)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
|
||||
|
||||
async def _execute_task(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
*,
|
||||
scheduled_for: datetime,
|
||||
manual: bool,
|
||||
run_row: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
# Each run uses its own throwaway thread so message history cannot accumulate
|
||||
# across runs and blow past the model's input window. The per-user
|
||||
# task["scheduler_thread_id"] is no longer used to drive the LLM call —
|
||||
# we keep it on the task row only for backwards compatibility.
|
||||
runtime_thread_id = str(uuid.uuid4())
|
||||
run_row = await self.store.create_run_once(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"task_id": task["task_id"],
|
||||
"user_id": task["user_id"],
|
||||
"scheduler_thread_id": runtime_thread_id,
|
||||
"scheduled_for": scheduled_for,
|
||||
"status": "running",
|
||||
"started_at": datetime.now(UTC),
|
||||
}
|
||||
)
|
||||
is_recovery = run_row is not None
|
||||
if run_row is None:
|
||||
return
|
||||
run_row = await self.store.create_run_once(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"task_id": task["task_id"],
|
||||
"user_id": task["user_id"],
|
||||
"scheduler_thread_id": str(uuid.uuid4()),
|
||||
"scheduled_for": scheduled_for,
|
||||
"status": "running",
|
||||
"started_at": datetime.now(UTC),
|
||||
}
|
||||
)
|
||||
if run_row is None:
|
||||
return
|
||||
else:
|
||||
# Recovery path: re-claim an existing (orphaned) run row.
|
||||
await self.store.update_run(run_row["id"], {"status": "running", "started_at": datetime.now(UTC)})
|
||||
|
||||
if not manual:
|
||||
# Advancing the schedule only happens once, when the run is first
|
||||
# created — recovery retries reuse the original scheduled_for.
|
||||
if not manual and not is_recovery:
|
||||
next_run_at = _next_run_at(task["cron_expr"], task.get("timezone") or "UTC", base=scheduled_for)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"next_run_at": next_run_at, "last_status": "running"})
|
||||
|
||||
try:
|
||||
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
|
||||
async with self._agent_run_lock:
|
||||
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
|
||||
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
|
||||
await self.store.update_run(run_row["id"], {"agent_run_id": record.run_id})
|
||||
if record.task is not None:
|
||||
await record.task
|
||||
result_text = await self._extract_latest_ai_text(runtime_thread_id)
|
||||
result_files = await self._extract_result_files(
|
||||
runtime_thread_id,
|
||||
task["user_id"],
|
||||
run_id=run_row["id"],
|
||||
output_snapshot=output_snapshot,
|
||||
run_id = run_row["id"]
|
||||
retry_count = int(run_row.get("retry_count") or 0)
|
||||
|
||||
while True:
|
||||
runtime_thread_id = str(uuid.uuid4())
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
self._run_agent_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
|
||||
timeout=_RUN_ATTEMPT_TIMEOUT_SECONDS,
|
||||
)
|
||||
result = _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
|
||||
except TimeoutError:
|
||||
# The attempt overran the 30-minute cap. Retry a bounded number
|
||||
# of times, then give up and mark the run failed.
|
||||
if retry_count >= _MAX_RUN_RETRIES:
|
||||
await self._fail_run(
|
||||
task,
|
||||
run_id,
|
||||
scheduled_for,
|
||||
error=f"执行超时:单次执行超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟,重试 {_MAX_RUN_RETRIES} 次后仍失败",
|
||||
retry_count=retry_count,
|
||||
)
|
||||
return
|
||||
retry_count += 1
|
||||
logger.warning("Scheduled task %s timed out; retry %s/%s", task["task_id"], retry_count, _MAX_RUN_RETRIES)
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{
|
||||
"retry_count": retry_count,
|
||||
"status": "running",
|
||||
"started_at": datetime.now(UTC),
|
||||
"error": f"执行超时,正在重试(第 {retry_count}/{_MAX_RUN_RETRIES} 次)",
|
||||
},
|
||||
)
|
||||
continue
|
||||
except Exception as exc:
|
||||
logger.exception("Scheduled task %s failed", task["task_id"])
|
||||
await self._fail_run(task, run_id, scheduled_for, error=str(exc), retry_count=retry_count)
|
||||
return
|
||||
|
||||
await self.store.update_run(
|
||||
run_row["id"],
|
||||
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result},
|
||||
run_id,
|
||||
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result, "retry_count": retry_count, "error": None},
|
||||
)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "succeeded"})
|
||||
await self._deliver_element_result(task, result["content"])
|
||||
except Exception as exc:
|
||||
logger.exception("Scheduled task %s failed", task["task_id"])
|
||||
result = _build_result(
|
||||
task,
|
||||
scheduled_for=scheduled_for,
|
||||
content=f"Scheduled task failed: {exc}",
|
||||
return
|
||||
|
||||
async def _run_agent_attempt(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
*,
|
||||
runtime_thread_id: str,
|
||||
scheduled_for: datetime,
|
||||
run_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the agent once and build the result payload.
|
||||
|
||||
Wrapped in :func:`asyncio.wait_for` by the caller so a hung agent loop
|
||||
cannot hold the shared run lock — and the whole scheduler — forever.
|
||||
"""
|
||||
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
|
||||
async with self._agent_run_lock:
|
||||
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
|
||||
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
|
||||
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
|
||||
if record.task is not None:
|
||||
await record.task
|
||||
result_text = await self._extract_latest_ai_text(runtime_thread_id)
|
||||
result_files = await self._extract_result_files(
|
||||
runtime_thread_id,
|
||||
task["user_id"],
|
||||
run_id=run_id,
|
||||
output_snapshot=output_snapshot,
|
||||
)
|
||||
await self.store.update_run(
|
||||
run_row["id"],
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": str(exc), "result": result},
|
||||
)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
|
||||
return _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
|
||||
|
||||
async def _ensure_thread_meta(self, user_id: str, thread_id: str) -> None:
|
||||
token = set_current_user(_RuntimeUser(id=user_id))
|
||||
|
||||
Reference in new issue
Block a user