feat: 定时任务管理 + 会话只读分享

定时任务:
- 管理员可查看/编辑/删除/触发全部用户的定时任务(新增 /api/scheduled-tasks/admin/* 接口与管理页)
- 卡死任务恢复:单次执行超 30 分钟超时重试(最多 3 次),耗尽后标记 failed;看门狗回收服务重启遗留的「执行中」任务

会话分享:
- 新增「只读链接」分享模式,生成公开免登录的只读对话快照页(/share/:code)
- 公开接口 /api/public/shares/* 提供对话快照与图片附件

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
吉鑫升andClaude Opus 4.7 committed 2026-05-18 15:39:40 +08:00
1 parent 4761677676
commit e0c1cb6581
45 files changed
+3235 -434

No files matched your search

@@ -0,0 +1,53 @@
"""Create thread_shares table (per-conversation share links).
Revision ID: 20260518_02
Revises: 20260518_01
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_02"
down_revision = "20260518_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_table("thread_shares"):
op.create_table(
"thread_shares",
sa.Column("share_code", sa.String(32), primary_key=True),
sa.Column("owner_user_id", sa.String(64), nullable=False),
sa.Column("thread_id", sa.String(64), nullable=False),
sa.Column("revoked", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_thread_shares_owner_user_id", "thread_shares", ["owner_user_id"])
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
elif not _has_column("thread_shares", "thread_id"):
# An earlier all-conversations variant of this table — add the column.
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
def downgrade() -> None:
if _has_table("thread_shares"):
op.drop_table("thread_shares")
@@ -0,0 +1,57 @@
"""Ensure thread_shares.thread_id exists (per-conversation share links).
Standalone migration so databases that already applied an earlier
all-conversations variant of ``20260518_02`` (which had no ``thread_id``
column) get the column added. Idempotent — a no-op when the column is
already present.
Revision ID: 20260518_03
Revises: 20260518_02
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_03"
down_revision = "20260518_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(ix["name"] == index_name for ix in inspector.get_indexes(table_name))
def upgrade() -> None:
if not _has_table("thread_shares"):
# Fresh database — 20260518_02 already created the table with thread_id.
return
if not _has_column("thread_shares", "thread_id"):
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
if not _has_index("thread_shares", "ix_thread_shares_thread_id"):
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
def downgrade() -> None:
# Leave the column in place — dropping it would break the share feature.
pass
@@ -0,0 +1,39 @@
"""Add retry_count to scheduled_task_runs.
Tracks how many recovery attempts a run has consumed so the scheduler can
cap retries on stuck/timed-out runs.
Revision ID: 20260518_04
Revises: 20260518_03
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_04"
down_revision = "20260518_03"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_column("scheduled_task_runs", "retry_count"):
op.add_column(
"scheduled_task_runs",
sa.Column("retry_count", sa.Integer(), nullable=False, server_default="0"),
)
def downgrade() -> None:
if _has_column("scheduled_task_runs", "retry_count"):
op.drop_column("scheduled_task_runs", "retry_count")
@@ -0,0 +1,44 @@
"""Add mode to thread_shares.
Distinguishes "import" shares (recipient copies the conversation into their
account) from "view" shares (a public, read-only static page).
Revision ID: 20260518_05
Revises: 20260518_04
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_05"
down_revision = "20260518_04"
branch_labels = None
depends_on = None
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
if table_name not in inspector.get_table_names():
return True # table missing entirely — skip (created with the column)
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_column("thread_shares", "mode"):
op.add_column(
"thread_shares",
sa.Column("mode", sa.String(16), nullable=False, server_default="import"),
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
if "thread_shares" in inspector.get_table_names() and any(
column["name"] == "mode" for column in inspector.get_columns("thread_shares")
):
op.drop_column("thread_shares", "mode")
@@ -32,6 +32,7 @@ from deerflow.persistence.scheduled_tasks.model import (
from deerflow.persistence.skills.model import SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.user.model import UserRow
__all__ = [
@@ -51,5 +52,6 @@ __all__ = [
"TagAssignmentRow",
"TagRow",
"ThreadMetaRow",
"ThreadShareRow",
"UserRow",
]
@@ -36,6 +36,35 @@ class ScheduledTaskStore(ABC):
async def due_tasks(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
raise NotImplementedError
# --- Admin (cross-user) ---
@abstractmethod
async def list_all_tasks(self) -> list[dict[str, Any]]:
"""Return every task across all users. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update a task by id without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def delete_task_any(self, task_id: str) -> bool:
"""Delete a task by id without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
"""Return runs for a task without restricting by owner. Admin-only callers."""
raise NotImplementedError
@abstractmethod
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
"""Return runs still in a non-terminal state whose ``started_at`` is older
than ``cutoff``. Used by the scheduler watchdog to recover runs that hung
or were orphaned by a service restart."""
raise NotImplementedError
@abstractmethod
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@@ -59,6 +59,37 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
due.sort(key=lambda task: task["next_run_at"])
return due[:limit]
# --- Admin (cross-user) ---
async def list_all_tasks(self) -> list[dict[str, Any]]:
return [dict(task) for task in self._tasks.values()]
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
task = self._tasks.get(task_id)
if task is None:
return None
task.update(data)
return dict(task)
async def delete_task_any(self, task_id: str) -> bool:
return self._tasks.pop(task_id, None) is not None
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
runs = [dict(run) for run in self._runs.values() if run["task_id"] == task_id]
runs.sort(key=lambda run: run.get("scheduled_for"), reverse=True)
return runs[:limit]
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
stuck = [
dict(run)
for run in self._runs.values()
if run.get("status") in ("running", "pending")
and run.get("started_at") is not None
and run["started_at"] <= cutoff
]
stuck.sort(key=lambda run: run["started_at"])
return stuck[:limit]
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
key = (data["task_id"], data["scheduled_for"].isoformat())
if key in self._runs_by_key:
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, Boolean, DateTime, String, Text, UniqueConstraint
from sqlalchemy import JSON, Boolean, DateTime, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -72,6 +72,10 @@ class ScheduledTaskRunRow(Base):
status: Mapped[str] = mapped_column(String(32), default="pending")
error: Mapped[str | None] = mapped_column(Text)
result_json: Mapped[dict | None] = mapped_column(JSON, default=dict)
# Number of recovery attempts made after the first try. A run is retried
# when it overruns the per-attempt timeout or is found orphaned in the
# "running" state (e.g. after a service restart). See ScheduledTaskService.
retry_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -124,6 +124,60 @@ class ScheduledTaskRepository(ScheduledTaskStore):
)
return [_to_dict(row) for row in result.scalars()]
# --- Admin (cross-user) ---
async def list_all_tasks(self) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(select(ScheduledTaskRow).order_by(ScheduledTaskRow.created_at.desc()))
return [_to_dict(row) for row in result.scalars()]
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ScheduledTaskRow, task_id)
if row is None:
return None
for key, value in data.items():
attr = "execution_context_json" if key == "execution_context" else key
if hasattr(row, attr):
setattr(row, attr, value)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def delete_task_any(self, task_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ScheduledTaskRow, task_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(ScheduledTaskRunRow)
.where(ScheduledTaskRunRow.task_id == task_id)
.order_by(ScheduledTaskRunRow.scheduled_for.desc())
.limit(limit)
)
return [_to_dict(row) for row in result.scalars()]
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(ScheduledTaskRunRow)
.where(
ScheduledTaskRunRow.status.in_(("running", "pending")),
ScheduledTaskRunRow.started_at.is_not(None),
ScheduledTaskRunRow.started_at <= cutoff,
)
.order_by(ScheduledTaskRunRow.started_at.asc())
.limit(limit)
)
return [_to_dict(row) for row in result.scalars()]
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
row = ScheduledTaskRunRow(
id=data["id"],
@@ -0,0 +1,19 @@
"""Persistence layer for conversation-history share links."""
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
from deerflow.persistence.thread_shares.memory import MemoryThreadShareStore
from deerflow.persistence.thread_shares.sql import ThreadShareRepository
__all__ = [
"ThreadShareStore",
"MemoryThreadShareStore",
"ThreadShareRepository",
"generate_share_code",
"make_thread_share_store",
]
def make_thread_share_store(session_factory):
if session_factory is None:
return MemoryThreadShareStore()
return ThreadShareRepository(session_factory)
@@ -0,0 +1,50 @@
"""Abstract persistence API for per-conversation share links."""
from __future__ import annotations
import secrets
from abc import ABC, abstractmethod
from typing import Any
# Unambiguous alphabet — no 0/O/1/I/L so codes are easy to read and type.
_CODE_ALPHABET = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"
def generate_share_code(length: int = 8) -> str:
"""Return a random, unguessable, human-readable share code."""
return "".join(secrets.choice(_CODE_ALPHABET) for _ in range(length))
class ThreadShareStore(ABC):
@abstractmethod
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
"""Create a new share link for one conversation and return its record.
``mode`` is ``"import"`` (recipient copies the conversation) or
``"view"`` (a public read-only static page).
"""
raise NotImplementedError
@abstractmethod
async def get(self, share_code: str) -> dict[str, Any] | None:
"""Look up a share by code regardless of owner (None if missing)."""
raise NotImplementedError
@abstractmethod
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
"""Return the owner's non-revoked share of ``mode`` for ``thread_id``, if any.
Used so re-sharing the same conversation in the same mode reuses its
existing code instead of minting a new one each time.
"""
raise NotImplementedError
@abstractmethod
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
"""List all share links created by ``owner_user_id`` (newest first)."""
raise NotImplementedError
@abstractmethod
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
"""Revoke a share link. Returns False if missing or not owned."""
raise NotImplementedError
@@ -0,0 +1,59 @@
"""In-memory ``ThreadShareStore`` implementation for tests / memory backend."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
class MemoryThreadShareStore(ThreadShareStore):
def __init__(self) -> None:
self._shares: dict[str, dict[str, Any]] = {}
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
code = generate_share_code()
while code in self._shares:
code = generate_share_code()
record = {
"share_code": code,
"owner_user_id": owner_user_id,
"thread_id": thread_id,
"revoked": False,
"mode": mode,
"created_at": datetime.now(UTC).isoformat(),
}
self._shares[code] = record
return copy.deepcopy(record)
async def get(self, share_code: str) -> dict[str, Any] | None:
record = self._shares.get(share_code)
return copy.deepcopy(record) if record is not None else None
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
matches = [
r
for r in self._shares.values()
if r["owner_user_id"] == owner_user_id
and r["thread_id"] == thread_id
and r.get("mode", "import") == mode
and not r["revoked"]
]
if not matches:
return None
matches.sort(key=lambda r: r["created_at"], reverse=True)
return copy.deepcopy(matches[0])
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
rows = [r for r in self._shares.values() if r["owner_user_id"] == owner_user_id]
rows.sort(key=lambda r: r["created_at"], reverse=True)
return [copy.deepcopy(r) for r in rows]
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
record = self._shares.get(share_code)
if record is None or record["owner_user_id"] != owner_user_id:
return False
record["revoked"] = True
return True
@@ -0,0 +1,36 @@
"""ORM model for conversation share links.
A share link is a short, revocable code created by a user for *one* of their
conversations. Redeeming it copies that single conversation into the redeemer's
account. The table only stores the link itself — the conversation is copied
lazily at import time.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class ThreadShareRow(Base):
__tablename__ = "thread_shares"
# Short, URL-safe, unguessable code embedded in the share link.
share_code: Mapped[str] = mapped_column(String(32), primary_key=True)
# The user who created the share.
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# The single conversation (thread) this link shares.
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# Soft-disable: a revoked link can no longer be previewed or imported.
revoked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
# Share mode:
# "import" — recipient (a logged-in user) copies the conversation into
# their own account and can keep chatting in the copy.
# "view" — a public, read-only static page anyone can open without
# logging in; viewers cannot send new messages.
mode: Mapped[str] = mapped_column(String(16), nullable=False, default="import", server_default="import")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
@@ -0,0 +1,80 @@
"""SQLAlchemy-backed per-conversation share-link repository."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
from deerflow.persistence.thread_shares.model import ThreadShareRow
def _row_to_dict(row: ThreadShareRow) -> dict[str, Any]:
data = row.to_dict()
created = data.get("created_at")
if isinstance(created, datetime):
data["created_at"] = created.isoformat()
return data
class ThreadShareRepository(ThreadShareStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
# Retry on the (astronomically unlikely) code collision.
async with self._sf() as session:
for _ in range(8):
code = generate_share_code()
if await session.get(ThreadShareRow, code) is not None:
continue
row = ThreadShareRow(
share_code=code,
owner_user_id=owner_user_id,
thread_id=thread_id,
revoked=False,
mode=mode,
)
session.add(row)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
raise RuntimeError("Failed to allocate a unique share code")
async def get(self, share_code: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ThreadShareRow, share_code)
return _row_to_dict(row) if row is not None else None
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
stmt = (
select(ThreadShareRow)
.where(
ThreadShareRow.owner_user_id == owner_user_id,
ThreadShareRow.thread_id == thread_id,
ThreadShareRow.mode == mode,
ThreadShareRow.revoked.is_(False),
)
.order_by(ThreadShareRow.created_at.desc())
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalars().first()
return _row_to_dict(row) if row is not None else None
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
stmt = select(ThreadShareRow).where(ThreadShareRow.owner_user_id == owner_user_id).order_by(ThreadShareRow.created_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ThreadShareRow, share_code)
if row is None or row.owner_user_id != owner_user_id:
return False
row.revoked = True
await session.commit()
return True
@@ -9,7 +9,7 @@ import re
import shutil
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any
@@ -27,6 +27,14 @@ from deerflow.utils.time import now_iso
logger = logging.getLogger(__name__)
# A single execution attempt may not exceed this wall-clock budget. When it
# does, the run is treated as stuck (the agent loop hung, a tool blocked, etc.)
# and retried. The watchdog uses the same window to recover runs orphaned by a
# crash or restart — those stay "running" in the DB with no in-process owner.
_RUN_ATTEMPT_TIMEOUT_SECONDS = 30 * 60 # 30 minutes
# Retries allowed after the first attempt before the run is marked "failed".
_MAX_RUN_RETRIES = 3
@dataclass
class _RuntimeUser:
@@ -398,7 +406,13 @@ class ScheduledTaskService:
)
return task
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
@staticmethod
def _build_task_update(data: dict[str, Any], existing: dict[str, Any] | None) -> dict[str, Any]:
"""Translate an API update payload into a storage update dict.
Shared by owner-scoped :meth:`update_task` and admin :meth:`admin_update_task`
so both apply identical schedule/timezone normalization.
"""
update_data: dict[str, Any] = {}
for key in ("name", "prompt", "enabled", "execution_agent_name"):
if key in data and data[key] is not None:
@@ -409,20 +423,74 @@ class ScheduledTaskService:
if data.get("schedule") is not None:
schedule = str(data["schedule"])
cron_expr = _schedule_to_cron(schedule)
tz = update_data.get("timezone")
if tz is None:
existing = await self.store.get_task(task_id, user_id)
tz = (existing or {}).get("timezone", "UTC")
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
update_data["schedule_text"] = schedule
update_data["cron_expr"] = cron_expr
update_data["next_run_at"] = _next_run_at(cron_expr, tz)
if data.get("execution_context") is not None:
update_data["execution_context"] = data["execution_context"]
# Re-enabling a task must also schedule it forward from now, otherwise a
# paused task with a stale/empty next_run_at would never fire again.
if update_data.get("enabled") is True and "next_run_at" not in update_data:
cron = (existing or {}).get("cron_expr")
if cron:
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
update_data["next_run_at"] = _next_run_at(cron, tz)
return update_data
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
existing = await self.store.get_task(task_id, user_id)
if existing is None:
return None
update_data = self._build_task_update(data, existing)
return await self.store.update_task(task_id, user_id, update_data)
async def delete_task(self, user_id: str, task_id: str) -> bool:
return await self.store.delete_task(task_id, user_id)
# --- Admin (cross-user management) ---
async def admin_list_tasks(self) -> list[dict[str, Any]]:
"""List every scheduled task across all users. Admin-only callers."""
tasks = await self.store.list_all_tasks()
for task in tasks:
task["role"] = "owner"
return tasks
async def admin_update_task(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update any task regardless of owner. Admin-only callers."""
existing = await self.store.get_task_any(task_id)
if existing is None:
return None
update_data = self._build_task_update(data, existing)
return await self.store.update_task_any(task_id, update_data)
async def admin_delete_task(self, task_id: str) -> bool:
"""Delete any task regardless of owner. Admin-only callers."""
return await self.store.delete_task_any(task_id)
async def admin_list_runs(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
"""List runs for any task regardless of owner. Admin-only callers."""
runs = await self.store.list_runs_any(task_id, limit=limit)
for run in runs:
run["can_edit"] = True
return runs
async def admin_get_run(self, run_id: str) -> dict[str, Any] | None:
"""Fetch a single run regardless of owner. Admin-only callers."""
run = await self.store.get_run_any(run_id)
if run is not None:
run["can_edit"] = True
return run
async def admin_trigger_task(self, task_id: str) -> dict[str, Any] | None:
"""Trigger an immediate run of any task regardless of owner."""
task = await self.store.get_task_any(task_id)
if task is None:
return None
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
return task
async def pause_task(self, user_id: str, task_id: str) -> dict[str, Any] | None:
return await self.store.update_task(task_id, user_id, {"enabled": False})
@@ -437,11 +505,19 @@ class ScheduledTaskService:
task = await self.store.get_task(task_id, user_id)
if task is None:
return None
asyncio.create_task(self._execute_task(task, scheduled_for=datetime.now(UTC), manual=True))
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
return task
async def _run_loop(self) -> None:
while not self._stopping.is_set():
# Recover hung / orphaned runs first so a single stuck task cannot
# silently wedge the scheduler indefinitely.
try:
await self._reconcile_stuck_runs()
except asyncio.CancelledError:
raise
except Exception:
logger.exception("Scheduled task watchdog scan failed")
try:
due = await self.store.due_tasks(datetime.now(UTC), limit=20)
for task in due:
@@ -466,64 +542,195 @@ class ScheduledTaskService:
finally:
self._running_task_ids.discard(task["task_id"])
async def _execute_task(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
async def _execute_tracked(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
"""Run a task while marking it busy so the watchdog won't double-drive it."""
self._running_task_ids.add(task["task_id"])
try:
await self._execute_task(task, scheduled_for=scheduled_for, manual=manual)
finally:
self._running_task_ids.discard(task["task_id"])
async def _reconcile_stuck_runs(self) -> None:
"""Recover runs left in the ``running`` state with no in-process owner.
A run becomes stuck when the agent loop hangs past the per-attempt
timeout, or when the process crashes / restarts mid-run (the original
asyncio task is gone but the DB row still says ``running``). For each
such run we retry up to ``_MAX_RUN_RETRIES`` times, then mark it
``failed`` so it no longer shows as perpetually executing.
"""
cutoff = datetime.now(UTC) - timedelta(seconds=_RUN_ATTEMPT_TIMEOUT_SECONDS)
stuck = await self.store.list_stuck_runs(cutoff, limit=20)
for run in stuck:
task_id = run["task_id"]
# A run still owned by a live in-process execution is handled by
# that execution's own timeout/retry loop — don't touch it here.
if task_id in self._running_task_ids:
continue
self._running_task_ids.add(task_id)
asyncio.create_task(self._recover_and_untrack(run))
async def _recover_and_untrack(self, run: dict[str, Any]) -> None:
try:
await self._recover_stuck_run(run)
except Exception:
logger.exception("Failed to recover stuck scheduled task run %s", run.get("id"))
finally:
self._running_task_ids.discard(run["task_id"])
async def _recover_stuck_run(self, run: dict[str, Any]) -> None:
run_id = run["id"]
retry_count = int(run.get("retry_count") or 0)
task = await self.store.get_task_any(run["task_id"])
scheduled_for = _coerce_datetime(run.get("scheduled_for")) or datetime.now(UTC)
if task is None:
# Parent task was deleted — just close out the orphaned run.
await self.store.update_run(
run_id,
{"status": "failed", "finished_at": datetime.now(UTC), "error": "执行超时:所属任务已被删除"},
)
return
if retry_count >= _MAX_RUN_RETRIES:
await self._fail_run(
task,
run_id,
scheduled_for,
error=f"执行超时:超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟未完成,重试 {_MAX_RUN_RETRIES} 次后仍失败",
retry_count=retry_count,
)
return
logger.warning("Recovering stuck scheduled task run %s (retry budget %s/%s)", run_id, retry_count, _MAX_RUN_RETRIES)
await self._execute_task(task, scheduled_for=scheduled_for, manual=False, run_row=run)
async def _fail_run(
self,
task: dict[str, Any],
run_id: str,
scheduled_for: datetime,
*,
error: str,
retry_count: int = 0,
) -> None:
"""Mark a run failed and reflect the outcome on the parent task."""
result = _build_result(task, scheduled_for=scheduled_for, content=f"Scheduled task failed: {error}")
await self.store.update_run(
run_id,
{"status": "failed", "finished_at": datetime.now(UTC), "error": error, "result": result, "retry_count": retry_count},
)
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
async def _execute_task(
self,
task: dict[str, Any],
*,
scheduled_for: datetime,
manual: bool,
run_row: dict[str, Any] | None = None,
) -> None:
# Each run uses its own throwaway thread so message history cannot accumulate
# across runs and blow past the model's input window. The per-user
# task["scheduler_thread_id"] is no longer used to drive the LLM call —
# we keep it on the task row only for backwards compatibility.
runtime_thread_id = str(uuid.uuid4())
run_row = await self.store.create_run_once(
{
"id": str(uuid.uuid4()),
"task_id": task["task_id"],
"user_id": task["user_id"],
"scheduler_thread_id": runtime_thread_id,
"scheduled_for": scheduled_for,
"status": "running",
"started_at": datetime.now(UTC),
}
)
is_recovery = run_row is not None
if run_row is None:
return
run_row = await self.store.create_run_once(
{
"id": str(uuid.uuid4()),
"task_id": task["task_id"],
"user_id": task["user_id"],
"scheduler_thread_id": str(uuid.uuid4()),
"scheduled_for": scheduled_for,
"status": "running",
"started_at": datetime.now(UTC),
}
)
if run_row is None:
return
else:
# Recovery path: re-claim an existing (orphaned) run row.
await self.store.update_run(run_row["id"], {"status": "running", "started_at": datetime.now(UTC)})
if not manual:
# Advancing the schedule only happens once, when the run is first
# created — recovery retries reuse the original scheduled_for.
if not manual and not is_recovery:
next_run_at = _next_run_at(task["cron_expr"], task.get("timezone") or "UTC", base=scheduled_for)
await self.store.update_task(task["task_id"], task["user_id"], {"next_run_at": next_run_at, "last_status": "running"})
try:
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
async with self._agent_run_lock:
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
await self.store.update_run(run_row["id"], {"agent_run_id": record.run_id})
if record.task is not None:
await record.task
result_text = await self._extract_latest_ai_text(runtime_thread_id)
result_files = await self._extract_result_files(
runtime_thread_id,
task["user_id"],
run_id=run_row["id"],
output_snapshot=output_snapshot,
run_id = run_row["id"]
retry_count = int(run_row.get("retry_count") or 0)
while True:
runtime_thread_id = str(uuid.uuid4())
try:
result = await asyncio.wait_for(
self._run_agent_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
timeout=_RUN_ATTEMPT_TIMEOUT_SECONDS,
)
result = _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
except TimeoutError:
# The attempt overran the 30-minute cap. Retry a bounded number
# of times, then give up and mark the run failed.
if retry_count >= _MAX_RUN_RETRIES:
await self._fail_run(
task,
run_id,
scheduled_for,
error=f"执行超时:单次执行超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟,重试 {_MAX_RUN_RETRIES} 次后仍失败",
retry_count=retry_count,
)
return
retry_count += 1
logger.warning("Scheduled task %s timed out; retry %s/%s", task["task_id"], retry_count, _MAX_RUN_RETRIES)
await self.store.update_run(
run_id,
{
"retry_count": retry_count,
"status": "running",
"started_at": datetime.now(UTC),
"error": f"执行超时,正在重试(第 {retry_count}/{_MAX_RUN_RETRIES} 次)",
},
)
continue
except Exception as exc:
logger.exception("Scheduled task %s failed", task["task_id"])
await self._fail_run(task, run_id, scheduled_for, error=str(exc), retry_count=retry_count)
return
await self.store.update_run(
run_row["id"],
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result},
run_id,
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result, "retry_count": retry_count, "error": None},
)
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "succeeded"})
await self._deliver_element_result(task, result["content"])
except Exception as exc:
logger.exception("Scheduled task %s failed", task["task_id"])
result = _build_result(
task,
scheduled_for=scheduled_for,
content=f"Scheduled task failed: {exc}",
return
async def _run_agent_attempt(
self,
task: dict[str, Any],
*,
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> dict[str, Any]:
"""Run the agent once and build the result payload.
Wrapped in :func:`asyncio.wait_for` by the caller so a hung agent loop
cannot hold the shared run lock — and the whole scheduler — forever.
"""
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
async with self._agent_run_lock:
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
if record.task is not None:
await record.task
result_text = await self._extract_latest_ai_text(runtime_thread_id)
result_files = await self._extract_result_files(
runtime_thread_id,
task["user_id"],
run_id=run_id,
output_snapshot=output_snapshot,
)
await self.store.update_run(
run_row["id"],
{"status": "failed", "finished_at": datetime.now(UTC), "error": str(exc), "result": result},
)
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
return _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
async def _ensure_thread_meta(self, user_id: str, thread_id: str) -> None:
token = set_current_user(_RuntimeUser(id=user_id))