Initial commit: deerflow-server (frontend-web + offline-backend-20260512)

This commit is contained in:
吉鑫升 committed 2026-05-13 09:36:15 +08:00
commit e7a455f22e
560 files changed
+101331

No files matched your search

@@ -0,0 +1,25 @@
"""Scheduled task persistence."""
from __future__ import annotations
from deerflow.persistence.scheduled_tasks.base import ScheduledTaskStore
from deerflow.persistence.scheduled_tasks.memory import MemoryScheduledTaskStore
from deerflow.persistence.scheduled_tasks.model import ScheduledTaskDeliveryProfileRow, ScheduledTaskRow, ScheduledTaskRunRow, SchedulerThreadRow
from deerflow.persistence.scheduled_tasks.sql import ScheduledTaskRepository
__all__ = [
"ScheduledTaskRepository",
"ScheduledTaskDeliveryProfileRow",
"MemoryScheduledTaskStore",
"ScheduledTaskRow",
"ScheduledTaskRunRow",
"ScheduledTaskStore",
"SchedulerThreadRow",
"make_scheduled_task_store",
]
def make_scheduled_task_store(session_factory):
if session_factory is None:
return MemoryScheduledTaskStore()
return ScheduledTaskRepository(session_factory)
@@ -0,0 +1,65 @@
"""Abstract scheduled-task persistence API."""
from __future__ import annotations
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any
class ScheduledTaskStore(ABC):
@abstractmethod
async def get_or_create_scheduler_thread(self, user_id: str, thread_id_factory) -> dict:
raise NotImplementedError
@abstractmethod
async def list_tasks(self, user_id: str) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update_task(self, task_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def delete_task(self, task_id: str, user_id: str) -> bool:
raise NotImplementedError
@abstractmethod
async def due_tasks(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def update_run(self, run_id: str, data: dict[str, Any]) -> None:
raise NotImplementedError
@abstractmethod
async def list_runs(self, task_id: str, user_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_run(self, run_id: str, user_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def delete_run(self, run_id: str, user_id: str) -> bool:
raise NotImplementedError
@abstractmethod
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update_delivery_profile(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@@ -0,0 +1,107 @@
"""In-memory scheduled-task store for memory persistence mode."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from deerflow.persistence.scheduled_tasks.base import ScheduledTaskStore
class MemoryScheduledTaskStore(ScheduledTaskStore):
def __init__(self) -> None:
self._threads: dict[str, dict[str, Any]] = {}
self._tasks: dict[str, dict[str, Any]] = {}
self._runs_by_key: set[tuple[str, str]] = set()
self._runs: dict[str, dict[str, Any]] = {}
self._profiles: dict[str, dict[str, Any]] = {}
async def get_or_create_scheduler_thread(self, user_id: str, thread_id_factory) -> dict:
if user_id not in self._threads:
self._threads[user_id] = {"user_id": user_id, "thread_id": thread_id_factory()}
return dict(self._threads[user_id])
async def list_tasks(self, user_id: str) -> list[dict[str, Any]]:
return [dict(task) for task in self._tasks.values() if task["user_id"] == user_id]
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
task = self._tasks.get(task_id)
if task is None or task["user_id"] != user_id:
return None
return dict(task)
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
self._tasks[data["task_id"]] = dict(data)
return dict(data)
async def update_task(self, task_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
task = self._tasks.get(task_id)
if task is None or task["user_id"] != user_id:
return None
task.update(data)
return dict(task)
async def delete_task(self, task_id: str, user_id: str) -> bool:
task = self._tasks.get(task_id)
if task is None or task["user_id"] != user_id:
return False
self._tasks.pop(task_id, None)
return True
async def due_tasks(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
due = [
dict(task)
for task in self._tasks.values()
if task.get("enabled") and task.get("next_run_at") is not None and task["next_run_at"] <= now
]
due.sort(key=lambda task: task["next_run_at"])
return due[:limit]
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
key = (data["task_id"], data["scheduled_for"].isoformat())
if key in self._runs_by_key:
return None
self._runs_by_key.add(key)
self._runs[data["id"]] = dict(data)
return dict(data)
async def update_run(self, run_id: str, data: dict[str, Any]) -> None:
if run_id in self._runs:
self._runs[run_id].update(data)
async def list_runs(self, task_id: str, user_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
runs = [
dict(run)
for run in self._runs.values()
if run["task_id"] == task_id and run["user_id"] == user_id
]
runs.sort(key=lambda run: run.get("scheduled_for"), reverse=True)
return runs[:limit]
async def get_run(self, run_id: str, user_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None or run["user_id"] != user_id:
return None
return dict(run)
async def delete_run(self, run_id: str, user_id: str) -> bool:
run = self._runs.get(run_id)
if run is None or run["user_id"] != user_id:
return False
self._runs.pop(run_id, None)
scheduled_for = run.get("scheduled_for")
if hasattr(scheduled_for, "isoformat"):
self._runs_by_key.discard((run["task_id"], scheduled_for.isoformat()))
return True
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
return dict(self._profiles.get(user_id) or {"user_id": user_id, "element_user_id": None})
async def update_delivery_profile(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
profile = self._profiles.setdefault(user_id, {"user_id": user_id, "element_user_id": None})
if "element_user_id" in data:
element_user_id = data["element_user_id"]
profile["element_user_id"] = element_user_id.strip() if isinstance(element_user_id, str) and element_user_id.strip() else None
if "element_room_id" in data:
profile["element_room_id"] = data["element_room_id"] or None
return dict(profile)
@@ -0,0 +1,72 @@
"""ORM models for user scheduled tasks."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, Boolean, DateTime, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class SchedulerThreadRow(Base):
__tablename__ = "scheduler_threads"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
thread_id: Mapped[str] = mapped_column(String(64), unique=True, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskRow(Base):
__tablename__ = "scheduled_tasks"
__table_args__ = (UniqueConstraint("user_id", "name", name="uq_scheduled_tasks_user_name"),)
task_id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), index=True)
scheduler_thread_id: Mapped[str] = mapped_column(String(64), index=True)
source_thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
source_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
execution_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
name: Mapped[str] = mapped_column(String(256))
prompt: Mapped[str] = mapped_column(Text)
schedule_text: Mapped[str] = mapped_column(String(256))
cron_expr: Mapped[str] = mapped_column(String(128))
timezone: Mapped[str] = mapped_column(String(64), default="UTC")
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_status: Mapped[str | None] = mapped_column(String(32))
execution_context_json: Mapped[dict] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskRunRow(Base):
__tablename__ = "scheduled_task_runs"
__table_args__ = (UniqueConstraint("task_id", "scheduled_for", name="uq_scheduled_task_runs_task_scheduled_for"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64), index=True)
user_id: Mapped[str] = mapped_column(String(64), index=True)
scheduler_thread_id: Mapped[str] = mapped_column(String(64), index=True)
agent_run_id: Mapped[str | None] = mapped_column(String(64), index=True)
scheduled_for: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
status: Mapped[str] = mapped_column(String(32), default="pending")
error: Mapped[str | None] = mapped_column(Text)
result_json: Mapped[dict | None] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class ScheduledTaskDeliveryProfileRow(Base):
__tablename__ = "scheduled_task_delivery_profiles"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
element_user_id: Mapped[str | None] = mapped_column(String(256), index=True)
element_room_id: Mapped[str | None] = mapped_column(String(256), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,196 @@
"""SQLAlchemy repository for scheduled tasks."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.scheduled_tasks.base import ScheduledTaskStore
from deerflow.persistence.scheduled_tasks.model import ScheduledTaskDeliveryProfileRow, ScheduledTaskRow, ScheduledTaskRunRow, SchedulerThreadRow
def _to_dict(row) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at", "next_run_at", "last_run_at", "scheduled_for", "started_at", "finished_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
if "execution_context_json" in data:
data["execution_context"] = data.pop("execution_context_json") or {}
if "result_json" in data:
data["result"] = data.pop("result_json") or None
return data
class ScheduledTaskRepository(ScheduledTaskStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get_or_create_scheduler_thread(self, user_id: str, thread_id_factory) -> dict:
async with self._sf() as session:
row = await session.get(SchedulerThreadRow, user_id)
if row is not None:
return _to_dict(row)
row = SchedulerThreadRow(user_id=user_id, thread_id=thread_id_factory())
session.add(row)
try:
await session.commit()
except IntegrityError:
await session.rollback()
row = await session.get(SchedulerThreadRow, user_id)
if row is None:
raise
await session.refresh(row)
return _to_dict(row)
async def list_tasks(self, user_id: str) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(select(ScheduledTaskRow).where(ScheduledTaskRow.user_id == user_id).order_by(ScheduledTaskRow.created_at.desc()))
return [_to_dict(row) for row in result.scalars()]
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ScheduledTaskRow, task_id)
if row is None or row.user_id != user_id:
return None
return _to_dict(row)
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
row = ScheduledTaskRow(
task_id=data["task_id"],
user_id=data["user_id"],
scheduler_thread_id=data["scheduler_thread_id"],
source_thread_id=data.get("source_thread_id"),
source_agent_name=data.get("source_agent_name"),
execution_agent_name=data.get("execution_agent_name"),
name=data["name"],
prompt=data["prompt"],
schedule_text=data["schedule_text"],
cron_expr=data["cron_expr"],
timezone=data["timezone"],
enabled=data.get("enabled", True),
next_run_at=data.get("next_run_at"),
last_status=data.get("last_status"),
execution_context_json=data.get("execution_context") or {},
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def update_task(self, task_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ScheduledTaskRow, task_id)
if row is None or row.user_id != user_id:
return None
for key, value in data.items():
attr = "execution_context_json" if key == "execution_context" else key
if hasattr(row, attr):
setattr(row, attr, value)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def delete_task(self, task_id: str, user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ScheduledTaskRow, task_id)
if row is None or row.user_id != user_id:
return False
await session.delete(row)
await session.commit()
return True
async def due_tasks(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(ScheduledTaskRow)
.where(ScheduledTaskRow.enabled.is_(True), ScheduledTaskRow.next_run_at.is_not(None), ScheduledTaskRow.next_run_at <= now)
.order_by(ScheduledTaskRow.next_run_at.asc())
.limit(limit)
)
return [_to_dict(row) for row in result.scalars()]
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
row = ScheduledTaskRunRow(
id=data["id"],
task_id=data["task_id"],
user_id=data["user_id"],
scheduler_thread_id=data["scheduler_thread_id"],
scheduled_for=data["scheduled_for"],
status=data.get("status", "pending"),
started_at=data.get("started_at"),
result_json=data.get("result") or None,
)
async with self._sf() as session:
session.add(row)
try:
await session.commit()
except IntegrityError:
await session.rollback()
return None
await session.refresh(row)
return _to_dict(row)
async def update_run(self, run_id: str, data: dict[str, Any]) -> None:
async with self._sf() as session:
values = dict(data)
if "result" in values:
values["result_json"] = values.pop("result")
values["updated_at"] = datetime.now(UTC)
await session.execute(update(ScheduledTaskRunRow).where(ScheduledTaskRunRow.id == run_id).values(**values))
await session.commit()
async def list_runs(self, task_id: str, user_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(ScheduledTaskRunRow)
.where(ScheduledTaskRunRow.task_id == task_id, ScheduledTaskRunRow.user_id == user_id)
.order_by(ScheduledTaskRunRow.scheduled_for.desc())
.limit(limit)
)
return [_to_dict(row) for row in result.scalars()]
async def get_run(self, run_id: str, user_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ScheduledTaskRunRow, run_id)
if row is None or row.user_id != user_id:
return None
return _to_dict(row)
async def delete_run(self, run_id: str, user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ScheduledTaskRunRow, run_id)
if row is None or row.user_id != user_id:
return False
await session.delete(row)
await session.commit()
return True
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(ScheduledTaskDeliveryProfileRow, user_id)
if row is None:
return {"user_id": user_id, "element_user_id": None}
return _to_dict(row)
async def update_delivery_profile(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(ScheduledTaskDeliveryProfileRow, user_id)
if row is None:
row = ScheduledTaskDeliveryProfileRow(user_id=user_id)
session.add(row)
if "element_user_id" in data:
element_user_id = data["element_user_id"]
row.element_user_id = element_user_id.strip() if isinstance(element_user_id, str) and element_user_id.strip() else None
if "element_room_id" in data:
row.element_room_id = data["element_room_id"] or None
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _to_dict(row)