Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into dhk-1
This commit is contained in:
commit
1e5e2581c8
86 files changed
+5887
-671
No files matched your search
+37
-15
@@ -43,9 +43,27 @@ logger = logging.getLogger(__name__)
|
||||
DEFAULT_IMAGE = "enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest"
|
||||
DEFAULT_PORT = 8080
|
||||
DEFAULT_CONTAINER_PREFIX = "deer-flow-sandbox"
|
||||
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
|
||||
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
|
||||
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
|
||||
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
|
||||
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
|
||||
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
|
||||
PACKAGE_MANAGER_ENV_KEYS = (
|
||||
"PIP_INDEX_URL",
|
||||
"PIP_EXTRA_INDEX_URL",
|
||||
"PIP_TRUSTED_HOST",
|
||||
"PIP_CERT",
|
||||
"PIP_CLIENT_CERT",
|
||||
"UV_INDEX_URL",
|
||||
"UV_EXTRA_INDEX_URL",
|
||||
"UV_DEFAULT_INDEX",
|
||||
"UV_INDEX",
|
||||
"UV_TRUSTED_HOST",
|
||||
"HTTP_PROXY",
|
||||
"HTTPS_PROXY",
|
||||
"NO_PROXY",
|
||||
"http_proxy",
|
||||
"https_proxy",
|
||||
"no_proxy",
|
||||
)
|
||||
|
||||
|
||||
def _lock_file_exclusive(lock_file) -> None:
|
||||
@@ -142,9 +160,9 @@ class AioSandboxProvider(SandboxProvider):
|
||||
Local provider manages container lifecycle directly (start/stop).
|
||||
"""
|
||||
provisioner_url = self._config.get("provisioner_url")
|
||||
if provisioner_url:
|
||||
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
|
||||
return RemoteSandboxBackend(provisioner_url=provisioner_url)
|
||||
if provisioner_url:
|
||||
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
|
||||
return RemoteSandboxBackend(provisioner_url=provisioner_url, environment=self._config["environment"])
|
||||
|
||||
logger.info("Using local container sandbox backend")
|
||||
return LocalContainerBackend(
|
||||
@@ -172,20 +190,24 @@ class AioSandboxProvider(SandboxProvider):
|
||||
"idle_timeout": idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT,
|
||||
"replicas": replicas if replicas is not None else DEFAULT_REPLICAS,
|
||||
"mounts": sandbox_config.mounts or [],
|
||||
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
|
||||
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
|
||||
# provisioner URL for dynamic pod management (e.g. http://provisioner:8002)
|
||||
"provisioner_url": getattr(sandbox_config, "provisioner_url", None) or "",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:
|
||||
"""Resolve environment variable references (values starting with $)."""
|
||||
resolved = {}
|
||||
for key, value in env_config.items():
|
||||
if isinstance(value, str) and value.startswith("$"):
|
||||
env_name = value[1:]
|
||||
resolved[key] = os.environ.get(env_name, "")
|
||||
else:
|
||||
def _resolve_env_vars(env_config: dict[str, str]) -> dict[str, str]:
|
||||
"""Resolve environment variable references (values starting with $)."""
|
||||
resolved = {}
|
||||
for key in PACKAGE_MANAGER_ENV_KEYS:
|
||||
value = os.environ.get(key)
|
||||
if value:
|
||||
resolved[key] = value
|
||||
for key, value in env_config.items():
|
||||
if isinstance(value, str) and value.startswith("$"):
|
||||
env_name = value[1:]
|
||||
resolved[key] = os.environ.get(env_name, "")
|
||||
else:
|
||||
resolved[key] = str(value)
|
||||
return resolved
|
||||
|
||||
|
||||
+16
-12
@@ -40,14 +40,17 @@ class RemoteSandboxBackend(SandboxBackend):
|
||||
provisioner_url: http://provisioner:8002
|
||||
"""
|
||||
|
||||
def __init__(self, provisioner_url: str):
|
||||
"""Initialize with the provisioner service URL.
|
||||
|
||||
Args:
|
||||
provisioner_url: URL of the provisioner service
|
||||
(e.g., ``http://provisioner:8002``).
|
||||
"""
|
||||
self._provisioner_url = provisioner_url.rstrip("/")
|
||||
def __init__(self, provisioner_url: str, environment: dict[str, str] | None = None):
|
||||
"""Initialize with the provisioner service URL.
|
||||
|
||||
Args:
|
||||
provisioner_url: URL of the provisioner service
|
||||
(e.g., ``http://provisioner:8002``).
|
||||
environment: Environment variables for sandbox pods. The remote
|
||||
provisioner may ignore this field if it does not support it.
|
||||
"""
|
||||
self._provisioner_url = provisioner_url.rstrip("/")
|
||||
self._environment = environment or {}
|
||||
|
||||
@property
|
||||
def provisioner_url(self) -> str:
|
||||
@@ -91,10 +94,11 @@ class RemoteSandboxBackend(SandboxBackend):
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self._provisioner_url}/api/sandboxes",
|
||||
json={
|
||||
"sandbox_id": sandbox_id,
|
||||
"thread_id": thread_id,
|
||||
},
|
||||
json={
|
||||
"sandbox_id": sandbox_id,
|
||||
"thread_id": thread_id,
|
||||
"environment": self._environment,
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
+47
-26
@@ -11,8 +11,13 @@ blocking, making a unified file safe for both workloads. Writers
|
||||
that contend for the lock wait via the default 5-second sqlite3
|
||||
busy timeout rather than failing immediately.
|
||||
|
||||
Postgres mode: both use the same database URL but maintain independent
|
||||
connection pools with different lifecycles.
|
||||
Postgres mode: both use the same database URL but maintain independent
|
||||
connection pools with different lifecycles.
|
||||
|
||||
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
|
||||
not have a built-in MySQL backend in this project, so deployments that choose
|
||||
MySQL for business tables should configure the legacy ``checkpointer:``
|
||||
section separately (usually sqlite) to preserve conversation checkpoints.
|
||||
|
||||
Memory mode: checkpointer uses MemorySaver, app uses in-memory stores.
|
||||
No database is initialized.
|
||||
@@ -21,12 +26,12 @@ Sensitive values (postgres_url) should use $VAR syntax in config.yaml
|
||||
to reference environment variables from .env:
|
||||
|
||||
database:
|
||||
backend: postgres
|
||||
postgres_url: $DATABASE_URL
|
||||
|
||||
The $VAR resolution is handled by AppConfig.resolve_env_variables()
|
||||
before this config is instantiated -- DatabaseConfig itself does not
|
||||
need to do any environment variable processing.
|
||||
backend: postgres
|
||||
postgres_url: $DATABASE_URL
|
||||
|
||||
The $VAR resolution is handled by AppConfig.resolve_env_variables()
|
||||
before this config is instantiated -- DatabaseConfig itself does not
|
||||
need to do any environment variable processing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -37,31 +42,40 @@ from typing import Literal
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class DatabaseConfig(BaseModel):
|
||||
backend: Literal["memory", "sqlite", "postgres"] = Field(
|
||||
default="memory",
|
||||
description=("Storage backend for both checkpointer and application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment."),
|
||||
)
|
||||
class DatabaseConfig(BaseModel):
|
||||
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
|
||||
default="memory",
|
||||
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
|
||||
)
|
||||
sqlite_dir: str = Field(
|
||||
default=".deer-flow/data",
|
||||
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
|
||||
)
|
||||
postgres_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"PostgreSQL connection URL, shared by checkpointer and app. "
|
||||
postgres_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"PostgreSQL connection URL, shared by checkpointer and app. "
|
||||
"Use $DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: postgresql://user:pass@host:5432/deerflow "
|
||||
"(the +asyncpg driver suffix is added automatically where needed)."
|
||||
),
|
||||
)
|
||||
),
|
||||
)
|
||||
mysql_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"MySQL connection URL for the application ORM. "
|
||||
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
|
||||
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
|
||||
),
|
||||
)
|
||||
echo_sql: bool = Field(
|
||||
default=False,
|
||||
description="Echo all SQL statements to log (debug only).",
|
||||
)
|
||||
pool_size: int = Field(
|
||||
default=5,
|
||||
description="Connection pool size for the app ORM engine (postgres only).",
|
||||
description="Connection pool size for the app ORM engine (postgres/mysql only).",
|
||||
)
|
||||
|
||||
# -- Derived helpers (not user-configured) --
|
||||
@@ -94,9 +108,16 @@ class DatabaseConfig(BaseModel):
|
||||
"""SQLAlchemy async URL for the application ORM engine."""
|
||||
if self.backend == "sqlite":
|
||||
return f"sqlite+aiosqlite:///{self.sqlite_path}"
|
||||
if self.backend == "postgres":
|
||||
url = self.postgres_url
|
||||
if url.startswith("postgresql://"):
|
||||
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||
return url
|
||||
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
|
||||
if self.backend == "postgres":
|
||||
url = self.postgres_url
|
||||
if url.startswith("postgresql://"):
|
||||
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||
return url
|
||||
if self.backend == "mysql":
|
||||
url = self.mysql_url
|
||||
if not url:
|
||||
raise ValueError("database.mysql_url is required for backend='mysql'")
|
||||
if url.startswith("mysql://"):
|
||||
url = url.replace("mysql://", "mysql+asyncmy://", 1)
|
||||
return url
|
||||
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
|
||||
+5
-4
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, String
|
||||
from sqlalchemy import Boolean, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class AgentRow(Base):
|
||||
@@ -16,9 +17,9 @@ class AgentRow(Base):
|
||||
# Stored as a plain string so the same id can be used as the filesystem
|
||||
# directory name under {base_dir}/agents/{id}/.
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False, index=True)
|
||||
name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -10,10 +10,13 @@ None and fall back to in-memory implementations.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
|
||||
def _json_serializer(obj: object) -> str:
|
||||
@@ -23,11 +26,37 @@ def _json_serializer(obj: object) -> str:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
|
||||
|
||||
The offline Docker image keeps the Python environment inside the image, while
|
||||
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
|
||||
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
|
||||
unchanged startup script can still import it after container recreation.
|
||||
"""
|
||||
try:
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
vendor_dir = runtime_home() / "python-vendor"
|
||||
except Exception:
|
||||
vendor_dir = Path(".deer-flow/python-vendor").resolve()
|
||||
|
||||
vendor = str(vendor_dir)
|
||||
if vendor_dir.exists() and vendor not in sys.path:
|
||||
sys.path.insert(0, vendor)
|
||||
|
||||
|
||||
def _mysql_driver_name(url: str) -> str:
|
||||
if url.startswith("mysql+aiomysql://"):
|
||||
return "aiomysql"
|
||||
return "asyncmy"
|
||||
|
||||
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
|
||||
|
||||
The target database name is extracted from *url*. The connection is
|
||||
@@ -65,8 +94,8 @@ async def init_engine(
|
||||
"""Create the async engine and session factory, then auto-create tables.
|
||||
|
||||
Args:
|
||||
backend: "memory", "sqlite", or "postgres".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres).
|
||||
backend: "memory", "sqlite", "postgres", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
|
||||
echo: Echo SQL to log.
|
||||
pool_size: Postgres connection pool size.
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
@@ -77,11 +106,20 @@ async def init_engine(
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
if find_spec(driver) is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
|
||||
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
|
||||
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
|
||||
)
|
||||
|
||||
if backend == "sqlite":
|
||||
import os
|
||||
@@ -112,16 +150,25 @@ async def init_engine(
|
||||
cursor.execute("PRAGMA foreign_keys=ON;")
|
||||
finally:
|
||||
cursor.close()
|
||||
elif backend == "postgres":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
elif backend == "postgres":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
elif backend == "mysql":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
@@ -162,10 +209,10 @@ async def init_engine_from_config(config) -> None:
|
||||
await init_engine("memory")
|
||||
return
|
||||
await init_engine(
|
||||
backend=config.backend,
|
||||
url=config.app_sqlalchemy_url,
|
||||
echo=config.echo_sql,
|
||||
pool_size=config.pool_size,
|
||||
backend=config.backend,
|
||||
url=config.app_sqlalchemy_url,
|
||||
echo=config.echo_sql,
|
||||
pool_size=config.pool_size,
|
||||
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
|
||||
)
|
||||
|
||||
|
||||
+3
-2
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, Text, UniqueConstraint
|
||||
from sqlalchemy import String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class FeedbackRow(Base):
|
||||
@@ -29,4 +30,4 @@ class FeedbackRow(Base):
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional text feedback from the user
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+3
-2
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Float, Integer, String, Text
|
||||
from sqlalchemy import Float, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class LlmCallMetricRow(Base):
|
||||
@@ -18,7 +19,7 @@ class LlmCallMetricRow(Base):
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Wall-clock time the LLM call ended. Indexed because admin views always
|
||||
# sort/filter by recency.
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
|
||||
+7
-6
@@ -44,12 +44,13 @@ def run_migrations_offline() -> None:
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection):
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=True, # Required for SQLite ALTER TABLE support
|
||||
)
|
||||
def do_run_migrations(connection):
|
||||
render_as_batch = connection.dialect.name == "sqlite"
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=render_as_batch, # Required for SQLite ALTER TABLE support
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
+8
-6
@@ -7,9 +7,11 @@ Create Date: 2026-05-12
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
import sqlalchemy as sa
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
|
||||
revision = "20260512_01"
|
||||
@@ -23,9 +25,9 @@ def _has_column(table_name: str, column_name: str) -> bool:
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_task_runs", "result_json"):
|
||||
op.add_column("scheduled_task_runs", sa.Column("result_json", sa.JSON(), nullable=True))
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_task_runs", "result_json"):
|
||||
op.add_column("scheduled_task_runs", sa.Column("result_json", PortableJSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ def upgrade() -> None:
|
||||
op.create_table(
|
||||
"agents",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("name", sa.String(191), nullable=False),
|
||||
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
|
||||
sa.Column("user_id", sa.String(64), nullable=True),
|
||||
sa.Column("published", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@ def upgrade() -> None:
|
||||
op.create_table(
|
||||
"tags",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("name", sa.String(159), nullable=False),
|
||||
sa.Column("tag_type", sa.String(32), nullable=False),
|
||||
sa.Column("color", sa.String(16), nullable=False, server_default="gray"),
|
||||
sa.Column("description", sa.String(1024), nullable=False, server_default=""),
|
||||
|
||||
+28
-18
@@ -76,29 +76,39 @@ def upgrade() -> None:
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
# 预置默认文章类型(幂等,跳过已存在的 key)
|
||||
# 预置默认文章类型(幂等,跳过已存在的 key)。
|
||||
# 用 SQLAlchemy Core 构造而非裸 SQL:``key`` 是 MySQL 保留字,裸
|
||||
# ``SELECT key`` 会触发 1064 语法错误。Core 编译器会按方言自动给保留字
|
||||
# 加引号(MySQL 反引号 / Postgres 双引号),保持跨后端可移植。
|
||||
article_types = sa.table(
|
||||
"ai_writing_article_types",
|
||||
sa.column("id", sa.String),
|
||||
sa.column("key", sa.String),
|
||||
sa.column("label", sa.String),
|
||||
sa.column("description", sa.Text),
|
||||
sa.column("prompt_hint", sa.Text),
|
||||
sa.column("default_word_count", sa.Integer),
|
||||
sa.column("sort_order", sa.Integer),
|
||||
sa.column("created_at", sa.DateTime),
|
||||
sa.column("updated_at", sa.DateTime),
|
||||
)
|
||||
conn = op.get_bind()
|
||||
existing_keys = {row[0] for row in conn.execute(sa.text("SELECT key FROM ai_writing_article_types")).fetchall()}
|
||||
existing_keys = {row[0] for row in conn.execute(sa.select(article_types.c.key)).fetchall()}
|
||||
now = datetime.now(UTC)
|
||||
for t in _DEFAULT_TYPES:
|
||||
if t["key"] not in existing_keys:
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"INSERT INTO ai_writing_article_types "
|
||||
"(id, key, label, description, prompt_hint, default_word_count, sort_order, created_at, updated_at) "
|
||||
"VALUES (:id, :key, :label, :description, :prompt_hint, :default_word_count, :sort_order, :created_at, :updated_at)"
|
||||
),
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"key": t["key"],
|
||||
"label": t["label"],
|
||||
"description": t["description"],
|
||||
"prompt_hint": t["prompt_hint"],
|
||||
"default_word_count": t["default_word_count"],
|
||||
"sort_order": t["sort_order"],
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
},
|
||||
article_types.insert().values(
|
||||
id=str(uuid.uuid4()),
|
||||
key=t["key"],
|
||||
label=t["label"],
|
||||
description=t["description"],
|
||||
prompt_hint=t["prompt_hint"],
|
||||
default_word_count=t["default_word_count"],
|
||||
sort_order=t["sort_order"],
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
"""Create thread_shares table (per-conversation share links).
|
||||
|
||||
Revision ID: 20260518_02
|
||||
Revises: 20260518_01
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_02"
|
||||
down_revision = "20260518_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("thread_shares"):
|
||||
op.create_table(
|
||||
"thread_shares",
|
||||
sa.Column("share_code", sa.String(32), primary_key=True),
|
||||
sa.Column("owner_user_id", sa.String(64), nullable=False),
|
||||
sa.Column("thread_id", sa.String(64), nullable=False),
|
||||
sa.Column("revoked", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_thread_shares_owner_user_id", "thread_shares", ["owner_user_id"])
|
||||
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
|
||||
elif not _has_column("thread_shares", "thread_id"):
|
||||
# An earlier all-conversations variant of this table — add the column.
|
||||
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
|
||||
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("thread_shares"):
|
||||
op.drop_table("thread_shares")
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
"""Ensure thread_shares.thread_id exists (per-conversation share links).
|
||||
|
||||
Standalone migration so databases that already applied an earlier
|
||||
all-conversations variant of ``20260518_02`` (which had no ``thread_id``
|
||||
column) get the column added. Idempotent — a no-op when the column is
|
||||
already present.
|
||||
|
||||
Revision ID: 20260518_03
|
||||
Revises: 20260518_02
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_03"
|
||||
down_revision = "20260518_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(ix["name"] == index_name for ix in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("thread_shares"):
|
||||
# Fresh database — 20260518_02 already created the table with thread_id.
|
||||
return
|
||||
if not _has_column("thread_shares", "thread_id"):
|
||||
op.add_column("thread_shares", sa.Column("thread_id", sa.String(64), nullable=False, server_default=""))
|
||||
if not _has_index("thread_shares", "ix_thread_shares_thread_id"):
|
||||
op.create_index("ix_thread_shares_thread_id", "thread_shares", ["thread_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Leave the column in place — dropping it would break the share feature.
|
||||
pass
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""Add retry_count to scheduled_task_runs.
|
||||
|
||||
Tracks how many recovery attempts a run has consumed so the scheduler can
|
||||
cap retries on stuck/timed-out runs.
|
||||
|
||||
Revision ID: 20260518_04
|
||||
Revises: 20260518_03
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_04"
|
||||
down_revision = "20260518_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("scheduled_task_runs", "retry_count"):
|
||||
op.add_column(
|
||||
"scheduled_task_runs",
|
||||
sa.Column("retry_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("scheduled_task_runs", "retry_count"):
|
||||
op.drop_column("scheduled_task_runs", "retry_count")
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""Add mode to thread_shares.
|
||||
|
||||
Distinguishes "import" shares (recipient copies the conversation into their
|
||||
account) from "view" shares (a public, read-only static page).
|
||||
|
||||
Revision ID: 20260518_05
|
||||
Revises: 20260518_04
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_05"
|
||||
down_revision = "20260518_04"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
if table_name not in inspector.get_table_names():
|
||||
return True # table missing entirely — skip (created with the column)
|
||||
return any(column["name"] == column_name for column in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_column("thread_shares", "mode"):
|
||||
op.add_column(
|
||||
"thread_shares",
|
||||
sa.Column("mode", sa.String(16), nullable=False, server_default="import"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if "thread_shares" in inspector.get_table_names() and any(
|
||||
column["name"] == "mode" for column in inspector.get_columns("thread_shares")
|
||||
):
|
||||
op.drop_column("thread_shares", "mode")
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
"""Merge the ai-writing and thread-shares migration heads.
|
||||
|
||||
Two independent migration branches diverged from ``20260515_01``: the AI
|
||||
writing feature (``20260517_01..03``) and the thread-shares / scheduled-task
|
||||
work (``20260518_01..05``). Merging the two branches into one repo left Alembic
|
||||
with multiple heads, so ``upgrade head`` failed. This is a no-op merge revision
|
||||
that joins both lineages — it changes no schema.
|
||||
|
||||
Revision ID: 20260518_06
|
||||
Revises: 20260517_03, 20260518_05
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
revision = "20260518_06"
|
||||
down_revision = ("20260517_03", "20260518_05")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
+2
@@ -32,6 +32,7 @@ from deerflow.persistence.scheduled_tasks.model import (
|
||||
from deerflow.persistence.skills.model import SkillRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
|
||||
__all__ = [
|
||||
@@ -51,5 +52,6 @@ __all__ = [
|
||||
"TagAssignmentRow",
|
||||
"TagRow",
|
||||
"ThreadMetaRow",
|
||||
"ThreadShareRow",
|
||||
"UserRow",
|
||||
]
|
||||
+4
-3
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy import Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class RunEventRow(Base):
|
||||
@@ -24,9 +25,9 @@ class RunEventRow(Base):
|
||||
category: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
# "message" | "trace" | "lifecycle"
|
||||
content: Mapped[str] = mapped_column(Text, default="")
|
||||
event_metadata: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
event_metadata: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
seq: Mapped[int] = mapped_column(nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("thread_id", "seq", name="uq_events_thread_seq"),
|
||||
|
||||
+4
-3
@@ -5,10 +5,11 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON, Boolean, DateTime, String, Text
|
||||
from sqlalchemy import Boolean, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class NotificationRow(Base):
|
||||
@@ -19,6 +20,6 @@ class NotificationRow(Base):
|
||||
type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
|
||||
body: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(PortableJSON(), nullable=False, default=dict)
|
||||
read: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), index=True)
|
||||
+4
-3
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text
|
||||
from sqlalchemy import Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class RecommendedQuestionRow(Base):
|
||||
@@ -17,5 +18,5 @@ class RecommendedQuestionRow(Base):
|
||||
title: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
content: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Index, String, Text
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class RunRow(Base):
|
||||
@@ -22,8 +23,8 @@ class RunRow(Base):
|
||||
|
||||
model_name: Mapped[str | None] = mapped_column(String(128))
|
||||
multitask_strategy: Mapped[str] = mapped_column(String(20), default="reject")
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
kwargs_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
kwargs_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
# Convenience fields (for listing pages without querying RunEventStore)
|
||||
@@ -43,7 +44,7 @@ class RunRow(Base):
|
||||
# Follow-up association
|
||||
follow_up_to_run_id: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
__table_args__ = (Index("ix_runs_thread_status", "thread_id", "status"),)
|
||||
+34
@@ -36,6 +36,35 @@ class ScheduledTaskStore(ABC):
|
||||
async def due_tasks(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Admin (cross-user) ---
|
||||
|
||||
@abstractmethod
|
||||
async def list_all_tasks(self) -> list[dict[str, Any]]:
|
||||
"""Return every task across all users. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update a task by id without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_task_any(self, task_id: str) -> bool:
|
||||
"""Delete a task by id without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""Return runs for a task without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""Return runs still in a non-terminal state whose ``started_at`` is older
|
||||
than ``cutoff``. Used by the scheduler watchdog to recover runs that hung
|
||||
or were orphaned by a service restart."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
@@ -56,6 +85,11 @@ class ScheduledTaskStore(ABC):
|
||||
async def delete_run(self, run_id: str, user_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_run_any(self, run_id: str) -> bool:
|
||||
"""Delete a run by id without restricting by owner. Admin-only callers."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
+43
-2
@@ -59,6 +59,37 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
|
||||
due.sort(key=lambda task: task["next_run_at"])
|
||||
return due[:limit]
|
||||
|
||||
# --- Admin (cross-user) ---
|
||||
|
||||
async def list_all_tasks(self) -> list[dict[str, Any]]:
|
||||
return [dict(task) for task in self._tasks.values()]
|
||||
|
||||
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
task = self._tasks.get(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
task.update(data)
|
||||
return dict(task)
|
||||
|
||||
async def delete_task_any(self, task_id: str) -> bool:
|
||||
return self._tasks.pop(task_id, None) is not None
|
||||
|
||||
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
runs = [dict(run) for run in self._runs.values() if run["task_id"] == task_id]
|
||||
runs.sort(key=lambda run: run.get("scheduled_for"), reverse=True)
|
||||
return runs[:limit]
|
||||
|
||||
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
stuck = [
|
||||
dict(run)
|
||||
for run in self._runs.values()
|
||||
if run.get("status") in ("running", "pending")
|
||||
and run.get("started_at") is not None
|
||||
and run["started_at"] <= cutoff
|
||||
]
|
||||
stuck.sort(key=lambda run: run["started_at"])
|
||||
return stuck[:limit]
|
||||
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
key = (data["task_id"], data["scheduled_for"].isoformat())
|
||||
if key in self._runs_by_key:
|
||||
@@ -90,11 +121,21 @@ class MemoryScheduledTaskStore(ScheduledTaskStore):
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["user_id"] != user_id:
|
||||
return False
|
||||
self._runs.pop(run_id, None)
|
||||
self._drop_run(run)
|
||||
return True
|
||||
|
||||
async def delete_run_any(self, run_id: str) -> bool:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return False
|
||||
self._drop_run(run)
|
||||
return True
|
||||
|
||||
def _drop_run(self, run: dict[str, Any]) -> None:
|
||||
self._runs.pop(run["id"], None)
|
||||
scheduled_for = run.get("scheduled_for")
|
||||
if hasattr(scheduled_for, "isoformat"):
|
||||
self._runs_by_key.discard((run["task_id"], scheduled_for.isoformat()))
|
||||
return True
|
||||
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
return dict(self._profiles.get(user_id) or {"user_id": user_id, "element_user_id": None})
|
||||
|
||||
+26
-21
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, Boolean, DateTime, String, Text, UniqueConstraint
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class SchedulerThreadRow(Base):
|
||||
@@ -15,8 +16,8 @@ class SchedulerThreadRow(Base):
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskRow(Base):
|
||||
@@ -29,20 +30,20 @@ class ScheduledTaskRow(Base):
|
||||
source_thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
source_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
execution_agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
name: Mapped[str] = mapped_column(String(127))
|
||||
prompt: Mapped[str] = mapped_column(Text)
|
||||
schedule_text: Mapped[str] = mapped_column(String(256))
|
||||
cron_expr: Mapped[str] = mapped_column(String(128))
|
||||
timezone: Mapped[str] = mapped_column(String(64), default="UTC")
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), index=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
last_status: Mapped[str | None] = mapped_column(String(32))
|
||||
execution_context_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
execution_context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
published: Mapped[bool] = mapped_column(Boolean, default=False, index=True)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskSubscriptionRow(Base):
|
||||
@@ -53,8 +54,8 @@ class ScheduledTaskSubscriptionRow(Base):
|
||||
task_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
notify_element: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskRunRow(Base):
|
||||
@@ -66,21 +67,25 @@ class ScheduledTaskRunRow(Base):
|
||||
user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
scheduler_thread_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
agent_run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
scheduled_for: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
scheduled_for: Mapped[datetime] = mapped_column(BeijingDateTime(), index=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending")
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
result_json: Mapped[dict | None] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
result_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
|
||||
# Number of recovery attempts made after the first try. A run is retried
|
||||
# when it overruns the per-attempt timeout or is found orphaned in the
|
||||
# "running" state (e.g. after a service restart). See ScheduledTaskService.
|
||||
retry_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ScheduledTaskDeliveryProfileRow(Base):
|
||||
__tablename__ = "scheduled_task_delivery_profiles"
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
element_user_id: Mapped[str | None] = mapped_column(String(256), index=True)
|
||||
element_user_id: Mapped[str | None] = mapped_column(String(191), index=True)
|
||||
element_room_id: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+71
-1
@@ -124,6 +124,60 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
# --- Admin (cross-user) ---
|
||||
|
||||
async def list_all_tasks(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(select(ScheduledTaskRow).order_by(ScheduledTaskRow.created_at.desc()))
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def update_task_any(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskRow, task_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key, value in data.items():
|
||||
attr = "execution_context_json" if key == "execution_context" else key
|
||||
if hasattr(row, attr):
|
||||
setattr(row, attr, value)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def delete_task_any(self, task_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskRow, task_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_runs_any(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(ScheduledTaskRunRow)
|
||||
.where(ScheduledTaskRunRow.task_id == task_id)
|
||||
.order_by(ScheduledTaskRunRow.scheduled_for.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_stuck_runs(self, cutoff: datetime, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(ScheduledTaskRunRow)
|
||||
.where(
|
||||
ScheduledTaskRunRow.status.in_(("running", "pending")),
|
||||
ScheduledTaskRunRow.started_at.is_not(None),
|
||||
ScheduledTaskRunRow.started_at <= cutoff,
|
||||
)
|
||||
.order_by(ScheduledTaskRunRow.started_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = ScheduledTaskRunRow(
|
||||
id=data["id"],
|
||||
@@ -180,6 +234,15 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def delete_run_any(self, run_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskRunRow, run_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledTaskDeliveryProfileRow, user_id)
|
||||
@@ -232,7 +295,14 @@ class ScheduledTaskRepository(ScheduledTaskStore):
|
||||
result = await session.execute(
|
||||
select(ScheduledTaskRow)
|
||||
.where(ScheduledTaskRow.published.is_(True))
|
||||
.order_by(ScheduledTaskRow.published_at.desc().nullslast(), ScheduledTaskRow.created_at.desc())
|
||||
# ``NULLS LAST`` is unsupported by MySQL — sort by an
|
||||
# ``IS NULL`` flag instead (False<True ⇒ non-null rows first),
|
||||
# which is portable across mysql/sqlite/postgres.
|
||||
.order_by(
|
||||
ScheduledTaskRow.published_at.is_(None),
|
||||
ScheduledTaskRow.published_at.desc(),
|
||||
ScheduledTaskRow.created_at.desc(),
|
||||
)
|
||||
.offset(max(0, offset))
|
||||
.limit(max(1, limit))
|
||||
)
|
||||
|
||||
+6
-5
@@ -14,10 +14,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, UniqueConstraint
|
||||
from sqlalchemy import String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
# Allowed tag/target types. Kept here so persistence + API share one source.
|
||||
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task")
|
||||
@@ -28,12 +29,12 @@ class TagRow(Base):
|
||||
__table_args__ = (UniqueConstraint("tag_type", "name", name="uq_tags_type_name"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(159), nullable=False)
|
||||
tag_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
color: Mapped[str] = mapped_column(String(16), nullable=False, default="gray")
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class TagAssignmentRow(Base):
|
||||
@@ -44,4 +45,4 @@ class TagAssignmentRow(Base):
|
||||
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
target_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
target_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
+5
-4
@@ -4,10 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, String
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class ThreadMetaRow(Base):
|
||||
@@ -18,6 +19,6 @@ class ThreadMetaRow(Base):
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
display_name: Mapped[str | None] = mapped_column(String(256))
|
||||
status: Mapped[str] = mapped_column(String(20), default="idle")
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"""Persistence layer for conversation-history share links."""
|
||||
|
||||
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
|
||||
from deerflow.persistence.thread_shares.memory import MemoryThreadShareStore
|
||||
from deerflow.persistence.thread_shares.sql import ThreadShareRepository
|
||||
|
||||
__all__ = [
|
||||
"ThreadShareStore",
|
||||
"MemoryThreadShareStore",
|
||||
"ThreadShareRepository",
|
||||
"generate_share_code",
|
||||
"make_thread_share_store",
|
||||
]
|
||||
|
||||
|
||||
def make_thread_share_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryThreadShareStore()
|
||||
return ThreadShareRepository(session_factory)
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
"""Abstract persistence API for per-conversation share links."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
# Unambiguous alphabet — no 0/O/1/I/L so codes are easy to read and type.
|
||||
_CODE_ALPHABET = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"
|
||||
|
||||
|
||||
def generate_share_code(length: int = 8) -> str:
|
||||
"""Return a random, unguessable, human-readable share code."""
|
||||
return "".join(secrets.choice(_CODE_ALPHABET) for _ in range(length))
|
||||
|
||||
|
||||
class ThreadShareStore(ABC):
|
||||
@abstractmethod
|
||||
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
|
||||
"""Create a new share link for one conversation and return its record.
|
||||
|
||||
``mode`` is ``"import"`` (recipient copies the conversation) or
|
||||
``"view"`` (a public read-only static page).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get(self, share_code: str) -> dict[str, Any] | None:
|
||||
"""Look up a share by code regardless of owner (None if missing)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
|
||||
"""Return the owner's non-revoked share of ``mode`` for ``thread_id``, if any.
|
||||
|
||||
Used so re-sharing the same conversation in the same mode reuses its
|
||||
existing code instead of minting a new one each time.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
|
||||
"""List all share links created by ``owner_user_id`` (newest first)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
|
||||
"""Revoke a share link. Returns False if missing or not owned."""
|
||||
raise NotImplementedError
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
"""In-memory ``ThreadShareStore`` implementation for tests / memory backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
|
||||
|
||||
|
||||
class MemoryThreadShareStore(ThreadShareStore):
|
||||
def __init__(self) -> None:
|
||||
self._shares: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
|
||||
code = generate_share_code()
|
||||
while code in self._shares:
|
||||
code = generate_share_code()
|
||||
record = {
|
||||
"share_code": code,
|
||||
"owner_user_id": owner_user_id,
|
||||
"thread_id": thread_id,
|
||||
"revoked": False,
|
||||
"mode": mode,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
self._shares[code] = record
|
||||
return copy.deepcopy(record)
|
||||
|
||||
async def get(self, share_code: str) -> dict[str, Any] | None:
|
||||
record = self._shares.get(share_code)
|
||||
return copy.deepcopy(record) if record is not None else None
|
||||
|
||||
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
|
||||
matches = [
|
||||
r
|
||||
for r in self._shares.values()
|
||||
if r["owner_user_id"] == owner_user_id
|
||||
and r["thread_id"] == thread_id
|
||||
and r.get("mode", "import") == mode
|
||||
and not r["revoked"]
|
||||
]
|
||||
if not matches:
|
||||
return None
|
||||
matches.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return copy.deepcopy(matches[0])
|
||||
|
||||
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._shares.values() if r["owner_user_id"] == owner_user_id]
|
||||
rows.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return [copy.deepcopy(r) for r in rows]
|
||||
|
||||
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
|
||||
record = self._shares.get(share_code)
|
||||
if record is None or record["owner_user_id"] != owner_user_id:
|
||||
return False
|
||||
record["revoked"] = True
|
||||
return True
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"""ORM model for conversation share links.
|
||||
|
||||
A share link is a short, revocable code created by a user for *one* of their
|
||||
conversations. Redeeming it copies that single conversation into the redeemer's
|
||||
account. The table only stores the link itself — the conversation is copied
|
||||
lazily at import time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ThreadShareRow(Base):
|
||||
__tablename__ = "thread_shares"
|
||||
|
||||
# Short, URL-safe, unguessable code embedded in the share link.
|
||||
share_code: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
# The user who created the share.
|
||||
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# The single conversation (thread) this link shares.
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# Soft-disable: a revoked link can no longer be previewed or imported.
|
||||
revoked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
# Share mode:
|
||||
# "import" — recipient (a logged-in user) copies the conversation into
|
||||
# their own account and can keep chatting in the copy.
|
||||
# "view" — a public, read-only static page anyone can open without
|
||||
# logging in; viewers cannot send new messages.
|
||||
mode: Mapped[str] = mapped_column(String(16), nullable=False, default="import", server_default="import")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
"""SQLAlchemy-backed per-conversation share-link repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.thread_shares.base import ThreadShareStore, generate_share_code
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
|
||||
|
||||
def _row_to_dict(row: ThreadShareRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
created = data.get("created_at")
|
||||
if isinstance(created, datetime):
|
||||
data["created_at"] = created.isoformat()
|
||||
return data
|
||||
|
||||
|
||||
class ThreadShareRepository(ThreadShareStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def create(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any]:
|
||||
# Retry on the (astronomically unlikely) code collision.
|
||||
async with self._sf() as session:
|
||||
for _ in range(8):
|
||||
code = generate_share_code()
|
||||
if await session.get(ThreadShareRow, code) is not None:
|
||||
continue
|
||||
row = ThreadShareRow(
|
||||
share_code=code,
|
||||
owner_user_id=owner_user_id,
|
||||
thread_id=thread_id,
|
||||
revoked=False,
|
||||
mode=mode,
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
raise RuntimeError("Failed to allocate a unique share code")
|
||||
|
||||
async def get(self, share_code: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ThreadShareRow, share_code)
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
|
||||
async def get_active_for_thread(self, owner_user_id: str, thread_id: str, *, mode: str = "import") -> dict[str, Any] | None:
|
||||
stmt = (
|
||||
select(ThreadShareRow)
|
||||
.where(
|
||||
ThreadShareRow.owner_user_id == owner_user_id,
|
||||
ThreadShareRow.thread_id == thread_id,
|
||||
ThreadShareRow.mode == mode,
|
||||
ThreadShareRow.revoked.is_(False),
|
||||
)
|
||||
.order_by(ThreadShareRow.created_at.desc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
|
||||
async def list_for_owner(self, owner_user_id: str) -> list[dict[str, Any]]:
|
||||
stmt = select(ThreadShareRow).where(ThreadShareRow.owner_user_id == owner_user_id).order_by(ThreadShareRow.created_at.desc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def revoke(self, share_code: str, owner_user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ThreadShareRow, share_code)
|
||||
if row is None or row.owner_user_id != owner_user_id:
|
||||
return False
|
||||
row.revoked = True
|
||||
await session.commit()
|
||||
return True
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Portable SQLAlchemy column types."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Text
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
|
||||
# China Standard Time. China has observed UTC+8 with no DST since 1991, so a
|
||||
# fixed offset is exact and avoids depending on the `tzdata` package (which the
|
||||
# stdlib `zoneinfo` needs on Windows / minimal containers).
|
||||
BEIJING_TZ = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
class PortableJSON(TypeDecorator):
|
||||
"""Use native JSON where available, and TEXT on older MySQL servers."""
|
||||
|
||||
impl = JSON
|
||||
cache_ok = True
|
||||
|
||||
def load_dialect_impl(self, dialect):
|
||||
if dialect.name == "mysql":
|
||||
return dialect.type_descriptor(Text())
|
||||
return dialect.type_descriptor(JSON())
|
||||
|
||||
def process_bind_param(self, value: Any, dialect) -> Any:
|
||||
if dialect.name != "mysql" or value is None:
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
def process_result_value(self, value: Any, dialect) -> Any:
|
||||
if dialect.name != "mysql" or value is None or not isinstance(value, str):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
class BeijingDateTime(TypeDecorator):
|
||||
"""Store timestamps as Beijing (UTC+8) wall-clock time.
|
||||
|
||||
The application works in UTC internally (``datetime.now(UTC)`` everywhere).
|
||||
MySQL ``DATETIME`` and SQLite have no timezone, so the driver serializes the
|
||||
wall-clock fields of whatever datetime it is given and drops ``tzinfo``.
|
||||
Without conversion a row stores the UTC wall clock and an operator browsing
|
||||
the table sees a time 8 hours behind Beijing.
|
||||
|
||||
This decorator converts UTC-aware values to Beijing on the way in (stored as
|
||||
naive Beijing wall-clock) and re-attaches the Beijing zone on the way out, so
|
||||
the database holds Beijing time while Python keeps receiving correct
|
||||
timezone-aware datetimes — instant comparisons stay correct either way.
|
||||
|
||||
PostgreSQL has a real ``TIMESTAMP WITH TIME ZONE`` type and stores the
|
||||
instant unambiguously, so values pass through unchanged there.
|
||||
"""
|
||||
|
||||
impl = DateTime
|
||||
cache_ok = True
|
||||
|
||||
def load_dialect_impl(self, dialect):
|
||||
# timezone=True keeps PostgreSQL on timestamptz; MySQL/SQLite ignore it.
|
||||
return dialect.type_descriptor(DateTime(timezone=True))
|
||||
|
||||
def process_bind_param(self, value: Any, dialect) -> Any:
|
||||
if not isinstance(value, datetime):
|
||||
return value
|
||||
if dialect.name == "postgresql":
|
||||
return value
|
||||
if value.tzinfo is None:
|
||||
# Naive values follow the codebase convention of being UTC.
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
|
||||
|
||||
def process_result_value(self, value: Any, dialect) -> Any:
|
||||
if not isinstance(value, datetime):
|
||||
return value
|
||||
if dialect.name == "postgresql":
|
||||
return value
|
||||
if value.tzinfo is None:
|
||||
# Stored as naive Beijing wall-clock; re-attach the zone.
|
||||
return value.replace(tzinfo=BEIJING_TZ)
|
||||
return value.astimezone(BEIJING_TZ)
|
||||
+4
-3
@@ -13,10 +13,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Index, String, text
|
||||
from sqlalchemy import Boolean, Index, String, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class UserRow(Base):
|
||||
@@ -25,7 +26,7 @@ class UserRow(Base):
|
||||
# UUIDs are stored as 36-char strings for cross-backend portability.
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False, index=True)
|
||||
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
|
||||
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
|
||||
@@ -33,7 +34,7 @@ class UserRow(Base):
|
||||
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
+1
@@ -87,6 +87,7 @@ class RunEventStore(abc.ABC):
|
||||
limit: int = 50,
|
||||
before_seq: int | None = None,
|
||||
after_seq: int | None = None,
|
||||
user_id: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Return displayable messages (category=message) for a specific run, ordered by seq ascending.
|
||||
|
||||
|
||||
+3
-1
@@ -152,7 +152,9 @@ class JsonlRunEventStore(RunEventStore):
|
||||
events = [e for e in events if e.get("event_type") in event_types]
|
||||
return events[:limit]
|
||||
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None):
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
|
||||
# JSONL events carry no per-event owner; user_id is accepted for
|
||||
# interface parity with DbRunEventStore but ignored.
|
||||
events = self._read_run_events(thread_id, run_id)
|
||||
filtered = [e for e in events if e.get("category") == "message"]
|
||||
if before_seq is not None:
|
||||
|
||||
+3
-1
@@ -97,7 +97,9 @@ class MemoryRunEventStore(RunEventStore):
|
||||
filtered = [e for e in filtered if e["event_type"] in event_types]
|
||||
return filtered[:limit]
|
||||
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None):
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
|
||||
# The in-memory store keeps no per-event owner, so user_id is accepted
|
||||
# for interface parity with DbRunEventStore but ignored.
|
||||
all_events = self._events.get(thread_id, [])
|
||||
filtered = [e for e in all_events if e["run_id"] == run_id and e["category"] == "message"]
|
||||
if before_seq is not None:
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
from .registry import get_scheduled_task_service, set_scheduled_task_service
|
||||
from .service import ScheduledTaskService
|
||||
from .service import DuplicateTaskNameError, ScheduledTaskService
|
||||
|
||||
__all__ = ["ScheduledTaskService", "get_scheduled_task_service", "set_scheduled_task_service"]
|
||||
__all__ = ["DuplicateTaskNameError", "ScheduledTaskService", "get_scheduled_task_service", "set_scheduled_task_service"]
|
||||
+313
-57
@@ -9,7 +9,7 @@ import re
|
||||
import shutil
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
@@ -27,6 +27,27 @@ from deerflow.utils.time import now_iso
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# A single execution attempt may not exceed this wall-clock budget. When it
|
||||
# does, the run is treated as stuck (the agent loop hung, a tool blocked, etc.)
|
||||
# and retried. The watchdog uses the same window to recover runs orphaned by a
|
||||
# crash or restart — those stay "running" in the DB with no in-process owner.
|
||||
_RUN_ATTEMPT_TIMEOUT_SECONDS = 30 * 60 # 30 minutes
|
||||
# Retries allowed after the first attempt before the run is marked "failed".
|
||||
_MAX_RUN_RETRIES = 3
|
||||
|
||||
|
||||
class DuplicateTaskNameError(Exception):
|
||||
"""A user already owns a scheduled task with the requested name.
|
||||
|
||||
Task names are unique per user (``uq_scheduled_tasks_user_name``). The
|
||||
service raises this instead of letting the raw DB ``IntegrityError`` bubble
|
||||
up so the API layer can return a friendly 409 rather than a 500.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str) -> None:
|
||||
self.name = name
|
||||
super().__init__(f"A scheduled task named {name!r} already exists")
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RuntimeUser:
|
||||
@@ -337,15 +358,20 @@ class ScheduledTaskService:
|
||||
"kind": item.get("kind"),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _remove_run_archive(run_id: str) -> None:
|
||||
"""Best-effort cleanup of a run's on-disk result archive."""
|
||||
archive_dir = get_paths().base_dir / "scheduled-task-runs" / run_id
|
||||
if archive_dir.exists():
|
||||
try:
|
||||
shutil.rmtree(archive_dir)
|
||||
except Exception:
|
||||
logger.warning("Failed to delete scheduled task run archive %s", archive_dir, exc_info=True)
|
||||
|
||||
async def delete_run(self, user_id: str, run_id: str) -> bool:
|
||||
deleted = await self.store.delete_run(run_id, user_id)
|
||||
if deleted:
|
||||
archive_dir = get_paths().base_dir / "scheduled-task-runs" / run_id
|
||||
if archive_dir.exists():
|
||||
try:
|
||||
shutil.rmtree(archive_dir)
|
||||
except Exception:
|
||||
logger.warning("Failed to delete scheduled task run archive %s", archive_dir, exc_info=True)
|
||||
self._remove_run_archive(run_id)
|
||||
return True
|
||||
candidate = await self.store.get_run_any(run_id)
|
||||
if candidate is not None and await self.store.get_subscription(candidate["task_id"], user_id) is not None:
|
||||
@@ -358,6 +384,20 @@ class ScheduledTaskService:
|
||||
async def update_delivery_profile(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
return await self.store.update_delivery_profile(user_id, data)
|
||||
|
||||
async def _assert_name_available(self, user_id: str, name: str, *, exclude_task_id: str | None = None) -> None:
|
||||
"""Raise :class:`DuplicateTaskNameError` if ``user_id`` already owns ``name``.
|
||||
|
||||
Enforces ``uq_scheduled_tasks_user_name`` proactively so callers get a
|
||||
clean conflict error instead of a raw DB ``IntegrityError``. The check
|
||||
compares the trimmed name; ``exclude_task_id`` skips the task being
|
||||
renamed so an unchanged name on update is not flagged as a collision.
|
||||
"""
|
||||
for existing in await self.store.list_tasks(user_id):
|
||||
if exclude_task_id is not None and existing.get("task_id") == exclude_task_id:
|
||||
continue
|
||||
if (existing.get("name") or "").strip() == name:
|
||||
raise DuplicateTaskNameError(name)
|
||||
|
||||
async def create_task(
|
||||
self,
|
||||
*,
|
||||
@@ -374,6 +414,8 @@ class ScheduledTaskService:
|
||||
tz = _normalize_timezone(timezone)
|
||||
cron_expr = _schedule_to_cron(schedule)
|
||||
next_run_at = _next_run_at(cron_expr, tz)
|
||||
final_name = name.strip() or "Scheduled task"
|
||||
await self._assert_name_available(user_id, final_name)
|
||||
scheduler_thread = await self.ensure_scheduler_thread(user_id)
|
||||
context = dict(execution_context or {})
|
||||
if execution_agent_name:
|
||||
@@ -386,7 +428,7 @@ class ScheduledTaskService:
|
||||
"source_thread_id": source_thread_id,
|
||||
"source_agent_name": source_agent_name,
|
||||
"execution_agent_name": execution_agent_name,
|
||||
"name": name.strip() or "Scheduled task",
|
||||
"name": final_name,
|
||||
"prompt": prompt.strip(),
|
||||
"schedule_text": schedule.strip(),
|
||||
"cron_expr": cron_expr,
|
||||
@@ -398,31 +440,106 @@ class ScheduledTaskService:
|
||||
)
|
||||
return task
|
||||
|
||||
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
@staticmethod
|
||||
def _build_task_update(data: dict[str, Any], existing: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Translate an API update payload into a storage update dict.
|
||||
|
||||
Shared by owner-scoped :meth:`update_task` and admin :meth:`admin_update_task`
|
||||
so both apply identical schedule/timezone normalization.
|
||||
"""
|
||||
update_data: dict[str, Any] = {}
|
||||
for key in ("name", "prompt", "enabled", "execution_agent_name"):
|
||||
if key in data and data[key] is not None:
|
||||
update_data[key] = data[key]
|
||||
# Names are matched against the per-user uniqueness constraint, so keep
|
||||
# the stored value trimmed to match create_task's normalization.
|
||||
if isinstance(update_data.get("name"), str):
|
||||
update_data["name"] = update_data["name"].strip() or "Scheduled task"
|
||||
timezone = data.get("timezone")
|
||||
if timezone is not None:
|
||||
update_data["timezone"] = _normalize_timezone(timezone)
|
||||
if data.get("schedule") is not None:
|
||||
schedule = str(data["schedule"])
|
||||
cron_expr = _schedule_to_cron(schedule)
|
||||
tz = update_data.get("timezone")
|
||||
if tz is None:
|
||||
existing = await self.store.get_task(task_id, user_id)
|
||||
tz = (existing or {}).get("timezone", "UTC")
|
||||
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
|
||||
update_data["schedule_text"] = schedule
|
||||
update_data["cron_expr"] = cron_expr
|
||||
update_data["next_run_at"] = _next_run_at(cron_expr, tz)
|
||||
if data.get("execution_context") is not None:
|
||||
update_data["execution_context"] = data["execution_context"]
|
||||
# Re-enabling a task must also schedule it forward from now, otherwise a
|
||||
# paused task with a stale/empty next_run_at would never fire again.
|
||||
if update_data.get("enabled") is True and "next_run_at" not in update_data:
|
||||
cron = (existing or {}).get("cron_expr")
|
||||
if cron:
|
||||
tz = update_data.get("timezone") or (existing or {}).get("timezone") or "UTC"
|
||||
update_data["next_run_at"] = _next_run_at(cron, tz)
|
||||
return update_data
|
||||
|
||||
async def update_task(self, user_id: str, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
existing = await self.store.get_task(task_id, user_id)
|
||||
if existing is None:
|
||||
return None
|
||||
update_data = self._build_task_update(data, existing)
|
||||
if "name" in update_data:
|
||||
await self._assert_name_available(user_id, update_data["name"], exclude_task_id=task_id)
|
||||
return await self.store.update_task(task_id, user_id, update_data)
|
||||
|
||||
async def delete_task(self, user_id: str, task_id: str) -> bool:
|
||||
return await self.store.delete_task(task_id, user_id)
|
||||
|
||||
# --- Admin (cross-user management) ---
|
||||
|
||||
async def admin_list_tasks(self) -> list[dict[str, Any]]:
|
||||
"""List every scheduled task across all users. Admin-only callers."""
|
||||
tasks = await self.store.list_all_tasks()
|
||||
for task in tasks:
|
||||
task["role"] = "owner"
|
||||
return tasks
|
||||
|
||||
async def admin_update_task(self, task_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update any task regardless of owner. Admin-only callers."""
|
||||
existing = await self.store.get_task_any(task_id)
|
||||
if existing is None:
|
||||
return None
|
||||
update_data = self._build_task_update(data, existing)
|
||||
if "name" in update_data:
|
||||
await self._assert_name_available(existing["user_id"], update_data["name"], exclude_task_id=task_id)
|
||||
return await self.store.update_task_any(task_id, update_data)
|
||||
|
||||
async def admin_delete_task(self, task_id: str) -> bool:
|
||||
"""Delete any task regardless of owner. Admin-only callers."""
|
||||
return await self.store.delete_task_any(task_id)
|
||||
|
||||
async def admin_delete_run(self, run_id: str) -> bool:
|
||||
"""Delete a single run regardless of owner. Admin-only callers."""
|
||||
deleted = await self.store.delete_run_any(run_id)
|
||||
if deleted:
|
||||
self._remove_run_archive(run_id)
|
||||
return deleted
|
||||
|
||||
async def admin_list_runs(self, task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""List runs for any task regardless of owner. Admin-only callers."""
|
||||
runs = await self.store.list_runs_any(task_id, limit=limit)
|
||||
for run in runs:
|
||||
run["can_edit"] = True
|
||||
return runs
|
||||
|
||||
async def admin_get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""Fetch a single run regardless of owner. Admin-only callers."""
|
||||
run = await self.store.get_run_any(run_id)
|
||||
if run is not None:
|
||||
run["can_edit"] = True
|
||||
return run
|
||||
|
||||
async def admin_trigger_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""Trigger an immediate run of any task regardless of owner."""
|
||||
task = await self.store.get_task_any(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
|
||||
return task
|
||||
|
||||
async def pause_task(self, user_id: str, task_id: str) -> dict[str, Any] | None:
|
||||
return await self.store.update_task(task_id, user_id, {"enabled": False})
|
||||
|
||||
@@ -437,11 +554,19 @@ class ScheduledTaskService:
|
||||
task = await self.store.get_task(task_id, user_id)
|
||||
if task is None:
|
||||
return None
|
||||
asyncio.create_task(self._execute_task(task, scheduled_for=datetime.now(UTC), manual=True))
|
||||
asyncio.create_task(self._execute_tracked(task, scheduled_for=datetime.now(UTC), manual=True))
|
||||
return task
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
while not self._stopping.is_set():
|
||||
# Recover hung / orphaned runs first so a single stuck task cannot
|
||||
# silently wedge the scheduler indefinitely.
|
||||
try:
|
||||
await self._reconcile_stuck_runs()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Scheduled task watchdog scan failed")
|
||||
try:
|
||||
due = await self.store.due_tasks(datetime.now(UTC), limit=20)
|
||||
for task in due:
|
||||
@@ -466,64 +591,195 @@ class ScheduledTaskService:
|
||||
finally:
|
||||
self._running_task_ids.discard(task["task_id"])
|
||||
|
||||
async def _execute_task(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
|
||||
async def _execute_tracked(self, task: dict[str, Any], *, scheduled_for: datetime, manual: bool) -> None:
|
||||
"""Run a task while marking it busy so the watchdog won't double-drive it."""
|
||||
self._running_task_ids.add(task["task_id"])
|
||||
try:
|
||||
await self._execute_task(task, scheduled_for=scheduled_for, manual=manual)
|
||||
finally:
|
||||
self._running_task_ids.discard(task["task_id"])
|
||||
|
||||
async def _reconcile_stuck_runs(self) -> None:
|
||||
"""Recover runs left in the ``running`` state with no in-process owner.
|
||||
|
||||
A run becomes stuck when the agent loop hangs past the per-attempt
|
||||
timeout, or when the process crashes / restarts mid-run (the original
|
||||
asyncio task is gone but the DB row still says ``running``). For each
|
||||
such run we retry up to ``_MAX_RUN_RETRIES`` times, then mark it
|
||||
``failed`` so it no longer shows as perpetually executing.
|
||||
"""
|
||||
cutoff = datetime.now(UTC) - timedelta(seconds=_RUN_ATTEMPT_TIMEOUT_SECONDS)
|
||||
stuck = await self.store.list_stuck_runs(cutoff, limit=20)
|
||||
for run in stuck:
|
||||
task_id = run["task_id"]
|
||||
# A run still owned by a live in-process execution is handled by
|
||||
# that execution's own timeout/retry loop — don't touch it here.
|
||||
if task_id in self._running_task_ids:
|
||||
continue
|
||||
self._running_task_ids.add(task_id)
|
||||
asyncio.create_task(self._recover_and_untrack(run))
|
||||
|
||||
async def _recover_and_untrack(self, run: dict[str, Any]) -> None:
|
||||
try:
|
||||
await self._recover_stuck_run(run)
|
||||
except Exception:
|
||||
logger.exception("Failed to recover stuck scheduled task run %s", run.get("id"))
|
||||
finally:
|
||||
self._running_task_ids.discard(run["task_id"])
|
||||
|
||||
async def _recover_stuck_run(self, run: dict[str, Any]) -> None:
|
||||
run_id = run["id"]
|
||||
retry_count = int(run.get("retry_count") or 0)
|
||||
task = await self.store.get_task_any(run["task_id"])
|
||||
scheduled_for = _coerce_datetime(run.get("scheduled_for")) or datetime.now(UTC)
|
||||
if task is None:
|
||||
# Parent task was deleted — just close out the orphaned run.
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": "执行超时:所属任务已被删除"},
|
||||
)
|
||||
return
|
||||
if retry_count >= _MAX_RUN_RETRIES:
|
||||
await self._fail_run(
|
||||
task,
|
||||
run_id,
|
||||
scheduled_for,
|
||||
error=f"执行超时:超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟未完成,重试 {_MAX_RUN_RETRIES} 次后仍失败",
|
||||
retry_count=retry_count,
|
||||
)
|
||||
return
|
||||
logger.warning("Recovering stuck scheduled task run %s (retry budget %s/%s)", run_id, retry_count, _MAX_RUN_RETRIES)
|
||||
await self._execute_task(task, scheduled_for=scheduled_for, manual=False, run_row=run)
|
||||
|
||||
async def _fail_run(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
run_id: str,
|
||||
scheduled_for: datetime,
|
||||
*,
|
||||
error: str,
|
||||
retry_count: int = 0,
|
||||
) -> None:
|
||||
"""Mark a run failed and reflect the outcome on the parent task."""
|
||||
result = _build_result(task, scheduled_for=scheduled_for, content=f"Scheduled task failed: {error}")
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": error, "result": result, "retry_count": retry_count},
|
||||
)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
|
||||
|
||||
async def _execute_task(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
*,
|
||||
scheduled_for: datetime,
|
||||
manual: bool,
|
||||
run_row: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
# Each run uses its own throwaway thread so message history cannot accumulate
|
||||
# across runs and blow past the model's input window. The per-user
|
||||
# task["scheduler_thread_id"] is no longer used to drive the LLM call —
|
||||
# we keep it on the task row only for backwards compatibility.
|
||||
runtime_thread_id = str(uuid.uuid4())
|
||||
run_row = await self.store.create_run_once(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"task_id": task["task_id"],
|
||||
"user_id": task["user_id"],
|
||||
"scheduler_thread_id": runtime_thread_id,
|
||||
"scheduled_for": scheduled_for,
|
||||
"status": "running",
|
||||
"started_at": datetime.now(UTC),
|
||||
}
|
||||
)
|
||||
is_recovery = run_row is not None
|
||||
if run_row is None:
|
||||
return
|
||||
run_row = await self.store.create_run_once(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"task_id": task["task_id"],
|
||||
"user_id": task["user_id"],
|
||||
"scheduler_thread_id": str(uuid.uuid4()),
|
||||
"scheduled_for": scheduled_for,
|
||||
"status": "running",
|
||||
"started_at": datetime.now(UTC),
|
||||
}
|
||||
)
|
||||
if run_row is None:
|
||||
return
|
||||
else:
|
||||
# Recovery path: re-claim an existing (orphaned) run row.
|
||||
await self.store.update_run(run_row["id"], {"status": "running", "started_at": datetime.now(UTC)})
|
||||
|
||||
if not manual:
|
||||
# Advancing the schedule only happens once, when the run is first
|
||||
# created — recovery retries reuse the original scheduled_for.
|
||||
if not manual and not is_recovery:
|
||||
next_run_at = _next_run_at(task["cron_expr"], task.get("timezone") or "UTC", base=scheduled_for)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"next_run_at": next_run_at, "last_status": "running"})
|
||||
|
||||
try:
|
||||
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
|
||||
async with self._agent_run_lock:
|
||||
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
|
||||
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
|
||||
await self.store.update_run(run_row["id"], {"agent_run_id": record.run_id})
|
||||
if record.task is not None:
|
||||
await record.task
|
||||
result_text = await self._extract_latest_ai_text(runtime_thread_id)
|
||||
result_files = await self._extract_result_files(
|
||||
runtime_thread_id,
|
||||
task["user_id"],
|
||||
run_id=run_row["id"],
|
||||
output_snapshot=output_snapshot,
|
||||
run_id = run_row["id"]
|
||||
retry_count = int(run_row.get("retry_count") or 0)
|
||||
|
||||
while True:
|
||||
runtime_thread_id = str(uuid.uuid4())
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
self._run_agent_attempt(task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id),
|
||||
timeout=_RUN_ATTEMPT_TIMEOUT_SECONDS,
|
||||
)
|
||||
result = _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
|
||||
except TimeoutError:
|
||||
# The attempt overran the 30-minute cap. Retry a bounded number
|
||||
# of times, then give up and mark the run failed.
|
||||
if retry_count >= _MAX_RUN_RETRIES:
|
||||
await self._fail_run(
|
||||
task,
|
||||
run_id,
|
||||
scheduled_for,
|
||||
error=f"执行超时:单次执行超过 {_RUN_ATTEMPT_TIMEOUT_SECONDS // 60} 分钟,重试 {_MAX_RUN_RETRIES} 次后仍失败",
|
||||
retry_count=retry_count,
|
||||
)
|
||||
return
|
||||
retry_count += 1
|
||||
logger.warning("Scheduled task %s timed out; retry %s/%s", task["task_id"], retry_count, _MAX_RUN_RETRIES)
|
||||
await self.store.update_run(
|
||||
run_id,
|
||||
{
|
||||
"retry_count": retry_count,
|
||||
"status": "running",
|
||||
"started_at": datetime.now(UTC),
|
||||
"error": f"执行超时,正在重试(第 {retry_count}/{_MAX_RUN_RETRIES} 次)",
|
||||
},
|
||||
)
|
||||
continue
|
||||
except Exception as exc:
|
||||
logger.exception("Scheduled task %s failed", task["task_id"])
|
||||
await self._fail_run(task, run_id, scheduled_for, error=str(exc), retry_count=retry_count)
|
||||
return
|
||||
|
||||
await self.store.update_run(
|
||||
run_row["id"],
|
||||
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result},
|
||||
run_id,
|
||||
{"status": "succeeded", "finished_at": datetime.now(UTC), "result": result, "retry_count": retry_count, "error": None},
|
||||
)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "succeeded"})
|
||||
await self._deliver_element_result(task, result["content"])
|
||||
except Exception as exc:
|
||||
logger.exception("Scheduled task %s failed", task["task_id"])
|
||||
result = _build_result(
|
||||
task,
|
||||
scheduled_for=scheduled_for,
|
||||
content=f"Scheduled task failed: {exc}",
|
||||
return
|
||||
|
||||
async def _run_agent_attempt(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
*,
|
||||
runtime_thread_id: str,
|
||||
scheduled_for: datetime,
|
||||
run_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the agent once and build the result payload.
|
||||
|
||||
Wrapped in :func:`asyncio.wait_for` by the caller so a hung agent loop
|
||||
cannot hold the shared run lock — and the whole scheduler — forever.
|
||||
"""
|
||||
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
|
||||
async with self._agent_run_lock:
|
||||
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"])
|
||||
record = await self._start_agent_run(task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
|
||||
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
|
||||
if record.task is not None:
|
||||
await record.task
|
||||
result_text = await self._extract_latest_ai_text(runtime_thread_id)
|
||||
result_files = await self._extract_result_files(
|
||||
runtime_thread_id,
|
||||
task["user_id"],
|
||||
run_id=run_id,
|
||||
output_snapshot=output_snapshot,
|
||||
)
|
||||
await self.store.update_run(
|
||||
run_row["id"],
|
||||
{"status": "failed", "finished_at": datetime.now(UTC), "error": str(exc), "result": result},
|
||||
)
|
||||
await self.store.update_task(task["task_id"], task["user_id"], {"last_run_at": datetime.now(UTC), "last_status": "failed"})
|
||||
return _build_result(task, scheduled_for=scheduled_for, content=result_text, files=result_files)
|
||||
|
||||
async def _ensure_thread_meta(self, user_id: str, thread_id: str) -> None:
|
||||
token = set_current_user(_RuntimeUser(id=user_id))
|
||||
|
||||
@@ -40,13 +40,14 @@ dependencies = [
|
||||
|
||||
[project.optional-dependencies]
|
||||
ollama = ["langchain-ollama>=0.3.0"]
|
||||
postgres = [
|
||||
"asyncpg>=0.29",
|
||||
"langgraph-checkpoint-postgres>=3.0.5",
|
||||
"psycopg[binary]>=3.3.3",
|
||||
"psycopg-pool>=3.3.0",
|
||||
]
|
||||
pymupdf = ["pymupdf4llm>=0.0.17"]
|
||||
postgres = [
|
||||
"asyncpg>=0.29",
|
||||
"langgraph-checkpoint-postgres>=3.0.5",
|
||||
"psycopg[binary]>=3.3.3",
|
||||
"psycopg-pool>=3.3.0",
|
||||
]
|
||||
mysql = ["asyncmy>=0.2.10"]
|
||||
pymupdf = ["pymupdf4llm>=0.0.17"]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
|
||||
Reference in new issue
Block a user