This commit is contained in:
董宏凯 committed 2026-06-07 21:25:54 +08:00
commit 05d14bb364
32 files changed
+2290 -507

No files matched your search

@@ -0,0 +1,5 @@
"""Persistence models for admin analytics snapshots."""
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
__all__ = ["AdminLeaderboardDailyStatRow"]
@@ -0,0 +1,31 @@
"""ORM models for admin analytics snapshots."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class AdminLeaderboardDailyStatRow(Base):
"""One precomputed leaderboard snapshot for a single local calendar day."""
__tablename__ = "admin_leaderboard_daily_stats"
__table_args__ = (
Index("ix_admin_leaderboard_daily_stats_status", "status"),
Index("ix_admin_leaderboard_daily_stats_updated_at", "updated_at"),
)
stat_date: Mapped[str] = mapped_column(String(10), primary_key=True)
settings_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
status: Mapped[str] = mapped_column(String(16), default="ready")
payload_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
message: Mapped[str | None] = mapped_column(Text)
error: Mapped[str | None] = mapped_column(Text)
generated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -11,16 +11,16 @@ None and fall back to in-memory implementations.
from __future__ import annotations
import asyncio
import json
import logging
import os
import sys
from importlib.util import find_spec
from pathlib import Path
from sqlalchemy import text
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
import json
import logging
import os
import sys
from importlib.util import find_spec
from pathlib import Path
from sqlalchemy import text
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
def _json_serializer(obj: object) -> str:
@@ -133,7 +133,7 @@ async def _run_alembic_upgrade(url: str) -> None:
logger.exception("Auto-applying Alembic migrations failed; continuing with create_all schema")
async def _auto_create_postgres_db(url: str) -> None:
async def _auto_create_postgres_db(url: str) -> None:
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
The target database name is extracted from *url*. The connection is
@@ -156,55 +156,57 @@ async def _auto_create_postgres_db(url: str) -> None:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL database: %s", db_name)
finally:
await maint_engine.dispose()
async def _auto_create_mysql_db(url: str) -> None:
"""Create the target MySQL database when it is missing.
``create_all`` can create tables, but MySQL requires the database/schema to
exist before the application URL can connect. We connect to the built-in
``mysql`` maintenance database on the same server and issue a conservative
``CREATE DATABASE IF NOT EXISTS``.
"""
parsed = make_url(url)
db_name = parsed.database
if not db_name:
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
maint_url = parsed.set(database="mysql")
maint_engine = create_async_engine(
maint_url.render_as_string(hide_password=False),
isolation_level="AUTOCOMMIT",
pool_pre_ping=True,
)
quoted = "`" + db_name.replace("`", "``") + "`"
try:
async with maint_engine.connect() as conn:
await conn.execute(
text(
f"CREATE DATABASE IF NOT EXISTS {quoted} "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
)
)
logger.info("Auto-created MySQL database if missing: %s", db_name)
finally:
await maint_engine.dispose()
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
async def init_engine(
finally:
await maint_engine.dispose()
async def _auto_create_mysql_db(url: str) -> None:
"""Create the target MySQL database when it is missing.
``create_all`` can create tables, but MySQL requires the database/schema to
exist before the application URL can connect. We connect to the built-in
``mysql`` maintenance database on the same server and issue a conservative
``CREATE DATABASE IF NOT EXISTS``.
"""
parsed = make_url(url)
db_name = parsed.database
if not db_name:
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
maint_url = parsed.set(database="mysql")
maint_engine = create_async_engine(
maint_url.render_as_string(hide_password=False),
isolation_level="AUTOCOMMIT",
pool_pre_ping=True,
)
quoted = "`" + db_name.replace("`", "``") + "`"
try:
async with maint_engine.connect() as conn:
await conn.execute(
text(
f"CREATE DATABASE IF NOT EXISTS {quoted} "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
)
)
logger.info("Auto-created MySQL database if missing: %s", db_name)
finally:
await maint_engine.dispose()
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
async def init_engine(
backend: str,
*,
url: str = "",
echo: bool = False,
pool_size: int = 5,
max_overflow: int = 10,
pool_timeout: int = 30,
sqlite_dir: str = "",
) -> None:
"""Create the async engine and session factory, then auto-create tables.
@@ -213,7 +215,9 @@ async def init_engine(
backend: "memory", "sqlite", "postgres", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
echo: Echo SQL to log.
pool_size: Postgres connection pool size.
pool_size: Base connection pool size (postgres/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
global _engine, _session_factory
@@ -271,6 +275,8 @@ async def init_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
json_serializer=_json_serializer,
)
@@ -279,6 +285,8 @@ async def init_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
@@ -302,37 +310,39 @@ async def init_engine(
# This is expected during initial scaffolding or minimal installs.
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend == "postgres" and "does not exist" in str(exc):
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend == "postgres" and "does not exist" in str(exc):
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database
await _engine.dispose()
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, pool_pre_ping=True, json_serializer=_json_serializer)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
else:
raise
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, json_serializer=_json_serializer)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=1800,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
else:
raise
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
# cannot do on its own. Failures don't abort startup — see the helper.
@@ -351,6 +361,8 @@ async def init_engine_from_config(config) -> None:
url=config.app_sqlalchemy_url,
echo=config.echo_sql,
pool_size=config.pool_size,
max_overflow=getattr(config, "max_overflow", 10),
pool_timeout=getattr(config, "pool_timeout", 30),
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
)
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Integer, String, Text
from sqlalchemy import Float, Index, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -15,6 +15,10 @@ class LlmCallMetricRow(Base):
"""One row per completed (or failed) LLM provider invocation."""
__tablename__ = "llm_call_metrics"
__table_args__ = (
Index("ix_llm_call_metrics_created_user", "created_at", "user_id"),
Index("ix_llm_call_metrics_run_created", "run_id", "created_at"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# Wall-clock time the LLM call ended. Indexed because admin views always
@@ -0,0 +1,85 @@
"""Create admin leaderboard daily snapshots and analytics indexes.
Revision ID: 20260608_01
Revises: 20260607_02
Create Date: 2026-06-08
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from deerflow.persistence.types import PortableJSON
revision = "20260608_01"
down_revision = "20260607_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def _create_index(table_name: str, index_name: str, columns: list[str]) -> None:
if _has_table(table_name) and not _has_index(table_name, index_name):
op.create_index(index_name, table_name, columns)
def _drop_index(table_name: str, index_name: str) -> None:
if _has_table(table_name) and _has_index(table_name, index_name):
op.drop_index(index_name, table_name=table_name)
def upgrade() -> None:
if not _has_table("admin_leaderboard_daily_stats"):
op.create_table(
"admin_leaderboard_daily_stats",
sa.Column("stat_date", sa.String(10), nullable=False),
sa.Column("settings_hash", sa.String(64), nullable=False),
sa.Column("status", sa.String(16), nullable=False, server_default="ready"),
sa.Column("payload_json", PortableJSON(), nullable=True),
sa.Column("message", sa.Text(), nullable=True),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("generated_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("stat_date", "settings_hash", name="pk_admin_leaderboard_daily_stats"),
)
op.create_index("ix_admin_leaderboard_daily_stats_status", "admin_leaderboard_daily_stats", ["status"])
op.create_index("ix_admin_leaderboard_daily_stats_updated_at", "admin_leaderboard_daily_stats", ["updated_at"])
_create_index("runs", "ix_runs_created_user_status", ["created_at", "user_id", "status"])
_create_index("runs", "ix_runs_user_created", ["user_id", "created_at"])
_create_index("runs", "ix_runs_thread_created", ["thread_id", "created_at"])
_create_index("tool_call_metrics", "ix_tool_call_metrics_created_user", ["created_at", "user_id"])
_create_index("tool_call_metrics", "ix_tool_call_metrics_run_created", ["run_id", "created_at"])
_create_index("llm_call_metrics", "ix_llm_call_metrics_created_user", ["created_at", "user_id"])
_create_index("llm_call_metrics", "ix_llm_call_metrics_run_created", ["run_id", "created_at"])
_create_index("threads_meta", "ix_threads_meta_created_user", ["created_at", "user_id"])
_create_index("threads_meta", "ix_threads_meta_user_created", ["user_id", "created_at"])
def downgrade() -> None:
_drop_index("threads_meta", "ix_threads_meta_user_created")
_drop_index("threads_meta", "ix_threads_meta_created_user")
_drop_index("llm_call_metrics", "ix_llm_call_metrics_run_created")
_drop_index("llm_call_metrics", "ix_llm_call_metrics_created_user")
_drop_index("tool_call_metrics", "ix_tool_call_metrics_run_created")
_drop_index("tool_call_metrics", "ix_tool_call_metrics_created_user")
_drop_index("runs", "ix_runs_thread_created")
_drop_index("runs", "ix_runs_user_created")
_drop_index("runs", "ix_runs_created_user_status")
if _has_table("admin_leaderboard_daily_stats"):
op.drop_table("admin_leaderboard_daily_stats")
@@ -21,6 +21,7 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
@@ -58,6 +59,7 @@ __all__ = [
"AgentFavoriteRow",
"AgentRow",
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"LlmCallMetricRow",
"NotificationRow",
"RecommendedQuestionRow",
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, String, Text
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -47,4 +47,9 @@ class RunRow(Base):
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
__table_args__ = (Index("ix_runs_thread_status", "thread_id", "status"),)
__table_args__ = (
Index("ix_runs_thread_status", "thread_id", "status"),
Index("ix_runs_created_user_status", "created_at", "user_id", "status"),
Index("ix_runs_user_created", "user_id", "created_at"),
Index("ix_runs_thread_created", "thread_id", "created_at"),
)
@@ -4,15 +4,19 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy import Index, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class ThreadMetaRow(Base):
__tablename__ = "threads_meta"
class ThreadMetaRow(Base):
__tablename__ = "threads_meta"
__table_args__ = (
Index("ix_threads_meta_created_user", "created_at", "user_id"),
Index("ix_threads_meta_user_created", "user_id", "created_at"),
)
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
assistant_id: Mapped[str | None] = mapped_column(String(128), index=True)
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String, Text
from sqlalchemy import Index, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -21,6 +21,10 @@ class ToolCallMetricRow(Base):
"""
__tablename__ = "tool_call_metrics"
__table_args__ = (
Index("ix_tool_call_metrics_created_user", "created_at", "user_id"),
Index("ix_tool_call_metrics_run_created", "run_id", "created_at"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)