初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,25 @@
"""DeerFlow application persistence layer (SQLAlchemy 2.0 async ORM).
This module manages DeerFlow's own application data -- runs metadata,
thread ownership, cron jobs, users. It is completely separate from
LangGraph's checkpointer, which manages graph execution state.
Usage:
from deerflow.persistence import init_engine, close_engine, get_session_factory
"""
from deerflow.persistence.engine import (
close_engine,
get_engine,
get_session_factory,
init_engine,
rebuild_workflow_schema_for_mysql,
)
__all__ = [
"close_engine",
"get_engine",
"get_session_factory",
"init_engine",
"rebuild_workflow_schema_for_mysql",
]
@@ -0,0 +1,5 @@
"""Persistence models for admin analytics snapshots."""
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
__all__ = ["AdminLeaderboardDailyStatRow"]
@@ -0,0 +1,47 @@
"""Pure scheduling helpers for the admin leaderboard daily snapshots.
Kept import-light on purpose (only stdlib ``datetime``) so the decision logic
is unit-testable without standing up the FastAPI app, the DB engine, or the
ORM. The background scheduler in ``app/gateway/app.py`` wires this to the real
snapshot rows.
"""
from __future__ import annotations
from datetime import datetime
def should_finalize_previous_day(
*,
status: str | None,
generated_at: datetime | None,
now: datetime,
finalize_hour: int,
) -> bool:
"""Decide whether *yesterday's* snapshot should be force-recomputed once.
The previous day's snapshot is normally built before midnight, so it can
miss runs / tool calls whose rows only land in the database shortly before
or after the day boundary. Once per day, on/after ``finalize_hour`` (local
Beijing time), we recompute it a single time so the full day is captured.
``now`` and ``generated_at`` are expected to be timezone-aware Beijing
datetimes (``generated_at`` may be ``None`` for legacy rows).
Returns ``True`` only when all hold:
- it is already on/after ``finalize_hour`` today;
- a snapshot row exists (``status`` is not ``None`` — a missing day is
left to the normal prewarm/queue path);
- that snapshot is not already ``queued``/``running`` (a finalize already
in flight, or normal processing);
- it was generated before today (i.e. not yet finalized for this day).
"""
if status is None:
return False
if now.hour < finalize_hour:
return False
if status in ("queued", "running"):
return False
if generated_at is not None and generated_at.date() >= now.date():
return False
return True
@@ -0,0 +1,31 @@
"""ORM models for admin analytics snapshots."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class AdminLeaderboardDailyStatRow(Base):
"""One precomputed leaderboard snapshot for a single local calendar day."""
__tablename__ = "admin_leaderboard_daily_stats"
__table_args__ = (
Index("ix_admin_leaderboard_daily_stats_status", "status"),
Index("ix_admin_leaderboard_daily_stats_updated_at", "updated_at"),
)
stat_date: Mapped[str] = mapped_column(String(10), primary_key=True)
settings_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
status: Mapped[str] = mapped_column(String(16), default="ready")
payload_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
message: Mapped[str | None] = mapped_column(Text)
error: Mapped[str | None] = mapped_column(Text)
generated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,30 @@
"""Custom agent persistence."""
from __future__ import annotations
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.memory import MemoryAgentStore
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.agents.sql import AgentRepository
__all__ = [
"AgentExtrasRow",
"AgentFavoriteRow",
"AgentRepository",
"AgentRow",
"AgentSkillRow",
"AgentStore",
"MemoryAgentStore",
"make_agent_store",
]
def make_agent_store(session_factory):
if session_factory is None:
return MemoryAgentStore()
return AgentRepository(session_factory)
@@ -0,0 +1,231 @@
"""Abstract persistence API for custom agents."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class AgentStore(ABC):
@abstractmethod
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update a built-in agent (user_id IS NULL).
Caller is responsible for authorization — this method does NOT check
any user identity and returns ``None`` if the target is not a built-in.
"""
raise NotImplementedError
@abstractmethod
async def delete(self, agent_id: str, user_id: str) -> bool:
raise NotImplementedError
@abstractmethod
async def delete_builtin(self, agent_id: str) -> bool:
"""Delete a built-in agent (user_id IS NULL).
Caller is responsible for authorization — this method does NOT check
any user identity and returns ``False`` if the target is not a built-in.
"""
raise NotImplementedError
@abstractmethod
async def count_owned(self, user_id: str) -> int:
"""Return the number of agents owned by ``user_id`` (user_id IS NOT NULL)."""
raise NotImplementedError
@abstractmethod
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
"""Set or clear the homepage-selector featured rank for an agent.
Returns the updated row, or ``None`` if the agent does not exist.
Caller is responsible for authorization (admin) and for verifying
that ``published`` is true when ``order`` is not ``None``.
"""
raise NotImplementedError
@abstractmethod
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
"""Return featured + still-visible-to-``user_id`` agents, ordered by ``featured_order`` asc."""
raise NotImplementedError
@abstractmethod
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
raise NotImplementedError
@abstractmethod
async def list_private(self) -> list[dict[str, Any]]:
"""Return every agent flagged ``is_private=True``.
Caller is responsible for authorization — typically the caller has
already verified the admin-configured private-square password.
"""
raise NotImplementedError
@abstractmethod
async def list_all(self) -> list[dict[str, Any]]:
"""Return every agent regardless of ownership/visibility.
Caller is responsible for authorization — typically the caller has
already verified ``system_role == "admin"``.
"""
raise NotImplementedError
@abstractmethod
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Update any agent row by id, regardless of ownership.
Caller is responsible for authorization (admin only). Returns ``None``
if the agent does not exist.
"""
raise NotImplementedError
@abstractmethod
async def delete_any(self, agent_id: str) -> bool:
"""Delete any agent row by id, regardless of ownership.
Caller is responsible for authorization (admin only). Returns
``False`` if the agent does not exist.
"""
raise NotImplementedError
@abstractmethod
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
) -> tuple[list[dict[str, Any]], int]:
"""Return one page of agents plus the total matching count.
``scope``:
- ``"mine"`` — owned by ``user_id`` OR favorited by ``user_id``.
- ``"square"`` — built-in or published, and not private-square.
- ``"private"``— flagged ``is_private=True`` (caller gates access).
- ``None`` — the default listing visibility (``list_visible``).
``admin=True`` bypasses the visibility/scope filter entirely (every
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
``search`` matches the raw name OR its desensitized display form.
``tag_ids`` keeps agents carrying ANY of the tags (OR). ``owner`` keeps
only agents published by that user (``user_id`` equality) — built-in
agents (no owner) never match. All filters, the COUNT, and LIMIT/OFFSET
all execute in SQL.
Returns ``(rows, total)`` where ``rows`` is at most ``page_size`` items
for the 1-based ``page``.
"""
raise NotImplementedError
# --- Listing cache (skills / tool_groups / model) ---
@abstractmethod
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
"""Replace the cached skill list for ``agent_id``.
``None`` and ``[]`` both clear the cache to "no skills" (mirrors the
tri-state semantics of ``AgentConfig.skills``).
"""
raise NotImplementedError
@abstractmethod
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``."""
raise NotImplementedError
@abstractmethod
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
"""Batch-fetch cached listing extras for ``agent_ids``.
Returns ``{agent_id: {"skills": list[str] | None, "tool_groups":
list[str] | None, "model": str | None}}``. An agent that was never
mirrored is absent from the mapping (caller treats it as unknown).
"""
raise NotImplementedError
@abstractmethod
async def delete_extras_for_agent(self, agent_id: str) -> None:
"""Drop the cached skills + extras rows for ``agent_id`` (on delete)."""
raise NotImplementedError
# --- Favorites ("收藏") ---
@abstractmethod
async def add_favorite(self, user_id: str, agent_id: str) -> None:
"""Bookmark ``agent_id`` for ``user_id``. Idempotent."""
raise NotImplementedError
@abstractmethod
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
"""Remove a bookmark. Returns ``False`` when none existed."""
raise NotImplementedError
@abstractmethod
async def list_favorite_ids(self, user_id: str) -> set[str]:
"""Return the set of agent ids ``user_id`` has favorited."""
raise NotImplementedError
@abstractmethod
async def delete_favorites_for_agent(self, agent_id: str) -> None:
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
# --- Per-user pins ("置顶") ---
@abstractmethod
async def add_pin(self, user_id: str, agent_id: str) -> None:
"""Pin ``agent_id`` for ``user_id``. Idempotent."""
raise NotImplementedError
@abstractmethod
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
"""Remove a per-user pin. Returns ``False`` when none existed."""
raise NotImplementedError
@abstractmethod
async def list_pin_ids(self, user_id: str) -> set[str]:
"""Return the set of agent ids pinned by ``user_id``."""
raise NotImplementedError
@abstractmethod
async def delete_pins_for_agent(self, agent_id: str) -> None:
"""Drop every user's pin of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
@@ -0,0 +1,325 @@
"""In-memory custom agent store for memory persistence mode."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.config.agent_name_desensitize import name_matches
from deerflow.persistence.agents.base import AgentStore
def _now() -> str:
return datetime.now(UTC).isoformat()
def _copy(data: dict[str, Any]) -> dict[str, Any]:
copied = dict(data)
copied["builtin"] = copied.get("user_id") is None
return copied
class MemoryAgentStore(AgentStore):
def __init__(self) -> None:
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._pins: set[tuple[str, str]] = set()
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
self._skills: dict[str, list[str]] = {}
self._extras: dict[str, dict[str, Any]] = {}
@staticmethod
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
"""Single-record visibility (``get_visible``) — the union of every
listing surface, so a list-visible agent always opens. Mirrors
``AgentRepository._visible_filter``: baseline visibility OR
private-square (password is the discovery gate, reads are not
re-gated) OR favorited (the "我的" listing keeps favorites even after
the owner unpublishes)."""
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
rows = [_copy(agent) for agent in self._agents.values() if self._is_visible(agent, user_id)]
rows.sort(key=lambda agent: agent.get("created_at") or "")
return rows
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or not self._is_fetchable(agent, user_id):
return None
return _copy(agent)
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
return _copy(agent) if agent is not None else None
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or agent.get("user_id") != user_id:
return None
return _copy(agent)
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
now = _now()
row = {
"id": data["id"],
"name": data["name"],
"description": data.get("description") or "",
"user_id": data.get("user_id"),
"published": bool(data.get("published", False)),
"is_private": bool(data.get("is_private", False)),
"square_id": str(data.get("square_id") or ""),
"featured_order": None,
"created_at": now,
"updated_at": now,
}
self._agents[row["id"]] = row
return _copy(row)
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
row = self._agents.get(data["id"])
if row is None:
row = {
"id": data["id"],
"name": data["name"],
"description": data.get("description") or "",
"user_id": None,
"published": bool(data.get("published", True)),
"featured_order": None,
"created_at": _now(),
"updated_at": _now(),
}
self._agents[row["id"]] = row
elif row.get("user_id") is None:
row["name"] = data["name"]
row["description"] = data.get("description") or ""
row["published"] = bool(data.get("published", True))
row["updated_at"] = _now()
return _copy(row)
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete(self, agent_id: str, user_id: str) -> bool:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return False
self._agents.pop(agent_id, None)
return True
async def delete_builtin(self, agent_id: str) -> bool:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return False
self._agents.pop(agent_id, None)
return True
async def count_owned(self, user_id: str) -> int:
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
row["featured_order"] = order
row["updated_at"] = _now()
return _copy(row)
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
candidates = [a for a in self._agents.values() if a.get("featured_order") is not None and self._is_visible(a, user_id)]
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
return [_copy(a) for a in candidates]
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
rows.sort(key=lambda a: order_map.get(a["id"], 0))
return [_copy(a) for a in rows]
async def list_private(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private"))]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def list_all(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values()]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete_any(self, agent_id: str) -> bool:
if agent_id not in self._agents:
return False
self._agents.pop(agent_id, None)
return True
# --- Paginated listing ---
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
if admin:
return True
if scope == "square":
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
if scope == "mine":
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
favorited = (user_id, agent["id"]) in self._favorites
return owned or favorited
if scope == "private":
return bool(agent.get("is_private"))
if scope == "builtin":
# 内置智能体 = user_id IS NULL(仅管理员;路由层 admin 闸门)。
return agent.get("user_id") is None
return self._is_visible(agent, user_id)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
) -> tuple[list[dict[str, Any]], int]:
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
owner_key = owner.strip() if owner and owner.strip() else None
square_key = square_id.strip() if square_id and square_id.strip() else None
def _square_ok(a: dict[str, Any]) -> bool:
if square_key is None:
return True
sid = str(a.get("square_id") or "")
return sid == square_key or (square_is_default and sid == "")
matched = [
a
for a in self._agents.values()
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
and (owner_key is None or a.get("user_id") == owner_key)
and _square_ok(a)
]
if sort == "time_asc":
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
elif sort == "updated_desc":
matched.sort(
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
reverse=True,
)
elif sort == "updated_asc":
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
elif sort == "name_asc":
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
elif sort == "name_desc":
matched.sort(
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
reverse=True,
)
else:
matched.sort(
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
reverse=True,
)
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
total = len(matched)
start = max(page - 1, 0) * page_size
page_rows = matched[start : start + page_size]
return [_copy(a) for a in page_rows], total
# --- Listing cache (skills / tool_groups / model) ---
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
self._extras[agent_id] = {"tool_groups": tool_groups, "model": model}
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
out: dict[str, dict[str, Any]] = {}
for agent_id in agent_ids:
has_extras = agent_id in self._extras
has_skills = agent_id in self._skills
if not has_extras and not has_skills:
continue
extras = self._extras.get(agent_id, {})
out[agent_id] = {
"skills": list(self._skills.get(agent_id, [])),
"tool_groups": extras.get("tool_groups"),
"model": extras.get("model"),
}
return out
async def delete_extras_for_agent(self, agent_id: str) -> None:
self._skills.pop(agent_id, None)
self._extras.pop(agent_id, None)
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
self._favorites.add((user_id, agent_id))
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
if (user_id, agent_id) in self._favorites:
self._favorites.discard((user_id, agent_id))
return True
return False
async def list_favorite_ids(self, user_id: str) -> set[str]:
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
self._pins.add((user_id, agent_id))
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
key = (user_id, agent_id)
if key not in self._pins:
return False
self._pins.remove(key)
return True
async def list_pin_ids(self, user_id: str) -> set[str]:
return {agent_id for uid, agent_id in self._pins if uid == user_id}
async def delete_pins_for_agent(self, agent_id: str) -> None:
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
@@ -0,0 +1,127 @@
"""ORM model for custom agents."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class AgentRow(Base):
__tablename__ = "agents"
# Stored as a plain string so the same id can be used as the filesystem
# directory name under {base_dir}/agents/{id}/.
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# When True, the agent is moved into the password-gated "private square"
# (私密广场) and is hidden from the regular public square / built-in list.
# The flag is independent of ``published``: a private agent is still
# visible to its owner in the "我的" tab, and visible to anyone in the
# private-square listing once the password has been verified.
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# Publish square (发布广场) this agent is filed under. A pure organizational
# category with no access control — independent of ``published`` /
# ``is_private``. Empty string = "not assigned" → the listing layer treats
# it as the admin-configured default square. Each agent belongs to at most
# one square (single column), so it can never be published to two squares.
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", server_default="", index=True)
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
# featured rank (lower sorts first). Only ``published=True`` agents are
# eligible; the router enforces that invariant on writes. BIGINT so the
# frontend can seed initial ranks from ``Date.now()`` (ms-resolution
# timestamps overflow a 4-byte INT on MySQL).
featured_order: Mapped[int | None] = mapped_column(BigInteger, nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class AgentFavoriteRow(Base):
"""A user's bookmark of a (typically public-square) agent.
Lets a user collect agents from the 广场 (square) into their "我的" tab
without owning them. One row per (user, agent); the unique constraint makes
favoriting idempotent.
"""
__tablename__ = "agent_favorites"
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_favorites_user_agent"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# Where this favorite came from: ``"user"`` = the user bookmarked it
# themselves; ``"position"`` = it was auto-granted by the user's 岗位
# (position) via the subscription sync engine. Sync only ever adds/removes
# ``"position"`` rows, so a user's own bookmarks are never clobbered.
origin: Mapped[str] = mapped_column(String(16), nullable=False, default="user", server_default="user")
# The position that granted this favorite (only set when ``origin='position'``).
position_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class AgentPinRow(Base):
"""A per-user pin that keeps an agent near the top of that user's lists."""
__tablename__ = "agent_pins"
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class AgentSkillRow(Base):
"""Denormalized cache of a custom agent's selected skills.
``config.yaml`` under ``.deer-flow/agents/{id}/`` stays the source of truth
for the agent runtime (the lead-agent prompt reads it directly). This table
is a read cache so the gallery list endpoint can render skill badges with a
single batched ``IN`` query instead of stat+YAML-parsing every agent's
``config.yaml`` on each request. Written through on create/update and
backfilled from disk by ``_sync_legacy_agents``.
One row per (agent, skill); ``position`` preserves the configured order.
"""
__tablename__ = "agent_skills"
# Prefix-unique instead of UniqueConstraint: (64 + 120) * 4 = 736 bytes
# fits MySQL's legacy 767-byte index limit (utf8mb4); a plain unique over
# agent_id + skill_name(191) would be 1020 bytes and fail create_all with
# error 1071 there. mysql_length is ignored on SQLite/Postgres.
__table_args__ = (Index("uq_agent_skills_agent_skill", "agent_id", "skill_name", unique=True, mysql_length={"skill_name": 120}),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
skill_name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
position: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
class AgentExtrasRow(Base):
"""Denormalized cache of a custom agent's ``tool_groups`` / ``model``.
Companion to :class:`AgentSkillRow` — same rationale (avoid per-agent
``config.yaml`` reads in the list path). One row per agent. The presence of
a row also marks "this agent has been mirrored": the list path treats a
missing row as "unknown / not yet backfilled" (``skills=None``) so it can
be told apart from an explicitly empty skill list.
``skills_synced_at`` records when the cache was last written from disk so
the backfill can skip agents whose ``config.yaml`` has not changed.
"""
__tablename__ = "agent_extras"
agent_id: Mapped[str] = mapped_column(String(64), primary_key=True)
# list[str] | None — the agent's tool_groups whitelist from config.yaml.
tool_groups: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True, default=None)
model: Mapped[str | None] = mapped_column(String(191), nullable=True, default=None)
skills_synced_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,573 @@
"""SQLAlchemy-backed custom agent repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import and_, case, delete, func, literal, not_, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.config.agent_name_desensitize import source_terms_for_needle
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentPinRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
from deerflow.persistence.tags.model import TagAssignmentRow
def _row_to_dict(row: AgentRow) -> dict[str, Any]:
data = row.to_dict()
data["builtin"] = data.get("user_id") is None
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
return data
class AgentRepository(AgentStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _business_chain_agent_exists(agent_id_col):
return (
select(RoundtableChainRow.id)
.where(
or_(
RoundtableChainRow.seats.like(literal('%"agent_id": "') + agent_id_col + literal('"%')),
RoundtableChainRow.seats.like(literal('%"agent_id":"') + agent_id_col + literal('"%')),
)
)
.exists()
)
@classmethod
def _valid_favorite_filter(cls):
return or_(
AgentFavoriteRow.origin == "user",
and_(
AgentFavoriteRow.origin == "position",
AgentRow.user_id.is_not(None),
not_(cls._business_chain_agent_exists(AgentFavoriteRow.agent_id)),
),
)
@staticmethod
def _visible_filter(user_id: str):
"""Visibility for single-record fetches (``get_visible``).
Lenient — must cover the union of every listing surface, so any agent
a user can see in a list always opens (detail + chat) instead of
404ing:
- built-in / owned / published — the baseline gallery visibility;
- ``is_private`` — the private-square password is the *discovery*
gate (listings); individual reads are not re-gated, so a private
agent opened from the square (or a bookmark/link) stays openable.
Note moving an agent into the private square forces
``published=false``, so this clause cannot lean on ``published``;
- favorited — the "我的" listing keeps favorited agents even after
the owner unpublishes them, so reads keep honoring the favorite.
"""
valid_favorite_exists = (
select(AgentFavoriteRow.id)
.where(
AgentFavoriteRow.user_id == user_id,
AgentFavoriteRow.agent_id == AgentRow.id,
AgentRepository._valid_favorite_filter(),
)
.exists()
)
return or_(
AgentRow.user_id.is_(None),
AgentRow.user_id == user_id,
AgentRow.published.is_(True),
AgentRow.is_private.is_(True),
valid_favorite_exists,
)
@staticmethod
def _listing_filter(user_id: str):
"""Visibility for *listings* — hides private-square agents from
non-owners so they only surface inside the password-gated endpoint.
Owners keep seeing their own private agents in '我的'.
"""
return or_(
AgentRow.user_id == user_id,
and_(
or_(AgentRow.user_id.is_(None), AgentRow.published.is_(True)),
AgentRow.is_private.is_(False),
),
)
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
stmt = select(AgentRow).where(self._listing_filter(user_id)).order_by(AgentRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
stmt = select(AgentRow).where(AgentRow.id == agent_id, self._visible_filter(user_id))
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _row_to_dict(row) if row is not None else None
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
return _row_to_dict(row) if row is not None else None
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
stmt = select(AgentRow).where(AgentRow.id == agent_id, AgentRow.user_id == user_id)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _row_to_dict(row) if row is not None else None
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = AgentRow(
id=data["id"],
name=data["name"],
description=data.get("description") or "",
user_id=data.get("user_id"),
published=bool(data.get("published", False)),
is_private=bool(data.get("is_private", False)),
square_id=str(data.get("square_id") or ""),
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(AgentRow, data["id"])
if row is None:
now = datetime.now(UTC)
row = AgentRow(
id=data["id"],
name=data["name"],
description=data.get("description") or "",
user_id=None,
published=bool(data.get("published", True)),
created_at=now,
updated_at=now,
)
session.add(row)
elif row.user_id is None:
row.name = data["name"]
row.description = data.get("description") or ""
row.published = bool(data.get("published", True))
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None or row.user_id != user_id:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None or row.user_id is not None:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def delete(self, agent_id: str, user_id: str) -> bool:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None or row.user_id != user_id:
return False
await session.delete(row)
await session.commit()
return True
async def delete_builtin(self, agent_id: str) -> bool:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None or row.user_id is not None:
return False
await session.delete(row)
await session.commit()
return True
async def count_owned(self, user_id: str) -> int:
stmt = select(func.count()).select_from(AgentRow).where(AgentRow.user_id == user_id)
async with self._sf() as session:
return await session.scalar(stmt) or 0
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None:
return None
row.featured_order = order
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
stmt = (
select(AgentRow)
.where(AgentRow.featured_order.is_not(None), self._listing_filter(user_id))
.order_by(AgentRow.featured_order.asc(), AgentRow.created_at.asc())
)
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
if not agent_ids:
return []
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
stmt = select(AgentRow).where(AgentRow.id.in_(agent_ids), self._listing_filter(user_id))
async with self._sf() as session:
result = await session.execute(stmt)
rows = list(result.scalars())
rows.sort(key=lambda r: order_map.get(r.id, len(order_map)))
return [_row_to_dict(r) for r in rows]
async def list_private(self) -> list[dict[str, Any]]:
stmt = select(AgentRow).where(AgentRow.is_private.is_(True)).order_by(AgentRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def list_all(self) -> list[dict[str, Any]]:
stmt = select(AgentRow).order_by(AgentRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def delete_any(self, agent_id: str) -> bool:
async with self._sf() as session:
row = await session.get(AgentRow, agent_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
# --- Paginated listing ---
def _scope_filter(self, *, user_id: str, scope: str | None, admin: bool):
"""Build the visibility predicate for a paginated listing."""
if admin:
return None # no visibility filter — every agent
if scope == "square":
return and_(
or_(AgentRow.user_id.is_(None), AgentRow.published.is_(True)),
AgentRow.is_private.is_(False),
)
if scope == "mine":
# owned OR valid favorites. Built-in agents and business-chain
# seats only enter "mine" when explicitly favorited by the user;
# the original position-granted behavior stays for ordinary agents.
valid_favorite_exists = (
select(AgentFavoriteRow.id)
.where(
AgentFavoriteRow.user_id == user_id,
AgentFavoriteRow.agent_id == AgentRow.id,
self._valid_favorite_filter(),
)
.exists()
)
return or_(AgentRow.user_id == user_id, valid_favorite_exists)
if scope == "private":
return AgentRow.is_private.is_(True)
if scope == "builtin":
# 内置智能体 = user_id IS NULL。仅管理员可用此 scope(路由层已 admin 闸门)。
return AgentRow.user_id.is_(None)
return self._listing_filter(user_id)
@staticmethod
def _search_clause(search: str):
"""Match the raw name OR (via the desensitize mirror) its display form."""
needle = search.strip().lower()
clauses = [func.lower(AgentRow.name).like(f"%{needle}%")]
for term in source_terms_for_needle(needle):
clauses.append(AgentRow.name.like(f"%{term}%"))
return or_(*clauses)
@staticmethod
def _tag_clause(tag_ids: list[str]):
subq = select(TagAssignmentRow.target_id).where(
TagAssignmentRow.target_type == "agent",
TagAssignmentRow.tag_id.in_(tag_ids),
)
return AgentRow.id.in_(subq)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
) -> tuple[list[dict[str, Any]], int]:
filters = []
scope_filter = self._scope_filter(user_id=user_id, scope=scope, admin=admin)
if scope_filter is not None:
filters.append(scope_filter)
if search and search.strip():
filters.append(self._search_clause(search))
if tag_ids:
filters.append(self._tag_clause(tag_ids))
if owner and owner.strip():
# Filter to one publisher. Built-in agents (user_id IS NULL) never match.
filters.append(AgentRow.user_id == owner.strip())
if square_id and square_id.strip():
sid = square_id.strip()
if square_is_default:
# Legacy / unassigned rows (square_id == "") belong to the
# default square, so include them alongside explicit matches.
filters.append(or_(AgentRow.square_id == sid, AgentRow.square_id == ""))
else:
filters.append(AgentRow.square_id == sid)
count_stmt = select(func.count()).select_from(AgentRow)
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
pin_rank = case((AgentRow.id.in_(pin_subq), 0), else_=1)
if sort == "time_asc":
order_by = (AgentRow.created_at.asc(), AgentRow.id.asc())
elif sort == "updated_desc":
order_by = (AgentRow.updated_at.desc(), AgentRow.id.asc())
elif sort == "updated_asc":
order_by = (AgentRow.updated_at.asc(), AgentRow.id.asc())
elif sort == "name_asc":
order_by = (func.lower(AgentRow.name).asc(), AgentRow.id.asc())
elif sort == "name_desc":
order_by = (func.lower(AgentRow.name).desc(), AgentRow.id.asc())
else:
order_by = (AgentRow.created_at.desc(), AgentRow.id.asc())
page_stmt = (
select(AgentRow)
.order_by(pin_rank.asc(), *order_by)
.limit(page_size)
.offset(max(page - 1, 0) * page_size)
)
if filters:
where = and_(*filters)
count_stmt = count_stmt.where(where)
page_stmt = page_stmt.where(where)
async with self._sf() as session:
total = await session.scalar(count_stmt) or 0
result = await session.execute(page_stmt)
rows = [_row_to_dict(row) for row in result.scalars()]
return rows, total
# --- Listing cache (skills / tool_groups / model) ---
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
async with self._sf() as session:
await session.execute(delete(AgentSkillRow).where(AgentSkillRow.agent_id == agent_id))
for position, name in enumerate(skills or []):
if not isinstance(name, str):
continue
session.add(
AgentSkillRow(
id=uuid4().hex,
agent_id=agent_id,
skill_name=name,
position=position,
)
)
await session.commit()
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
async with self._sf() as session:
row = await session.get(AgentExtrasRow, agent_id)
now = datetime.now(UTC)
if row is None:
session.add(
AgentExtrasRow(
agent_id=agent_id,
tool_groups=tool_groups,
model=model,
skills_synced_at=now,
)
)
else:
row.tool_groups = tool_groups
row.model = model
row.skills_synced_at = now
await session.commit()
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
if not agent_ids:
return {}
out: dict[str, dict[str, Any]] = {}
async with self._sf() as session:
extras = await session.execute(
select(AgentExtrasRow).where(AgentExtrasRow.agent_id.in_(agent_ids))
)
for row in extras.scalars():
out[row.agent_id] = {
"skills": [], # default; filled below if skill rows exist
"tool_groups": row.tool_groups,
"model": row.model,
}
skill_rows = await session.execute(
select(AgentSkillRow.agent_id, AgentSkillRow.skill_name)
.where(AgentSkillRow.agent_id.in_(agent_ids))
.order_by(AgentSkillRow.agent_id.asc(), AgentSkillRow.position.asc())
)
for agent_id, skill_name in skill_rows.all():
entry = out.setdefault(
agent_id, {"skills": [], "tool_groups": None, "model": None}
)
entry["skills"].append(skill_name)
return out
async def delete_extras_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentSkillRow).where(AgentSkillRow.agent_id == agent_id))
await session.execute(delete(AgentExtrasRow).where(AgentExtrasRow.agent_id == agent_id))
await session.commit()
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
async with self._sf() as session:
existing = (
await session.execute(
select(AgentFavoriteRow)
.where(AgentFavoriteRow.user_id == user_id, AgentFavoriteRow.agent_id == agent_id)
.limit(1)
)
).scalar_one_or_none()
if existing is not None:
if existing.origin != "user" or existing.position_id is not None:
existing.origin = "user"
existing.position_id = None
await session.commit()
return
session.add(AgentFavoriteRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
try:
await session.commit()
except IntegrityError:
# Lost a race against a concurrent favorite of the same agent —
# the unique constraint already guarantees the desired state.
await session.rollback()
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(AgentFavoriteRow).where(
AgentFavoriteRow.user_id == user_id,
AgentFavoriteRow.agent_id == agent_id,
)
)
await session.commit()
return (result.rowcount or 0) > 0
async def list_favorite_ids(self, user_id: str) -> set[str]:
async with self._sf() as session:
result = await session.execute(
select(AgentFavoriteRow.agent_id)
.join(AgentRow, AgentRow.id == AgentFavoriteRow.agent_id)
.where(
AgentFavoriteRow.user_id == user_id,
self._valid_favorite_filter(),
)
)
return {row[0] for row in result.all()}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
await session.commit()
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
async with self._sf() as session:
existing = (
await session.execute(
select(AgentPinRow.id)
.where(AgentPinRow.user_id == user_id, AgentPinRow.agent_id == agent_id)
.limit(1)
)
).scalar_one_or_none()
if existing is not None:
return
session.add(AgentPinRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
try:
await session.commit()
except IntegrityError:
await session.rollback()
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(AgentPinRow).where(
AgentPinRow.user_id == user_id,
AgentPinRow.agent_id == agent_id,
)
)
await session.commit()
return (result.rowcount or 0) > 0
async def list_pin_ids(self, user_id: str) -> set[str]:
async with self._sf() as session:
result = await session.execute(
select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
)
return {row[0] for row in result.all()}
async def delete_pins_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentPinRow).where(AgentPinRow.agent_id == agent_id))
await session.commit()
@@ -0,0 +1,46 @@
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
from deerflow.persistence.ai_writing_sessions.sql import AIWritingSessionRepository
def make_ai_writing_session_store(session_factory) -> AIWritingSessionRepository | None:
if session_factory is None:
return None
return AIWritingSessionRepository(session_factory)
# ── 进程级默认 repo 单例 ──────────────────────────────────────────────────────
#
# AI 写作 graph 注册成 LangGraph Server assistant 后,节点跑在 harness 进程里,
# 拿不到 FastAPI 的 ``request.app.state``;harness → app 又是禁止的 import 方向。
# 这里提供一个 app 启动时注册、harness 节点运行时读取的「跨边界单例」通道。
#
# 写入:app/gateway/deps.py 在创建完 session_factory 之后调
# ``register_default_store(app.state.ai_writing_session_store)``
# 读取:节点里 ``get_default_store()``,None 表示未配置 DB(跳过写入即可)。
_default_store: AIWritingSessionRepository | None = None
def register_default_store(store: AIWritingSessionRepository | None) -> None:
"""登记进程级默认 AI 写作 session store。
幂等:可多次调用,覆盖前值。传 ``None`` 用于显式清理(主要给测试)。
"""
global _default_store
_default_store = store
def get_default_store() -> AIWritingSessionRepository | None:
"""返回进程级默认 AI 写作 session store;未配置时为 ``None``。
节点调用方应在拿到 ``None`` 时跳过 DB 写入(写入是「锦上添花」非致命)。
"""
return _default_store
__all__ = [
"AIWritingSessionRow",
"AIWritingSessionRepository",
"make_ai_writing_session_store",
"register_default_store",
"get_default_store",
]
@@ -0,0 +1,38 @@
"""ORM model for AI writing sessions."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class AIWritingSessionRow(Base):
__tablename__ = "ai_writing_sessions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
user_intent: Mapped[str | None] = mapped_column(String(512), nullable=True)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="in_progress")
draft_title: Mapped[str | None] = mapped_column(String(512), nullable=True)
draft_markdown: Mapped[str | None] = mapped_column(Text, nullable=True)
completed_interventions: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array
review_result: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显。
# 用 PortableLongText:MySQL 上映射为 LONGTEXT,避免 TEXT 64KB 上限导致保存 500。
transcript: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
# 时间列必须用 BeijingDateTime(项目规范,与 threads_meta 等其它表对齐)—— MySQL
# DATETIME 不存时区,raw ``DateTime(timezone=True)`` 会把 UTC wall clock 写入
# 列,读出来比真实北京时间晚 8 小时。``BeijingDateTime`` 在 bind 时 UTC→Beijing、
# 在 result 时 attach +08:00,前后端拿到的就是北京时间且 tz-aware。
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,258 @@
"""SQLAlchemy-backed AI writing session storage."""
from __future__ import annotations
import json
from datetime import UTC, datetime
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
_UNSET = object()
class AIWritingSessionRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: AIWritingSessionRow) -> dict:
d = row.to_dict()
for k in ("created_at", "updated_at"):
if isinstance(d.get(k), datetime):
d[k] = d[k].isoformat()
for k in ("completed_interventions", "review_result", "transcript"):
val = d.get(k)
if isinstance(val, str):
try:
d[k] = json.loads(val)
except Exception:
d[k] = None
return d
async def create(
self,
*,
id: str,
title: str,
user_intent: str | None = None,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.create")
now = datetime.now(UTC)
row = AIWritingSessionRow(
id=id,
user_id=resolved_user_id,
title=title,
user_intent=user_intent,
status="in_progress",
draft_title=None,
draft_markdown=None,
completed_interventions=None,
review_result=None,
transcript=None,
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def upsert_session(
self,
*,
id: str,
title: str,
user_intent: str | None = None,
user_id: str | None | _AutoSentinel = AUTO,
status: str = "in_progress",
) -> dict:
"""幂等:行不存在则创建,行存在则保留原数据(不覆盖 draft / review 等已有写入)。
AI 写作 graph 的 ``intent_parser_node`` 在每次进入时调一次(首启 + resume 时
都会跑),所以必须幂等。resume 时若直接 ``create`` 会因主键冲突抛错;用
``update`` 又找不到行而 no-op。这个方法把两条路合一:行不存在新建,行存在
只刷新 ``updated_at`` 以便列表页排序前置。
"""
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.upsert_session")
now = datetime.now(UTC)
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, id)
if row is None:
row = AIWritingSessionRow(
id=id,
user_id=resolved_user_id,
title=title,
user_intent=user_intent,
status=status,
draft_title=None,
draft_markdown=None,
completed_interventions=None,
review_result=None,
transcript=None,
created_at=now,
updated_at=now,
)
session.add(row)
else:
# 行已存在(resume 路径):仅刷新 updated_at,保留 user_id / title / draft / review 等。
# 若 user_id 之前为空(migration 兼容场景),用解析出来的填上。
if row.user_id is None and resolved_user_id is not None:
row.user_id = resolved_user_id
row.updated_at = now
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def update(
self,
session_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
title=_UNSET,
status=_UNSET,
draft_title=_UNSET,
draft_markdown=_UNSET,
completed_interventions=_UNSET,
review_result=_UNSET,
transcript=_UNSET,
) -> dict | None:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.update")
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, session_id)
if row is None:
return None
if resolved_user_id is not None and row.user_id != resolved_user_id:
return None
if title is not _UNSET:
row.title = title
if status is not _UNSET:
row.status = status
if draft_title is not _UNSET:
row.draft_title = draft_title
if draft_markdown is not _UNSET:
row.draft_markdown = draft_markdown
if completed_interventions is not _UNSET:
row.completed_interventions = (
json.dumps(completed_interventions, ensure_ascii=False)
if completed_interventions is not None
else None
)
if review_result is not _UNSET:
row.review_result = (
json.dumps(review_result, ensure_ascii=False)
if review_result is not None
else None
)
if transcript is not _UNSET:
row.transcript = (
json.dumps(transcript, ensure_ascii=False)
if transcript is not None
else None
)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def get(
self,
session_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict | None:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.get")
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, session_id)
if row is None:
return None
if resolved_user_id is not None and row.user_id != resolved_user_id:
return None
return self._row_to_dict(row)
async def list(
self,
*,
user_id: str | None | _AutoSentinel = AUTO,
limit: int = 50,
) -> list[dict]:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.list")
stmt = select(AIWritingSessionRow)
if resolved_user_id is not None:
stmt = stmt.where(AIWritingSessionRow.user_id == resolved_user_id)
stmt = stmt.order_by(AIWritingSessionRow.updated_at.desc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
async def delete(
self,
session_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.delete")
async with self._sf() as session:
row = await session.get(AIWritingSessionRow, session_id)
if row is None:
return False
if resolved_user_id is not None and row.user_id != resolved_user_id:
return False
await session.delete(row)
await session.commit()
return True
async def list_by_status(self, status: str, *, limit: int = 500) -> list[dict]:
"""列出指定 status 的会话(仅 id + user_id,用于后台驱动重启续跑 reconcile)。
不按 user_id 过滤——进程级 reconcile 要把所有用户「挂起中」的会话都拉起。
"""
stmt = (
select(AIWritingSessionRow.id, AIWritingSessionRow.user_id)
.where(AIWritingSessionRow.status == status)
.order_by(AIWritingSessionRow.updated_at.asc())
.limit(limit)
)
async with self._sf() as session:
result = await session.execute(stmt)
return [{"id": row[0], "user_id": row[1]} for row in result.all()]
async def list_older_than(
self,
cutoff: datetime,
*,
only_finished: bool = False,
limit: int = 500,
) -> list[str]:
"""列出 updated_at 早于 cutoff 的会话 id(最多 limit 个)。
清理任务先 list 出 id 列表,再用这些 id 联动删 checkpointer
里对应的 thread,最后再删 MySQL 行 —— 保证两边一致。
Args:
cutoff: 截止时间,updated_at < cutoff 的会话被认为「过期」
only_finished: True 时只清 status in (done/error),False 不看状态
limit: 单批最多返回多少条,用于分批处理避免大事务
"""
stmt = select(AIWritingSessionRow.id).where(AIWritingSessionRow.updated_at < cutoff)
if only_finished:
stmt = stmt.where(AIWritingSessionRow.status.in_(("done", "error")))
stmt = stmt.order_by(AIWritingSessionRow.updated_at.asc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [row[0] for row in result.all()]
async def delete_by_ids(self, session_ids: list[str]) -> int:
"""按 id 列表批量删除。返回实际删除的行数。"""
if not session_ids:
return 0
stmt = delete(AIWritingSessionRow).where(AIWritingSessionRow.id.in_(session_ids))
async with self._sf() as session:
result = await session.execute(stmt)
await session.commit()
return int(result.rowcount or 0)
@@ -0,0 +1,11 @@
from deerflow.persistence.article_types.model import ArticleTypeRow
from deerflow.persistence.article_types.sql import ArticleTypeRepository
def make_article_type_store(session_factory) -> ArticleTypeRepository | None:
if session_factory is None:
return None
return ArticleTypeRepository(session_factory)
__all__ = ["ArticleTypeRow", "ArticleTypeRepository", "make_article_type_store"]
@@ -0,0 +1,24 @@
"""ORM model for AI writing article types."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class ArticleTypeRow(Base):
__tablename__ = "ai_writing_article_types"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
key: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
label: Mapped[str] = mapped_column(String(128), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
prompt_hint: Mapped[str | None] = mapped_column(Text, nullable=True)
default_word_count: Mapped[int] = mapped_column(Integer, nullable=False, default=800)
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
@@ -0,0 +1,123 @@
"""SQLAlchemy-backed article type storage."""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.article_types.model import ArticleTypeRow
_UNSET = object()
class ArticleTypeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: ArticleTypeRow) -> dict:
d = row.to_dict()
for k in ("created_at", "updated_at"):
if isinstance(d.get(k), datetime):
d[k] = d[k].isoformat()
return d
async def create(
self,
*,
key: str,
label: str,
description: str | None = None,
prompt_hint: str | None = None,
default_word_count: int = 800,
sort_order: int = 0,
) -> dict:
now = datetime.now(UTC)
row = ArticleTypeRow(
id=str(uuid.uuid4()),
key=key,
label=label,
description=description,
prompt_hint=prompt_hint,
default_word_count=default_word_count,
sort_order=sort_order,
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def update(
self,
article_type_id: str,
*,
key=_UNSET,
label=_UNSET,
description=_UNSET,
prompt_hint=_UNSET,
default_word_count=_UNSET,
sort_order=_UNSET,
) -> dict | None:
async with self._sf() as session:
row = await session.get(ArticleTypeRow, article_type_id)
if row is None:
return None
if key is not _UNSET:
row.key = key
if label is not _UNSET:
row.label = label
if description is not _UNSET:
row.description = description
if prompt_hint is not _UNSET:
row.prompt_hint = prompt_hint
if default_word_count is not _UNSET:
row.default_word_count = default_word_count
if sort_order is not _UNSET:
row.sort_order = sort_order
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def get(self, article_type_id: str) -> dict | None:
async with self._sf() as session:
row = await session.get(ArticleTypeRow, article_type_id)
if row is None:
return None
return self._row_to_dict(row)
async def get_by_key(self, key: str) -> dict | None:
async with self._sf() as session:
stmt = select(ArticleTypeRow).where(ArticleTypeRow.key == key)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
if row is None:
return None
return self._row_to_dict(row)
async def list(self) -> list[dict]:
stmt = select(ArticleTypeRow).order_by(ArticleTypeRow.sort_order.asc(), ArticleTypeRow.created_at.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
async def delete(self, article_type_id: str) -> bool:
async with self._sf() as session:
row = await session.get(ArticleTypeRow, article_type_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def list_keys(self) -> list[str]:
stmt = select(ArticleTypeRow.key).order_by(ArticleTypeRow.sort_order.asc())
async with self._sf() as session:
result = await session.execute(stmt)
return list(result.scalars())
@@ -0,0 +1,4 @@
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
@@ -0,0 +1,175 @@
"""ORM model for DeerFlow-native assistant knowledge bases."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Index, Integer, LargeBinary, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class AssistantKnowledgeBaseRow(Base):
__tablename__ = "assistant_knowledge_base"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
description: Mapped[str | None] = mapped_column(String(2048), nullable=True)
# ``global`` is the full-library aggregation base. ``custom`` bases are
# user-created assistant knowledge bases fed from WeKnora export packages.
# Nullable keeps startup column reconciliation safe for existing databases.
base_kind: Mapped[str | None] = mapped_column(String(24), nullable=True, index=True)
status: Mapped[str] = mapped_column(String(64), default="active")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeSourceRow(Base):
__tablename__ = "assistant_knowledge_sources"
__table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_type: Mapped[str] = mapped_column(String(32), index=True)
source_key: Mapped[str] = mapped_column(String(255))
knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_name: Mapped[str] = mapped_column(String(255), default="")
last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeImportJobRow(Base):
__tablename__ = "assistant_knowledge_import_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="exporting")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeWikiPageRow(Base):
__tablename__ = "assistant_knowledge_wiki_pages"
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
slug: Mapped[str] = mapped_column(String(512), index=True)
title: Mapped[str] = mapped_column(String(512), index=True)
page_type: Mapped[str] = mapped_column(String(32), index=True)
current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeWikiRevisionRow(Base):
__tablename__ = "assistant_knowledge_wiki_revisions"
__table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
revision_no: Mapped[int] = mapped_column(Integer)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
change_type: Mapped[str] = mapped_column(String(32), default="import")
status: Mapped[str] = mapped_column(String(24), default="published")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeVectorRow(Base):
__tablename__ = "assistant_knowledge_vectors"
__table_args__ = (
Index("ix_assistant_knowledge_vector_base_fingerprint", "base_id", "embedding_fingerprint"),
Index("ix_assistant_knowledge_vector_page", "page_id"),
Index("ix_assistant_knowledge_vector_source", "source_id"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_vector_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
section_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_model: Mapped[str | None] = mapped_column(String(255), nullable=True)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class AssistantKnowledgeEntityRow(Base):
__tablename__ = "assistant_knowledge_entities"
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512), index=True)
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeEntityContributionRow(Base):
__tablename__ = "assistant_knowledge_entity_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class AssistantKnowledgeRelationRow(Base):
__tablename__ = "assistant_knowledge_relations"
__table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeRelationContributionRow(Base):
__tablename__ = "assistant_knowledge_relation_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -0,0 +1,40 @@
"""SQLAlchemy declarative base with automatic to_dict support.
All DeerFlow ORM models inherit from this Base. It provides a generic
to_dict() method via SQLAlchemy's inspect() so individual models don't
need to write their own serialization logic.
LangGraph's checkpointer tables are NOT managed by this Base.
"""
from __future__ import annotations
from sqlalchemy import inspect as sa_inspect
from sqlalchemy.orm import DeclarativeBase
class Base(DeclarativeBase):
"""Base class for all DeerFlow ORM models.
Provides:
- Automatic to_dict() via SQLAlchemy column inspection.
- Standard __repr__() showing all column values.
"""
def to_dict(self, *, exclude: set[str] | None = None) -> dict:
"""Convert ORM instance to plain dict.
Uses SQLAlchemy's inspect() to iterate mapped column attributes.
Args:
exclude: Optional set of column keys to omit.
Returns:
Dict of {column_key: value} for all mapped columns.
"""
exclude = exclude or set()
return {c.key: getattr(self, c.key) for c in sa_inspect(type(self)).mapper.column_attrs if c.key not in exclude}
def __repr__(self) -> str:
cols = ", ".join(f"{c.key}={getattr(self, c.key)!r}" for c in sa_inspect(type(self)).mapper.column_attrs)
return f"{type(self).__name__}({cols})"
@@ -0,0 +1,18 @@
"""Global business → agent / chain mapping persistence (业务链条管理)."""
from __future__ import annotations
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.business_mapping.sql import BusinessMappingRepository
__all__ = [
"BusinessMappingRepository",
"BusinessMappingRow",
"make_business_mapping_store",
]
def make_business_mapping_store(session_factory):
if session_factory is None:
return None
return BusinessMappingRepository(session_factory)
@@ -0,0 +1,53 @@
"""ORM model for the global business → agent / chain mapping (业务链条管理).
A "business mapping" row binds a fixed business code (3Q / 6BF / 6-1…6-4 / 7BF /
8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
account can resolve it). Used to auto-pick the business chain when a task page
launches the multi-agent roundtable.
Unlike ``roundtable_chains`` (per-user), this mapping is **global / shared** —
there is no ``user_id`` column; all accounts see and use the same configuration.
The fixed business rows are seeded on startup; users only fill in the
agent / chain columns from the management page.
``agent_name`` / ``chain_title`` are denormalized display snapshots so the table
renders without cross-referencing and survives a later agent/chain rename or
delete (the picked id still drives the auto-flow).
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class BusinessMappingRow(Base):
__tablename__ = "business_mapping"
# Business code is the natural primary key (e.g. "3Q", "6BF"). Global, no user scope.
business_code: Mapped[str] = mapped_column(String(32), primary_key=True)
# Display label shown in the 业务 column (e.g. "任务FX").
label: Mapped[str] = mapped_column(String(128), nullable=False, default="")
# Picked 智能体 (暂存不用,自动流程当前不消费此字段). NULL = 未配置.
agent_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
agent_name: Mapped[str | None] = mapped_column(String(191), nullable=True)
# Picked 业务链条 → roundtable_chains.id (须为 public). NULL = 未配置.
chain_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
chain_title: Mapped[str | None] = mapped_column(String(512), nullable=True)
# Fixed display order for the management table.
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,143 @@
"""SQLAlchemy-backed global business → agent / chain mapping store.
Global / shared (no ``user_id`` scoping): every account reads and writes the same
rows. Fixed business rows are seeded once on startup (``ensure_seed_defaults``)
with empty agent/chain; users fill those in from the management page via
``update_mapping``.
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.business_mapping.model import BusinessMappingRow
_UNSET = object()
# Fixed business rows (业务码 → 展示名). Seeded on first boot; agent/chain left empty.
# 8BF 页面尚未建好,先占位保留行。顺序即管理表的展示顺序。
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
# 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
{"business_code": "3Q", "label": "3QFX"},
{"business_code": "6BF", "label": "任务FX"},
{"business_code": "7BF", "label": "XDFX"},
{"business_code": "8BF", "label": "8BF"},
{"business_code": "AFX", "label": "AGENTFX"},
# 独立配置位,与上方业务码无关;固定排在表尾。
{"business_code": "6-1", "label": "6-1"},
{"business_code": "6-2", "label": "6-2"},
{"business_code": "6-3", "label": "6-3"},
{"business_code": "6-4", "label": "6-4"},
{"business_code": "7-1", "label": "7-1"},
{"business_code": "7-2", "label": "7-2"},
{"business_code": "7-3", "label": "7-3"},
{"business_code": "7-4", "label": "7-4"},
]
class BusinessMappingRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _to_dict(row: BusinessMappingRow) -> dict[str, Any]:
return {
"business_code": row.business_code,
"label": row.label,
"agent_id": row.agent_id,
"agent_name": row.agent_name,
"chain_id": row.chain_id,
"chain_title": row.chain_title,
"sort_order": row.sort_order,
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
async def list_mappings(self) -> list[dict[str, Any]]:
stmt = select(BusinessMappingRow).order_by(
BusinessMappingRow.sort_order.asc(), BusinessMappingRow.business_code.asc()
)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._to_dict(row) for row in result.scalars()]
async def get_mapping(self, business_code: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(BusinessMappingRow, business_code)
return self._to_dict(row) if row is not None else None
async def update_mapping(
self,
business_code: str,
*,
agent_id=_UNSET,
agent_name=_UNSET,
chain_id=_UNSET,
chain_title=_UNSET,
) -> dict[str, Any] | None:
"""Update an existing (seeded) business row. Returns None if the code is unknown.
Only forwarded fields are touched; explicit ``None`` clears that field.
"""
async with self._sf() as session:
row = await session.get(BusinessMappingRow, business_code)
if row is None:
return None
if agent_id is not _UNSET:
row.agent_id = agent_id or None
if agent_name is not _UNSET:
row.agent_name = agent_name or None
if chain_id is not _UNSET:
row.chain_id = chain_id or None
if chain_title is not _UNSET:
row.chain_title = chain_title or None
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return self._to_dict(row)
async def ensure_seed_defaults(self) -> int:
"""Idempotently insert the fixed business rows if missing. Returns inserted count.
Never overwrites an existing row's agent/chain (respects user config).
Re-syncs ``sort_order`` (and empty ``label``) so newly inserted codes keep
the intended table order next to older seeded rows.
"""
inserted = 0
dirty = False
now = datetime.now(UTC)
async with self._sf() as session:
for idx, seed in enumerate(DEFAULT_BUSINESS_ROWS):
existing = await session.get(BusinessMappingRow, seed["business_code"])
if existing is not None:
if existing.sort_order != idx:
existing.sort_order = idx
dirty = True
seed_label = str(seed.get("label") or "")
if seed_label and not (existing.label or "").strip():
existing.label = seed_label
dirty = True
continue
session.add(
BusinessMappingRow(
business_code=seed["business_code"],
label=seed.get("label", ""),
agent_id=None,
agent_name=None,
chain_id=None,
chain_title=None,
sort_order=idx,
created_at=now,
updated_at=now,
)
)
inserted += 1
dirty = True
if dirty:
await session.commit()
return inserted
@@ -0,0 +1,30 @@
"""Compact/sidebar-summary menu override persistence."""
from __future__ import annotations
from deerflow.persistence.compact_menu_overrides.base import CompactMenuOverrideStore
from deerflow.persistence.compact_menu_overrides.keys import (
ALLOWED_COMPACT_MENU_TYPES,
DEFAULT_COMPACT_MENU_TYPE,
normalize_compact_menu_type,
)
from deerflow.persistence.compact_menu_overrides.memory import MemoryCompactMenuOverrideStore
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.compact_menu_overrides.sql import CompactMenuOverrideRepository
__all__ = [
"ALLOWED_COMPACT_MENU_TYPES",
"CompactMenuOverrideRepository",
"CompactMenuOverrideRow",
"CompactMenuOverrideStore",
"DEFAULT_COMPACT_MENU_TYPE",
"MemoryCompactMenuOverrideStore",
"make_compact_menu_override_store",
"normalize_compact_menu_type",
]
def make_compact_menu_override_store(session_factory):
if session_factory is None:
return MemoryCompactMenuOverrideStore()
return CompactMenuOverrideRepository(session_factory)
@@ -0,0 +1,24 @@
"""Abstract compact-sidebar menu override persistence API."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class CompactMenuOverrideStore(ABC):
@abstractmethod
async def list_overrides(self, menu_type: str | None = None) -> list[dict[str, Any]]:
"""Return compact-menu override rows for one type (default A)."""
raise NotImplementedError
@abstractmethod
async def replace_all(
self,
rows: list[dict[str, Any]],
*,
menu_type: str | None = None,
updated_by: str | None = None,
) -> list[dict[str, Any]]:
"""Atomically replace the compact-menu override set for one type."""
raise NotImplementedError
@@ -0,0 +1,47 @@
"""Compact-menu type helpers.
Two website deployments can keep independent compact-sidebar layouts.
Type ``A`` is the historical default (existing rows stay as-is). Type ``B``
is stored under a prefixed primary key so it never collides with type A on
databases that still use ``node_id`` as the sole PK.
"""
from __future__ import annotations
DEFAULT_COMPACT_MENU_TYPE = "A"
ALLOWED_COMPACT_MENU_TYPES = ("A", "B")
_STORAGE_SEP = "::"
_TYPE_ALIASES = {
"A": "A",
"1": "A",
"DEFAULT": "A",
"B": "B",
"2": "B",
"ALT": "B",
}
def normalize_compact_menu_type(value: str | None) -> str:
raw = (value or "").strip().upper()
return _TYPE_ALIASES.get(raw, DEFAULT_COMPACT_MENU_TYPE)
def storage_node_id(menu_type: str, node_id: str) -> str:
"""PK value that stays unique across types on a node_id-only primary key."""
t = normalize_compact_menu_type(menu_type)
nid = str(node_id)
if t == DEFAULT_COMPACT_MENU_TYPE:
prefix = f"{t}{_STORAGE_SEP}"
return nid[len(prefix) :] if nid.startswith(prefix) else nid
prefix = f"{t}{_STORAGE_SEP}"
return nid if nid.startswith(prefix) else f"{prefix}{nid}"
def public_node_id(stored: str, menu_type: str | None = None) -> str:
"""Strip the type prefix so API callers always see the real menu node id."""
t = normalize_compact_menu_type(menu_type)
nid = str(stored)
prefix = f"{t}{_STORAGE_SEP}"
if nid.startswith(prefix):
return nid[len(prefix) :]
return nid
@@ -0,0 +1,64 @@
"""In-memory compact-menu override store."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.compact_menu_overrides.base import CompactMenuOverrideStore
from deerflow.persistence.compact_menu_overrides.keys import (
normalize_compact_menu_type,
public_node_id,
storage_node_id,
)
def _normalize(data: dict[str, Any], menu_type: str, updated_by: str | None) -> dict[str, Any]:
placement = str(data.get("placement") or "sidebar")
node_id = public_node_id(str(data["node_id"]), menu_type)
return {
"node_id": storage_node_id(menu_type, node_id),
"menu_type": menu_type,
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
"sort_order": int(data.get("sort_order", 0)),
"label": (data.get("label") or None),
"updated_by": updated_by,
"updated_at": datetime.now(UTC).isoformat(),
}
def _public(row: dict[str, Any]) -> dict[str, Any]:
menu_type = normalize_compact_menu_type(row.get("menu_type"))
out = dict(row)
out["menu_type"] = menu_type
out["node_id"] = public_node_id(str(row.get("node_id") or ""), menu_type)
return out
class MemoryCompactMenuOverrideStore(CompactMenuOverrideStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
async def list_overrides(self, menu_type: str | None = None) -> list[dict[str, Any]]:
t = normalize_compact_menu_type(menu_type)
rows = [r for r in self._rows.values() if normalize_compact_menu_type(r.get("menu_type")) == t]
return [_public(r) for r in sorted(rows, key=lambda r: r.get("sort_order", 0))]
async def replace_all(
self,
rows: list[dict[str, Any]],
*,
menu_type: str | None = None,
updated_by: str | None = None,
) -> list[dict[str, Any]]:
t = normalize_compact_menu_type(menu_type)
self._rows = {
k: v for k, v in self._rows.items() if normalize_compact_menu_type(v.get("menu_type")) != t
}
for r in rows:
node_id = r.get("node_id")
if not node_id:
continue
stored = _normalize(r, t, updated_by)
self._rows[str(stored["node_id"])] = stored
return await self.list_overrides(t)
@@ -0,0 +1,30 @@
"""ORM model for compact/sidebar-summary menu overrides.
This table only controls the compact left menu presentation. The full sidebar
menu layout continues to use ``menu_overrides`` and is unaffected.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class CompactMenuOverrideRow(Base):
__tablename__ = "compact_menu_overrides"
__table_args__ = (Index("ix_compact_menu_overrides_menu_type", "menu_type"),)
node_id: Mapped[str] = mapped_column(String(128), primary_key=True)
menu_type: Mapped[str] = mapped_column(String(32), default="A", server_default="A")
placement: Mapped[str] = mapped_column(String(32), default="sidebar")
sort_order: Mapped[int] = mapped_column(Integer, default=0)
label: Mapped[str | None] = mapped_column(String(255))
updated_by: Mapped[str | None] = mapped_column(String(64))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
@@ -0,0 +1,87 @@
"""SQLAlchemy-backed compact-menu override store."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import delete, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.compact_menu_overrides.base import CompactMenuOverrideStore
from deerflow.persistence.compact_menu_overrides.keys import (
DEFAULT_COMPACT_MENU_TYPE,
normalize_compact_menu_type,
public_node_id,
storage_node_id,
)
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
def _to_dict(row: CompactMenuOverrideRow) -> dict[str, Any]:
data = row.to_dict()
menu_type = normalize_compact_menu_type(data.get("menu_type"))
data["menu_type"] = menu_type
data["node_id"] = public_node_id(str(data.get("node_id") or ""), menu_type)
value = data.get("updated_at")
if isinstance(value, datetime):
data["updated_at"] = value.isoformat()
return data
def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> CompactMenuOverrideRow:
placement = str(data.get("placement") or "sidebar")
node_id = str(data["node_id"])
return CompactMenuOverrideRow(
node_id=storage_node_id(menu_type, node_id),
menu_type=menu_type,
placement=placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
sort_order=int(data.get("sort_order", 0)),
label=(data.get("label") or None),
updated_by=updated_by,
)
class CompactMenuOverrideRepository(CompactMenuOverrideStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_overrides(self, menu_type: str | None = None) -> list[dict[str, Any]]:
t = normalize_compact_menu_type(menu_type)
stmt = select(CompactMenuOverrideRow).order_by(CompactMenuOverrideRow.sort_order.asc())
stmt = stmt.where(_menu_type_filter(t))
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def replace_all(
self,
rows: list[dict[str, Any]],
*,
menu_type: str | None = None,
updated_by: str | None = None,
) -> list[dict[str, Any]]:
t = normalize_compact_menu_type(menu_type)
by_id: dict[str, dict[str, Any]] = {}
for r in rows:
node_id = r.get("node_id")
if node_id:
by_id[public_node_id(str(node_id), t)] = r
async with self._sf() as session:
await session.execute(delete(CompactMenuOverrideRow).where(_menu_type_filter(t)))
new_rows = [_row_from(r, t, updated_by) for r in by_id.values()]
session.add_all(new_rows)
await session.commit()
for row in new_rows:
await session.refresh(row)
return [_to_dict(row) for row in new_rows]
def _menu_type_filter(menu_type: str):
if menu_type == DEFAULT_COMPACT_MENU_TYPE:
return or_(
CompactMenuOverrideRow.menu_type == menu_type,
CompactMenuOverrideRow.menu_type.is_(None),
CompactMenuOverrideRow.menu_type == "",
)
return CompactMenuOverrideRow.menu_type == menu_type
@@ -0,0 +1,15 @@
"""Concurrency (system-pressure) sampling persistence."""
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
from deerflow.persistence.concurrency.store import (
ConcurrencyBucket,
ConcurrencySampleStore,
make_concurrency_sample_store,
)
__all__ = [
"ConcurrencyBucket",
"ConcurrencySampleRow",
"ConcurrencySampleStore",
"make_concurrency_sample_store",
]
@@ -0,0 +1,31 @@
"""ORM model for periodic concurrency (system-pressure) samples.
A background sampler writes one row every few seconds holding the number of
in-flight model runs at that instant. The admin monitor aggregates these into
per-minute / per-hour buckets to draw the 并发量 pressure line chart, even for
time windows before the admin opened the page.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ConcurrencySampleRow(Base):
"""One sampled snapshot of concurrent in-flight model runs."""
__tablename__ = "concurrency_samples"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
# Wall-clock time the sample was taken (Beijing time on disk, tz-aware in Python).
sampled_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), index=True, default=lambda: datetime.now(UTC)
)
# Number of runs whose status was pending/running at ``sampled_at``.
active_count: Mapped[int] = mapped_column(Integer, default=0)
@@ -0,0 +1,142 @@
"""Async store for concurrency samples + minute/hour aggregation."""
from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
logger = logging.getLogger(__name__)
# How far back the aggregation will fetch raw rows before bucketing. A safety
# cap so a degenerate range never tries to load the whole table into memory.
_MAX_ROWS = 60_000
@dataclass
class ConcurrencyBucket:
"""One minute/hour bucket of the concurrency series."""
bucket: str # ISO start of the bucket
peak: int # max concurrent runs observed in the bucket
avg: float # mean concurrent runs in the bucket
samples: int # how many raw samples fell into the bucket
def _bucket_start(dt: datetime, granularity: str) -> datetime:
"""Truncate a timestamp to the start of its minute/hour bucket."""
if granularity == "hour":
return dt.replace(minute=0, second=0, microsecond=0)
return dt.replace(second=0, microsecond=0)
class ConcurrencySampleStore:
"""Persists periodic concurrency snapshots and aggregates them."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def record(self, active_count: int, *, sampled_at: datetime | None = None) -> None:
"""Insert one sample. Best-effort — never raises into the caller."""
try:
async with self._sf() as session:
session.add(
ConcurrencySampleRow(
active_count=max(0, int(active_count)),
sampled_at=sampled_at or datetime.now(UTC),
)
)
await session.commit()
except Exception: # pragma: no cover - sampling must never break the app
logger.warning("Failed to record concurrency sample", exc_info=True)
async def aggregate(
self,
*,
since: datetime,
until: datetime,
granularity: str = "minute",
) -> list[ConcurrencyBucket]:
"""Return per-minute / per-hour buckets of peak + average concurrency.
Bucketing is done in Python (not SQL ``date_trunc``/``strftime``) so the
logic is identical across the sqlite/mysql/postgres backends.
"""
gran = "hour" if granularity == "hour" else "minute"
try:
async with self._sf() as session:
rows = (
await session.execute(
select(ConcurrencySampleRow.sampled_at, ConcurrencySampleRow.active_count)
.where(
ConcurrencySampleRow.sampled_at >= since,
ConcurrencySampleRow.sampled_at <= until,
)
.order_by(ConcurrencySampleRow.sampled_at)
.limit(_MAX_ROWS)
)
).all()
except Exception:
logger.warning("Failed to aggregate concurrency samples", exc_info=True)
return []
buckets: dict[datetime, dict[str, float]] = {}
for sampled_at, active_count in rows:
if sampled_at is None:
continue
if sampled_at.tzinfo is None:
sampled_at = sampled_at.replace(tzinfo=UTC)
key = _bucket_start(sampled_at, gran)
value = int(active_count or 0)
entry = buckets.get(key)
if entry is None:
buckets[key] = {"peak": value, "sum": value, "count": 1}
else:
entry["peak"] = max(entry["peak"], value)
entry["sum"] += value
entry["count"] += 1
result: list[ConcurrencyBucket] = []
for key in sorted(buckets):
entry = buckets[key]
count = max(1, int(entry["count"]))
result.append(
ConcurrencyBucket(
bucket=key.isoformat(),
peak=int(entry["peak"]),
avg=round(entry["sum"] / count, 2),
samples=count,
)
)
return result
async def purge_older_than(self, cutoff: datetime) -> int:
"""Delete samples older than ``cutoff``. Returns rows removed."""
try:
async with self._sf() as session:
result = await session.execute(
delete(ConcurrencySampleRow).where(ConcurrencySampleRow.sampled_at < cutoff)
)
await session.commit()
return int(result.rowcount or 0)
except Exception: # pragma: no cover
logger.warning("Failed to purge concurrency samples", exc_info=True)
return 0
def make_concurrency_sample_store(
session_factory: async_sessionmaker[AsyncSession] | None,
) -> ConcurrencySampleStore | None:
"""Factory mirroring the other ``make_*_store`` helpers.
Returns ``None`` on the memory backend (no SQL session factory), in which
case the monitor simply reports a live snapshot with no historical chart.
"""
if session_factory is None:
return None
return ConcurrencySampleStore(session_factory)
@@ -0,0 +1,18 @@
"""大屏绘制智能体 session persistence (按 taskId 存取,不分权)."""
from __future__ import annotations
from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow
from deerflow.persistence.dashboard_sessions.sql import DashboardSessionRepository
__all__ = [
"DashboardSessionRepository",
"DashboardSessionRow",
"make_dashboard_session_store",
]
def make_dashboard_session_store(session_factory):
if session_factory is None:
return None
return DashboardSessionRepository(session_factory)
@@ -0,0 +1,49 @@
"""ORM model for the standalone 大屏绘制智能体 session (按 taskId 存取).
One row per external ``task_id`` — the standalone dashboard page persists its
chat transcript + the latest structured ``report_json`` so that returning to the
same task either shows the finished dashboard directly (valid report_json) or
restores the generation process.
**Global / shared by task** — like ``roundtable_drafts`` task-bound drafts, the
store keys on ``task_id`` and does **not** scope by ``user_id`` (open read +
shared writes). ``user_id`` is recorded for audit only.
``transcript`` / ``report_json`` are large JSON blobs (chat history + report
data) stored as serialized strings in ``PortableLongText`` (MySQL ``LONGTEXT``)
to dodge MySQL's 64 KB ``TEXT`` cap — same approach as ``roundtable_drafts``.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DashboardSessionRow(Base):
__tablename__ = "dashboard_sessions"
# One session per task — keep it unique so upsert-by-task is unambiguous.
__table_args__ = (UniqueConstraint("task_id", name="uq_dashboard_sessions_task_id"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# External task id (deep-link). The session is shared by task, not by user.
task_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
# Audit only — who last wrote. Never used for access control.
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
# Chat transcript JSON (the dashboard agent's dialogue bubbles).
transcript: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Latest structured dashboard data (report-json string). Presence of a
# parseable value = "有效结果" → the page shows the dashboard directly.
report_json: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,104 @@
"""SQLAlchemy-backed 大屏绘制智能体 session store (按 taskId,不分权).
``transcript`` / ``report_json`` are JSON serialized into ``PortableLongText``
and round-tripped here with ``json.dumps`` / ``json.loads`` — same approach as
``roundtable_drafts``. All access keys on ``task_id`` and ignores ``user_id``
(open read + shared writes); ``user_id`` is recorded for audit only.
"""
from __future__ import annotations
import json
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow
def _loads(value: Any) -> Any:
if not isinstance(value, str):
return value
if not value:
return None
try:
return json.loads(value)
except Exception:
return None
def _dumps(value: Any) -> str | None:
if value is None:
return None
if isinstance(value, str):
return value # already a JSON string (report_json) — store verbatim
return json.dumps(value, ensure_ascii=False)
class DashboardSessionRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _to_dict(row: DashboardSessionRow) -> dict[str, Any]:
return {
"id": row.id,
"task_id": row.task_id,
"user_id": row.user_id,
"transcript": _loads(row.transcript),
# report_json is a structured-data JSON string; return it as a string
# so the frontend parses it with the same robust parser it uses live.
"report_json": row.report_json,
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
async def get_by_task(self, task_id: str) -> dict[str, Any] | None:
"""取该 task 的会话(**忽略 user_id**,不分权)。无则 None。"""
stmt = select(DashboardSessionRow).where(DashboardSessionRow.task_id == task_id).limit(1)
async with self._sf() as session:
result = await session.execute(stmt)
row = result.scalars().first()
return self._to_dict(row) if row is not None else None
async def upsert_by_task(
self,
task_id: str,
*,
transcript: Any = None,
report_json: Any = None,
user_id: str | None = None,
) -> dict[str, Any]:
"""按 task_id upsert:无则建、有则更新(**不分权**)。
``transcript`` / ``report_json`` 仅在传入非 None 时覆盖,便于只更新其中一项。
"""
now = datetime.now(UTC)
async with self._sf() as session:
stmt = select(DashboardSessionRow).where(DashboardSessionRow.task_id == task_id).limit(1)
row = (await session.execute(stmt)).scalars().first()
if row is None:
row = DashboardSessionRow(
id=uuid4().hex,
task_id=task_id,
user_id=user_id,
transcript=_dumps(transcript),
report_json=_dumps(report_json),
created_at=now,
updated_at=now,
)
session.add(row)
else:
if transcript is not None:
row.transcript = _dumps(transcript)
if report_json is not None:
row.report_json = _dumps(report_json)
if user_id is not None:
row.user_id = user_id
row.updated_at = now
await session.commit()
await session.refresh(row)
return self._to_dict(row)
@@ -0,0 +1,33 @@
"""Deep-research event-log persistence."""
from __future__ import annotations
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
from deerflow.persistence.deep_research_events.sql import DeepResearchEventRepository
def make_deep_research_event_store(session_factory) -> DeepResearchEventRepository | None:
if session_factory is None:
return None
return DeepResearchEventRepository(session_factory)
_default_store: DeepResearchEventRepository | None = None
def register_default_store(store: DeepResearchEventRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchEventRepository | None:
return _default_store
__all__ = [
"DeepResearchEventRepository",
"DeepResearchEventRow",
"get_default_store",
"make_deep_research_event_store",
"register_default_store",
]
@@ -0,0 +1,43 @@
"""ORM model for the deep-research event log.
Unlike ``roundtable_jobs`` (whole-row DB-poll SSE), deep research persists a
genuine **append-only event stream** so the frontend can reconnect with
``?after=<seq>`` and replay missed events (§15.3). ``seq`` is monotonic per
job and enforced unique by a composite constraint.
Events are small: payloads carry summaries + locators, never full material
text or model reasoning (§15.2 / §19.5).
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchEventRow(Base):
__tablename__ = "deep_research_events"
__table_args__ = (
# Monotonic per-job sequence — the SSE ``Last-Event-ID`` / ``?after`` cursor.
UniqueConstraint("job_id", "seq", name="uq_deep_research_events_job_seq"),
# Composite index for "all events of a job in seq order".
sa.Index("ix_deep_research_events_job_seq", "job_id", "seq"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
job_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
seq: Mapped[int] = mapped_column(Integer, nullable=False)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -0,0 +1,143 @@
"""SQLAlchemy repository for the deep-research event log.
The ``append()`` method assigns the next monotonic ``seq`` per job inside the
same transaction as the INSERT (``SELECT MAX(seq) … FOR`` + insert). Because
only **one** worker holds the job lease at a time, contention on seq is
low; the unique ``(job_id, seq)`` constraint catches any collision as a safety net.
"""
from __future__ import annotations
import json
import logging
from datetime import date, datetime
from enum import Enum
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
logger = logging.getLogger(__name__)
def _json_default(value: Any) -> Any:
"""Make event payloads replayable when source metadata has native dates.
Search adapters intentionally preserve ``published_at`` as a ``datetime``
until it reaches the source store. Events carry the same source envelope,
so the append-only log must normalise it instead of silently falling back to
a live-only frame.
"""
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, Enum):
return value.value
if hasattr(value, "model_dump"):
return value.model_dump(mode="json")
return str(value)
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False, default=_json_default)
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
class DeepResearchEventRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: DeepResearchEventRow) -> dict[str, Any]:
d = row.to_dict()
if "payload" in d:
d["payload"] = _loads(d["payload"])
# created_at → ISO for SSE.
if d.get("created_at"):
d["created_at"] = d["created_at"].isoformat() if hasattr(d["created_at"], "isoformat") else d["created_at"]
return d
async def append(
self,
*,
session_id: str,
job_id: str,
event_type: str,
phase: str = "initializing",
payload: dict[str, Any] | None = None,
max_retries: int = 3,
) -> dict[str, Any]:
"""Insert the next event, assigning ``seq = max(seq) + 1`` for this job.
Retries on the unique-constraint collision (only happens if two writers
race for the same seq — shouldn't occur under single-lease ownership
but is handled defensively).
"""
for attempt in range(max_retries):
async with self._sf() as session:
current_max = (await session.execute(select(func.max(DeepResearchEventRow.seq)).where(DeepResearchEventRow.job_id == job_id))).scalar() or 0
next_seq = int(current_max) + 1
row = DeepResearchEventRow(
job_id=job_id,
session_id=session_id,
seq=next_seq,
event_type=event_type,
phase=phase,
payload=_dumps(payload),
)
session.add(row)
try:
# Do not issue ``session.refresh(row)`` after commit. On a
# MySQL read/write-splitting deployment that read can land
# on a lagging replica, which cannot see the just-committed
# event and raises ``Could not refresh instance``. Flush
# assigns generated primary-key/default values on the writer
# connection, so the plain dict is complete before commit.
await session.flush()
result = self._row_to_dict(row)
await session.commit()
return result
except IntegrityError:
await session.rollback()
continue # retry with recomputed max
raise RuntimeError(f"deep_research_events.append: seq collision persisted after {max_retries} retries (job={job_id})")
async def list_after(self, job_id: str, *, after: int = 0, limit: int = 200) -> list[dict[str, Any]]:
"""Events with ``seq > after`` in ascending order (SSE replay, §15.3)."""
async with self._sf() as session:
stmt = select(DeepResearchEventRow).where(DeepResearchEventRow.job_id == job_id).where(DeepResearchEventRow.seq > after).order_by(DeepResearchEventRow.seq.asc()).limit(limit)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r) for r in rows]
async def last_seq(self, job_id: str) -> int:
async with self._sf() as session:
mx = (await session.execute(select(func.max(DeepResearchEventRow.seq)).where(DeepResearchEventRow.job_id == job_id))).scalar()
return int(mx or 0)
async def delete_by_job(self, job_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(delete(DeepResearchEventRow).where(DeepResearchEventRow.job_id == job_id))
await session.commit()
return result.rowcount
async def delete_by_session(self, session_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(delete(DeepResearchEventRow).where(DeepResearchEventRow.session_id == session_id))
await session.commit()
return result.rowcount
@@ -0,0 +1,48 @@
"""Deep-research durable-job persistence.
Mirrors ``roundtable_jobs``: store factory + process-level default-store singleton
so harness-side runner code can write progress without reaching into FastAPI
``request.app.state`` (harness → app imports forbidden).
"""
from __future__ import annotations
from deerflow.persistence.deep_research_jobs.model import DeepResearchJobRow
from deerflow.persistence.deep_research_jobs.sql import (
ACTIVE_STATUSES,
CANCELLABLE_STATUSES,
TERMINAL_STATUSES,
DeepResearchJobRepository,
compute_active_dedupe_key,
)
def make_deep_research_job_store(session_factory) -> DeepResearchJobRepository | None:
if session_factory is None:
return None
return DeepResearchJobRepository(session_factory)
_default_store: DeepResearchJobRepository | None = None
def register_default_store(store: DeepResearchJobRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchJobRepository | None:
return _default_store
__all__ = [
"ACTIVE_STATUSES",
"CANCELLABLE_STATUSES",
"DeepResearchJobRepository",
"DeepResearchJobRow",
"TERMINAL_STATUSES",
"compute_active_dedupe_key",
"get_default_store",
"make_deep_research_job_store",
"register_default_store",
]
@@ -0,0 +1,89 @@
"""ORM model for deep-research durable jobs.
A *job* is one **backend execution** of the research pipeline against a session.
Detailed / deep / multi-agent research can run for minutes, so jobs are
**durable**: persisted to the DB, claimed via atomic lease, heartbeat-renewed,
and recoverable after a crash or server restart (§14).
The design mirrors ``roundtable_jobs`` (Phase 3 lease + dedupe):
- ``active_dedupe_key`` — unique index; occupied while the job is non-terminal,
cleared (NULL) on terminal. INSERT conflict = idempotent "active job exists".
- ``lease_owner`` / ``lease_until`` — atomic conditional claim + heartbeat.
- ``version`` — row CAS counter, bumped on every state-changing UPDATE.
- Two-phase cancel: ``cancel_requested`` flag → executor watcher → ``cancelled``.
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchJobRow(Base):
__tablename__ = "deep_research_jobs"
__table_args__ = (
# Same trick as roundtable_jobs: unique key occupied only while active.
UniqueConstraint("active_dedupe_key", name="uq_deep_research_jobs_active_dedupe_key"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# ── idempotency + dedupe ────────────────────────────────────────────────
request_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
request_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
active_dedupe_key: Mapped[str | None] = mapped_column(String(64), nullable=True)
# ── frozen input (rebuilt by any worker after a takeover) ───────────────
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# ── lease / ownership ───────────────────────────────────────────────────
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
# ── status machine (§14.2) ──────────────────────────────────────────────
# queued / running / awaiting_input / completed / failed / cancelled.
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
# Fine-grained phase (§14.3): initializing / planning / collecting / …
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
progress: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
current_query: Mapped[str | None] = mapped_column(Text, nullable=True)
# ── two-phase cancel ────────────────────────────────────────────────────
cancel_requested: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=sa.false()
)
cancel_requested_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
# ── interrupt / human-review resume (§11.2) ─────────────────────────────
awaiting_interrupt_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
awaiting_payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# ── error classification (§14.5) ────────────────────────────────────────
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
# ── result locator ──────────────────────────────────────────────────────
result_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
@@ -0,0 +1,554 @@
"""SQLAlchemy repository for deep-research durable jobs.
Mirrors ``roundtable_jobs`` Phase-3 lease/dedupe/CAS logic exactly:
- ``compute_active_dedupe_key(session_id, user_id)`` → sha256 scope; NULL on terminal.
- ``try_create_or_get_active()`` — INSERT-conflict = idempotent active hit.
- ``claim_job()`` — atomic conditional UPDATE (rowcount-based mutual exclusion).
- ``renew_lease()`` — heartbeat (only lease owner, only while ``running``).
- ``update_progress()`` — lease-gated conditional UPDATE (CAS via ``version``).
- ``request_cancel()`` → ``finalize_cancel()`` — two-phase cancel.
- ``queue_for_resume()`` — atomic ``awaiting_input`` → ``queued``.
Recovery story (§14): ``input_snapshot`` lets any worker rebuild execution
after a takeover; expired-lease ``running`` rows are re-claimable via
``list_claimable``.
"""
from __future__ import annotations
import hashlib
import json
import logging
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_jobs.model import DeepResearchJobRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
logger = logging.getLogger(__name__)
_UNSET = object()
# ── status constants (§14.2) ────────────────────────────────────────────────
TERMINAL_STATUSES = ("completed", "failed", "cancelled")
ACTIVE_STATUSES = ("queued", "running", "awaiting_input")
CANCELLABLE_STATUSES = ("queued", "running", "awaiting_input")
def compute_active_dedupe_key(session_id: str, user_id: str | None) -> str:
"""``sha256(session_id + user_id)`` — one active job per session+user scope."""
scope = f"session:{session_id}:{user_id or ''}"
return hashlib.sha256(scope.encode("utf-8")).hexdigest()
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
class DeepResearchJobRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# ── row ↔ dict ──────────────────────────────────────────────────────────
@staticmethod
def _row_to_dict(row: DeepResearchJobRow) -> dict[str, Any]:
d = row.to_dict()
for k in ("input_snapshot", "result_snapshot", "awaiting_payload"):
if k in d:
d[k] = _loads(d[k])
return d
# ── create / idempotent active lookup ───────────────────────────────────
async def try_create_or_get_active(
self,
*,
id: str,
session_id: str,
user_id: str | _AutoSentinel = AUTO,
request_id: str | None = None,
request_hash: str | None = None,
input_snapshot: dict[str, Any] | None = None,
status: str = "queued",
phase: str = "initializing",
) -> tuple[dict[str, Any], bool]:
"""Insert a new job, or return the existing active job for this scope.
Returns ``(job_dict, created)``. ``created=False`` = an active job
already exists (idempotent hit), driven by the ``active_dedupe_key``
unique index.
"""
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.try_create_or_get_active")
dedupe_key = compute_active_dedupe_key(session_id, resolved)
for _ in range(2):
try:
async with self._sf() as session:
row = DeepResearchJobRow(
id=id,
session_id=session_id,
user_id=resolved,
request_id=request_id,
request_hash=request_hash,
active_dedupe_key=dedupe_key,
input_snapshot=_dumps(input_snapshot),
status=status,
phase=phase,
)
session.add(row)
# Do not issue ``session.refresh(row)`` after commit. A
# MySQL read/write-splitting proxy can route that read to
# a lagging replica, even though the INSERT has already
# committed on the writer. The caller would receive 500
# while an invisible queued job remains in the database.
# ``flush`` materializes ORM defaults on the writer, so we
# can return a complete, durable snapshot without a
# replica-dependent readback.
await session.flush()
result = self._row_to_dict(row)
await session.commit()
return result, True
except IntegrityError:
# Unique-index conflict → an active job already exists.
pass
existing = await self._get_by_dedupe_key(dedupe_key)
if existing is not None:
return existing, False
# Winner entered terminal between conflict and readback → retry INSERT.
# Should not happen in practice.
raise RuntimeError("try_create_or_get_active: race condition not resolved after retries")
async def replace_input_snapshot(self, job_id: str, input_snapshot: dict[str, Any]) -> dict[str, Any] | None:
"""Overwrite the frozen start payload on a still-queued job.
Idempotent retries used to keep the first snapshot forever. A first
write that raced ahead of the browser evidence (or a reused empty
job) then ran without ``collector_materials`` even though the later
request carried them.
"""
async with self._sf() as session:
result = await session.execute(
update(DeepResearchJobRow)
.where(
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status.in_(("queued", "running")),
)
.values(
input_snapshot=_dumps(input_snapshot),
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
await session.rollback()
return None
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
async def _get_by_dedupe_key(self, dedupe_key: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = (
await session.execute(
select(DeepResearchJobRow).where(
DeepResearchJobRow.active_dedupe_key == dedupe_key
)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── read ────────────────────────────────────────────────────────────────
async def get(
self, job_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.get")
async with self._sf() as session:
stmt = select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
if resolved is not None:
stmt = stmt.where(DeepResearchJobRow.user_id == resolved)
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def get_unscoped(self, job_id: str) -> dict[str, Any] | None:
"""Read without user filter (executor/dispatcher use this)."""
async with self._sf() as session:
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def get_active_for_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.get_active_for_session")
dedupe_key = compute_active_dedupe_key(session_id, resolved)
return await self._get_by_dedupe_key(dedupe_key)
async def list_by_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, limit: int = 20
) -> list[dict[str, Any]]:
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.list_by_session")
async with self._sf() as session:
stmt = select(DeepResearchJobRow).where(DeepResearchJobRow.session_id == session_id)
if resolved is not None:
stmt = stmt.where(DeepResearchJobRow.user_id == resolved)
stmt = stmt.order_by(DeepResearchJobRow.created_at.desc()).limit(limit)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r) for r in rows]
# ── dispatcher: scan + claim ────────────────────────────────────────────
async def list_claimable(self, *, now: datetime, limit: int = 20) -> list[str]:
"""Candidate job ids: queued, or running with an expired lease (zombie)."""
async with self._sf() as session:
stmt = (
select(DeepResearchJobRow.id)
.where(
DeepResearchJobRow.cancel_requested.is_(False),
or_(
DeepResearchJobRow.status == "queued",
(DeepResearchJobRow.status == "running")
& DeepResearchJobRow.lease_until.is_not(None)
& (DeepResearchJobRow.lease_until < now),
),
)
.order_by(DeepResearchJobRow.updated_at.asc())
.limit(limit)
)
return [r for r in (await session.execute(stmt)).scalars().all()]
async def claim_job(
self, job_id: str, *, lease_owner: str, lease_until: datetime
) -> dict[str, Any] | None:
"""Atomic conditional claim — rowcount==0 means another worker won."""
async with self._sf() as session:
values = {
"status": "running",
"lease_owner": lease_owner,
"lease_until": lease_until,
"attempt": DeepResearchJobRow.attempt + 1,
"version": DeepResearchJobRow.version + 1,
"started_at": datetime.now(UTC),
"updated_at": datetime.now(UTC),
"cancel_requested": False,
}
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.cancel_requested.is_(False),
or_(
DeepResearchJobRow.status == "queued",
(DeepResearchJobRow.status == "running")
& DeepResearchJobRow.lease_until.is_not(None)
& (DeepResearchJobRow.lease_until < datetime.now(UTC)),
),
]
result = await session.execute(
update(DeepResearchJobRow).where(*where).values(**values)
)
if result.rowcount == 0:
return None
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
# ── heartbeat ───────────────────────────────────────────────────────────
async def renew_lease(
self, job_id: str, *, lease_owner: str, lease_until: datetime
) -> bool:
async with self._sf() as session:
result = await session.execute(
update(DeepResearchJobRow)
.where(
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.lease_owner == lease_owner,
DeepResearchJobRow.status == "running",
)
.values(lease_until=lease_until, updated_at=datetime.now(UTC))
)
await session.commit()
return result.rowcount > 0
async def release_lease(self, job_id: str, *, lease_owner: str | None = None) -> bool:
async with self._sf() as session:
where = [DeepResearchJobRow.id == job_id]
if lease_owner is not None:
where.append(DeepResearchJobRow.lease_owner == lease_owner)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(lease_owner=None, lease_until=None, updated_at=datetime.now(UTC))
)
await session.commit()
return result.rowcount > 0
# ── progress update (lease-gated CAS) ───────────────────────────────────
async def update_progress(
self,
job_id: str,
*,
lease_owner: str | None = None,
user_id: str | _AutoSentinel = AUTO,
status: str | object = _UNSET,
phase: str | object = _UNSET,
progress: int | object = _UNSET,
current_query: str | None | object = _UNSET,
error_code: str | None | object = _UNSET,
error_message: str | None | object = _UNSET,
result_snapshot: dict | None | object = _UNSET,
awaiting_interrupt_id: str | None | object = _UNSET,
awaiting_payload: dict | None | object = _UNSET,
) -> dict[str, Any] | None:
"""Lease-gated conditional progress update.
When ``lease_owner`` is given, the write is rejected unless this worker
still owns the lease and the job hasn't been cancelled — preventing a
zombie worker from clobbering a takeover. Returns ``None`` on rejection.
Note: when ``lease_owner`` is set, the lease ownership *is* the
authorization (the executor calls this from a background task with no
user context), so ``user_id`` is left unresolved in that path.
"""
if lease_owner is not None and isinstance(user_id, _AutoSentinel):
resolved = None # lease-gated path — ownership is the gate, not user_id
else:
resolved = resolve_user_id(
user_id, method_name="DeepResearchJobRepository.update_progress"
)
values: dict[str, Any] = {"updated_at": datetime.now(UTC)}
if status is not _UNSET and status is not None:
values["status"] = status
if status in TERMINAL_STATUSES:
# Release the scope key + lease on terminal.
values["active_dedupe_key"] = None
values["lease_owner"] = None
values["lease_until"] = None
values["finished_at"] = datetime.now(UTC)
elif status == "awaiting_input":
# Paused: release the lease so a worker isn't pinned, keep the scope key.
values["lease_owner"] = None
values["lease_until"] = None
if phase is not _UNSET:
values["phase"] = phase
if progress is not _UNSET:
values["progress"] = progress
if current_query is not _UNSET:
values["current_query"] = current_query
if error_code is not _UNSET:
values["error_code"] = error_code
if error_message is not _UNSET:
values["error_message"] = error_message
if result_snapshot is not _UNSET:
values["result_snapshot"] = _dumps(result_snapshot)
if awaiting_interrupt_id is not _UNSET:
values["awaiting_interrupt_id"] = awaiting_interrupt_id
if awaiting_payload is not _UNSET:
values["awaiting_payload"] = _dumps(awaiting_payload)
values["version"] = DeepResearchJobRow.version + 1
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status.not_in(["cancelled"]),
]
if lease_owner is not None:
where.append(DeepResearchJobRow.lease_owner == lease_owner)
if resolved is not None and lease_owner is None:
where.append(DeepResearchJobRow.user_id == resolved)
async with self._sf() as session:
result = await session.execute(
update(DeepResearchJobRow).where(*where).values(**values)
)
if result.rowcount == 0:
return None
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
# ── two-phase cancel ────────────────────────────────────────────────────
async def request_cancel(
self, job_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> dict[str, Any] | None:
"""Phase 1: atomically flip ``cancel_requested`` + flag for the executor."""
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.request_cancel")
async with self._sf() as session:
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status.in_(list(CANCELLABLE_STATUSES)),
]
if resolved is not None:
where.append(DeepResearchJobRow.user_id == resolved)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(
cancel_requested=True,
cancel_requested_at=datetime.now(UTC),
version=DeepResearchJobRow.version + 1,
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
return None
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
async def finalize_cancel(
self, job_id: str, *, lease_owner: str | None = None
) -> dict[str, Any] | None:
"""Phase 2: ``cancel_requested`` → terminal ``cancelled`` (clears dedupe key)."""
async with self._sf() as session:
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.cancel_requested.is_(True),
]
if lease_owner is not None:
where.append(DeepResearchJobRow.lease_owner == lease_owner)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(
status="cancelled",
active_dedupe_key=None,
lease_owner=None,
lease_until=None,
version=DeepResearchJobRow.version + 1,
finished_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
return None
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
# ── resume after human-review interrupt ─────────────────────────────────
async def queue_for_resume(
self,
job_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
resume_payload: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Atomic ``awaiting_input`` → ``queued`` so the dispatcher re-claims it."""
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.queue_for_resume")
async with self._sf() as session:
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status == "awaiting_input",
]
if resolved is not None:
where.append(DeepResearchJobRow.user_id == resolved)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(
status="queued",
awaiting_interrupt_id=None,
awaiting_payload=_dumps(resume_payload),
lease_owner=None,
lease_until=None,
version=DeepResearchJobRow.version + 1,
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
return None
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
# ── cleanup ─────────────────────────────────────────────────────────────
async def list_older_than(self, *, cutoff: datetime, limit: int = 100) -> list[str]:
async with self._sf() as session:
stmt = (
select(DeepResearchJobRow.id)
.where(DeepResearchJobRow.status.in_(list(TERMINAL_STATUSES)))
.where(DeepResearchJobRow.updated_at < cutoff)
.limit(limit)
)
return [r for r in (await session.execute(stmt)).scalars().all()]
async def list_cancel_reapable(self, *, now: datetime, limit: int = 20) -> list[str]:
"""``cancel_requested`` jobs whose lease expired or was never held.
These are orphaned cancellations (the worker that set the flag crashed
before finalizing). The dispatcher reaps them so they don't stick in a
half-cancelled state forever (§14.3).
"""
async with self._sf() as session:
stmt = (
select(DeepResearchJobRow.id)
.where(DeepResearchJobRow.cancel_requested.is_(True))
.where(DeepResearchJobRow.status.not_in(list(TERMINAL_STATUSES)))
.where(
or_(
DeepResearchJobRow.lease_owner.is_(None),
(
DeepResearchJobRow.lease_until.is_not(None)
& (DeepResearchJobRow.lease_until < now)
),
)
)
.limit(limit)
)
return [r for r in (await session.execute(stmt)).scalars().all()]
@@ -0,0 +1,33 @@
"""Deep-research follow-up message persistence."""
from __future__ import annotations
from deerflow.persistence.deep_research_messages.model import DeepResearchMessageRow
from deerflow.persistence.deep_research_messages.sql import DeepResearchMessageRepository
def make_deep_research_message_store(session_factory) -> DeepResearchMessageRepository | None:
if session_factory is None:
return None
return DeepResearchMessageRepository(session_factory)
_default_store: DeepResearchMessageRepository | None = None
def register_default_store(store: DeepResearchMessageRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchMessageRepository | None:
return _default_store
__all__ = [
"DeepResearchMessageRepository",
"DeepResearchMessageRow",
"get_default_store",
"make_deep_research_message_store",
"register_default_store",
]
@@ -0,0 +1,35 @@
"""ORM model for deep-research follow-up messages.
After a report is complete, the user can ask follow-up questions that are
answered against the session's report + sources only (``allow_new_research=false``
default; §13.5 / §16.6). This table persists that conversation.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchMessageRow(Base):
__tablename__ = "deep_research_messages"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
role: Mapped[str] = mapped_column(String(16), nullable=False) # user / assistant
content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
# Source ids cited in an assistant reply.
citation_source_ids: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Token/cost usage for this message (JSON).
usage_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -0,0 +1,162 @@
"""SQLAlchemy repository for deep-research follow-up messages."""
from __future__ import annotations
import json
import logging
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_messages.model import DeepResearchMessageRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
logger = logging.getLogger(__name__)
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
class DeepResearchMessageRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: DeepResearchMessageRow) -> dict[str, Any]:
d = row.to_dict()
if "citation_source_ids" in d:
d["citation_source_ids"] = _loads(d["citation_source_ids"]) or []
if "usage_snapshot" in d:
d["usage_snapshot"] = _loads(d["usage_snapshot"])
return d
async def append(
self,
*,
id: str,
session_id: str,
user_id: str | _AutoSentinel = AUTO,
role: str,
content: str,
citation_source_ids: list[str] | None = None,
usage_snapshot: dict[str, Any] | None = None,
) -> dict[str, Any]:
resolved = resolve_user_id(user_id, method_name="DeepResearchMessageRepository.append")
async with self._sf() as session:
current_max = (
await session.execute(
select(func.max(DeepResearchMessageRow.seq)).where(
DeepResearchMessageRow.session_id == session_id
)
)
).scalar() or 0
row = DeepResearchMessageRow(
id=id,
session_id=session_id,
user_id=resolved,
role=role,
content=content,
citation_source_ids=_dumps(citation_source_ids),
usage_snapshot=_dumps(usage_snapshot),
seq=int(current_max) + 1,
)
session.add(row)
# Keep the successful append independent of a later replica read.
# The explicit id/seq and Python-side defaults are available after
# flush, before the transaction releases its writer connection.
await session.flush()
snapshot = self._row_to_dict(row)
await session.commit()
return snapshot
async def list_by_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, limit: int = 100
) -> list[dict[str, Any]]:
resolved = resolve_user_id(user_id, method_name="DeepResearchMessageRepository.list_by_session")
async with self._sf() as session:
stmt = select(DeepResearchMessageRow).where(
DeepResearchMessageRow.session_id == session_id
)
if resolved is not None:
stmt = stmt.where(DeepResearchMessageRow.user_id == resolved)
stmt = stmt.order_by(DeepResearchMessageRow.seq.asc()).limit(limit)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r) for r in rows]
async def get(
self,
message_id: str,
*,
session_id: str,
user_id: str | _AutoSentinel = AUTO,
) -> dict[str, Any] | None:
"""Fetch one follow-up message inside its owner-scoped session."""
resolved = resolve_user_id(user_id, method_name="DeepResearchMessageRepository.get")
async with self._sf() as session:
stmt = select(DeepResearchMessageRow).where(
DeepResearchMessageRow.id == message_id,
DeepResearchMessageRow.session_id == session_id,
)
if resolved is not None:
stmt = stmt.where(DeepResearchMessageRow.user_id == resolved)
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row) if row is not None else None
async def update_usage_snapshot(
self,
message_id: str,
*,
session_id: str,
usage_snapshot: dict[str, Any] | None,
user_id: str | _AutoSentinel = AUTO,
) -> dict[str, Any] | None:
"""Update durable UI metadata without altering the message transcript."""
resolved = resolve_user_id(
user_id,
method_name="DeepResearchMessageRepository.update_usage_snapshot",
)
async with self._sf() as session:
where = [
DeepResearchMessageRow.id == message_id,
DeepResearchMessageRow.session_id == session_id,
]
if resolved is not None:
where.append(DeepResearchMessageRow.user_id == resolved)
result = await session.execute(
update(DeepResearchMessageRow)
.where(*where)
.values(usage_snapshot=_dumps(usage_snapshot))
)
if not result.rowcount:
await session.rollback()
return None
stmt = select(DeepResearchMessageRow).where(*where)
row = (await session.execute(stmt)).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row is not None else None
await session.commit()
return snapshot
async def delete_by_session(self, session_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(
delete(DeepResearchMessageRow).where(
DeepResearchMessageRow.session_id == session_id
)
)
await session.commit()
return result.rowcount
@@ -0,0 +1,40 @@
"""Deep-research session persistence.
Exposes a store factory + a process-level default-store singleton, mirroring
``roundtable_jobs``: harness-side graph nodes call ``get_default_store()``;
``app/gateway/deps.py`` registers it at startup.
"""
from __future__ import annotations
from deerflow.persistence.deep_research_sessions.model import DeepResearchSessionRow
from deerflow.persistence.deep_research_sessions.sql import DeepResearchSessionRepository
def make_deep_research_session_store(
session_factory,
) -> DeepResearchSessionRepository | None:
if session_factory is None:
return None
return DeepResearchSessionRepository(session_factory)
_default_store: DeepResearchSessionRepository | None = None
def register_default_store(store: DeepResearchSessionRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchSessionRepository | None:
return _default_store
__all__ = [
"DeepResearchSessionRepository",
"DeepResearchSessionRow",
"get_default_store",
"make_deep_research_session_store",
"register_default_store",
]
@@ -0,0 +1,62 @@
"""ORM model for deep-research sessions.
A *session* is one research topic owned by a user: it holds the frozen config
snapshot, the latest report/plan, and points at the hidden runtime thread that
owns the artifact files. Multiple *jobs* (runs) can execute against a session
(re-run with different settings, resume after human review), but at most one
job is active at a time (enforced by the jobs table's ``active_dedupe_key``).
Field design follows ``roundtable_drafts`` / ``ai_writing_sessions``:
big JSON/text columns use ``PortableLongText``; timestamps use ``BeijingDateTime``.
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchSessionRow(Base):
__tablename__ = "deep_research_sessions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
query: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
# Research mode (basic/detailed/deep/multi_agent/quick) — denormalised for list views.
mode: Mapped[str] = mapped_column(String(32), nullable=False, default="basic", index=True)
# Session status machine: draft / running / awaiting_input / completed / failed / cancelled.
status: Mapped[str] = mapped_column(String(32), nullable=False, default="draft", index=True)
# Frozen config at creation/start time (full DeepResearchConfig JSON).
config_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# The hidden runtime thread that owns the artifacts (metadata.thread_type=deep_research).
runtime_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True, unique=True)
# Latest research plan (JSON) — updated by the planner node.
plan_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Latest report outputs.
report_markdown: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
report_html: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Denormalised counts for list views (avoid joining sources table).
source_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
# Token/cost usage snapshot (JSON: {model: {prompt, completion, total}}).
usage_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Current active job id (null when none running).
active_job_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
# User-facing error summary on failure.
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,274 @@
"""SQLAlchemy repository for deep-research sessions.
Mirrors the ``roundtable_jobs`` / ``ai_writing_sessions`` repository conventions:
constructed with an ``async_sessionmaker``, methods are ``async``, open
``async with self._sf() as session:`` blocks, and return **plain dicts**.
JSON columns round-trip through ``json.dumps`` / ``json.loads``.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_sessions.model import DeepResearchSessionRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
logger = logging.getLogger(__name__)
# Only these states have a durable job that consumes a per-user research slot.
# A draft is just an unsent composer record; counting it here makes a few saved
# drafts permanently trigger the start-job 429 guard.
IN_FLIGHT_STATUSES = ("running", "awaiting_input")
TERMINAL_STATUSES = ("completed", "failed", "cancelled")
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
class DeepResearchSessionRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# ── row ↔ dict ──────────────────────────────────────────────────────────
@staticmethod
def _row_to_dict(row: DeepResearchSessionRow) -> dict[str, Any]:
d = row.to_dict()
# Deserialize JSON columns.
for k in ("config_snapshot", "plan_snapshot", "usage_snapshot"):
if k in d:
d[k] = _loads(d[k])
return d
# ── create ──────────────────────────────────────────────────────────────
async def create(
self,
*,
id: str,
user_id: str | _AutoSentinel = AUTO,
title: str = "",
query: str = "",
mode: str = "basic",
status: str = "draft",
config_snapshot: dict[str, Any] | None = None,
runtime_thread_id: str | None = None,
) -> dict[str, Any]:
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.create")
async with self._sf() as session:
row = DeepResearchSessionRow(
id=id,
user_id=resolved,
title=title,
query=query,
mode=mode,
status=status,
config_snapshot=_dumps(config_snapshot),
runtime_thread_id=runtime_thread_id,
)
session.add(row)
# New sessions are a control-plane write. Avoid a post-commit
# refresh because MySQL read/write-splitting can make a just
# created session look absent on a lagging replica.
await session.flush()
snapshot = self._row_to_dict(row)
await session.commit()
return snapshot
# ── read ────────────────────────────────────────────────────────────────
async def get(self, session_id: str, *, user_id: str | _AutoSentinel = AUTO) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.get")
async with self._sf() as session:
stmt = select(DeepResearchSessionRow).where(DeepResearchSessionRow.id == session_id)
if resolved is not None:
stmt = stmt.where(DeepResearchSessionRow.user_id == resolved)
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def list_by_user(
self,
*,
user_id: str | _AutoSentinel = AUTO,
status: str | None = None,
limit: int = 20,
offset: int = 0,
include_config: bool = False,
) -> list[dict[str, Any]]:
"""Paginated session list for the history panel (excludes heavy report text)."""
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.list_by_user")
async with self._sf() as session:
stmt = select(DeepResearchSessionRow)
if resolved is not None:
stmt = stmt.where(DeepResearchSessionRow.user_id == resolved)
if status:
stmt = stmt.where(DeepResearchSessionRow.status == status)
stmt = stmt.order_by(DeepResearchSessionRow.updated_at.desc()).limit(limit).offset(offset)
rows = (await session.execute(stmt)).scalars().all()
# Strip heavy columns for list views.
out: list[dict[str, Any]] = []
for r in rows:
d = self._row_to_dict(r)
d.pop("report_markdown", None)
d.pop("report_html", None)
if not include_config:
d.pop("config_snapshot", None)
d.pop("plan_snapshot", None)
d.pop("usage_snapshot", None)
d.pop("error", None)
out.append(d)
return out
async def count_by_user(
self, *, user_id: str | _AutoSentinel = AUTO, status: str | None = None
) -> int:
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.count_by_user")
async with self._sf() as session:
stmt = select(func.count(DeepResearchSessionRow.id))
if resolved is not None:
stmt = stmt.where(DeepResearchSessionRow.user_id == resolved)
if status:
stmt = stmt.where(DeepResearchSessionRow.status == status)
return int((await session.execute(stmt)).scalar() or 0)
async def count_active_by_user(self, *, user_id: str | _AutoSentinel = AUTO) -> int:
"""Count sessions with an in-flight job for the per-user concurrency limit.
Draft sessions deliberately do not count: they have no job row and a
user may safely keep any number of unfinished research prompts.
"""
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.count_active_by_user")
async with self._sf() as session:
stmt = (
select(func.count(DeepResearchSessionRow.id))
.where(DeepResearchSessionRow.user_id == resolved)
.where(DeepResearchSessionRow.status.in_(list(IN_FLIGHT_STATUSES)))
)
return int((await session.execute(stmt)).scalar() or 0)
# ── update ──────────────────────────────────────────────────────────────
async def update(
self,
session_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
**fields: Any,
) -> dict[str, Any] | None:
"""Generic field update; JSON-valued fields are auto-serialized."""
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.update")
# Serialize known JSON columns.
if "config_snapshot" in fields and fields["config_snapshot"] is not None:
fields["config_snapshot"] = _dumps(fields["config_snapshot"])
if "plan_snapshot" in fields and fields["plan_snapshot"] is not None:
fields["plan_snapshot"] = _dumps(fields["plan_snapshot"])
if "usage_snapshot" in fields and fields["usage_snapshot"] is not None:
fields["usage_snapshot"] = _dumps(fields["usage_snapshot"])
fields["updated_at"] = datetime.now(UTC)
async with self._sf() as session:
where = [DeepResearchSessionRow.id == session_id]
if resolved is not None:
where.append(DeepResearchSessionRow.user_id == resolved)
result = await session.execute(
update(DeepResearchSessionRow).where(*where).values(**fields)
)
if result.rowcount == 0:
return None
row = (
await session.execute(
select(DeepResearchSessionRow).where(DeepResearchSessionRow.id == session_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
async def replace_report_if_unchanged(
self,
session_id: str,
*,
expected_report: str,
report_markdown: str,
user_id: str | _AutoSentinel = AUTO,
) -> dict[str, Any] | None:
"""Atomically replace a report only when its exact prior text still matches.
The caller may use a hash for user-facing diagnostics, but SQL compares
the text itself so another request cannot slip in between a Python-side
hash check and the update.
"""
resolved = resolve_user_id(
user_id,
method_name="DeepResearchSessionRepository.replace_report_if_unchanged",
)
async with self._sf() as session:
clauses = [
DeepResearchSessionRow.id == session_id,
DeepResearchSessionRow.report_markdown == expected_report,
]
if resolved is not None:
clauses.append(DeepResearchSessionRow.user_id == resolved)
result = await session.execute(
update(DeepResearchSessionRow)
.where(*clauses)
.values(
report_markdown=report_markdown,
report_html=None,
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
return None
row = (
await session.execute(
select(DeepResearchSessionRow).where(DeepResearchSessionRow.id == session_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
# ── delete ──────────────────────────────────────────────────────────────
async def delete(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> bool:
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.delete")
async with self._sf() as session:
where = [DeepResearchSessionRow.id == session_id]
if resolved is not None:
where.append(DeepResearchSessionRow.user_id == resolved)
result = await session.execute(delete(DeepResearchSessionRow).where(*where))
await session.commit()
return result.rowcount > 0
async def get_runtime_thread_id(self, session_id: str) -> str | None:
"""Look up the hidden thread id (for artifact cleanup on delete)."""
async with self._sf() as session:
row = (
await session.execute(
select(DeepResearchSessionRow.runtime_thread_id).where(
DeepResearchSessionRow.id == session_id
)
)
).scalar_one_or_none()
return row
@@ -0,0 +1,33 @@
"""Deep-research source-material persistence."""
from __future__ import annotations
from deerflow.persistence.deep_research_sources.model import DeepResearchSourceRow
from deerflow.persistence.deep_research_sources.sql import DeepResearchSourceRepository
def make_deep_research_source_store(session_factory) -> DeepResearchSourceRepository | None:
if session_factory is None:
return None
return DeepResearchSourceRepository(session_factory)
_default_store: DeepResearchSourceRepository | None = None
def register_default_store(store: DeepResearchSourceRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchSourceRepository | None:
return _default_store
__all__ = [
"DeepResearchSourceRepository",
"DeepResearchSourceRow",
"get_default_store",
"make_deep_research_source_store",
"register_default_store",
]
@@ -0,0 +1,68 @@
"""ORM model for deep-research source materials.
Stores every :class:`~deerflow.agents.deep_research.types.ResearchMaterial`
collected during a session, with full provenance (title, url, source,
``rec_uuid``, publish time, relevance) so report citations can trace back to
the originating DeerFlow record (§8.2 / §13.4).
Full text uses ``PortableLongText`` (MySQL ``LONGTEXT``) since web pages can be
large; the list endpoint returns summaries only (§16.5).
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Boolean, Float, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchSourceRow(Base):
__tablename__ = "deep_research_sources"
__table_args__ = (
# MySQL requires a prefix length when indexing TEXT columns. The
# column remains unbounded so long URLs are preserved; the prefix is
# only an index implementation detail and equality filters still
# compare the complete URL value.
sa.Index("ix_deep_research_sources_url", "url", mysql_length=512),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
job_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# The query that first surfaced this material.
first_seen_query: Mapped[str | None] = mapped_column(Text, nullable=True)
title: Mapped[str] = mapped_column(Text, nullable=False, default="")
url: Mapped[str | None] = mapped_column(Text, nullable=True)
raw_content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
snippet: Mapped[str | None] = mapped_column(Text, nullable=True)
source: Mapped[str] = mapped_column(String(256), nullable=False, default="")
source_type: Mapped[str] = mapped_column(String(32), nullable=False, default="web")
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
relevance_score: Mapped[float | None] = mapped_column(Float, nullable=True)
rec_uuid: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="")
# Whether this source was selected into the final context.
selected: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=sa.false()
)
selection_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
# Stable citation key assigned during report writing (e.g. "1", "2").
citation_key: Mapped[str | None] = mapped_column(String(16), nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,243 @@
"""SQLAlchemy repository for deep-research source materials."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_sources.model import DeepResearchSourceRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
logger = logging.getLogger(__name__)
_JSON_COLS = ("metadata",)
class DeepResearchSourceRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: DeepResearchSourceRow, *, include_content: bool = True) -> dict[str, Any]:
d = row.to_dict()
if not include_content:
d.pop("raw_content", None)
d.pop("snippet", None)
return d
# ── upsert (dedupe by rec_uuid / url / content_hash) ────────────────────
async def upsert(self, *, session_id: str, job_id: str | None = None, **fields: Any) -> dict[str, Any]:
"""Insert a source, or return the existing one if rec_uuid/url/hash matches."""
# Try insert.
async with self._sf() as session:
row = DeepResearchSourceRow(session_id=session_id, job_id=job_id, **fields)
session.add(row)
try:
# The source was produced on the worker's write path. Build
# its response on that same connection so a replica-lag read
# cannot turn a successfully collected source into a failed
# research run.
await session.flush()
snapshot = self._row_to_dict(row)
await session.commit()
return snapshot
except Exception:
await session.rollback()
# Fallback: find existing by rec_uuid or url.
existing = await self._find_existing(
session_id=session_id,
rec_uuid=fields.get("rec_uuid"),
url=fields.get("url"),
content_hash=fields.get("content_hash", ""),
)
return existing or self._row_to_dict(row)
async def _find_existing(
self, *, session_id: str, rec_uuid: str | None, url: str | None, content_hash: str
) -> dict[str, Any] | None:
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(
DeepResearchSourceRow.session_id == session_id
)
if rec_uuid:
stmt = stmt.where(DeepResearchSourceRow.rec_uuid == rec_uuid)
elif url:
stmt = stmt.where(DeepResearchSourceRow.url == url)
elif content_hash:
stmt = stmt.where(DeepResearchSourceRow.content_hash == content_hash)
else:
return None
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── read ────────────────────────────────────────────────────────────────
async def get(
self,
source_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
include_content: bool = True,
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.get")
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(DeepResearchSourceRow.id == source_id)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row, include_content=include_content) if row else None
async def list_by_session(
self,
session_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
selected: bool | None = None,
limit: int = 50,
offset: int = 0,
include_content: bool = False,
) -> list[dict[str, Any]]:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.list_by_session")
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(DeepResearchSourceRow.session_id == session_id)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
if selected is not None:
stmt = stmt.where(DeepResearchSourceRow.selected == selected)
stmt = stmt.order_by(DeepResearchSourceRow.created_at.asc()).limit(limit).offset(offset)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r, include_content=include_content) for r in rows]
async def count_by_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, selected: bool | None = None
) -> int:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.count_by_session")
async with self._sf() as session:
stmt = select(func.count(DeepResearchSourceRow.id)).where(
DeepResearchSourceRow.session_id == session_id
)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
if selected is not None:
stmt = stmt.where(DeepResearchSourceRow.selected == selected)
return int((await session.execute(stmt)).scalar() or 0)
async def clone_selected_to_session(
self,
source_session_id: str,
target_session_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
) -> dict[str, str]:
"""Copy the selected evidence into a new report session.
Source ids are globally unique primary keys, so a report variant cannot
reuse the parent rows. Return the ``old_id -> new_id`` mapping to let
the caller repair the copied Markdown citation markers before the
variant becomes visible.
"""
resolved = resolve_user_id(
user_id,
method_name="DeepResearchSourceRepository.clone_selected_to_session",
)
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(
DeepResearchSourceRow.session_id == source_session_id,
DeepResearchSourceRow.selected.is_(True),
)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
source_rows = (await session.execute(stmt)).scalars().all()
if not source_rows:
# Collector persist marks rows selected, but an older session
# may only have unselected harvest rows. Cloning nothing would
# make 「选择结构再生成」 write with no materials.
stmt = select(DeepResearchSourceRow).where(
DeepResearchSourceRow.session_id == source_session_id,
)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
source_rows = (await session.execute(stmt)).scalars().all()
id_map: dict[str, str] = {}
cloned_rows: list[DeepResearchSourceRow] = []
for source in source_rows:
cloned_id = f"drs_src_{uuid.uuid4().hex}"
id_map[source.id] = cloned_id
cloned_rows.append(
DeepResearchSourceRow(
id=cloned_id,
session_id=target_session_id,
job_id=None,
user_id=resolved if resolved is not None else source.user_id,
first_seen_query=source.first_seen_query,
title=source.title,
url=source.url,
raw_content=source.raw_content,
snippet=source.snippet,
source=source.source,
source_type=source.source_type,
published_at=source.published_at,
relevance_score=source.relevance_score,
rec_uuid=source.rec_uuid,
content_hash=source.content_hash,
selected=True,
selection_reason=source.selection_reason,
citation_key=source.citation_key,
)
)
if cloned_rows:
session.add_all(cloned_rows)
await session.flush()
await session.commit()
return id_map
# ── update selection ────────────────────────────────────────────────────
async def set_selected(
self,
source_id: str,
*,
selected: bool,
reason: str | None = None,
citation_key: str | None = None,
user_id: str | _AutoSentinel = AUTO,
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.set_selected")
async with self._sf() as session:
where = [DeepResearchSourceRow.id == source_id]
if resolved is not None:
where.append(DeepResearchSourceRow.user_id == resolved)
values: dict[str, Any] = {"selected": selected}
if reason is not None:
values["selection_reason"] = reason
if citation_key is not None:
values["citation_key"] = citation_key
result = await session.execute(
update(DeepResearchSourceRow).where(*where).values(**values)
)
if result.rowcount == 0:
return None
row = (
await session.execute(
select(DeepResearchSourceRow).where(DeepResearchSourceRow.id == source_id)
)
).scalar_one_or_none()
snapshot = self._row_to_dict(row) if row else None
await session.commit()
return snapshot
async def delete_by_session(self, session_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(
delete(DeepResearchSourceRow).where(DeepResearchSourceRow.session_id == session_id)
)
await session.commit()
return result.rowcount
@@ -0,0 +1,17 @@
"""Document rewrite version persistence factory."""
from deerflow.persistence.document_rewrite_versions.model import DocumentRewriteVersionRow
from deerflow.persistence.document_rewrite_versions.sql import DocumentRewriteVersionRepository
def make_document_rewrite_version_store(session_factory) -> DocumentRewriteVersionRepository | None:
if session_factory is None:
return None
return DocumentRewriteVersionRepository(session_factory)
__all__ = [
"DocumentRewriteVersionRepository",
"DocumentRewriteVersionRow",
"make_document_rewrite_version_store",
]
@@ -0,0 +1,34 @@
"""Persistent, undoable snapshots for whole-document AI rewrites."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DocumentRewriteVersionRow(Base):
"""The immutable pre-rewrite document retained for one safe undo action."""
__tablename__ = "document_rewrite_versions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# ``artifact`` uses ``source_id=thread_id`` and a virtual sandbox path;
# ``deep_research_report`` uses ``source_id=deep-research session id``.
source_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
source_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
source_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
original_content: Mapped[str] = mapped_column(PortableLongText, nullable=False)
original_hash: Mapped[str] = mapped_column(String(64), nullable=False)
committed_hash: Mapped[str] = mapped_column(String(64), nullable=False)
instruction: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
model_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
reverted_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -0,0 +1,153 @@
"""Repository for document-rewrite snapshots.
The caller performs the actual content compare-and-swap. This repository keeps
the retained original version owner-scoped and marks a snapshot consumed after a
successful undo, preventing an older undo from silently overwriting newer work.
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.document_rewrite_versions.model import DocumentRewriteVersionRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
class DocumentRewriteVersionRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: DocumentRewriteVersionRow) -> dict[str, Any]:
return row.to_dict()
async def create(
self,
*,
id: str,
source_type: str,
source_id: str,
source_path: str | None,
original_content: str,
original_hash: str,
committed_hash: str,
instruction: str | None = None,
model_name: str | None = None,
user_id: str | _AutoSentinel = AUTO,
) -> dict[str, Any]:
resolved = resolve_user_id(user_id, method_name="DocumentRewriteVersionRepository.create")
async with self._sf() as session:
row = DocumentRewriteVersionRow(
id=id,
user_id=resolved,
source_type=source_type,
source_id=source_id,
source_path=source_path,
original_content=original_content,
original_hash=original_hash,
committed_hash=committed_hash,
instruction=instruction,
model_name=model_name,
)
session.add(row)
# Do not refresh after commit. In a read/write-splitting MySQL
# deployment that read can be routed to a lagging replica even
# though the INSERT succeeded on the writer. The caller treats a
# version-record failure as a reason to restore the document, so
# that replica-only error would otherwise undo a valid rewrite.
# Flush materializes the Python/SQLAlchemy defaults on the writer
# and lets us return a complete snapshot without a second query.
await session.flush()
snapshot = self._row_to_dict(row)
await session.commit()
return snapshot
async def get(
self,
version_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
source_type: str | None = None,
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DocumentRewriteVersionRepository.get")
async with self._sf() as session:
clauses = [DocumentRewriteVersionRow.id == version_id]
if resolved is not None:
clauses.append(DocumentRewriteVersionRow.user_id == resolved)
if source_type:
clauses.append(DocumentRewriteVersionRow.source_type == source_type)
row = (await session.execute(select(DocumentRewriteVersionRow).where(*clauses))).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def mark_reverted(
self,
version_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
) -> bool:
resolved = resolve_user_id(user_id, method_name="DocumentRewriteVersionRepository.mark_reverted")
async with self._sf() as session:
clauses = [
DocumentRewriteVersionRow.id == version_id,
DocumentRewriteVersionRow.reverted_at.is_(None),
]
if resolved is not None:
clauses.append(DocumentRewriteVersionRow.user_id == resolved)
result = await session.execute(
update(DocumentRewriteVersionRow)
.where(*clauses)
.values(reverted_at=datetime.now(UTC))
)
await session.commit()
return result.rowcount > 0
async def prune_source(
self,
*,
source_type: str,
source_id: str,
source_path: str | None,
keep: int = 20,
user_id: str | _AutoSentinel = AUTO,
) -> int:
"""Retain the newest snapshots for one document without crossing owners.
Old rewrite snapshots contain complete document bodies, so this is both a
storage guardrail and an explicit product boundary: users can undo the
recent history, not retain an unbounded hidden copy of every document.
"""
if keep < 1:
raise ValueError("keep must be at least one")
resolved = resolve_user_id(user_id, method_name="DocumentRewriteVersionRepository.prune_source")
async with self._sf() as session:
clauses = [
DocumentRewriteVersionRow.source_type == source_type,
DocumentRewriteVersionRow.source_id == source_id,
DocumentRewriteVersionRow.source_path == source_path,
]
if resolved is not None:
clauses.append(DocumentRewriteVersionRow.user_id == resolved)
stale_ids = list(
(
await session.scalars(
select(DocumentRewriteVersionRow.id)
.where(*clauses)
.order_by(
DocumentRewriteVersionRow.created_at.desc(),
DocumentRewriteVersionRow.id.desc(),
)
.offset(keep)
)
).all()
)
if not stale_ids:
return 0
result = await session.execute(
delete(DocumentRewriteVersionRow).where(DocumentRewriteVersionRow.id.in_(stale_ids))
)
await session.commit()
return int(result.rowcount or 0)
@@ -0,0 +1,18 @@
"""Persistence for iframe embed session → thread mappings."""
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
from deerflow.persistence.embed_sessions.memory import MemoryEmbedSessionStore
from deerflow.persistence.embed_sessions.sql import EmbedSessionRepository
__all__ = [
"EmbedSessionStore",
"MemoryEmbedSessionStore",
"EmbedSessionRepository",
"make_embed_session_store",
]
def make_embed_session_store(session_factory):
if session_factory is None:
return MemoryEmbedSessionStore()
return EmbedSessionRepository(session_factory)
@@ -0,0 +1,18 @@
"""Abstract persistence API for iframe embed session → thread mappings."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class EmbedSessionStore(ABC):
@abstractmethod
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
"""Create or update the mapping for an embed session."""
raise NotImplementedError
@abstractmethod
async def get(self, session_id: str) -> dict[str, Any] | None:
"""Return ``{session_id, thread_id, created_at, updated_at}`` or None."""
raise NotImplementedError
@@ -0,0 +1,37 @@
"""In-memory ``EmbedSessionStore`` for tests / memory backend."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
class MemoryEmbedSessionStore(EmbedSessionStore):
def __init__(self) -> None:
self._sessions: dict[str, dict[str, Any]] = {}
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
now = datetime.now(UTC).isoformat()
existing = self._sessions.get(session_id)
if existing is None:
record = {
"session_id": session_id,
"thread_id": thread_id,
"created_at": now,
"updated_at": now,
}
else:
record = {
**existing,
"thread_id": thread_id,
"updated_at": now,
}
self._sessions[session_id] = record
return copy.deepcopy(record)
async def get(self, session_id: str) -> dict[str, Any] | None:
record = self._sessions.get(session_id)
return copy.deepcopy(record) if record is not None else None
@@ -0,0 +1,21 @@
"""ORM model for iframe embed session → thread lookups."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class EmbedSessionRow(Base):
__tablename__ = "embed_sessions"
# Opaque id generated by the parent page and passed into the iframe URL.
session_id: Mapped[str] = mapped_column(String(128), primary_key=True)
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -0,0 +1,49 @@
"""SQLAlchemy-backed embed session repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
def _row_to_dict(row: EmbedSessionRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
return data
class EmbedSessionRepository(EmbedSessionStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(EmbedSessionRow, session_id)
now = datetime.now(UTC)
if row is None:
row = EmbedSessionRow(
session_id=session_id,
thread_id=thread_id,
created_at=now,
updated_at=now,
)
session.add(row)
else:
row.thread_id = thread_id
row.updated_at = now
await session.commit()
await session.refresh(row)
return _row_to_dict(row)
async def get(self, session_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(EmbedSessionRow, session_id)
return _row_to_dict(row) if row is not None else None
@@ -0,0 +1,735 @@
"""Async SQLAlchemy engine lifecycle management.
Initializes at Gateway startup, provides session factory for
repositories, disposes at shutdown.
When database.backend="memory", init_engine is a no-op and
get_session_factory() returns None. Repositories must check for
None and fall back to in-memory implementations.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import sys
from importlib.util import find_spec
from pathlib import Path
from sqlalchemy import text
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
def _json_serializer(obj: object) -> str:
"""JSON serializer with ensure_ascii=False for Chinese character support."""
return json.dumps(obj, ensure_ascii=False)
logger = logging.getLogger(__name__)
_engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
# The event loop on which the async engine was created. asyncmy/asyncpg connection
# pools bind their connections to the loop that opened them; awaiting a write on a
# *different* loop reuses those connections cross-loop and raises
# "got Future attached to a different loop". We capture the engine's home loop here so
# best-effort writes coming from worker threads / other loops can be routed back to it
# (see ``fire_and_forget_db_write``).
_engine_loop: asyncio.AbstractEventLoop | None = None
# These are the complete workflow-studio persistence tables owned by DeerFlow.
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
# wildcard: another application might share a schema and have its own table
# with that prefix.
_WORKFLOW_SCHEMA_TABLE_NAMES = (
"workflow_definitions",
"workflow_versions",
"workflow_runs",
"workflow_node_runs",
"workflow_run_events",
"workflow_run_artifacts",
"workflow_data_sources",
)
def _add_runtime_vendor_path() -> None:
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
The offline Docker image keeps the Python environment inside the image, while
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
unchanged startup script can still import it after container recreation.
"""
try:
from deerflow.config.runtime_paths import runtime_home
vendor_dir = runtime_home() / "python-vendor"
except Exception:
vendor_dir = Path(".deer-flow/python-vendor").resolve()
vendor = str(vendor_dir)
if vendor_dir.exists() and vendor not in sys.path:
sys.path.insert(0, vendor)
def _mysql_driver_name(url: str) -> str:
if url.startswith("mysql+aiomysql://"):
return "aiomysql"
return "asyncmy"
async def _run_alembic_upgrade(url: str) -> None:
"""Apply pending Alembic migrations against ``url`` (``alembic upgrade head``).
Runs after ``Base.metadata.create_all`` so brand-new deployments still get
a complete schema even without Alembic, while existing deployments pick up
in-place schema fixes (column widenings, new indexes, etc.) at the next
restart — no manual ``alembic upgrade`` step required.
Notes:
* Each historical migration in ``migrations/versions/`` already guards
itself with ``_has_table`` / ``_has_column``, so re-running them against
a database that was originally built via ``create_all`` is a sequence of
no-ops up to the first migration that actually has work to do.
* The Alembic ``env.py`` calls ``asyncio.run`` internally, which cannot run
inside an already-running loop — so we hop into a worker thread.
* Failures here are logged but do not abort startup. The app remains
operable on the ``create_all`` schema; the operator just won't get
in-place fixes until they investigate.
"""
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in ("1", "true", "yes"):
logger.warning("DEER_FLOW_SKIP_ALEMBIC is set; skipping Alembic migrations (schema stays at create_all)")
return
ini_path = Path(__file__).parent / "migrations" / "alembic.ini"
if not ini_path.exists():
logger.warning("Alembic config not found at %s; skipping auto-migrate", ini_path)
return
# Alembic's ``Config`` is backed by ConfigParser, which performs ``%``
# interpolation on every option value. URL-encoded credentials such as
# ``kland%4088124932`` (the ``@`` in ``kland@88124932``) trip that up with
# ``ValueError: invalid interpolation syntax ... at position N``. Doubling
# the percent signs neutralises the interpolation; ConfigParser un-doubles
# them back to the original value when read.
escaped_url = url.replace("%", "%%")
def _do_upgrade() -> None:
# Import inside the worker so a missing Alembic install can't break
# the engine import chain.
from alembic import command
from alembic.config import Config
cfg = Config(str(ini_path))
cfg.set_main_option("sqlalchemy.url", escaped_url)
# The ``script_location`` in alembic.ini is ``%(here)s`` which points
# at the directory of alembic.ini itself — already correct for us.
command.upgrade(cfg, "head")
# Bound the upgrade so a stuck DDL (e.g. MySQL metadata-lock wait) cannot
# hang startup forever. Tunable via DEER_FLOW_ALEMBIC_TIMEOUT (seconds).
# NOTE: asyncio.wait_for cancels the *await* on timeout; the worker thread
# itself keeps running (Python can't kill threads). That's acceptable —
# the orphan thread will exit when its MySQL connection eventually errors
# or when the process restarts. The point here is to unblock startup.
try:
timeout_s = float(os.getenv("DEER_FLOW_ALEMBIC_TIMEOUT", "60"))
except ValueError:
timeout_s = 60.0
try:
await asyncio.wait_for(asyncio.to_thread(_do_upgrade), timeout=timeout_s)
logger.info("Alembic migrations applied (or already up-to-date)")
except TimeoutError:
logger.error(
"Alembic upgrade did not finish within %.0fs; continuing startup on create_all schema. "
"Likely cause: MySQL metadata-lock wait — check SHOW FULL PROCESSLIST for a blocking transaction.",
timeout_s,
)
except Exception:
logger.exception("Auto-applying Alembic migrations failed; continuing with create_all schema")
async def _auto_create_postgres_db(url: str) -> None:
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
The target database name is extracted from *url*. The connection is
made to the default ``postgres`` database on the same server using
``AUTOCOMMIT`` isolation (CREATE DATABASE cannot run inside a
transaction).
"""
from sqlalchemy import text
from sqlalchemy.engine.url import make_url
parsed = make_url(url)
db_name = parsed.database
if not db_name:
raise ValueError("Cannot auto-create database: no database name in URL")
# Connect to the default 'postgres' database to issue CREATE DATABASE
maint_url = parsed.set(database="postgres")
maint_engine = create_async_engine(maint_url, isolation_level="AUTOCOMMIT")
try:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
finally:
await maint_engine.dispose()
async def _auto_create_mysql_db(url: str) -> None:
"""Create the target MySQL database when it is missing.
``create_all`` can create tables, but MySQL requires the database/schema to
exist before the application URL can connect. We connect to the built-in
``mysql`` maintenance database on the same server and issue a conservative
``CREATE DATABASE IF NOT EXISTS``.
"""
parsed = make_url(url)
db_name = parsed.database
if not db_name:
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
maint_url = parsed.set(database="mysql")
maint_engine = create_async_engine(
maint_url.render_as_string(hide_password=False),
isolation_level="AUTOCOMMIT",
pool_pre_ping=True,
)
quoted = "`" + db_name.replace("`", "``") + "`"
try:
async with maint_engine.connect() as conn:
await conn.execute(
text(
f"CREATE DATABASE IF NOT EXISTS {quoted} "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
)
)
logger.info("Auto-created MySQL database if missing: %s", db_name)
finally:
await maint_engine.dispose()
def _ensure_orm_columns_sync(sync_conn) -> None:
"""Add ORM-declared columns that are missing from already-existing tables.
``Base.metadata.create_all`` creates brand-new tables but never patches
existing ones, and Alembic auto-upgrade is disabled (see the commented-out
``_run_alembic_upgrade`` call) — so on an existing database a new
``mapped_column`` on an ORM model would otherwise error at first query
(e.g. ``no such column: roundtable_drafts.task_id`` on SQLite).
This sweep compares each existing table against the ORM metadata and issues
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
safely (nullable, or carrying a server_default). Indexes covering a newly
added column are created as well. Table-level ``UniqueConstraint`` objects
that are declared on the model but missing from the table are also created
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
on an existing database, weakening dedupe.
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
``BeijingDateTime``) resolve to the right backend type. Everything is
per-column best-effort: a failure is logged and startup continues.
"""
from sqlalchemy import UniqueConstraint
from sqlalchemy import inspect as sa_inspect
from sqlalchemy.schema import CreateColumn
from deerflow.persistence.base import Base
try:
import deerflow.persistence.models # noqa: F401 — register all ORM models on Base.metadata
except ImportError:
pass
inspector = sa_inspect(sync_conn)
existing_tables = set(inspector.get_table_names())
preparer = sync_conn.dialect.identifier_preparer
for table in Base.metadata.sorted_tables:
if table.name not in existing_tables:
continue
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
missing = [c for c in table.columns if c.name not in existing_cols]
if not missing:
continue
added: set[str] = set()
for col in missing:
if not col.nullable and col.server_default is None:
logger.warning(
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
table.name,
col.name,
)
continue
try:
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
added.add(col.name)
logger.info("Auto-added missing column %s.%s", table.name, col.name)
except Exception:
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
if not added:
continue
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
continue
try:
idx.create(sync_conn)
logger.info("Auto-created index %s on %s", idx.name, table.name)
except Exception:
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
# ── 表级 UNIQUE 约束补齐 ──────────────────────────────────────────────────
# create_all 只为全新表建 UniqueConstraint;既有表上新增声明的唯一约束(如
# roundtable_jobs.active_dedupe_key)会静默缺失,去重/幂等的正确性就靠它,故
# 这里显式补建。迁移可能用 CREATE UNIQUE INDEX(同名)实现唯一性,故同时查
# get_unique_constraints 与 get_indexes 的名字集合,避免重复建。
for table in Base.metadata.sorted_tables:
if table.name not in existing_tables:
continue
named_uqs = [
c for c in table.constraints
if isinstance(c, UniqueConstraint) and c.name is not None
]
if not named_uqs:
continue
try:
existing_uq_names = {
uq["name"] for uq in inspector.get_unique_constraints(table.name) if uq.get("name")
}
except Exception:
existing_uq_names = set()
try:
existing_idx_names = {
ix["name"] for ix in inspector.get_indexes(table.name) if ix.get("name")
}
except Exception:
existing_idx_names = set()
for uq in named_uqs:
if uq.name in existing_uq_names or uq.name in existing_idx_names:
continue
# 用 CREATE UNIQUE INDEX(而非 ALTER TABLE ADD CONSTRAINT):迁移本身也用
# CREATE UNIQUE INDEX,且它在 SQLite/MySQL/PostgreSQL 都受支持(SQLite 不
# 支持 ALTER TABLE ADD CONSTRAINT),功能等价地强制唯一性。
cols = ", ".join(preparer.quote(c.name) for c in uq.columns)
try:
sync_conn.exec_driver_sql(
f"CREATE UNIQUE INDEX {preparer.quote(uq.name)} "
f"ON {preparer.quote(table.name)} ({cols})"
)
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
async def init_engine(
backend: str,
*,
url: str = "",
echo: bool = False,
pool_size: int = 5,
max_overflow: int = 10,
pool_timeout: int = 30,
pool_recycle: int = 1800,
sqlite_dir: str = "",
) -> None:
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
echo: Echo SQL to log.
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
global _engine, _session_factory, _engine_loop
if backend == "memory":
logger.info("Persistence backend=memory -- ORM engine not initialized")
return
# Remember the loop that owns the engine's connection pool, so off-loop
# best-effort writes (metrics from worker threads, etc.) can be routed here
# instead of reusing pooled connections cross-loop.
try:
_engine_loop = asyncio.get_running_loop()
except RuntimeError:
_engine_loop = None
if backend in {"postgres", "gauss"}:
try:
import asyncpg # noqa: F401
except ImportError:
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
if find_spec(driver) is None:
raise ImportError(
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
)
if backend == "sqlite":
import os
from sqlalchemy import event
os.makedirs(sqlite_dir or ".", exist_ok=True)
_engine = create_async_engine(url, echo=echo, json_serializer=_json_serializer)
# Enable WAL on every new connection. SQLite PRAGMA settings are
# per-connection, so we wire the listener instead of running PRAGMA
# once at startup. WAL gives concurrent reads + writers without
# blocking and is the standard recommendation for any production
# SQLite deployment (TC-UPG-06 in AUTH_TEST_PLAN.md). The companion
# ``synchronous=NORMAL`` is the safe-and-fast pairing — fsync only
# at WAL checkpoint boundaries instead of every commit.
# Note: we do not set PRAGMA busy_timeout here — Python's sqlite3
# driver already defaults to a 5-second busy timeout (see the
# ``timeout`` kwarg of ``sqlite3.connect``), and aiosqlite /
# SQLAlchemy's aiosqlite dialect inherit that default. Setting
# it again would be a no-op.
@event.listens_for(_engine.sync_engine, "connect")
def _enable_sqlite_wal(dbapi_conn, _record): # noqa: ARG001 — SQLAlchemy contract
cursor = dbapi_conn.cursor()
try:
cursor.execute("PRAGMA journal_mode=WAL;")
cursor.execute("PRAGMA synchronous=NORMAL;")
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend in {"postgres", "gauss"}:
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
elif backend == "mysql":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
else:
raise ValueError(f"Unknown persistence backend: {backend!r}")
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
# Auto-create tables for fresh deployments, then let Alembic apply any
# in-place schema fixes on top (see ``_run_alembic_upgrade`` below). Both
# are idempotent so repeating them across restarts is harmless.
from deerflow.persistence.base import Base
# Import all models so Base.metadata discovers them.
# When no models exist yet (scaffolding phase), this is a no-op.
try:
import deerflow.persistence.models # noqa: F401
except ImportError:
# Models package not yet available — tables won't be auto-created.
# This is expected during initial scaffolding or minimal installs.
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database
await _engine.dispose()
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, pool_recycle=pool_recycle, json_serializer=_json_serializer)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
else:
raise
# Patch existing tables with ORM-declared columns that ``create_all``
# cannot add (it only creates brand-new tables). Best-effort: a failure is
# logged and startup continues on the current schema.
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
except Exception:
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
async def init_engine_from_config(config) -> None:
"""Convenience: init engine from a DatabaseConfig object."""
if config.backend == "memory":
await init_engine("memory")
return
await init_engine(
backend=config.backend,
url=config.app_sqlalchemy_url,
echo=config.echo_sql,
pool_size=config.pool_size,
max_overflow=getattr(config, "max_overflow", 10),
pool_timeout=getattr(config, "pool_timeout", 30),
pool_recycle=getattr(config, "pool_recycle", 1800),
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
)
def get_session_factory() -> async_sessionmaker[AsyncSession] | None:
"""Return the async session factory, or None if backend=memory."""
return _session_factory
def get_engine() -> AsyncEngine | None:
"""Return the async engine, or None if not initialized."""
return _engine
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
"""Return the event loop that owns the engine's connection pool (or None)."""
return _engine_loop
async def rebuild_workflow_schema_for_mysql() -> list[str]:
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
This is a deliberately narrow recovery operation for an unused Workflow
Studio installation. It never uses a prefix wildcard, never changes the
selected database, and refuses to run unless the engine is connected to
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
Returns the allow-listed workflow tables that existed before the rebuild.
"""
engine = get_engine()
if engine is None:
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
if engine.dialect.name != "mysql":
raise RuntimeError(
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
)
expected_database = engine.url.database
if not expected_database:
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
# Make the operation safe when this helper is invoked independently from
# normal app startup as well.
import deerflow.persistence.models # noqa: F401 — register ORM models
from deerflow.persistence.base import Base
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
if missing_models:
raise RuntimeError(
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
)
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
def _rebuild(sync_conn) -> list[str]:
from sqlalchemy import inspect as sa_inspect
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
if current_database != expected_database:
raise RuntimeError(
"Refusing workflow schema rebuild: connected database "
f"{current_database!r} does not match configured database {expected_database!r}"
)
existing = set(sa_inspect(sync_conn).get_table_names())
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
# SQLAlchemy orders parent/child tables around the foreign key from
# workflow_versions to workflow_definitions. Only tables in the exact
# allow-list above are ever passed to drop_all/create_all.
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
return present
async with engine.begin() as conn:
rebuilt = await conn.run_sync(_rebuild)
logger.warning(
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
expected_database,
", ".join(rebuilt) if rebuilt else "none (created fresh)",
)
return rebuilt
def fire_and_forget_db_write(coro) -> None:
"""Schedule a best-effort DB write on the engine's home loop, from any loop/thread.
The async engine's connection pool is bound to the loop that created it
(``init_engine``, normally the Gateway's main loop). Running a write on a
*different* loop — e.g. a model/tool call that LangGraph executed in a worker
thread, which then does ``asyncio.run(store.record(...))`` — reuses those pooled
connections cross-loop and raises ``got Future attached to a different loop``.
Under concurrency this happens constantly (many calls land in worker threads at
once), spamming the log and churning the pool; single-threaded it almost never does.
This routes every such write back onto the engine's home loop:
- already on that loop -> ``create_task`` (in-loop fire-and-forget);
- on another loop/thread -> ``run_coroutine_threadsafe`` to the home loop.
The write itself stays best-effort: if no engine loop is captured (memory backend
/ not initialized) the coroutine is closed (avoids a "never awaited" warning) and
the call is a no-op. Never raises.
"""
loop = _engine_loop
if loop is None or loop.is_closed():
close = getattr(coro, "close", None)
if callable(close):
close()
return
try:
running = asyncio.get_running_loop()
except RuntimeError:
running = None
try:
if running is loop:
loop.create_task(coro)
else:
asyncio.run_coroutine_threadsafe(coro, loop)
except Exception: # noqa: BLE001 — best-effort; scheduling must never break the caller
close = getattr(coro, "close", None)
if callable(close):
close()
logger.debug("fire_and_forget_db_write: failed to schedule DB write", exc_info=True)
def _run_coro_on_throwaway_loop(coro):
"""Last-resort: run *coro* to completion on a fresh event loop.
Only reached when there is no usable engine home loop, or when the caller is
*synchronously* inside the home loop (routing back would deadlock). This
touches the shared connection pool from a throwaway loop, so it can still leak
a dead-loop-bound connection — but the common worker-thread path no longer
gets here (see ``run_db_blocking``).
"""
try:
asyncio.get_running_loop()
in_loop = True
except RuntimeError:
in_loop = False
if in_loop:
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
return pool.submit(asyncio.run, coro).result()
return asyncio.run(coro)
def run_db_blocking(coro, *, timeout: float | None = 30.0):
"""Run an awaited DB coroutine from sync code, safe w.r.t. the engine's home loop.
The blocking-read sibling of :func:`fire_and_forget_db_write`: where that one
drops the result, this returns it — used by the sync→async bridges that must
read a row before proceeding (thread-path owner resolution, always-on skills,
skill visibility).
The async engine's connection pool is bound to the loop that created it
(``init_engine``). Running a DB coroutine on a *throwaway* loop
(``asyncio.run``) checks out pooled connections cross-loop and **leaks** them:
the connection ends up bound to a loop that is then closed, and when the pool
later recycles/disposes it, asyncmy's ``_terminate_graceful_close`` fires on the
wrong loop and raises ``got Future <...> attached to a different loop``. Under
concurrency (scheduler / subagent / sandbox-tool worker threads each spinning a
fresh loop) this floods the logs.
So when we are **off** the home loop — the common case, a worker thread with no
running loop — route the coroutine onto the home loop and block the *caller*
thread for its result. The pool is only ever touched on its own loop, so nothing
leaks. Only when there is no home loop, or we are synchronously inside it (where
blocking would deadlock), do we fall back to a throwaway loop.
Never swallows the coroutine's own exception — the sync bridges wrap this call in
their own try/except and degrade gracefully.
"""
loop = _engine_loop
try:
running = asyncio.get_running_loop()
except RuntimeError:
running = None
if loop is not None and not loop.is_closed() and running is not loop:
# Off the home loop (worker thread, or a different loop): execute the coro
# on the engine's loop and block this thread for the result. asyncmy's
# pooled connections stay on their own loop, so none leak cross-loop.
return asyncio.run_coroutine_threadsafe(coro, loop).result(timeout)
# No home loop, or we're already synchronously inside it: last resort.
return _run_coro_on_throwaway_loop(coro)
async def close_engine() -> None:
"""Dispose the engine, release all connections."""
global _engine, _session_factory, _engine_loop
if _engine is not None:
await _engine.dispose()
logger.info("Persistence engine closed")
_engine = None
_session_factory = None
_engine_loop = None
@@ -0,0 +1,31 @@
"""Enterprise-research task persistence."""
from deerflow.persistence.enterprise_research.base import EnterpriseResearchReportJobStore, EnterpriseResearchTaskStore
from deerflow.persistence.enterprise_research.memory import MemoryEnterpriseResearchReportJobStore, MemoryEnterpriseResearchTaskStore
from deerflow.persistence.enterprise_research.model import EnterpriseResearchReportJobRow, EnterpriseResearchTaskRow
from deerflow.persistence.enterprise_research.sql import EnterpriseResearchReportJobRepository, EnterpriseResearchTaskRepository
__all__ = [
"EnterpriseResearchTaskRow",
"EnterpriseResearchReportJobRow",
"EnterpriseResearchTaskRepository",
"EnterpriseResearchReportJobRepository",
"EnterpriseResearchTaskStore",
"EnterpriseResearchReportJobStore",
"MemoryEnterpriseResearchTaskStore",
"MemoryEnterpriseResearchReportJobStore",
"make_enterprise_research_task_store",
"make_enterprise_research_report_job_store",
]
def make_enterprise_research_task_store(session_factory):
if session_factory is None:
return MemoryEnterpriseResearchTaskStore()
return EnterpriseResearchTaskRepository(session_factory)
def make_enterprise_research_report_job_store(session_factory):
if session_factory is None:
return MemoryEnterpriseResearchReportJobStore()
return EnterpriseResearchReportJobRepository(session_factory)
@@ -0,0 +1,75 @@
"""Persistence contract for the independent enterprise-research workbench."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class EnterpriseResearchTaskStore(ABC):
"""Owner-scoped task storage; evidence snapshots are DeerFlow-owned."""
@abstractmethod
async def list_tasks(self, user_id: str, *, limit: int = 20) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update_task(self, task_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def delete_task(self, task_id: str, user_id: str) -> bool:
raise NotImplementedError
class EnterpriseResearchReportJobStore(ABC):
"""Owner-scoped, frozen-input jobs for enterprise report writing."""
@abstractmethod
async def create_job(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def get_job(self, job_id: str, user_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def get_active_for_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def claim_job(self, job_id: str, user_id: str) -> bool:
"""Atomically move one queued job to running."""
raise NotImplementedError
@abstractmethod
async def requeue_job(self, job_id: str, user_id: str, *, expected_updated_at: str | None) -> bool:
"""Atomically requeue the exact running snapshot seen during recovery."""
raise NotImplementedError
@abstractmethod
async def transition_job(
self, job_id: str, user_id: str, *, expected_statuses: set[str], data: dict[str, Any]
) -> dict[str, Any] | None:
"""Apply a terminal/state transition only from one of the expected states."""
raise NotImplementedError
@abstractmethod
async def update_job(self, job_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_jobs(self, task_id: str, user_id: str, *, limit: int = 10) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def list_recoverable(self, *, limit: int = 100) -> list[dict[str, Any]]:
raise NotImplementedError
@@ -0,0 +1,172 @@
"""In-memory fallback for enterprise-research tasks."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.enterprise_research.base import EnterpriseResearchReportJobStore, EnterpriseResearchTaskStore
def _now() -> str:
return datetime.now(UTC).isoformat()
class MemoryEnterpriseResearchTaskStore(EnterpriseResearchTaskStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
async def list_tasks(self, user_id: str, *, limit: int = 20) -> list[dict[str, Any]]:
rows = [dict(row) for row in self._rows.values() if row["user_id"] == user_id]
return sorted(rows, key=lambda row: row["updated_at"], reverse=True)[:limit]
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
row = self._rows.get(task_id)
return dict(row) if row is not None and row["user_id"] == user_id else None
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
now = _now()
row = {
"id": str(data["id"]),
"user_id": str(data["user_id"]),
"subject": str(data["subject"]),
"focus": str(data.get("focus") or ""),
"template_id": str(data["template_id"]),
"knowledge_base_ids": list(data.get("knowledge_base_ids") or []),
"queries": list(data.get("queries") or []),
"selected_source_ids": list(data.get("selected_source_ids") or []),
"sources": list(data.get("sources") or []),
"plan_markdown": data.get("plan_markdown"),
"report_markdown": data.get("report_markdown"),
"report_summary": data.get("report_summary"),
"report_model_name": data.get("report_model_name"),
"active_report_job_id": data.get("active_report_job_id"),
"status": str(data.get("status") or "draft"),
"warning": data.get("warning"),
"created_at": now,
"updated_at": now,
}
self._rows[row["id"]] = row
return dict(row)
async def update_task(self, task_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._rows.get(task_id)
if row is None or row["user_id"] != user_id:
return None
for key in (
"status", "warning", "sources", "selected_source_ids", "plan_markdown",
"report_markdown", "report_summary", "report_model_name", "active_report_job_id",
):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return dict(row)
async def delete_task(self, task_id: str, user_id: str) -> bool:
row = self._rows.get(task_id)
if row is None or row["user_id"] != user_id:
return False
del self._rows[task_id]
return True
class MemoryEnterpriseResearchReportJobStore(EnterpriseResearchReportJobStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
async def create_job(self, data: dict[str, Any]) -> dict[str, Any]:
now = _now()
status = str(data.get("status") or "queued")
active_dedupe_key = (
f"{data['user_id']}:{data['task_id']}" if status in {"queued", "running"} else None
)
if active_dedupe_key:
existing = next(
(row for row in self._rows.values() if row.get("active_dedupe_key") == active_dedupe_key),
None,
)
if existing is not None:
return dict(existing)
row = {
"id": str(data["id"]), "task_id": str(data["task_id"]), "user_id": str(data["user_id"]),
"status": status, "phase": str(data.get("phase") or "queued"),
"model_name": data.get("model_name"), "action": str(data.get("action") or "initial"),
"active_dedupe_key": active_dedupe_key,
"instruction": data.get("instruction"), "style": data.get("style"),
"target_length": data.get("target_length"), "plan_snapshot": str(data["plan_snapshot"]),
"sources": list(data.get("sources") or []), "report_markdown": str(data.get("report_markdown") or ""),
"original_report": data.get("original_report"),
"report_summary": data.get("report_summary"), "error_message": data.get("error_message"),
"created_at": now, "updated_at": now,
}
self._rows[row["id"]] = row
return dict(row)
async def get_job(self, job_id: str, user_id: str) -> dict[str, Any] | None:
row = self._rows.get(job_id)
return dict(row) if row is not None and row["user_id"] == user_id else None
async def get_active_for_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
rows = [row for row in self._rows.values() if row["task_id"] == task_id and row["user_id"] == user_id and row["status"] in {"queued", "running"}]
return dict(max(rows, key=lambda row: row["updated_at"])) if rows else None
async def claim_job(self, job_id: str, user_id: str) -> bool:
row = self._rows.get(job_id)
if row is None or row["user_id"] != user_id or row["status"] != "queued":
return False
row["status"] = "running"
row["updated_at"] = _now()
return True
async def requeue_job(self, job_id: str, user_id: str, *, expected_updated_at: str | None) -> bool:
row = self._rows.get(job_id)
if (
row is None
or row["user_id"] != user_id
or row["status"] != "running"
or row.get("updated_at") != expected_updated_at
):
return False
row.update({"status": "queued", "phase": "queued", "report_markdown": "", "error_message": None})
row["updated_at"] = _now()
return True
async def transition_job(
self, job_id: str, user_id: str, *, expected_statuses: set[str], data: dict[str, Any]
) -> dict[str, Any] | None:
row = self._rows.get(job_id)
if row is None or row["user_id"] != user_id or row["status"] not in expected_statuses:
return None
for key in (
"status", "phase", "report_markdown", "report_summary", "error_message", "model_name",
"action", "instruction", "style", "target_length", "original_report",
):
if key in data:
row[key] = data[key]
if data.get("status") in {"completed", "failed", "cancelled"}:
row["active_dedupe_key"] = None
row["updated_at"] = _now()
return dict(row)
async def update_job(self, job_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._rows.get(job_id)
if row is None or row["user_id"] != user_id:
return None
for key in (
"status", "phase", "report_markdown", "report_summary", "error_message", "model_name",
"action", "instruction", "style", "target_length", "original_report",
):
if key in data:
row[key] = data[key]
if data.get("status") in {"completed", "failed", "cancelled"}:
row["active_dedupe_key"] = None
row["updated_at"] = _now()
return dict(row)
async def list_jobs(self, task_id: str, user_id: str, *, limit: int = 10) -> list[dict[str, Any]]:
rows = [dict(row) for row in self._rows.values() if row["task_id"] == task_id and row["user_id"] == user_id]
return sorted(rows, key=lambda row: row["updated_at"], reverse=True)[:limit]
async def list_recoverable(self, *, limit: int = 100) -> list[dict[str, Any]]:
rows = [dict(row) for row in self._rows.values() if row["status"] in {"queued", "running"}]
return sorted(rows, key=lambda row: row["updated_at"])[:limit]
@@ -0,0 +1,77 @@
"""ORM rows for the independent enterprise-research workbench."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class EnterpriseResearchTaskRow(Base):
"""A user-owned research task plus its bounded WeKnora evidence snapshot."""
__tablename__ = "enterprise_research_tasks"
__table_args__ = (Index("ix_enterprise_research_tasks_owner_updated", "user_id", "updated_at"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
subject: Mapped[str] = mapped_column(String(256), nullable=False)
focus: Mapped[str] = mapped_column(Text, nullable=False, default="", server_default="")
template_id: Mapped[str] = mapped_column(String(64), nullable=False)
knowledge_base_ids: Mapped[list[str]] = mapped_column(PortableJSON(), nullable=False, default=list)
queries: Mapped[list[str]] = mapped_column(PortableJSON(), nullable=False, default=list)
# Nullable keeps upgrades safe: the generic schema reconciler can add this
# field to a phase-one table without an unsafe MySQL TEXT default.
selected_source_ids: Mapped[list[str] | None] = mapped_column(PortableJSON(), nullable=True, default=list)
# A WeKnora result can contain a long document chunk. MySQL TEXT is only
# 64 KiB, so evidence uses LONGTEXT through PortableLongText.
sources_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="[]", server_default="[]")
plan_markdown: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
report_markdown: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
report_summary: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
report_model_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
active_report_job_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="draft", server_default="draft")
warning: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
class EnterpriseResearchReportJobRow(Base):
"""A report job whose plan/evidence are frozen when the user starts it."""
__tablename__ = "enterprise_research_report_jobs"
__table_args__ = (
Index("ix_enterprise_research_report_jobs_task_owner_updated", "task_id", "user_id", "updated_at"),
UniqueConstraint("active_dedupe_key", name="uq_enterprise_research_report_jobs_active"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", server_default="queued")
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", server_default="queued")
model_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
# Present only for queued/running rows. NULL terminal rows keep unlimited
# history while this key rejects cross-process duplicate starts.
active_dedupe_key: Mapped[str | None] = mapped_column(String(256), nullable=True)
action: Mapped[str | None] = mapped_column(String(32), nullable=True, default="initial")
instruction: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
style: Mapped[str | None] = mapped_column(String(32), nullable=True)
target_length: Mapped[int | None] = mapped_column(nullable=True)
plan_snapshot: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
sources_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
original_report: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
report_markdown: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="", server_default="")
report_summary: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
error_message: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
@@ -0,0 +1,320 @@
"""SQLAlchemy store for enterprise-research tasks."""
from __future__ import annotations
import json
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.enterprise_research.base import EnterpriseResearchReportJobStore, EnterpriseResearchTaskStore
from deerflow.persistence.enterprise_research.model import EnterpriseResearchReportJobRow, EnterpriseResearchTaskRow
def _to_dict(row: EnterpriseResearchTaskRow) -> dict[str, Any]:
try:
sources = json.loads(row.sources_json or "[]")
except (TypeError, ValueError):
sources = []
data: dict[str, Any] = {
"id": row.id,
"user_id": row.user_id,
"subject": row.subject,
"focus": row.focus,
"template_id": row.template_id,
"knowledge_base_ids": list(row.knowledge_base_ids or []),
"queries": list(row.queries or []),
"selected_source_ids": list(row.selected_source_ids or []),
"sources": sources if isinstance(sources, list) else [],
"plan_markdown": row.plan_markdown,
"report_markdown": row.report_markdown,
"report_summary": row.report_summary,
"report_model_name": row.report_model_name,
"active_report_job_id": row.active_report_job_id,
"status": row.status,
"warning": row.warning,
"created_at": row.created_at.isoformat() if row.created_at else None,
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
}
return data
class EnterpriseResearchTaskRepository(EnterpriseResearchTaskStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_tasks(self, user_id: str, *, limit: int = 20) -> list[dict[str, Any]]:
stmt = (
select(EnterpriseResearchTaskRow)
.where(EnterpriseResearchTaskRow.user_id == user_id)
.order_by(EnterpriseResearchTaskRow.updated_at.desc())
.limit(limit)
)
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
stmt = select(EnterpriseResearchTaskRow).where(
EnterpriseResearchTaskRow.id == task_id,
EnterpriseResearchTaskRow.user_id == user_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row is not None else None
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = EnterpriseResearchTaskRow(
id=str(data["id"]),
user_id=str(data["user_id"]),
subject=str(data["subject"]),
focus=str(data.get("focus") or ""),
template_id=str(data["template_id"]),
knowledge_base_ids=list(data.get("knowledge_base_ids") or []),
queries=list(data.get("queries") or []),
selected_source_ids=list(data.get("selected_source_ids") or []),
sources_json=json.dumps(list(data.get("sources") or []), ensure_ascii=False),
plan_markdown=data.get("plan_markdown"),
report_markdown=data.get("report_markdown"),
report_summary=data.get("report_summary"),
report_model_name=data.get("report_model_name"),
active_report_job_id=data.get("active_report_job_id"),
status=str(data.get("status") or "draft"),
warning=data.get("warning"),
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
# Do not refresh after commit: a MySQL read/write split can route it
# to a lagging replica. The flushed in-memory row is authoritative.
await session.flush()
payload = _to_dict(row)
await session.commit()
return payload
async def update_task(self, task_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
stmt = select(EnterpriseResearchTaskRow).where(
EnterpriseResearchTaskRow.id == task_id,
EnterpriseResearchTaskRow.user_id == user_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
return None
if "status" in data:
row.status = str(data["status"])
if "warning" in data:
row.warning = data["warning"]
if "sources" in data:
row.sources_json = json.dumps(list(data["sources"] or []), ensure_ascii=False)
if "selected_source_ids" in data:
row.selected_source_ids = list(data["selected_source_ids"] or [])
if "plan_markdown" in data:
row.plan_markdown = data["plan_markdown"]
for key in ("report_markdown", "report_summary", "report_model_name", "active_report_job_id"):
if key in data:
setattr(row, key, data[key])
row.updated_at = datetime.now(UTC)
await session.flush()
payload = _to_dict(row)
await session.commit()
return payload
async def delete_task(self, task_id: str, user_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(EnterpriseResearchTaskRow).where(
EnterpriseResearchTaskRow.id == task_id,
EnterpriseResearchTaskRow.user_id == user_id,
)
)
await session.commit()
return bool(result.rowcount)
def _job_to_dict(row: EnterpriseResearchReportJobRow) -> dict[str, Any]:
try:
sources = json.loads(row.sources_json or "[]")
except (TypeError, ValueError):
sources = []
return {
"id": row.id, "task_id": row.task_id, "user_id": row.user_id, "status": row.status,
"phase": row.phase, "model_name": row.model_name, "action": row.action or "initial",
"instruction": row.instruction, "style": row.style, "target_length": row.target_length,
"plan_snapshot": row.plan_snapshot,
"sources": sources if isinstance(sources, list) else [], "report_markdown": row.report_markdown,
"original_report": row.original_report,
"report_summary": row.report_summary, "error_message": row.error_message,
"created_at": row.created_at.isoformat() if row.created_at else None,
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
}
class EnterpriseResearchReportJobRepository(EnterpriseResearchReportJobStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create_job(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
status = str(data.get("status") or "queued")
active_dedupe_key = (
f"{data['user_id']}:{data['task_id']}" if status in {"queued", "running"} else None
)
row = EnterpriseResearchReportJobRow(
id=str(data["id"]), task_id=str(data["task_id"]), user_id=str(data["user_id"]),
status=status, phase=str(data.get("phase") or "queued"),
model_name=data.get("model_name"), action=str(data.get("action") or "initial"),
active_dedupe_key=active_dedupe_key,
instruction=data.get("instruction"), style=data.get("style"), target_length=data.get("target_length"),
plan_snapshot=str(data["plan_snapshot"]),
sources_json=json.dumps(list(data.get("sources") or []), ensure_ascii=False),
original_report=data.get("original_report"),
report_markdown=str(data.get("report_markdown") or ""), report_summary=data.get("report_summary"),
error_message=data.get("error_message"), created_at=now, updated_at=now,
)
async with self._sf() as session:
session.add(row)
try:
await session.flush()
payload = _job_to_dict(row)
await session.commit()
return payload
except IntegrityError:
await session.rollback()
if not active_dedupe_key:
raise
existing = (await session.execute(
select(EnterpriseResearchReportJobRow)
.where(EnterpriseResearchReportJobRow.active_dedupe_key == active_dedupe_key)
.limit(1)
)).scalar_one_or_none()
if existing is None:
raise
return _job_to_dict(existing)
async def get_job(self, job_id: str, user_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = (await session.execute(select(EnterpriseResearchReportJobRow).where(
EnterpriseResearchReportJobRow.id == job_id, EnterpriseResearchReportJobRow.user_id == user_id,
))).scalar_one_or_none()
return _job_to_dict(row) if row is not None else None
async def get_active_for_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = (await session.execute(select(EnterpriseResearchReportJobRow).where(
EnterpriseResearchReportJobRow.task_id == task_id,
EnterpriseResearchReportJobRow.user_id == user_id,
EnterpriseResearchReportJobRow.status.in_(["queued", "running"]),
).order_by(EnterpriseResearchReportJobRow.updated_at.desc()).limit(1))).scalar_one_or_none()
return _job_to_dict(row) if row is not None else None
async def claim_job(self, job_id: str, user_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
update(EnterpriseResearchReportJobRow)
.where(
EnterpriseResearchReportJobRow.id == job_id,
EnterpriseResearchReportJobRow.user_id == user_id,
EnterpriseResearchReportJobRow.status == "queued",
)
.values(status="running", updated_at=datetime.now(UTC))
)
await session.commit()
return bool(result.rowcount)
async def requeue_job(self, job_id: str, user_id: str, *, expected_updated_at: str | None) -> bool:
if not expected_updated_at:
return False
try:
expected = datetime.fromisoformat(expected_updated_at)
except ValueError:
return False
async with self._sf() as session:
result = await session.execute(
update(EnterpriseResearchReportJobRow)
.where(
EnterpriseResearchReportJobRow.id == job_id,
EnterpriseResearchReportJobRow.user_id == user_id,
EnterpriseResearchReportJobRow.status == "running",
EnterpriseResearchReportJobRow.updated_at == expected,
)
.values(
status="queued",
phase="queued",
report_markdown="",
error_message=None,
updated_at=datetime.now(UTC),
)
)
await session.commit()
return bool(result.rowcount)
async def transition_job(
self, job_id: str, user_id: str, *, expected_statuses: set[str], data: dict[str, Any]
) -> dict[str, Any] | None:
allowed = {
"status", "phase", "report_markdown", "report_summary", "error_message", "model_name",
"action", "instruction", "style", "target_length", "original_report",
}
values = {key: value for key, value in data.items() if key in allowed}
if data.get("status") in {"completed", "failed", "cancelled"}:
values["active_dedupe_key"] = None
values["updated_at"] = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(EnterpriseResearchReportJobRow)
.where(
EnterpriseResearchReportJobRow.id == job_id,
EnterpriseResearchReportJobRow.user_id == user_id,
EnterpriseResearchReportJobRow.status.in_(expected_statuses),
)
.values(**values)
)
if not result.rowcount:
await session.rollback()
return None
await session.commit()
return await self.get_job(job_id, user_id)
async def update_job(self, job_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = (await session.execute(select(EnterpriseResearchReportJobRow).where(
EnterpriseResearchReportJobRow.id == job_id, EnterpriseResearchReportJobRow.user_id == user_id,
))).scalar_one_or_none()
if row is None:
return None
for key in (
"status", "phase", "report_markdown", "report_summary", "error_message", "model_name",
"action", "instruction", "style", "target_length", "original_report",
):
if key in data:
setattr(row, key, data[key])
if data.get("status") in {"completed", "failed", "cancelled"}:
row.active_dedupe_key = None
row.updated_at = datetime.now(UTC)
await session.flush()
payload = _job_to_dict(row)
await session.commit()
return payload
async def list_jobs(self, task_id: str, user_id: str, *, limit: int = 10) -> list[dict[str, Any]]:
async with self._sf() as session:
rows = (await session.execute(select(EnterpriseResearchReportJobRow).where(
EnterpriseResearchReportJobRow.task_id == task_id, EnterpriseResearchReportJobRow.user_id == user_id,
).order_by(EnterpriseResearchReportJobRow.updated_at.desc()).limit(limit))).scalars()
return [_job_to_dict(row) for row in rows]
async def list_recoverable(self, *, limit: int = 100) -> list[dict[str, Any]]:
async with self._sf() as session:
rows = (await session.execute(
select(EnterpriseResearchReportJobRow)
.where(EnterpriseResearchReportJobRow.status.in_(["queued", "running"]))
.order_by(EnterpriseResearchReportJobRow.updated_at.asc())
.limit(limit)
)).scalars()
return [_job_to_dict(row) for row in rows]
@@ -0,0 +1,6 @@
"""Feedback persistence — ORM and SQL repository."""
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.feedback.sql import FeedbackRepository
__all__ = ["FeedbackRepository", "FeedbackRow"]
@@ -0,0 +1,38 @@
"""ORM model for user feedback on runs."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class FeedbackRow(Base):
__tablename__ = "feedback"
# Prefix-unique instead of UniqueConstraint: a plain unique over three
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
# mysql_length is ignored on SQLite/Postgres.
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
message_id: Mapped[str | None] = mapped_column(String(64))
# message_id is an optional RunEventStore event identifier —
# allows feedback to target a specific message or the entire run
rating: Mapped[int] = mapped_column(nullable=False)
# +1 (thumbs-up) or -1 (thumbs-down)
comment: Mapped[str | None] = mapped_column(Text)
# Optional text feedback from the user
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -0,0 +1,217 @@
"""SQLAlchemy-backed feedback storage.
Each method acquires its own short-lived session.
"""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from sqlalchemy import case, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
class FeedbackRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: FeedbackRow) -> dict:
d = row.to_dict()
val = d.get("created_at")
if isinstance(val, datetime):
d["created_at"] = val.isoformat()
return d
async def create(
self,
*,
run_id: str,
thread_id: str,
rating: int,
user_id: str | None | _AutoSentinel = AUTO,
message_id: str | None = None,
comment: str | None = None,
) -> dict:
"""Create a feedback record. rating must be +1 or -1."""
if rating not in (1, -1):
raise ValueError(f"rating must be +1 or -1, got {rating}")
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.create")
row = FeedbackRow(
feedback_id=str(uuid.uuid4()),
run_id=run_id,
thread_id=thread_id,
user_id=resolved_user_id,
message_id=message_id,
rating=rating,
comment=comment,
created_at=datetime.now(UTC),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def get(
self,
feedback_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict | None:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.get")
async with self._sf() as session:
row = await session.get(FeedbackRow, feedback_id)
if row is None:
return None
if resolved_user_id is not None and row.user_id != resolved_user_id:
return None
return self._row_to_dict(row)
async def list_by_run(
self,
thread_id: str,
run_id: str,
*,
limit: int = 100,
user_id: str | None | _AutoSentinel = AUTO,
) -> list[dict]:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_run")
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
async def list_by_thread(
self,
thread_id: str,
*,
limit: int = 100,
user_id: str | None | _AutoSentinel = AUTO,
) -> list[dict]:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread")
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
async def delete(
self,
feedback_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete")
async with self._sf() as session:
row = await session.get(FeedbackRow, feedback_id)
if row is None:
return False
if resolved_user_id is not None and row.user_id != resolved_user_id:
return False
await session.delete(row)
await session.commit()
return True
async def upsert(
self,
*,
run_id: str,
thread_id: str,
rating: int,
user_id: str | None | _AutoSentinel = AUTO,
comment: str | None = None,
) -> dict:
"""Create or update feedback for (thread_id, run_id, user_id). rating must be +1 or -1."""
if rating not in (1, -1):
raise ValueError(f"rating must be +1 or -1, got {rating}")
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.upsert")
async with self._sf() as session:
stmt = select(FeedbackRow).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
FeedbackRow.user_id == resolved_user_id,
)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
if row is not None:
row.rating = rating
row.comment = comment
row.created_at = datetime.now(UTC)
else:
row = FeedbackRow(
feedback_id=str(uuid.uuid4()),
run_id=run_id,
thread_id=thread_id,
user_id=resolved_user_id,
rating=rating,
comment=comment,
created_at=datetime.now(UTC),
)
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def delete_by_run(
self,
*,
thread_id: str,
run_id: str,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
"""Delete the current user's feedback for a run. Returns True if a record was deleted."""
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete_by_run")
async with self._sf() as session:
stmt = select(FeedbackRow).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
FeedbackRow.user_id == resolved_user_id,
)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def list_by_thread_grouped(
self,
thread_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict[str, dict]:
"""Return feedback grouped by run_id for a thread: {run_id: feedback_dict}."""
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread_grouped")
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
async with self._sf() as session:
result = await session.execute(stmt)
return {row.run_id: self._row_to_dict(row) for row in result.scalars()}
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict:
"""Aggregate feedback stats for a run using database-side counting."""
stmt = select(
func.count().label("total"),
func.coalesce(func.sum(case((FeedbackRow.rating == 1, 1), else_=0)), 0).label("positive"),
func.coalesce(func.sum(case((FeedbackRow.rating == -1, 1), else_=0)), 0).label("negative"),
).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
async with self._sf() as session:
row = (await session.execute(stmt)).one()
return {
"run_id": run_id,
"total": row.total,
"positive": row.positive,
"negative": row.negative,
}
@@ -0,0 +1,22 @@
"""Exact-match fixed-question persistence."""
from __future__ import annotations
from deerflow.persistence.fixed_questions.base import FixedQuestionStore
from deerflow.persistence.fixed_questions.memory import MemoryFixedQuestionStore
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
from deerflow.persistence.fixed_questions.sql import FixedQuestionRepository
__all__ = [
"FixedQuestionRepository",
"FixedQuestionRow",
"FixedQuestionStore",
"MemoryFixedQuestionStore",
"make_fixed_question_store",
]
def make_fixed_question_store(session_factory):
if session_factory is None:
return MemoryFixedQuestionStore()
return FixedQuestionRepository(session_factory)
@@ -0,0 +1,30 @@
"""Abstract persistence API for administrator-managed fixed questions."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class FixedQuestionStore(ABC):
"""Global fixed-question configuration shared by every account."""
@abstractmethod
async def list_questions(self) -> list[dict[str, Any]]:
"""Return every configured question in display order."""
raise NotImplementedError
@abstractmethod
async def find_enabled(self, question: str) -> dict[str, Any] | None:
"""Return the enabled row whose question exactly equals ``question``."""
raise NotImplementedError
@abstractmethod
async def replace_all(
self,
rows: list[dict[str, Any]],
*,
updated_by: str | None = None,
) -> list[dict[str, Any]]:
"""Atomically replace the complete fixed-question configuration."""
raise NotImplementedError
@@ -0,0 +1,56 @@
"""In-memory fixed-question store used by memory deployments and tests."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.fixed_questions.base import FixedQuestionStore
def _normalize(data: dict[str, Any], updated_by: str | None) -> dict[str, Any]:
return {
"id": str(data["id"]),
"question": str(data.get("question") or ""),
"answer": str(data.get("answer") or ""),
"enabled": bool(data.get("enabled", True)),
"tokens_per_second": int(data.get("tokens_per_second", 100)),
"sort_order": int(data.get("sort_order", 0)),
"updated_by": updated_by,
"updated_at": datetime.now(UTC).isoformat(),
}
class MemoryFixedQuestionStore(FixedQuestionStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
async def list_questions(self) -> list[dict[str, Any]]:
rows = sorted(
self._rows.values(),
key=lambda row: (row.get("sort_order", 0), row.get("question", "")),
)
return [dict(row) for row in rows]
async def find_enabled(self, question: str) -> dict[str, Any] | None:
for row in self._rows.values():
if row["enabled"] and row["question"] == question:
return dict(row)
return None
async def replace_all(
self,
rows: list[dict[str, Any]],
*,
updated_by: str | None = None,
) -> list[dict[str, Any]]:
# Last duplicate question wins, mirroring the SQL repository.
by_id: dict[str, dict[str, Any]] = {}
for row in rows:
question = row.get("question")
row_id = row.get("id")
if row_id and question not in (None, ""):
by_id[str(row_id)] = row
by_question = {str(row["question"]): row for row in by_id.values()}
self._rows = {str(row["id"]): _normalize(row, updated_by) for row in by_question.values()}
return await self.list_questions()
@@ -0,0 +1,41 @@
"""ORM model for exact-match, administrator-managed fixed questions."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class FixedQuestionRow(Base):
"""One globally shared question/answer pair.
``question_hash`` provides a portable indexed lookup for questions stored in
``Text``. The repository always compares the original question as well, so a
theoretical SHA-256 collision cannot cause a false match.
"""
__tablename__ = "fixed_questions"
__table_args__ = (UniqueConstraint("question_hash", name="uq_fixed_questions_question_hash"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
question: Mapped[str] = mapped_column(Text, nullable=False)
question_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
answer: Mapped[str] = mapped_column(PortableLongText, nullable=False)
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="1")
tokens_per_second: Mapped[int] = mapped_column(
Integer,
default=100,
server_default="100",
)
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,88 @@
"""SQLAlchemy-backed fixed-question store."""
from __future__ import annotations
import hashlib
from datetime import datetime
from typing import Any
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.fixed_questions.base import FixedQuestionStore
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
def question_hash(question: str) -> str:
return hashlib.sha256(question.encode("utf-8")).hexdigest()
def _to_dict(row: FixedQuestionRow) -> dict[str, Any]:
data = row.to_dict()
data.pop("question_hash", None)
value = data.get("updated_at")
if isinstance(value, datetime):
data["updated_at"] = value.isoformat()
return data
def _row_from(data: dict[str, Any], updated_by: str | None) -> FixedQuestionRow:
question = str(data.get("question") or "")
return FixedQuestionRow(
id=str(data["id"]),
question=question,
question_hash=question_hash(question),
answer=str(data.get("answer") or ""),
enabled=bool(data.get("enabled", True)),
tokens_per_second=int(data.get("tokens_per_second", 100)),
sort_order=int(data.get("sort_order", 0)),
updated_by=updated_by,
)
class FixedQuestionRepository(FixedQuestionStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_questions(self) -> list[dict[str, Any]]:
stmt = select(FixedQuestionRow).order_by(
FixedQuestionRow.sort_order.asc(),
FixedQuestionRow.question.asc(),
)
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def find_enabled(self, question: str) -> dict[str, Any] | None:
stmt = select(FixedQuestionRow).where(
FixedQuestionRow.question_hash == question_hash(question),
FixedQuestionRow.enabled.is_(True),
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None or row.question != question:
return None
return _to_dict(row)
async def replace_all(
self,
rows: list[dict[str, Any]],
*,
updated_by: str | None = None,
) -> list[dict[str, Any]]:
# Last duplicate question wins, preventing a unique-hash violation even
# if a non-UI caller submits malformed data.
by_id: dict[str, dict[str, Any]] = {}
for row in rows:
question = row.get("question")
row_id = row.get("id")
if row_id and question not in (None, ""):
by_id[str(row_id)] = row
by_question = {str(row["question"]): row for row in by_id.values()}
async with self._sf() as session:
await session.execute(delete(FixedQuestionRow))
new_rows = [_row_from(row, updated_by) for row in by_question.values()]
session.add_all(new_rows)
await session.commit()
return [_to_dict(row) for row in new_rows]
@@ -0,0 +1,23 @@
"""Generated HTML page + favorite persistence."""
from __future__ import annotations
from deerflow.persistence.html_pages.base import HtmlPageStore
from deerflow.persistence.html_pages.memory import MemoryHtmlPageStore
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
from deerflow.persistence.html_pages.sql import HtmlPageRepository
__all__ = [
"HtmlPageStore",
"HtmlPageRepository",
"MemoryHtmlPageStore",
"HtmlPageFavoriteRow",
"ScheduledHtmlPageRow",
"make_html_page_store",
]
def make_html_page_store(session_factory):
if session_factory is None:
return MemoryHtmlPageStore()
return HtmlPageRepository(session_factory)
@@ -0,0 +1,53 @@
"""Abstract persistence API for generated HTML pages and favorites."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class HtmlPageStore(ABC):
# --- Generated pages ---
@abstractmethod
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def get_page(self, page_id: str) -> dict[str, Any] | None:
"""Return a page by id without restricting by owner."""
raise NotImplementedError
@abstractmethod
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
"""Return the page produced by a given run, if any."""
raise NotImplementedError
@abstractmethod
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
"""Return the most recently created page for a task, if any."""
raise NotImplementedError
# --- Favorites ---
@abstractmethod
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
"""Return a favorite by id without restricting by owner."""
raise NotImplementedError
@abstractmethod
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
"""Return the user's existing favorite for a page, if any (dedup check)."""
raise NotImplementedError
@abstractmethod
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
raise NotImplementedError
@@ -0,0 +1,102 @@
"""In-memory HtmlPageStore for no-database / test deployments."""
from __future__ import annotations
import copy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.html_pages.base import HtmlPageStore
def _now_iso() -> str:
return datetime.now(UTC).isoformat()
class MemoryHtmlPageStore(HtmlPageStore):
def __init__(self) -> None:
self._pages: dict[str, dict[str, Any]] = {}
self._favorites: dict[str, dict[str, Any]] = {}
# --- Generated pages ---
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
row = {
"id": data["id"],
"task_id": data["task_id"],
"run_id": data["run_id"],
"user_id": data["user_id"],
"title": data.get("title") or "",
"html_content": data.get("html_content") or "",
"html_file_path": data.get("html_file_path"),
"data_snapshot": data.get("data_snapshot") or {},
"prompt_snapshot": data.get("prompt_snapshot") or {},
"reference_favorite_id": data.get("reference_favorite_id"),
"style_summary": data.get("style_summary"),
"layout_summary": data.get("layout_summary"),
"created_at": _now_iso(),
"updated_at": _now_iso(),
}
self._pages[row["id"]] = row
return copy.deepcopy(row)
async def get_page(self, page_id: str) -> dict[str, Any] | None:
row = self._pages.get(page_id)
return copy.deepcopy(row) if row is not None else None
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
matches = [r for r in self._pages.values() if r["run_id"] == run_id]
if not matches:
return None
matches.sort(key=lambda r: r["created_at"], reverse=True)
return copy.deepcopy(matches[0])
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
matches = [r for r in self._pages.values() if r["task_id"] == task_id]
if not matches:
return None
matches.sort(key=lambda r: r["created_at"], reverse=True)
return copy.deepcopy(matches[0])
# --- Favorites ---
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
row = {
"id": data["id"],
"user_id": data["user_id"],
"page_id": data["page_id"],
"title": data.get("title") or "",
"description": data.get("description"),
"tags": data.get("tags") or [],
"html_snapshot": data.get("html_snapshot") or "",
"style_summary": data.get("style_summary"),
"layout_summary": data.get("layout_summary"),
"content_structure_summary": data.get("content_structure_summary"),
"reuse_prompt": data.get("reuse_prompt"),
"created_at": _now_iso(),
"updated_at": _now_iso(),
}
self._favorites[row["id"]] = row
return copy.deepcopy(row)
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
matches = [r for r in self._favorites.values() if r["user_id"] == user_id]
matches.sort(key=lambda r: r["created_at"], reverse=True)
return [copy.deepcopy(r) for r in matches[: max(1, limit)]]
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
row = self._favorites.get(favorite_id)
return copy.deepcopy(row) if row is not None else None
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
for row in self._favorites.values():
if row["user_id"] == user_id and row["page_id"] == page_id:
return copy.deepcopy(row)
return None
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
row = self._favorites.get(favorite_id)
if row is None or row["user_id"] != user_id:
return False
del self._favorites[favorite_id]
return True
@@ -0,0 +1,65 @@
"""ORM models for generated HTML pages and user favorites.
These back the "HTML 页面定时任务" feature: a scheduled task of kind
``html_page`` renders a full HTML document each run. Every rendered page is
recorded in :class:`ScheduledHtmlPageRow`; a user may bookmark one into
:class:`HtmlPageFavoriteRow`, which also stores a reusable style/layout
summary so a later task can be generated "in the style of" the favorite.
Both tables are brand new, so ``Base.metadata.create_all`` creates them on the
next startup without touching any existing schema.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class ScheduledHtmlPageRow(Base):
"""One HTML page produced by a single run of an ``html_page`` task."""
__tablename__ = "scheduled_html_pages"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64), index=True)
run_id: Mapped[str] = mapped_column(String(64), index=True)
user_id: Mapped[str] = mapped_column(String(64), index=True)
title: Mapped[str] = mapped_column(String(255))
# Full HTML document — can be large, so LONGTEXT on MySQL.
html_content: Mapped[str] = mapped_column(PortableLongText())
html_file_path: Mapped[str | None] = mapped_column(Text)
data_snapshot_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
prompt_snapshot_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
reference_favorite_id: Mapped[str | None] = mapped_column(String(64), index=True)
style_summary: Mapped[str | None] = mapped_column(Text)
layout_summary: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class HtmlPageFavoriteRow(Base):
"""A user-bookmarked HTML page, with a reusable style/layout summary."""
__tablename__ = "html_page_favorites"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), index=True)
page_id: Mapped[str] = mapped_column(String(64), index=True)
title: Mapped[str] = mapped_column(String(255))
description: Mapped[str | None] = mapped_column(Text)
tags_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list)
# Snapshot of the page HTML at favorite time, decoupled from the source run
# so deleting the run never breaks the favorite reference.
html_snapshot: Mapped[str] = mapped_column(PortableLongText())
style_summary: Mapped[str | None] = mapped_column(Text)
layout_summary: Mapped[str | None] = mapped_column(Text)
content_structure_summary: Mapped[str | None] = mapped_column(Text)
reuse_prompt: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,141 @@
"""SQLAlchemy repository for generated HTML pages and favorites."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.html_pages.base import HtmlPageStore
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
def _to_dict(row) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
# Normalize the JSON column names to API-friendly keys.
if "data_snapshot_json" in data:
data["data_snapshot"] = data.pop("data_snapshot_json") or {}
if "prompt_snapshot_json" in data:
data["prompt_snapshot"] = data.pop("prompt_snapshot_json") or {}
if "tags_json" in data:
data["tags"] = data.pop("tags_json") or []
return data
class HtmlPageRepository(HtmlPageStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# --- Generated pages ---
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
row = ScheduledHtmlPageRow(
id=data["id"],
task_id=data["task_id"],
run_id=data["run_id"],
user_id=data["user_id"],
title=data.get("title") or "",
html_content=data.get("html_content") or "",
html_file_path=data.get("html_file_path"),
data_snapshot_json=data.get("data_snapshot") or {},
prompt_snapshot_json=data.get("prompt_snapshot") or {},
reference_favorite_id=data.get("reference_favorite_id"),
style_summary=data.get("style_summary"),
layout_summary=data.get("layout_summary"),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def get_page(self, page_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ScheduledHtmlPageRow, page_id)
return _to_dict(row) if row is not None else None
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(ScheduledHtmlPageRow)
.where(ScheduledHtmlPageRow.run_id == run_id)
.order_by(ScheduledHtmlPageRow.created_at.desc())
.limit(1)
)
row = result.scalar_one_or_none()
return _to_dict(row) if row is not None else None
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(ScheduledHtmlPageRow)
.where(ScheduledHtmlPageRow.task_id == task_id)
.order_by(ScheduledHtmlPageRow.created_at.desc())
.limit(1)
)
row = result.scalar_one_or_none()
return _to_dict(row) if row is not None else None
# --- Favorites ---
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
row = HtmlPageFavoriteRow(
id=data["id"],
user_id=data["user_id"],
page_id=data["page_id"],
title=data.get("title") or "",
description=data.get("description"),
tags_json=data.get("tags") or [],
html_snapshot=data.get("html_snapshot") or "",
style_summary=data.get("style_summary"),
layout_summary=data.get("layout_summary"),
content_structure_summary=data.get("content_structure_summary"),
reuse_prompt=data.get("reuse_prompt"),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(HtmlPageFavoriteRow)
.where(HtmlPageFavoriteRow.user_id == user_id)
.order_by(HtmlPageFavoriteRow.created_at.desc())
.limit(max(1, limit))
)
return [_to_dict(row) for row in result.scalars()]
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(HtmlPageFavoriteRow, favorite_id)
return _to_dict(row) if row is not None else None
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(HtmlPageFavoriteRow)
.where(HtmlPageFavoriteRow.user_id == user_id, HtmlPageFavoriteRow.page_id == page_id)
.limit(1)
)
row = result.scalar_one_or_none()
return _to_dict(row) if row is not None else None
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(HtmlPageFavoriteRow).where(
HtmlPageFavoriteRow.id == favorite_id,
HtmlPageFavoriteRow.user_id == user_id,
)
)
await session.commit()
return (result.rowcount or 0) > 0
@@ -0,0 +1,25 @@
"""Intent-turn single-flight persistence (Phase 4).
Exposes a store factory; ``app/gateway/deps.py`` registers it on
``app.state.intent_turn_store`` at startup. The intent router reads it from
``request.app.state`` (no cross-boundary default needed — intent runs only in the
gateway process).
"""
from __future__ import annotations
from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.intent_turns.sql import IntentTurnRepository
def make_intent_turn_store(session_factory) -> IntentTurnRepository | None:
if session_factory is None:
return None
return IntentTurnRepository(session_factory)
__all__ = [
"IntentTurnRow",
"IntentTurnRepository",
"make_intent_turn_store",
]
@@ -0,0 +1,60 @@
"""ORM model for intent-determination turn single-flight (Phase 4).
一条 ``intent_turns`` 行 = 第一步「确定意图」对话的**一个回合**对 ``(thread_id,
client_turn_id)`` 的占用记录。前端为每个回合生成稳定的 ``client_turn_id``,重试 / 双击 /
网络层重发都复用它;后端用 ``UNIQUE(thread_id, client_turn_id)`` 做跨 worker 单飞:
* 同一 ``(thread_id, client_turn_id)`` 至多一条行 —— INSERT 冲突即幂等命中:
- 已 ``done`` → 原样回放缓存的终态帧,**不再启 run**(杜绝双发意图 / 重复澄清卡);
- 在跑且租约未过期 → 该请求是重复的 in-flight 请求,丢弃(原请求仍在流式吐结果);
- 在跑但租约过期(worker 崩溃)→ 抢占续跑;
- ``message_hash`` 不同(同一 client_turn_id 却是不同内容)→ 409 客户端 bug。
一致性只来自 DB 唯一键 + 条件 UPDATE(租约 CAS),不依赖进程内 ``RunManager``
(后者是每进程内存态,多 worker 下看不到彼此的 run,无法单飞)。
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class IntentTurnRow(Base):
__tablename__ = "intent_turns"
__table_args__ = (
# 跨 worker 单飞的根:同一 (thread_id, client_turn_id) 至多一条。INSERT 冲突即幂等。
UniqueConstraint("thread_id", "client_turn_id", name="uq_intent_turns_thread_client"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
thread_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
# 前端每回合生成的稳定幂等键;重试 / 双击复用同一个。
client_turn_id: Mapped[str] = mapped_column(String(64), nullable=False)
# 本回合 message 的 sha256:检测「同 client_turn_id 但内容变了」的客户端 bug(→ 409)。
message_hash: Mapped[str] = mapped_column(String(64), nullable=False)
# running(在跑)/ done(已出终态帧,可回放)/ error。
status: Mapped[str] = mapped_column(
String(16), nullable=False, default="running", server_default="running"
)
# 终态帧的完整 JSON(done/error 时写入):重复请求命中时原样回放,不再启 run。
result_json: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
# 租约:领走的 worker 标记;过期(worker 崩溃)后可被条件 UPDATE 抢占续跑。
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,225 @@
"""SQLAlchemy-backed intent-turn single-flight storage (Phase 4).
跨 worker 单飞 + 幂等回放,一致性只来自 DB(唯一键 + 条件 UPDATE),不依赖进程内
``RunManager``(多 worker 下彼此不可见)。两条核心路径:
* ``try_claim`` —— INSERT 占用 ``(thread_id, client_turn_id)``;唯一键冲突即幂等命中,
按「已 done→回放 / 在跑且租约有效→丢弃 / 租约过期或 error→抢占续跑 / 内容不一致→409」分流。
* ``record_result`` —— 持租者把终态帧写回(条件 UPDATE 要求租约匹配),清租约。
"""
from __future__ import annotations
import json
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.intent_turns.model import IntentTurnRow
def _loads(value: Any) -> Any:
if isinstance(value, str) and value:
try:
return json.loads(value)
except (json.JSONDecodeError, TypeError):
return value
return value
class IntentTurnRepository:
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: IntentTurnRow | None) -> dict[str, Any] | None:
if row is None:
return None
return {
"id": row.id,
"thread_id": row.thread_id,
"client_turn_id": row.client_turn_id,
"message_hash": row.message_hash,
"status": row.status,
"result_json": _loads(row.result_json),
"error": row.error,
"lease_owner": row.lease_owner,
"lease_until": row.lease_until.isoformat() if row.lease_until else None,
"created_at": row.created_at,
"updated_at": row.updated_at,
}
async def get(self, thread_id: str, client_turn_id: str) -> dict[str, Any] | None:
stmt = select(IntentTurnRow).where(
IntentTurnRow.thread_id == thread_id,
IntentTurnRow.client_turn_id == client_turn_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row)
async def try_claim(
self,
*,
thread_id: str,
client_turn_id: str,
message_hash: str,
lease_owner: str,
lease_until: datetime,
) -> dict[str, Any]:
"""占用一个意图回合 —— 跨 worker 单飞 + 幂等的入口。
返回 ``{"action": ..., "row": {...}}``,``action`` 取值:
* ``"run"`` —— 本调用方赢得占用(新建 / 抢占续跑),应启 run 并在终态写回。
* ``"replay"`` —— 该回合已 ``done``,原样回放 ``row.result_json``,**不再启 run**。
* ``"duplicate"``—— 该回合仍在跑且租约有效(或刚被别处抢占),本请求是重复的
in-flight 请求,应丢弃(原请求仍在流式吐结果)。
* ``"conflict"`` —— 同一 ``client_turn_id`` 但 ``message_hash`` 不同 → 客户端 bug,409。
"""
# 1) 先尝试 INSERT 占用。
new_id = uuid4().hex
try:
async with self._sf() as session:
session.add(
IntentTurnRow(
id=new_id,
thread_id=thread_id,
client_turn_id=client_turn_id,
message_hash=message_hash,
status="running",
lease_owner=lease_owner,
lease_until=lease_until,
)
)
await session.commit()
row = await self.get(thread_id, client_turn_id)
return {"action": "run", "row": row}
except IntegrityError:
# 唯一键 (thread_id, client_turn_id) 冲突 → 幂等命中,读回既有行分流。
pass
existing = await self.get(thread_id, client_turn_id)
if existing is None:
# 极端竞态:冲突与回读之间行被清掉。降级为「可新建」再插一次(best-effort)。
return {"action": "run", "row": None}
# 2) 内容指纹不一致 → 客户端 bug(同 client_turn_id 却换了 message)。
if existing.get("message_hash") != message_hash:
return {"action": "conflict", "row": existing}
status = existing.get("status") or "running"
# 3) 已完成 → 幂等回放终态帧。
if status == "done":
return {"action": "replay", "row": existing}
# 4) 在跑或出错 → 尝试条件 UPDATE 抢占续跑(error 强制;running 要求租约已过期)。
reclaimed = await self._conditional_reclaim(
thread_id=thread_id,
client_turn_id=client_turn_id,
prev_status=status,
lease_owner=lease_owner,
lease_until=lease_until,
)
if reclaimed is not None:
return {"action": "run", "row": reclaimed}
# 5) 抢占失败:仍在跑且租约有效(或刚被别处抢占)→ 本请求是重复 in-flight,丢弃。
return {"action": "duplicate", "row": existing}
async def _conditional_reclaim(
self,
*,
thread_id: str,
client_turn_id: str,
prev_status: str,
lease_owner: str,
lease_until: datetime,
) -> dict[str, Any] | None:
"""原子条件 UPDATE 抢占:``error`` 强制续跑;``running`` 仅当租约已过期。
成功则置 ``status='running'``、写新租约,返回行;否则 None(仍在跑 / 刚被抢占)。
"""
now = datetime.now(UTC)
where = [
IntentTurnRow.thread_id == thread_id,
IntentTurnRow.client_turn_id == client_turn_id,
]
if prev_status == "error":
where.append(IntentTurnRow.status == "error")
else:
where.extend(
[
IntentTurnRow.status == "running",
IntentTurnRow.lease_until.is_not(None),
IntentTurnRow.lease_until < now,
]
)
values = {
"status": "running",
"lease_owner": lease_owner,
"lease_until": lease_until,
"updated_at": now,
}
async with self._sf() as session:
result = await session.execute(
update(IntentTurnRow).where(*where).values(**values)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(IntentTurnRow).where(
IntentTurnRow.thread_id == thread_id,
IntentTurnRow.client_turn_id == client_turn_id,
)
)
).scalar_one_or_none()
return self._row_to_dict(row)
async def record_result(
self,
*,
thread_id: str,
client_turn_id: str,
lease_owner: str,
status: str,
result_json: Any = None,
error: str | None = None,
) -> bool:
"""持租者把终态帧写回(条件 UPDATE 要求租约匹配),并清租约。
只有赢得 ``try_claim``(``action='run'``)的调用方才应调用,且 ``lease_owner``
必须与领取时一致 —— 防止「抢占续跑的新 worker」被「旧 worker 的迟到收尾」覆盖。
"""
now = datetime.now(UTC)
values: dict[str, Any] = {
"status": status,
"lease_owner": None,
"lease_until": None,
"updated_at": now,
}
if result_json is not None:
values["result_json"] = json.dumps(result_json, ensure_ascii=False)
if error is not None:
values["error"] = error
where = [
IntentTurnRow.thread_id == thread_id,
IntentTurnRow.client_turn_id == client_turn_id,
IntentTurnRow.lease_owner == lease_owner,
IntentTurnRow.status == "running",
]
async with self._sf() as session:
result = await session.execute(
update(IntentTurnRow).where(*where).values(**values)
)
if result.rowcount == 0:
return False
await session.commit()
return True
@@ -0,0 +1,23 @@
"""Light-app registry persistence (轻应用中心)."""
from __future__ import annotations
from deerflow.persistence.light_apps.base import DuplicateLightAppNameError, LightAppStore
from deerflow.persistence.light_apps.memory import MemoryLightAppStore
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.light_apps.sql import LightAppRepository
__all__ = [
"DuplicateLightAppNameError",
"LightAppRepository",
"LightAppRow",
"LightAppStore",
"MemoryLightAppStore",
"make_light_app_store",
]
def make_light_app_store(session_factory):
if session_factory is None:
return MemoryLightAppStore()
return LightAppRepository(session_factory)
@@ -0,0 +1,46 @@
"""Abstract light-app registry persistence API.
Global / shared store: no ``user_id`` scoping. Reads are open to any caller; the
router gates writes behind admin. ``created_by`` / ``updated_by`` are audit-only.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class DuplicateLightAppNameError(ValueError):
"""Raised when creating/updating an app would collide on the unique name."""
class LightAppStore(ABC):
@abstractmethod
async def list_apps(
self,
*,
app_type: str | None = None,
mount_location: str | None = None,
enabled_only: bool = False,
) -> list[dict[str, Any]]:
"""Return registered apps (newest first), optionally filtered."""
raise NotImplementedError
@abstractmethod
async def get_app(self, app_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_app(self, data: dict[str, Any]) -> dict[str, Any]:
"""Create an app. Raises ``DuplicateLightAppNameError`` on a name clash."""
raise NotImplementedError
@abstractmethod
async def update_app(self, app_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
"""Partial update. Returns None if the app does not exist. Raises
``DuplicateLightAppNameError`` when renaming onto an existing name."""
raise NotImplementedError
@abstractmethod
async def delete_app(self, app_id: str) -> bool:
raise NotImplementedError
@@ -0,0 +1,97 @@
"""In-memory light-app registry store (used when no DB is configured / tests)."""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.light_apps.base import DuplicateLightAppNameError, LightAppStore
_WRITABLE = {
"name",
"description",
"app_type",
"url",
"route_params",
"mount_location",
"icon",
"enabled",
"sort_order",
"updated_by",
}
class MemoryLightAppStore(LightAppStore):
def __init__(self) -> None:
self._apps: dict[str, dict[str, Any]] = {}
@staticmethod
def _public(row: dict[str, Any]) -> dict[str, Any]:
out = dict(row)
out["route_params"] = list(out.get("route_params") or [])
return out
def _name_taken(self, name: str, *, exclude_id: str | None = None) -> bool:
return any(r["name"] == name and r["app_id"] != exclude_id for r in self._apps.values())
async def list_apps(
self,
*,
app_type: str | None = None,
mount_location: str | None = None,
enabled_only: bool = False,
) -> list[dict[str, Any]]:
rows = list(self._apps.values())
if app_type:
rows = [r for r in rows if r["app_type"] == app_type]
if mount_location:
rows = [r for r in rows if r.get("mount_location") == mount_location]
if enabled_only:
rows = [r for r in rows if r.get("enabled", True)]
rows.sort(key=lambda r: (r.get("sort_order", 0), r.get("created_at", "")))
return [self._public(r) for r in rows]
async def get_app(self, app_id: str) -> dict[str, Any] | None:
row = self._apps.get(app_id)
return self._public(row) if row is not None else None
async def create_app(self, data: dict[str, Any]) -> dict[str, Any]:
if self._name_taken(data["name"]):
raise DuplicateLightAppNameError(data["name"])
now = datetime.now(UTC).isoformat()
app_id = data.get("app_id") or uuid.uuid4().hex[:12]
row = {
"app_id": app_id,
"name": data["name"],
"description": data.get("description"),
"app_type": data["app_type"],
"url": data["url"],
"route_params": data.get("route_params") or [],
"mount_location": data.get("mount_location"),
"icon": data.get("icon"),
"enabled": data.get("enabled", True),
"sort_order": data.get("sort_order", 0),
"created_by": data.get("created_by"),
"updated_by": data.get("created_by"),
"created_at": now,
"updated_at": now,
}
self._apps[app_id] = row
return self._public(row)
async def update_app(self, app_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._apps.get(app_id)
if row is None:
return None
new_name = data.get("name")
if new_name is not None and new_name != row["name"] and self._name_taken(new_name, exclude_id=app_id):
raise DuplicateLightAppNameError(new_name)
for key, value in data.items():
if key in _WRITABLE:
row[key] = value
row["updated_at"] = datetime.now(UTC).isoformat()
return self._public(row)
async def delete_app(self, app_id: str) -> bool:
return self._apps.pop(app_id, None) is not None
@@ -0,0 +1,42 @@
"""ORM model for the light-app registry (轻应用中心).
Global / shared — there is no ``user_id`` scoping; every account sees the same
registered apps. ``created_by`` / ``updated_by`` are audit-only.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class LightAppRow(Base):
__tablename__ = "light_apps"
__table_args__ = (UniqueConstraint("name", name="uq_light_apps_name"),)
app_id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(127))
description: Mapped[str | None] = mapped_column(Text)
# "window_open" (open in a new tab) | "iframe" (embed under a parent menu)
app_type: Mapped[str] = mapped_column(String(32))
url: Mapped[str] = mapped_column(Text)
# List of route-param descriptors resolved at launch time by the frontend:
# {"key": str, "paramType": "login"|"theme", "source": str,
# "customValue": str, "themeValue": str}
route_params_json: Mapped[list] = mapped_column(PortableJSON(), default=list)
# iframe only: the parent sidebar menu id to mount under
# (e.g. "xk-center-task" / "pg-center-task" / "ty-center-task" /
# "comprehensive-management"). NULL for window_open apps.
mount_location: Mapped[str | None] = mapped_column(String(64), index=True)
icon: Mapped[str | None] = mapped_column(String(64))
enabled: Mapped[bool] = mapped_column(Boolean, default=True, index=True)
sort_order: Mapped[int] = mapped_column(Integer, default=0)
created_by: Mapped[str | None] = mapped_column(String(64))
updated_by: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,135 @@
"""SQLAlchemy-backed light-app registry store (global / shared)."""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.light_apps.base import DuplicateLightAppNameError, LightAppStore
from deerflow.persistence.light_apps.model import LightAppRow
# Columns a caller may set/patch (everything except identity + timestamps).
_WRITABLE = {
"name",
"description",
"app_type",
"url",
"route_params",
"mount_location",
"icon",
"enabled",
"sort_order",
"updated_by",
}
def _to_dict(row: LightAppRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
data["route_params"] = data.pop("route_params_json", None) or []
return data
def _column_for(key: str) -> str:
return "route_params_json" if key == "route_params" else key
class LightAppRepository(LightAppStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_apps(
self,
*,
app_type: str | None = None,
mount_location: str | None = None,
enabled_only: bool = False,
) -> list[dict[str, Any]]:
stmt = select(LightAppRow)
if app_type:
stmt = stmt.where(LightAppRow.app_type == app_type)
if mount_location:
stmt = stmt.where(LightAppRow.mount_location == mount_location)
if enabled_only:
stmt = stmt.where(LightAppRow.enabled.is_(True))
stmt = stmt.order_by(LightAppRow.sort_order.asc(), LightAppRow.created_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def get_app(self, app_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LightAppRow, app_id)
return _to_dict(row) if row is not None else None
async def _name_taken(self, session: AsyncSession, name: str, *, exclude_id: str | None = None) -> bool:
stmt = select(LightAppRow.app_id).where(LightAppRow.name == name)
if exclude_id is not None:
stmt = stmt.where(LightAppRow.app_id != exclude_id)
result = await session.execute(stmt.limit(1))
return result.first() is not None
async def create_app(self, data: dict[str, Any]) -> dict[str, Any]:
row = LightAppRow(
app_id=data.get("app_id") or uuid.uuid4().hex[:12],
name=data["name"],
description=data.get("description"),
app_type=data["app_type"],
url=data["url"],
route_params_json=data.get("route_params") or [],
mount_location=data.get("mount_location"),
icon=data.get("icon"),
enabled=data.get("enabled", True),
sort_order=data.get("sort_order", 0),
created_by=data.get("created_by"),
updated_by=data.get("created_by"),
)
async with self._sf() as session:
if await self._name_taken(session, row.name):
raise DuplicateLightAppNameError(row.name)
session.add(row)
try:
await session.commit()
except IntegrityError as exc:
await session.rollback()
raise DuplicateLightAppNameError(row.name) from exc
await session.refresh(row)
return _to_dict(row)
async def update_app(self, app_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LightAppRow, app_id)
if row is None:
return None
new_name = data.get("name")
if new_name is not None and new_name != row.name and await self._name_taken(session, new_name, exclude_id=app_id):
raise DuplicateLightAppNameError(new_name)
for key, value in data.items():
if key not in _WRITABLE:
continue
setattr(row, _column_for(key), value)
row.updated_at = datetime.now(UTC)
try:
await session.commit()
except IntegrityError as exc:
await session.rollback()
raise DuplicateLightAppNameError(new_name or row.name) from exc
await session.refresh(row)
return _to_dict(row)
async def delete_app(self, app_id: str) -> bool:
async with self._sf() as session:
row = await session.get(LightAppRow, app_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
@@ -0,0 +1,25 @@
"""Persistence layer for per-LLM-call metrics.
Each completed LLM invocation (success or failure) writes a single row so
administrators can audit usage and speed. Designed to be cheap to write —
the middleware fires once per provider call and never blocks streaming.
"""
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsStore
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llm_metrics.sql import SqlLlmMetricsStore
__all__ = [
"LlmCallMetricRow",
"LlmMetricRecord",
"LlmMetricsStore",
"SqlLlmMetricsStore",
"make_llm_metrics_store",
]
def make_llm_metrics_store(session_factory) -> LlmMetricsStore | None:
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
if session_factory is None:
return None
return SqlLlmMetricsStore(session_factory)
@@ -0,0 +1,69 @@
"""Abstract repository contract for LLM call metrics."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any
@dataclass
class LlmMetricRecord:
"""In-memory shape of a metric row before persistence."""
id: str
created_at: datetime
user_id: str | None = None
thread_id: str | None = None
run_id: str | None = None
agent_name: str | None = None
model_name: str | None = None
duration_ms: int = 0
input_tokens: int | None = None
output_tokens: int | None = None
total_tokens: int | None = None
tokens_per_sec: float | None = None
status: str = "success"
error_type: str | None = None
error_message: str | None = None
@dataclass
class LlmMetricsQuery:
"""Filter set for the admin list / export endpoints."""
user_id: str | None = None
# Multiple user ids — set when a fuzzy username search resolves to several
# accounts. Takes precedence over ``user_id`` when non-empty.
user_ids: list[str] | None = None
model_name: str | None = None
status: str | None = None
since: datetime | None = None
until: datetime | None = None
limit: int = 100
offset: int = 0
class LlmMetricsStore(ABC):
"""Persistence contract for the metrics middleware + admin API."""
@abstractmethod
async def record(self, metric: LlmMetricRecord) -> None:
"""Persist one metric row. Must never raise on best-effort writes."""
raise NotImplementedError
@abstractmethod
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
"""Return the matching rows (newest first), paginated by ``query``."""
raise NotImplementedError
@abstractmethod
async def count(self, query: LlmMetricsQuery) -> int:
"""Return the unpaginated row count for ``query``."""
raise NotImplementedError
@abstractmethod
async def iter_all(self, query: LlmMetricsQuery):
"""Yield rows matching ``query`` without pagination — used by export."""
raise NotImplementedError
@@ -0,0 +1,68 @@
"""ORM model for per-LLM-call metrics."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Index, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class LlmCallMetricRow(Base):
"""One row per completed (or failed) LLM provider invocation."""
__tablename__ = "llm_call_metrics"
__table_args__ = (
Index("ix_llm_call_metrics_created_user", "created_at", "user_id"),
Index("ix_llm_call_metrics_run_created", "run_id", "created_at"),
Index("ix_llm_call_metrics_created_model_user", "created_at", "model_name", "user_id"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# Wall-clock time the LLM call ended. Indexed because admin views always
# sort/filter by recency.
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
# Wall-clock duration of the provider invocation in milliseconds. Always set.
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
input_tokens: Mapped[int | None] = mapped_column(Integer)
output_tokens: Mapped[int | None] = mapped_column(Integer)
total_tokens: Mapped[int | None] = mapped_column(Integer)
# Output tokens / seconds; null when there is no completion (errors).
tokens_per_sec: Mapped[float | None] = mapped_column(Float)
# "success" or "error". Errors store the classification and original message
# so admins can scan for transient vs auth/quota issues.
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
error_type: Mapped[str | None] = mapped_column(String(64))
error_message: Mapped[str | None] = mapped_column(Text)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
"user_id": self.user_id,
"thread_id": self.thread_id,
"run_id": self.run_id,
"agent_name": self.agent_name,
"model_name": self.model_name,
"duration_ms": self.duration_ms,
"input_tokens": self.input_tokens,
"output_tokens": self.output_tokens,
"total_tokens": self.total_tokens,
"tokens_per_sec": self.tokens_per_sec,
"status": self.status,
"error_type": self.error_type,
"error_message": self.error_message,
}
@@ -0,0 +1,112 @@
"""SQLAlchemy-backed LLM metrics repository."""
from __future__ import annotations
import logging
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import desc, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery, LlmMetricsStore
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
logger = logging.getLogger(__name__)
def _row_to_dict(row: LlmCallMetricRow) -> dict[str, Any]:
return row.to_dict()
def _normalize_created_at(value: datetime) -> datetime:
return value if value.tzinfo else value.replace(tzinfo=UTC)
class SqlLlmMetricsStore(LlmMetricsStore):
"""SQLite/Postgres-backed implementation; writes are intentionally fire-and-forget."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def record(self, metric: LlmMetricRecord) -> None:
row = LlmCallMetricRow(
id=metric.id,
created_at=_normalize_created_at(metric.created_at),
user_id=metric.user_id,
thread_id=metric.thread_id,
run_id=metric.run_id,
agent_name=metric.agent_name,
model_name=metric.model_name,
duration_ms=metric.duration_ms,
input_tokens=metric.input_tokens,
output_tokens=metric.output_tokens,
total_tokens=metric.total_tokens,
tokens_per_sec=metric.tokens_per_sec,
status=metric.status,
error_type=metric.error_type,
error_message=metric.error_message,
)
try:
async with self._sf() as session:
session.add(row)
await session.commit()
except Exception:
# Metrics are best-effort. Never let a persistence failure surface
# to the agent runtime — log and move on.
logger.warning("Failed to persist llm_call_metrics row %s", metric.id, exc_info=True)
def _apply_filters(self, stmt, query: LlmMetricsQuery):
if query.user_ids:
stmt = stmt.where(LlmCallMetricRow.user_id.in_(query.user_ids))
elif query.user_id:
stmt = stmt.where(LlmCallMetricRow.user_id == query.user_id)
if query.model_name:
stmt = stmt.where(LlmCallMetricRow.model_name == query.model_name)
if query.status:
stmt = stmt.where(LlmCallMetricRow.status == query.status)
if query.since is not None:
stmt = stmt.where(LlmCallMetricRow.created_at >= _normalize_created_at(query.since))
if query.until is not None:
stmt = stmt.where(LlmCallMetricRow.created_at <= _normalize_created_at(query.until))
return stmt
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
stmt = select(LlmCallMetricRow)
stmt = self._apply_filters(stmt, query)
stmt = stmt.order_by(desc(LlmCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
async with self._sf() as session:
result = await session.execute(stmt)
return [_row_to_dict(row) for row in result.scalars()]
async def count(self, query: LlmMetricsQuery) -> int:
stmt = select(func.count()).select_from(LlmCallMetricRow)
stmt = self._apply_filters(stmt, query)
async with self._sf() as session:
return (await session.scalar(stmt)) or 0
async def iter_all(self, query: LlmMetricsQuery) -> AsyncIterator[dict[str, Any]]:
# Page through the result so the Excel export stays bounded in memory
# for large date windows.
page_size = 500
offset = 0
while True:
page_query = LlmMetricsQuery(
user_id=query.user_id,
user_ids=query.user_ids,
model_name=query.model_name,
status=query.status,
since=query.since,
until=query.until,
limit=page_size,
offset=offset,
)
page = await self.list(page_query)
if not page:
return
for item in page:
yield item
if len(page) < page_size:
return
offset += page_size
@@ -0,0 +1,22 @@
"""DeerFlow-owned LLMWiki authorization metadata."""
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.memory import MemoryLlmWikiStore
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki.sql import LlmWikiRepository
__all__ = [
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
"LlmWikiRepository",
"LlmWikiScope",
"LlmWikiStore",
"MemoryLlmWikiStore",
"make_llmwiki_store",
]
def make_llmwiki_store(session_factory):
if session_factory is None:
return MemoryLlmWikiStore()
return LlmWikiRepository(session_factory)
@@ -0,0 +1,89 @@
"""Persistence contract for DeerFlow-owned LLMWiki authorization metadata."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, Literal
LlmWikiScope = Literal["personal", "public", "all"]
class LlmWikiStore(ABC):
@abstractmethod
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]: ...
@abstractmethod
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]: ...
@abstractmethod
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None: ...
@abstractmethod
async def adopt_conversation_deposit_mapping(
self,
mapping_id: str,
*,
name: str,
description: str,
) -> dict[str, Any] | None: ...
@abstractmethod
async def delete_mapping(self, mapping_id: str) -> bool: ...
@abstractmethod
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None: ...
@abstractmethod
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]: ...
@abstractmethod
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None: ...
@abstractmethod
async def get_conversation_deposit(
self,
*,
thread_id: str,
assistant_message_id: str,
) -> dict[str, Any] | None: ...
@abstractmethod
async def upsert_conversation_deposit(
self,
*,
thread_id: str,
human_message_id: str | None,
assistant_message_id: str,
knowledge_base_mapping_id: str,
weknora_knowledge_id: str | None,
title: str,
created_by_user_id: str | None,
) -> dict[str, Any]: ...
@abstractmethod
async def delete_conversation_deposits(
self,
*,
thread_id: str,
message_ids: list[str] | None = None,
) -> list[dict[str, Any]]: ...
@@ -0,0 +1,231 @@
"""In-memory LLMWiki metadata store for tests and DB-less deployments."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
def _now() -> str:
return datetime.now(UTC).isoformat()
class MemoryLlmWikiStore(LlmWikiStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._conversation_deposits: dict[str, dict[str, Any]] = {}
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
if any(row["weknora_id"] == weknora_id for row in self._rows.values()):
raise ValueError("WeKnora knowledge base is already mapped")
now = _now()
row = {
"id": str(uuid4()),
"weknora_id": weknora_id,
"owner_user_id": owner_user_id,
"name": name,
"description": description,
"kb_type": kb_type,
"publication_status": "private",
"published_by": None,
"reviewed_by": None,
"published_at": None,
"wiki_index_enabled": True,
"external_search_enabled": False,
"external_search_updated_at": None,
"created_at": now,
"updated_at": now,
}
self._rows[row["id"]] = row
return dict(row)
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
rows = list(self._rows.values())
if scope == "personal":
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id]
elif scope == "public":
rows = [row for row in rows if row["publication_status"] == "published"]
else:
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id or row["publication_status"] == "published"]
return [dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
if is_admin or row["owner_user_id"] == user_id:
return dict(row)
if not write and row["publication_status"] == "published":
return dict(row)
return None
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
for row in self._rows.values():
if row["weknora_id"] == weknora_id:
return dict(row)
return None
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
if name is not None:
row["name"] = name
if description is not None:
row["description"] = description
if wiki_index_enabled is not None:
row["wiki_index_enabled"] = wiki_index_enabled
if external_search_enabled is not None:
row["external_search_enabled"] = external_search_enabled
row["external_search_updated_at"] = _now()
row["updated_at"] = _now()
return dict(row)
async def adopt_conversation_deposit_mapping(
self,
mapping_id: str,
*,
name: str,
description: str,
) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
now = _now()
row.update(
{
"owner_user_id": "system",
"name": name,
"description": description,
"publication_status": "published",
"published_by": "system",
"reviewed_by": None,
"published_at": row.get("published_at") or now,
"updated_at": now,
}
)
return dict(row)
async def delete_mapping(self, mapping_id: str) -> bool:
return self._rows.pop(mapping_id, None) is not None
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None or (not is_admin and row["owner_user_id"] != actor_user_id):
return None
row["publication_status"] = "published"
row["published_by"] = actor_user_id
row["reviewed_by"] = None
row["published_at"] = _now()
row["updated_at"] = _now()
return dict(row)
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None or row["publication_status"] != "pending":
return None
row["publication_status"] = "published" if approved else "rejected"
row["reviewed_by"] = reviewer_user_id
row["published_at"] = _now() if approved else None
row["updated_at"] = _now()
return dict(row)
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None or (not is_admin and row["owner_user_id"] != actor_user_id):
return None
row["publication_status"] = "private"
row["reviewed_by"] = actor_user_id if is_admin else row.get("reviewed_by")
row["published_at"] = None
row["updated_at"] = _now()
return dict(row)
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
rows = [row for row in self._rows.values() if row["publication_status"] != "private"]
if status:
rows = [row for row in rows if row["publication_status"] == status]
return [dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
for row in self._rows.values():
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
return dict(row)
return None
async def get_conversation_deposit(
self,
*,
thread_id: str,
assistant_message_id: str,
) -> dict[str, Any] | None:
key = f"{thread_id}:{assistant_message_id}"
row = self._conversation_deposits.get(key)
return dict(row) if row else None
async def upsert_conversation_deposit(
self,
*,
thread_id: str,
human_message_id: str | None,
assistant_message_id: str,
knowledge_base_mapping_id: str,
weknora_knowledge_id: str | None,
title: str,
created_by_user_id: str | None,
) -> dict[str, Any]:
key = f"{thread_id}:{assistant_message_id}"
now = _now()
row = self._conversation_deposits.get(key)
if row is None:
row = {
"id": str(uuid4()),
"thread_id": thread_id,
"human_message_id": human_message_id,
"assistant_message_id": assistant_message_id,
"knowledge_base_mapping_id": knowledge_base_mapping_id,
"weknora_knowledge_id": weknora_knowledge_id,
"title": title,
"created_by_user_id": created_by_user_id,
"created_at": now,
"updated_at": now,
}
self._conversation_deposits[key] = row
else:
row.update(
{
"human_message_id": human_message_id,
"knowledge_base_mapping_id": knowledge_base_mapping_id,
"weknora_knowledge_id": weknora_knowledge_id,
"title": title,
"updated_at": now,
}
)
return dict(row)
async def delete_conversation_deposits(
self,
*,
thread_id: str,
message_ids: list[str] | None = None,
) -> list[dict[str, Any]]:
ids = set(message_ids or [])
deleted: list[dict[str, Any]] = []
for key, row in list(self._conversation_deposits.items()):
if row.get("thread_id") != thread_id:
continue
if ids and row.get("human_message_id") not in ids and row.get("assistant_message_id") not in ids:
continue
deleted.append(dict(row))
self._conversation_deposits.pop(key, None)
return deleted
@@ -0,0 +1,66 @@
"""ORM mapping between DeerFlow ownership and WeKnora knowledge bases."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class LlmWikiKnowledgeBaseRow(Base):
__tablename__ = "llmwiki_knowledge_bases"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
weknora_id: Mapped[str] = mapped_column(String(128), unique=True, index=True)
owner_user_id: Mapped[str] = mapped_column(String(64), index=True)
name: Mapped[str] = mapped_column(String(255), default="")
description: Mapped[str] = mapped_column(String(2048), default="")
kb_type: Mapped[str] = mapped_column(String(32), default="document")
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
wiki_index_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true(), index=True)
external_search_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
external_search_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
def to_dict(self) -> dict:
return {column.name: getattr(self, column.name) for column in self.__table__.columns}
class LlmWikiConversationDepositRow(Base):
"""Mapping from one DeerFlow answer turn to its WeKnora manual document.
The actual content lives in WeKnora. DeerFlow keeps only the stable ids it
needs to rewrite/delete the document when the user regenerates or removes a
conversation turn.
"""
__tablename__ = "llmwiki_conversation_deposits"
__table_args__ = (
UniqueConstraint(
"thread_id",
"assistant_message_id",
name="uq_llmwiki_conversation_deposit_answer",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
thread_id: Mapped[str] = mapped_column(String(128), index=True)
human_message_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
assistant_message_id: Mapped[str] = mapped_column(String(128), index=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), index=True)
weknora_knowledge_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
title: Mapped[str] = mapped_column(String(255), default="")
created_by_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
def to_dict(self) -> dict:
return {column.name: getattr(self, column.name) for column in self.__table__.columns}
@@ -0,0 +1,281 @@
"""SQLAlchemy LLMWiki metadata repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
def _now() -> datetime:
return datetime.now(UTC)
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
return data
class LlmWikiRepository(LlmWikiStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
row = LlmWikiKnowledgeBaseRow(
id=str(uuid4()),
weknora_id=weknora_id,
owner_user_id=owner_user_id,
name=name,
description=description,
kb_type=kb_type,
publication_status="private",
created_at=_now(),
updated_at=_now(),
)
async with self._sf() as session:
session.add(row)
await session.commit()
return _to_dict(row)
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
stmt = select(LlmWikiKnowledgeBaseRow)
if scope == "personal" and not is_admin:
stmt = stmt.where(LlmWikiKnowledgeBaseRow.owner_user_id == user_id)
elif scope == "public":
stmt = stmt.where(LlmWikiKnowledgeBaseRow.publication_status == "published")
elif scope == "all" and not is_admin:
stmt = stmt.where(
or_(
LlmWikiKnowledgeBaseRow.owner_user_id == user_id,
LlmWikiKnowledgeBaseRow.publication_status == "published",
)
)
stmt = stmt.order_by(LlmWikiKnowledgeBaseRow.created_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.id == mapping_id)
if not is_admin:
condition = LlmWikiKnowledgeBaseRow.owner_user_id == user_id
if not write:
condition = or_(condition, LlmWikiKnowledgeBaseRow.publication_status == "published")
stmt = stmt.where(condition)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.weknora_id == weknora_id)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
return None
if name is not None:
row.name = name
if description is not None:
row.description = description
if wiki_index_enabled is not None:
row.wiki_index_enabled = wiki_index_enabled
if external_search_enabled is not None:
row.external_search_enabled = external_search_enabled
row.external_search_updated_at = _now()
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def adopt_conversation_deposit_mapping(
self,
mapping_id: str,
*,
name: str,
description: str,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
return None
now = _now()
row.owner_user_id = "system"
row.name = name
row.description = description
row.publication_status = "published"
row.published_by = "system"
row.reviewed_by = None
row.published_at = row.published_at or now
row.updated_at = now
await session.commit()
return _to_dict(row)
async def delete_mapping(self, mapping_id: str) -> bool:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None or (not is_admin and row.owner_user_id != actor_user_id):
return None
row.publication_status = "published"
row.published_by = actor_user_id
row.reviewed_by = None
row.published_at = _now()
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None or row.publication_status != "pending":
return None
row.publication_status = "published" if approved else "rejected"
row.reviewed_by = reviewer_user_id
row.published_at = _now() if approved else None
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None or (not is_admin and row.owner_user_id != actor_user_id):
return None
row.publication_status = "private"
if is_admin:
row.reviewed_by = actor_user_id
row.published_at = None
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.publication_status != "private")
if status:
stmt = stmt.where(LlmWikiKnowledgeBaseRow.publication_status == status)
stmt = stmt.order_by(LlmWikiKnowledgeBaseRow.updated_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
stmt = (
select(LlmWikiKnowledgeBaseRow)
.where(
LlmWikiKnowledgeBaseRow.owner_user_id == "system",
LlmWikiKnowledgeBaseRow.name == "对话沉淀",
LlmWikiKnowledgeBaseRow.publication_status == "published",
)
.order_by(LlmWikiKnowledgeBaseRow.created_at.asc())
.limit(1)
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def get_conversation_deposit(
self,
*,
thread_id: str,
assistant_message_id: str,
) -> dict[str, Any] | None:
stmt = select(LlmWikiConversationDepositRow).where(
LlmWikiConversationDepositRow.thread_id == thread_id,
LlmWikiConversationDepositRow.assistant_message_id == assistant_message_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def upsert_conversation_deposit(
self,
*,
thread_id: str,
human_message_id: str | None,
assistant_message_id: str,
knowledge_base_mapping_id: str,
weknora_knowledge_id: str | None,
title: str,
created_by_user_id: str | None,
) -> dict[str, Any]:
now = _now()
async with self._sf() as session:
stmt = select(LlmWikiConversationDepositRow).where(
LlmWikiConversationDepositRow.thread_id == thread_id,
LlmWikiConversationDepositRow.assistant_message_id == assistant_message_id,
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = LlmWikiConversationDepositRow(
id=str(uuid4()),
thread_id=thread_id,
human_message_id=human_message_id,
assistant_message_id=assistant_message_id,
knowledge_base_mapping_id=knowledge_base_mapping_id,
weknora_knowledge_id=weknora_knowledge_id,
title=title,
created_by_user_id=created_by_user_id,
created_at=now,
updated_at=now,
)
session.add(row)
else:
row.human_message_id = human_message_id
row.knowledge_base_mapping_id = knowledge_base_mapping_id
row.weknora_knowledge_id = weknora_knowledge_id
row.title = title
row.updated_at = now
await session.commit()
return _to_dict(row)
async def delete_conversation_deposits(
self,
*,
thread_id: str,
message_ids: list[str] | None = None,
) -> list[dict[str, Any]]:
conditions = [LlmWikiConversationDepositRow.thread_id == thread_id]
if message_ids:
conditions.append(
or_(
LlmWikiConversationDepositRow.human_message_id.in_(message_ids),
LlmWikiConversationDepositRow.assistant_message_id.in_(message_ids),
)
)
stmt = select(LlmWikiConversationDepositRow).where(*conditions)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
deleted = [_to_dict(row) for row in rows]
if rows:
await session.execute(delete(LlmWikiConversationDepositRow).where(LlmWikiConversationDepositRow.id.in_([row.id for row in rows])))
await session.commit()
return deleted
@@ -0,0 +1,25 @@
"""Persistence facade for the DeerFlow-local Wiki vector index."""
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
return (
MemoryLlmWikiIndexStore()
if session_factory is None
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
)
__all__ = [
"LlmWikiIndexStore",
"MemoryLlmWikiIndexStore",
"SqlLlmWikiIndexStore",
"LlmWikiWikiPageRow",
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"make_llmwiki_index_store",
]
@@ -0,0 +1,85 @@
"""Storage contract for local Wiki pages, vectors, revisions and sync leases."""
from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from typing import Any
class LlmWikiIndexStore(ABC):
@abstractmethod
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None: ...
@abstractmethod
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
@abstractmethod
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
@abstractmethod
async def replace_page_vectors(
self,
*,
page: dict[str, Any],
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
) -> dict[str, Any]: ...
@abstractmethod
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
@abstractmethod
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
@abstractmethod
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
async def iter_vector_snapshot_rows(
self,
mapping_id: str,
fingerprint: str,
*,
batch_size: int = 1000,
) -> AsyncIterator[dict[str, Any]]:
"""Iterate vector rows for streaming export.
Implementations with a database backend should override this method so
very large vector packages do not need to be materialized in memory.
"""
snapshot = await self.load_vector_snapshot(mapping_id, fingerprint)
for row in snapshot.get("rows") or []:
if isinstance(row, dict):
yield row
@abstractmethod
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
@abstractmethod
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool: ...
@abstractmethod
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool: ...
@abstractmethod
async def update_sync_progress(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
remote_page_count: int,
) -> bool: ...
@abstractmethod
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
@abstractmethod
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
@abstractmethod
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]: ...
@abstractmethod
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int: ...
@@ -0,0 +1,202 @@
"""In-memory local Wiki index store used by tests and DB-less mode."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
def _now() -> datetime:
return datetime.now(UTC)
class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self) -> None:
self.pages: dict[str, dict[str, Any]] = {}
self.vectors: dict[str, list[dict[str, Any]]] = {}
self.states: dict[str, dict[str, Any]] = {}
def _state(self, mapping_id: str) -> dict[str, Any]:
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
return dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
if page_id in self.pages:
self.pages[page_id]["last_seen_sync_id"] = sync_id
self.pages[page_id]["last_error"] = None
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
page_id = existing["id"] if existing else str(uuid4())
now = _now().isoformat()
row = {
**(existing or {}),
**page,
"id": page_id,
"index_status": "ready",
"indexed_fingerprint": fingerprint,
"indexed_at": now,
"last_error": None,
"is_remote_deleted": False,
"last_seen_sync_id": sync_id,
"created_at": (existing or {}).get("created_at", now),
"updated_at": now,
}
self.pages[page_id] = row
self.vectors[page_id] = [{**item, "page_id": page_id, "knowledge_base_mapping_id": page["knowledge_base_mapping_id"], "embedding_fingerprint": fingerprint} for item in vectors]
state = self._state(page["knowledge_base_mapping_id"])
state["index_revision"] += 1
state["embedding_fingerprint"] = fingerprint
return dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
if existing:
self.pages[existing["id"]].update(
{
"last_seen_sync_id": sync_id,
"last_error": error,
# Preserve the last atomically completed generation.
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
}
)
return
page_id = str(uuid4())
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
changed = 0
for page in self.pages.values():
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
page.update({"is_remote_deleted": True, "index_status": "deleted"})
self.vectors.pop(page["id"], None)
changed += 1
if changed:
self._state(mapping_id)["index_revision"] += 1
return changed
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
rows = []
for page_id, vectors in self.vectors.items():
page = self.pages[page_id]
if page["knowledge_base_mapping_id"] != mapping_id or page.get("is_remote_deleted") or page.get("status") not in {"published", "draft"} or page.get("index_status") != "ready":
continue
rows.extend({**vector, "page": dict(page)} for vector in vectors if vector["embedding_fingerprint"] == fingerprint)
state = self._state(mapping_id)
return {"revision": state["index_revision"], "fingerprint": fingerprint, "rows": rows}
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
state = self._state(mapping_id)
return int(state["index_revision"]), state.get("embedding_fingerprint"), str(state.get("state") or "idle")
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
state = self._state(mapping_id)
now = _now()
expires = state.get("lease_expires_at")
if state.get("lease_owner") is not None and isinstance(expires, datetime) and expires >= now:
return False
state.update(
{
"state": "rebuilding" if state.get("state") == "rebuild_required" else "running",
"active_sync_id": sync_id,
"lease_owner": owner,
"lease_expires_at": now + timedelta(seconds=lease_seconds),
"last_started_at": now,
}
)
return True
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
state = self._state(mapping_id)
if state.get("lease_owner") != owner:
return False
state["lease_expires_at"] = _now() + timedelta(seconds=lease_seconds)
return True
async def update_sync_progress(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
remote_page_count: int,
) -> bool:
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
return False
state["remote_page_count"] = max(0, remote_page_count)
state["updated_at"] = _now()
return True
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
return
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
was_rebuilding = state.get("state") == "rebuilding"
state.update(
{
"state": "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed",
"lease_owner": None,
"lease_expires_at": None,
"active_sync_id": None,
"last_error": error,
"remote_page_count": remote_page_count,
"local_page_count": len(pages),
"ready_page_count": sum(p.get("index_status") == "ready" for p in pages),
"failed_page_count": sum(p.get("index_status") in {"failed", "stale"} or p.get("last_error") is not None for p in pages),
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
"last_success_at": _now() if success else state.get("last_success_at"),
}
)
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
wanted = set(mapping_ids or [])
return [dict(value) for key, value in self.states.items() if not wanted or key in wanted]
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
result = []
for page in sorted(
(
page
for page in self.pages.values()
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
),
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
):
result.append(
{
"id": page["id"],
"slug": page["slug"],
"title": page["title"],
"page_type": page["page_type"],
"status": page["status"],
"index_status": page["index_status"],
"vector_count": len(self.vectors.get(page["id"], [])),
"indexed_at": page.get("indexed_at"),
"last_error": page.get("last_error"),
"last_seen_sync_id": page.get("last_seen_sync_id"),
"updated_at": page.get("updated_at"),
}
)
return result
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
for mapping_id in mapping_ids:
state = self._state(mapping_id)
state.update({"state": "rebuild_required", "embedding_fingerprint": fingerprint})
for page in self.pages.values():
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted"):
page["index_status"] = "stale"
return len(mapping_ids)
@@ -0,0 +1,94 @@
"""Portable ORM rows for the DeerFlow-local WeKnora Wiki mirror/index."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, false
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class LlmWikiWikiPageRow(Base):
__tablename__ = "llmwiki_wiki_pages"
__table_args__ = (
UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
Index("ix_llmwiki_wiki_page_mapping_visibility", "knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False, index=True)
remote_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
slug: Mapped[str] = mapped_column(String(1024), nullable=False)
slug_hash: Mapped[str] = mapped_column(String(64), nullable=False)
title: Mapped[str] = mapped_column(String(512), nullable=False, default="", index=True)
summary: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
content_md: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
page_type: Mapped[str] = mapped_column(String(32), nullable=False, default="article", index=True)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="published", index=True)
aliases: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
category_path: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
parent_slug: Mapped[str | None] = mapped_column(String(1024), nullable=True)
wiki_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
source_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
chunk_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
page_version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
remote_created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
remote_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
index_status: Mapped[str] = mapped_column(String(24), nullable=False, default="pending", index=True)
indexed_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
indexed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
is_remote_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
last_seen_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now, index=True)
class LlmWikiWikiVectorRow(Base):
__tablename__ = "llmwiki_wiki_vectors"
__table_args__ = (
UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
Index("ix_llmwiki_wiki_vector_snapshot", "knowledge_base_mapping_id", "embedding_fingerprint"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
page_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False, index=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
section_index: Mapped[int] = mapped_column(Integer, nullable=False)
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
class LlmWikiWikiSyncStateRow(Base):
__tablename__ = "llmwiki_wiki_sync_states"
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True)
state: Mapped[str] = mapped_column(String(24), nullable=False, default="idle")
active_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_success_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
remote_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
local_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
ready_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
vector_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
@@ -0,0 +1,400 @@
"""SQLAlchemy implementation of the local Wiki index store."""
from __future__ import annotations
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import case, delete, func, insert, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row) -> dict[str, Any]:
value = row.to_dict()
for key, item in tuple(value.items()):
if isinstance(item, datetime):
value[key] = item.isoformat()
return value
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None:
self._sf = session_factory
self.write_batch_size = max(1, min(int(write_batch_size), 500))
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if row is None:
row = LlmWikiWikiSyncStateRow(knowledge_base_mapping_id=mapping_id, state="idle", index_revision=0, updated_at=_now())
session.add(row)
await session.flush()
return row
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
async with self._sf() as session:
result = (await session.execute(stmt)).one_or_none()
return {**_dict(result[0]), "vector_count": result[1]} if result else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
async with self._sf() as session:
await session.execute(
update(LlmWikiWikiPageRow)
.where(LlmWikiWikiPageRow.id == page_id)
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
)
await session.commit()
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
row = (await session.execute(stmt)).scalar_one_or_none()
if row is not None and row.slug != page["slug"]:
raise ValueError("Wiki slug hash collision detected")
now = _now()
if row is None:
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
session.add(row)
else:
for key, value in page.items():
setattr(row, key, value)
row.index_status = "ready"
row.indexed_fingerprint = fingerprint
row.indexed_at = now
row.last_error = None
row.is_remote_deleted = False
row.last_seen_sync_id = sync_id
row.updated_at = now
await session.flush()
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
vector_rows = [
{
"id": str(uuid4()),
"page_id": row.id,
"knowledge_base_mapping_id": mapping_id,
"section_index": int(item["section_index"]),
"heading": item.get("heading"),
"section_content": str(item["section_content"]),
"content_hash": str(item["content_hash"]),
"embedding_fingerprint": fingerprint,
"embedding_dimensions": int(item["embedding_dimensions"]),
"vector_blob": bytes(item["vector_blob"]),
"created_at": now,
}
for item in vectors
]
# Keep the page replacement atomic, but cap each SQL statement so
# large Wiki pages cannot flood the database with one huge insert.
for start in range(0, len(vector_rows), self.write_batch_size):
await session.execute(
insert(LlmWikiWikiVectorRow),
vector_rows[start : start + self.write_batch_size],
)
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.embedding_fingerprint = fingerprint
state.updated_at = now
await session.commit()
return _dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
mapping_id = str(page["knowledge_base_mapping_id"])
safe_error = error[:4000]
async with self._sf() as session:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
row = (await session.execute(stmt)).scalar_one_or_none()
now = _now()
if row is None:
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
session.add(row)
row.index_status = "failed"
else:
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))).scalar_one())
# Preserve the last atomically completed generation while the
# newer remote version remains pending after an embedding error.
row.index_status = "ready" if vector_count else "failed"
row.last_seen_sync_id = sync_id
row.last_error = safe_error
row.updated_at = now
await self._ensure_state(session, mapping_id)
await session.commit()
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
async with self._sf() as session:
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
or_(LlmWikiWikiPageRow.last_seen_sync_id.is_(None), LlmWikiWikiPageRow.last_seen_sync_id != sync_id),
)
.with_for_update()
)
rows = list((await session.execute(stmt)).scalars())
if not rows:
return 0
now = _now()
page_ids = [row.id for row in rows]
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id.in_(page_ids)))
for row in rows:
row.is_remote_deleted = True
row.index_status = "deleted"
row.updated_at = now
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return len(rows)
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
stmt = (
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
.where(
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
LlmWikiWikiPageRow.index_status == "ready",
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
)
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
pairs = list((await session.execute(stmt)).all())
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
async def iter_vector_snapshot_rows(
self,
mapping_id: str,
fingerprint: str,
*,
batch_size: int = 1000,
) -> AsyncIterator[dict[str, Any]]:
batch = max(1, min(int(batch_size), 5000))
offset = 0
while True:
stmt = (
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
.where(
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
LlmWikiWikiPageRow.index_status == "ready",
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
.limit(batch)
.offset(offset)
)
async with self._sf() as session:
pairs = list((await session.execute(stmt)).all())
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
if not rows:
break
for row in rows:
yield row
if len(rows) < batch:
break
offset += len(rows)
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if state is None:
return 0, None, "not_synced"
return int(state.index_revision or 0), state.embedding_fingerprint, state.state
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
now = _now()
async with self._sf() as session:
try:
await self._ensure_state(session, mapping_id)
await session.commit()
except IntegrityError:
# Another worker created the one-per-KB state row between our
# read and insert. Roll back this isolated bootstrap session;
# the conditional UPDATE below is the actual lease election.
await session.rollback()
async with self._sf() as session:
stmt = (
update(LlmWikiWikiSyncStateRow)
.where(
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
or_(
LlmWikiWikiSyncStateRow.lease_owner.is_(None),
LlmWikiWikiSyncStateRow.lease_expires_at.is_(None),
LlmWikiWikiSyncStateRow.lease_expires_at < now,
),
)
.values(
state=case((LlmWikiWikiSyncStateRow.state == "rebuild_required", "rebuilding"), else_="running"),
active_sync_id=sync_id,
lease_owner=owner,
lease_expires_at=now + timedelta(seconds=lease_seconds),
last_started_at=now,
last_error=None,
updated_at=now,
)
)
result = await session.execute(stmt)
await session.commit()
return bool(result.rowcount)
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
now = _now()
async with self._sf() as session:
result = await session.execute(
update(LlmWikiWikiSyncStateRow)
.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiSyncStateRow.lease_owner == owner)
.values(lease_expires_at=now + timedelta(seconds=lease_seconds), updated_at=now)
)
await session.commit()
return bool(result.rowcount)
async def update_sync_progress(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
remote_page_count: int,
) -> bool:
async with self._sf() as session:
result = await session.execute(
update(LlmWikiWikiSyncStateRow)
.where(
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiSyncStateRow.lease_owner == owner,
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
)
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
)
await session.commit()
return bool(result.rowcount)
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
return
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
failed_count = int(
(
await session.execute(
select(func.count())
.select_from(LlmWikiWikiPageRow)
.where(
*visible,
or_(
LlmWikiWikiPageRow.index_status.in_(["failed", "stale"]),
LlmWikiWikiPageRow.last_error.is_not(None),
),
)
)
).scalar_one()
)
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id))).scalar_one())
now = _now()
was_rebuilding = state.state == "rebuilding"
state.state = "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed"
state.active_sync_id = None
state.lease_owner = None
state.lease_expires_at = None
state.last_error = (error or "")[:4000] or None
state.remote_page_count = remote_page_count
state.local_page_count = local_count
state.ready_page_count = ready_count
state.failed_page_count = failed_count
state.vector_count = vector_count
if success and not error:
state.last_completed_at = now
if success:
state.last_success_at = now
state.updated_at = now
await session.commit()
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
stmt = select(LlmWikiWikiSyncStateRow)
if mapping_ids:
stmt = stmt.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id.in_(mapping_ids))
stmt = stmt.order_by(LlmWikiWikiSyncStateRow.updated_at.desc())
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
vector_counts = (
select(
LlmWikiWikiVectorRow.page_id.label("page_id"),
func.count(LlmWikiWikiVectorRow.id).label("vector_count"),
)
.where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id)
.group_by(LlmWikiWikiVectorRow.page_id)
.subquery()
)
stmt = (
select(LlmWikiWikiPageRow, func.coalesce(vector_counts.c.vector_count, 0))
.outerjoin(vector_counts, vector_counts.c.page_id == LlmWikiWikiPageRow.id)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
)
.order_by(LlmWikiWikiPageRow.wiki_path, LlmWikiWikiPageRow.slug)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
result: list[dict[str, Any]] = []
for row, vector_count in rows:
page = _dict(row)
result.append(
{
"id": page["id"],
"slug": page["slug"],
"title": page["title"],
"page_type": page["page_type"],
"status": page["status"],
"index_status": page["index_status"],
"vector_count": int(vector_count or 0),
"indexed_at": page.get("indexed_at"),
"last_error": page.get("last_error"),
"last_seen_sync_id": page.get("last_seen_sync_id"),
"updated_at": page.get("updated_at"),
}
)
return result
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
if not mapping_ids:
return 0
now = _now()
async with self._sf() as session:
for mapping_id in mapping_ids:
state = await self._ensure_state(session, mapping_id)
state.state = "rebuild_required"
state.embedding_fingerprint = fingerprint
state.updated_at = now
await session.execute(update(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(mapping_ids), LlmWikiWikiPageRow.is_remote_deleted.is_(False)).values(index_status="stale", updated_at=now))
await session.commit()
return len(mapping_ids)
@@ -0,0 +1,23 @@
"""Per-user memory configuration override persistence."""
from __future__ import annotations
from deerflow.persistence.memory_config.base import UserMemoryConfigStore
from deerflow.persistence.memory_config.memory import MemoryUserMemoryConfigStore
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.memory_config.sql import SqlUserMemoryConfigRepository
def make_user_memory_config_store(session_factory) -> UserMemoryConfigStore:
if session_factory is None:
return MemoryUserMemoryConfigStore()
return SqlUserMemoryConfigRepository(session_factory)
__all__ = [
"UserMemoryConfigStore",
"UserMemoryConfigRow",
"MemoryUserMemoryConfigStore",
"SqlUserMemoryConfigRepository",
"make_user_memory_config_store",
]
@@ -0,0 +1,32 @@
"""Abstract persistence API for per-user memory config overrides."""
from __future__ import annotations
from abc import ABC, abstractmethod
class UserMemoryConfigStore(ABC):
@abstractmethod
async def get_recall_budget(self, user_id: str) -> str | None:
"""Return the user's recall_budget override, or None if not set."""
raise NotImplementedError
@abstractmethod
async def set_recall_budget(self, user_id: str, value: str) -> None:
"""Upsert the user's recall_budget override."""
raise NotImplementedError
@abstractmethod
async def delete(self, user_id: str) -> None:
"""Remove all per-user overrides for the given user."""
raise NotImplementedError
@abstractmethod
async def load_all(self) -> dict[str, str]:
"""Bulk-load all user recall_budget overrides.
Returns {user_id: recall_budget} for all rows where recall_budget IS NOT NULL.
Called once at startup to warm the in-memory cache.
"""
raise NotImplementedError
Loaded 100 of 3008 files, more files were not shown because too many files have changed in this diff. Show more