初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
@@ -0,0 +1,25 @@
|
||||
"""DeerFlow application persistence layer (SQLAlchemy 2.0 async ORM).
|
||||
|
||||
This module manages DeerFlow's own application data -- runs metadata,
|
||||
thread ownership, cron jobs, users. It is completely separate from
|
||||
LangGraph's checkpointer, which manages graph execution state.
|
||||
|
||||
Usage:
|
||||
from deerflow.persistence import init_engine, close_engine, get_session_factory
|
||||
"""
|
||||
|
||||
from deerflow.persistence.engine import (
|
||||
close_engine,
|
||||
get_engine,
|
||||
get_session_factory,
|
||||
init_engine,
|
||||
rebuild_workflow_schema_for_mysql,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"close_engine",
|
||||
"get_engine",
|
||||
"get_session_factory",
|
||||
"init_engine",
|
||||
"rebuild_workflow_schema_for_mysql",
|
||||
]
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
"""Persistence models for admin analytics snapshots."""
|
||||
|
||||
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
|
||||
|
||||
__all__ = ["AdminLeaderboardDailyStatRow"]
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
"""Pure scheduling helpers for the admin leaderboard daily snapshots.
|
||||
|
||||
Kept import-light on purpose (only stdlib ``datetime``) so the decision logic
|
||||
is unit-testable without standing up the FastAPI app, the DB engine, or the
|
||||
ORM. The background scheduler in ``app/gateway/app.py`` wires this to the real
|
||||
snapshot rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def should_finalize_previous_day(
|
||||
*,
|
||||
status: str | None,
|
||||
generated_at: datetime | None,
|
||||
now: datetime,
|
||||
finalize_hour: int,
|
||||
) -> bool:
|
||||
"""Decide whether *yesterday's* snapshot should be force-recomputed once.
|
||||
|
||||
The previous day's snapshot is normally built before midnight, so it can
|
||||
miss runs / tool calls whose rows only land in the database shortly before
|
||||
or after the day boundary. Once per day, on/after ``finalize_hour`` (local
|
||||
Beijing time), we recompute it a single time so the full day is captured.
|
||||
|
||||
``now`` and ``generated_at`` are expected to be timezone-aware Beijing
|
||||
datetimes (``generated_at`` may be ``None`` for legacy rows).
|
||||
|
||||
Returns ``True`` only when all hold:
|
||||
- it is already on/after ``finalize_hour`` today;
|
||||
- a snapshot row exists (``status`` is not ``None`` — a missing day is
|
||||
left to the normal prewarm/queue path);
|
||||
- that snapshot is not already ``queued``/``running`` (a finalize already
|
||||
in flight, or normal processing);
|
||||
- it was generated before today (i.e. not yet finalized for this day).
|
||||
"""
|
||||
if status is None:
|
||||
return False
|
||||
if now.hour < finalize_hour:
|
||||
return False
|
||||
if status in ("queued", "running"):
|
||||
return False
|
||||
if generated_at is not None and generated_at.date() >= now.date():
|
||||
return False
|
||||
return True
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
"""ORM models for admin analytics snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class AdminLeaderboardDailyStatRow(Base):
|
||||
"""One precomputed leaderboard snapshot for a single local calendar day."""
|
||||
|
||||
__tablename__ = "admin_leaderboard_daily_stats"
|
||||
__table_args__ = (
|
||||
Index("ix_admin_leaderboard_daily_stats_status", "status"),
|
||||
Index("ix_admin_leaderboard_daily_stats_updated_at", "updated_at"),
|
||||
)
|
||||
|
||||
stat_date: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||
settings_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
status: Mapped[str] = mapped_column(String(16), default="ready")
|
||||
payload_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
message: Mapped[str | None] = mapped_column(Text)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
generated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime())
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""Custom agent persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.agents.base import AgentStore
|
||||
from deerflow.persistence.agents.memory import MemoryAgentStore
|
||||
from deerflow.persistence.agents.model import (
|
||||
AgentExtrasRow,
|
||||
AgentFavoriteRow,
|
||||
AgentRow,
|
||||
AgentSkillRow,
|
||||
)
|
||||
from deerflow.persistence.agents.sql import AgentRepository
|
||||
|
||||
__all__ = [
|
||||
"AgentExtrasRow",
|
||||
"AgentFavoriteRow",
|
||||
"AgentRepository",
|
||||
"AgentRow",
|
||||
"AgentSkillRow",
|
||||
"AgentStore",
|
||||
"MemoryAgentStore",
|
||||
"make_agent_store",
|
||||
]
|
||||
|
||||
|
||||
def make_agent_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryAgentStore()
|
||||
return AgentRepository(session_factory)
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
"""Abstract persistence API for custom agents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class AgentStore(ABC):
|
||||
@abstractmethod
|
||||
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update a built-in agent (user_id IS NULL).
|
||||
|
||||
Caller is responsible for authorization — this method does NOT check
|
||||
any user identity and returns ``None`` if the target is not a built-in.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete(self, agent_id: str, user_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_builtin(self, agent_id: str) -> bool:
|
||||
"""Delete a built-in agent (user_id IS NULL).
|
||||
|
||||
Caller is responsible for authorization — this method does NOT check
|
||||
any user identity and returns ``False`` if the target is not a built-in.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
"""Return the number of agents owned by ``user_id`` (user_id IS NOT NULL)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
|
||||
"""Set or clear the homepage-selector featured rank for an agent.
|
||||
|
||||
Returns the updated row, or ``None`` if the agent does not exist.
|
||||
Caller is responsible for authorization (admin) and for verifying
|
||||
that ``published`` is true when ``order`` is not ``None``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
"""Return featured + still-visible-to-``user_id`` agents, ordered by ``featured_order`` asc."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_private(self) -> list[dict[str, Any]]:
|
||||
"""Return every agent flagged ``is_private=True``.
|
||||
|
||||
Caller is responsible for authorization — typically the caller has
|
||||
already verified the admin-configured private-square password.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
"""Return every agent regardless of ownership/visibility.
|
||||
|
||||
Caller is responsible for authorization — typically the caller has
|
||||
already verified ``system_role == "admin"``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update any agent row by id, regardless of ownership.
|
||||
|
||||
Caller is responsible for authorization (admin only). Returns ``None``
|
||||
if the agent does not exist.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_any(self, agent_id: str) -> bool:
|
||||
"""Delete any agent row by id, regardless of ownership.
|
||||
|
||||
Caller is responsible for authorization (admin only). Returns
|
||||
``False`` if the agent does not exist.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Return one page of agents plus the total matching count.
|
||||
|
||||
``scope``:
|
||||
- ``"mine"`` — owned by ``user_id`` OR favorited by ``user_id``.
|
||||
- ``"square"`` — built-in or published, and not private-square.
|
||||
- ``"private"``— flagged ``is_private=True`` (caller gates access).
|
||||
- ``None`` — the default listing visibility (``list_visible``).
|
||||
``admin=True`` bypasses the visibility/scope filter entirely (every
|
||||
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
|
||||
|
||||
``search`` matches the raw name OR its desensitized display form.
|
||||
``tag_ids`` keeps agents carrying ANY of the tags (OR). ``owner`` keeps
|
||||
only agents published by that user (``user_id`` equality) — built-in
|
||||
agents (no owner) never match. All filters, the COUNT, and LIMIT/OFFSET
|
||||
all execute in SQL.
|
||||
|
||||
Returns ``(rows, total)`` where ``rows`` is at most ``page_size`` items
|
||||
for the 1-based ``page``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Listing cache (skills / tool_groups / model) ---
|
||||
|
||||
@abstractmethod
|
||||
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
|
||||
"""Replace the cached skill list for ``agent_id``.
|
||||
|
||||
``None`` and ``[]`` both clear the cache to "no skills" (mirrors the
|
||||
tri-state semantics of ``AgentConfig.skills``).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
|
||||
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
|
||||
"""Batch-fetch cached listing extras for ``agent_ids``.
|
||||
|
||||
Returns ``{agent_id: {"skills": list[str] | None, "tool_groups":
|
||||
list[str] | None, "model": str | None}}``. An agent that was never
|
||||
mirrored is absent from the mapping (caller treats it as unknown).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_extras_for_agent(self, agent_id: str) -> None:
|
||||
"""Drop the cached skills + extras rows for ``agent_id`` (on delete)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Favorites ("收藏") ---
|
||||
|
||||
@abstractmethod
|
||||
async def add_favorite(self, user_id: str, agent_id: str) -> None:
|
||||
"""Bookmark ``agent_id`` for ``user_id``. Idempotent."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
|
||||
"""Remove a bookmark. Returns ``False`` when none existed."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_favorite_ids(self, user_id: str) -> set[str]:
|
||||
"""Return the set of agent ids ``user_id`` has favorited."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Per-user pins ("置顶") ---
|
||||
|
||||
@abstractmethod
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
"""Pin ``agent_id`` for ``user_id``. Idempotent."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
"""Remove a per-user pin. Returns ``False`` when none existed."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
"""Return the set of agent ids pinned by ``user_id``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
"""Drop every user's pin of ``agent_id`` (called on agent delete)."""
|
||||
raise NotImplementedError
|
||||
+325
@@ -0,0 +1,325 @@
|
||||
"""In-memory custom agent store for memory persistence mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.agent_name_desensitize import name_matches
|
||||
from deerflow.persistence.agents.base import AgentStore
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _copy(data: dict[str, Any]) -> dict[str, Any]:
|
||||
copied = dict(data)
|
||||
copied["builtin"] = copied.get("user_id") is None
|
||||
return copied
|
||||
|
||||
|
||||
class MemoryAgentStore(AgentStore):
|
||||
def __init__(self) -> None:
|
||||
self._agents: dict[str, dict[str, Any]] = {}
|
||||
self._favorites: set[tuple[str, str]] = set()
|
||||
self._pins: set[tuple[str, str]] = set()
|
||||
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
|
||||
self._skills: dict[str, list[str]] = {}
|
||||
self._extras: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
|
||||
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
|
||||
|
||||
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
|
||||
"""Single-record visibility (``get_visible``) — the union of every
|
||||
listing surface, so a list-visible agent always opens. Mirrors
|
||||
``AgentRepository._visible_filter``: baseline visibility OR
|
||||
private-square (password is the discovery gate, reads are not
|
||||
re-gated) OR favorited (the "我的" listing keeps favorites even after
|
||||
the owner unpublishes)."""
|
||||
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
|
||||
|
||||
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
|
||||
rows = [_copy(agent) for agent in self._agents.values() if self._is_visible(agent, user_id)]
|
||||
rows.sort(key=lambda agent: agent.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or not self._is_fetchable(agent, user_id):
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
return _copy(agent) if agent is not None else None
|
||||
|
||||
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or agent.get("user_id") != user_id:
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = _now()
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"name": data["name"],
|
||||
"description": data.get("description") or "",
|
||||
"user_id": data.get("user_id"),
|
||||
"published": bool(data.get("published", False)),
|
||||
"is_private": bool(data.get("is_private", False)),
|
||||
"square_id": str(data.get("square_id") or ""),
|
||||
"featured_order": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._agents[row["id"]] = row
|
||||
return _copy(row)
|
||||
|
||||
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = self._agents.get(data["id"])
|
||||
if row is None:
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"name": data["name"],
|
||||
"description": data.get("description") or "",
|
||||
"user_id": None,
|
||||
"published": bool(data.get("published", True)),
|
||||
"featured_order": None,
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
self._agents[row["id"]] = row
|
||||
elif row.get("user_id") is None:
|
||||
row["name"] = data["name"]
|
||||
row["description"] = data.get("description") or ""
|
||||
row["published"] = bool(data.get("published", True))
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete(self, agent_id: str, user_id: str) -> bool:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
async def delete_builtin(self, agent_id: str) -> bool:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
|
||||
|
||||
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
row["featured_order"] = order
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
candidates = [a for a in self._agents.values() if a.get("featured_order") is not None and self._is_visible(a, user_id)]
|
||||
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
|
||||
return [_copy(a) for a in candidates]
|
||||
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
|
||||
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
|
||||
rows.sort(key=lambda a: order_map.get(a["id"], 0))
|
||||
return [_copy(a) for a in rows]
|
||||
|
||||
async def list_private(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private"))]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values()]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete_any(self, agent_id: str) -> bool:
|
||||
if agent_id not in self._agents:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
# --- Paginated listing ---
|
||||
|
||||
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
|
||||
if admin:
|
||||
return True
|
||||
if scope == "square":
|
||||
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
|
||||
if scope == "mine":
|
||||
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
|
||||
favorited = (user_id, agent["id"]) in self._favorites
|
||||
return owned or favorited
|
||||
if scope == "private":
|
||||
return bool(agent.get("is_private"))
|
||||
if scope == "builtin":
|
||||
# 内置智能体 = user_id IS NULL(仅管理员;路由层 admin 闸门)。
|
||||
return agent.get("user_id") is None
|
||||
return self._is_visible(agent, user_id)
|
||||
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
|
||||
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
|
||||
owner_key = owner.strip() if owner and owner.strip() else None
|
||||
square_key = square_id.strip() if square_id and square_id.strip() else None
|
||||
|
||||
def _square_ok(a: dict[str, Any]) -> bool:
|
||||
if square_key is None:
|
||||
return True
|
||||
sid = str(a.get("square_id") or "")
|
||||
return sid == square_key or (square_is_default and sid == "")
|
||||
|
||||
matched = [
|
||||
a
|
||||
for a in self._agents.values()
|
||||
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
|
||||
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
|
||||
and (owner_key is None or a.get("user_id") == owner_key)
|
||||
and _square_ok(a)
|
||||
]
|
||||
if sort == "time_asc":
|
||||
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
|
||||
elif sort == "updated_desc":
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
elif sort == "updated_asc":
|
||||
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
|
||||
elif sort == "name_asc":
|
||||
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
|
||||
elif sort == "name_desc":
|
||||
matched.sort(
|
||||
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
else:
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
|
||||
total = len(matched)
|
||||
start = max(page - 1, 0) * page_size
|
||||
page_rows = matched[start : start + page_size]
|
||||
return [_copy(a) for a in page_rows], total
|
||||
|
||||
# --- Listing cache (skills / tool_groups / model) ---
|
||||
|
||||
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
|
||||
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
|
||||
|
||||
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
|
||||
self._extras[agent_id] = {"tool_groups": tool_groups, "model": model}
|
||||
|
||||
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for agent_id in agent_ids:
|
||||
has_extras = agent_id in self._extras
|
||||
has_skills = agent_id in self._skills
|
||||
if not has_extras and not has_skills:
|
||||
continue
|
||||
extras = self._extras.get(agent_id, {})
|
||||
out[agent_id] = {
|
||||
"skills": list(self._skills.get(agent_id, [])),
|
||||
"tool_groups": extras.get("tool_groups"),
|
||||
"model": extras.get("model"),
|
||||
}
|
||||
return out
|
||||
|
||||
async def delete_extras_for_agent(self, agent_id: str) -> None:
|
||||
self._skills.pop(agent_id, None)
|
||||
self._extras.pop(agent_id, None)
|
||||
|
||||
# --- Favorites ("收藏") ---
|
||||
|
||||
async def add_favorite(self, user_id: str, agent_id: str) -> None:
|
||||
self._favorites.add((user_id, agent_id))
|
||||
|
||||
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
|
||||
if (user_id, agent_id) in self._favorites:
|
||||
self._favorites.discard((user_id, agent_id))
|
||||
return True
|
||||
return False
|
||||
|
||||
async def list_favorite_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
|
||||
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
|
||||
|
||||
# --- Pins ("置顶") ---
|
||||
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
self._pins.add((user_id, agent_id))
|
||||
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
key = (user_id, agent_id)
|
||||
if key not in self._pins:
|
||||
return False
|
||||
self._pins.remove(key)
|
||||
return True
|
||||
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
"""ORM model for custom agents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class AgentRow(Base):
|
||||
__tablename__ = "agents"
|
||||
|
||||
# Stored as a plain string so the same id can be used as the filesystem
|
||||
# directory name under {base_dir}/agents/{id}/.
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="")
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# When True, the agent is moved into the password-gated "private square"
|
||||
# (私密广场) and is hidden from the regular public square / built-in list.
|
||||
# The flag is independent of ``published``: a private agent is still
|
||||
# visible to its owner in the "我的" tab, and visible to anyone in the
|
||||
# private-square listing once the password has been verified.
|
||||
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# Publish square (发布广场) this agent is filed under. A pure organizational
|
||||
# category with no access control — independent of ``published`` /
|
||||
# ``is_private``. Empty string = "not assigned" → the listing layer treats
|
||||
# it as the admin-configured default square. Each agent belongs to at most
|
||||
# one square (single column), so it can never be published to two squares.
|
||||
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", server_default="", index=True)
|
||||
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
|
||||
# featured rank (lower sorts first). Only ``published=True`` agents are
|
||||
# eligible; the router enforces that invariant on writes. BIGINT so the
|
||||
# frontend can seed initial ranks from ``Date.now()`` (ms-resolution
|
||||
# timestamps overflow a 4-byte INT on MySQL).
|
||||
featured_order: Mapped[int | None] = mapped_column(BigInteger, nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentFavoriteRow(Base):
|
||||
"""A user's bookmark of a (typically public-square) agent.
|
||||
|
||||
Lets a user collect agents from the 广场 (square) into their "我的" tab
|
||||
without owning them. One row per (user, agent); the unique constraint makes
|
||||
favoriting idempotent.
|
||||
"""
|
||||
|
||||
__tablename__ = "agent_favorites"
|
||||
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_favorites_user_agent"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# Where this favorite came from: ``"user"`` = the user bookmarked it
|
||||
# themselves; ``"position"`` = it was auto-granted by the user's 岗位
|
||||
# (position) via the subscription sync engine. Sync only ever adds/removes
|
||||
# ``"position"`` rows, so a user's own bookmarks are never clobbered.
|
||||
origin: Mapped[str] = mapped_column(String(16), nullable=False, default="user", server_default="user")
|
||||
# The position that granted this favorite (only set when ``origin='position'``).
|
||||
position_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentPinRow(Base):
|
||||
"""A per-user pin that keeps an agent near the top of that user's lists."""
|
||||
|
||||
__tablename__ = "agent_pins"
|
||||
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentSkillRow(Base):
|
||||
"""Denormalized cache of a custom agent's selected skills.
|
||||
|
||||
``config.yaml`` under ``.deer-flow/agents/{id}/`` stays the source of truth
|
||||
for the agent runtime (the lead-agent prompt reads it directly). This table
|
||||
is a read cache so the gallery list endpoint can render skill badges with a
|
||||
single batched ``IN`` query instead of stat+YAML-parsing every agent's
|
||||
``config.yaml`` on each request. Written through on create/update and
|
||||
backfilled from disk by ``_sync_legacy_agents``.
|
||||
|
||||
One row per (agent, skill); ``position`` preserves the configured order.
|
||||
"""
|
||||
|
||||
__tablename__ = "agent_skills"
|
||||
# Prefix-unique instead of UniqueConstraint: (64 + 120) * 4 = 736 bytes
|
||||
# fits MySQL's legacy 767-byte index limit (utf8mb4); a plain unique over
|
||||
# agent_id + skill_name(191) would be 1020 bytes and fail create_all with
|
||||
# error 1071 there. mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_agent_skills_agent_skill", "agent_id", "skill_name", unique=True, mysql_length={"skill_name": 120}),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
skill_name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
|
||||
position: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
|
||||
class AgentExtrasRow(Base):
|
||||
"""Denormalized cache of a custom agent's ``tool_groups`` / ``model``.
|
||||
|
||||
Companion to :class:`AgentSkillRow` — same rationale (avoid per-agent
|
||||
``config.yaml`` reads in the list path). One row per agent. The presence of
|
||||
a row also marks "this agent has been mirrored": the list path treats a
|
||||
missing row as "unknown / not yet backfilled" (``skills=None``) so it can
|
||||
be told apart from an explicitly empty skill list.
|
||||
|
||||
``skills_synced_at`` records when the cache was last written from disk so
|
||||
the backfill can skip agents whose ``config.yaml`` has not changed.
|
||||
"""
|
||||
|
||||
__tablename__ = "agent_extras"
|
||||
|
||||
agent_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# list[str] | None — the agent's tool_groups whitelist from config.yaml.
|
||||
tool_groups: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True, default=None)
|
||||
model: Mapped[str | None] = mapped_column(String(191), nullable=True, default=None)
|
||||
skills_synced_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -0,0 +1,573 @@
|
||||
"""SQLAlchemy-backed custom agent repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import and_, case, delete, func, literal, not_, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.config.agent_name_desensitize import source_terms_for_needle
|
||||
from deerflow.persistence.agents.base import AgentStore
|
||||
from deerflow.persistence.agents.model import (
|
||||
AgentExtrasRow,
|
||||
AgentFavoriteRow,
|
||||
AgentPinRow,
|
||||
AgentRow,
|
||||
AgentSkillRow,
|
||||
)
|
||||
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow
|
||||
|
||||
|
||||
def _row_to_dict(row: AgentRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["builtin"] = data.get("user_id") is None
|
||||
for key in ("created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
return data
|
||||
|
||||
|
||||
class AgentRepository(AgentStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _business_chain_agent_exists(agent_id_col):
|
||||
return (
|
||||
select(RoundtableChainRow.id)
|
||||
.where(
|
||||
or_(
|
||||
RoundtableChainRow.seats.like(literal('%"agent_id": "') + agent_id_col + literal('"%')),
|
||||
RoundtableChainRow.seats.like(literal('%"agent_id":"') + agent_id_col + literal('"%')),
|
||||
)
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _valid_favorite_filter(cls):
|
||||
return or_(
|
||||
AgentFavoriteRow.origin == "user",
|
||||
and_(
|
||||
AgentFavoriteRow.origin == "position",
|
||||
AgentRow.user_id.is_not(None),
|
||||
not_(cls._business_chain_agent_exists(AgentFavoriteRow.agent_id)),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _visible_filter(user_id: str):
|
||||
"""Visibility for single-record fetches (``get_visible``).
|
||||
|
||||
Lenient — must cover the union of every listing surface, so any agent
|
||||
a user can see in a list always opens (detail + chat) instead of
|
||||
404ing:
|
||||
|
||||
- built-in / owned / published — the baseline gallery visibility;
|
||||
- ``is_private`` — the private-square password is the *discovery*
|
||||
gate (listings); individual reads are not re-gated, so a private
|
||||
agent opened from the square (or a bookmark/link) stays openable.
|
||||
Note moving an agent into the private square forces
|
||||
``published=false``, so this clause cannot lean on ``published``;
|
||||
- favorited — the "我的" listing keeps favorited agents even after
|
||||
the owner unpublishes them, so reads keep honoring the favorite.
|
||||
"""
|
||||
valid_favorite_exists = (
|
||||
select(AgentFavoriteRow.id)
|
||||
.where(
|
||||
AgentFavoriteRow.user_id == user_id,
|
||||
AgentFavoriteRow.agent_id == AgentRow.id,
|
||||
AgentRepository._valid_favorite_filter(),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
return or_(
|
||||
AgentRow.user_id.is_(None),
|
||||
AgentRow.user_id == user_id,
|
||||
AgentRow.published.is_(True),
|
||||
AgentRow.is_private.is_(True),
|
||||
valid_favorite_exists,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _listing_filter(user_id: str):
|
||||
"""Visibility for *listings* — hides private-square agents from
|
||||
non-owners so they only surface inside the password-gated endpoint.
|
||||
Owners keep seeing their own private agents in '我的'.
|
||||
"""
|
||||
return or_(
|
||||
AgentRow.user_id == user_id,
|
||||
and_(
|
||||
or_(AgentRow.user_id.is_(None), AgentRow.published.is_(True)),
|
||||
AgentRow.is_private.is_(False),
|
||||
),
|
||||
)
|
||||
|
||||
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
|
||||
stmt = select(AgentRow).where(self._listing_filter(user_id)).order_by(AgentRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
stmt = select(AgentRow).where(AgentRow.id == agent_id, self._visible_filter(user_id))
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
|
||||
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
|
||||
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
stmt = select(AgentRow).where(AgentRow.id == agent_id, AgentRow.user_id == user_id)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
|
||||
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = AgentRow(
|
||||
id=data["id"],
|
||||
name=data["name"],
|
||||
description=data.get("description") or "",
|
||||
user_id=data.get("user_id"),
|
||||
published=bool(data.get("published", False)),
|
||||
is_private=bool(data.get("is_private", False)),
|
||||
square_id=str(data.get("square_id") or ""),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
|
||||
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, data["id"])
|
||||
if row is None:
|
||||
now = datetime.now(UTC)
|
||||
row = AgentRow(
|
||||
id=data["id"],
|
||||
name=data["name"],
|
||||
description=data.get("description") or "",
|
||||
user_id=None,
|
||||
published=bool(data.get("published", True)),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
elif row.user_id is None:
|
||||
row.name = data["name"]
|
||||
row.description = data.get("description") or ""
|
||||
row.published = bool(data.get("published", True))
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
|
||||
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
|
||||
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None or row.user_id is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
|
||||
async def delete(self, agent_id: str, user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def delete_builtin(self, agent_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None or row.user_id is not None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
stmt = select(func.count()).select_from(AgentRow).where(AgentRow.user_id == user_id)
|
||||
async with self._sf() as session:
|
||||
return await session.scalar(stmt) or 0
|
||||
|
||||
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
row.featured_order = order
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(AgentRow)
|
||||
.where(AgentRow.featured_order.is_not(None), self._listing_filter(user_id))
|
||||
.order_by(AgentRow.featured_order.asc(), AgentRow.created_at.asc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
if not agent_ids:
|
||||
return []
|
||||
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
|
||||
stmt = select(AgentRow).where(AgentRow.id.in_(agent_ids), self._listing_filter(user_id))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
rows = list(result.scalars())
|
||||
rows.sort(key=lambda r: order_map.get(r.id, len(order_map)))
|
||||
return [_row_to_dict(r) for r in rows]
|
||||
|
||||
async def list_private(self) -> list[dict[str, Any]]:
|
||||
stmt = select(AgentRow).where(AgentRow.is_private.is_(True)).order_by(AgentRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
stmt = select(AgentRow).order_by(AgentRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
|
||||
async def delete_any(self, agent_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
# --- Paginated listing ---
|
||||
|
||||
def _scope_filter(self, *, user_id: str, scope: str | None, admin: bool):
|
||||
"""Build the visibility predicate for a paginated listing."""
|
||||
if admin:
|
||||
return None # no visibility filter — every agent
|
||||
if scope == "square":
|
||||
return and_(
|
||||
or_(AgentRow.user_id.is_(None), AgentRow.published.is_(True)),
|
||||
AgentRow.is_private.is_(False),
|
||||
)
|
||||
if scope == "mine":
|
||||
# owned OR valid favorites. Built-in agents and business-chain
|
||||
# seats only enter "mine" when explicitly favorited by the user;
|
||||
# the original position-granted behavior stays for ordinary agents.
|
||||
valid_favorite_exists = (
|
||||
select(AgentFavoriteRow.id)
|
||||
.where(
|
||||
AgentFavoriteRow.user_id == user_id,
|
||||
AgentFavoriteRow.agent_id == AgentRow.id,
|
||||
self._valid_favorite_filter(),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
return or_(AgentRow.user_id == user_id, valid_favorite_exists)
|
||||
if scope == "private":
|
||||
return AgentRow.is_private.is_(True)
|
||||
if scope == "builtin":
|
||||
# 内置智能体 = user_id IS NULL。仅管理员可用此 scope(路由层已 admin 闸门)。
|
||||
return AgentRow.user_id.is_(None)
|
||||
return self._listing_filter(user_id)
|
||||
|
||||
@staticmethod
|
||||
def _search_clause(search: str):
|
||||
"""Match the raw name OR (via the desensitize mirror) its display form."""
|
||||
needle = search.strip().lower()
|
||||
clauses = [func.lower(AgentRow.name).like(f"%{needle}%")]
|
||||
for term in source_terms_for_needle(needle):
|
||||
clauses.append(AgentRow.name.like(f"%{term}%"))
|
||||
return or_(*clauses)
|
||||
|
||||
@staticmethod
|
||||
def _tag_clause(tag_ids: list[str]):
|
||||
subq = select(TagAssignmentRow.target_id).where(
|
||||
TagAssignmentRow.target_type == "agent",
|
||||
TagAssignmentRow.tag_id.in_(tag_ids),
|
||||
)
|
||||
return AgentRow.id.in_(subq)
|
||||
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
filters = []
|
||||
scope_filter = self._scope_filter(user_id=user_id, scope=scope, admin=admin)
|
||||
if scope_filter is not None:
|
||||
filters.append(scope_filter)
|
||||
if search and search.strip():
|
||||
filters.append(self._search_clause(search))
|
||||
if tag_ids:
|
||||
filters.append(self._tag_clause(tag_ids))
|
||||
if owner and owner.strip():
|
||||
# Filter to one publisher. Built-in agents (user_id IS NULL) never match.
|
||||
filters.append(AgentRow.user_id == owner.strip())
|
||||
if square_id and square_id.strip():
|
||||
sid = square_id.strip()
|
||||
if square_is_default:
|
||||
# Legacy / unassigned rows (square_id == "") belong to the
|
||||
# default square, so include them alongside explicit matches.
|
||||
filters.append(or_(AgentRow.square_id == sid, AgentRow.square_id == ""))
|
||||
else:
|
||||
filters.append(AgentRow.square_id == sid)
|
||||
|
||||
count_stmt = select(func.count()).select_from(AgentRow)
|
||||
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
|
||||
pin_rank = case((AgentRow.id.in_(pin_subq), 0), else_=1)
|
||||
|
||||
if sort == "time_asc":
|
||||
order_by = (AgentRow.created_at.asc(), AgentRow.id.asc())
|
||||
elif sort == "updated_desc":
|
||||
order_by = (AgentRow.updated_at.desc(), AgentRow.id.asc())
|
||||
elif sort == "updated_asc":
|
||||
order_by = (AgentRow.updated_at.asc(), AgentRow.id.asc())
|
||||
elif sort == "name_asc":
|
||||
order_by = (func.lower(AgentRow.name).asc(), AgentRow.id.asc())
|
||||
elif sort == "name_desc":
|
||||
order_by = (func.lower(AgentRow.name).desc(), AgentRow.id.asc())
|
||||
else:
|
||||
order_by = (AgentRow.created_at.desc(), AgentRow.id.asc())
|
||||
|
||||
page_stmt = (
|
||||
select(AgentRow)
|
||||
.order_by(pin_rank.asc(), *order_by)
|
||||
.limit(page_size)
|
||||
.offset(max(page - 1, 0) * page_size)
|
||||
)
|
||||
if filters:
|
||||
where = and_(*filters)
|
||||
count_stmt = count_stmt.where(where)
|
||||
page_stmt = page_stmt.where(where)
|
||||
|
||||
async with self._sf() as session:
|
||||
total = await session.scalar(count_stmt) or 0
|
||||
result = await session.execute(page_stmt)
|
||||
rows = [_row_to_dict(row) for row in result.scalars()]
|
||||
return rows, total
|
||||
|
||||
# --- Listing cache (skills / tool_groups / model) ---
|
||||
|
||||
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(AgentSkillRow).where(AgentSkillRow.agent_id == agent_id))
|
||||
for position, name in enumerate(skills or []):
|
||||
if not isinstance(name, str):
|
||||
continue
|
||||
session.add(
|
||||
AgentSkillRow(
|
||||
id=uuid4().hex,
|
||||
agent_id=agent_id,
|
||||
skill_name=name,
|
||||
position=position,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AgentExtrasRow, agent_id)
|
||||
now = datetime.now(UTC)
|
||||
if row is None:
|
||||
session.add(
|
||||
AgentExtrasRow(
|
||||
agent_id=agent_id,
|
||||
tool_groups=tool_groups,
|
||||
model=model,
|
||||
skills_synced_at=now,
|
||||
)
|
||||
)
|
||||
else:
|
||||
row.tool_groups = tool_groups
|
||||
row.model = model
|
||||
row.skills_synced_at = now
|
||||
await session.commit()
|
||||
|
||||
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
|
||||
if not agent_ids:
|
||||
return {}
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
async with self._sf() as session:
|
||||
extras = await session.execute(
|
||||
select(AgentExtrasRow).where(AgentExtrasRow.agent_id.in_(agent_ids))
|
||||
)
|
||||
for row in extras.scalars():
|
||||
out[row.agent_id] = {
|
||||
"skills": [], # default; filled below if skill rows exist
|
||||
"tool_groups": row.tool_groups,
|
||||
"model": row.model,
|
||||
}
|
||||
skill_rows = await session.execute(
|
||||
select(AgentSkillRow.agent_id, AgentSkillRow.skill_name)
|
||||
.where(AgentSkillRow.agent_id.in_(agent_ids))
|
||||
.order_by(AgentSkillRow.agent_id.asc(), AgentSkillRow.position.asc())
|
||||
)
|
||||
for agent_id, skill_name in skill_rows.all():
|
||||
entry = out.setdefault(
|
||||
agent_id, {"skills": [], "tool_groups": None, "model": None}
|
||||
)
|
||||
entry["skills"].append(skill_name)
|
||||
return out
|
||||
|
||||
async def delete_extras_for_agent(self, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(AgentSkillRow).where(AgentSkillRow.agent_id == agent_id))
|
||||
await session.execute(delete(AgentExtrasRow).where(AgentExtrasRow.agent_id == agent_id))
|
||||
await session.commit()
|
||||
|
||||
# --- Favorites ("收藏") ---
|
||||
|
||||
async def add_favorite(self, user_id: str, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
existing = (
|
||||
await session.execute(
|
||||
select(AgentFavoriteRow)
|
||||
.where(AgentFavoriteRow.user_id == user_id, AgentFavoriteRow.agent_id == agent_id)
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
if existing.origin != "user" or existing.position_id is not None:
|
||||
existing.origin = "user"
|
||||
existing.position_id = None
|
||||
await session.commit()
|
||||
return
|
||||
session.add(AgentFavoriteRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Lost a race against a concurrent favorite of the same agent —
|
||||
# the unique constraint already guarantees the desired state.
|
||||
await session.rollback()
|
||||
|
||||
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(AgentFavoriteRow).where(
|
||||
AgentFavoriteRow.user_id == user_id,
|
||||
AgentFavoriteRow.agent_id == agent_id,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return (result.rowcount or 0) > 0
|
||||
|
||||
async def list_favorite_ids(self, user_id: str) -> set[str]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(AgentFavoriteRow.agent_id)
|
||||
.join(AgentRow, AgentRow.id == AgentFavoriteRow.agent_id)
|
||||
.where(
|
||||
AgentFavoriteRow.user_id == user_id,
|
||||
self._valid_favorite_filter(),
|
||||
)
|
||||
)
|
||||
return {row[0] for row in result.all()}
|
||||
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
|
||||
await session.commit()
|
||||
|
||||
# --- Pins ("置顶") ---
|
||||
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
existing = (
|
||||
await session.execute(
|
||||
select(AgentPinRow.id)
|
||||
.where(AgentPinRow.user_id == user_id, AgentPinRow.agent_id == agent_id)
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return
|
||||
session.add(AgentPinRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(AgentPinRow).where(
|
||||
AgentPinRow.user_id == user_id,
|
||||
AgentPinRow.agent_id == agent_id,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return (result.rowcount or 0) > 0
|
||||
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
|
||||
)
|
||||
return {row[0] for row in result.all()}
|
||||
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(AgentPinRow).where(AgentPinRow.agent_id == agent_id))
|
||||
await session.commit()
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
from deerflow.persistence.ai_writing_sessions.sql import AIWritingSessionRepository
|
||||
|
||||
|
||||
def make_ai_writing_session_store(session_factory) -> AIWritingSessionRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return AIWritingSessionRepository(session_factory)
|
||||
|
||||
|
||||
# ── 进程级默认 repo 单例 ──────────────────────────────────────────────────────
|
||||
#
|
||||
# AI 写作 graph 注册成 LangGraph Server assistant 后,节点跑在 harness 进程里,
|
||||
# 拿不到 FastAPI 的 ``request.app.state``;harness → app 又是禁止的 import 方向。
|
||||
# 这里提供一个 app 启动时注册、harness 节点运行时读取的「跨边界单例」通道。
|
||||
#
|
||||
# 写入:app/gateway/deps.py 在创建完 session_factory 之后调
|
||||
# ``register_default_store(app.state.ai_writing_session_store)``
|
||||
# 读取:节点里 ``get_default_store()``,None 表示未配置 DB(跳过写入即可)。
|
||||
_default_store: AIWritingSessionRepository | None = None
|
||||
|
||||
|
||||
def register_default_store(store: AIWritingSessionRepository | None) -> None:
|
||||
"""登记进程级默认 AI 写作 session store。
|
||||
|
||||
幂等:可多次调用,覆盖前值。传 ``None`` 用于显式清理(主要给测试)。
|
||||
"""
|
||||
global _default_store
|
||||
_default_store = store
|
||||
|
||||
|
||||
def get_default_store() -> AIWritingSessionRepository | None:
|
||||
"""返回进程级默认 AI 写作 session store;未配置时为 ``None``。
|
||||
|
||||
节点调用方应在拿到 ``None`` 时跳过 DB 写入(写入是「锦上添花」非致命)。
|
||||
"""
|
||||
return _default_store
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AIWritingSessionRow",
|
||||
"AIWritingSessionRepository",
|
||||
"make_ai_writing_session_store",
|
||||
"register_default_store",
|
||||
"get_default_store",
|
||||
]
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
"""ORM model for AI writing sessions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class AIWritingSessionRow(Base):
|
||||
__tablename__ = "ai_writing_sessions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
user_intent: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="in_progress")
|
||||
draft_title: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
draft_markdown: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
completed_interventions: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON array
|
||||
review_result: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON object
|
||||
# 完整对话时间线(前端组装的 progressEvents + completedInterventions),用于历史回显。
|
||||
# 用 PortableLongText:MySQL 上映射为 LONGTEXT,避免 TEXT 64KB 上限导致保存 500。
|
||||
transcript: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # JSON object
|
||||
# 时间列必须用 BeijingDateTime(项目规范,与 threads_meta 等其它表对齐)—— MySQL
|
||||
# DATETIME 不存时区,raw ``DateTime(timezone=True)`` 会把 UTC wall clock 写入
|
||||
# 列,读出来比真实北京时间晚 8 小时。``BeijingDateTime`` 在 bind 时 UTC→Beijing、
|
||||
# 在 result 时 attach +08:00,前后端拿到的就是北京时间且 tz-aware。
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+258
@@ -0,0 +1,258 @@
|
||||
"""SQLAlchemy-backed AI writing session storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
class AIWritingSessionRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: AIWritingSessionRow) -> dict:
|
||||
d = row.to_dict()
|
||||
for k in ("created_at", "updated_at"):
|
||||
if isinstance(d.get(k), datetime):
|
||||
d[k] = d[k].isoformat()
|
||||
for k in ("completed_interventions", "review_result", "transcript"):
|
||||
val = d.get(k)
|
||||
if isinstance(val, str):
|
||||
try:
|
||||
d[k] = json.loads(val)
|
||||
except Exception:
|
||||
d[k] = None
|
||||
return d
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
id: str,
|
||||
title: str,
|
||||
user_intent: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.create")
|
||||
now = datetime.now(UTC)
|
||||
row = AIWritingSessionRow(
|
||||
id=id,
|
||||
user_id=resolved_user_id,
|
||||
title=title,
|
||||
user_intent=user_intent,
|
||||
status="in_progress",
|
||||
draft_title=None,
|
||||
draft_markdown=None,
|
||||
completed_interventions=None,
|
||||
review_result=None,
|
||||
transcript=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def upsert_session(
|
||||
self,
|
||||
*,
|
||||
id: str,
|
||||
title: str,
|
||||
user_intent: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
status: str = "in_progress",
|
||||
) -> dict:
|
||||
"""幂等:行不存在则创建,行存在则保留原数据(不覆盖 draft / review 等已有写入)。
|
||||
|
||||
AI 写作 graph 的 ``intent_parser_node`` 在每次进入时调一次(首启 + resume 时
|
||||
都会跑),所以必须幂等。resume 时若直接 ``create`` 会因主键冲突抛错;用
|
||||
``update`` 又找不到行而 no-op。这个方法把两条路合一:行不存在新建,行存在
|
||||
只刷新 ``updated_at`` 以便列表页排序前置。
|
||||
"""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.upsert_session")
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, id)
|
||||
if row is None:
|
||||
row = AIWritingSessionRow(
|
||||
id=id,
|
||||
user_id=resolved_user_id,
|
||||
title=title,
|
||||
user_intent=user_intent,
|
||||
status=status,
|
||||
draft_title=None,
|
||||
draft_markdown=None,
|
||||
completed_interventions=None,
|
||||
review_result=None,
|
||||
transcript=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
# 行已存在(resume 路径):仅刷新 updated_at,保留 user_id / title / draft / review 等。
|
||||
# 若 user_id 之前为空(migration 兼容场景),用解析出来的填上。
|
||||
if row.user_id is None and resolved_user_id is not None:
|
||||
row.user_id = resolved_user_id
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def update(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
title=_UNSET,
|
||||
status=_UNSET,
|
||||
draft_title=_UNSET,
|
||||
draft_markdown=_UNSET,
|
||||
completed_interventions=_UNSET,
|
||||
review_result=_UNSET,
|
||||
transcript=_UNSET,
|
||||
) -> dict | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.update")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, session_id)
|
||||
if row is None:
|
||||
return None
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return None
|
||||
if title is not _UNSET:
|
||||
row.title = title
|
||||
if status is not _UNSET:
|
||||
row.status = status
|
||||
if draft_title is not _UNSET:
|
||||
row.draft_title = draft_title
|
||||
if draft_markdown is not _UNSET:
|
||||
row.draft_markdown = draft_markdown
|
||||
if completed_interventions is not _UNSET:
|
||||
row.completed_interventions = (
|
||||
json.dumps(completed_interventions, ensure_ascii=False)
|
||||
if completed_interventions is not None
|
||||
else None
|
||||
)
|
||||
if review_result is not _UNSET:
|
||||
row.review_result = (
|
||||
json.dumps(review_result, ensure_ascii=False)
|
||||
if review_result is not None
|
||||
else None
|
||||
)
|
||||
if transcript is not _UNSET:
|
||||
row.transcript = (
|
||||
json.dumps(transcript, ensure_ascii=False)
|
||||
if transcript is not None
|
||||
else None
|
||||
)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.get")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, session_id)
|
||||
if row is None:
|
||||
return None
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def list(
|
||||
self,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
limit: int = 50,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.list")
|
||||
stmt = select(AIWritingSessionRow)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(AIWritingSessionRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(AIWritingSessionRow.updated_at.desc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="AIWritingSessionRepository.delete")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AIWritingSessionRow, session_id)
|
||||
if row is None:
|
||||
return False
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_by_status(self, status: str, *, limit: int = 500) -> list[dict]:
|
||||
"""列出指定 status 的会话(仅 id + user_id,用于后台驱动重启续跑 reconcile)。
|
||||
|
||||
不按 user_id 过滤——进程级 reconcile 要把所有用户「挂起中」的会话都拉起。
|
||||
"""
|
||||
stmt = (
|
||||
select(AIWritingSessionRow.id, AIWritingSessionRow.user_id)
|
||||
.where(AIWritingSessionRow.status == status)
|
||||
.order_by(AIWritingSessionRow.updated_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [{"id": row[0], "user_id": row[1]} for row in result.all()]
|
||||
|
||||
async def list_older_than(
|
||||
self,
|
||||
cutoff: datetime,
|
||||
*,
|
||||
only_finished: bool = False,
|
||||
limit: int = 500,
|
||||
) -> list[str]:
|
||||
"""列出 updated_at 早于 cutoff 的会话 id(最多 limit 个)。
|
||||
|
||||
清理任务先 list 出 id 列表,再用这些 id 联动删 checkpointer
|
||||
里对应的 thread,最后再删 MySQL 行 —— 保证两边一致。
|
||||
|
||||
Args:
|
||||
cutoff: 截止时间,updated_at < cutoff 的会话被认为「过期」
|
||||
only_finished: True 时只清 status in (done/error),False 不看状态
|
||||
limit: 单批最多返回多少条,用于分批处理避免大事务
|
||||
"""
|
||||
stmt = select(AIWritingSessionRow.id).where(AIWritingSessionRow.updated_at < cutoff)
|
||||
if only_finished:
|
||||
stmt = stmt.where(AIWritingSessionRow.status.in_(("done", "error")))
|
||||
stmt = stmt.order_by(AIWritingSessionRow.updated_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [row[0] for row in result.all()]
|
||||
|
||||
async def delete_by_ids(self, session_ids: list[str]) -> int:
|
||||
"""按 id 列表批量删除。返回实际删除的行数。"""
|
||||
if not session_ids:
|
||||
return 0
|
||||
stmt = delete(AIWritingSessionRow).where(AIWritingSessionRow.id.in_(session_ids))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
from deerflow.persistence.article_types.model import ArticleTypeRow
|
||||
from deerflow.persistence.article_types.sql import ArticleTypeRepository
|
||||
|
||||
|
||||
def make_article_type_store(session_factory) -> ArticleTypeRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return ArticleTypeRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = ["ArticleTypeRow", "ArticleTypeRepository", "make_article_type_store"]
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
"""ORM model for AI writing article types."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class ArticleTypeRow(Base):
|
||||
__tablename__ = "ai_writing_article_types"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
label: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
prompt_hint: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
default_word_count: Mapped[int] = mapped_column(Integer, nullable=False, default=800)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
"""SQLAlchemy-backed article type storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.article_types.model import ArticleTypeRow
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
class ArticleTypeRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: ArticleTypeRow) -> dict:
|
||||
d = row.to_dict()
|
||||
for k in ("created_at", "updated_at"):
|
||||
if isinstance(d.get(k), datetime):
|
||||
d[k] = d[k].isoformat()
|
||||
return d
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
key: str,
|
||||
label: str,
|
||||
description: str | None = None,
|
||||
prompt_hint: str | None = None,
|
||||
default_word_count: int = 800,
|
||||
sort_order: int = 0,
|
||||
) -> dict:
|
||||
now = datetime.now(UTC)
|
||||
row = ArticleTypeRow(
|
||||
id=str(uuid.uuid4()),
|
||||
key=key,
|
||||
label=label,
|
||||
description=description,
|
||||
prompt_hint=prompt_hint,
|
||||
default_word_count=default_word_count,
|
||||
sort_order=sort_order,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def update(
|
||||
self,
|
||||
article_type_id: str,
|
||||
*,
|
||||
key=_UNSET,
|
||||
label=_UNSET,
|
||||
description=_UNSET,
|
||||
prompt_hint=_UNSET,
|
||||
default_word_count=_UNSET,
|
||||
sort_order=_UNSET,
|
||||
) -> dict | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArticleTypeRow, article_type_id)
|
||||
if row is None:
|
||||
return None
|
||||
if key is not _UNSET:
|
||||
row.key = key
|
||||
if label is not _UNSET:
|
||||
row.label = label
|
||||
if description is not _UNSET:
|
||||
row.description = description
|
||||
if prompt_hint is not _UNSET:
|
||||
row.prompt_hint = prompt_hint
|
||||
if default_word_count is not _UNSET:
|
||||
row.default_word_count = default_word_count
|
||||
if sort_order is not _UNSET:
|
||||
row.sort_order = sort_order
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(self, article_type_id: str) -> dict | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArticleTypeRow, article_type_id)
|
||||
if row is None:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get_by_key(self, key: str) -> dict | None:
|
||||
async with self._sf() as session:
|
||||
stmt = select(ArticleTypeRow).where(ArticleTypeRow.key == key)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def list(self) -> list[dict]:
|
||||
stmt = select(ArticleTypeRow).order_by(ArticleTypeRow.sort_order.asc(), ArticleTypeRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def delete(self, article_type_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArticleTypeRow, article_type_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_keys(self) -> list[str]:
|
||||
stmt = select(ArticleTypeRow.key).order_by(ArticleTypeRow.sort_order.asc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return list(result.scalars())
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
|
||||
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
|
||||
|
||||
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
"""ORM model for DeerFlow-native assistant knowledge bases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Float, Index, Integer, LargeBinary, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class AssistantKnowledgeBaseRow(Base):
|
||||
__tablename__ = "assistant_knowledge_base"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
|
||||
description: Mapped[str | None] = mapped_column(String(2048), nullable=True)
|
||||
# ``global`` is the full-library aggregation base. ``custom`` bases are
|
||||
# user-created assistant knowledge bases fed from WeKnora export packages.
|
||||
# Nullable keeps startup column reconciliation safe for existing databases.
|
||||
base_kind: Mapped[str | None] = mapped_column(String(24), nullable=True, index=True)
|
||||
status: Mapped[str] = mapped_column(String(64), default="active")
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeSourceRow(Base):
|
||||
__tablename__ = "assistant_knowledge_sources"
|
||||
__table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
source_key: Mapped[str] = mapped_column(String(255))
|
||||
knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
source_name: Mapped[str] = mapped_column(String(255), default="")
|
||||
last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeImportJobRow(Base):
|
||||
__tablename__ = "assistant_knowledge_import_jobs"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
|
||||
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
|
||||
phase: Mapped[str] = mapped_column(String(32), default="exporting")
|
||||
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
|
||||
class AssistantKnowledgeWikiPageRow(Base):
|
||||
__tablename__ = "assistant_knowledge_wiki_pages"
|
||||
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
slug: Mapped[str] = mapped_column(String(512), index=True)
|
||||
title: Mapped[str] = mapped_column(String(512), index=True)
|
||||
page_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
|
||||
summary: Mapped[str] = mapped_column(String(2048), default="")
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeWikiRevisionRow(Base):
|
||||
__tablename__ = "assistant_knowledge_wiki_revisions"
|
||||
__table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
page_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
revision_no: Mapped[int] = mapped_column(Integer)
|
||||
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
|
||||
summary: Mapped[str] = mapped_column(String(2048), default="")
|
||||
source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
change_type: Mapped[str] = mapped_column(String(32), default="import")
|
||||
status: Mapped[str] = mapped_column(String(24), default="published")
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
|
||||
class AssistantKnowledgeVectorRow(Base):
|
||||
__tablename__ = "assistant_knowledge_vectors"
|
||||
__table_args__ = (
|
||||
Index("ix_assistant_knowledge_vector_base_fingerprint", "base_id", "embedding_fingerprint"),
|
||||
Index("ix_assistant_knowledge_vector_page", "page_id"),
|
||||
Index("ix_assistant_knowledge_vector_source", "source_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
page_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_vector_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
section_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
embedding_model: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeEntityRow(Base):
|
||||
__tablename__ = "assistant_knowledge_entities"
|
||||
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
entity_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
normalized_name: Mapped[str] = mapped_column(String(512))
|
||||
display_name: Mapped[str] = mapped_column(String(512), index=True)
|
||||
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
|
||||
|
||||
class AssistantKnowledgeEntityContributionRow(Base):
|
||||
__tablename__ = "assistant_knowledge_entity_contributions"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
|
||||
|
||||
class AssistantKnowledgeRelationRow(Base):
|
||||
__tablename__ = "assistant_knowledge_relations"
|
||||
__table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
predicate: Mapped[str] = mapped_column(String(128), index=True)
|
||||
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
description: Mapped[str] = mapped_column(String(2048), default="")
|
||||
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
|
||||
|
||||
class AssistantKnowledgeRelationContributionRow(Base):
|
||||
__tablename__ = "assistant_knowledge_relation_contributions"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
relation_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
+2360
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,40 @@
|
||||
"""SQLAlchemy declarative base with automatic to_dict support.
|
||||
|
||||
All DeerFlow ORM models inherit from this Base. It provides a generic
|
||||
to_dict() method via SQLAlchemy's inspect() so individual models don't
|
||||
need to write their own serialization logic.
|
||||
|
||||
LangGraph's checkpointer tables are NOT managed by this Base.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
"""Base class for all DeerFlow ORM models.
|
||||
|
||||
Provides:
|
||||
- Automatic to_dict() via SQLAlchemy column inspection.
|
||||
- Standard __repr__() showing all column values.
|
||||
"""
|
||||
|
||||
def to_dict(self, *, exclude: set[str] | None = None) -> dict:
|
||||
"""Convert ORM instance to plain dict.
|
||||
|
||||
Uses SQLAlchemy's inspect() to iterate mapped column attributes.
|
||||
|
||||
Args:
|
||||
exclude: Optional set of column keys to omit.
|
||||
|
||||
Returns:
|
||||
Dict of {column_key: value} for all mapped columns.
|
||||
"""
|
||||
exclude = exclude or set()
|
||||
return {c.key: getattr(self, c.key) for c in sa_inspect(type(self)).mapper.column_attrs if c.key not in exclude}
|
||||
|
||||
def __repr__(self) -> str:
|
||||
cols = ", ".join(f"{c.key}={getattr(self, c.key)!r}" for c in sa_inspect(type(self)).mapper.column_attrs)
|
||||
return f"{type(self).__name__}({cols})"
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
"""Global business → agent / chain mapping persistence (业务链条管理)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.business_mapping.model import BusinessMappingRow
|
||||
from deerflow.persistence.business_mapping.sql import BusinessMappingRepository
|
||||
|
||||
__all__ = [
|
||||
"BusinessMappingRepository",
|
||||
"BusinessMappingRow",
|
||||
"make_business_mapping_store",
|
||||
]
|
||||
|
||||
|
||||
def make_business_mapping_store(session_factory):
|
||||
if session_factory is None:
|
||||
return None
|
||||
return BusinessMappingRepository(session_factory)
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
"""ORM model for the global business → agent / chain mapping (业务链条管理).
|
||||
|
||||
A "business mapping" row binds a fixed business code (3Q / 6BF / 6-1…6-4 / 7BF /
|
||||
8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
|
||||
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
|
||||
account can resolve it). Used to auto-pick the business chain when a task page
|
||||
launches the multi-agent roundtable.
|
||||
|
||||
Unlike ``roundtable_chains`` (per-user), this mapping is **global / shared** —
|
||||
there is no ``user_id`` column; all accounts see and use the same configuration.
|
||||
The fixed business rows are seeded on startup; users only fill in the
|
||||
agent / chain columns from the management page.
|
||||
|
||||
``agent_name`` / ``chain_title`` are denormalized display snapshots so the table
|
||||
renders without cross-referencing and survives a later agent/chain rename or
|
||||
delete (the picked id still drives the auto-flow).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class BusinessMappingRow(Base):
|
||||
__tablename__ = "business_mapping"
|
||||
|
||||
# Business code is the natural primary key (e.g. "3Q", "6BF"). Global, no user scope.
|
||||
business_code: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
# Display label shown in the 业务 column (e.g. "任务FX").
|
||||
label: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
# Picked 智能体 (暂存不用,自动流程当前不消费此字段). NULL = 未配置.
|
||||
agent_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(191), nullable=True)
|
||||
# Picked 业务链条 → roundtable_chains.id (须为 public). NULL = 未配置.
|
||||
chain_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
chain_title: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
# Fixed display order for the management table.
|
||||
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
"""SQLAlchemy-backed global business → agent / chain mapping store.
|
||||
|
||||
Global / shared (no ``user_id`` scoping): every account reads and writes the same
|
||||
rows. Fixed business rows are seeded once on startup (``ensure_seed_defaults``)
|
||||
with empty agent/chain; users fill those in from the management page via
|
||||
``update_mapping``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.business_mapping.model import BusinessMappingRow
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
# Fixed business rows (业务码 → 展示名). Seeded on first boot; agent/chain left empty.
|
||||
# 8BF 页面尚未建好,先占位保留行。顺序即管理表的展示顺序。
|
||||
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
|
||||
# 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
|
||||
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
{"business_code": "3Q", "label": "3QFX"},
|
||||
{"business_code": "6BF", "label": "任务FX"},
|
||||
{"business_code": "7BF", "label": "XDFX"},
|
||||
{"business_code": "8BF", "label": "8BF"},
|
||||
{"business_code": "AFX", "label": "AGENTFX"},
|
||||
# 独立配置位,与上方业务码无关;固定排在表尾。
|
||||
{"business_code": "6-1", "label": "6-1"},
|
||||
{"business_code": "6-2", "label": "6-2"},
|
||||
{"business_code": "6-3", "label": "6-3"},
|
||||
{"business_code": "6-4", "label": "6-4"},
|
||||
{"business_code": "7-1", "label": "7-1"},
|
||||
{"business_code": "7-2", "label": "7-2"},
|
||||
{"business_code": "7-3", "label": "7-3"},
|
||||
{"business_code": "7-4", "label": "7-4"},
|
||||
]
|
||||
|
||||
|
||||
class BusinessMappingRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _to_dict(row: BusinessMappingRow) -> dict[str, Any]:
|
||||
return {
|
||||
"business_code": row.business_code,
|
||||
"label": row.label,
|
||||
"agent_id": row.agent_id,
|
||||
"agent_name": row.agent_name,
|
||||
"chain_id": row.chain_id,
|
||||
"chain_title": row.chain_title,
|
||||
"sort_order": row.sort_order,
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
}
|
||||
|
||||
async def list_mappings(self) -> list[dict[str, Any]]:
|
||||
stmt = select(BusinessMappingRow).order_by(
|
||||
BusinessMappingRow.sort_order.asc(), BusinessMappingRow.business_code.asc()
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_mapping(self, business_code: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(BusinessMappingRow, business_code)
|
||||
return self._to_dict(row) if row is not None else None
|
||||
|
||||
async def update_mapping(
|
||||
self,
|
||||
business_code: str,
|
||||
*,
|
||||
agent_id=_UNSET,
|
||||
agent_name=_UNSET,
|
||||
chain_id=_UNSET,
|
||||
chain_title=_UNSET,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Update an existing (seeded) business row. Returns None if the code is unknown.
|
||||
|
||||
Only forwarded fields are touched; explicit ``None`` clears that field.
|
||||
"""
|
||||
async with self._sf() as session:
|
||||
row = await session.get(BusinessMappingRow, business_code)
|
||||
if row is None:
|
||||
return None
|
||||
if agent_id is not _UNSET:
|
||||
row.agent_id = agent_id or None
|
||||
if agent_name is not _UNSET:
|
||||
row.agent_name = agent_name or None
|
||||
if chain_id is not _UNSET:
|
||||
row.chain_id = chain_id or None
|
||||
if chain_title is not _UNSET:
|
||||
row.chain_title = chain_title or None
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
|
||||
async def ensure_seed_defaults(self) -> int:
|
||||
"""Idempotently insert the fixed business rows if missing. Returns inserted count.
|
||||
|
||||
Never overwrites an existing row's agent/chain (respects user config).
|
||||
Re-syncs ``sort_order`` (and empty ``label``) so newly inserted codes keep
|
||||
the intended table order next to older seeded rows.
|
||||
"""
|
||||
inserted = 0
|
||||
dirty = False
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
for idx, seed in enumerate(DEFAULT_BUSINESS_ROWS):
|
||||
existing = await session.get(BusinessMappingRow, seed["business_code"])
|
||||
if existing is not None:
|
||||
if existing.sort_order != idx:
|
||||
existing.sort_order = idx
|
||||
dirty = True
|
||||
seed_label = str(seed.get("label") or "")
|
||||
if seed_label and not (existing.label or "").strip():
|
||||
existing.label = seed_label
|
||||
dirty = True
|
||||
continue
|
||||
session.add(
|
||||
BusinessMappingRow(
|
||||
business_code=seed["business_code"],
|
||||
label=seed.get("label", ""),
|
||||
agent_id=None,
|
||||
agent_name=None,
|
||||
chain_id=None,
|
||||
chain_title=None,
|
||||
sort_order=idx,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
inserted += 1
|
||||
dirty = True
|
||||
if dirty:
|
||||
await session.commit()
|
||||
return inserted
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""Compact/sidebar-summary menu override persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.compact_menu_overrides.base import CompactMenuOverrideStore
|
||||
from deerflow.persistence.compact_menu_overrides.keys import (
|
||||
ALLOWED_COMPACT_MENU_TYPES,
|
||||
DEFAULT_COMPACT_MENU_TYPE,
|
||||
normalize_compact_menu_type,
|
||||
)
|
||||
from deerflow.persistence.compact_menu_overrides.memory import MemoryCompactMenuOverrideStore
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
|
||||
from deerflow.persistence.compact_menu_overrides.sql import CompactMenuOverrideRepository
|
||||
|
||||
__all__ = [
|
||||
"ALLOWED_COMPACT_MENU_TYPES",
|
||||
"CompactMenuOverrideRepository",
|
||||
"CompactMenuOverrideRow",
|
||||
"CompactMenuOverrideStore",
|
||||
"DEFAULT_COMPACT_MENU_TYPE",
|
||||
"MemoryCompactMenuOverrideStore",
|
||||
"make_compact_menu_override_store",
|
||||
"normalize_compact_menu_type",
|
||||
]
|
||||
|
||||
|
||||
def make_compact_menu_override_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryCompactMenuOverrideStore()
|
||||
return CompactMenuOverrideRepository(session_factory)
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
"""Abstract compact-sidebar menu override persistence API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class CompactMenuOverrideStore(ABC):
|
||||
@abstractmethod
|
||||
async def list_overrides(self, menu_type: str | None = None) -> list[dict[str, Any]]:
|
||||
"""Return compact-menu override rows for one type (default A)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def replace_all(
|
||||
self,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
menu_type: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Atomically replace the compact-menu override set for one type."""
|
||||
raise NotImplementedError
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
"""Compact-menu type helpers.
|
||||
|
||||
Two website deployments can keep independent compact-sidebar layouts.
|
||||
Type ``A`` is the historical default (existing rows stay as-is). Type ``B``
|
||||
is stored under a prefixed primary key so it never collides with type A on
|
||||
databases that still use ``node_id`` as the sole PK.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
DEFAULT_COMPACT_MENU_TYPE = "A"
|
||||
ALLOWED_COMPACT_MENU_TYPES = ("A", "B")
|
||||
_STORAGE_SEP = "::"
|
||||
_TYPE_ALIASES = {
|
||||
"A": "A",
|
||||
"1": "A",
|
||||
"DEFAULT": "A",
|
||||
"B": "B",
|
||||
"2": "B",
|
||||
"ALT": "B",
|
||||
}
|
||||
|
||||
|
||||
def normalize_compact_menu_type(value: str | None) -> str:
|
||||
raw = (value or "").strip().upper()
|
||||
return _TYPE_ALIASES.get(raw, DEFAULT_COMPACT_MENU_TYPE)
|
||||
|
||||
|
||||
def storage_node_id(menu_type: str, node_id: str) -> str:
|
||||
"""PK value that stays unique across types on a node_id-only primary key."""
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
nid = str(node_id)
|
||||
if t == DEFAULT_COMPACT_MENU_TYPE:
|
||||
prefix = f"{t}{_STORAGE_SEP}"
|
||||
return nid[len(prefix) :] if nid.startswith(prefix) else nid
|
||||
prefix = f"{t}{_STORAGE_SEP}"
|
||||
return nid if nid.startswith(prefix) else f"{prefix}{nid}"
|
||||
|
||||
|
||||
def public_node_id(stored: str, menu_type: str | None = None) -> str:
|
||||
"""Strip the type prefix so API callers always see the real menu node id."""
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
nid = str(stored)
|
||||
prefix = f"{t}{_STORAGE_SEP}"
|
||||
if nid.startswith(prefix):
|
||||
return nid[len(prefix) :]
|
||||
return nid
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
"""In-memory compact-menu override store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.compact_menu_overrides.base import CompactMenuOverrideStore
|
||||
from deerflow.persistence.compact_menu_overrides.keys import (
|
||||
normalize_compact_menu_type,
|
||||
public_node_id,
|
||||
storage_node_id,
|
||||
)
|
||||
|
||||
|
||||
def _normalize(data: dict[str, Any], menu_type: str, updated_by: str | None) -> dict[str, Any]:
|
||||
placement = str(data.get("placement") or "sidebar")
|
||||
node_id = public_node_id(str(data["node_id"]), menu_type)
|
||||
return {
|
||||
"node_id": storage_node_id(menu_type, node_id),
|
||||
"menu_type": menu_type,
|
||||
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"label": (data.get("label") or None),
|
||||
"updated_by": updated_by,
|
||||
"updated_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def _public(row: dict[str, Any]) -> dict[str, Any]:
|
||||
menu_type = normalize_compact_menu_type(row.get("menu_type"))
|
||||
out = dict(row)
|
||||
out["menu_type"] = menu_type
|
||||
out["node_id"] = public_node_id(str(row.get("node_id") or ""), menu_type)
|
||||
return out
|
||||
|
||||
|
||||
class MemoryCompactMenuOverrideStore(CompactMenuOverrideStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def list_overrides(self, menu_type: str | None = None) -> list[dict[str, Any]]:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
rows = [r for r in self._rows.values() if normalize_compact_menu_type(r.get("menu_type")) == t]
|
||||
return [_public(r) for r in sorted(rows, key=lambda r: r.get("sort_order", 0))]
|
||||
|
||||
async def replace_all(
|
||||
self,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
menu_type: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
self._rows = {
|
||||
k: v for k, v in self._rows.items() if normalize_compact_menu_type(v.get("menu_type")) != t
|
||||
}
|
||||
for r in rows:
|
||||
node_id = r.get("node_id")
|
||||
if not node_id:
|
||||
continue
|
||||
stored = _normalize(r, t, updated_by)
|
||||
self._rows[str(stored["node_id"])] = stored
|
||||
return await self.list_overrides(t)
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""ORM model for compact/sidebar-summary menu overrides.
|
||||
|
||||
This table only controls the compact left menu presentation. The full sidebar
|
||||
menu layout continues to use ``menu_overrides`` and is unaffected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class CompactMenuOverrideRow(Base):
|
||||
__tablename__ = "compact_menu_overrides"
|
||||
__table_args__ = (Index("ix_compact_menu_overrides_menu_type", "menu_type"),)
|
||||
|
||||
node_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
menu_type: Mapped[str] = mapped_column(String(32), default="A", server_default="A")
|
||||
placement: Mapped[str] = mapped_column(String(32), default="sidebar")
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0)
|
||||
label: Mapped[str | None] = mapped_column(String(255))
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
"""SQLAlchemy-backed compact-menu override store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.compact_menu_overrides.base import CompactMenuOverrideStore
|
||||
from deerflow.persistence.compact_menu_overrides.keys import (
|
||||
DEFAULT_COMPACT_MENU_TYPE,
|
||||
normalize_compact_menu_type,
|
||||
public_node_id,
|
||||
storage_node_id,
|
||||
)
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
|
||||
|
||||
|
||||
def _to_dict(row: CompactMenuOverrideRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
menu_type = normalize_compact_menu_type(data.get("menu_type"))
|
||||
data["menu_type"] = menu_type
|
||||
data["node_id"] = public_node_id(str(data.get("node_id") or ""), menu_type)
|
||||
value = data.get("updated_at")
|
||||
if isinstance(value, datetime):
|
||||
data["updated_at"] = value.isoformat()
|
||||
return data
|
||||
|
||||
|
||||
def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> CompactMenuOverrideRow:
|
||||
placement = str(data.get("placement") or "sidebar")
|
||||
node_id = str(data["node_id"])
|
||||
return CompactMenuOverrideRow(
|
||||
node_id=storage_node_id(menu_type, node_id),
|
||||
menu_type=menu_type,
|
||||
placement=placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
sort_order=int(data.get("sort_order", 0)),
|
||||
label=(data.get("label") or None),
|
||||
updated_by=updated_by,
|
||||
)
|
||||
|
||||
|
||||
class CompactMenuOverrideRepository(CompactMenuOverrideStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_overrides(self, menu_type: str | None = None) -> list[dict[str, Any]]:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
stmt = select(CompactMenuOverrideRow).order_by(CompactMenuOverrideRow.sort_order.asc())
|
||||
stmt = stmt.where(_menu_type_filter(t))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def replace_all(
|
||||
self,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
menu_type: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for r in rows:
|
||||
node_id = r.get("node_id")
|
||||
if node_id:
|
||||
by_id[public_node_id(str(node_id), t)] = r
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(CompactMenuOverrideRow).where(_menu_type_filter(t)))
|
||||
new_rows = [_row_from(r, t, updated_by) for r in by_id.values()]
|
||||
session.add_all(new_rows)
|
||||
await session.commit()
|
||||
for row in new_rows:
|
||||
await session.refresh(row)
|
||||
return [_to_dict(row) for row in new_rows]
|
||||
|
||||
|
||||
def _menu_type_filter(menu_type: str):
|
||||
if menu_type == DEFAULT_COMPACT_MENU_TYPE:
|
||||
return or_(
|
||||
CompactMenuOverrideRow.menu_type == menu_type,
|
||||
CompactMenuOverrideRow.menu_type.is_(None),
|
||||
CompactMenuOverrideRow.menu_type == "",
|
||||
)
|
||||
return CompactMenuOverrideRow.menu_type == menu_type
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
"""Concurrency (system-pressure) sampling persistence."""
|
||||
|
||||
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
|
||||
from deerflow.persistence.concurrency.store import (
|
||||
ConcurrencyBucket,
|
||||
ConcurrencySampleStore,
|
||||
make_concurrency_sample_store,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ConcurrencyBucket",
|
||||
"ConcurrencySampleRow",
|
||||
"ConcurrencySampleStore",
|
||||
"make_concurrency_sample_store",
|
||||
]
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
"""ORM model for periodic concurrency (system-pressure) samples.
|
||||
|
||||
A background sampler writes one row every few seconds holding the number of
|
||||
in-flight model runs at that instant. The admin monitor aggregates these into
|
||||
per-minute / per-hour buckets to draw the 并发量 pressure line chart, even for
|
||||
time windows before the admin opened the page.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ConcurrencySampleRow(Base):
|
||||
"""One sampled snapshot of concurrent in-flight model runs."""
|
||||
|
||||
__tablename__ = "concurrency_samples"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
# Wall-clock time the sample was taken (Beijing time on disk, tz-aware in Python).
|
||||
sampled_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), index=True, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
# Number of runs whose status was pending/running at ``sampled_at``.
|
||||
active_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
"""Async store for concurrency samples + minute/hour aggregation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# How far back the aggregation will fetch raw rows before bucketing. A safety
|
||||
# cap so a degenerate range never tries to load the whole table into memory.
|
||||
_MAX_ROWS = 60_000
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConcurrencyBucket:
|
||||
"""One minute/hour bucket of the concurrency series."""
|
||||
|
||||
bucket: str # ISO start of the bucket
|
||||
peak: int # max concurrent runs observed in the bucket
|
||||
avg: float # mean concurrent runs in the bucket
|
||||
samples: int # how many raw samples fell into the bucket
|
||||
|
||||
|
||||
def _bucket_start(dt: datetime, granularity: str) -> datetime:
|
||||
"""Truncate a timestamp to the start of its minute/hour bucket."""
|
||||
if granularity == "hour":
|
||||
return dt.replace(minute=0, second=0, microsecond=0)
|
||||
return dt.replace(second=0, microsecond=0)
|
||||
|
||||
|
||||
class ConcurrencySampleStore:
|
||||
"""Persists periodic concurrency snapshots and aggregates them."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def record(self, active_count: int, *, sampled_at: datetime | None = None) -> None:
|
||||
"""Insert one sample. Best-effort — never raises into the caller."""
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
session.add(
|
||||
ConcurrencySampleRow(
|
||||
active_count=max(0, int(active_count)),
|
||||
sampled_at=sampled_at or datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
except Exception: # pragma: no cover - sampling must never break the app
|
||||
logger.warning("Failed to record concurrency sample", exc_info=True)
|
||||
|
||||
async def aggregate(
|
||||
self,
|
||||
*,
|
||||
since: datetime,
|
||||
until: datetime,
|
||||
granularity: str = "minute",
|
||||
) -> list[ConcurrencyBucket]:
|
||||
"""Return per-minute / per-hour buckets of peak + average concurrency.
|
||||
|
||||
Bucketing is done in Python (not SQL ``date_trunc``/``strftime``) so the
|
||||
logic is identical across the sqlite/mysql/postgres backends.
|
||||
"""
|
||||
gran = "hour" if granularity == "hour" else "minute"
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(ConcurrencySampleRow.sampled_at, ConcurrencySampleRow.active_count)
|
||||
.where(
|
||||
ConcurrencySampleRow.sampled_at >= since,
|
||||
ConcurrencySampleRow.sampled_at <= until,
|
||||
)
|
||||
.order_by(ConcurrencySampleRow.sampled_at)
|
||||
.limit(_MAX_ROWS)
|
||||
)
|
||||
).all()
|
||||
except Exception:
|
||||
logger.warning("Failed to aggregate concurrency samples", exc_info=True)
|
||||
return []
|
||||
|
||||
buckets: dict[datetime, dict[str, float]] = {}
|
||||
for sampled_at, active_count in rows:
|
||||
if sampled_at is None:
|
||||
continue
|
||||
if sampled_at.tzinfo is None:
|
||||
sampled_at = sampled_at.replace(tzinfo=UTC)
|
||||
key = _bucket_start(sampled_at, gran)
|
||||
value = int(active_count or 0)
|
||||
entry = buckets.get(key)
|
||||
if entry is None:
|
||||
buckets[key] = {"peak": value, "sum": value, "count": 1}
|
||||
else:
|
||||
entry["peak"] = max(entry["peak"], value)
|
||||
entry["sum"] += value
|
||||
entry["count"] += 1
|
||||
|
||||
result: list[ConcurrencyBucket] = []
|
||||
for key in sorted(buckets):
|
||||
entry = buckets[key]
|
||||
count = max(1, int(entry["count"]))
|
||||
result.append(
|
||||
ConcurrencyBucket(
|
||||
bucket=key.isoformat(),
|
||||
peak=int(entry["peak"]),
|
||||
avg=round(entry["sum"] / count, 2),
|
||||
samples=count,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
async def purge_older_than(self, cutoff: datetime) -> int:
|
||||
"""Delete samples older than ``cutoff``. Returns rows removed."""
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(ConcurrencySampleRow).where(ConcurrencySampleRow.sampled_at < cutoff)
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
except Exception: # pragma: no cover
|
||||
logger.warning("Failed to purge concurrency samples", exc_info=True)
|
||||
return 0
|
||||
|
||||
|
||||
def make_concurrency_sample_store(
|
||||
session_factory: async_sessionmaker[AsyncSession] | None,
|
||||
) -> ConcurrencySampleStore | None:
|
||||
"""Factory mirroring the other ``make_*_store`` helpers.
|
||||
|
||||
Returns ``None`` on the memory backend (no SQL session factory), in which
|
||||
case the monitor simply reports a live snapshot with no historical chart.
|
||||
"""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return ConcurrencySampleStore(session_factory)
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
"""大屏绘制智能体 session persistence (按 taskId 存取,不分权)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow
|
||||
from deerflow.persistence.dashboard_sessions.sql import DashboardSessionRepository
|
||||
|
||||
__all__ = [
|
||||
"DashboardSessionRepository",
|
||||
"DashboardSessionRow",
|
||||
"make_dashboard_session_store",
|
||||
]
|
||||
|
||||
|
||||
def make_dashboard_session_store(session_factory):
|
||||
if session_factory is None:
|
||||
return None
|
||||
return DashboardSessionRepository(session_factory)
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""ORM model for the standalone 大屏绘制智能体 session (按 taskId 存取).
|
||||
|
||||
One row per external ``task_id`` — the standalone dashboard page persists its
|
||||
chat transcript + the latest structured ``report_json`` so that returning to the
|
||||
same task either shows the finished dashboard directly (valid report_json) or
|
||||
restores the generation process.
|
||||
|
||||
**Global / shared by task** — like ``roundtable_drafts`` task-bound drafts, the
|
||||
store keys on ``task_id`` and does **not** scope by ``user_id`` (open read +
|
||||
shared writes). ``user_id`` is recorded for audit only.
|
||||
|
||||
``transcript`` / ``report_json`` are large JSON blobs (chat history + report
|
||||
data) stored as serialized strings in ``PortableLongText`` (MySQL ``LONGTEXT``)
|
||||
to dodge MySQL's 64 KB ``TEXT`` cap — same approach as ``roundtable_drafts``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class DashboardSessionRow(Base):
|
||||
__tablename__ = "dashboard_sessions"
|
||||
# One session per task — keep it unique so upsert-by-task is unambiguous.
|
||||
__table_args__ = (UniqueConstraint("task_id", name="uq_dashboard_sessions_task_id"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# External task id (deep-link). The session is shared by task, not by user.
|
||||
task_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
# Audit only — who last wrote. Never used for access control.
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
# Chat transcript JSON (the dashboard agent's dialogue bubbles).
|
||||
transcript: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# Latest structured dashboard data (report-json string). Presence of a
|
||||
# parseable value = "有效结果" → the page shows the dashboard directly.
|
||||
report_json: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
"""SQLAlchemy-backed 大屏绘制智能体 session store (按 taskId,不分权).
|
||||
|
||||
``transcript`` / ``report_json`` are JSON serialized into ``PortableLongText``
|
||||
and round-tripped here with ``json.dumps`` / ``json.loads`` — same approach as
|
||||
``roundtable_drafts``. All access keys on ``task_id`` and ignores ``user_id``
|
||||
(open read + shared writes); ``user_id`` is recorded for audit only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _dumps(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
return value # already a JSON string (report_json) — store verbatim
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
class DashboardSessionRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _to_dict(row: DashboardSessionRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"task_id": row.task_id,
|
||||
"user_id": row.user_id,
|
||||
"transcript": _loads(row.transcript),
|
||||
# report_json is a structured-data JSON string; return it as a string
|
||||
# so the frontend parses it with the same robust parser it uses live.
|
||||
"report_json": row.report_json,
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
}
|
||||
|
||||
async def get_by_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""取该 task 的会话(**忽略 user_id**,不分权)。无则 None。"""
|
||||
stmt = select(DashboardSessionRow).where(DashboardSessionRow.task_id == task_id).limit(1)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalars().first()
|
||||
return self._to_dict(row) if row is not None else None
|
||||
|
||||
async def upsert_by_task(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
transcript: Any = None,
|
||||
report_json: Any = None,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""按 task_id upsert:无则建、有则更新(**不分权**)。
|
||||
|
||||
``transcript`` / ``report_json`` 仅在传入非 None 时覆盖,便于只更新其中一项。
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
stmt = select(DashboardSessionRow).where(DashboardSessionRow.task_id == task_id).limit(1)
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
if row is None:
|
||||
row = DashboardSessionRow(
|
||||
id=uuid4().hex,
|
||||
task_id=task_id,
|
||||
user_id=user_id,
|
||||
transcript=_dumps(transcript),
|
||||
report_json=_dumps(report_json),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
if transcript is not None:
|
||||
row.transcript = _dumps(transcript)
|
||||
if report_json is not None:
|
||||
row.report_json = _dumps(report_json)
|
||||
if user_id is not None:
|
||||
row.user_id = user_id
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
"""Deep-research event-log persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
|
||||
from deerflow.persistence.deep_research_events.sql import DeepResearchEventRepository
|
||||
|
||||
|
||||
def make_deep_research_event_store(session_factory) -> DeepResearchEventRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return DeepResearchEventRepository(session_factory)
|
||||
|
||||
|
||||
_default_store: DeepResearchEventRepository | None = None
|
||||
|
||||
|
||||
def register_default_store(store: DeepResearchEventRepository | None) -> None:
|
||||
global _default_store
|
||||
_default_store = store
|
||||
|
||||
|
||||
def get_default_store() -> DeepResearchEventRepository | None:
|
||||
return _default_store
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DeepResearchEventRepository",
|
||||
"DeepResearchEventRow",
|
||||
"get_default_store",
|
||||
"make_deep_research_event_store",
|
||||
"register_default_store",
|
||||
]
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
"""ORM model for the deep-research event log.
|
||||
|
||||
Unlike ``roundtable_jobs`` (whole-row DB-poll SSE), deep research persists a
|
||||
genuine **append-only event stream** so the frontend can reconnect with
|
||||
``?after=<seq>`` and replay missed events (§15.3). ``seq`` is monotonic per
|
||||
job and enforced unique by a composite constraint.
|
||||
|
||||
Events are small: payloads carry summaries + locators, never full material
|
||||
text or model reasoning (§15.2 / §19.5).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class DeepResearchEventRow(Base):
|
||||
__tablename__ = "deep_research_events"
|
||||
__table_args__ = (
|
||||
# Monotonic per-job sequence — the SSE ``Last-Event-ID`` / ``?after`` cursor.
|
||||
UniqueConstraint("job_id", "seq", name="uq_deep_research_events_job_seq"),
|
||||
# Composite index for "all events of a job in seq order".
|
||||
sa.Index("ix_deep_research_events_job_seq", "job_id", "seq"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
job_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
seq: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
|
||||
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
"""SQLAlchemy repository for the deep-research event log.
|
||||
|
||||
The ``append()`` method assigns the next monotonic ``seq`` per job inside the
|
||||
same transaction as the INSERT (``SELECT MAX(seq) … FOR`` + insert). Because
|
||||
only **one** worker holds the job lease at a time, contention on seq is
|
||||
low; the unique ``(job_id, seq)`` constraint catches any collision as a safety net.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _json_default(value: Any) -> Any:
|
||||
"""Make event payloads replayable when source metadata has native dates.
|
||||
|
||||
Search adapters intentionally preserve ``published_at`` as a ``datetime``
|
||||
until it reaches the source store. Events carry the same source envelope,
|
||||
so the append-only log must normalise it instead of silently falling back to
|
||||
a live-only frame.
|
||||
"""
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, Enum):
|
||||
return value.value
|
||||
if hasattr(value, "model_dump"):
|
||||
return value.model_dump(mode="json")
|
||||
return str(value)
|
||||
|
||||
|
||||
def _dumps(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False, default=_json_default)
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
class DeepResearchEventRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: DeepResearchEventRow) -> dict[str, Any]:
|
||||
d = row.to_dict()
|
||||
if "payload" in d:
|
||||
d["payload"] = _loads(d["payload"])
|
||||
# created_at → ISO for SSE.
|
||||
if d.get("created_at"):
|
||||
d["created_at"] = d["created_at"].isoformat() if hasattr(d["created_at"], "isoformat") else d["created_at"]
|
||||
return d
|
||||
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
session_id: str,
|
||||
job_id: str,
|
||||
event_type: str,
|
||||
phase: str = "initializing",
|
||||
payload: dict[str, Any] | None = None,
|
||||
max_retries: int = 3,
|
||||
) -> dict[str, Any]:
|
||||
"""Insert the next event, assigning ``seq = max(seq) + 1`` for this job.
|
||||
|
||||
Retries on the unique-constraint collision (only happens if two writers
|
||||
race for the same seq — shouldn't occur under single-lease ownership
|
||||
but is handled defensively).
|
||||
"""
|
||||
for attempt in range(max_retries):
|
||||
async with self._sf() as session:
|
||||
current_max = (await session.execute(select(func.max(DeepResearchEventRow.seq)).where(DeepResearchEventRow.job_id == job_id))).scalar() or 0
|
||||
next_seq = int(current_max) + 1
|
||||
row = DeepResearchEventRow(
|
||||
job_id=job_id,
|
||||
session_id=session_id,
|
||||
seq=next_seq,
|
||||
event_type=event_type,
|
||||
phase=phase,
|
||||
payload=_dumps(payload),
|
||||
)
|
||||
session.add(row)
|
||||
try:
|
||||
# Do not issue ``session.refresh(row)`` after commit. On a
|
||||
# MySQL read/write-splitting deployment that read can land
|
||||
# on a lagging replica, which cannot see the just-committed
|
||||
# event and raises ``Could not refresh instance``. Flush
|
||||
# assigns generated primary-key/default values on the writer
|
||||
# connection, so the plain dict is complete before commit.
|
||||
await session.flush()
|
||||
result = self._row_to_dict(row)
|
||||
await session.commit()
|
||||
return result
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
continue # retry with recomputed max
|
||||
raise RuntimeError(f"deep_research_events.append: seq collision persisted after {max_retries} retries (job={job_id})")
|
||||
|
||||
async def list_after(self, job_id: str, *, after: int = 0, limit: int = 200) -> list[dict[str, Any]]:
|
||||
"""Events with ``seq > after`` in ascending order (SSE replay, §15.3)."""
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchEventRow).where(DeepResearchEventRow.job_id == job_id).where(DeepResearchEventRow.seq > after).order_by(DeepResearchEventRow.seq.asc()).limit(limit)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [self._row_to_dict(r) for r in rows]
|
||||
|
||||
async def last_seq(self, job_id: str) -> int:
|
||||
async with self._sf() as session:
|
||||
mx = (await session.execute(select(func.max(DeepResearchEventRow.seq)).where(DeepResearchEventRow.job_id == job_id))).scalar()
|
||||
return int(mx or 0)
|
||||
|
||||
async def delete_by_job(self, job_id: str) -> int:
|
||||
from sqlalchemy import delete
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(DeepResearchEventRow).where(DeepResearchEventRow.job_id == job_id))
|
||||
await session.commit()
|
||||
return result.rowcount
|
||||
|
||||
async def delete_by_session(self, session_id: str) -> int:
|
||||
from sqlalchemy import delete
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(DeepResearchEventRow).where(DeepResearchEventRow.session_id == session_id))
|
||||
await session.commit()
|
||||
return result.rowcount
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""Deep-research durable-job persistence.
|
||||
|
||||
Mirrors ``roundtable_jobs``: store factory + process-level default-store singleton
|
||||
so harness-side runner code can write progress without reaching into FastAPI
|
||||
``request.app.state`` (harness → app imports forbidden).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.deep_research_jobs.model import DeepResearchJobRow
|
||||
from deerflow.persistence.deep_research_jobs.sql import (
|
||||
ACTIVE_STATUSES,
|
||||
CANCELLABLE_STATUSES,
|
||||
TERMINAL_STATUSES,
|
||||
DeepResearchJobRepository,
|
||||
compute_active_dedupe_key,
|
||||
)
|
||||
|
||||
|
||||
def make_deep_research_job_store(session_factory) -> DeepResearchJobRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return DeepResearchJobRepository(session_factory)
|
||||
|
||||
|
||||
_default_store: DeepResearchJobRepository | None = None
|
||||
|
||||
|
||||
def register_default_store(store: DeepResearchJobRepository | None) -> None:
|
||||
global _default_store
|
||||
_default_store = store
|
||||
|
||||
|
||||
def get_default_store() -> DeepResearchJobRepository | None:
|
||||
return _default_store
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ACTIVE_STATUSES",
|
||||
"CANCELLABLE_STATUSES",
|
||||
"DeepResearchJobRepository",
|
||||
"DeepResearchJobRow",
|
||||
"TERMINAL_STATUSES",
|
||||
"compute_active_dedupe_key",
|
||||
"get_default_store",
|
||||
"make_deep_research_job_store",
|
||||
"register_default_store",
|
||||
]
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
"""ORM model for deep-research durable jobs.
|
||||
|
||||
A *job* is one **backend execution** of the research pipeline against a session.
|
||||
Detailed / deep / multi-agent research can run for minutes, so jobs are
|
||||
**durable**: persisted to the DB, claimed via atomic lease, heartbeat-renewed,
|
||||
and recoverable after a crash or server restart (§14).
|
||||
|
||||
The design mirrors ``roundtable_jobs`` (Phase 3 lease + dedupe):
|
||||
|
||||
- ``active_dedupe_key`` — unique index; occupied while the job is non-terminal,
|
||||
cleared (NULL) on terminal. INSERT conflict = idempotent "active job exists".
|
||||
- ``lease_owner`` / ``lease_until`` — atomic conditional claim + heartbeat.
|
||||
- ``version`` — row CAS counter, bumped on every state-changing UPDATE.
|
||||
- Two-phase cancel: ``cancel_requested`` flag → executor watcher → ``cancelled``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class DeepResearchJobRow(Base):
|
||||
__tablename__ = "deep_research_jobs"
|
||||
__table_args__ = (
|
||||
# Same trick as roundtable_jobs: unique key occupied only while active.
|
||||
UniqueConstraint("active_dedupe_key", name="uq_deep_research_jobs_active_dedupe_key"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
|
||||
# ── idempotency + dedupe ────────────────────────────────────────────────
|
||||
request_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
request_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
active_dedupe_key: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
|
||||
# ── frozen input (rebuilt by any worker after a takeover) ───────────────
|
||||
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
|
||||
# ── lease / ownership ───────────────────────────────────────────────────
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
|
||||
|
||||
# ── status machine (§14.2) ──────────────────────────────────────────────
|
||||
# queued / running / awaiting_input / completed / failed / cancelled.
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
|
||||
# Fine-grained phase (§14.3): initializing / planning / collecting / …
|
||||
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
|
||||
progress: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
|
||||
current_query: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
# ── two-phase cancel ────────────────────────────────────────────────────
|
||||
cancel_requested: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default=sa.false()
|
||||
)
|
||||
cancel_requested_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
# ── interrupt / human-review resume (§11.2) ─────────────────────────────
|
||||
awaiting_interrupt_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
awaiting_payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
|
||||
# ── error classification (§14.5) ────────────────────────────────────────
|
||||
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
# ── result locator ──────────────────────────────────────────────────────
|
||||
result_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
+554
@@ -0,0 +1,554 @@
|
||||
"""SQLAlchemy repository for deep-research durable jobs.
|
||||
|
||||
Mirrors ``roundtable_jobs`` Phase-3 lease/dedupe/CAS logic exactly:
|
||||
|
||||
- ``compute_active_dedupe_key(session_id, user_id)`` → sha256 scope; NULL on terminal.
|
||||
- ``try_create_or_get_active()`` — INSERT-conflict = idempotent active hit.
|
||||
- ``claim_job()`` — atomic conditional UPDATE (rowcount-based mutual exclusion).
|
||||
- ``renew_lease()`` — heartbeat (only lease owner, only while ``running``).
|
||||
- ``update_progress()`` — lease-gated conditional UPDATE (CAS via ``version``).
|
||||
- ``request_cancel()`` → ``finalize_cancel()`` — two-phase cancel.
|
||||
- ``queue_for_resume()`` — atomic ``awaiting_input`` → ``queued``.
|
||||
|
||||
Recovery story (§14): ``input_snapshot`` lets any worker rebuild execution
|
||||
after a takeover; expired-lease ``running`` rows are re-claimable via
|
||||
``list_claimable``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.deep_research_jobs.model import DeepResearchJobRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
# ── status constants (§14.2) ────────────────────────────────────────────────
|
||||
TERMINAL_STATUSES = ("completed", "failed", "cancelled")
|
||||
ACTIVE_STATUSES = ("queued", "running", "awaiting_input")
|
||||
CANCELLABLE_STATUSES = ("queued", "running", "awaiting_input")
|
||||
|
||||
|
||||
def compute_active_dedupe_key(session_id: str, user_id: str | None) -> str:
|
||||
"""``sha256(session_id + user_id)`` — one active job per session+user scope."""
|
||||
scope = f"session:{session_id}:{user_id or ''}"
|
||||
return hashlib.sha256(scope.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _dumps(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
class DeepResearchJobRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# ── row ↔ dict ──────────────────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: DeepResearchJobRow) -> dict[str, Any]:
|
||||
d = row.to_dict()
|
||||
for k in ("input_snapshot", "result_snapshot", "awaiting_payload"):
|
||||
if k in d:
|
||||
d[k] = _loads(d[k])
|
||||
return d
|
||||
|
||||
# ── create / idempotent active lookup ───────────────────────────────────
|
||||
|
||||
async def try_create_or_get_active(
|
||||
self,
|
||||
*,
|
||||
id: str,
|
||||
session_id: str,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
request_id: str | None = None,
|
||||
request_hash: str | None = None,
|
||||
input_snapshot: dict[str, Any] | None = None,
|
||||
status: str = "queued",
|
||||
phase: str = "initializing",
|
||||
) -> tuple[dict[str, Any], bool]:
|
||||
"""Insert a new job, or return the existing active job for this scope.
|
||||
|
||||
Returns ``(job_dict, created)``. ``created=False`` = an active job
|
||||
already exists (idempotent hit), driven by the ``active_dedupe_key``
|
||||
unique index.
|
||||
"""
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.try_create_or_get_active")
|
||||
dedupe_key = compute_active_dedupe_key(session_id, resolved)
|
||||
|
||||
for _ in range(2):
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
row = DeepResearchJobRow(
|
||||
id=id,
|
||||
session_id=session_id,
|
||||
user_id=resolved,
|
||||
request_id=request_id,
|
||||
request_hash=request_hash,
|
||||
active_dedupe_key=dedupe_key,
|
||||
input_snapshot=_dumps(input_snapshot),
|
||||
status=status,
|
||||
phase=phase,
|
||||
)
|
||||
session.add(row)
|
||||
# Do not issue ``session.refresh(row)`` after commit. A
|
||||
# MySQL read/write-splitting proxy can route that read to
|
||||
# a lagging replica, even though the INSERT has already
|
||||
# committed on the writer. The caller would receive 500
|
||||
# while an invisible queued job remains in the database.
|
||||
# ``flush`` materializes ORM defaults on the writer, so we
|
||||
# can return a complete, durable snapshot without a
|
||||
# replica-dependent readback.
|
||||
await session.flush()
|
||||
result = self._row_to_dict(row)
|
||||
await session.commit()
|
||||
return result, True
|
||||
except IntegrityError:
|
||||
# Unique-index conflict → an active job already exists.
|
||||
pass
|
||||
|
||||
existing = await self._get_by_dedupe_key(dedupe_key)
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
# Winner entered terminal between conflict and readback → retry INSERT.
|
||||
|
||||
# Should not happen in practice.
|
||||
raise RuntimeError("try_create_or_get_active: race condition not resolved after retries")
|
||||
|
||||
async def replace_input_snapshot(self, job_id: str, input_snapshot: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Overwrite the frozen start payload on a still-queued job.
|
||||
|
||||
Idempotent retries used to keep the first snapshot forever. A first
|
||||
write that raced ahead of the browser evidence (or a reused empty
|
||||
job) then ran without ``collector_materials`` even though the later
|
||||
request carried them.
|
||||
"""
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(DeepResearchJobRow)
|
||||
.where(
|
||||
DeepResearchJobRow.id == job_id,
|
||||
DeepResearchJobRow.status.in_(("queued", "running")),
|
||||
)
|
||||
.values(
|
||||
input_snapshot=_dumps(input_snapshot),
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
async def _get_by_dedupe_key(self, dedupe_key: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchJobRow).where(
|
||||
DeepResearchJobRow.active_dedupe_key == dedupe_key
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return self._row_to_dict(row) if row else None
|
||||
|
||||
# ── read ────────────────────────────────────────────────────────────────
|
||||
|
||||
async def get(
|
||||
self, job_id: str, *, user_id: str | _AutoSentinel = AUTO
|
||||
) -> dict[str, Any] | None:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.get")
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchJobRow.user_id == resolved)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return self._row_to_dict(row) if row else None
|
||||
|
||||
async def get_unscoped(self, job_id: str) -> dict[str, Any] | None:
|
||||
"""Read without user filter (executor/dispatcher use this)."""
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return self._row_to_dict(row) if row else None
|
||||
|
||||
async def get_active_for_session(
|
||||
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO
|
||||
) -> dict[str, Any] | None:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.get_active_for_session")
|
||||
dedupe_key = compute_active_dedupe_key(session_id, resolved)
|
||||
return await self._get_by_dedupe_key(dedupe_key)
|
||||
|
||||
async def list_by_session(
|
||||
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, limit: int = 20
|
||||
) -> list[dict[str, Any]]:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.list_by_session")
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchJobRow).where(DeepResearchJobRow.session_id == session_id)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchJobRow.user_id == resolved)
|
||||
stmt = stmt.order_by(DeepResearchJobRow.created_at.desc()).limit(limit)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [self._row_to_dict(r) for r in rows]
|
||||
|
||||
# ── dispatcher: scan + claim ────────────────────────────────────────────
|
||||
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 20) -> list[str]:
|
||||
"""Candidate job ids: queued, or running with an expired lease (zombie)."""
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(DeepResearchJobRow.id)
|
||||
.where(
|
||||
DeepResearchJobRow.cancel_requested.is_(False),
|
||||
or_(
|
||||
DeepResearchJobRow.status == "queued",
|
||||
(DeepResearchJobRow.status == "running")
|
||||
& DeepResearchJobRow.lease_until.is_not(None)
|
||||
& (DeepResearchJobRow.lease_until < now),
|
||||
),
|
||||
)
|
||||
.order_by(DeepResearchJobRow.updated_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [r for r in (await session.execute(stmt)).scalars().all()]
|
||||
|
||||
async def claim_job(
|
||||
self, job_id: str, *, lease_owner: str, lease_until: datetime
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomic conditional claim — rowcount==0 means another worker won."""
|
||||
async with self._sf() as session:
|
||||
values = {
|
||||
"status": "running",
|
||||
"lease_owner": lease_owner,
|
||||
"lease_until": lease_until,
|
||||
"attempt": DeepResearchJobRow.attempt + 1,
|
||||
"version": DeepResearchJobRow.version + 1,
|
||||
"started_at": datetime.now(UTC),
|
||||
"updated_at": datetime.now(UTC),
|
||||
"cancel_requested": False,
|
||||
}
|
||||
where = [
|
||||
DeepResearchJobRow.id == job_id,
|
||||
DeepResearchJobRow.cancel_requested.is_(False),
|
||||
or_(
|
||||
DeepResearchJobRow.status == "queued",
|
||||
(DeepResearchJobRow.status == "running")
|
||||
& DeepResearchJobRow.lease_until.is_not(None)
|
||||
& (DeepResearchJobRow.lease_until < datetime.now(UTC)),
|
||||
),
|
||||
]
|
||||
result = await session.execute(
|
||||
update(DeepResearchJobRow).where(*where).values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
# ── heartbeat ───────────────────────────────────────────────────────────
|
||||
|
||||
async def renew_lease(
|
||||
self, job_id: str, *, lease_owner: str, lease_until: datetime
|
||||
) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(DeepResearchJobRow)
|
||||
.where(
|
||||
DeepResearchJobRow.id == job_id,
|
||||
DeepResearchJobRow.lease_owner == lease_owner,
|
||||
DeepResearchJobRow.status == "running",
|
||||
)
|
||||
.values(lease_until=lease_until, updated_at=datetime.now(UTC))
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount > 0
|
||||
|
||||
async def release_lease(self, job_id: str, *, lease_owner: str | None = None) -> bool:
|
||||
async with self._sf() as session:
|
||||
where = [DeepResearchJobRow.id == job_id]
|
||||
if lease_owner is not None:
|
||||
where.append(DeepResearchJobRow.lease_owner == lease_owner)
|
||||
result = await session.execute(
|
||||
update(DeepResearchJobRow)
|
||||
.where(*where)
|
||||
.values(lease_owner=None, lease_until=None, updated_at=datetime.now(UTC))
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount > 0
|
||||
|
||||
# ── progress update (lease-gated CAS) ───────────────────────────────────
|
||||
|
||||
async def update_progress(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
status: str | object = _UNSET,
|
||||
phase: str | object = _UNSET,
|
||||
progress: int | object = _UNSET,
|
||||
current_query: str | None | object = _UNSET,
|
||||
error_code: str | None | object = _UNSET,
|
||||
error_message: str | None | object = _UNSET,
|
||||
result_snapshot: dict | None | object = _UNSET,
|
||||
awaiting_interrupt_id: str | None | object = _UNSET,
|
||||
awaiting_payload: dict | None | object = _UNSET,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Lease-gated conditional progress update.
|
||||
|
||||
When ``lease_owner`` is given, the write is rejected unless this worker
|
||||
still owns the lease and the job hasn't been cancelled — preventing a
|
||||
zombie worker from clobbering a takeover. Returns ``None`` on rejection.
|
||||
|
||||
Note: when ``lease_owner`` is set, the lease ownership *is* the
|
||||
authorization (the executor calls this from a background task with no
|
||||
user context), so ``user_id`` is left unresolved in that path.
|
||||
"""
|
||||
if lease_owner is not None and isinstance(user_id, _AutoSentinel):
|
||||
resolved = None # lease-gated path — ownership is the gate, not user_id
|
||||
else:
|
||||
resolved = resolve_user_id(
|
||||
user_id, method_name="DeepResearchJobRepository.update_progress"
|
||||
)
|
||||
values: dict[str, Any] = {"updated_at": datetime.now(UTC)}
|
||||
|
||||
if status is not _UNSET and status is not None:
|
||||
values["status"] = status
|
||||
if status in TERMINAL_STATUSES:
|
||||
# Release the scope key + lease on terminal.
|
||||
values["active_dedupe_key"] = None
|
||||
values["lease_owner"] = None
|
||||
values["lease_until"] = None
|
||||
values["finished_at"] = datetime.now(UTC)
|
||||
elif status == "awaiting_input":
|
||||
# Paused: release the lease so a worker isn't pinned, keep the scope key.
|
||||
values["lease_owner"] = None
|
||||
values["lease_until"] = None
|
||||
if phase is not _UNSET:
|
||||
values["phase"] = phase
|
||||
if progress is not _UNSET:
|
||||
values["progress"] = progress
|
||||
if current_query is not _UNSET:
|
||||
values["current_query"] = current_query
|
||||
if error_code is not _UNSET:
|
||||
values["error_code"] = error_code
|
||||
if error_message is not _UNSET:
|
||||
values["error_message"] = error_message
|
||||
if result_snapshot is not _UNSET:
|
||||
values["result_snapshot"] = _dumps(result_snapshot)
|
||||
if awaiting_interrupt_id is not _UNSET:
|
||||
values["awaiting_interrupt_id"] = awaiting_interrupt_id
|
||||
if awaiting_payload is not _UNSET:
|
||||
values["awaiting_payload"] = _dumps(awaiting_payload)
|
||||
|
||||
values["version"] = DeepResearchJobRow.version + 1
|
||||
|
||||
where = [
|
||||
DeepResearchJobRow.id == job_id,
|
||||
DeepResearchJobRow.status.not_in(["cancelled"]),
|
||||
]
|
||||
if lease_owner is not None:
|
||||
where.append(DeepResearchJobRow.lease_owner == lease_owner)
|
||||
if resolved is not None and lease_owner is None:
|
||||
where.append(DeepResearchJobRow.user_id == resolved)
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(DeepResearchJobRow).where(*where).values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
# ── two-phase cancel ────────────────────────────────────────────────────
|
||||
|
||||
async def request_cancel(
|
||||
self, job_id: str, *, user_id: str | _AutoSentinel = AUTO
|
||||
) -> dict[str, Any] | None:
|
||||
"""Phase 1: atomically flip ``cancel_requested`` + flag for the executor."""
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.request_cancel")
|
||||
async with self._sf() as session:
|
||||
where = [
|
||||
DeepResearchJobRow.id == job_id,
|
||||
DeepResearchJobRow.status.in_(list(CANCELLABLE_STATUSES)),
|
||||
]
|
||||
if resolved is not None:
|
||||
where.append(DeepResearchJobRow.user_id == resolved)
|
||||
result = await session.execute(
|
||||
update(DeepResearchJobRow)
|
||||
.where(*where)
|
||||
.values(
|
||||
cancel_requested=True,
|
||||
cancel_requested_at=datetime.now(UTC),
|
||||
version=DeepResearchJobRow.version + 1,
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
async def finalize_cancel(
|
||||
self, job_id: str, *, lease_owner: str | None = None
|
||||
) -> dict[str, Any] | None:
|
||||
"""Phase 2: ``cancel_requested`` → terminal ``cancelled`` (clears dedupe key)."""
|
||||
async with self._sf() as session:
|
||||
where = [
|
||||
DeepResearchJobRow.id == job_id,
|
||||
DeepResearchJobRow.cancel_requested.is_(True),
|
||||
]
|
||||
if lease_owner is not None:
|
||||
where.append(DeepResearchJobRow.lease_owner == lease_owner)
|
||||
result = await session.execute(
|
||||
update(DeepResearchJobRow)
|
||||
.where(*where)
|
||||
.values(
|
||||
status="cancelled",
|
||||
active_dedupe_key=None,
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
version=DeepResearchJobRow.version + 1,
|
||||
finished_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
# ── resume after human-review interrupt ─────────────────────────────────
|
||||
|
||||
async def queue_for_resume(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
resume_payload: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomic ``awaiting_input`` → ``queued`` so the dispatcher re-claims it."""
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.queue_for_resume")
|
||||
async with self._sf() as session:
|
||||
where = [
|
||||
DeepResearchJobRow.id == job_id,
|
||||
DeepResearchJobRow.status == "awaiting_input",
|
||||
]
|
||||
if resolved is not None:
|
||||
where.append(DeepResearchJobRow.user_id == resolved)
|
||||
result = await session.execute(
|
||||
update(DeepResearchJobRow)
|
||||
.where(*where)
|
||||
.values(
|
||||
status="queued",
|
||||
awaiting_interrupt_id=None,
|
||||
awaiting_payload=_dumps(resume_payload),
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
version=DeepResearchJobRow.version + 1,
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
# ── cleanup ─────────────────────────────────────────────────────────────
|
||||
|
||||
async def list_older_than(self, *, cutoff: datetime, limit: int = 100) -> list[str]:
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(DeepResearchJobRow.id)
|
||||
.where(DeepResearchJobRow.status.in_(list(TERMINAL_STATUSES)))
|
||||
.where(DeepResearchJobRow.updated_at < cutoff)
|
||||
.limit(limit)
|
||||
)
|
||||
return [r for r in (await session.execute(stmt)).scalars().all()]
|
||||
|
||||
async def list_cancel_reapable(self, *, now: datetime, limit: int = 20) -> list[str]:
|
||||
"""``cancel_requested`` jobs whose lease expired or was never held.
|
||||
|
||||
These are orphaned cancellations (the worker that set the flag crashed
|
||||
before finalizing). The dispatcher reaps them so they don't stick in a
|
||||
half-cancelled state forever (§14.3).
|
||||
"""
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(DeepResearchJobRow.id)
|
||||
.where(DeepResearchJobRow.cancel_requested.is_(True))
|
||||
.where(DeepResearchJobRow.status.not_in(list(TERMINAL_STATUSES)))
|
||||
.where(
|
||||
or_(
|
||||
DeepResearchJobRow.lease_owner.is_(None),
|
||||
(
|
||||
DeepResearchJobRow.lease_until.is_not(None)
|
||||
& (DeepResearchJobRow.lease_until < now)
|
||||
),
|
||||
)
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
return [r for r in (await session.execute(stmt)).scalars().all()]
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
"""Deep-research follow-up message persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.deep_research_messages.model import DeepResearchMessageRow
|
||||
from deerflow.persistence.deep_research_messages.sql import DeepResearchMessageRepository
|
||||
|
||||
|
||||
def make_deep_research_message_store(session_factory) -> DeepResearchMessageRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return DeepResearchMessageRepository(session_factory)
|
||||
|
||||
|
||||
_default_store: DeepResearchMessageRepository | None = None
|
||||
|
||||
|
||||
def register_default_store(store: DeepResearchMessageRepository | None) -> None:
|
||||
global _default_store
|
||||
_default_store = store
|
||||
|
||||
|
||||
def get_default_store() -> DeepResearchMessageRepository | None:
|
||||
return _default_store
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DeepResearchMessageRepository",
|
||||
"DeepResearchMessageRow",
|
||||
"get_default_store",
|
||||
"make_deep_research_message_store",
|
||||
"register_default_store",
|
||||
]
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
"""ORM model for deep-research follow-up messages.
|
||||
|
||||
After a report is complete, the user can ask follow-up questions that are
|
||||
answered against the session's report + sources only (``allow_new_research=false``
|
||||
default; §13.5 / §16.6). This table persists that conversation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class DeepResearchMessageRow(Base):
|
||||
__tablename__ = "deep_research_messages"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
role: Mapped[str] = mapped_column(String(16), nullable=False) # user / assistant
|
||||
content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
|
||||
# Source ids cited in an assistant reply.
|
||||
citation_source_ids: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# Token/cost usage for this message (JSON).
|
||||
usage_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
"""SQLAlchemy repository for deep-research follow-up messages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.deep_research_messages.model import DeepResearchMessageRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _dumps(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
class DeepResearchMessageRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: DeepResearchMessageRow) -> dict[str, Any]:
|
||||
d = row.to_dict()
|
||||
if "citation_source_ids" in d:
|
||||
d["citation_source_ids"] = _loads(d["citation_source_ids"]) or []
|
||||
if "usage_snapshot" in d:
|
||||
d["usage_snapshot"] = _loads(d["usage_snapshot"])
|
||||
return d
|
||||
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
id: str,
|
||||
session_id: str,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
role: str,
|
||||
content: str,
|
||||
citation_source_ids: list[str] | None = None,
|
||||
usage_snapshot: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchMessageRepository.append")
|
||||
async with self._sf() as session:
|
||||
current_max = (
|
||||
await session.execute(
|
||||
select(func.max(DeepResearchMessageRow.seq)).where(
|
||||
DeepResearchMessageRow.session_id == session_id
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
row = DeepResearchMessageRow(
|
||||
id=id,
|
||||
session_id=session_id,
|
||||
user_id=resolved,
|
||||
role=role,
|
||||
content=content,
|
||||
citation_source_ids=_dumps(citation_source_ids),
|
||||
usage_snapshot=_dumps(usage_snapshot),
|
||||
seq=int(current_max) + 1,
|
||||
)
|
||||
session.add(row)
|
||||
# Keep the successful append independent of a later replica read.
|
||||
# The explicit id/seq and Python-side defaults are available after
|
||||
# flush, before the transaction releases its writer connection.
|
||||
await session.flush()
|
||||
snapshot = self._row_to_dict(row)
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
async def list_by_session(
|
||||
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, limit: int = 100
|
||||
) -> list[dict[str, Any]]:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchMessageRepository.list_by_session")
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchMessageRow).where(
|
||||
DeepResearchMessageRow.session_id == session_id
|
||||
)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchMessageRow.user_id == resolved)
|
||||
stmt = stmt.order_by(DeepResearchMessageRow.seq.asc()).limit(limit)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [self._row_to_dict(r) for r in rows]
|
||||
|
||||
async def get(
|
||||
self,
|
||||
message_id: str,
|
||||
*,
|
||||
session_id: str,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Fetch one follow-up message inside its owner-scoped session."""
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchMessageRepository.get")
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchMessageRow).where(
|
||||
DeepResearchMessageRow.id == message_id,
|
||||
DeepResearchMessageRow.session_id == session_id,
|
||||
)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchMessageRow.user_id == resolved)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return self._row_to_dict(row) if row is not None else None
|
||||
|
||||
async def update_usage_snapshot(
|
||||
self,
|
||||
message_id: str,
|
||||
*,
|
||||
session_id: str,
|
||||
usage_snapshot: dict[str, Any] | None,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Update durable UI metadata without altering the message transcript."""
|
||||
resolved = resolve_user_id(
|
||||
user_id,
|
||||
method_name="DeepResearchMessageRepository.update_usage_snapshot",
|
||||
)
|
||||
async with self._sf() as session:
|
||||
where = [
|
||||
DeepResearchMessageRow.id == message_id,
|
||||
DeepResearchMessageRow.session_id == session_id,
|
||||
]
|
||||
if resolved is not None:
|
||||
where.append(DeepResearchMessageRow.user_id == resolved)
|
||||
result = await session.execute(
|
||||
update(DeepResearchMessageRow)
|
||||
.where(*where)
|
||||
.values(usage_snapshot=_dumps(usage_snapshot))
|
||||
)
|
||||
if not result.rowcount:
|
||||
await session.rollback()
|
||||
return None
|
||||
stmt = select(DeepResearchMessageRow).where(*where)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row is not None else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
async def delete_by_session(self, session_id: str) -> int:
|
||||
from sqlalchemy import delete
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(DeepResearchMessageRow).where(
|
||||
DeepResearchMessageRow.session_id == session_id
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
"""Deep-research session persistence.
|
||||
|
||||
Exposes a store factory + a process-level default-store singleton, mirroring
|
||||
``roundtable_jobs``: harness-side graph nodes call ``get_default_store()``;
|
||||
``app/gateway/deps.py`` registers it at startup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.deep_research_sessions.model import DeepResearchSessionRow
|
||||
from deerflow.persistence.deep_research_sessions.sql import DeepResearchSessionRepository
|
||||
|
||||
|
||||
def make_deep_research_session_store(
|
||||
session_factory,
|
||||
) -> DeepResearchSessionRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return DeepResearchSessionRepository(session_factory)
|
||||
|
||||
|
||||
_default_store: DeepResearchSessionRepository | None = None
|
||||
|
||||
|
||||
def register_default_store(store: DeepResearchSessionRepository | None) -> None:
|
||||
global _default_store
|
||||
_default_store = store
|
||||
|
||||
|
||||
def get_default_store() -> DeepResearchSessionRepository | None:
|
||||
return _default_store
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DeepResearchSessionRepository",
|
||||
"DeepResearchSessionRow",
|
||||
"get_default_store",
|
||||
"make_deep_research_session_store",
|
||||
"register_default_store",
|
||||
]
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""ORM model for deep-research sessions.
|
||||
|
||||
A *session* is one research topic owned by a user: it holds the frozen config
|
||||
snapshot, the latest report/plan, and points at the hidden runtime thread that
|
||||
owns the artifact files. Multiple *jobs* (runs) can execute against a session
|
||||
(re-run with different settings, resume after human review), but at most one
|
||||
job is active at a time (enforced by the jobs table's ``active_dedupe_key``).
|
||||
|
||||
Field design follows ``roundtable_drafts`` / ``ai_writing_sessions``:
|
||||
big JSON/text columns use ``PortableLongText``; timestamps use ``BeijingDateTime``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class DeepResearchSessionRow(Base):
|
||||
__tablename__ = "deep_research_sessions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
|
||||
query: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
|
||||
# Research mode (basic/detailed/deep/multi_agent/quick) — denormalised for list views.
|
||||
mode: Mapped[str] = mapped_column(String(32), nullable=False, default="basic", index=True)
|
||||
# Session status machine: draft / running / awaiting_input / completed / failed / cancelled.
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="draft", index=True)
|
||||
# Frozen config at creation/start time (full DeepResearchConfig JSON).
|
||||
config_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# The hidden runtime thread that owns the artifacts (metadata.thread_type=deep_research).
|
||||
runtime_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True, unique=True)
|
||||
# Latest research plan (JSON) — updated by the planner node.
|
||||
plan_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# Latest report outputs.
|
||||
report_markdown: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
report_html: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# Denormalised counts for list views (avoid joining sources table).
|
||||
source_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
|
||||
# Token/cost usage snapshot (JSON: {model: {prompt, completion, total}}).
|
||||
usage_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# Current active job id (null when none running).
|
||||
active_job_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
# User-facing error summary on failure.
|
||||
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+274
@@ -0,0 +1,274 @@
|
||||
"""SQLAlchemy repository for deep-research sessions.
|
||||
|
||||
Mirrors the ``roundtable_jobs`` / ``ai_writing_sessions`` repository conventions:
|
||||
constructed with an ``async_sessionmaker``, methods are ``async``, open
|
||||
``async with self._sf() as session:`` blocks, and return **plain dicts**.
|
||||
JSON columns round-trip through ``json.dumps`` / ``json.loads``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.deep_research_sessions.model import DeepResearchSessionRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Only these states have a durable job that consumes a per-user research slot.
|
||||
# A draft is just an unsent composer record; counting it here makes a few saved
|
||||
# drafts permanently trigger the start-job 429 guard.
|
||||
IN_FLIGHT_STATUSES = ("running", "awaiting_input")
|
||||
TERMINAL_STATUSES = ("completed", "failed", "cancelled")
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _dumps(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
class DeepResearchSessionRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# ── row ↔ dict ──────────────────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: DeepResearchSessionRow) -> dict[str, Any]:
|
||||
d = row.to_dict()
|
||||
# Deserialize JSON columns.
|
||||
for k in ("config_snapshot", "plan_snapshot", "usage_snapshot"):
|
||||
if k in d:
|
||||
d[k] = _loads(d[k])
|
||||
return d
|
||||
|
||||
# ── create ──────────────────────────────────────────────────────────────
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
id: str,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
title: str = "",
|
||||
query: str = "",
|
||||
mode: str = "basic",
|
||||
status: str = "draft",
|
||||
config_snapshot: dict[str, Any] | None = None,
|
||||
runtime_thread_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.create")
|
||||
async with self._sf() as session:
|
||||
row = DeepResearchSessionRow(
|
||||
id=id,
|
||||
user_id=resolved,
|
||||
title=title,
|
||||
query=query,
|
||||
mode=mode,
|
||||
status=status,
|
||||
config_snapshot=_dumps(config_snapshot),
|
||||
runtime_thread_id=runtime_thread_id,
|
||||
)
|
||||
session.add(row)
|
||||
# New sessions are a control-plane write. Avoid a post-commit
|
||||
# refresh because MySQL read/write-splitting can make a just
|
||||
# created session look absent on a lagging replica.
|
||||
await session.flush()
|
||||
snapshot = self._row_to_dict(row)
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
# ── read ────────────────────────────────────────────────────────────────
|
||||
|
||||
async def get(self, session_id: str, *, user_id: str | _AutoSentinel = AUTO) -> dict[str, Any] | None:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.get")
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchSessionRow).where(DeepResearchSessionRow.id == session_id)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchSessionRow.user_id == resolved)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return self._row_to_dict(row) if row else None
|
||||
|
||||
async def list_by_user(
|
||||
self,
|
||||
*,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
status: str | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
include_config: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Paginated session list for the history panel (excludes heavy report text)."""
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.list_by_user")
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchSessionRow)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchSessionRow.user_id == resolved)
|
||||
if status:
|
||||
stmt = stmt.where(DeepResearchSessionRow.status == status)
|
||||
stmt = stmt.order_by(DeepResearchSessionRow.updated_at.desc()).limit(limit).offset(offset)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
# Strip heavy columns for list views.
|
||||
out: list[dict[str, Any]] = []
|
||||
for r in rows:
|
||||
d = self._row_to_dict(r)
|
||||
d.pop("report_markdown", None)
|
||||
d.pop("report_html", None)
|
||||
if not include_config:
|
||||
d.pop("config_snapshot", None)
|
||||
d.pop("plan_snapshot", None)
|
||||
d.pop("usage_snapshot", None)
|
||||
d.pop("error", None)
|
||||
out.append(d)
|
||||
return out
|
||||
|
||||
async def count_by_user(
|
||||
self, *, user_id: str | _AutoSentinel = AUTO, status: str | None = None
|
||||
) -> int:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.count_by_user")
|
||||
async with self._sf() as session:
|
||||
stmt = select(func.count(DeepResearchSessionRow.id))
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchSessionRow.user_id == resolved)
|
||||
if status:
|
||||
stmt = stmt.where(DeepResearchSessionRow.status == status)
|
||||
return int((await session.execute(stmt)).scalar() or 0)
|
||||
|
||||
async def count_active_by_user(self, *, user_id: str | _AutoSentinel = AUTO) -> int:
|
||||
"""Count sessions with an in-flight job for the per-user concurrency limit.
|
||||
|
||||
Draft sessions deliberately do not count: they have no job row and a
|
||||
user may safely keep any number of unfinished research prompts.
|
||||
"""
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.count_active_by_user")
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(func.count(DeepResearchSessionRow.id))
|
||||
.where(DeepResearchSessionRow.user_id == resolved)
|
||||
.where(DeepResearchSessionRow.status.in_(list(IN_FLIGHT_STATUSES)))
|
||||
)
|
||||
return int((await session.execute(stmt)).scalar() or 0)
|
||||
|
||||
# ── update ──────────────────────────────────────────────────────────────
|
||||
|
||||
async def update(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Generic field update; JSON-valued fields are auto-serialized."""
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.update")
|
||||
# Serialize known JSON columns.
|
||||
if "config_snapshot" in fields and fields["config_snapshot"] is not None:
|
||||
fields["config_snapshot"] = _dumps(fields["config_snapshot"])
|
||||
if "plan_snapshot" in fields and fields["plan_snapshot"] is not None:
|
||||
fields["plan_snapshot"] = _dumps(fields["plan_snapshot"])
|
||||
if "usage_snapshot" in fields and fields["usage_snapshot"] is not None:
|
||||
fields["usage_snapshot"] = _dumps(fields["usage_snapshot"])
|
||||
fields["updated_at"] = datetime.now(UTC)
|
||||
|
||||
async with self._sf() as session:
|
||||
where = [DeepResearchSessionRow.id == session_id]
|
||||
if resolved is not None:
|
||||
where.append(DeepResearchSessionRow.user_id == resolved)
|
||||
result = await session.execute(
|
||||
update(DeepResearchSessionRow).where(*where).values(**fields)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchSessionRow).where(DeepResearchSessionRow.id == session_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
async def replace_report_if_unchanged(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
expected_report: str,
|
||||
report_markdown: str,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomically replace a report only when its exact prior text still matches.
|
||||
|
||||
The caller may use a hash for user-facing diagnostics, but SQL compares
|
||||
the text itself so another request cannot slip in between a Python-side
|
||||
hash check and the update.
|
||||
"""
|
||||
resolved = resolve_user_id(
|
||||
user_id,
|
||||
method_name="DeepResearchSessionRepository.replace_report_if_unchanged",
|
||||
)
|
||||
async with self._sf() as session:
|
||||
clauses = [
|
||||
DeepResearchSessionRow.id == session_id,
|
||||
DeepResearchSessionRow.report_markdown == expected_report,
|
||||
]
|
||||
if resolved is not None:
|
||||
clauses.append(DeepResearchSessionRow.user_id == resolved)
|
||||
result = await session.execute(
|
||||
update(DeepResearchSessionRow)
|
||||
.where(*clauses)
|
||||
.values(
|
||||
report_markdown=report_markdown,
|
||||
report_html=None,
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchSessionRow).where(DeepResearchSessionRow.id == session_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
# ── delete ──────────────────────────────────────────────────────────────
|
||||
|
||||
async def delete(
|
||||
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO
|
||||
) -> bool:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.delete")
|
||||
async with self._sf() as session:
|
||||
where = [DeepResearchSessionRow.id == session_id]
|
||||
if resolved is not None:
|
||||
where.append(DeepResearchSessionRow.user_id == resolved)
|
||||
result = await session.execute(delete(DeepResearchSessionRow).where(*where))
|
||||
await session.commit()
|
||||
return result.rowcount > 0
|
||||
|
||||
async def get_runtime_thread_id(self, session_id: str) -> str | None:
|
||||
"""Look up the hidden thread id (for artifact cleanup on delete)."""
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchSessionRow.runtime_thread_id).where(
|
||||
DeepResearchSessionRow.id == session_id
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return row
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
"""Deep-research source-material persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.deep_research_sources.model import DeepResearchSourceRow
|
||||
from deerflow.persistence.deep_research_sources.sql import DeepResearchSourceRepository
|
||||
|
||||
|
||||
def make_deep_research_source_store(session_factory) -> DeepResearchSourceRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return DeepResearchSourceRepository(session_factory)
|
||||
|
||||
|
||||
_default_store: DeepResearchSourceRepository | None = None
|
||||
|
||||
|
||||
def register_default_store(store: DeepResearchSourceRepository | None) -> None:
|
||||
global _default_store
|
||||
_default_store = store
|
||||
|
||||
|
||||
def get_default_store() -> DeepResearchSourceRepository | None:
|
||||
return _default_store
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DeepResearchSourceRepository",
|
||||
"DeepResearchSourceRow",
|
||||
"get_default_store",
|
||||
"make_deep_research_source_store",
|
||||
"register_default_store",
|
||||
]
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
"""ORM model for deep-research source materials.
|
||||
|
||||
Stores every :class:`~deerflow.agents.deep_research.types.ResearchMaterial`
|
||||
collected during a session, with full provenance (title, url, source,
|
||||
``rec_uuid``, publish time, relevance) so report citations can trace back to
|
||||
the originating DeerFlow record (§8.2 / §13.4).
|
||||
|
||||
Full text uses ``PortableLongText`` (MySQL ``LONGTEXT``) since web pages can be
|
||||
large; the list endpoint returns summaries only (§16.5).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Boolean, Float, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class DeepResearchSourceRow(Base):
|
||||
__tablename__ = "deep_research_sources"
|
||||
__table_args__ = (
|
||||
# MySQL requires a prefix length when indexing TEXT columns. The
|
||||
# column remains unbounded so long URLs are preserved; the prefix is
|
||||
# only an index implementation detail and equality filters still
|
||||
# compare the complete URL value.
|
||||
sa.Index("ix_deep_research_sources_url", "url", mysql_length=512),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
job_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
|
||||
# The query that first surfaced this material.
|
||||
first_seen_query: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
title: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
raw_content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
|
||||
snippet: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
source: Mapped[str] = mapped_column(String(256), nullable=False, default="")
|
||||
source_type: Mapped[str] = mapped_column(String(32), nullable=False, default="web")
|
||||
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
relevance_score: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
rec_uuid: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
|
||||
# Whether this source was selected into the final context.
|
||||
selected: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default=sa.false()
|
||||
)
|
||||
selection_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Stable citation key assigned during report writing (e.g. "1", "2").
|
||||
citation_key: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+243
@@ -0,0 +1,243 @@
|
||||
"""SQLAlchemy repository for deep-research source materials."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.deep_research_sources.model import DeepResearchSourceRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_JSON_COLS = ("metadata",)
|
||||
|
||||
|
||||
class DeepResearchSourceRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: DeepResearchSourceRow, *, include_content: bool = True) -> dict[str, Any]:
|
||||
d = row.to_dict()
|
||||
if not include_content:
|
||||
d.pop("raw_content", None)
|
||||
d.pop("snippet", None)
|
||||
return d
|
||||
|
||||
# ── upsert (dedupe by rec_uuid / url / content_hash) ────────────────────
|
||||
|
||||
async def upsert(self, *, session_id: str, job_id: str | None = None, **fields: Any) -> dict[str, Any]:
|
||||
"""Insert a source, or return the existing one if rec_uuid/url/hash matches."""
|
||||
# Try insert.
|
||||
async with self._sf() as session:
|
||||
row = DeepResearchSourceRow(session_id=session_id, job_id=job_id, **fields)
|
||||
session.add(row)
|
||||
try:
|
||||
# The source was produced on the worker's write path. Build
|
||||
# its response on that same connection so a replica-lag read
|
||||
# cannot turn a successfully collected source into a failed
|
||||
# research run.
|
||||
await session.flush()
|
||||
snapshot = self._row_to_dict(row)
|
||||
await session.commit()
|
||||
return snapshot
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
# Fallback: find existing by rec_uuid or url.
|
||||
existing = await self._find_existing(
|
||||
session_id=session_id,
|
||||
rec_uuid=fields.get("rec_uuid"),
|
||||
url=fields.get("url"),
|
||||
content_hash=fields.get("content_hash", ""),
|
||||
)
|
||||
return existing or self._row_to_dict(row)
|
||||
|
||||
async def _find_existing(
|
||||
self, *, session_id: str, rec_uuid: str | None, url: str | None, content_hash: str
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchSourceRow).where(
|
||||
DeepResearchSourceRow.session_id == session_id
|
||||
)
|
||||
if rec_uuid:
|
||||
stmt = stmt.where(DeepResearchSourceRow.rec_uuid == rec_uuid)
|
||||
elif url:
|
||||
stmt = stmt.where(DeepResearchSourceRow.url == url)
|
||||
elif content_hash:
|
||||
stmt = stmt.where(DeepResearchSourceRow.content_hash == content_hash)
|
||||
else:
|
||||
return None
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return self._row_to_dict(row) if row else None
|
||||
|
||||
# ── read ────────────────────────────────────────────────────────────────
|
||||
|
||||
async def get(
|
||||
self,
|
||||
source_id: str,
|
||||
*,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
include_content: bool = True,
|
||||
) -> dict[str, Any] | None:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.get")
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchSourceRow).where(DeepResearchSourceRow.id == source_id)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return self._row_to_dict(row, include_content=include_content) if row else None
|
||||
|
||||
async def list_by_session(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
selected: bool | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
include_content: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.list_by_session")
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchSourceRow).where(DeepResearchSourceRow.session_id == session_id)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
|
||||
if selected is not None:
|
||||
stmt = stmt.where(DeepResearchSourceRow.selected == selected)
|
||||
stmt = stmt.order_by(DeepResearchSourceRow.created_at.asc()).limit(limit).offset(offset)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [self._row_to_dict(r, include_content=include_content) for r in rows]
|
||||
|
||||
async def count_by_session(
|
||||
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, selected: bool | None = None
|
||||
) -> int:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.count_by_session")
|
||||
async with self._sf() as session:
|
||||
stmt = select(func.count(DeepResearchSourceRow.id)).where(
|
||||
DeepResearchSourceRow.session_id == session_id
|
||||
)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
|
||||
if selected is not None:
|
||||
stmt = stmt.where(DeepResearchSourceRow.selected == selected)
|
||||
return int((await session.execute(stmt)).scalar() or 0)
|
||||
|
||||
async def clone_selected_to_session(
|
||||
self,
|
||||
source_session_id: str,
|
||||
target_session_id: str,
|
||||
*,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
) -> dict[str, str]:
|
||||
"""Copy the selected evidence into a new report session.
|
||||
|
||||
Source ids are globally unique primary keys, so a report variant cannot
|
||||
reuse the parent rows. Return the ``old_id -> new_id`` mapping to let
|
||||
the caller repair the copied Markdown citation markers before the
|
||||
variant becomes visible.
|
||||
"""
|
||||
resolved = resolve_user_id(
|
||||
user_id,
|
||||
method_name="DeepResearchSourceRepository.clone_selected_to_session",
|
||||
)
|
||||
async with self._sf() as session:
|
||||
stmt = select(DeepResearchSourceRow).where(
|
||||
DeepResearchSourceRow.session_id == source_session_id,
|
||||
DeepResearchSourceRow.selected.is_(True),
|
||||
)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
|
||||
source_rows = (await session.execute(stmt)).scalars().all()
|
||||
if not source_rows:
|
||||
# Collector persist marks rows selected, but an older session
|
||||
# may only have unselected harvest rows. Cloning nothing would
|
||||
# make 「选择结构再生成」 write with no materials.
|
||||
stmt = select(DeepResearchSourceRow).where(
|
||||
DeepResearchSourceRow.session_id == source_session_id,
|
||||
)
|
||||
if resolved is not None:
|
||||
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
|
||||
source_rows = (await session.execute(stmt)).scalars().all()
|
||||
|
||||
id_map: dict[str, str] = {}
|
||||
cloned_rows: list[DeepResearchSourceRow] = []
|
||||
for source in source_rows:
|
||||
cloned_id = f"drs_src_{uuid.uuid4().hex}"
|
||||
id_map[source.id] = cloned_id
|
||||
cloned_rows.append(
|
||||
DeepResearchSourceRow(
|
||||
id=cloned_id,
|
||||
session_id=target_session_id,
|
||||
job_id=None,
|
||||
user_id=resolved if resolved is not None else source.user_id,
|
||||
first_seen_query=source.first_seen_query,
|
||||
title=source.title,
|
||||
url=source.url,
|
||||
raw_content=source.raw_content,
|
||||
snippet=source.snippet,
|
||||
source=source.source,
|
||||
source_type=source.source_type,
|
||||
published_at=source.published_at,
|
||||
relevance_score=source.relevance_score,
|
||||
rec_uuid=source.rec_uuid,
|
||||
content_hash=source.content_hash,
|
||||
selected=True,
|
||||
selection_reason=source.selection_reason,
|
||||
citation_key=source.citation_key,
|
||||
)
|
||||
)
|
||||
if cloned_rows:
|
||||
session.add_all(cloned_rows)
|
||||
await session.flush()
|
||||
await session.commit()
|
||||
return id_map
|
||||
|
||||
# ── update selection ────────────────────────────────────────────────────
|
||||
|
||||
async def set_selected(
|
||||
self,
|
||||
source_id: str,
|
||||
*,
|
||||
selected: bool,
|
||||
reason: str | None = None,
|
||||
citation_key: str | None = None,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
) -> dict[str, Any] | None:
|
||||
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.set_selected")
|
||||
async with self._sf() as session:
|
||||
where = [DeepResearchSourceRow.id == source_id]
|
||||
if resolved is not None:
|
||||
where.append(DeepResearchSourceRow.user_id == resolved)
|
||||
values: dict[str, Any] = {"selected": selected}
|
||||
if reason is not None:
|
||||
values["selection_reason"] = reason
|
||||
if citation_key is not None:
|
||||
values["citation_key"] = citation_key
|
||||
result = await session.execute(
|
||||
update(DeepResearchSourceRow).where(*where).values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DeepResearchSourceRow).where(DeepResearchSourceRow.id == source_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
snapshot = self._row_to_dict(row) if row else None
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
async def delete_by_session(self, session_id: str) -> int:
|
||||
from sqlalchemy import delete
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(DeepResearchSourceRow).where(DeepResearchSourceRow.session_id == session_id)
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
"""Document rewrite version persistence factory."""
|
||||
|
||||
from deerflow.persistence.document_rewrite_versions.model import DocumentRewriteVersionRow
|
||||
from deerflow.persistence.document_rewrite_versions.sql import DocumentRewriteVersionRepository
|
||||
|
||||
|
||||
def make_document_rewrite_version_store(session_factory) -> DocumentRewriteVersionRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return DocumentRewriteVersionRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DocumentRewriteVersionRepository",
|
||||
"DocumentRewriteVersionRow",
|
||||
"make_document_rewrite_version_store",
|
||||
]
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"""Persistent, undoable snapshots for whole-document AI rewrites."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class DocumentRewriteVersionRow(Base):
|
||||
"""The immutable pre-rewrite document retained for one safe undo action."""
|
||||
|
||||
__tablename__ = "document_rewrite_versions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
# ``artifact`` uses ``source_id=thread_id`` and a virtual sandbox path;
|
||||
# ``deep_research_report`` uses ``source_id=deep-research session id``.
|
||||
source_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
source_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
original_content: Mapped[str] = mapped_column(PortableLongText, nullable=False)
|
||||
original_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
committed_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
instruction: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
model_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
reverted_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
"""Repository for document-rewrite snapshots.
|
||||
|
||||
The caller performs the actual content compare-and-swap. This repository keeps
|
||||
the retained original version owner-scoped and marks a snapshot consumed after a
|
||||
successful undo, preventing an older undo from silently overwriting newer work.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.document_rewrite_versions.model import DocumentRewriteVersionRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
|
||||
class DocumentRewriteVersionRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: DocumentRewriteVersionRow) -> dict[str, Any]:
|
||||
return row.to_dict()
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
id: str,
|
||||
source_type: str,
|
||||
source_id: str,
|
||||
source_path: str | None,
|
||||
original_content: str,
|
||||
original_hash: str,
|
||||
committed_hash: str,
|
||||
instruction: str | None = None,
|
||||
model_name: str | None = None,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
) -> dict[str, Any]:
|
||||
resolved = resolve_user_id(user_id, method_name="DocumentRewriteVersionRepository.create")
|
||||
async with self._sf() as session:
|
||||
row = DocumentRewriteVersionRow(
|
||||
id=id,
|
||||
user_id=resolved,
|
||||
source_type=source_type,
|
||||
source_id=source_id,
|
||||
source_path=source_path,
|
||||
original_content=original_content,
|
||||
original_hash=original_hash,
|
||||
committed_hash=committed_hash,
|
||||
instruction=instruction,
|
||||
model_name=model_name,
|
||||
)
|
||||
session.add(row)
|
||||
# Do not refresh after commit. In a read/write-splitting MySQL
|
||||
# deployment that read can be routed to a lagging replica even
|
||||
# though the INSERT succeeded on the writer. The caller treats a
|
||||
# version-record failure as a reason to restore the document, so
|
||||
# that replica-only error would otherwise undo a valid rewrite.
|
||||
# Flush materializes the Python/SQLAlchemy defaults on the writer
|
||||
# and lets us return a complete snapshot without a second query.
|
||||
await session.flush()
|
||||
snapshot = self._row_to_dict(row)
|
||||
await session.commit()
|
||||
return snapshot
|
||||
|
||||
async def get(
|
||||
self,
|
||||
version_id: str,
|
||||
*,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
source_type: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
resolved = resolve_user_id(user_id, method_name="DocumentRewriteVersionRepository.get")
|
||||
async with self._sf() as session:
|
||||
clauses = [DocumentRewriteVersionRow.id == version_id]
|
||||
if resolved is not None:
|
||||
clauses.append(DocumentRewriteVersionRow.user_id == resolved)
|
||||
if source_type:
|
||||
clauses.append(DocumentRewriteVersionRow.source_type == source_type)
|
||||
row = (await session.execute(select(DocumentRewriteVersionRow).where(*clauses))).scalar_one_or_none()
|
||||
return self._row_to_dict(row) if row else None
|
||||
|
||||
async def mark_reverted(
|
||||
self,
|
||||
version_id: str,
|
||||
*,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved = resolve_user_id(user_id, method_name="DocumentRewriteVersionRepository.mark_reverted")
|
||||
async with self._sf() as session:
|
||||
clauses = [
|
||||
DocumentRewriteVersionRow.id == version_id,
|
||||
DocumentRewriteVersionRow.reverted_at.is_(None),
|
||||
]
|
||||
if resolved is not None:
|
||||
clauses.append(DocumentRewriteVersionRow.user_id == resolved)
|
||||
result = await session.execute(
|
||||
update(DocumentRewriteVersionRow)
|
||||
.where(*clauses)
|
||||
.values(reverted_at=datetime.now(UTC))
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount > 0
|
||||
|
||||
async def prune_source(
|
||||
self,
|
||||
*,
|
||||
source_type: str,
|
||||
source_id: str,
|
||||
source_path: str | None,
|
||||
keep: int = 20,
|
||||
user_id: str | _AutoSentinel = AUTO,
|
||||
) -> int:
|
||||
"""Retain the newest snapshots for one document without crossing owners.
|
||||
|
||||
Old rewrite snapshots contain complete document bodies, so this is both a
|
||||
storage guardrail and an explicit product boundary: users can undo the
|
||||
recent history, not retain an unbounded hidden copy of every document.
|
||||
"""
|
||||
if keep < 1:
|
||||
raise ValueError("keep must be at least one")
|
||||
resolved = resolve_user_id(user_id, method_name="DocumentRewriteVersionRepository.prune_source")
|
||||
async with self._sf() as session:
|
||||
clauses = [
|
||||
DocumentRewriteVersionRow.source_type == source_type,
|
||||
DocumentRewriteVersionRow.source_id == source_id,
|
||||
DocumentRewriteVersionRow.source_path == source_path,
|
||||
]
|
||||
if resolved is not None:
|
||||
clauses.append(DocumentRewriteVersionRow.user_id == resolved)
|
||||
stale_ids = list(
|
||||
(
|
||||
await session.scalars(
|
||||
select(DocumentRewriteVersionRow.id)
|
||||
.where(*clauses)
|
||||
.order_by(
|
||||
DocumentRewriteVersionRow.created_at.desc(),
|
||||
DocumentRewriteVersionRow.id.desc(),
|
||||
)
|
||||
.offset(keep)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not stale_ids:
|
||||
return 0
|
||||
result = await session.execute(
|
||||
delete(DocumentRewriteVersionRow).where(DocumentRewriteVersionRow.id.in_(stale_ids))
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
"""Persistence for iframe embed session → thread mappings."""
|
||||
|
||||
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
|
||||
from deerflow.persistence.embed_sessions.memory import MemoryEmbedSessionStore
|
||||
from deerflow.persistence.embed_sessions.sql import EmbedSessionRepository
|
||||
|
||||
__all__ = [
|
||||
"EmbedSessionStore",
|
||||
"MemoryEmbedSessionStore",
|
||||
"EmbedSessionRepository",
|
||||
"make_embed_session_store",
|
||||
]
|
||||
|
||||
|
||||
def make_embed_session_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryEmbedSessionStore()
|
||||
return EmbedSessionRepository(session_factory)
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
"""Abstract persistence API for iframe embed session → thread mappings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class EmbedSessionStore(ABC):
|
||||
@abstractmethod
|
||||
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
|
||||
"""Create or update the mapping for an embed session."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get(self, session_id: str) -> dict[str, Any] | None:
|
||||
"""Return ``{session_id, thread_id, created_at, updated_at}`` or None."""
|
||||
raise NotImplementedError
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"""In-memory ``EmbedSessionStore`` for tests / memory backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
|
||||
|
||||
|
||||
class MemoryEmbedSessionStore(EmbedSessionStore):
|
||||
def __init__(self) -> None:
|
||||
self._sessions: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
existing = self._sessions.get(session_id)
|
||||
if existing is None:
|
||||
record = {
|
||||
"session_id": session_id,
|
||||
"thread_id": thread_id,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
else:
|
||||
record = {
|
||||
**existing,
|
||||
"thread_id": thread_id,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._sessions[session_id] = record
|
||||
return copy.deepcopy(record)
|
||||
|
||||
async def get(self, session_id: str) -> dict[str, Any] | None:
|
||||
record = self._sessions.get(session_id)
|
||||
return copy.deepcopy(record) if record is not None else None
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
"""ORM model for iframe embed session → thread lookups."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class EmbedSessionRow(Base):
|
||||
__tablename__ = "embed_sessions"
|
||||
|
||||
# Opaque id generated by the parent page and passed into the iframe URL.
|
||||
session_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""SQLAlchemy-backed embed session repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.embed_sessions.base import EmbedSessionStore
|
||||
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
|
||||
|
||||
|
||||
def _row_to_dict(row: EmbedSessionRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
return data
|
||||
|
||||
|
||||
class EmbedSessionRepository(EmbedSessionStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def upsert(self, session_id: str, thread_id: str) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(EmbedSessionRow, session_id)
|
||||
now = datetime.now(UTC)
|
||||
if row is None:
|
||||
row = EmbedSessionRow(
|
||||
session_id=session_id,
|
||||
thread_id=thread_id,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.thread_id = thread_id
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _row_to_dict(row)
|
||||
|
||||
async def get(self, session_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(EmbedSessionRow, session_id)
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
@@ -0,0 +1,735 @@
|
||||
"""Async SQLAlchemy engine lifecycle management.
|
||||
|
||||
Initializes at Gateway startup, provides session factory for
|
||||
repositories, disposes at shutdown.
|
||||
|
||||
When database.backend="memory", init_engine is a no-op and
|
||||
get_session_factory() returns None. Repositories must check for
|
||||
None and fall back to in-memory implementations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
|
||||
def _json_serializer(obj: object) -> str:
|
||||
"""JSON serializer with ensure_ascii=False for Chinese character support."""
|
||||
return json.dumps(obj, ensure_ascii=False)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
# The event loop on which the async engine was created. asyncmy/asyncpg connection
|
||||
# pools bind their connections to the loop that opened them; awaiting a write on a
|
||||
# *different* loop reuses those connections cross-loop and raises
|
||||
# "got Future attached to a different loop". We capture the engine's home loop here so
|
||||
# best-effort writes coming from worker threads / other loops can be routed back to it
|
||||
# (see ``fire_and_forget_db_write``).
|
||||
_engine_loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
# These are the complete workflow-studio persistence tables owned by DeerFlow.
|
||||
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
|
||||
# wildcard: another application might share a schema and have its own table
|
||||
# with that prefix.
|
||||
_WORKFLOW_SCHEMA_TABLE_NAMES = (
|
||||
"workflow_definitions",
|
||||
"workflow_versions",
|
||||
"workflow_runs",
|
||||
"workflow_node_runs",
|
||||
"workflow_run_events",
|
||||
"workflow_run_artifacts",
|
||||
"workflow_data_sources",
|
||||
)
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
"""Allow offline deployments to place optional DB drivers in DEER_FLOW_HOME.
|
||||
|
||||
The offline Docker image keeps the Python environment inside the image, while
|
||||
``/app/backend/.deer-flow`` is a persistent bind mount. The MySQL switch
|
||||
script installs ``asyncmy`` into ``{DEER_FLOW_HOME}/python-vendor`` so the
|
||||
unchanged startup script can still import it after container recreation.
|
||||
"""
|
||||
try:
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
vendor_dir = runtime_home() / "python-vendor"
|
||||
except Exception:
|
||||
vendor_dir = Path(".deer-flow/python-vendor").resolve()
|
||||
|
||||
vendor = str(vendor_dir)
|
||||
if vendor_dir.exists() and vendor not in sys.path:
|
||||
sys.path.insert(0, vendor)
|
||||
|
||||
|
||||
def _mysql_driver_name(url: str) -> str:
|
||||
if url.startswith("mysql+aiomysql://"):
|
||||
return "aiomysql"
|
||||
return "asyncmy"
|
||||
|
||||
|
||||
async def _run_alembic_upgrade(url: str) -> None:
|
||||
"""Apply pending Alembic migrations against ``url`` (``alembic upgrade head``).
|
||||
|
||||
Runs after ``Base.metadata.create_all`` so brand-new deployments still get
|
||||
a complete schema even without Alembic, while existing deployments pick up
|
||||
in-place schema fixes (column widenings, new indexes, etc.) at the next
|
||||
restart — no manual ``alembic upgrade`` step required.
|
||||
|
||||
Notes:
|
||||
|
||||
* Each historical migration in ``migrations/versions/`` already guards
|
||||
itself with ``_has_table`` / ``_has_column``, so re-running them against
|
||||
a database that was originally built via ``create_all`` is a sequence of
|
||||
no-ops up to the first migration that actually has work to do.
|
||||
* The Alembic ``env.py`` calls ``asyncio.run`` internally, which cannot run
|
||||
inside an already-running loop — so we hop into a worker thread.
|
||||
* Failures here are logged but do not abort startup. The app remains
|
||||
operable on the ``create_all`` schema; the operator just won't get
|
||||
in-place fixes until they investigate.
|
||||
"""
|
||||
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in ("1", "true", "yes"):
|
||||
logger.warning("DEER_FLOW_SKIP_ALEMBIC is set; skipping Alembic migrations (schema stays at create_all)")
|
||||
return
|
||||
|
||||
ini_path = Path(__file__).parent / "migrations" / "alembic.ini"
|
||||
if not ini_path.exists():
|
||||
logger.warning("Alembic config not found at %s; skipping auto-migrate", ini_path)
|
||||
return
|
||||
|
||||
# Alembic's ``Config`` is backed by ConfigParser, which performs ``%``
|
||||
# interpolation on every option value. URL-encoded credentials such as
|
||||
# ``kland%4088124932`` (the ``@`` in ``kland@88124932``) trip that up with
|
||||
# ``ValueError: invalid interpolation syntax ... at position N``. Doubling
|
||||
# the percent signs neutralises the interpolation; ConfigParser un-doubles
|
||||
# them back to the original value when read.
|
||||
escaped_url = url.replace("%", "%%")
|
||||
|
||||
def _do_upgrade() -> None:
|
||||
# Import inside the worker so a missing Alembic install can't break
|
||||
# the engine import chain.
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
|
||||
cfg = Config(str(ini_path))
|
||||
cfg.set_main_option("sqlalchemy.url", escaped_url)
|
||||
# The ``script_location`` in alembic.ini is ``%(here)s`` which points
|
||||
# at the directory of alembic.ini itself — already correct for us.
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
# Bound the upgrade so a stuck DDL (e.g. MySQL metadata-lock wait) cannot
|
||||
# hang startup forever. Tunable via DEER_FLOW_ALEMBIC_TIMEOUT (seconds).
|
||||
# NOTE: asyncio.wait_for cancels the *await* on timeout; the worker thread
|
||||
# itself keeps running (Python can't kill threads). That's acceptable —
|
||||
# the orphan thread will exit when its MySQL connection eventually errors
|
||||
# or when the process restarts. The point here is to unblock startup.
|
||||
try:
|
||||
timeout_s = float(os.getenv("DEER_FLOW_ALEMBIC_TIMEOUT", "60"))
|
||||
except ValueError:
|
||||
timeout_s = 60.0
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.to_thread(_do_upgrade), timeout=timeout_s)
|
||||
logger.info("Alembic migrations applied (or already up-to-date)")
|
||||
except TimeoutError:
|
||||
logger.error(
|
||||
"Alembic upgrade did not finish within %.0fs; continuing startup on create_all schema. "
|
||||
"Likely cause: MySQL metadata-lock wait — check SHOW FULL PROCESSLIST for a blocking transaction.",
|
||||
timeout_s,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Auto-applying Alembic migrations failed; continuing with create_all schema")
|
||||
|
||||
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
|
||||
|
||||
The target database name is extracted from *url*. The connection is
|
||||
made to the default ``postgres`` database on the same server using
|
||||
``AUTOCOMMIT`` isolation (CREATE DATABASE cannot run inside a
|
||||
transaction).
|
||||
"""
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine.url import make_url
|
||||
|
||||
parsed = make_url(url)
|
||||
db_name = parsed.database
|
||||
if not db_name:
|
||||
raise ValueError("Cannot auto-create database: no database name in URL")
|
||||
|
||||
# Connect to the default 'postgres' database to issue CREATE DATABASE
|
||||
maint_url = parsed.set(database="postgres")
|
||||
maint_engine = create_async_engine(maint_url, isolation_level="AUTOCOMMIT")
|
||||
try:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
|
||||
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
async def _auto_create_mysql_db(url: str) -> None:
|
||||
"""Create the target MySQL database when it is missing.
|
||||
|
||||
``create_all`` can create tables, but MySQL requires the database/schema to
|
||||
exist before the application URL can connect. We connect to the built-in
|
||||
``mysql`` maintenance database on the same server and issue a conservative
|
||||
``CREATE DATABASE IF NOT EXISTS``.
|
||||
"""
|
||||
|
||||
parsed = make_url(url)
|
||||
db_name = parsed.database
|
||||
if not db_name:
|
||||
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
|
||||
|
||||
maint_url = parsed.set(database="mysql")
|
||||
maint_engine = create_async_engine(
|
||||
maint_url.render_as_string(hide_password=False),
|
||||
isolation_level="AUTOCOMMIT",
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
quoted = "`" + db_name.replace("`", "``") + "`"
|
||||
try:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(
|
||||
text(
|
||||
f"CREATE DATABASE IF NOT EXISTS {quoted} "
|
||||
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
||||
)
|
||||
)
|
||||
logger.info("Auto-created MySQL database if missing: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
"""Add ORM-declared columns that are missing from already-existing tables.
|
||||
|
||||
``Base.metadata.create_all`` creates brand-new tables but never patches
|
||||
existing ones, and Alembic auto-upgrade is disabled (see the commented-out
|
||||
``_run_alembic_upgrade`` call) — so on an existing database a new
|
||||
``mapped_column`` on an ORM model would otherwise error at first query
|
||||
(e.g. ``no such column: roundtable_drafts.task_id`` on SQLite).
|
||||
|
||||
This sweep compares each existing table against the ORM metadata and issues
|
||||
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
|
||||
safely (nullable, or carrying a server_default). Indexes covering a newly
|
||||
added column are created as well. Table-level ``UniqueConstraint`` objects
|
||||
that are declared on the model but missing from the table are also created
|
||||
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
|
||||
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
|
||||
on an existing database, weakening dedupe.
|
||||
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
|
||||
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
|
||||
``BeijingDateTime``) resolve to the right backend type. Everything is
|
||||
per-column best-effort: a failure is logged and startup continues.
|
||||
"""
|
||||
from sqlalchemy import UniqueConstraint
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
from sqlalchemy.schema import CreateColumn
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
try:
|
||||
import deerflow.persistence.models # noqa: F401 — register all ORM models on Base.metadata
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
inspector = sa_inspect(sync_conn)
|
||||
existing_tables = set(inspector.get_table_names())
|
||||
preparer = sync_conn.dialect.identifier_preparer
|
||||
|
||||
for table in Base.metadata.sorted_tables:
|
||||
if table.name not in existing_tables:
|
||||
continue
|
||||
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
|
||||
missing = [c for c in table.columns if c.name not in existing_cols]
|
||||
if not missing:
|
||||
continue
|
||||
|
||||
added: set[str] = set()
|
||||
for col in missing:
|
||||
if not col.nullable and col.server_default is None:
|
||||
logger.warning(
|
||||
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
|
||||
table.name,
|
||||
col.name,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
added.add(col.name)
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
|
||||
|
||||
if not added:
|
||||
continue
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
logger.info("Auto-created index %s on %s", idx.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
|
||||
|
||||
# ── 表级 UNIQUE 约束补齐 ──────────────────────────────────────────────────
|
||||
# create_all 只为全新表建 UniqueConstraint;既有表上新增声明的唯一约束(如
|
||||
# roundtable_jobs.active_dedupe_key)会静默缺失,去重/幂等的正确性就靠它,故
|
||||
# 这里显式补建。迁移可能用 CREATE UNIQUE INDEX(同名)实现唯一性,故同时查
|
||||
# get_unique_constraints 与 get_indexes 的名字集合,避免重复建。
|
||||
for table in Base.metadata.sorted_tables:
|
||||
if table.name not in existing_tables:
|
||||
continue
|
||||
named_uqs = [
|
||||
c for c in table.constraints
|
||||
if isinstance(c, UniqueConstraint) and c.name is not None
|
||||
]
|
||||
if not named_uqs:
|
||||
continue
|
||||
try:
|
||||
existing_uq_names = {
|
||||
uq["name"] for uq in inspector.get_unique_constraints(table.name) if uq.get("name")
|
||||
}
|
||||
except Exception:
|
||||
existing_uq_names = set()
|
||||
try:
|
||||
existing_idx_names = {
|
||||
ix["name"] for ix in inspector.get_indexes(table.name) if ix.get("name")
|
||||
}
|
||||
except Exception:
|
||||
existing_idx_names = set()
|
||||
for uq in named_uqs:
|
||||
if uq.name in existing_uq_names or uq.name in existing_idx_names:
|
||||
continue
|
||||
# 用 CREATE UNIQUE INDEX(而非 ALTER TABLE ADD CONSTRAINT):迁移本身也用
|
||||
# CREATE UNIQUE INDEX,且它在 SQLite/MySQL/PostgreSQL 都受支持(SQLite 不
|
||||
# 支持 ALTER TABLE ADD CONSTRAINT),功能等价地强制唯一性。
|
||||
cols = ", ".join(preparer.quote(c.name) for c in uq.columns)
|
||||
try:
|
||||
sync_conn.exec_driver_sql(
|
||||
f"CREATE UNIQUE INDEX {preparer.quote(uq.name)} "
|
||||
f"ON {preparer.quote(table.name)} ({cols})"
|
||||
)
|
||||
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
|
||||
|
||||
async def init_engine(
|
||||
backend: str,
|
||||
*,
|
||||
url: str = "",
|
||||
echo: bool = False,
|
||||
pool_size: int = 5,
|
||||
max_overflow: int = 10,
|
||||
pool_timeout: int = 30,
|
||||
pool_recycle: int = 1800,
|
||||
sqlite_dir: str = "",
|
||||
) -> None:
|
||||
"""Create the async engine and session factory, then auto-create tables.
|
||||
|
||||
Args:
|
||||
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
|
||||
echo: Echo SQL to log.
|
||||
pool_size: Base connection pool size (postgres/gauss/mysql).
|
||||
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
|
||||
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
|
||||
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
"""
|
||||
global _engine, _session_factory, _engine_loop
|
||||
|
||||
if backend == "memory":
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
|
||||
# Remember the loop that owns the engine's connection pool, so off-loop
|
||||
# best-effort writes (metrics from worker threads, etc.) can be routed here
|
||||
# instead of reusing pooled connections cross-loop.
|
||||
try:
|
||||
_engine_loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
_engine_loop = None
|
||||
|
||||
if backend in {"postgres", "gauss"}:
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
extra = "gauss" if backend == "gauss" else "postgres"
|
||||
raise ImportError(
|
||||
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
|
||||
f"Install it with:\n uv sync --extra {extra}\n"
|
||||
"Or switch to backend: sqlite in config.yaml for single-node deployment."
|
||||
) from None
|
||||
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
|
||||
"is not installed.\nInstall it with:\n uv sync --extra gauss"
|
||||
)
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
if find_spec(driver) is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'mysql' but the MySQL async driver is not installed.\n"
|
||||
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
|
||||
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
|
||||
)
|
||||
|
||||
if backend == "sqlite":
|
||||
import os
|
||||
|
||||
from sqlalchemy import event
|
||||
|
||||
os.makedirs(sqlite_dir or ".", exist_ok=True)
|
||||
_engine = create_async_engine(url, echo=echo, json_serializer=_json_serializer)
|
||||
|
||||
# Enable WAL on every new connection. SQLite PRAGMA settings are
|
||||
# per-connection, so we wire the listener instead of running PRAGMA
|
||||
# once at startup. WAL gives concurrent reads + writers without
|
||||
# blocking and is the standard recommendation for any production
|
||||
# SQLite deployment (TC-UPG-06 in AUTH_TEST_PLAN.md). The companion
|
||||
# ``synchronous=NORMAL`` is the safe-and-fast pairing — fsync only
|
||||
# at WAL checkpoint boundaries instead of every commit.
|
||||
# Note: we do not set PRAGMA busy_timeout here — Python's sqlite3
|
||||
# driver already defaults to a 5-second busy timeout (see the
|
||||
# ``timeout`` kwarg of ``sqlite3.connect``), and aiosqlite /
|
||||
# SQLAlchemy's aiosqlite dialect inherit that default. Setting
|
||||
# it again would be a no-op.
|
||||
@event.listens_for(_engine.sync_engine, "connect")
|
||||
def _enable_sqlite_wal(dbapi_conn, _record): # noqa: ARG001 — SQLAlchemy contract
|
||||
cursor = dbapi_conn.cursor()
|
||||
try:
|
||||
cursor.execute("PRAGMA journal_mode=WAL;")
|
||||
cursor.execute("PRAGMA synchronous=NORMAL;")
|
||||
cursor.execute("PRAGMA foreign_keys=ON;")
|
||||
finally:
|
||||
cursor.close()
|
||||
elif backend in {"postgres", "gauss"}:
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
elif backend == "mysql":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
# Auto-create tables for fresh deployments, then let Alembic apply any
|
||||
# in-place schema fixes on top (see ``_run_alembic_upgrade`` below). Both
|
||||
# are idempotent so repeating them across restarts is harmless.
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
# Import all models so Base.metadata discovers them.
|
||||
# When no models exist yet (scaffolding phase), this is a no-op.
|
||||
try:
|
||||
import deerflow.persistence.models # noqa: F401
|
||||
except ImportError:
|
||||
# Models package not yet available — tables won't be auto-created.
|
||||
# This is expected during initial scaffolding or minimal installs.
|
||||
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
|
||||
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
except Exception as exc:
|
||||
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
|
||||
# Database not yet created — attempt to auto-create it, then retry.
|
||||
await _auto_create_postgres_db(url)
|
||||
# Rebuild engine against the now-existing database
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, pool_recycle=pool_recycle, json_serializer=_json_serializer)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
|
||||
# Database not yet created. Create it, rebuild the app engine
|
||||
# against the target DB, then let SQLAlchemy create all tables.
|
||||
await _auto_create_mysql_db(url)
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
else:
|
||||
raise
|
||||
|
||||
# Patch existing tables with ORM-declared columns that ``create_all``
|
||||
# cannot add (it only creates brand-new tables). Best-effort: a failure is
|
||||
# logged and startup continues on the current schema.
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(_ensure_orm_columns_sync)
|
||||
except Exception:
|
||||
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
|
||||
|
||||
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
# await _run_alembic_upgrade(url)
|
||||
|
||||
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
|
||||
|
||||
|
||||
async def init_engine_from_config(config) -> None:
|
||||
"""Convenience: init engine from a DatabaseConfig object."""
|
||||
if config.backend == "memory":
|
||||
await init_engine("memory")
|
||||
return
|
||||
await init_engine(
|
||||
backend=config.backend,
|
||||
url=config.app_sqlalchemy_url,
|
||||
echo=config.echo_sql,
|
||||
pool_size=config.pool_size,
|
||||
max_overflow=getattr(config, "max_overflow", 10),
|
||||
pool_timeout=getattr(config, "pool_timeout", 30),
|
||||
pool_recycle=getattr(config, "pool_recycle", 1800),
|
||||
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
|
||||
)
|
||||
|
||||
|
||||
def get_session_factory() -> async_sessionmaker[AsyncSession] | None:
|
||||
"""Return the async session factory, or None if backend=memory."""
|
||||
return _session_factory
|
||||
|
||||
|
||||
def get_engine() -> AsyncEngine | None:
|
||||
"""Return the async engine, or None if not initialized."""
|
||||
return _engine
|
||||
|
||||
|
||||
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
|
||||
"""Return the event loop that owns the engine's connection pool (or None)."""
|
||||
return _engine_loop
|
||||
|
||||
|
||||
async def rebuild_workflow_schema_for_mysql() -> list[str]:
|
||||
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
|
||||
|
||||
This is a deliberately narrow recovery operation for an unused Workflow
|
||||
Studio installation. It never uses a prefix wildcard, never changes the
|
||||
selected database, and refuses to run unless the engine is connected to
|
||||
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
|
||||
|
||||
Returns the allow-listed workflow tables that existed before the rebuild.
|
||||
"""
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
|
||||
if engine.dialect.name != "mysql":
|
||||
raise RuntimeError(
|
||||
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
|
||||
)
|
||||
|
||||
expected_database = engine.url.database
|
||||
if not expected_database:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
|
||||
|
||||
# Make the operation safe when this helper is invoked independently from
|
||||
# normal app startup as well.
|
||||
import deerflow.persistence.models # noqa: F401 — register ORM models
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
|
||||
if missing_models:
|
||||
raise RuntimeError(
|
||||
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
|
||||
)
|
||||
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
|
||||
|
||||
def _rebuild(sync_conn) -> list[str]:
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
|
||||
if current_database != expected_database:
|
||||
raise RuntimeError(
|
||||
"Refusing workflow schema rebuild: connected database "
|
||||
f"{current_database!r} does not match configured database {expected_database!r}"
|
||||
)
|
||||
|
||||
existing = set(sa_inspect(sync_conn).get_table_names())
|
||||
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
|
||||
# SQLAlchemy orders parent/child tables around the foreign key from
|
||||
# workflow_versions to workflow_definitions. Only tables in the exact
|
||||
# allow-list above are ever passed to drop_all/create_all.
|
||||
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
return present
|
||||
|
||||
async with engine.begin() as conn:
|
||||
rebuilt = await conn.run_sync(_rebuild)
|
||||
|
||||
logger.warning(
|
||||
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
|
||||
expected_database,
|
||||
", ".join(rebuilt) if rebuilt else "none (created fresh)",
|
||||
)
|
||||
return rebuilt
|
||||
|
||||
|
||||
def fire_and_forget_db_write(coro) -> None:
|
||||
"""Schedule a best-effort DB write on the engine's home loop, from any loop/thread.
|
||||
|
||||
The async engine's connection pool is bound to the loop that created it
|
||||
(``init_engine``, normally the Gateway's main loop). Running a write on a
|
||||
*different* loop — e.g. a model/tool call that LangGraph executed in a worker
|
||||
thread, which then does ``asyncio.run(store.record(...))`` — reuses those pooled
|
||||
connections cross-loop and raises ``got Future attached to a different loop``.
|
||||
Under concurrency this happens constantly (many calls land in worker threads at
|
||||
once), spamming the log and churning the pool; single-threaded it almost never does.
|
||||
|
||||
This routes every such write back onto the engine's home loop:
|
||||
- already on that loop -> ``create_task`` (in-loop fire-and-forget);
|
||||
- on another loop/thread -> ``run_coroutine_threadsafe`` to the home loop.
|
||||
|
||||
The write itself stays best-effort: if no engine loop is captured (memory backend
|
||||
/ not initialized) the coroutine is closed (avoids a "never awaited" warning) and
|
||||
the call is a no-op. Never raises.
|
||||
"""
|
||||
loop = _engine_loop
|
||||
if loop is None or loop.is_closed():
|
||||
close = getattr(coro, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
return
|
||||
try:
|
||||
running = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
running = None
|
||||
try:
|
||||
if running is loop:
|
||||
loop.create_task(coro)
|
||||
else:
|
||||
asyncio.run_coroutine_threadsafe(coro, loop)
|
||||
except Exception: # noqa: BLE001 — best-effort; scheduling must never break the caller
|
||||
close = getattr(coro, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
logger.debug("fire_and_forget_db_write: failed to schedule DB write", exc_info=True)
|
||||
|
||||
|
||||
def _run_coro_on_throwaway_loop(coro):
|
||||
"""Last-resort: run *coro* to completion on a fresh event loop.
|
||||
|
||||
Only reached when there is no usable engine home loop, or when the caller is
|
||||
*synchronously* inside the home loop (routing back would deadlock). This
|
||||
touches the shared connection pool from a throwaway loop, so it can still leak
|
||||
a dead-loop-bound connection — but the common worker-thread path no longer
|
||||
gets here (see ``run_db_blocking``).
|
||||
"""
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
except RuntimeError:
|
||||
in_loop = False
|
||||
|
||||
if in_loop:
|
||||
import concurrent.futures
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(asyncio.run, coro).result()
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def run_db_blocking(coro, *, timeout: float | None = 30.0):
|
||||
"""Run an awaited DB coroutine from sync code, safe w.r.t. the engine's home loop.
|
||||
|
||||
The blocking-read sibling of :func:`fire_and_forget_db_write`: where that one
|
||||
drops the result, this returns it — used by the sync→async bridges that must
|
||||
read a row before proceeding (thread-path owner resolution, always-on skills,
|
||||
skill visibility).
|
||||
|
||||
The async engine's connection pool is bound to the loop that created it
|
||||
(``init_engine``). Running a DB coroutine on a *throwaway* loop
|
||||
(``asyncio.run``) checks out pooled connections cross-loop and **leaks** them:
|
||||
the connection ends up bound to a loop that is then closed, and when the pool
|
||||
later recycles/disposes it, asyncmy's ``_terminate_graceful_close`` fires on the
|
||||
wrong loop and raises ``got Future <...> attached to a different loop``. Under
|
||||
concurrency (scheduler / subagent / sandbox-tool worker threads each spinning a
|
||||
fresh loop) this floods the logs.
|
||||
|
||||
So when we are **off** the home loop — the common case, a worker thread with no
|
||||
running loop — route the coroutine onto the home loop and block the *caller*
|
||||
thread for its result. The pool is only ever touched on its own loop, so nothing
|
||||
leaks. Only when there is no home loop, or we are synchronously inside it (where
|
||||
blocking would deadlock), do we fall back to a throwaway loop.
|
||||
|
||||
Never swallows the coroutine's own exception — the sync bridges wrap this call in
|
||||
their own try/except and degrade gracefully.
|
||||
"""
|
||||
loop = _engine_loop
|
||||
try:
|
||||
running = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
running = None
|
||||
if loop is not None and not loop.is_closed() and running is not loop:
|
||||
# Off the home loop (worker thread, or a different loop): execute the coro
|
||||
# on the engine's loop and block this thread for the result. asyncmy's
|
||||
# pooled connections stay on their own loop, so none leak cross-loop.
|
||||
return asyncio.run_coroutine_threadsafe(coro, loop).result(timeout)
|
||||
# No home loop, or we're already synchronously inside it: last resort.
|
||||
return _run_coro_on_throwaway_loop(coro)
|
||||
|
||||
|
||||
async def close_engine() -> None:
|
||||
"""Dispose the engine, release all connections."""
|
||||
global _engine, _session_factory, _engine_loop
|
||||
if _engine is not None:
|
||||
await _engine.dispose()
|
||||
logger.info("Persistence engine closed")
|
||||
_engine = None
|
||||
_session_factory = None
|
||||
_engine_loop = None
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
"""Enterprise-research task persistence."""
|
||||
|
||||
from deerflow.persistence.enterprise_research.base import EnterpriseResearchReportJobStore, EnterpriseResearchTaskStore
|
||||
from deerflow.persistence.enterprise_research.memory import MemoryEnterpriseResearchReportJobStore, MemoryEnterpriseResearchTaskStore
|
||||
from deerflow.persistence.enterprise_research.model import EnterpriseResearchReportJobRow, EnterpriseResearchTaskRow
|
||||
from deerflow.persistence.enterprise_research.sql import EnterpriseResearchReportJobRepository, EnterpriseResearchTaskRepository
|
||||
|
||||
__all__ = [
|
||||
"EnterpriseResearchTaskRow",
|
||||
"EnterpriseResearchReportJobRow",
|
||||
"EnterpriseResearchTaskRepository",
|
||||
"EnterpriseResearchReportJobRepository",
|
||||
"EnterpriseResearchTaskStore",
|
||||
"EnterpriseResearchReportJobStore",
|
||||
"MemoryEnterpriseResearchTaskStore",
|
||||
"MemoryEnterpriseResearchReportJobStore",
|
||||
"make_enterprise_research_task_store",
|
||||
"make_enterprise_research_report_job_store",
|
||||
]
|
||||
|
||||
|
||||
def make_enterprise_research_task_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryEnterpriseResearchTaskStore()
|
||||
return EnterpriseResearchTaskRepository(session_factory)
|
||||
|
||||
|
||||
def make_enterprise_research_report_job_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryEnterpriseResearchReportJobStore()
|
||||
return EnterpriseResearchReportJobRepository(session_factory)
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
"""Persistence contract for the independent enterprise-research workbench."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class EnterpriseResearchTaskStore(ABC):
|
||||
"""Owner-scoped task storage; evidence snapshots are DeerFlow-owned."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_tasks(self, user_id: str, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_task(self, task_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_task(self, task_id: str, user_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class EnterpriseResearchReportJobStore(ABC):
|
||||
"""Owner-scoped, frozen-input jobs for enterprise report writing."""
|
||||
|
||||
@abstractmethod
|
||||
async def create_job(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_job(self, job_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_active_for_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def claim_job(self, job_id: str, user_id: str) -> bool:
|
||||
"""Atomically move one queued job to running."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def requeue_job(self, job_id: str, user_id: str, *, expected_updated_at: str | None) -> bool:
|
||||
"""Atomically requeue the exact running snapshot seen during recovery."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def transition_job(
|
||||
self, job_id: str, user_id: str, *, expected_statuses: set[str], data: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
"""Apply a terminal/state transition only from one of the expected states."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_job(self, job_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_jobs(self, task_id: str, user_id: str, *, limit: int = 10) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_recoverable(self, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
"""In-memory fallback for enterprise-research tasks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.enterprise_research.base import EnterpriseResearchReportJobStore, EnterpriseResearchTaskStore
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class MemoryEnterpriseResearchTaskStore(EnterpriseResearchTaskStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def list_tasks(self, user_id: str, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
rows = [dict(row) for row in self._rows.values() if row["user_id"] == user_id]
|
||||
return sorted(rows, key=lambda row: row["updated_at"], reverse=True)[:limit]
|
||||
|
||||
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(task_id)
|
||||
return dict(row) if row is not None and row["user_id"] == user_id else None
|
||||
|
||||
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = _now()
|
||||
row = {
|
||||
"id": str(data["id"]),
|
||||
"user_id": str(data["user_id"]),
|
||||
"subject": str(data["subject"]),
|
||||
"focus": str(data.get("focus") or ""),
|
||||
"template_id": str(data["template_id"]),
|
||||
"knowledge_base_ids": list(data.get("knowledge_base_ids") or []),
|
||||
"queries": list(data.get("queries") or []),
|
||||
"selected_source_ids": list(data.get("selected_source_ids") or []),
|
||||
"sources": list(data.get("sources") or []),
|
||||
"plan_markdown": data.get("plan_markdown"),
|
||||
"report_markdown": data.get("report_markdown"),
|
||||
"report_summary": data.get("report_summary"),
|
||||
"report_model_name": data.get("report_model_name"),
|
||||
"active_report_job_id": data.get("active_report_job_id"),
|
||||
"status": str(data.get("status") or "draft"),
|
||||
"warning": data.get("warning"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._rows[row["id"]] = row
|
||||
return dict(row)
|
||||
|
||||
async def update_task(self, task_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._rows.get(task_id)
|
||||
if row is None or row["user_id"] != user_id:
|
||||
return None
|
||||
for key in (
|
||||
"status", "warning", "sources", "selected_source_ids", "plan_markdown",
|
||||
"report_markdown", "report_summary", "report_model_name", "active_report_job_id",
|
||||
):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
|
||||
async def delete_task(self, task_id: str, user_id: str) -> bool:
|
||||
row = self._rows.get(task_id)
|
||||
if row is None or row["user_id"] != user_id:
|
||||
return False
|
||||
del self._rows[task_id]
|
||||
return True
|
||||
|
||||
|
||||
class MemoryEnterpriseResearchReportJobStore(EnterpriseResearchReportJobStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def create_job(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = _now()
|
||||
status = str(data.get("status") or "queued")
|
||||
active_dedupe_key = (
|
||||
f"{data['user_id']}:{data['task_id']}" if status in {"queued", "running"} else None
|
||||
)
|
||||
if active_dedupe_key:
|
||||
existing = next(
|
||||
(row for row in self._rows.values() if row.get("active_dedupe_key") == active_dedupe_key),
|
||||
None,
|
||||
)
|
||||
if existing is not None:
|
||||
return dict(existing)
|
||||
row = {
|
||||
"id": str(data["id"]), "task_id": str(data["task_id"]), "user_id": str(data["user_id"]),
|
||||
"status": status, "phase": str(data.get("phase") or "queued"),
|
||||
"model_name": data.get("model_name"), "action": str(data.get("action") or "initial"),
|
||||
"active_dedupe_key": active_dedupe_key,
|
||||
"instruction": data.get("instruction"), "style": data.get("style"),
|
||||
"target_length": data.get("target_length"), "plan_snapshot": str(data["plan_snapshot"]),
|
||||
"sources": list(data.get("sources") or []), "report_markdown": str(data.get("report_markdown") or ""),
|
||||
"original_report": data.get("original_report"),
|
||||
"report_summary": data.get("report_summary"), "error_message": data.get("error_message"),
|
||||
"created_at": now, "updated_at": now,
|
||||
}
|
||||
self._rows[row["id"]] = row
|
||||
return dict(row)
|
||||
|
||||
async def get_job(self, job_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(job_id)
|
||||
return dict(row) if row is not None and row["user_id"] == user_id else None
|
||||
|
||||
async def get_active_for_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
rows = [row for row in self._rows.values() if row["task_id"] == task_id and row["user_id"] == user_id and row["status"] in {"queued", "running"}]
|
||||
return dict(max(rows, key=lambda row: row["updated_at"])) if rows else None
|
||||
|
||||
async def claim_job(self, job_id: str, user_id: str) -> bool:
|
||||
row = self._rows.get(job_id)
|
||||
if row is None or row["user_id"] != user_id or row["status"] != "queued":
|
||||
return False
|
||||
row["status"] = "running"
|
||||
row["updated_at"] = _now()
|
||||
return True
|
||||
|
||||
async def requeue_job(self, job_id: str, user_id: str, *, expected_updated_at: str | None) -> bool:
|
||||
row = self._rows.get(job_id)
|
||||
if (
|
||||
row is None
|
||||
or row["user_id"] != user_id
|
||||
or row["status"] != "running"
|
||||
or row.get("updated_at") != expected_updated_at
|
||||
):
|
||||
return False
|
||||
row.update({"status": "queued", "phase": "queued", "report_markdown": "", "error_message": None})
|
||||
row["updated_at"] = _now()
|
||||
return True
|
||||
|
||||
async def transition_job(
|
||||
self, job_id: str, user_id: str, *, expected_statuses: set[str], data: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
row = self._rows.get(job_id)
|
||||
if row is None or row["user_id"] != user_id or row["status"] not in expected_statuses:
|
||||
return None
|
||||
for key in (
|
||||
"status", "phase", "report_markdown", "report_summary", "error_message", "model_name",
|
||||
"action", "instruction", "style", "target_length", "original_report",
|
||||
):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
if data.get("status") in {"completed", "failed", "cancelled"}:
|
||||
row["active_dedupe_key"] = None
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
|
||||
async def update_job(self, job_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._rows.get(job_id)
|
||||
if row is None or row["user_id"] != user_id:
|
||||
return None
|
||||
for key in (
|
||||
"status", "phase", "report_markdown", "report_summary", "error_message", "model_name",
|
||||
"action", "instruction", "style", "target_length", "original_report",
|
||||
):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
if data.get("status") in {"completed", "failed", "cancelled"}:
|
||||
row["active_dedupe_key"] = None
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
|
||||
async def list_jobs(self, task_id: str, user_id: str, *, limit: int = 10) -> list[dict[str, Any]]:
|
||||
rows = [dict(row) for row in self._rows.values() if row["task_id"] == task_id and row["user_id"] == user_id]
|
||||
return sorted(rows, key=lambda row: row["updated_at"], reverse=True)[:limit]
|
||||
|
||||
async def list_recoverable(self, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
rows = [dict(row) for row in self._rows.values() if row["status"] in {"queued", "running"}]
|
||||
return sorted(rows, key=lambda row: row["updated_at"])[:limit]
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
"""ORM rows for the independent enterprise-research workbench."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class EnterpriseResearchTaskRow(Base):
|
||||
"""A user-owned research task plus its bounded WeKnora evidence snapshot."""
|
||||
|
||||
__tablename__ = "enterprise_research_tasks"
|
||||
__table_args__ = (Index("ix_enterprise_research_tasks_owner_updated", "user_id", "updated_at"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
subject: Mapped[str] = mapped_column(String(256), nullable=False)
|
||||
focus: Mapped[str] = mapped_column(Text, nullable=False, default="", server_default="")
|
||||
template_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
knowledge_base_ids: Mapped[list[str]] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
queries: Mapped[list[str]] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
# Nullable keeps upgrades safe: the generic schema reconciler can add this
|
||||
# field to a phase-one table without an unsafe MySQL TEXT default.
|
||||
selected_source_ids: Mapped[list[str] | None] = mapped_column(PortableJSON(), nullable=True, default=list)
|
||||
# A WeKnora result can contain a long document chunk. MySQL TEXT is only
|
||||
# 64 KiB, so evidence uses LONGTEXT through PortableLongText.
|
||||
sources_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="[]", server_default="[]")
|
||||
plan_markdown: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
report_markdown: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
report_summary: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
report_model_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
active_report_job_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="draft", server_default="draft")
|
||||
warning: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
|
||||
class EnterpriseResearchReportJobRow(Base):
|
||||
"""A report job whose plan/evidence are frozen when the user starts it."""
|
||||
|
||||
__tablename__ = "enterprise_research_report_jobs"
|
||||
__table_args__ = (
|
||||
Index("ix_enterprise_research_report_jobs_task_owner_updated", "task_id", "user_id", "updated_at"),
|
||||
UniqueConstraint("active_dedupe_key", name="uq_enterprise_research_report_jobs_active"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", server_default="queued")
|
||||
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", server_default="queued")
|
||||
model_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
# Present only for queued/running rows. NULL terminal rows keep unlimited
|
||||
# history while this key rejects cross-process duplicate starts.
|
||||
active_dedupe_key: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
action: Mapped[str | None] = mapped_column(String(32), nullable=True, default="initial")
|
||||
instruction: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
style: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
target_length: Mapped[int | None] = mapped_column(nullable=True)
|
||||
plan_snapshot: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
|
||||
sources_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
|
||||
original_report: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
report_markdown: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="", server_default="")
|
||||
report_summary: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
+320
@@ -0,0 +1,320 @@
|
||||
"""SQLAlchemy store for enterprise-research tasks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.enterprise_research.base import EnterpriseResearchReportJobStore, EnterpriseResearchTaskStore
|
||||
from deerflow.persistence.enterprise_research.model import EnterpriseResearchReportJobRow, EnterpriseResearchTaskRow
|
||||
|
||||
|
||||
def _to_dict(row: EnterpriseResearchTaskRow) -> dict[str, Any]:
|
||||
try:
|
||||
sources = json.loads(row.sources_json or "[]")
|
||||
except (TypeError, ValueError):
|
||||
sources = []
|
||||
data: dict[str, Any] = {
|
||||
"id": row.id,
|
||||
"user_id": row.user_id,
|
||||
"subject": row.subject,
|
||||
"focus": row.focus,
|
||||
"template_id": row.template_id,
|
||||
"knowledge_base_ids": list(row.knowledge_base_ids or []),
|
||||
"queries": list(row.queries or []),
|
||||
"selected_source_ids": list(row.selected_source_ids or []),
|
||||
"sources": sources if isinstance(sources, list) else [],
|
||||
"plan_markdown": row.plan_markdown,
|
||||
"report_markdown": row.report_markdown,
|
||||
"report_summary": row.report_summary,
|
||||
"report_model_name": row.report_model_name,
|
||||
"active_report_job_id": row.active_report_job_id,
|
||||
"status": row.status,
|
||||
"warning": row.warning,
|
||||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||||
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
class EnterpriseResearchTaskRepository(EnterpriseResearchTaskStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_tasks(self, user_id: str, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(EnterpriseResearchTaskRow)
|
||||
.where(EnterpriseResearchTaskRow.user_id == user_id)
|
||||
.order_by(EnterpriseResearchTaskRow.updated_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
stmt = select(EnterpriseResearchTaskRow).where(
|
||||
EnterpriseResearchTaskRow.id == task_id,
|
||||
EnterpriseResearchTaskRow.user_id == user_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = EnterpriseResearchTaskRow(
|
||||
id=str(data["id"]),
|
||||
user_id=str(data["user_id"]),
|
||||
subject=str(data["subject"]),
|
||||
focus=str(data.get("focus") or ""),
|
||||
template_id=str(data["template_id"]),
|
||||
knowledge_base_ids=list(data.get("knowledge_base_ids") or []),
|
||||
queries=list(data.get("queries") or []),
|
||||
selected_source_ids=list(data.get("selected_source_ids") or []),
|
||||
sources_json=json.dumps(list(data.get("sources") or []), ensure_ascii=False),
|
||||
plan_markdown=data.get("plan_markdown"),
|
||||
report_markdown=data.get("report_markdown"),
|
||||
report_summary=data.get("report_summary"),
|
||||
report_model_name=data.get("report_model_name"),
|
||||
active_report_job_id=data.get("active_report_job_id"),
|
||||
status=str(data.get("status") or "draft"),
|
||||
warning=data.get("warning"),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
# Do not refresh after commit: a MySQL read/write split can route it
|
||||
# to a lagging replica. The flushed in-memory row is authoritative.
|
||||
await session.flush()
|
||||
payload = _to_dict(row)
|
||||
await session.commit()
|
||||
return payload
|
||||
|
||||
async def update_task(self, task_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
stmt = select(EnterpriseResearchTaskRow).where(
|
||||
EnterpriseResearchTaskRow.id == task_id,
|
||||
EnterpriseResearchTaskRow.user_id == user_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
if "status" in data:
|
||||
row.status = str(data["status"])
|
||||
if "warning" in data:
|
||||
row.warning = data["warning"]
|
||||
if "sources" in data:
|
||||
row.sources_json = json.dumps(list(data["sources"] or []), ensure_ascii=False)
|
||||
if "selected_source_ids" in data:
|
||||
row.selected_source_ids = list(data["selected_source_ids"] or [])
|
||||
if "plan_markdown" in data:
|
||||
row.plan_markdown = data["plan_markdown"]
|
||||
for key in ("report_markdown", "report_summary", "report_model_name", "active_report_job_id"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.flush()
|
||||
payload = _to_dict(row)
|
||||
await session.commit()
|
||||
return payload
|
||||
|
||||
async def delete_task(self, task_id: str, user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(EnterpriseResearchTaskRow).where(
|
||||
EnterpriseResearchTaskRow.id == task_id,
|
||||
EnterpriseResearchTaskRow.user_id == user_id,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
|
||||
def _job_to_dict(row: EnterpriseResearchReportJobRow) -> dict[str, Any]:
|
||||
try:
|
||||
sources = json.loads(row.sources_json or "[]")
|
||||
except (TypeError, ValueError):
|
||||
sources = []
|
||||
return {
|
||||
"id": row.id, "task_id": row.task_id, "user_id": row.user_id, "status": row.status,
|
||||
"phase": row.phase, "model_name": row.model_name, "action": row.action or "initial",
|
||||
"instruction": row.instruction, "style": row.style, "target_length": row.target_length,
|
||||
"plan_snapshot": row.plan_snapshot,
|
||||
"sources": sources if isinstance(sources, list) else [], "report_markdown": row.report_markdown,
|
||||
"original_report": row.original_report,
|
||||
"report_summary": row.report_summary, "error_message": row.error_message,
|
||||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||||
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
class EnterpriseResearchReportJobRepository(EnterpriseResearchReportJobStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def create_job(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
status = str(data.get("status") or "queued")
|
||||
active_dedupe_key = (
|
||||
f"{data['user_id']}:{data['task_id']}" if status in {"queued", "running"} else None
|
||||
)
|
||||
row = EnterpriseResearchReportJobRow(
|
||||
id=str(data["id"]), task_id=str(data["task_id"]), user_id=str(data["user_id"]),
|
||||
status=status, phase=str(data.get("phase") or "queued"),
|
||||
model_name=data.get("model_name"), action=str(data.get("action") or "initial"),
|
||||
active_dedupe_key=active_dedupe_key,
|
||||
instruction=data.get("instruction"), style=data.get("style"), target_length=data.get("target_length"),
|
||||
plan_snapshot=str(data["plan_snapshot"]),
|
||||
sources_json=json.dumps(list(data.get("sources") or []), ensure_ascii=False),
|
||||
original_report=data.get("original_report"),
|
||||
report_markdown=str(data.get("report_markdown") or ""), report_summary=data.get("report_summary"),
|
||||
error_message=data.get("error_message"), created_at=now, updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
try:
|
||||
await session.flush()
|
||||
payload = _job_to_dict(row)
|
||||
await session.commit()
|
||||
return payload
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
if not active_dedupe_key:
|
||||
raise
|
||||
existing = (await session.execute(
|
||||
select(EnterpriseResearchReportJobRow)
|
||||
.where(EnterpriseResearchReportJobRow.active_dedupe_key == active_dedupe_key)
|
||||
.limit(1)
|
||||
)).scalar_one_or_none()
|
||||
if existing is None:
|
||||
raise
|
||||
return _job_to_dict(existing)
|
||||
|
||||
async def get_job(self, job_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(EnterpriseResearchReportJobRow).where(
|
||||
EnterpriseResearchReportJobRow.id == job_id, EnterpriseResearchReportJobRow.user_id == user_id,
|
||||
))).scalar_one_or_none()
|
||||
return _job_to_dict(row) if row is not None else None
|
||||
|
||||
async def get_active_for_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(EnterpriseResearchReportJobRow).where(
|
||||
EnterpriseResearchReportJobRow.task_id == task_id,
|
||||
EnterpriseResearchReportJobRow.user_id == user_id,
|
||||
EnterpriseResearchReportJobRow.status.in_(["queued", "running"]),
|
||||
).order_by(EnterpriseResearchReportJobRow.updated_at.desc()).limit(1))).scalar_one_or_none()
|
||||
return _job_to_dict(row) if row is not None else None
|
||||
|
||||
async def claim_job(self, job_id: str, user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(EnterpriseResearchReportJobRow)
|
||||
.where(
|
||||
EnterpriseResearchReportJobRow.id == job_id,
|
||||
EnterpriseResearchReportJobRow.user_id == user_id,
|
||||
EnterpriseResearchReportJobRow.status == "queued",
|
||||
)
|
||||
.values(status="running", updated_at=datetime.now(UTC))
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def requeue_job(self, job_id: str, user_id: str, *, expected_updated_at: str | None) -> bool:
|
||||
if not expected_updated_at:
|
||||
return False
|
||||
try:
|
||||
expected = datetime.fromisoformat(expected_updated_at)
|
||||
except ValueError:
|
||||
return False
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(EnterpriseResearchReportJobRow)
|
||||
.where(
|
||||
EnterpriseResearchReportJobRow.id == job_id,
|
||||
EnterpriseResearchReportJobRow.user_id == user_id,
|
||||
EnterpriseResearchReportJobRow.status == "running",
|
||||
EnterpriseResearchReportJobRow.updated_at == expected,
|
||||
)
|
||||
.values(
|
||||
status="queued",
|
||||
phase="queued",
|
||||
report_markdown="",
|
||||
error_message=None,
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def transition_job(
|
||||
self, job_id: str, user_id: str, *, expected_statuses: set[str], data: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
allowed = {
|
||||
"status", "phase", "report_markdown", "report_summary", "error_message", "model_name",
|
||||
"action", "instruction", "style", "target_length", "original_report",
|
||||
}
|
||||
values = {key: value for key, value in data.items() if key in allowed}
|
||||
if data.get("status") in {"completed", "failed", "cancelled"}:
|
||||
values["active_dedupe_key"] = None
|
||||
values["updated_at"] = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(EnterpriseResearchReportJobRow)
|
||||
.where(
|
||||
EnterpriseResearchReportJobRow.id == job_id,
|
||||
EnterpriseResearchReportJobRow.user_id == user_id,
|
||||
EnterpriseResearchReportJobRow.status.in_(expected_statuses),
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
if not result.rowcount:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
return await self.get_job(job_id, user_id)
|
||||
|
||||
async def update_job(self, job_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(EnterpriseResearchReportJobRow).where(
|
||||
EnterpriseResearchReportJobRow.id == job_id, EnterpriseResearchReportJobRow.user_id == user_id,
|
||||
))).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
for key in (
|
||||
"status", "phase", "report_markdown", "report_summary", "error_message", "model_name",
|
||||
"action", "instruction", "style", "target_length", "original_report",
|
||||
):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
if data.get("status") in {"completed", "failed", "cancelled"}:
|
||||
row.active_dedupe_key = None
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.flush()
|
||||
payload = _job_to_dict(row)
|
||||
await session.commit()
|
||||
return payload
|
||||
|
||||
async def list_jobs(self, task_id: str, user_id: str, *, limit: int = 10) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(select(EnterpriseResearchReportJobRow).where(
|
||||
EnterpriseResearchReportJobRow.task_id == task_id, EnterpriseResearchReportJobRow.user_id == user_id,
|
||||
).order_by(EnterpriseResearchReportJobRow.updated_at.desc()).limit(limit))).scalars()
|
||||
return [_job_to_dict(row) for row in rows]
|
||||
|
||||
async def list_recoverable(self, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(
|
||||
select(EnterpriseResearchReportJobRow)
|
||||
.where(EnterpriseResearchReportJobRow.status.in_(["queued", "running"]))
|
||||
.order_by(EnterpriseResearchReportJobRow.updated_at.asc())
|
||||
.limit(limit)
|
||||
)).scalars()
|
||||
return [_job_to_dict(row) for row in rows]
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
"""Feedback persistence — ORM and SQL repository."""
|
||||
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.feedback.sql import FeedbackRepository
|
||||
|
||||
__all__ = ["FeedbackRepository", "FeedbackRow"]
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
"""ORM model for user feedback on runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
# Prefix-unique instead of UniqueConstraint: a plain unique over three
|
||||
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
|
||||
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
|
||||
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
|
||||
# mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
|
||||
|
||||
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
message_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# message_id is an optional RunEventStore event identifier —
|
||||
# allows feedback to target a specific message or the entire run
|
||||
|
||||
rating: Mapped[int] = mapped_column(nullable=False)
|
||||
# +1 (thumbs-up) or -1 (thumbs-down)
|
||||
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional text feedback from the user
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
"""SQLAlchemy-backed feedback storage.
|
||||
|
||||
Each method acquires its own short-lived session.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
|
||||
class FeedbackRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: FeedbackRow) -> dict:
|
||||
d = row.to_dict()
|
||||
val = d.get("created_at")
|
||||
if isinstance(val, datetime):
|
||||
d["created_at"] = val.isoformat()
|
||||
return d
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
thread_id: str,
|
||||
rating: int,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
message_id: str | None = None,
|
||||
comment: str | None = None,
|
||||
) -> dict:
|
||||
"""Create a feedback record. rating must be +1 or -1."""
|
||||
if rating not in (1, -1):
|
||||
raise ValueError(f"rating must be +1 or -1, got {rating}")
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.create")
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
message_id=message_id,
|
||||
rating=rating,
|
||||
comment=comment,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
feedback_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.get")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(FeedbackRow, feedback_id)
|
||||
if row is None:
|
||||
return None
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def list_by_run(
|
||||
self,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_run")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def list_by_thread(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
feedback_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(FeedbackRow, feedback_id)
|
||||
if row is None:
|
||||
return False
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def upsert(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
thread_id: str,
|
||||
rating: int,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
comment: str | None = None,
|
||||
) -> dict:
|
||||
"""Create or update feedback for (thread_id, run_id, user_id). rating must be +1 or -1."""
|
||||
if rating not in (1, -1):
|
||||
raise ValueError(f"rating must be +1 or -1, got {rating}")
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.upsert")
|
||||
async with self._sf() as session:
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is not None:
|
||||
row.rating = rating
|
||||
row.comment = comment
|
||||
row.created_at = datetime.now(UTC)
|
||||
else:
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
rating=rating,
|
||||
comment=comment,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def delete_by_run(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
"""Delete the current user's feedback for a run. Returns True if a record was deleted."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete_by_run")
|
||||
async with self._sf() as session:
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_by_thread_grouped(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict[str, dict]:
|
||||
"""Return feedback grouped by run_id for a thread: {run_id: feedback_dict}."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread_grouped")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return {row.run_id: self._row_to_dict(row) for row in result.scalars()}
|
||||
|
||||
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict:
|
||||
"""Aggregate feedback stats for a run using database-side counting."""
|
||||
stmt = select(
|
||||
func.count().label("total"),
|
||||
func.coalesce(func.sum(case((FeedbackRow.rating == 1, 1), else_=0)), 0).label("positive"),
|
||||
func.coalesce(func.sum(case((FeedbackRow.rating == -1, 1), else_=0)), 0).label("negative"),
|
||||
).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).one()
|
||||
return {
|
||||
"run_id": run_id,
|
||||
"total": row.total,
|
||||
"positive": row.positive,
|
||||
"negative": row.negative,
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Exact-match fixed-question persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.fixed_questions.base import FixedQuestionStore
|
||||
from deerflow.persistence.fixed_questions.memory import MemoryFixedQuestionStore
|
||||
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
|
||||
from deerflow.persistence.fixed_questions.sql import FixedQuestionRepository
|
||||
|
||||
__all__ = [
|
||||
"FixedQuestionRepository",
|
||||
"FixedQuestionRow",
|
||||
"FixedQuestionStore",
|
||||
"MemoryFixedQuestionStore",
|
||||
"make_fixed_question_store",
|
||||
]
|
||||
|
||||
|
||||
def make_fixed_question_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryFixedQuestionStore()
|
||||
return FixedQuestionRepository(session_factory)
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""Abstract persistence API for administrator-managed fixed questions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class FixedQuestionStore(ABC):
|
||||
"""Global fixed-question configuration shared by every account."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_questions(self) -> list[dict[str, Any]]:
|
||||
"""Return every configured question in display order."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def find_enabled(self, question: str) -> dict[str, Any] | None:
|
||||
"""Return the enabled row whose question exactly equals ``question``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def replace_all(
|
||||
self,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Atomically replace the complete fixed-question configuration."""
|
||||
raise NotImplementedError
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
"""In-memory fixed-question store used by memory deployments and tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.fixed_questions.base import FixedQuestionStore
|
||||
|
||||
|
||||
def _normalize(data: dict[str, Any], updated_by: str | None) -> dict[str, Any]:
|
||||
return {
|
||||
"id": str(data["id"]),
|
||||
"question": str(data.get("question") or ""),
|
||||
"answer": str(data.get("answer") or ""),
|
||||
"enabled": bool(data.get("enabled", True)),
|
||||
"tokens_per_second": int(data.get("tokens_per_second", 100)),
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"updated_by": updated_by,
|
||||
"updated_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
class MemoryFixedQuestionStore(FixedQuestionStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def list_questions(self) -> list[dict[str, Any]]:
|
||||
rows = sorted(
|
||||
self._rows.values(),
|
||||
key=lambda row: (row.get("sort_order", 0), row.get("question", "")),
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def find_enabled(self, question: str) -> dict[str, Any] | None:
|
||||
for row in self._rows.values():
|
||||
if row["enabled"] and row["question"] == question:
|
||||
return dict(row)
|
||||
return None
|
||||
|
||||
async def replace_all(
|
||||
self,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
# Last duplicate question wins, mirroring the SQL repository.
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
question = row.get("question")
|
||||
row_id = row.get("id")
|
||||
if row_id and question not in (None, ""):
|
||||
by_id[str(row_id)] = row
|
||||
by_question = {str(row["question"]): row for row in by_id.values()}
|
||||
self._rows = {str(row["id"]): _normalize(row, updated_by) for row in by_question.values()}
|
||||
return await self.list_questions()
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
"""ORM model for exact-match, administrator-managed fixed questions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class FixedQuestionRow(Base):
|
||||
"""One globally shared question/answer pair.
|
||||
|
||||
``question_hash`` provides a portable indexed lookup for questions stored in
|
||||
``Text``. The repository always compares the original question as well, so a
|
||||
theoretical SHA-256 collision cannot cause a false match.
|
||||
"""
|
||||
|
||||
__tablename__ = "fixed_questions"
|
||||
__table_args__ = (UniqueConstraint("question_hash", name="uq_fixed_questions_question_hash"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
question: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
question_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
answer: Mapped[str] = mapped_column(PortableLongText, nullable=False)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="1")
|
||||
tokens_per_second: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
default=100,
|
||||
server_default="100",
|
||||
)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
"""SQLAlchemy-backed fixed-question store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.fixed_questions.base import FixedQuestionStore
|
||||
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
|
||||
|
||||
|
||||
def question_hash(question: str) -> str:
|
||||
return hashlib.sha256(question.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _to_dict(row: FixedQuestionRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data.pop("question_hash", None)
|
||||
value = data.get("updated_at")
|
||||
if isinstance(value, datetime):
|
||||
data["updated_at"] = value.isoformat()
|
||||
return data
|
||||
|
||||
|
||||
def _row_from(data: dict[str, Any], updated_by: str | None) -> FixedQuestionRow:
|
||||
question = str(data.get("question") or "")
|
||||
return FixedQuestionRow(
|
||||
id=str(data["id"]),
|
||||
question=question,
|
||||
question_hash=question_hash(question),
|
||||
answer=str(data.get("answer") or ""),
|
||||
enabled=bool(data.get("enabled", True)),
|
||||
tokens_per_second=int(data.get("tokens_per_second", 100)),
|
||||
sort_order=int(data.get("sort_order", 0)),
|
||||
updated_by=updated_by,
|
||||
)
|
||||
|
||||
|
||||
class FixedQuestionRepository(FixedQuestionStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_questions(self) -> list[dict[str, Any]]:
|
||||
stmt = select(FixedQuestionRow).order_by(
|
||||
FixedQuestionRow.sort_order.asc(),
|
||||
FixedQuestionRow.question.asc(),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def find_enabled(self, question: str) -> dict[str, Any] | None:
|
||||
stmt = select(FixedQuestionRow).where(
|
||||
FixedQuestionRow.question_hash == question_hash(question),
|
||||
FixedQuestionRow.enabled.is_(True),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None or row.question != question:
|
||||
return None
|
||||
return _to_dict(row)
|
||||
|
||||
async def replace_all(
|
||||
self,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
# Last duplicate question wins, preventing a unique-hash violation even
|
||||
# if a non-UI caller submits malformed data.
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
question = row.get("question")
|
||||
row_id = row.get("id")
|
||||
if row_id and question not in (None, ""):
|
||||
by_id[str(row_id)] = row
|
||||
by_question = {str(row["question"]): row for row in by_id.values()}
|
||||
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(FixedQuestionRow))
|
||||
new_rows = [_row_from(row, updated_by) for row in by_question.values()]
|
||||
session.add_all(new_rows)
|
||||
await session.commit()
|
||||
return [_to_dict(row) for row in new_rows]
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
"""Generated HTML page + favorite persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.html_pages.base import HtmlPageStore
|
||||
from deerflow.persistence.html_pages.memory import MemoryHtmlPageStore
|
||||
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
|
||||
from deerflow.persistence.html_pages.sql import HtmlPageRepository
|
||||
|
||||
__all__ = [
|
||||
"HtmlPageStore",
|
||||
"HtmlPageRepository",
|
||||
"MemoryHtmlPageStore",
|
||||
"HtmlPageFavoriteRow",
|
||||
"ScheduledHtmlPageRow",
|
||||
"make_html_page_store",
|
||||
]
|
||||
|
||||
|
||||
def make_html_page_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryHtmlPageStore()
|
||||
return HtmlPageRepository(session_factory)
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
"""Abstract persistence API for generated HTML pages and favorites."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class HtmlPageStore(ABC):
|
||||
# --- Generated pages ---
|
||||
|
||||
@abstractmethod
|
||||
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_page(self, page_id: str) -> dict[str, Any] | None:
|
||||
"""Return a page by id without restricting by owner."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""Return the page produced by a given run, if any."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""Return the most recently created page for a task, if any."""
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Favorites ---
|
||||
|
||||
@abstractmethod
|
||||
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
|
||||
"""Return a favorite by id without restricting by owner."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
|
||||
"""Return the user's existing favorite for a page, if any (dedup check)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
"""In-memory HtmlPageStore for no-database / test deployments."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.html_pages.base import HtmlPageStore
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class MemoryHtmlPageStore(HtmlPageStore):
|
||||
def __init__(self) -> None:
|
||||
self._pages: dict[str, dict[str, Any]] = {}
|
||||
self._favorites: dict[str, dict[str, Any]] = {}
|
||||
|
||||
# --- Generated pages ---
|
||||
|
||||
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"task_id": data["task_id"],
|
||||
"run_id": data["run_id"],
|
||||
"user_id": data["user_id"],
|
||||
"title": data.get("title") or "",
|
||||
"html_content": data.get("html_content") or "",
|
||||
"html_file_path": data.get("html_file_path"),
|
||||
"data_snapshot": data.get("data_snapshot") or {},
|
||||
"prompt_snapshot": data.get("prompt_snapshot") or {},
|
||||
"reference_favorite_id": data.get("reference_favorite_id"),
|
||||
"style_summary": data.get("style_summary"),
|
||||
"layout_summary": data.get("layout_summary"),
|
||||
"created_at": _now_iso(),
|
||||
"updated_at": _now_iso(),
|
||||
}
|
||||
self._pages[row["id"]] = row
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def get_page(self, page_id: str) -> dict[str, Any] | None:
|
||||
row = self._pages.get(page_id)
|
||||
return copy.deepcopy(row) if row is not None else None
|
||||
|
||||
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
matches = [r for r in self._pages.values() if r["run_id"] == run_id]
|
||||
if not matches:
|
||||
return None
|
||||
matches.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return copy.deepcopy(matches[0])
|
||||
|
||||
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
matches = [r for r in self._pages.values() if r["task_id"] == task_id]
|
||||
if not matches:
|
||||
return None
|
||||
matches.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return copy.deepcopy(matches[0])
|
||||
|
||||
# --- Favorites ---
|
||||
|
||||
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"user_id": data["user_id"],
|
||||
"page_id": data["page_id"],
|
||||
"title": data.get("title") or "",
|
||||
"description": data.get("description"),
|
||||
"tags": data.get("tags") or [],
|
||||
"html_snapshot": data.get("html_snapshot") or "",
|
||||
"style_summary": data.get("style_summary"),
|
||||
"layout_summary": data.get("layout_summary"),
|
||||
"content_structure_summary": data.get("content_structure_summary"),
|
||||
"reuse_prompt": data.get("reuse_prompt"),
|
||||
"created_at": _now_iso(),
|
||||
"updated_at": _now_iso(),
|
||||
}
|
||||
self._favorites[row["id"]] = row
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
matches = [r for r in self._favorites.values() if r["user_id"] == user_id]
|
||||
matches.sort(key=lambda r: r["created_at"], reverse=True)
|
||||
return [copy.deepcopy(r) for r in matches[: max(1, limit)]]
|
||||
|
||||
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
|
||||
row = self._favorites.get(favorite_id)
|
||||
return copy.deepcopy(row) if row is not None else None
|
||||
|
||||
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
|
||||
for row in self._favorites.values():
|
||||
if row["user_id"] == user_id and row["page_id"] == page_id:
|
||||
return copy.deepcopy(row)
|
||||
return None
|
||||
|
||||
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
|
||||
row = self._favorites.get(favorite_id)
|
||||
if row is None or row["user_id"] != user_id:
|
||||
return False
|
||||
del self._favorites[favorite_id]
|
||||
return True
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
"""ORM models for generated HTML pages and user favorites.
|
||||
|
||||
These back the "HTML 页面定时任务" feature: a scheduled task of kind
|
||||
``html_page`` renders a full HTML document each run. Every rendered page is
|
||||
recorded in :class:`ScheduledHtmlPageRow`; a user may bookmark one into
|
||||
:class:`HtmlPageFavoriteRow`, which also stores a reusable style/layout
|
||||
summary so a later task can be generated "in the style of" the favorite.
|
||||
|
||||
Both tables are brand new, so ``Base.metadata.create_all`` creates them on the
|
||||
next startup without touching any existing schema.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class ScheduledHtmlPageRow(Base):
|
||||
"""One HTML page produced by a single run of an ``html_page`` task."""
|
||||
|
||||
__tablename__ = "scheduled_html_pages"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
title: Mapped[str] = mapped_column(String(255))
|
||||
# Full HTML document — can be large, so LONGTEXT on MySQL.
|
||||
html_content: Mapped[str] = mapped_column(PortableLongText())
|
||||
html_file_path: Mapped[str | None] = mapped_column(Text)
|
||||
data_snapshot_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
|
||||
prompt_snapshot_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict)
|
||||
reference_favorite_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
style_summary: Mapped[str | None] = mapped_column(Text)
|
||||
layout_summary: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class HtmlPageFavoriteRow(Base):
|
||||
"""A user-bookmarked HTML page, with a reusable style/layout summary."""
|
||||
|
||||
__tablename__ = "html_page_favorites"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
page_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
title: Mapped[str] = mapped_column(String(255))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
tags_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list)
|
||||
# Snapshot of the page HTML at favorite time, decoupled from the source run
|
||||
# so deleting the run never breaks the favorite reference.
|
||||
html_snapshot: Mapped[str] = mapped_column(PortableLongText())
|
||||
style_summary: Mapped[str | None] = mapped_column(Text)
|
||||
layout_summary: Mapped[str | None] = mapped_column(Text)
|
||||
content_structure_summary: Mapped[str | None] = mapped_column(Text)
|
||||
reuse_prompt: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
"""SQLAlchemy repository for generated HTML pages and favorites."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.html_pages.base import HtmlPageStore
|
||||
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
|
||||
|
||||
|
||||
def _to_dict(row) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
# Normalize the JSON column names to API-friendly keys.
|
||||
if "data_snapshot_json" in data:
|
||||
data["data_snapshot"] = data.pop("data_snapshot_json") or {}
|
||||
if "prompt_snapshot_json" in data:
|
||||
data["prompt_snapshot"] = data.pop("prompt_snapshot_json") or {}
|
||||
if "tags_json" in data:
|
||||
data["tags"] = data.pop("tags_json") or []
|
||||
return data
|
||||
|
||||
|
||||
class HtmlPageRepository(HtmlPageStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# --- Generated pages ---
|
||||
|
||||
async def create_page(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = ScheduledHtmlPageRow(
|
||||
id=data["id"],
|
||||
task_id=data["task_id"],
|
||||
run_id=data["run_id"],
|
||||
user_id=data["user_id"],
|
||||
title=data.get("title") or "",
|
||||
html_content=data.get("html_content") or "",
|
||||
html_file_path=data.get("html_file_path"),
|
||||
data_snapshot_json=data.get("data_snapshot") or {},
|
||||
prompt_snapshot_json=data.get("prompt_snapshot") or {},
|
||||
reference_favorite_id=data.get("reference_favorite_id"),
|
||||
style_summary=data.get("style_summary"),
|
||||
layout_summary=data.get("layout_summary"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def get_page(self, page_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ScheduledHtmlPageRow, page_id)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def get_page_by_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(ScheduledHtmlPageRow)
|
||||
.where(ScheduledHtmlPageRow.run_id == run_id)
|
||||
.order_by(ScheduledHtmlPageRow.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def latest_page_for_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(ScheduledHtmlPageRow)
|
||||
.where(ScheduledHtmlPageRow.task_id == task_id)
|
||||
.order_by(ScheduledHtmlPageRow.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
# --- Favorites ---
|
||||
|
||||
async def create_favorite(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = HtmlPageFavoriteRow(
|
||||
id=data["id"],
|
||||
user_id=data["user_id"],
|
||||
page_id=data["page_id"],
|
||||
title=data.get("title") or "",
|
||||
description=data.get("description"),
|
||||
tags_json=data.get("tags") or [],
|
||||
html_snapshot=data.get("html_snapshot") or "",
|
||||
style_summary=data.get("style_summary"),
|
||||
layout_summary=data.get("layout_summary"),
|
||||
content_structure_summary=data.get("content_structure_summary"),
|
||||
reuse_prompt=data.get("reuse_prompt"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def list_favorites(self, user_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(HtmlPageFavoriteRow)
|
||||
.where(HtmlPageFavoriteRow.user_id == user_id)
|
||||
.order_by(HtmlPageFavoriteRow.created_at.desc())
|
||||
.limit(max(1, limit))
|
||||
)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_favorite(self, favorite_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(HtmlPageFavoriteRow, favorite_id)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def get_favorite_by_page(self, user_id: str, page_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(HtmlPageFavoriteRow)
|
||||
.where(HtmlPageFavoriteRow.user_id == user_id, HtmlPageFavoriteRow.page_id == page_id)
|
||||
.limit(1)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def delete_favorite(self, favorite_id: str, user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(HtmlPageFavoriteRow).where(
|
||||
HtmlPageFavoriteRow.id == favorite_id,
|
||||
HtmlPageFavoriteRow.user_id == user_id,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return (result.rowcount or 0) > 0
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
"""Intent-turn single-flight persistence (Phase 4).
|
||||
|
||||
Exposes a store factory; ``app/gateway/deps.py`` registers it on
|
||||
``app.state.intent_turn_store`` at startup. The intent router reads it from
|
||||
``request.app.state`` (no cross-boundary default needed — intent runs only in the
|
||||
gateway process).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
from deerflow.persistence.intent_turns.sql import IntentTurnRepository
|
||||
|
||||
|
||||
def make_intent_turn_store(session_factory) -> IntentTurnRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return IntentTurnRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"IntentTurnRow",
|
||||
"IntentTurnRepository",
|
||||
"make_intent_turn_store",
|
||||
]
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
"""ORM model for intent-determination turn single-flight (Phase 4).
|
||||
|
||||
一条 ``intent_turns`` 行 = 第一步「确定意图」对话的**一个回合**对 ``(thread_id,
|
||||
client_turn_id)`` 的占用记录。前端为每个回合生成稳定的 ``client_turn_id``,重试 / 双击 /
|
||||
网络层重发都复用它;后端用 ``UNIQUE(thread_id, client_turn_id)`` 做跨 worker 单飞:
|
||||
|
||||
* 同一 ``(thread_id, client_turn_id)`` 至多一条行 —— INSERT 冲突即幂等命中:
|
||||
- 已 ``done`` → 原样回放缓存的终态帧,**不再启 run**(杜绝双发意图 / 重复澄清卡);
|
||||
- 在跑且租约未过期 → 该请求是重复的 in-flight 请求,丢弃(原请求仍在流式吐结果);
|
||||
- 在跑但租约过期(worker 崩溃)→ 抢占续跑;
|
||||
- ``message_hash`` 不同(同一 client_turn_id 却是不同内容)→ 409 客户端 bug。
|
||||
|
||||
一致性只来自 DB 唯一键 + 条件 UPDATE(租约 CAS),不依赖进程内 ``RunManager``
|
||||
(后者是每进程内存态,多 worker 下看不到彼此的 run,无法单飞)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class IntentTurnRow(Base):
|
||||
__tablename__ = "intent_turns"
|
||||
__table_args__ = (
|
||||
# 跨 worker 单飞的根:同一 (thread_id, client_turn_id) 至多一条。INSERT 冲突即幂等。
|
||||
UniqueConstraint("thread_id", "client_turn_id", name="uq_intent_turns_thread_client"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
# 前端每回合生成的稳定幂等键;重试 / 双击复用同一个。
|
||||
client_turn_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# 本回合 message 的 sha256:检测「同 client_turn_id 但内容变了」的客户端 bug(→ 409)。
|
||||
message_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# running(在跑)/ done(已出终态帧,可回放)/ error。
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="running", server_default="running"
|
||||
)
|
||||
# 终态帧的完整 JSON(done/error 时写入):重复请求命中时原样回放,不再启 run。
|
||||
result_json: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# 租约:领走的 worker 标记;过期(worker 崩溃)后可被条件 UPDATE 抢占续跑。
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
"""SQLAlchemy-backed intent-turn single-flight storage (Phase 4).
|
||||
|
||||
跨 worker 单飞 + 幂等回放,一致性只来自 DB(唯一键 + 条件 UPDATE),不依赖进程内
|
||||
``RunManager``(多 worker 下彼此不可见)。两条核心路径:
|
||||
|
||||
* ``try_claim`` —— INSERT 占用 ``(thread_id, client_turn_id)``;唯一键冲突即幂等命中,
|
||||
按「已 done→回放 / 在跑且租约有效→丢弃 / 租约过期或 error→抢占续跑 / 内容不一致→409」分流。
|
||||
* ``record_result`` —— 持租者把终态帧写回(条件 UPDATE 要求租约匹配),清租约。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return value
|
||||
return value
|
||||
|
||||
|
||||
class IntentTurnRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: IntentTurnRow | None) -> dict[str, Any] | None:
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"id": row.id,
|
||||
"thread_id": row.thread_id,
|
||||
"client_turn_id": row.client_turn_id,
|
||||
"message_hash": row.message_hash,
|
||||
"status": row.status,
|
||||
"result_json": _loads(row.result_json),
|
||||
"error": row.error,
|
||||
"lease_owner": row.lease_owner,
|
||||
"lease_until": row.lease_until.isoformat() if row.lease_until else None,
|
||||
"created_at": row.created_at,
|
||||
"updated_at": row.updated_at,
|
||||
}
|
||||
|
||||
async def get(self, thread_id: str, client_turn_id: str) -> dict[str, Any] | None:
|
||||
stmt = select(IntentTurnRow).where(
|
||||
IntentTurnRow.thread_id == thread_id,
|
||||
IntentTurnRow.client_turn_id == client_turn_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def try_claim(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
client_turn_id: str,
|
||||
message_hash: str,
|
||||
lease_owner: str,
|
||||
lease_until: datetime,
|
||||
) -> dict[str, Any]:
|
||||
"""占用一个意图回合 —— 跨 worker 单飞 + 幂等的入口。
|
||||
|
||||
返回 ``{"action": ..., "row": {...}}``,``action`` 取值:
|
||||
|
||||
* ``"run"`` —— 本调用方赢得占用(新建 / 抢占续跑),应启 run 并在终态写回。
|
||||
* ``"replay"`` —— 该回合已 ``done``,原样回放 ``row.result_json``,**不再启 run**。
|
||||
* ``"duplicate"``—— 该回合仍在跑且租约有效(或刚被别处抢占),本请求是重复的
|
||||
in-flight 请求,应丢弃(原请求仍在流式吐结果)。
|
||||
* ``"conflict"`` —— 同一 ``client_turn_id`` 但 ``message_hash`` 不同 → 客户端 bug,409。
|
||||
"""
|
||||
# 1) 先尝试 INSERT 占用。
|
||||
new_id = uuid4().hex
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
session.add(
|
||||
IntentTurnRow(
|
||||
id=new_id,
|
||||
thread_id=thread_id,
|
||||
client_turn_id=client_turn_id,
|
||||
message_hash=message_hash,
|
||||
status="running",
|
||||
lease_owner=lease_owner,
|
||||
lease_until=lease_until,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
row = await self.get(thread_id, client_turn_id)
|
||||
return {"action": "run", "row": row}
|
||||
except IntegrityError:
|
||||
# 唯一键 (thread_id, client_turn_id) 冲突 → 幂等命中,读回既有行分流。
|
||||
pass
|
||||
|
||||
existing = await self.get(thread_id, client_turn_id)
|
||||
if existing is None:
|
||||
# 极端竞态:冲突与回读之间行被清掉。降级为「可新建」再插一次(best-effort)。
|
||||
return {"action": "run", "row": None}
|
||||
|
||||
# 2) 内容指纹不一致 → 客户端 bug(同 client_turn_id 却换了 message)。
|
||||
if existing.get("message_hash") != message_hash:
|
||||
return {"action": "conflict", "row": existing}
|
||||
|
||||
status = existing.get("status") or "running"
|
||||
|
||||
# 3) 已完成 → 幂等回放终态帧。
|
||||
if status == "done":
|
||||
return {"action": "replay", "row": existing}
|
||||
|
||||
# 4) 在跑或出错 → 尝试条件 UPDATE 抢占续跑(error 强制;running 要求租约已过期)。
|
||||
reclaimed = await self._conditional_reclaim(
|
||||
thread_id=thread_id,
|
||||
client_turn_id=client_turn_id,
|
||||
prev_status=status,
|
||||
lease_owner=lease_owner,
|
||||
lease_until=lease_until,
|
||||
)
|
||||
if reclaimed is not None:
|
||||
return {"action": "run", "row": reclaimed}
|
||||
|
||||
# 5) 抢占失败:仍在跑且租约有效(或刚被别处抢占)→ 本请求是重复 in-flight,丢弃。
|
||||
return {"action": "duplicate", "row": existing}
|
||||
|
||||
async def _conditional_reclaim(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
client_turn_id: str,
|
||||
prev_status: str,
|
||||
lease_owner: str,
|
||||
lease_until: datetime,
|
||||
) -> dict[str, Any] | None:
|
||||
"""原子条件 UPDATE 抢占:``error`` 强制续跑;``running`` 仅当租约已过期。
|
||||
|
||||
成功则置 ``status='running'``、写新租约,返回行;否则 None(仍在跑 / 刚被抢占)。
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
where = [
|
||||
IntentTurnRow.thread_id == thread_id,
|
||||
IntentTurnRow.client_turn_id == client_turn_id,
|
||||
]
|
||||
if prev_status == "error":
|
||||
where.append(IntentTurnRow.status == "error")
|
||||
else:
|
||||
where.extend(
|
||||
[
|
||||
IntentTurnRow.status == "running",
|
||||
IntentTurnRow.lease_until.is_not(None),
|
||||
IntentTurnRow.lease_until < now,
|
||||
]
|
||||
)
|
||||
values = {
|
||||
"status": "running",
|
||||
"lease_owner": lease_owner,
|
||||
"lease_until": lease_until,
|
||||
"updated_at": now,
|
||||
}
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(IntentTurnRow).where(*where).values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return None
|
||||
await session.commit()
|
||||
row = (
|
||||
await session.execute(
|
||||
select(IntentTurnRow).where(
|
||||
IntentTurnRow.thread_id == thread_id,
|
||||
IntentTurnRow.client_turn_id == client_turn_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def record_result(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
client_turn_id: str,
|
||||
lease_owner: str,
|
||||
status: str,
|
||||
result_json: Any = None,
|
||||
error: str | None = None,
|
||||
) -> bool:
|
||||
"""持租者把终态帧写回(条件 UPDATE 要求租约匹配),并清租约。
|
||||
|
||||
只有赢得 ``try_claim``(``action='run'``)的调用方才应调用,且 ``lease_owner``
|
||||
必须与领取时一致 —— 防止「抢占续跑的新 worker」被「旧 worker 的迟到收尾」覆盖。
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
values: dict[str, Any] = {
|
||||
"status": status,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
"updated_at": now,
|
||||
}
|
||||
if result_json is not None:
|
||||
values["result_json"] = json.dumps(result_json, ensure_ascii=False)
|
||||
if error is not None:
|
||||
values["error"] = error
|
||||
where = [
|
||||
IntentTurnRow.thread_id == thread_id,
|
||||
IntentTurnRow.client_turn_id == client_turn_id,
|
||||
IntentTurnRow.lease_owner == lease_owner,
|
||||
IntentTurnRow.status == "running",
|
||||
]
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(IntentTurnRow).where(*where).values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return False
|
||||
await session.commit()
|
||||
return True
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
"""Light-app registry persistence (轻应用中心)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.light_apps.base import DuplicateLightAppNameError, LightAppStore
|
||||
from deerflow.persistence.light_apps.memory import MemoryLightAppStore
|
||||
from deerflow.persistence.light_apps.model import LightAppRow
|
||||
from deerflow.persistence.light_apps.sql import LightAppRepository
|
||||
|
||||
__all__ = [
|
||||
"DuplicateLightAppNameError",
|
||||
"LightAppRepository",
|
||||
"LightAppRow",
|
||||
"LightAppStore",
|
||||
"MemoryLightAppStore",
|
||||
"make_light_app_store",
|
||||
]
|
||||
|
||||
|
||||
def make_light_app_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryLightAppStore()
|
||||
return LightAppRepository(session_factory)
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
"""Abstract light-app registry persistence API.
|
||||
|
||||
Global / shared store: no ``user_id`` scoping. Reads are open to any caller; the
|
||||
router gates writes behind admin. ``created_by`` / ``updated_by`` are audit-only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class DuplicateLightAppNameError(ValueError):
|
||||
"""Raised when creating/updating an app would collide on the unique name."""
|
||||
|
||||
|
||||
class LightAppStore(ABC):
|
||||
@abstractmethod
|
||||
async def list_apps(
|
||||
self,
|
||||
*,
|
||||
app_type: str | None = None,
|
||||
mount_location: str | None = None,
|
||||
enabled_only: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return registered apps (newest first), optionally filtered."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_app(self, app_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_app(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create an app. Raises ``DuplicateLightAppNameError`` on a name clash."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_app(self, app_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Partial update. Returns None if the app does not exist. Raises
|
||||
``DuplicateLightAppNameError`` when renaming onto an existing name."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_app(self, app_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
"""In-memory light-app registry store (used when no DB is configured / tests)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.light_apps.base import DuplicateLightAppNameError, LightAppStore
|
||||
|
||||
_WRITABLE = {
|
||||
"name",
|
||||
"description",
|
||||
"app_type",
|
||||
"url",
|
||||
"route_params",
|
||||
"mount_location",
|
||||
"icon",
|
||||
"enabled",
|
||||
"sort_order",
|
||||
"updated_by",
|
||||
}
|
||||
|
||||
|
||||
class MemoryLightAppStore(LightAppStore):
|
||||
def __init__(self) -> None:
|
||||
self._apps: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _public(row: dict[str, Any]) -> dict[str, Any]:
|
||||
out = dict(row)
|
||||
out["route_params"] = list(out.get("route_params") or [])
|
||||
return out
|
||||
|
||||
def _name_taken(self, name: str, *, exclude_id: str | None = None) -> bool:
|
||||
return any(r["name"] == name and r["app_id"] != exclude_id for r in self._apps.values())
|
||||
|
||||
async def list_apps(
|
||||
self,
|
||||
*,
|
||||
app_type: str | None = None,
|
||||
mount_location: str | None = None,
|
||||
enabled_only: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = list(self._apps.values())
|
||||
if app_type:
|
||||
rows = [r for r in rows if r["app_type"] == app_type]
|
||||
if mount_location:
|
||||
rows = [r for r in rows if r.get("mount_location") == mount_location]
|
||||
if enabled_only:
|
||||
rows = [r for r in rows if r.get("enabled", True)]
|
||||
rows.sort(key=lambda r: (r.get("sort_order", 0), r.get("created_at", "")))
|
||||
return [self._public(r) for r in rows]
|
||||
|
||||
async def get_app(self, app_id: str) -> dict[str, Any] | None:
|
||||
row = self._apps.get(app_id)
|
||||
return self._public(row) if row is not None else None
|
||||
|
||||
async def create_app(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
if self._name_taken(data["name"]):
|
||||
raise DuplicateLightAppNameError(data["name"])
|
||||
now = datetime.now(UTC).isoformat()
|
||||
app_id = data.get("app_id") or uuid.uuid4().hex[:12]
|
||||
row = {
|
||||
"app_id": app_id,
|
||||
"name": data["name"],
|
||||
"description": data.get("description"),
|
||||
"app_type": data["app_type"],
|
||||
"url": data["url"],
|
||||
"route_params": data.get("route_params") or [],
|
||||
"mount_location": data.get("mount_location"),
|
||||
"icon": data.get("icon"),
|
||||
"enabled": data.get("enabled", True),
|
||||
"sort_order": data.get("sort_order", 0),
|
||||
"created_by": data.get("created_by"),
|
||||
"updated_by": data.get("created_by"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._apps[app_id] = row
|
||||
return self._public(row)
|
||||
|
||||
async def update_app(self, app_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._apps.get(app_id)
|
||||
if row is None:
|
||||
return None
|
||||
new_name = data.get("name")
|
||||
if new_name is not None and new_name != row["name"] and self._name_taken(new_name, exclude_id=app_id):
|
||||
raise DuplicateLightAppNameError(new_name)
|
||||
for key, value in data.items():
|
||||
if key in _WRITABLE:
|
||||
row[key] = value
|
||||
row["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return self._public(row)
|
||||
|
||||
async def delete_app(self, app_id: str) -> bool:
|
||||
return self._apps.pop(app_id, None) is not None
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""ORM model for the light-app registry (轻应用中心).
|
||||
|
||||
Global / shared — there is no ``user_id`` scoping; every account sees the same
|
||||
registered apps. ``created_by`` / ``updated_by`` are audit-only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class LightAppRow(Base):
|
||||
__tablename__ = "light_apps"
|
||||
__table_args__ = (UniqueConstraint("name", name="uq_light_apps_name"),)
|
||||
|
||||
app_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(127))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
# "window_open" (open in a new tab) | "iframe" (embed under a parent menu)
|
||||
app_type: Mapped[str] = mapped_column(String(32))
|
||||
url: Mapped[str] = mapped_column(Text)
|
||||
# List of route-param descriptors resolved at launch time by the frontend:
|
||||
# {"key": str, "paramType": "login"|"theme", "source": str,
|
||||
# "customValue": str, "themeValue": str}
|
||||
route_params_json: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
# iframe only: the parent sidebar menu id to mount under
|
||||
# (e.g. "xk-center-task" / "pg-center-task" / "ty-center-task" /
|
||||
# "comprehensive-management"). NULL for window_open apps.
|
||||
mount_location: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
icon: Mapped[str | None] = mapped_column(String(64))
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, index=True)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64))
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
"""SQLAlchemy-backed light-app registry store (global / shared)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.light_apps.base import DuplicateLightAppNameError, LightAppStore
|
||||
from deerflow.persistence.light_apps.model import LightAppRow
|
||||
|
||||
# Columns a caller may set/patch (everything except identity + timestamps).
|
||||
_WRITABLE = {
|
||||
"name",
|
||||
"description",
|
||||
"app_type",
|
||||
"url",
|
||||
"route_params",
|
||||
"mount_location",
|
||||
"icon",
|
||||
"enabled",
|
||||
"sort_order",
|
||||
"updated_by",
|
||||
}
|
||||
|
||||
|
||||
def _to_dict(row: LightAppRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
data["route_params"] = data.pop("route_params_json", None) or []
|
||||
return data
|
||||
|
||||
|
||||
def _column_for(key: str) -> str:
|
||||
return "route_params_json" if key == "route_params" else key
|
||||
|
||||
|
||||
class LightAppRepository(LightAppStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_apps(
|
||||
self,
|
||||
*,
|
||||
app_type: str | None = None,
|
||||
mount_location: str | None = None,
|
||||
enabled_only: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = select(LightAppRow)
|
||||
if app_type:
|
||||
stmt = stmt.where(LightAppRow.app_type == app_type)
|
||||
if mount_location:
|
||||
stmt = stmt.where(LightAppRow.mount_location == mount_location)
|
||||
if enabled_only:
|
||||
stmt = stmt.where(LightAppRow.enabled.is_(True))
|
||||
stmt = stmt.order_by(LightAppRow.sort_order.asc(), LightAppRow.created_at.desc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_app(self, app_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LightAppRow, app_id)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def _name_taken(self, session: AsyncSession, name: str, *, exclude_id: str | None = None) -> bool:
|
||||
stmt = select(LightAppRow.app_id).where(LightAppRow.name == name)
|
||||
if exclude_id is not None:
|
||||
stmt = stmt.where(LightAppRow.app_id != exclude_id)
|
||||
result = await session.execute(stmt.limit(1))
|
||||
return result.first() is not None
|
||||
|
||||
async def create_app(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = LightAppRow(
|
||||
app_id=data.get("app_id") or uuid.uuid4().hex[:12],
|
||||
name=data["name"],
|
||||
description=data.get("description"),
|
||||
app_type=data["app_type"],
|
||||
url=data["url"],
|
||||
route_params_json=data.get("route_params") or [],
|
||||
mount_location=data.get("mount_location"),
|
||||
icon=data.get("icon"),
|
||||
enabled=data.get("enabled", True),
|
||||
sort_order=data.get("sort_order", 0),
|
||||
created_by=data.get("created_by"),
|
||||
updated_by=data.get("created_by"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
if await self._name_taken(session, row.name):
|
||||
raise DuplicateLightAppNameError(row.name)
|
||||
session.add(row)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError as exc:
|
||||
await session.rollback()
|
||||
raise DuplicateLightAppNameError(row.name) from exc
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def update_app(self, app_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LightAppRow, app_id)
|
||||
if row is None:
|
||||
return None
|
||||
new_name = data.get("name")
|
||||
if new_name is not None and new_name != row.name and await self._name_taken(session, new_name, exclude_id=app_id):
|
||||
raise DuplicateLightAppNameError(new_name)
|
||||
for key, value in data.items():
|
||||
if key not in _WRITABLE:
|
||||
continue
|
||||
setattr(row, _column_for(key), value)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError as exc:
|
||||
await session.rollback()
|
||||
raise DuplicateLightAppNameError(new_name or row.name) from exc
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def delete_app(self, app_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LightAppRow, app_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
"""Persistence layer for per-LLM-call metrics.
|
||||
|
||||
Each completed LLM invocation (success or failure) writes a single row so
|
||||
administrators can audit usage and speed. Designed to be cheap to write —
|
||||
the middleware fires once per provider call and never blocks streaming.
|
||||
"""
|
||||
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsStore
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llm_metrics.sql import SqlLlmMetricsStore
|
||||
|
||||
__all__ = [
|
||||
"LlmCallMetricRow",
|
||||
"LlmMetricRecord",
|
||||
"LlmMetricsStore",
|
||||
"SqlLlmMetricsStore",
|
||||
"make_llm_metrics_store",
|
||||
]
|
||||
|
||||
|
||||
def make_llm_metrics_store(session_factory) -> LlmMetricsStore | None:
|
||||
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return SqlLlmMetricsStore(session_factory)
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
"""Abstract repository contract for LLM call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmMetricRecord:
|
||||
"""In-memory shape of a metric row before persistence."""
|
||||
|
||||
id: str
|
||||
created_at: datetime
|
||||
user_id: str | None = None
|
||||
thread_id: str | None = None
|
||||
run_id: str | None = None
|
||||
agent_name: str | None = None
|
||||
model_name: str | None = None
|
||||
duration_ms: int = 0
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
tokens_per_sec: float | None = None
|
||||
status: str = "success"
|
||||
error_type: str | None = None
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmMetricsQuery:
|
||||
"""Filter set for the admin list / export endpoints."""
|
||||
|
||||
user_id: str | None = None
|
||||
# Multiple user ids — set when a fuzzy username search resolves to several
|
||||
# accounts. Takes precedence over ``user_id`` when non-empty.
|
||||
user_ids: list[str] | None = None
|
||||
model_name: str | None = None
|
||||
status: str | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
limit: int = 100
|
||||
offset: int = 0
|
||||
|
||||
|
||||
class LlmMetricsStore(ABC):
|
||||
"""Persistence contract for the metrics middleware + admin API."""
|
||||
|
||||
@abstractmethod
|
||||
async def record(self, metric: LlmMetricRecord) -> None:
|
||||
"""Persist one metric row. Must never raise on best-effort writes."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
|
||||
"""Return the matching rows (newest first), paginated by ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count(self, query: LlmMetricsQuery) -> int:
|
||||
"""Return the unpaginated row count for ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def iter_all(self, query: LlmMetricsQuery):
|
||||
"""Yield rows matching ``query`` without pagination — used by export."""
|
||||
raise NotImplementedError
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
"""ORM model for per-LLM-call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Float, Index, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class LlmCallMetricRow(Base):
|
||||
"""One row per completed (or failed) LLM provider invocation."""
|
||||
|
||||
__tablename__ = "llm_call_metrics"
|
||||
__table_args__ = (
|
||||
Index("ix_llm_call_metrics_created_user", "created_at", "user_id"),
|
||||
Index("ix_llm_call_metrics_run_created", "run_id", "created_at"),
|
||||
Index("ix_llm_call_metrics_created_model_user", "created_at", "model_name", "user_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Wall-clock time the LLM call ended. Indexed because admin views always
|
||||
# sort/filter by recency.
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
|
||||
# Wall-clock duration of the provider invocation in milliseconds. Always set.
|
||||
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
input_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
output_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
total_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
# Output tokens / seconds; null when there is no completion (errors).
|
||||
tokens_per_sec: Mapped[float | None] = mapped_column(Float)
|
||||
|
||||
# "success" or "error". Errors store the classification and original message
|
||||
# so admins can scan for transient vs auth/quota issues.
|
||||
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
|
||||
"user_id": self.user_id,
|
||||
"thread_id": self.thread_id,
|
||||
"run_id": self.run_id,
|
||||
"agent_name": self.agent_name,
|
||||
"model_name": self.model_name,
|
||||
"duration_ms": self.duration_ms,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"total_tokens": self.total_tokens,
|
||||
"tokens_per_sec": self.tokens_per_sec,
|
||||
"status": self.status,
|
||||
"error_type": self.error_type,
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
"""SQLAlchemy-backed LLM metrics repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery, LlmMetricsStore
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _row_to_dict(row: LlmCallMetricRow) -> dict[str, Any]:
|
||||
return row.to_dict()
|
||||
|
||||
|
||||
def _normalize_created_at(value: datetime) -> datetime:
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
class SqlLlmMetricsStore(LlmMetricsStore):
|
||||
"""SQLite/Postgres-backed implementation; writes are intentionally fire-and-forget."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def record(self, metric: LlmMetricRecord) -> None:
|
||||
row = LlmCallMetricRow(
|
||||
id=metric.id,
|
||||
created_at=_normalize_created_at(metric.created_at),
|
||||
user_id=metric.user_id,
|
||||
thread_id=metric.thread_id,
|
||||
run_id=metric.run_id,
|
||||
agent_name=metric.agent_name,
|
||||
model_name=metric.model_name,
|
||||
duration_ms=metric.duration_ms,
|
||||
input_tokens=metric.input_tokens,
|
||||
output_tokens=metric.output_tokens,
|
||||
total_tokens=metric.total_tokens,
|
||||
tokens_per_sec=metric.tokens_per_sec,
|
||||
status=metric.status,
|
||||
error_type=metric.error_type,
|
||||
error_message=metric.error_message,
|
||||
)
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
# Metrics are best-effort. Never let a persistence failure surface
|
||||
# to the agent runtime — log and move on.
|
||||
logger.warning("Failed to persist llm_call_metrics row %s", metric.id, exc_info=True)
|
||||
|
||||
def _apply_filters(self, stmt, query: LlmMetricsQuery):
|
||||
if query.user_ids:
|
||||
stmt = stmt.where(LlmCallMetricRow.user_id.in_(query.user_ids))
|
||||
elif query.user_id:
|
||||
stmt = stmt.where(LlmCallMetricRow.user_id == query.user_id)
|
||||
if query.model_name:
|
||||
stmt = stmt.where(LlmCallMetricRow.model_name == query.model_name)
|
||||
if query.status:
|
||||
stmt = stmt.where(LlmCallMetricRow.status == query.status)
|
||||
if query.since is not None:
|
||||
stmt = stmt.where(LlmCallMetricRow.created_at >= _normalize_created_at(query.since))
|
||||
if query.until is not None:
|
||||
stmt = stmt.where(LlmCallMetricRow.created_at <= _normalize_created_at(query.until))
|
||||
return stmt
|
||||
|
||||
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
stmt = stmt.order_by(desc(LlmCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def count(self, query: LlmMetricsQuery) -> int:
|
||||
stmt = select(func.count()).select_from(LlmCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
async with self._sf() as session:
|
||||
return (await session.scalar(stmt)) or 0
|
||||
|
||||
async def iter_all(self, query: LlmMetricsQuery) -> AsyncIterator[dict[str, Any]]:
|
||||
# Page through the result so the Excel export stays bounded in memory
|
||||
# for large date windows.
|
||||
page_size = 500
|
||||
offset = 0
|
||||
while True:
|
||||
page_query = LlmMetricsQuery(
|
||||
user_id=query.user_id,
|
||||
user_ids=query.user_ids,
|
||||
model_name=query.model_name,
|
||||
status=query.status,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
limit=page_size,
|
||||
offset=offset,
|
||||
)
|
||||
page = await self.list(page_query)
|
||||
if not page:
|
||||
return
|
||||
for item in page:
|
||||
yield item
|
||||
if len(page) < page_size:
|
||||
return
|
||||
offset += page_size
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""DeerFlow-owned LLMWiki authorization metadata."""
|
||||
|
||||
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
|
||||
from deerflow.persistence.llmwiki.memory import MemoryLlmWikiStore
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
from deerflow.persistence.llmwiki.sql import LlmWikiRepository
|
||||
|
||||
__all__ = [
|
||||
"LlmWikiKnowledgeBaseRow",
|
||||
"LlmWikiConversationDepositRow",
|
||||
"LlmWikiRepository",
|
||||
"LlmWikiScope",
|
||||
"LlmWikiStore",
|
||||
"MemoryLlmWikiStore",
|
||||
"make_llmwiki_store",
|
||||
]
|
||||
|
||||
|
||||
def make_llmwiki_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryLlmWikiStore()
|
||||
return LlmWikiRepository(session_factory)
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
"""Persistence contract for DeerFlow-owned LLMWiki authorization metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Literal
|
||||
|
||||
LlmWikiScope = Literal["personal", "public", "all"]
|
||||
|
||||
|
||||
class LlmWikiStore(ABC):
|
||||
@abstractmethod
|
||||
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def update_mapping(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def adopt_conversation_deposit_mapping(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
name: str,
|
||||
description: str,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def delete_mapping(self, mapping_id: str) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_conversation_deposit(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
assistant_message_id: str,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def upsert_conversation_deposit(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
human_message_id: str | None,
|
||||
assistant_message_id: str,
|
||||
knowledge_base_mapping_id: str,
|
||||
weknora_knowledge_id: str | None,
|
||||
title: str,
|
||||
created_by_user_id: str | None,
|
||||
) -> dict[str, Any]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def delete_conversation_deposits(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
message_ids: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]: ...
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
"""In-memory LLMWiki metadata store for tests and DB-less deployments."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class MemoryLlmWikiStore(LlmWikiStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._conversation_deposits: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
|
||||
if any(row["weknora_id"] == weknora_id for row in self._rows.values()):
|
||||
raise ValueError("WeKnora knowledge base is already mapped")
|
||||
now = _now()
|
||||
row = {
|
||||
"id": str(uuid4()),
|
||||
"weknora_id": weknora_id,
|
||||
"owner_user_id": owner_user_id,
|
||||
"name": name,
|
||||
"description": description,
|
||||
"kb_type": kb_type,
|
||||
"publication_status": "private",
|
||||
"published_by": None,
|
||||
"reviewed_by": None,
|
||||
"published_at": None,
|
||||
"wiki_index_enabled": True,
|
||||
"external_search_enabled": False,
|
||||
"external_search_updated_at": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._rows[row["id"]] = row
|
||||
return dict(row)
|
||||
|
||||
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
|
||||
rows = list(self._rows.values())
|
||||
if scope == "personal":
|
||||
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id]
|
||||
elif scope == "public":
|
||||
rows = [row for row in rows if row["publication_status"] == "published"]
|
||||
else:
|
||||
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id or row["publication_status"] == "published"]
|
||||
return [dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
|
||||
|
||||
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None:
|
||||
return None
|
||||
if is_admin or row["owner_user_id"] == user_id:
|
||||
return dict(row)
|
||||
if not write and row["publication_status"] == "published":
|
||||
return dict(row)
|
||||
return None
|
||||
|
||||
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
|
||||
for row in self._rows.values():
|
||||
if row["weknora_id"] == weknora_id:
|
||||
return dict(row)
|
||||
return None
|
||||
|
||||
async def update_mapping(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None:
|
||||
return None
|
||||
if name is not None:
|
||||
row["name"] = name
|
||||
if description is not None:
|
||||
row["description"] = description
|
||||
if wiki_index_enabled is not None:
|
||||
row["wiki_index_enabled"] = wiki_index_enabled
|
||||
if external_search_enabled is not None:
|
||||
row["external_search_enabled"] = external_search_enabled
|
||||
row["external_search_updated_at"] = _now()
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
|
||||
async def adopt_conversation_deposit_mapping(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
name: str,
|
||||
description: str,
|
||||
) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None:
|
||||
return None
|
||||
now = _now()
|
||||
row.update(
|
||||
{
|
||||
"owner_user_id": "system",
|
||||
"name": name,
|
||||
"description": description,
|
||||
"publication_status": "published",
|
||||
"published_by": "system",
|
||||
"reviewed_by": None,
|
||||
"published_at": row.get("published_at") or now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
return dict(row)
|
||||
|
||||
async def delete_mapping(self, mapping_id: str) -> bool:
|
||||
return self._rows.pop(mapping_id, None) is not None
|
||||
|
||||
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None or (not is_admin and row["owner_user_id"] != actor_user_id):
|
||||
return None
|
||||
row["publication_status"] = "published"
|
||||
row["published_by"] = actor_user_id
|
||||
row["reviewed_by"] = None
|
||||
row["published_at"] = _now()
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
|
||||
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None or row["publication_status"] != "pending":
|
||||
return None
|
||||
row["publication_status"] = "published" if approved else "rejected"
|
||||
row["reviewed_by"] = reviewer_user_id
|
||||
row["published_at"] = _now() if approved else None
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
|
||||
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None or (not is_admin and row["owner_user_id"] != actor_user_id):
|
||||
return None
|
||||
row["publication_status"] = "private"
|
||||
row["reviewed_by"] = actor_user_id if is_admin else row.get("reviewed_by")
|
||||
row["published_at"] = None
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
|
||||
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
|
||||
rows = [row for row in self._rows.values() if row["publication_status"] != "private"]
|
||||
if status:
|
||||
rows = [row for row in rows if row["publication_status"] == status]
|
||||
return [dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
|
||||
|
||||
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
|
||||
for row in self._rows.values():
|
||||
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
|
||||
return dict(row)
|
||||
return None
|
||||
|
||||
async def get_conversation_deposit(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
assistant_message_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
key = f"{thread_id}:{assistant_message_id}"
|
||||
row = self._conversation_deposits.get(key)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def upsert_conversation_deposit(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
human_message_id: str | None,
|
||||
assistant_message_id: str,
|
||||
knowledge_base_mapping_id: str,
|
||||
weknora_knowledge_id: str | None,
|
||||
title: str,
|
||||
created_by_user_id: str | None,
|
||||
) -> dict[str, Any]:
|
||||
key = f"{thread_id}:{assistant_message_id}"
|
||||
now = _now()
|
||||
row = self._conversation_deposits.get(key)
|
||||
if row is None:
|
||||
row = {
|
||||
"id": str(uuid4()),
|
||||
"thread_id": thread_id,
|
||||
"human_message_id": human_message_id,
|
||||
"assistant_message_id": assistant_message_id,
|
||||
"knowledge_base_mapping_id": knowledge_base_mapping_id,
|
||||
"weknora_knowledge_id": weknora_knowledge_id,
|
||||
"title": title,
|
||||
"created_by_user_id": created_by_user_id,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._conversation_deposits[key] = row
|
||||
else:
|
||||
row.update(
|
||||
{
|
||||
"human_message_id": human_message_id,
|
||||
"knowledge_base_mapping_id": knowledge_base_mapping_id,
|
||||
"weknora_knowledge_id": weknora_knowledge_id,
|
||||
"title": title,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
return dict(row)
|
||||
|
||||
async def delete_conversation_deposits(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
message_ids: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
ids = set(message_ids or [])
|
||||
deleted: list[dict[str, Any]] = []
|
||||
for key, row in list(self._conversation_deposits.items()):
|
||||
if row.get("thread_id") != thread_id:
|
||||
continue
|
||||
if ids and row.get("human_message_id") not in ids and row.get("assistant_message_id") not in ids:
|
||||
continue
|
||||
deleted.append(dict(row))
|
||||
self._conversation_deposits.pop(key, None)
|
||||
return deleted
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""ORM mapping between DeerFlow ownership and WeKnora knowledge bases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class LlmWikiKnowledgeBaseRow(Base):
|
||||
__tablename__ = "llmwiki_knowledge_bases"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
weknora_id: Mapped[str] = mapped_column(String(128), unique=True, index=True)
|
||||
owner_user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
name: Mapped[str] = mapped_column(String(255), default="")
|
||||
description: Mapped[str] = mapped_column(String(2048), default="")
|
||||
kb_type: Mapped[str] = mapped_column(String(32), default="document")
|
||||
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
|
||||
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
wiki_index_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true(), index=True)
|
||||
external_search_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
|
||||
external_search_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {column.name: getattr(self, column.name) for column in self.__table__.columns}
|
||||
|
||||
|
||||
class LlmWikiConversationDepositRow(Base):
|
||||
"""Mapping from one DeerFlow answer turn to its WeKnora manual document.
|
||||
|
||||
The actual content lives in WeKnora. DeerFlow keeps only the stable ids it
|
||||
needs to rewrite/delete the document when the user regenerates or removes a
|
||||
conversation turn.
|
||||
"""
|
||||
|
||||
__tablename__ = "llmwiki_conversation_deposits"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"thread_id",
|
||||
"assistant_message_id",
|
||||
name="uq_llmwiki_conversation_deposit_answer",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(128), index=True)
|
||||
human_message_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
assistant_message_id: Mapped[str] = mapped_column(String(128), index=True)
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
weknora_knowledge_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
title: Mapped[str] = mapped_column(String(255), default="")
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {column.name: getattr(self, column.name) for column in self.__table__.columns}
|
||||
+281
@@ -0,0 +1,281 @@
|
||||
"""SQLAlchemy LLMWiki metadata repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
return data
|
||||
|
||||
|
||||
class LlmWikiRepository(LlmWikiStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
|
||||
row = LlmWikiKnowledgeBaseRow(
|
||||
id=str(uuid4()),
|
||||
weknora_id=weknora_id,
|
||||
owner_user_id=owner_user_id,
|
||||
name=name,
|
||||
description=description,
|
||||
kb_type=kb_type,
|
||||
publication_status="private",
|
||||
created_at=_now(),
|
||||
updated_at=_now(),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmWikiKnowledgeBaseRow)
|
||||
if scope == "personal" and not is_admin:
|
||||
stmt = stmt.where(LlmWikiKnowledgeBaseRow.owner_user_id == user_id)
|
||||
elif scope == "public":
|
||||
stmt = stmt.where(LlmWikiKnowledgeBaseRow.publication_status == "published")
|
||||
elif scope == "all" and not is_admin:
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
LlmWikiKnowledgeBaseRow.owner_user_id == user_id,
|
||||
LlmWikiKnowledgeBaseRow.publication_status == "published",
|
||||
)
|
||||
)
|
||||
stmt = stmt.order_by(LlmWikiKnowledgeBaseRow.created_at.desc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.id == mapping_id)
|
||||
if not is_admin:
|
||||
condition = LlmWikiKnowledgeBaseRow.owner_user_id == user_id
|
||||
if not write:
|
||||
condition = or_(condition, LlmWikiKnowledgeBaseRow.publication_status == "published")
|
||||
stmt = stmt.where(condition)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _to_dict(row) if row else None
|
||||
|
||||
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.weknora_id == weknora_id)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _to_dict(row) if row else None
|
||||
|
||||
async def update_mapping(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
|
||||
if row is None:
|
||||
return None
|
||||
if name is not None:
|
||||
row.name = name
|
||||
if description is not None:
|
||||
row.description = description
|
||||
if wiki_index_enabled is not None:
|
||||
row.wiki_index_enabled = wiki_index_enabled
|
||||
if external_search_enabled is not None:
|
||||
row.external_search_enabled = external_search_enabled
|
||||
row.external_search_updated_at = _now()
|
||||
row.updated_at = _now()
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
async def adopt_conversation_deposit_mapping(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
name: str,
|
||||
description: str,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
|
||||
if row is None:
|
||||
return None
|
||||
now = _now()
|
||||
row.owner_user_id = "system"
|
||||
row.name = name
|
||||
row.description = description
|
||||
row.publication_status = "published"
|
||||
row.published_by = "system"
|
||||
row.reviewed_by = None
|
||||
row.published_at = row.published_at or now
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
async def delete_mapping(self, mapping_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def request_publish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
|
||||
if row is None or (not is_admin and row.owner_user_id != actor_user_id):
|
||||
return None
|
||||
row.publication_status = "published"
|
||||
row.published_by = actor_user_id
|
||||
row.reviewed_by = None
|
||||
row.published_at = _now()
|
||||
row.updated_at = _now()
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
|
||||
if row is None or row.publication_status != "pending":
|
||||
return None
|
||||
row.publication_status = "published" if approved else "rejected"
|
||||
row.reviewed_by = reviewer_user_id
|
||||
row.published_at = _now() if approved else None
|
||||
row.updated_at = _now()
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
|
||||
if row is None or (not is_admin and row.owner_user_id != actor_user_id):
|
||||
return None
|
||||
row.publication_status = "private"
|
||||
if is_admin:
|
||||
row.reviewed_by = actor_user_id
|
||||
row.published_at = None
|
||||
row.updated_at = _now()
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmWikiKnowledgeBaseRow).where(LlmWikiKnowledgeBaseRow.publication_status != "private")
|
||||
if status:
|
||||
stmt = stmt.where(LlmWikiKnowledgeBaseRow.publication_status == status)
|
||||
stmt = stmt.order_by(LlmWikiKnowledgeBaseRow.updated_at.desc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
|
||||
stmt = (
|
||||
select(LlmWikiKnowledgeBaseRow)
|
||||
.where(
|
||||
LlmWikiKnowledgeBaseRow.owner_user_id == "system",
|
||||
LlmWikiKnowledgeBaseRow.name == "对话沉淀",
|
||||
LlmWikiKnowledgeBaseRow.publication_status == "published",
|
||||
)
|
||||
.order_by(LlmWikiKnowledgeBaseRow.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _to_dict(row) if row else None
|
||||
|
||||
async def get_conversation_deposit(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
assistant_message_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiConversationDepositRow).where(
|
||||
LlmWikiConversationDepositRow.thread_id == thread_id,
|
||||
LlmWikiConversationDepositRow.assistant_message_id == assistant_message_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _to_dict(row) if row else None
|
||||
|
||||
async def upsert_conversation_deposit(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
human_message_id: str | None,
|
||||
assistant_message_id: str,
|
||||
knowledge_base_mapping_id: str,
|
||||
weknora_knowledge_id: str | None,
|
||||
title: str,
|
||||
created_by_user_id: str | None,
|
||||
) -> dict[str, Any]:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
stmt = select(LlmWikiConversationDepositRow).where(
|
||||
LlmWikiConversationDepositRow.thread_id == thread_id,
|
||||
LlmWikiConversationDepositRow.assistant_message_id == assistant_message_id,
|
||||
)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = LlmWikiConversationDepositRow(
|
||||
id=str(uuid4()),
|
||||
thread_id=thread_id,
|
||||
human_message_id=human_message_id,
|
||||
assistant_message_id=assistant_message_id,
|
||||
knowledge_base_mapping_id=knowledge_base_mapping_id,
|
||||
weknora_knowledge_id=weknora_knowledge_id,
|
||||
title=title,
|
||||
created_by_user_id=created_by_user_id,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.human_message_id = human_message_id
|
||||
row.knowledge_base_mapping_id = knowledge_base_mapping_id
|
||||
row.weknora_knowledge_id = weknora_knowledge_id
|
||||
row.title = title
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
async def delete_conversation_deposits(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
message_ids: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
conditions = [LlmWikiConversationDepositRow.thread_id == thread_id]
|
||||
if message_ids:
|
||||
conditions.append(
|
||||
or_(
|
||||
LlmWikiConversationDepositRow.human_message_id.in_(message_ids),
|
||||
LlmWikiConversationDepositRow.assistant_message_id.in_(message_ids),
|
||||
)
|
||||
)
|
||||
stmt = select(LlmWikiConversationDepositRow).where(*conditions)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
deleted = [_to_dict(row) for row in rows]
|
||||
if rows:
|
||||
await session.execute(delete(LlmWikiConversationDepositRow).where(LlmWikiConversationDepositRow.id.in_([row.id for row in rows])))
|
||||
await session.commit()
|
||||
return deleted
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
"""Persistence facade for the DeerFlow-local Wiki vector index."""
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
|
||||
|
||||
|
||||
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
|
||||
return (
|
||||
MemoryLlmWikiIndexStore()
|
||||
if session_factory is None
|
||||
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LlmWikiIndexStore",
|
||||
"MemoryLlmWikiIndexStore",
|
||||
"SqlLlmWikiIndexStore",
|
||||
"LlmWikiWikiPageRow",
|
||||
"LlmWikiWikiVectorRow",
|
||||
"LlmWikiWikiSyncStateRow",
|
||||
"make_llmwiki_index_store",
|
||||
]
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
"""Storage contract for local Wiki pages, vectors, revisions and sync leases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
|
||||
class LlmWikiIndexStore(ABC):
|
||||
@abstractmethod
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def replace_page_vectors(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
|
||||
|
||||
@abstractmethod
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
|
||||
|
||||
async def iter_vector_snapshot_rows(
|
||||
self,
|
||||
mapping_id: str,
|
||||
fingerprint: str,
|
||||
*,
|
||||
batch_size: int = 1000,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Iterate vector rows for streaming export.
|
||||
|
||||
Implementations with a database backend should override this method so
|
||||
very large vector packages do not need to be materialized in memory.
|
||||
"""
|
||||
snapshot = await self.load_vector_snapshot(mapping_id, fingerprint)
|
||||
for row in snapshot.get("rows") or []:
|
||||
if isinstance(row, dict):
|
||||
yield row
|
||||
|
||||
@abstractmethod
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def update_sync_progress(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int: ...
|
||||
+202
@@ -0,0 +1,202 @@
|
||||
"""In-memory local Wiki index store used by tests and DB-less mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self) -> None:
|
||||
self.pages: dict[str, dict[str, Any]] = {}
|
||||
self.vectors: dict[str, list[dict[str, Any]]] = {}
|
||||
self.states: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def _state(self, mapping_id: str) -> dict[str, Any]:
|
||||
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
|
||||
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
|
||||
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
if page_id in self.pages:
|
||||
self.pages[page_id]["last_seen_sync_id"] = sync_id
|
||||
self.pages[page_id]["last_error"] = None
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
page_id = existing["id"] if existing else str(uuid4())
|
||||
now = _now().isoformat()
|
||||
row = {
|
||||
**(existing or {}),
|
||||
**page,
|
||||
"id": page_id,
|
||||
"index_status": "ready",
|
||||
"indexed_fingerprint": fingerprint,
|
||||
"indexed_at": now,
|
||||
"last_error": None,
|
||||
"is_remote_deleted": False,
|
||||
"last_seen_sync_id": sync_id,
|
||||
"created_at": (existing or {}).get("created_at", now),
|
||||
"updated_at": now,
|
||||
}
|
||||
self.pages[page_id] = row
|
||||
self.vectors[page_id] = [{**item, "page_id": page_id, "knowledge_base_mapping_id": page["knowledge_base_mapping_id"], "embedding_fingerprint": fingerprint} for item in vectors]
|
||||
state = self._state(page["knowledge_base_mapping_id"])
|
||||
state["index_revision"] += 1
|
||||
state["embedding_fingerprint"] = fingerprint
|
||||
return dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
if existing:
|
||||
self.pages[existing["id"]].update(
|
||||
{
|
||||
"last_seen_sync_id": sync_id,
|
||||
"last_error": error,
|
||||
# Preserve the last atomically completed generation.
|
||||
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
|
||||
}
|
||||
)
|
||||
return
|
||||
page_id = str(uuid4())
|
||||
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
changed = 0
|
||||
for page in self.pages.values():
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
|
||||
page.update({"is_remote_deleted": True, "index_status": "deleted"})
|
||||
self.vectors.pop(page["id"], None)
|
||||
changed += 1
|
||||
if changed:
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return changed
|
||||
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
|
||||
rows = []
|
||||
for page_id, vectors in self.vectors.items():
|
||||
page = self.pages[page_id]
|
||||
if page["knowledge_base_mapping_id"] != mapping_id or page.get("is_remote_deleted") or page.get("status") not in {"published", "draft"} or page.get("index_status") != "ready":
|
||||
continue
|
||||
rows.extend({**vector, "page": dict(page)} for vector in vectors if vector["embedding_fingerprint"] == fingerprint)
|
||||
state = self._state(mapping_id)
|
||||
return {"revision": state["index_revision"], "fingerprint": fingerprint, "rows": rows}
|
||||
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
|
||||
state = self._state(mapping_id)
|
||||
return int(state["index_revision"]), state.get("embedding_fingerprint"), str(state.get("state") or "idle")
|
||||
|
||||
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
now = _now()
|
||||
expires = state.get("lease_expires_at")
|
||||
if state.get("lease_owner") is not None and isinstance(expires, datetime) and expires >= now:
|
||||
return False
|
||||
state.update(
|
||||
{
|
||||
"state": "rebuilding" if state.get("state") == "rebuild_required" else "running",
|
||||
"active_sync_id": sync_id,
|
||||
"lease_owner": owner,
|
||||
"lease_expires_at": now + timedelta(seconds=lease_seconds),
|
||||
"last_started_at": now,
|
||||
}
|
||||
)
|
||||
return True
|
||||
|
||||
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner:
|
||||
return False
|
||||
state["lease_expires_at"] = _now() + timedelta(seconds=lease_seconds)
|
||||
return True
|
||||
|
||||
async def update_sync_progress(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
return False
|
||||
state["remote_page_count"] = max(0, remote_page_count)
|
||||
state["updated_at"] = _now()
|
||||
return True
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
return
|
||||
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
|
||||
was_rebuilding = state.get("state") == "rebuilding"
|
||||
state.update(
|
||||
{
|
||||
"state": "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed",
|
||||
"lease_owner": None,
|
||||
"lease_expires_at": None,
|
||||
"active_sync_id": None,
|
||||
"last_error": error,
|
||||
"remote_page_count": remote_page_count,
|
||||
"local_page_count": len(pages),
|
||||
"ready_page_count": sum(p.get("index_status") == "ready" for p in pages),
|
||||
"failed_page_count": sum(p.get("index_status") in {"failed", "stale"} or p.get("last_error") is not None for p in pages),
|
||||
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
|
||||
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
|
||||
"last_success_at": _now() if success else state.get("last_success_at"),
|
||||
}
|
||||
)
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
wanted = set(mapping_ids or [])
|
||||
return [dict(value) for key, value in self.states.items() if not wanted or key in wanted]
|
||||
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for page in sorted(
|
||||
(
|
||||
page
|
||||
for page in self.pages.values()
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
|
||||
),
|
||||
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
|
||||
):
|
||||
result.append(
|
||||
{
|
||||
"id": page["id"],
|
||||
"slug": page["slug"],
|
||||
"title": page["title"],
|
||||
"page_type": page["page_type"],
|
||||
"status": page["status"],
|
||||
"index_status": page["index_status"],
|
||||
"vector_count": len(self.vectors.get(page["id"], [])),
|
||||
"indexed_at": page.get("indexed_at"),
|
||||
"last_error": page.get("last_error"),
|
||||
"last_seen_sync_id": page.get("last_seen_sync_id"),
|
||||
"updated_at": page.get("updated_at"),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
|
||||
for mapping_id in mapping_ids:
|
||||
state = self._state(mapping_id)
|
||||
state.update({"state": "rebuild_required", "embedding_fingerprint": fingerprint})
|
||||
for page in self.pages.values():
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted"):
|
||||
page["index_status"] = "stale"
|
||||
return len(mapping_ids)
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
"""Portable ORM rows for the DeerFlow-local WeKnora Wiki mirror/index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, false
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class LlmWikiWikiPageRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_pages"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
|
||||
Index("ix_llmwiki_wiki_page_mapping_visibility", "knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
remote_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
slug: Mapped[str] = mapped_column(String(1024), nullable=False)
|
||||
slug_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
title: Mapped[str] = mapped_column(String(512), nullable=False, default="", index=True)
|
||||
summary: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
content_md: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
page_type: Mapped[str] = mapped_column(String(32), nullable=False, default="article", index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="published", index=True)
|
||||
aliases: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
category_path: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
parent_slug: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
wiki_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
source_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
chunk_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
page_version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
remote_created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
remote_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
|
||||
index_status: Mapped[str] = mapped_column(String(24), nullable=False, default="pending", index=True)
|
||||
indexed_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
indexed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
is_remote_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
|
||||
last_seen_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now, index=True)
|
||||
|
||||
|
||||
class LlmWikiWikiVectorRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_vectors"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
|
||||
Index("ix_llmwiki_wiki_vector_snapshot", "knowledge_base_mapping_id", "embedding_fingerprint"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
page_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
section_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
embedding_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
|
||||
|
||||
class LlmWikiWikiSyncStateRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_sync_states"
|
||||
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True)
|
||||
state: Mapped[str] = mapped_column(String(24), nullable=False, default="idle")
|
||||
active_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_success_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
remote_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
local_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
ready_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
vector_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
|
||||
+400
@@ -0,0 +1,400 @@
|
||||
"""SQLAlchemy implementation of the local Wiki index store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import case, delete, func, insert, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _dict(row) -> dict[str, Any]:
|
||||
value = row.to_dict()
|
||||
for key, item in tuple(value.items()):
|
||||
if isinstance(item, datetime):
|
||||
value[key] = item.isoformat()
|
||||
return value
|
||||
|
||||
|
||||
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None:
|
||||
self._sf = session_factory
|
||||
self.write_batch_size = max(1, min(int(write_batch_size), 500))
|
||||
|
||||
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
|
||||
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if row is None:
|
||||
row = LlmWikiWikiSyncStateRow(knowledge_base_mapping_id=mapping_id, state="idle", index_revision=0, updated_at=_now())
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
return row
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
|
||||
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
|
||||
async with self._sf() as session:
|
||||
result = (await session.execute(stmt)).one_or_none()
|
||||
return {**_dict(result[0]), "vector_count": result[1]} if result else None
|
||||
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(LlmWikiWikiPageRow)
|
||||
.where(LlmWikiWikiPageRow.id == page_id)
|
||||
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
async with self._sf() as session:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is not None and row.slug != page["slug"]:
|
||||
raise ValueError("Wiki slug hash collision detected")
|
||||
now = _now()
|
||||
if row is None:
|
||||
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
|
||||
session.add(row)
|
||||
else:
|
||||
for key, value in page.items():
|
||||
setattr(row, key, value)
|
||||
row.index_status = "ready"
|
||||
row.indexed_fingerprint = fingerprint
|
||||
row.indexed_at = now
|
||||
row.last_error = None
|
||||
row.is_remote_deleted = False
|
||||
row.last_seen_sync_id = sync_id
|
||||
row.updated_at = now
|
||||
await session.flush()
|
||||
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
|
||||
vector_rows = [
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"page_id": row.id,
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"section_index": int(item["section_index"]),
|
||||
"heading": item.get("heading"),
|
||||
"section_content": str(item["section_content"]),
|
||||
"content_hash": str(item["content_hash"]),
|
||||
"embedding_fingerprint": fingerprint,
|
||||
"embedding_dimensions": int(item["embedding_dimensions"]),
|
||||
"vector_blob": bytes(item["vector_blob"]),
|
||||
"created_at": now,
|
||||
}
|
||||
for item in vectors
|
||||
]
|
||||
# Keep the page replacement atomic, but cap each SQL statement so
|
||||
# large Wiki pages cannot flood the database with one huge insert.
|
||||
for start in range(0, len(vector_rows), self.write_batch_size):
|
||||
await session.execute(
|
||||
insert(LlmWikiWikiVectorRow),
|
||||
vector_rows[start : start + self.write_batch_size],
|
||||
)
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.embedding_fingerprint = fingerprint
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return _dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
safe_error = error[:4000]
|
||||
async with self._sf() as session:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
now = _now()
|
||||
if row is None:
|
||||
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
|
||||
session.add(row)
|
||||
row.index_status = "failed"
|
||||
else:
|
||||
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))).scalar_one())
|
||||
# Preserve the last atomically completed generation while the
|
||||
# newer remote version remains pending after an embedding error.
|
||||
row.index_status = "ready" if vector_count else "failed"
|
||||
row.last_seen_sync_id = sync_id
|
||||
row.last_error = safe_error
|
||||
row.updated_at = now
|
||||
await self._ensure_state(session, mapping_id)
|
||||
await session.commit()
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
or_(LlmWikiWikiPageRow.last_seen_sync_id.is_(None), LlmWikiWikiPageRow.last_seen_sync_id != sync_id),
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
return 0
|
||||
now = _now()
|
||||
page_ids = [row.id for row in rows]
|
||||
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id.in_(page_ids)))
|
||||
for row in rows:
|
||||
row.is_remote_deleted = True
|
||||
row.index_status = "deleted"
|
||||
row.updated_at = now
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
|
||||
stmt = (
|
||||
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
|
||||
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
|
||||
.where(
|
||||
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
|
||||
LlmWikiWikiPageRow.index_status == "ready",
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
|
||||
)
|
||||
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
pairs = list((await session.execute(stmt)).all())
|
||||
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
|
||||
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
|
||||
|
||||
async def iter_vector_snapshot_rows(
|
||||
self,
|
||||
mapping_id: str,
|
||||
fingerprint: str,
|
||||
*,
|
||||
batch_size: int = 1000,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
batch = max(1, min(int(batch_size), 5000))
|
||||
offset = 0
|
||||
while True:
|
||||
stmt = (
|
||||
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
|
||||
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
|
||||
.where(
|
||||
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
|
||||
LlmWikiWikiPageRow.index_status == "ready",
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
|
||||
)
|
||||
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
|
||||
.limit(batch)
|
||||
.offset(offset)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
pairs = list((await session.execute(stmt)).all())
|
||||
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
yield row
|
||||
if len(rows) < batch:
|
||||
break
|
||||
offset += len(rows)
|
||||
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if state is None:
|
||||
return 0, None, "not_synced"
|
||||
return int(state.index_revision or 0), state.embedding_fingerprint, state.state
|
||||
|
||||
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
try:
|
||||
await self._ensure_state(session, mapping_id)
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Another worker created the one-per-KB state row between our
|
||||
# read and insert. Roll back this isolated bootstrap session;
|
||||
# the conditional UPDATE below is the actual lease election.
|
||||
await session.rollback()
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
or_(
|
||||
LlmWikiWikiSyncStateRow.lease_owner.is_(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at.is_(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at < now,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state=case((LlmWikiWikiSyncStateRow.state == "rebuild_required", "rebuilding"), else_="running"),
|
||||
active_sync_id=sync_id,
|
||||
lease_owner=owner,
|
||||
lease_expires_at=now + timedelta(seconds=lease_seconds),
|
||||
last_started_at=now,
|
||||
last_error=None,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiSyncStateRow.lease_owner == owner)
|
||||
.values(lease_expires_at=now + timedelta(seconds=lease_seconds), updated_at=now)
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def update_sync_progress(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiSyncStateRow.lease_owner == owner,
|
||||
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
|
||||
)
|
||||
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
|
||||
return
|
||||
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
|
||||
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
|
||||
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
|
||||
failed_count = int(
|
||||
(
|
||||
await session.execute(
|
||||
select(func.count())
|
||||
.select_from(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
*visible,
|
||||
or_(
|
||||
LlmWikiWikiPageRow.index_status.in_(["failed", "stale"]),
|
||||
LlmWikiWikiPageRow.last_error.is_not(None),
|
||||
),
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
)
|
||||
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id))).scalar_one())
|
||||
now = _now()
|
||||
was_rebuilding = state.state == "rebuilding"
|
||||
state.state = "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed"
|
||||
state.active_sync_id = None
|
||||
state.lease_owner = None
|
||||
state.lease_expires_at = None
|
||||
state.last_error = (error or "")[:4000] or None
|
||||
state.remote_page_count = remote_page_count
|
||||
state.local_page_count = local_count
|
||||
state.ready_page_count = ready_count
|
||||
state.failed_page_count = failed_count
|
||||
state.vector_count = vector_count
|
||||
if success and not error:
|
||||
state.last_completed_at = now
|
||||
if success:
|
||||
state.last_success_at = now
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmWikiWikiSyncStateRow)
|
||||
if mapping_ids:
|
||||
stmt = stmt.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id.in_(mapping_ids))
|
||||
stmt = stmt.order_by(LlmWikiWikiSyncStateRow.updated_at.desc())
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
|
||||
vector_counts = (
|
||||
select(
|
||||
LlmWikiWikiVectorRow.page_id.label("page_id"),
|
||||
func.count(LlmWikiWikiVectorRow.id).label("vector_count"),
|
||||
)
|
||||
.where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id)
|
||||
.group_by(LlmWikiWikiVectorRow.page_id)
|
||||
.subquery()
|
||||
)
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow, func.coalesce(vector_counts.c.vector_count, 0))
|
||||
.outerjoin(vector_counts, vector_counts.c.page_id == LlmWikiWikiPageRow.id)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
)
|
||||
.order_by(LlmWikiWikiPageRow.wiki_path, LlmWikiWikiPageRow.slug)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
result: list[dict[str, Any]] = []
|
||||
for row, vector_count in rows:
|
||||
page = _dict(row)
|
||||
result.append(
|
||||
{
|
||||
"id": page["id"],
|
||||
"slug": page["slug"],
|
||||
"title": page["title"],
|
||||
"page_type": page["page_type"],
|
||||
"status": page["status"],
|
||||
"index_status": page["index_status"],
|
||||
"vector_count": int(vector_count or 0),
|
||||
"indexed_at": page.get("indexed_at"),
|
||||
"last_error": page.get("last_error"),
|
||||
"last_seen_sync_id": page.get("last_seen_sync_id"),
|
||||
"updated_at": page.get("updated_at"),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
|
||||
if not mapping_ids:
|
||||
return 0
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
for mapping_id in mapping_ids:
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.state = "rebuild_required"
|
||||
state.embedding_fingerprint = fingerprint
|
||||
state.updated_at = now
|
||||
await session.execute(update(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(mapping_ids), LlmWikiWikiPageRow.is_remote_deleted.is_(False)).values(index_status="stale", updated_at=now))
|
||||
await session.commit()
|
||||
return len(mapping_ids)
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
"""Per-user memory configuration override persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.memory_config.base import UserMemoryConfigStore
|
||||
from deerflow.persistence.memory_config.memory import MemoryUserMemoryConfigStore
|
||||
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
|
||||
from deerflow.persistence.memory_config.sql import SqlUserMemoryConfigRepository
|
||||
|
||||
|
||||
def make_user_memory_config_store(session_factory) -> UserMemoryConfigStore:
|
||||
if session_factory is None:
|
||||
return MemoryUserMemoryConfigStore()
|
||||
return SqlUserMemoryConfigRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"UserMemoryConfigStore",
|
||||
"UserMemoryConfigRow",
|
||||
"MemoryUserMemoryConfigStore",
|
||||
"SqlUserMemoryConfigRepository",
|
||||
"make_user_memory_config_store",
|
||||
]
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
"""Abstract persistence API for per-user memory config overrides."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class UserMemoryConfigStore(ABC):
|
||||
|
||||
@abstractmethod
|
||||
async def get_recall_budget(self, user_id: str) -> str | None:
|
||||
"""Return the user's recall_budget override, or None if not set."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_recall_budget(self, user_id: str, value: str) -> None:
|
||||
"""Upsert the user's recall_budget override."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete(self, user_id: str) -> None:
|
||||
"""Remove all per-user overrides for the given user."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def load_all(self) -> dict[str, str]:
|
||||
"""Bulk-load all user recall_budget overrides.
|
||||
|
||||
Returns {user_id: recall_budget} for all rows where recall_budget IS NOT NULL.
|
||||
Called once at startup to warm the in-memory cache.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
Loaded 100 of 3008 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user