sync update from LAN repo (source master @ ca48c4ec)

This commit is contained in:
gozeros committed 2026-11-05 10:53:54 +08:00
1 parent 8bc8b0e082
commit ea9dcff9c0
873 files changed
+107866 -14011

No files matched your search

@@ -77,6 +77,11 @@ class AgentStore(ABC):
"""Return featured + still-visible-to-``user_id`` agents, ordered by ``featured_order`` asc."""
raise NotImplementedError
@abstractmethod
async def list_sidebar(self, user_id: str, *, limit: int = 3) -> list[dict[str, Any]]:
"""Return only the id/name fields needed by the workspace sidebar."""
raise NotImplementedError
@abstractmethod
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
@@ -133,6 +138,7 @@ class AgentStore(ABC):
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
hidden: bool | None = None,
) -> tuple[list[dict[str, Any]], int]:
"""Return one page of agents plus the total matching count.
@@ -143,6 +149,9 @@ class AgentStore(ABC):
- ``None`` — the default listing visibility (``list_visible``).
``admin=True`` bypasses the visibility/scope filter entirely (every
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
Admin-hidden agents (``hidden=True`` rows) are excluded from every
non-admin listing; under ``admin=True`` the ``hidden`` param selects
only hidden / only visible rows (``None`` = both).
``search`` matches the raw name OR its desensitized display form.
``tag_ids`` keeps agents carrying ANY of the tags (OR). ``owner`` keeps
@@ -167,8 +176,20 @@ class AgentStore(ABC):
raise NotImplementedError
@abstractmethod
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``."""
async def set_extras(
self,
agent_id: str,
*,
tool_groups: list[str] | None,
model: str | None,
chat_link: Any = None,
update_chat_link: bool = False,
) -> None:
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``.
``chat_link`` is written only when ``update_chat_link`` is true, so a
skills/model refresh does not clear a previously mirrored jump link.
"""
raise NotImplementedError
@abstractmethod
@@ -1,325 +1,398 @@
"""In-memory custom agent store for memory persistence mode."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.config.agent_name_desensitize import name_matches
from deerflow.persistence.agents.base import AgentStore
def _now() -> str:
return datetime.now(UTC).isoformat()
def _copy(data: dict[str, Any]) -> dict[str, Any]:
copied = dict(data)
copied["builtin"] = copied.get("user_id") is None
return copied
class MemoryAgentStore(AgentStore):
def __init__(self) -> None:
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._pins: set[tuple[str, str]] = set()
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
self._skills: dict[str, list[str]] = {}
self._extras: dict[str, dict[str, Any]] = {}
@staticmethod
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
"""Single-record visibility (``get_visible``) — the union of every
listing surface, so a list-visible agent always opens. Mirrors
``AgentRepository._visible_filter``: baseline visibility OR
private-square (password is the discovery gate, reads are not
re-gated) OR favorited (the "我的" listing keeps favorites even after
the owner unpublishes)."""
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
rows = [_copy(agent) for agent in self._agents.values() if self._is_visible(agent, user_id)]
rows.sort(key=lambda agent: agent.get("created_at") or "")
return rows
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or not self._is_fetchable(agent, user_id):
return None
return _copy(agent)
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
return _copy(agent) if agent is not None else None
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or agent.get("user_id") != user_id:
return None
return _copy(agent)
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
now = _now()
row = {
"id": data["id"],
"name": data["name"],
"description": data.get("description") or "",
"user_id": data.get("user_id"),
"published": bool(data.get("published", False)),
"is_private": bool(data.get("is_private", False)),
"square_id": str(data.get("square_id") or ""),
"featured_order": None,
"created_at": now,
"updated_at": now,
}
self._agents[row["id"]] = row
return _copy(row)
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
row = self._agents.get(data["id"])
if row is None:
row = {
"id": data["id"],
"name": data["name"],
"description": data.get("description") or "",
"user_id": None,
"published": bool(data.get("published", True)),
"featured_order": None,
"created_at": _now(),
"updated_at": _now(),
}
self._agents[row["id"]] = row
elif row.get("user_id") is None:
row["name"] = data["name"]
row["description"] = data.get("description") or ""
row["published"] = bool(data.get("published", True))
row["updated_at"] = _now()
return _copy(row)
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete(self, agent_id: str, user_id: str) -> bool:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return False
self._agents.pop(agent_id, None)
return True
async def delete_builtin(self, agent_id: str) -> bool:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return False
self._agents.pop(agent_id, None)
return True
async def count_owned(self, user_id: str) -> int:
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
row["featured_order"] = order
row["updated_at"] = _now()
return _copy(row)
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
candidates = [a for a in self._agents.values() if a.get("featured_order") is not None and self._is_visible(a, user_id)]
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
return [_copy(a) for a in candidates]
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
rows.sort(key=lambda a: order_map.get(a["id"], 0))
return [_copy(a) for a in rows]
async def list_private(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private"))]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def list_all(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values()]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete_any(self, agent_id: str) -> bool:
if agent_id not in self._agents:
return False
self._agents.pop(agent_id, None)
return True
# --- Paginated listing ---
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
if admin:
return True
if scope == "square":
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
if scope == "mine":
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
favorited = (user_id, agent["id"]) in self._favorites
return owned or favorited
if scope == "private":
return bool(agent.get("is_private"))
if scope == "builtin":
# 内置智能体 = user_id IS NULL(仅管理员;路由层 admin 闸门)。
return agent.get("user_id") is None
return self._is_visible(agent, user_id)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
) -> tuple[list[dict[str, Any]], int]:
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
owner_key = owner.strip() if owner and owner.strip() else None
square_key = square_id.strip() if square_id and square_id.strip() else None
def _square_ok(a: dict[str, Any]) -> bool:
if square_key is None:
return True
sid = str(a.get("square_id") or "")
return sid == square_key or (square_is_default and sid == "")
matched = [
a
for a in self._agents.values()
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
and (owner_key is None or a.get("user_id") == owner_key)
and _square_ok(a)
]
if sort == "time_asc":
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
elif sort == "updated_desc":
matched.sort(
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
reverse=True,
)
elif sort == "updated_asc":
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
elif sort == "name_asc":
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
elif sort == "name_desc":
matched.sort(
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
reverse=True,
)
else:
matched.sort(
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
reverse=True,
)
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
total = len(matched)
start = max(page - 1, 0) * page_size
page_rows = matched[start : start + page_size]
return [_copy(a) for a in page_rows], total
# --- Listing cache (skills / tool_groups / model) ---
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
self._extras[agent_id] = {"tool_groups": tool_groups, "model": model}
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
out: dict[str, dict[str, Any]] = {}
for agent_id in agent_ids:
has_extras = agent_id in self._extras
has_skills = agent_id in self._skills
if not has_extras and not has_skills:
continue
extras = self._extras.get(agent_id, {})
out[agent_id] = {
"skills": list(self._skills.get(agent_id, [])),
"tool_groups": extras.get("tool_groups"),
"model": extras.get("model"),
}
return out
async def delete_extras_for_agent(self, agent_id: str) -> None:
self._skills.pop(agent_id, None)
self._extras.pop(agent_id, None)
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
self._favorites.add((user_id, agent_id))
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
if (user_id, agent_id) in self._favorites:
self._favorites.discard((user_id, agent_id))
return True
return False
async def list_favorite_ids(self, user_id: str) -> set[str]:
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
self._pins.add((user_id, agent_id))
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
key = (user_id, agent_id)
if key not in self._pins:
return False
self._pins.remove(key)
return True
async def list_pin_ids(self, user_id: str) -> set[str]:
return {agent_id for uid, agent_id in self._pins if uid == user_id}
async def delete_pins_for_agent(self, agent_id: str) -> None:
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
"""In-memory custom agent store for memory persistence mode."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.config.agent_name_desensitize import name_matches
from deerflow.persistence.agents.base import AgentStore
def _now() -> str:
return datetime.now(UTC).isoformat()
def _copy(data: dict[str, Any]) -> dict[str, Any]:
copied = dict(data)
copied["builtin"] = copied.get("user_id") is None
return copied
class MemoryAgentStore(AgentStore):
def __init__(self) -> None:
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._pins: set[tuple[str, str]] = set()
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
self._skills: dict[str, list[str]] = {}
self._extras: dict[str, dict[str, Any]] = {}
@staticmethod
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
"""Single-record visibility (``get_visible``) — the union of every
listing surface, so a list-visible agent always opens. Mirrors
``AgentRepository._visible_filter``: baseline visibility OR
private-square (password is the discovery gate, reads are not
re-gated) OR favorited (the "我的" listing keeps favorites even after
the owner unpublishes)."""
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
rows = [
_copy(agent)
for agent in self._agents.values()
if self._is_visible(agent, user_id) and not agent.get("hidden")
]
rows.sort(key=lambda agent: agent.get("created_at") or "")
return rows
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or not self._is_fetchable(agent, user_id):
return None
return _copy(agent)
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
return _copy(agent) if agent is not None else None
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or agent.get("user_id") != user_id:
return None
return _copy(agent)
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
now = _now()
row = {
"id": data["id"],
"name": data["name"],
"description": data.get("description") or "",
"user_id": data.get("user_id"),
"published": bool(data.get("published", False)),
"is_private": bool(data.get("is_private", False)),
"square_id": str(data.get("square_id") or ""),
"hidden": bool(data.get("hidden", False)),
"featured_order": None,
"created_at": now,
"updated_at": now,
}
self._agents[row["id"]] = row
return _copy(row)
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
row = self._agents.get(data["id"])
if row is None:
row = {
"id": data["id"],
"name": data["name"],
"description": data.get("description") or "",
"user_id": None,
"published": bool(data.get("published", True)),
"hidden": bool(data.get("hidden", False)),
"featured_order": None,
"created_at": _now(),
"updated_at": _now(),
}
self._agents[row["id"]] = row
elif row.get("user_id") is None:
row["name"] = data["name"]
row["description"] = data.get("description") or ""
row["published"] = bool(data.get("published", True))
row["updated_at"] = _now()
return _copy(row)
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return None
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return None
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete(self, agent_id: str, user_id: str) -> bool:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return False
self._agents.pop(agent_id, None)
return True
async def delete_builtin(self, agent_id: str) -> bool:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return False
self._agents.pop(agent_id, None)
return True
async def count_owned(self, user_id: str) -> int:
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
row["featured_order"] = order
row["updated_at"] = _now()
return _copy(row)
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
candidates = [
a
for a in self._agents.values()
if a.get("featured_order") is not None and self._is_visible(a, user_id) and not a.get("hidden")
]
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
return [_copy(a) for a in candidates]
async def list_sidebar(self, user_id: str, *, limit: int = 3) -> list[dict[str, Any]]:
mine = [
a for a in self._agents.values()
if not a.get("hidden")
and ((a.get("user_id") == user_id) or (user_id, a["id"]) in self._favorites)
]
square = [
a for a in self._agents.values()
if not a.get("hidden")
and (a.get("user_id") is None or a.get("published"))
and not a.get("is_private")
]
for rows in (mine, square):
rows.sort(
key=lambda a: (str(a.get("created_at") or ""), str(a.get("id") or "")),
reverse=True,
)
rows.sort(key=lambda a: (user_id, a["id"]) in self._pins, reverse=True)
out: list[dict[str, Any]] = []
seen: set[str] = set()
def add(agent: dict[str, Any]) -> None:
if len(out) >= limit or agent["id"] in seen:
return
seen.add(agent["id"])
out.append({"id": agent["id"], "name": agent.get("name") or ""})
for agent in [*mine, *square]:
if (user_id, agent["id"]) in self._pins:
add(agent)
for agent in mine:
add(agent)
for agent in square:
add(agent)
return out
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
rows.sort(key=lambda a: order_map.get(a["id"], 0))
return [_copy(a) for a in rows]
async def list_private(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private")) and not a.get("hidden")]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def list_all(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values()]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete_any(self, agent_id: str) -> bool:
if agent_id not in self._agents:
return False
self._agents.pop(agent_id, None)
return True
# --- Paginated listing ---
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
if admin:
return True # every agent — the hidden filter is a separate param
if agent.get("hidden"):
return False # admin-hidden agents vanish from every non-admin scope
if scope == "square":
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
if scope == "mine":
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
favorited = (user_id, agent["id"]) in self._favorites
return owned or favorited
if scope == "private":
return bool(agent.get("is_private"))
if scope == "builtin":
# 内置智能体 = user_id IS NULL(仅管理员;路由层 admin 闸门)。
return agent.get("user_id") is None
return self._is_visible(agent, user_id)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
hidden: bool | None = None,
) -> tuple[list[dict[str, Any]], int]:
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
owner_key = owner.strip() if owner and owner.strip() else None
square_key = square_id.strip() if square_id and square_id.strip() else None
def _square_ok(a: dict[str, Any]) -> bool:
if square_key is None:
return True
sid = str(a.get("square_id") or "")
return sid == square_key or (square_is_default and sid == "")
def _hidden_ok(a: dict[str, Any]) -> bool:
if admin:
# Admin console filter: keep only hidden / only visible agents.
return hidden is None or bool(a.get("hidden")) == hidden
return not a.get("hidden")
matched = [
a
for a in self._agents.values()
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
and _hidden_ok(a)
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
and (owner_key is None or a.get("user_id") == owner_key)
and _square_ok(a)
]
if sort == "time_asc":
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
elif sort == "updated_desc":
matched.sort(
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
reverse=True,
)
elif sort == "updated_asc":
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
elif sort == "name_asc":
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
elif sort == "name_desc":
matched.sort(
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
reverse=True,
)
else:
matched.sort(
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
reverse=True,
)
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
total = len(matched)
start = max(page - 1, 0) * page_size
page_rows = matched[start : start + page_size]
return [_copy(a) for a in page_rows], total
# --- Listing cache (skills / tool_groups / model) ---
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
async def set_extras(
self,
agent_id: str,
*,
tool_groups: list[str] | None,
model: str | None,
chat_link: Any = None,
update_chat_link: bool = False,
) -> None:
previous = self._extras.get(agent_id, {})
stored = {
"tool_groups": tool_groups,
"model": model,
"chat_link": previous.get("chat_link"),
}
if update_chat_link:
stored["chat_link"] = chat_link
self._extras[agent_id] = stored
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
out: dict[str, dict[str, Any]] = {}
for agent_id in agent_ids:
has_extras = agent_id in self._extras
has_skills = agent_id in self._skills
if not has_extras and not has_skills:
continue
extras = self._extras.get(agent_id, {})
out[agent_id] = {
"skills": list(self._skills.get(agent_id, [])),
"tool_groups": extras.get("tool_groups"),
"model": extras.get("model"),
"chat_link": extras.get("chat_link"),
}
return out
async def delete_extras_for_agent(self, agent_id: str) -> None:
self._skills.pop(agent_id, None)
self._extras.pop(agent_id, None)
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
self._favorites.add((user_id, agent_id))
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
if (user_id, agent_id) in self._favorites:
self._favorites.discard((user_id, agent_id))
return True
return False
async def list_favorite_ids(self, user_id: str) -> set[str]:
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
self._pins.add((user_id, agent_id))
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
key = (user_id, agent_id)
if key not in self._pins:
return False
self._pins.remove(key)
return True
async def list_pin_ids(self, user_id: str) -> set[str]:
return {agent_id for uid, agent_id in self._pins if uid == user_id}
async def delete_pins_for_agent(self, agent_id: str) -> None:
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -13,6 +13,13 @@ from deerflow.persistence.types import BeijingDateTime, PortableJSON
class AgentRow(Base):
__tablename__ = "agents"
# Composite indexes for the gallery tabs. Single-column indexes cannot
# serve "广场" (hidden + private + square) or "我的" (owner + hidden)
# without a wide scan once the catalog grows.
__table_args__ = (
Index("ix_agents_catalog_square", "hidden", "is_private", "square_id", "created_at"),
Index("ix_agents_owner_hidden", "user_id", "hidden", "created_at"),
)
# Stored as a plain string so the same id can be used as the filesystem
# directory name under {base_dir}/agents/{id}/.
@@ -27,6 +34,14 @@ class AgentRow(Base):
# visible to its owner in the "我的" tab, and visible to anyone in the
# private-square listing once the password has been verified.
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# When True the agent is hidden from every non-admin listing surface
# (gallery / 我的 / 广场 / homepage selector / private square). Only the
# admin agents console (?admin=true) still lists it. Purely a catalog
# switch: single-record access (detail / chat / run) keeps working, so
# existing conversations with a hidden agent are unaffected.
# server_default keeps the startup ORM-column sweep able to ADD COLUMN
# on existing deployments (NOT NULL needs a default).
hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"), index=True)
# Publish square (发布广场) this agent is filed under. A pure organizational
# category with no access control — independent of ``published`` /
# ``is_private``. Empty string = "not assigned" → the listing layer treats
@@ -124,4 +139,8 @@ class AgentExtrasRow(Base):
# list[str] | None — the agent's tool_groups whitelist from config.yaml.
tool_groups: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True, default=None)
model: Mapped[str | None] = mapped_column(String(191), nullable=True, default=None)
# Gallery 「对话」 jump target mirrored from config.yaml so the list endpoint
# can redirect without reading every agent file. Nullable JSON so existing
# databases pick the column up on the startup ORM sweep.
chat_link: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True, default=None)
skills_synced_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,23 @@
"""MySQL-persisted administrator switch for the optional API Redis cache."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, Boolean, Integer, String, text, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ApiReadCacheSettingRow(Base):
__tablename__ = "api_read_cache_settings"
key: Mapped[str] = mapped_column(String(32), primary_key=True, default="default")
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
ttl_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=300, server_default=text("300"))
collection_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
cache_epoch: Mapped[str] = mapped_column(String(32), nullable=False, default="0", server_default="0")
browser_status_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true())
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -4,7 +4,7 @@ A "business mapping" row binds a fixed business code (3Q / 6BF / 3q-0…3q-6 /
6-1…6-4 / 7BF / 7-1…7-4 / 8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
account can resolve it). Used to auto-pick the business chain when a task page
launches the multi-agent roundtable.
launches the multi-agent roundtable. ``3q-1…3q-6`` are seeded as custom-type rows.
Unlike ``roundtable_chains`` (per-user), this mapping is **global / shared** —
there is no ``user_id`` column; all accounts see and use the same configuration.
@@ -20,7 +20,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String
from sqlalchemy import Boolean, Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -34,6 +34,16 @@ class BusinessMappingRow(Base):
business_code: Mapped[str] = mapped_column(String(32), primary_key=True)
# Display label shown in the 业务 column (e.g. "任务FX").
label: Mapped[str] = mapped_column(String(128), nullable=False, default="")
# A user-visible category. Built-in rows keep ``内置业务``; rows created from
# the configuration page consistently use ``自定义类型``.
mapping_type: Mapped[str] = mapped_column(
String(64), nullable=False, default="内置业务", server_default="内置业务"
)
# Built-in rows are seeded and cannot be redefined. Custom rows are created
# by an administrator and their business_code doubles as a login goPath.
is_custom: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default="0"
)
# Picked 智能体 (暂存不用,自动流程当前不消费此字段). NULL = 未配置.
agent_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
agent_name: Mapped[str | None] = mapped_column(String(191), nullable=True)
@@ -11,7 +11,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.business_mapping.model import BusinessMappingRow
@@ -21,21 +21,25 @@ _UNSET = object()
# Fixed business rows (业务码 → 展示名). Seeded on first boot; agent/chain left empty.
# 8BF 页面尚未建好,先占位保留行。顺序即管理表的展示顺序。
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
# b5-1 = 外部写入本系统详情后,由本系统读取的任务分析单智能体深链。
# 3q-0…3q-6 / 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
# 3q-1…3q-6 默认按「自定义类型」落库(goPath 走自定义深链);配置管理页默认不展示这 6 行。
_CUSTOM_SLOT = {"is_custom": True, "mapping_type": "自定义类型"}
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
{"business_code": "3Q", "label": "3QFX"},
{"business_code": "6BF", "label": "任务FX"},
{"business_code": "7BF", "label": "XDFX"},
{"business_code": "8BF", "label": "8BF"},
{"business_code": "AFX", "label": "AGENTFX"},
{"business_code": "b5-1", "label": "任务分析"},
# 独立配置位,与上方业务码无关;固定排在表尾。
{"business_code": "3q-0", "label": "3q-0"},
{"business_code": "3q-1", "label": "3q-1"},
{"business_code": "3q-2", "label": "3q-2"},
{"business_code": "3q-3", "label": "3q-3"},
{"business_code": "3q-4", "label": "3q-4"},
{"business_code": "3q-5", "label": "3q-5"},
{"business_code": "3q-6", "label": "3q-6"},
{"business_code": "3q-1", "label": "3q-1", **_CUSTOM_SLOT},
{"business_code": "3q-2", "label": "3q-2", **_CUSTOM_SLOT},
{"business_code": "3q-3", "label": "3q-3", **_CUSTOM_SLOT},
{"business_code": "3q-4", "label": "3q-4", **_CUSTOM_SLOT},
{"business_code": "3q-5", "label": "3q-5", **_CUSTOM_SLOT},
{"business_code": "3q-6", "label": "3q-6", **_CUSTOM_SLOT},
{"business_code": "6-1", "label": "6-1"},
{"business_code": "6-2", "label": "6-2"},
{"business_code": "6-3", "label": "6-3"},
@@ -47,6 +51,14 @@ DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
]
def _seed_type(seed: dict[str, Any]) -> tuple[bool, str]:
is_custom = bool(seed.get("is_custom", False))
mapping_type = str(seed.get("mapping_type") or "").strip()
if not mapping_type:
mapping_type = "自定义类型" if is_custom else "内置业务"
return is_custom, mapping_type
class BusinessMappingRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -56,6 +68,8 @@ class BusinessMappingRepository:
return {
"business_code": row.business_code,
"label": row.label,
"mapping_type": row.mapping_type or "内置业务",
"is_custom": bool(row.is_custom),
"agent_id": row.agent_id,
"agent_name": row.agent_name,
"chain_id": row.chain_id,
@@ -108,18 +122,62 @@ class BusinessMappingRepository:
await session.refresh(row)
return self._to_dict(row)
async def create_custom_mapping(
self,
*,
business_code: str,
mapping_type: str,
) -> dict[str, Any] | None:
"""Create one administrator-defined mapping.
``business_code`` is deliberately the externally supplied goPath. The
case-insensitive uniqueness check mirrors frontend deep-link matching,
preventing two rows that could resolve from the same login URL.
"""
normalized_code = business_code.strip()
normalized_type = mapping_type.strip()
async with self._sf() as session:
existing = await session.scalar(
select(BusinessMappingRow).where(
func.lower(BusinessMappingRow.business_code) == normalized_code.lower()
)
)
if existing is not None:
return None
max_sort_order = await session.scalar(select(func.max(BusinessMappingRow.sort_order)))
now = datetime.now(UTC)
row = BusinessMappingRow(
business_code=normalized_code,
label=normalized_code,
mapping_type=normalized_type,
is_custom=True,
agent_id=None,
agent_name=None,
chain_id=None,
chain_title=None,
sort_order=(int(max_sort_order) if max_sort_order is not None else -1) + 1,
created_at=now,
updated_at=now,
)
session.add(row)
await session.commit()
await session.refresh(row)
return self._to_dict(row)
async def ensure_seed_defaults(self) -> int:
"""Idempotently insert the fixed business rows if missing. Returns inserted count.
Never overwrites an existing row's agent/chain (respects user config).
Re-syncs ``sort_order`` (and empty ``label``) so newly inserted codes keep
the intended table order next to older seeded rows.
Re-syncs ``sort_order``, empty ``label``, and seed ``is_custom`` /
``mapping_type`` so newly inserted codes keep the intended table order
and 3q-1…3q-6 stay classified as custom type.
"""
inserted = 0
dirty = False
now = datetime.now(UTC)
async with self._sf() as session:
for idx, seed in enumerate(DEFAULT_BUSINESS_ROWS):
is_custom, mapping_type = _seed_type(seed)
existing = await session.get(BusinessMappingRow, seed["business_code"])
if existing is not None:
if existing.sort_order != idx:
@@ -129,11 +187,19 @@ class BusinessMappingRepository:
if seed_label and not (existing.label or "").strip():
existing.label = seed_label
dirty = True
if bool(existing.is_custom) != is_custom:
existing.is_custom = is_custom
dirty = True
if (existing.mapping_type or "") != mapping_type:
existing.mapping_type = mapping_type
dirty = True
continue
session.add(
BusinessMappingRow(
business_code=seed["business_code"],
label=seed.get("label", ""),
mapping_type=mapping_type,
is_custom=is_custom,
agent_id=None,
agent_name=None,
chain_id=None,
@@ -22,6 +22,7 @@ def _normalize(data: dict[str, Any], menu_type: str, updated_by: str | None) ->
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
"sort_order": int(data.get("sort_order", 0)),
"label": (data.get("label") or None),
"disabled": bool(data.get("disabled", False)),
"updated_by": updated_by,
"updated_at": datetime.now(UTC).isoformat(),
}
@@ -1,14 +1,15 @@
"""ORM model for compact/sidebar-summary menu overrides.
This table only controls the compact left menu presentation. The full sidebar
menu layout continues to use ``menu_overrides`` and is unaffected.
This table only controls compact menu presentation: lane, order, label and
enablement for the left menu, settings menu and toolbox. The full sidebar menu
layout continues to use ``menu_overrides`` and is unaffected.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, Integer, String
from sqlalchemy import Boolean, Index, Integer, String, false
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -24,6 +25,7 @@ class CompactMenuOverrideRow(Base):
placement: Mapped[str] = mapped_column(String(32), default="sidebar")
sort_order: Mapped[int] = mapped_column(Integer, default=0)
label: Mapped[str | None] = mapped_column(String(255))
disabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
updated_by: Mapped[str | None] = mapped_column(String(64))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
@@ -41,6 +41,7 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C
placement=placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
sort_order=int(data.get("sort_order", 0)),
label=(data.get("label") or None),
disabled=bool(data.get("disabled", False)),
updated_by=updated_by,
)
@@ -53,6 +54,7 @@ def _normalize_user_row(data: dict[str, Any], menu_type: str) -> dict[str, Any]:
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
"sort_order": int(data.get("sort_order", 0)),
"label": (data.get("label") or None),
"disabled": bool(data.get("disabled", False)),
}
@@ -11,14 +11,20 @@ None and fall back to in-memory implementations.
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import os
import re
import sys
import time
import weakref
from contextvars import ContextVar, Token
from importlib.util import find_spec
from pathlib import Path
from typing import Any
from sqlalchemy import text
from sqlalchemy import event, text
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
@@ -38,21 +44,167 @@ _session_factory: async_sessionmaker[AsyncSession] | None = None
# "got Future attached to a different loop". We capture the engine's home loop here so
# best-effort writes coming from worker threads / other loops can be routed back to it
# (see ``fire_and_forget_db_write``).
_engine_loop: asyncio.AbstractEventLoop | None = None
# These are the complete workflow-studio persistence tables owned by DeerFlow.
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
# wildcard: another application might share a schema and have its own table
# with that prefix.
_WORKFLOW_SCHEMA_TABLE_NAMES = (
"workflow_definitions",
"workflow_versions",
"workflow_runs",
"workflow_node_runs",
"workflow_run_events",
"workflow_run_artifacts",
"workflow_data_sources",
)
_engine_loop: asyncio.AbstractEventLoop | None = None
_performance_request_context: ContextVar[tuple[str, Any] | None] = ContextVar("deerflow_performance_request", default=None)
def bind_performance_request(request_id: str) -> Token:
"""Bind a request ID to this task so slow SQL logs can be correlated safely."""
try:
task = asyncio.current_task()
except RuntimeError:
task = None
return _performance_request_context.set((request_id, weakref.ref(task) if task is not None else None))
def reset_performance_request(token: Token) -> None:
_performance_request_context.reset(token)
def _current_performance_request_id() -> str:
context = _performance_request_context.get()
if context is None:
return "-"
try:
task = asyncio.current_task()
except RuntimeError:
task = None
return context[0] if task is not None and context[1] is not None and context[1]() is task else "-"
def _slow_sql_threshold_ms() -> float:
try:
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_SQL_LOG_MS", "500")), 60_000.0))
except (TypeError, ValueError):
return 500.0
def _sql_operation(statement: str) -> str:
parts = statement.lstrip().split(None, 1)
candidate = parts[0].upper() if parts else "EMPTY"
return candidate if re.fullmatch(r"[A-Z_]+", candidate) else "OTHER"
def _sql_table(statement: str) -> str:
"""Return the target/source table only, without query text or values."""
match = re.search(
r"\b(?:INSERT\s+INTO|UPDATE|DELETE\s+FROM)\s+([`\"\w.]+)",
statement,
re.IGNORECASE,
)
if match is None:
match = re.search(r"\bFROM\s+([`\"\w.]+)", statement, re.IGNORECASE)
if match is None:
return "unknown"
return match.group(1).strip("`\".")[-128:] or "unknown"
def _sql_fingerprint(statement: str) -> str:
return hashlib.sha256(statement.encode("utf-8", "replace")).hexdigest()[:12]
def _database_error_metadata(exception: BaseException | None) -> tuple[str, str, str]:
"""Extract safe driver diagnostics without logging SQL or bound values."""
if exception is None:
return "unknown", "-", "-"
code = getattr(exception, "errno", None) or getattr(exception, "code", None)
args = getattr(exception, "args", ())
if code is None and args and isinstance(args[0], (int, str)):
candidate = str(args[0])
if candidate.isdigit():
code = candidate
sqlstate = getattr(exception, "sqlstate", None) or getattr(exception, "pgcode", None)
diagnostic = getattr(exception, "diag", None)
constraint = getattr(diagnostic, "constraint_name", None)
def _safe_identifier(value: Any) -> str:
text_value = str(value or "")
return text_value[:128] if re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", text_value) else "-"
return _safe_identifier(code), _safe_identifier(sqlstate), _safe_identifier(constraint)
def _install_slow_sql_logging(engine: AsyncEngine, backend: str) -> None:
"""Log slow SQL timings without logging query text or bound values."""
sync_engine = engine.sync_engine
def before_cursor_execute(_conn, _cursor, statement, _parameters, context, _executemany):
if context is not None:
context._deerflow_perf_started = time.perf_counter()
context._deerflow_perf_operation = _sql_operation(statement)
context._deerflow_perf_statement = statement
def after_cursor_execute(_conn, _cursor, _statement, _parameters, context, executemany):
if context is None:
return
started = getattr(context, "_deerflow_perf_started", None)
if started is None:
return
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
if elapsed_ms >= _slow_sql_threshold_ms():
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", ""))
logger.warning(
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%.2f executemany=%s fingerprint=%s outcome=ok",
_current_performance_request_id(),
backend,
getattr(context, "_deerflow_perf_operation", "OTHER"),
_sql_table(getattr(context, "_deerflow_perf_statement", "")),
elapsed_ms,
bool(executemany),
fingerprint,
)
def handle_error(exception_context):
context = exception_context.execution_context
started = getattr(context, "_deerflow_perf_started", None) if context is not None else None
elapsed_ms = round((time.perf_counter() - started) * 1000, 2) if started is not None else None
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
original_exception = exception_context.original_exception
error_code, sqlstate, constraint = _database_error_metadata(original_exception)
table = _sql_table(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
duplicate_key = error_code == "1062" or sqlstate == "23505" or (
table == "scheduled_task_runs"
and "UNIQUE constraint failed" in str(original_exception or "")
)
log = logger.info if table == "scheduled_task_runs" and duplicate_key else logger.error
outcome = "duplicate_ignored" if table == "scheduled_task_runs" and duplicate_key else "error"
log(
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%s executemany=%s fingerprint=%s outcome=%s error_type=%s error_code=%s sqlstate=%s constraint=%s",
_current_performance_request_id(),
backend,
getattr(context, "_deerflow_perf_operation", "OTHER") if context is not None else "OTHER",
table,
elapsed_ms,
bool(getattr(context, "executemany", False)) if context is not None else False,
fingerprint,
outcome,
type(original_exception).__name__ if original_exception is not None else "unknown",
error_code,
sqlstate,
constraint,
)
event.listen(sync_engine, "before_cursor_execute", before_cursor_execute)
event.listen(sync_engine, "after_cursor_execute", after_cursor_execute)
event.listen(sync_engine, "handle_error", handle_error)
# These are the complete workflow-studio persistence tables owned by DeerFlow.
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
# wildcard: another application might share a schema and have its own table
# with that prefix.
_WORKFLOW_SCHEMA_TABLE_NAMES = (
"workflow_definitions",
"workflow_versions",
"workflow_runs",
"workflow_node_runs",
"workflow_run_events",
"workflow_run_artifacts",
"workflow_data_sources",
)
def _add_runtime_vendor_path() -> None:
@@ -81,6 +233,42 @@ def _mysql_driver_name(url: str) -> str:
return "asyncmy"
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
kwargs = dict(
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
# These are not MySQL/psycopg connection arguments.
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
kwargs["connect_args"] = {"command_timeout": command_timeout}
return kwargs
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
kwargs = dict(
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
# These are not MySQL/psycopg connection arguments.
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
kwargs["connect_args"] = {"command_timeout": command_timeout}
return kwargs
async def _run_alembic_upgrade(url: str) -> None:
"""Apply pending Alembic migrations against ``url`` (``alembic upgrade head``).
@@ -176,7 +364,7 @@ async def _auto_create_postgres_db(url: str) -> None:
try:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
finally:
await maint_engine.dispose()
@@ -215,7 +403,7 @@ async def _auto_create_mysql_db(url: str) -> None:
await maint_engine.dispose()
def _ensure_orm_columns_sync(sync_conn) -> None:
def _ensure_orm_columns_sync(sync_conn) -> None:
"""Add ORM-declared columns that are missing from already-existing tables.
``Base.metadata.create_all`` creates brand-new tables but never patches
@@ -226,14 +414,14 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
This sweep compares each existing table against the ORM metadata and issues
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
safely (nullable, or carrying a server_default). Missing ORM indexes are
created as well. Table-level ``UniqueConstraint`` objects
safely (nullable, or carrying a server_default). Missing ORM indexes are
created as well. Table-level ``UniqueConstraint`` objects
that are declared on the model but missing from the table are also created
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
on an existing database, weakening dedupe.
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
``BeijingDateTime``) resolve to the right backend type. Everything is
per-column best-effort: a failure is logged and startup continues.
@@ -258,10 +446,8 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
continue
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
missing = [c for c in table.columns if c.name not in existing_cols]
if not missing:
continue
for col in missing:
for col in missing:
if not col.nullable and col.server_default is None:
logger.warning(
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
@@ -270,21 +456,21 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
)
continue
try:
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
logger.info("Auto-added missing column %s.%s", table.name, col.name)
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
logger.info("Auto-added missing column %s.%s", table.name, col.name)
except Exception:
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if idx.name in existing_idx:
continue
try:
idx.create(sync_conn)
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if not idx.name or idx.name in existing_idx:
continue
try:
idx.create(sync_conn)
logger.info("Auto-created index %s on %s", idx.name, table.name)
except Exception:
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
@@ -328,67 +514,67 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
f"ON {preparer.quote(table.name)} ({cols})"
)
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
Alembic auto-upgrade is intentionally disabled in deployed Gateway
processes, while the ORM column synchronizer is always run. The latter
can add the task lookup columns and indexes but cannot derive values from
JSON metadata. Backfill task rows once at startup so an upgrade never
temporarily hides every historical task conversation from ``/threads/search``.
"""
from sqlalchemy import inspect as sa_inspect
table_name = "threads_meta"
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
try:
inspector = sa_inspect(sync_conn)
if table_name not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns(table_name)}
if not required_columns <= columns:
return
dialect = sync_conn.dialect.name
if dialect == "sqlite":
task_id = "json_extract(metadata_json, '$.taskId')"
agent_id = "json_extract(metadata_json, '$.agent_id')"
action_id = "json_extract(metadata_json, '$.actionId')"
valid_json = "json_valid(metadata_json)"
elif dialect == "mysql":
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
valid_json = "JSON_VALID(metadata_json)"
elif dialect in {"postgresql", "opengauss"}:
task_id = "metadata_json ->> 'taskId'"
agent_id = "metadata_json ->> 'agent_id'"
action_id = "metadata_json ->> 'actionId'"
valid_json = "TRUE"
else:
return
result = sync_conn.execute(
text(
"UPDATE threads_meta "
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
)
)
if result.rowcount:
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
except Exception:
# A malformed legacy JSON row or a backend-specific JSON function must
# not prevent startup. The generic metadata search remains available,
# and the failure is visible to the operator for remediation.
logger.exception("Failed to backfill thread task lookup projections")
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
Alembic auto-upgrade is intentionally disabled in deployed Gateway
processes, while the ORM column synchronizer is always run. The latter
can add the task lookup columns and indexes but cannot derive values from
JSON metadata. Backfill task rows once at startup so an upgrade never
temporarily hides every historical task conversation from ``/threads/search``.
"""
from sqlalchemy import inspect as sa_inspect
table_name = "threads_meta"
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
try:
inspector = sa_inspect(sync_conn)
if table_name not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns(table_name)}
if not required_columns <= columns:
return
dialect = sync_conn.dialect.name
if dialect == "sqlite":
task_id = "json_extract(metadata_json, '$.taskId')"
agent_id = "json_extract(metadata_json, '$.agent_id')"
action_id = "json_extract(metadata_json, '$.actionId')"
valid_json = "json_valid(metadata_json)"
elif dialect == "mysql":
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
valid_json = "JSON_VALID(metadata_json)"
elif dialect in {"postgresql", "opengauss"}:
task_id = "metadata_json ->> 'taskId'"
agent_id = "metadata_json ->> 'agent_id'"
action_id = "metadata_json ->> 'actionId'"
valid_json = "TRUE"
else:
return
result = sync_conn.execute(
text(
"UPDATE threads_meta "
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
)
)
if result.rowcount:
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
except Exception:
# A malformed legacy JSON row or a backend-specific JSON function must
# not prevent startup. The generic metadata search remains available,
# and the failure is visible to the operator for remediation.
logger.exception("Failed to backfill thread task lookup projections")
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
@@ -441,24 +627,32 @@ async def init_engine(
pool_size: int = 5,
max_overflow: int = 10,
pool_timeout: int = 30,
pool_recycle: int = 1800,
pool_recycle: int | None = None,
command_timeout: float | None = 30,
sqlite_dir: str = "",
) -> None:
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
echo: Echo SQL to log.
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
command_timeout: asyncpg command deadline, including pool pre-ping (None disables).
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
global _engine, _session_factory, _engine_loop
if pool_recycle is None:
pool_recycle = 300 if backend == "postgres" else 1800
if pool_recycle is None:
pool_recycle = 300 if backend == "postgres" else 1800
if backend == "memory":
logger.info("Persistence backend=memory -- ORM engine not initialized")
return
@@ -475,25 +669,25 @@ async def init_engine(
except RuntimeError:
_engine_loop = None
if backend in {"postgres", "gauss"}:
if backend in {"postgres", "gauss"}:
try:
import asyncpg # noqa: F401
except ImportError:
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "gauss":
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
install_gauss_dialect_compatibility()
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "gauss":
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
install_gauss_dialect_compatibility()
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
@@ -509,10 +703,6 @@ async def init_engine(
install_gauss_mysql_dialect_compatibility(driver)
if backend == "sqlite":
import os
from sqlalchemy import event
os.makedirs(sqlite_dir or ".", exist_ok=True)
_engine = create_async_engine(url, echo=echo, json_serializer=_json_serializer)
@@ -537,31 +727,28 @@ async def init_engine(
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend in {"postgres", "gauss"}:
elif backend in {"postgres", "gauss"}:
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
elif backend == "mysql":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
else:
raise ValueError(f"Unknown persistence backend: {backend!r}")
_install_slow_sql_logging(_engine, backend)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
# Auto-create tables for fresh deployments, then let Alembic apply any
@@ -612,14 +799,13 @@ async def init_engine(
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
_install_slow_sql_logging(_engine, backend)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await _create_all_with_product_integrity(conn)
@@ -630,14 +816,13 @@ async def init_engine(
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
_install_slow_sql_logging(_engine, backend)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await _create_all_with_product_integrity(conn)
@@ -675,7 +860,7 @@ async def init_engine(
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
async def init_engine_from_config(config) -> None:
@@ -690,14 +875,17 @@ async def init_engine_from_config(config) -> None:
pool_size=config.pool_size,
max_overflow=getattr(config, "max_overflow", 10),
pool_timeout=getattr(config, "pool_timeout", 30),
pool_recycle=getattr(config, "pool_recycle", 1800),
pool_recycle=getattr(config, "pool_recycle", None),
command_timeout=getattr(config, "command_timeout", 30),
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
)
def get_session_factory() -> async_sessionmaker[AsyncSession] | None:
"""Return the async session factory, or None if backend=memory."""
return _session_factory
from deerflow.persistence.loop_isolation import isolated_session_factory
return isolated_session_factory() or _session_factory
def get_engine() -> AsyncEngine | None:
@@ -705,73 +893,73 @@ def get_engine() -> AsyncEngine | None:
return _engine
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
"""Return the event loop that owns the engine's connection pool (or None)."""
return _engine_loop
async def rebuild_workflow_schema_for_mysql() -> list[str]:
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
This is a deliberately narrow recovery operation for an unused Workflow
Studio installation. It never uses a prefix wildcard, never changes the
selected database, and refuses to run unless the engine is connected to
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
Returns the allow-listed workflow tables that existed before the rebuild.
"""
engine = get_engine()
if engine is None:
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
if engine.dialect.name != "mysql":
raise RuntimeError(
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
)
expected_database = engine.url.database
if not expected_database:
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
# Make the operation safe when this helper is invoked independently from
# normal app startup as well.
import deerflow.persistence.models # noqa: F401 — register ORM models
from deerflow.persistence.base import Base
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
if missing_models:
raise RuntimeError(
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
)
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
def _rebuild(sync_conn) -> list[str]:
from sqlalchemy import inspect as sa_inspect
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
if current_database != expected_database:
raise RuntimeError(
"Refusing workflow schema rebuild: connected database "
f"{current_database!r} does not match configured database {expected_database!r}"
)
existing = set(sa_inspect(sync_conn).get_table_names())
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
# SQLAlchemy orders parent/child tables around the foreign key from
# workflow_versions to workflow_definitions. Only tables in the exact
# allow-list above are ever passed to drop_all/create_all.
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
return present
async with engine.begin() as conn:
rebuilt = await conn.run_sync(_rebuild)
logger.warning(
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
expected_database,
", ".join(rebuilt) if rebuilt else "none (created fresh)",
)
return rebuilt
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
"""Return the event loop that owns the engine's connection pool (or None)."""
return _engine_loop
async def rebuild_workflow_schema_for_mysql() -> list[str]:
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
This is a deliberately narrow recovery operation for an unused Workflow
Studio installation. It never uses a prefix wildcard, never changes the
selected database, and refuses to run unless the engine is connected to
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
Returns the allow-listed workflow tables that existed before the rebuild.
"""
engine = get_engine()
if engine is None:
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
if engine.dialect.name != "mysql":
raise RuntimeError(
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
)
expected_database = engine.url.database
if not expected_database:
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
# Make the operation safe when this helper is invoked independently from
# normal app startup as well.
import deerflow.persistence.models # noqa: F401 — register ORM models
from deerflow.persistence.base import Base
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
if missing_models:
raise RuntimeError(
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
)
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
def _rebuild(sync_conn) -> list[str]:
from sqlalchemy import inspect as sa_inspect
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
if current_database != expected_database:
raise RuntimeError(
"Refusing workflow schema rebuild: connected database "
f"{current_database!r} does not match configured database {expected_database!r}"
)
existing = set(sa_inspect(sync_conn).get_table_names())
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
# SQLAlchemy orders parent/child tables around the foreign key from
# workflow_versions to workflow_definitions. Only tables in the exact
# allow-list above are ever passed to drop_all/create_all.
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
return present
async with engine.begin() as conn:
rebuilt = await conn.run_sync(_rebuild)
logger.warning(
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
expected_database,
", ".join(rebuilt) if rebuilt else "none (created fresh)",
)
return rebuilt
def fire_and_forget_db_write(coro) -> None:
@@ -820,10 +1008,13 @@ def _run_coro_on_throwaway_loop(coro):
Only reached when there is no usable engine home loop, or when the caller is
*synchronously* inside the home loop (routing back would deadlock). This
touches the shared connection pool from a throwaway loop, so it can still leak
a dead-loop-bound connection — but the common worker-thread path no longer
gets here (see ``run_db_blocking``).
uses an isolated, non-pooled engine for compatibility reads. The coroutine
must obtain its session factory inside its async body so the isolated
context applies; no connection is returned to the Gateway pool.
"""
from deerflow.persistence.loop_isolation import run_isolated_db_read
coro = run_isolated_db_read(coro)
try:
asyncio.get_running_loop()
in_loop = True
@@ -0,0 +1,160 @@
"""Bound JSON predicates for SQLite, PostgreSQL and MySQL.
Adapted from upstream #2865. MySQL business tables store JSON in TEXT, so
these predicates use JSON functions rather than assuming a native JSON column.
Unsupported keys/values must stay in the repository's residual filter; never
silently discard a predicate (in particular a shared task's taskId filter).
"""
from __future__ import annotations
import math
import re
from typing import Any
from sqlalchemy import BigInteger, Float, String, bindparam
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.visitors import InternalTraversal
from sqlalchemy.types import Boolean
_SAFE_KEY = re.compile(r"[A-Za-z0-9_-]+\Z")
_STRIP_CHARS = "".join(chr(i) for i in range(0x3001) if chr(i).isspace())
def supports_json_filter(key: Any, value: Any) -> bool:
if not isinstance(key, str) or not _SAFE_KEY.fullmatch(key):
return False
if value is None or isinstance(value, (str, bool)):
return True
if isinstance(value, int):
return -(2**63) <= value < 2**63
return isinstance(value, float) and math.isfinite(value)
class JsonPredicate(ColumnElement):
"""A scalar equality, Python truthiness, or nonblank-string predicate."""
inherit_cache = True
type = Boolean()
_is_implicitly_boolean = True
_traverse_internals = [
("column", InternalTraversal.dp_clauseelement),
("key", InternalTraversal.dp_string),
("operation", InternalTraversal.dp_string),
("value", InternalTraversal.dp_plain_obj),
("value_kind", InternalTraversal.dp_string),
]
def __init__(self, column, key: str, operation: str = "equal", value: Any = None):
if not supports_json_filter(key, value):
raise ValueError("Unsupported JSON filter key or value")
if operation not in {"equal", "truthy", "nonblank"}:
raise ValueError("Unsupported JSON predicate operation")
self.column = column
self.key = key
self.operation = operation
self.value = value
# Keep Python value types separate in SQLAlchemy's compiled cache.
self.value_kind = type(value).__name__
super().__init__()
def _compile(element, compiler, family, **kw):
if not _SAFE_KEY.fullmatch(element.key):
raise ValueError("Unsafe JSON key")
column = compiler.process(element.column, **kw)
def bound(value, type_=String()):
return compiler.process(bindparam(None, value, type_=type_), **kw)
if family == "postgres":
raw = f"({column} -> '{element.key}')"
kind = f"json_typeof({raw})"
text = f"({column} ->> '{element.key}')"
string_kind, number_kinds = "string", "'number'"
else:
path = f"'$.\"{element.key}\"'"
raw = f"json_extract({column}, {path})"
if family == "mysql":
kind, text = f"lower(json_type({raw}))", f"json_unquote({raw})"
string_kind, number_kinds = "string", "'integer', 'double', 'decimal'"
else:
kind, text = f"json_type({column}, {path})", raw
string_kind, number_kinds = "text", "'integer', 'real'"
if element.operation == "nonblank":
if family == "mysql":
# MySQL 5.7 regexes are byte-oriented; a Unicode whitespace regex
# can misclassify CJK text. Removing each complete whitespace
# character preserves str.strip()'s nonempty test on 5.7 and 8.x.
stripped = text
for char in _STRIP_CHARS:
stripped = f"replace({stripped}, {bound(char)}, '')"
has_text = f"length({stripped}) > 0"
else:
trim = "btrim" if family == "postgres" else "trim"
# SQLite length(TEXT) stops at NUL, unlike Python str.strip().
has_text = f"{trim}({text}, {bound(_STRIP_CHARS)}) <> ''"
clause = f"({kind} = '{string_kind}' AND {has_text})"
elif element.operation == "truthy":
if family == "sqlite":
containers = f"json({raw}) NOT IN ('[]', '{{}}')"
boolean = f"{kind} = 'true'"
elif family == "mysql":
containers = f"json_length({raw}) > 0"
boolean = f"{text} = 'true'"
else:
containers = f"CAST({raw} AS jsonb) NOT IN ('[]'::jsonb, '{{}}'::jsonb)"
boolean = f"{text} = 'true'"
cast = "NUMERIC" if family == "postgres" else "DOUBLE PRECISION"
number = f"({text} + 0e0)" if family == "mysql" else f"CAST({text} AS {cast})"
nonempty = f"{text} <> ''" if family == "sqlite" else f"length({text}) > 0"
clause = (
f"CASE WHEN {kind} IN ({number_kinds}) THEN {number} <> 0 "
f"WHEN {kind} = '{string_kind}' THEN {nonempty} "
f"WHEN {kind} IN ('array', 'object') THEN {containers} "
f"WHEN {kind} IN ('boolean', 'true', 'false') THEN {boolean} ELSE FALSE END"
)
elif element.value is None:
# Preserve this project's md.get(key) == None semantics for old rows.
clause = f"({kind} IS NULL OR {kind} = 'null')"
elif isinstance(element.value, (bool, int, float)):
if family != "sqlite":
raise NotImplementedError("Server numeric metadata requires the exact residual filter")
value = int(element.value) if isinstance(element.value, bool) else element.value
# Preserve md.get(key) == value: Python considers True == 1 == 1.0.
# Text ids remain distinct from numbers. SQLite compares its extracted
# integers/reals natively; server numeric filters use the residual path.
param = bound(value, Float() if isinstance(value, float) else BigInteger())
clause = f"({kind} IN ({number_kinds}, 'true', 'false') AND {text} = {param})"
else:
# Binary comparison is essential for task/agent ids: MySQL TEXT can
# inherit a case-insensitive collation and ignores trailing spaces.
extract = f"CAST({text} AS BINARY)" if family == "mysql" else text
param = bound(element.value)
if family == "mysql":
param = f"CAST({param} AS BINARY)"
clause = f"({kind} = '{string_kind}' AND {extract} = {param})"
# Missing/null values must remain false under NOT (system exclusions).
return f"COALESCE(({clause}), FALSE)"
@compiles(JsonPredicate, "sqlite")
def _sqlite(element, compiler, **kw):
return _compile(element, compiler, "sqlite", **kw)
@compiles(JsonPredicate, "postgresql")
def _postgres(element, compiler, **kw):
return _compile(element, compiler, "postgres", **kw)
@compiles(JsonPredicate, "mysql")
def _mysql(element, compiler, **kw):
return _compile(element, compiler, "mysql", **kw)
@compiles(JsonPredicate)
def _unsupported(element, compiler, **kw):
raise NotImplementedError(f"JSON predicates unsupported for {compiler.dialect.name}")
@@ -0,0 +1,437 @@
"""Private knowledge-base Q&A sessions with shared Markdown publications.
This persistence unit deliberately contains no Gateway/application imports. A
session and its LangGraph thread belong to exactly one user. Once published,
only the saved Markdown snapshot is shared with readers of the bound knowledge
base; the private draft and thread identifier never become publication data.
"""
from __future__ import annotations
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from copy import deepcopy
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import String, delete, select, update
from sqlalchemy.exc import OperationalError, ProgrammingError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class KnowledgeResearchSessionRow(Base):
__tablename__ = "knowledge_research_sessions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
# A separate opaque id is the only identity exposed on the shared bookshelf.
publication_id: Mapped[str] = mapped_column(String(36), unique=True, index=True)
thread_id: Mapped[str] = mapped_column(String(128), unique=True, index=True)
knowledge_base_id: Mapped[str] = mapped_column(String(128), index=True)
user_id: Mapped[str] = mapped_column(String(255), index=True)
title: Mapped[str] = mapped_column(String(255))
draft_markdown: Mapped[str] = mapped_column(PortableLongText, default="")
wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
wiki_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
wiki_folder_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
saved_title: Mapped[str | None] = mapped_column(String(255), nullable=True)
saved_markdown: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
saved_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
saved_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
nullable=False,
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
nullable=False,
)
def _is_missing_table(exc: Exception) -> bool:
message = str(exc).lower()
return (
"no such table" in message
or "doesn't exist" in message
or "does not exist" in message
or "unknown table" in message
)
def _iso(value: Any) -> Any:
return value.isoformat() if isinstance(value, datetime) else value
def _row_dict(row: KnowledgeResearchSessionRow | dict[str, Any]) -> dict[str, Any]:
if isinstance(row, dict):
return deepcopy(row)
return {
"id": row.id,
"publication_id": row.publication_id,
"thread_id": row.thread_id,
"knowledge_base_id": row.knowledge_base_id,
"user_id": row.user_id,
"title": row.title,
"draft_markdown": row.draft_markdown or "",
"wiki_slug": row.wiki_slug,
"wiki_page_id": row.wiki_page_id,
"wiki_folder_id": row.wiki_folder_id,
"saved_title": row.saved_title,
"saved_markdown": row.saved_markdown,
"saved_hash": row.saved_hash,
"saved_at": _iso(row.saved_at),
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
class _PublicationLease:
"""One transaction-scoped mutable publication row."""
def __init__(self, row: KnowledgeResearchSessionRow | dict[str, Any]) -> None:
self._row = row
def snapshot(self) -> dict[str, Any]:
return _row_dict(self._row)
def save(
self,
*,
title: str,
markdown: str,
saved_hash: str,
wiki_slug: str,
wiki_page_id: str | None,
wiki_folder_id: str | None,
) -> dict[str, Any]:
now = datetime.now(UTC)
values = {
"wiki_slug": wiki_slug,
"wiki_page_id": wiki_page_id,
"wiki_folder_id": wiki_folder_id,
"saved_title": title,
"saved_markdown": markdown,
"saved_hash": saved_hash,
"saved_at": now,
"updated_at": now,
}
if isinstance(self._row, dict):
self._row.update(
{
**values,
"saved_at": now.isoformat(),
"updated_at": now.isoformat(),
}
)
else:
for key, value in values.items():
setattr(self._row, key, value)
return self.snapshot()
class KnowledgeResearchStore:
"""SQL store with a behaviorally equivalent memory fallback."""
def __init__(
self,
session_factory: async_sessionmaker[AsyncSession] | None,
) -> None:
self._sf = session_factory
self._rows: dict[str, dict[str, Any]] = {}
self._publication_locks: dict[str, asyncio.Lock] = {}
@asynccontextmanager
async def locked_publication(
self,
session_id: str,
user_id: str,
) -> AsyncIterator[_PublicationLease | None]:
"""Serialize version-check + remote write + local publication commit.
SQL deployments hold a row lock for the context lifetime, so separate
application workers cannot both pass the same expected-hash check.
SQLite has no ``FOR UPDATE`` support, so a no-op update first acquires
its database write lock. The memory fallback uses one asyncio lock per
research session and commits a copy only on successful exit.
"""
if self._sf is None:
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
async with lock:
original = self._rows.get(session_id)
if original is None or original["user_id"] != user_id:
yield None
return
working = deepcopy(original)
lease = _PublicationLease(working)
yield lease
self._rows[session_id] = working
return
async with self._sf() as session:
async with session.begin():
bind = session.get_bind()
if bind.dialect.name == "sqlite":
await session.execute(
update(KnowledgeResearchSessionRow)
.where(
KnowledgeResearchSessionRow.id == session_id,
KnowledgeResearchSessionRow.user_id == user_id,
)
.values(
updated_at=KnowledgeResearchSessionRow.updated_at
)
)
row = (
await session.execute(
select(KnowledgeResearchSessionRow)
.where(
KnowledgeResearchSessionRow.id == session_id,
KnowledgeResearchSessionRow.user_id == user_id,
)
.with_for_update()
)
).scalar_one_or_none()
yield _PublicationLease(row) if row is not None else None
async def create(
self,
*,
knowledge_base_id: str,
user_id: str,
title: str,
session_id: str | None = None,
publication_id: str | None = None,
thread_id: str | None = None,
) -> dict[str, Any]:
now = datetime.now(UTC)
row = KnowledgeResearchSessionRow(
id=session_id or str(uuid4()),
publication_id=publication_id or str(uuid4()),
thread_id=thread_id or str(uuid4()),
knowledge_base_id=knowledge_base_id,
user_id=user_id,
title=title,
draft_markdown="",
created_at=now,
updated_at=now,
)
if self._sf is None:
data = _row_dict(row)
self._rows[row.id] = data
return deepcopy(data)
async with self._sf() as session:
session.add(row)
await session.commit()
return _row_dict(row)
async def get(self, session_id: str) -> dict[str, Any] | None:
if self._sf is None:
row = self._rows.get(session_id)
return deepcopy(row) if row else None
async with self._sf() as session:
row = await session.get(KnowledgeResearchSessionRow, session_id)
return _row_dict(row) if row else None
async def get_owned(self, session_id: str, user_id: str) -> dict[str, Any] | None:
row = await self.get(session_id)
return row if row and row["user_id"] == user_id else None
async def get_by_thread(self, thread_id: str) -> dict[str, Any] | None:
if self._sf is None:
for row in self._rows.values():
if row["thread_id"] == thread_id:
return deepcopy(row)
return None
async with self._sf() as session:
try:
row = (
await session.execute(
select(KnowledgeResearchSessionRow).where(
KnowledgeResearchSessionRow.thread_id == thread_id
)
)
).scalar_one_or_none()
except (OperationalError, ProgrammingError) as exc:
# Ordinary Q&A hits this lookup on every run. A missing table
# (older DB before create_all registered the model) must not
# abort /runs/stream; treat it as "not a research thread".
if _is_missing_table(exc):
return None
raise
return _row_dict(row) if row else None
async def get_publication(self, publication_id: str) -> dict[str, Any] | None:
if self._sf is None:
for row in self._rows.values():
if row["publication_id"] == publication_id and row.get("saved_hash"):
return deepcopy(row)
return None
async with self._sf() as session:
row = (
await session.execute(
select(KnowledgeResearchSessionRow).where(
KnowledgeResearchSessionRow.publication_id == publication_id,
KnowledgeResearchSessionRow.saved_hash.is_not(None),
)
)
).scalar_one_or_none()
return _row_dict(row) if row else None
async def list_owned(
self,
knowledge_base_id: str,
user_id: str,
) -> list[dict[str, Any]]:
if self._sf is None:
rows = [
deepcopy(row)
for row in self._rows.values()
if row["knowledge_base_id"] == knowledge_base_id
and row["user_id"] == user_id
]
return sorted(rows, key=lambda item: str(item["updated_at"]), reverse=True)
async with self._sf() as session:
rows = (
await session.execute(
select(KnowledgeResearchSessionRow)
.where(
KnowledgeResearchSessionRow.knowledge_base_id
== knowledge_base_id,
KnowledgeResearchSessionRow.user_id == user_id,
)
.order_by(KnowledgeResearchSessionRow.updated_at.desc())
)
).scalars()
return [_row_dict(row) for row in rows]
async def list_published(
self,
knowledge_base_id: str,
) -> list[dict[str, Any]]:
if self._sf is None:
rows = [
deepcopy(row)
for row in self._rows.values()
if row["knowledge_base_id"] == knowledge_base_id
and row.get("saved_hash")
]
return sorted(
rows,
key=lambda item: str(item.get("saved_at") or item["updated_at"]),
reverse=True,
)
async with self._sf() as session:
rows = (
await session.execute(
select(KnowledgeResearchSessionRow)
.where(
KnowledgeResearchSessionRow.knowledge_base_id
== knowledge_base_id,
KnowledgeResearchSessionRow.saved_hash.is_not(None),
)
.order_by(KnowledgeResearchSessionRow.saved_at.desc())
)
).scalars()
return [_row_dict(row) for row in rows]
async def update_owned(
self,
session_id: str,
user_id: str,
*,
title: str | None = None,
draft_markdown: str | None = None,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
if self._sf is None:
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
async with lock:
row = self._rows.get(session_id)
if row is None or row["user_id"] != user_id:
return None
if title is not None:
row["title"] = title
if draft_markdown is not None:
row["draft_markdown"] = draft_markdown
row["updated_at"] = now.isoformat()
return deepcopy(row)
async with self._sf() as session:
row = (
await session.execute(
select(KnowledgeResearchSessionRow)
.where(
KnowledgeResearchSessionRow.id == session_id,
KnowledgeResearchSessionRow.user_id == user_id,
)
.with_for_update()
)
).scalar_one_or_none()
if row is None:
return None
if title is not None:
row.title = title
if draft_markdown is not None:
row.draft_markdown = draft_markdown
row.updated_at = now
await session.commit()
return _row_dict(row)
async def save_publication(
self,
session_id: str,
user_id: str,
*,
title: str,
markdown: str,
saved_hash: str,
wiki_slug: str,
wiki_page_id: str | None,
wiki_folder_id: str | None,
) -> dict[str, Any] | None:
async with self.locked_publication(session_id, user_id) as publication:
if publication is None:
return None
return publication.save(
title=title,
markdown=markdown,
saved_hash=saved_hash,
wiki_slug=wiki_slug,
wiki_page_id=wiki_page_id,
wiki_folder_id=wiki_folder_id,
)
async def delete(self, session_id: str) -> None:
if self._sf is None:
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
async with lock:
self._rows.pop(session_id, None)
return
async with self._sf() as session:
await session.execute(
delete(KnowledgeResearchSessionRow).where(
KnowledgeResearchSessionRow.id == session_id
)
)
await session.commit()
def make_knowledge_research_store(session_factory):
return KnowledgeResearchStore(session_factory)
__all__ = [
"KnowledgeResearchSessionRow",
"KnowledgeResearchStore",
"make_knowledge_research_store",
]
@@ -15,6 +15,19 @@ class DuplicateLightAppNameError(ValueError):
class LightAppStore(ABC):
async def list_sidebar_apps(self) -> list[dict[str, Any]]:
"""Return only the columns needed to render iframe app menu entries."""
rows = await self.list_apps(app_type="iframe", enabled_only=True)
return [
{
"app_id": row.get("app_id"),
"name": row.get("name"),
"mount_location": row.get("mount_location"),
"sort_order": row.get("sort_order", 0),
}
for row in rows
]
@abstractmethod
async def list_apps(
self,
@@ -65,6 +65,22 @@ class LightAppRepository(LightAppStore):
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def list_sidebar_apps(self) -> list[dict[str, Any]]:
"""Avoid loading descriptions, icons, route JSON, tags, and audit data."""
stmt = (
select(
LightAppRow.app_id,
LightAppRow.name,
LightAppRow.mount_location,
LightAppRow.sort_order,
)
.where(LightAppRow.app_type == "iframe", LightAppRow.enabled.is_(True))
.order_by(LightAppRow.sort_order.asc(), LightAppRow.created_at.desc())
)
async with self._sf() as session:
result = await session.execute(stmt)
return [dict(row._mapping) for row in result]
async def get_app(self, app_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LightAppRow, app_id)
@@ -67,3 +67,12 @@ class LlmMetricsStore(ABC):
async def iter_all(self, query: LlmMetricsQuery):
"""Yield rows matching ``query`` without pagination — used by export."""
raise NotImplementedError
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
"""Delete one bounded batch of audit rows older than ``cutoff``.
The default keeps lightweight/in-memory test implementations compatible.
SQL-backed stores override it for the scheduled admin-audit retention
sweep.
"""
return 0
@@ -7,7 +7,7 @@ from collections.abc import AsyncIterator
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import desc, func, select
from sqlalchemy import delete, desc, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery, LlmMetricsStore
@@ -110,3 +110,34 @@ class SqlLlmMetricsStore(LlmMetricsStore):
if len(page) < page_size:
return
offset += page_size
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
"""Delete one small, oldest-first retention batch.
Selecting primary keys first keeps this portable across SQLite, MySQL,
and PostgreSQL (which do not share ``DELETE ... LIMIT`` syntax) and
avoids a long-running delete transaction on a busy audit table.
"""
# Keep below SQLite's traditional 999 bind-variable limit. The
# retention worker requests batches repeatedly, so portability is more
# valuable than one giant delete statement.
size = max(1, min(int(batch_size), 500))
normalized_cutoff = _normalize_created_at(cutoff)
async with self._sf() as session:
ids = list(
(
await session.scalars(
select(LlmCallMetricRow.id)
.where(LlmCallMetricRow.created_at < normalized_cutoff)
.order_by(LlmCallMetricRow.created_at.asc(), LlmCallMetricRow.id.asc())
.limit(size)
)
).all()
)
if not ids:
return 0
result = await session.execute(
delete(LlmCallMetricRow).where(LlmCallMetricRow.id.in_(ids))
)
await session.commit()
return int(result.rowcount or 0)
@@ -0,0 +1,29 @@
"""Keep synchronous compatibility reads off the Gateway's async connection pool."""
import asyncio
from contextvars import ContextVar
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.pool import NullPool
_isolated_factory = ContextVar("isolated_db_factory", default=None)
def isolated_session_factory():
return _isolated_factory.get()
async def run_isolated_db_read(coro):
from deerflow.persistence.engine import get_engine
source = get_engine()
if source is None:
return await coro
# Never return a connection created on this temporary loop to the main pool.
engine = create_async_engine(source.url, poolclass=NullPool)
token = _isolated_factory.set(async_sessionmaker(engine, expire_on_commit=False))
try:
return await asyncio.wait_for(coro, 30)
finally:
_isolated_factory.reset(token)
await engine.dispose()
@@ -0,0 +1,28 @@
"""Helpers shared by the online Alembic environment and its tests."""
from __future__ import annotations
import os
from typing import Any
from sqlalchemy import text
def configure_mysql_migration_lock_waits(connection: Any) -> None:
"""Set bounded MySQL DDL lock waits without leaving a transaction open.
SQLAlchemy starts an implicit transaction for ``SET SESSION`` on MySQL.
Alembic's migration transaction then reuses it, and closing the connection
rolls back the ``alembic_version`` updates even though MySQL DDL itself has
already committed. Commit the session-local settings before Alembic starts
its transaction so successful migrations are recorded durably.
"""
try:
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
except ValueError:
lock_wait = 15
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
connection.commit()
@@ -9,14 +9,13 @@ from __future__ import annotations
import asyncio
import logging
import os
from logging.config import fileConfig
from alembic import context
from sqlalchemy import text
from sqlalchemy.ext.asyncio import create_async_engine
from deerflow.persistence.base import Base
from deerflow.persistence.migration_utils import configure_mysql_migration_lock_waits
# Import all models so metadata is populated.
try:
@@ -57,12 +56,7 @@ def do_run_migrations(connection):
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
if connection.dialect.name == "mysql" and not is_gauss_mysql_product():
try:
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
except ValueError:
lock_wait = 15
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
configure_mysql_migration_lock_waits(connection)
context.configure(
connection=connection,
@@ -0,0 +1,60 @@
"""Add snapshots supplied to the b5-1 task-analysis deep link.
Revision ID: 20261021_01
Revises: 20261020_01
Create Date: 2026-10-20
This file originally shipped with revision id ``20261020_01`` — the same id as
``20261020_01_knowledge_management_advanced_tools`` — which left the migration
graph with a duplicate revision and no single head. Re-issued as
``20261021_01`` chained after the canonical ``20261020_01``; its upgrade is
fully guarded (``if _has_table(): return``) so databases that already created
the table under the old id re-run it as a no-op.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261021_01"
down_revision = "20261020_01"
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_details"
def _has_table() -> bool:
return _TABLE in inspect(op.get_bind()).get_table_names()
def _json_type() -> sa.types.TypeEngine:
# Keep the migration consistent with PortableJSON, whose MySQL form is
# TEXT for compatibility with older deployments.
return sa.Text() if op.get_bind().dialect.name == "mysql" else sa.JSON()
def upgrade() -> None:
if _has_table():
return
op.create_table(
_TABLE,
sa.Column("detail_type", sa.String(32), primary_key=True),
sa.Column("detail_id", sa.String(128), primary_key=True),
sa.Column("task_name", sa.String(512), nullable=False),
sa.Column("task_content", sa.Text(), nullable=False),
sa.Column("overview", sa.Text(), nullable=False),
sa.Column("extra_json", _json_type(), nullable=False),
sa.Column("version", sa.Integer(), nullable=False),
sa.Column("created_by", sa.String(64), nullable=True),
sa.Column("updated_by", sa.String(64), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
)
def downgrade() -> None:
if _has_table():
op.drop_table(_TABLE)
@@ -0,0 +1,66 @@
"""Admin catalog switch: ``agents.hidden``.
Hidden agents disappear from every non-admin listing surface (gallery / 我的 /
广场 / homepage selector / private square) while staying fully usable — only
the admin agents console (?admin=true) keeps listing them.
The Gateway's startup ORM-column sweep also adds this column on existing
databases (Alembic auto-upgrade is disabled there), so this migration mostly
serves environments that run Alembic manually. Fully guarded and idempotent.
Revision ID: 20261022_03
Revises: 20261021_01
Create Date: 2026-10-22
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261022_03"
down_revision = "20261021_01"
branch_labels = None
depends_on = None
_TABLE = "agents"
_COLUMN = "hidden"
_INDEX = "ix_agents_hidden"
def _has_table(table_name: str) -> bool:
return table_name in inspect(op.get_bind()).get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
if _has_table(_TABLE) and not _has_column(_TABLE, _COLUMN):
# server_default mirrors the ORM declaration so the sweep on live
# databases and this migration produce the same NOT NULL column.
op.add_column(
_TABLE,
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.text("0")),
)
if _has_table(_TABLE) and not _has_index(_TABLE, _INDEX):
op.create_index(_INDEX, _TABLE, [_COLUMN])
def downgrade() -> None:
if _has_table(_TABLE) and _has_index(_TABLE, _INDEX):
op.drop_index(_INDEX, table_name=_TABLE)
if _has_table(_TABLE) and _has_column(_TABLE, _COLUMN):
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,72 @@
"""Persist private knowledge-research sessions and shared publications.
Revision ID: 20261022_01
Revises: 20261014_02
Create Date: 2026-10-22
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableLongText
revision = "20261022_01"
down_revision = "20261014_02"
branch_labels = None
depends_on = None
_TABLE = "knowledge_research_sessions"
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if inspector.has_table(_TABLE):
return
op.create_table(
_TABLE,
sa.Column("id", sa.String(length=36), primary_key=True),
sa.Column("publication_id", sa.String(length=36), nullable=False),
sa.Column("thread_id", sa.String(length=128), nullable=False),
sa.Column("knowledge_base_id", sa.String(length=128), nullable=False),
sa.Column("user_id", sa.String(length=255), nullable=False),
sa.Column("title", sa.String(length=255), nullable=False),
sa.Column("draft_markdown", PortableLongText(), nullable=False),
sa.Column("wiki_slug", sa.String(length=512), nullable=True),
sa.Column("wiki_page_id", sa.String(length=128), nullable=True),
sa.Column("wiki_folder_id", sa.String(length=128), nullable=True),
sa.Column("saved_title", sa.String(length=255), nullable=True),
sa.Column("saved_markdown", PortableLongText(), nullable=True),
sa.Column("saved_hash", sa.String(length=64), nullable=True),
sa.Column("saved_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("publication_id", name="uq_knowledge_research_publication_id"),
sa.UniqueConstraint("thread_id", name="uq_knowledge_research_thread_id"),
)
op.create_index(
"ix_knowledge_research_sessions_publication_id",
_TABLE,
["publication_id"],
)
op.create_index(
"ix_knowledge_research_sessions_thread_id",
_TABLE,
["thread_id"],
)
op.create_index(
"ix_knowledge_research_sessions_knowledge_base_id",
_TABLE,
["knowledge_base_id"],
)
op.create_index(
"ix_knowledge_research_sessions_user_id",
_TABLE,
["user_id"],
)
def downgrade() -> None:
if sa.inspect(op.get_bind()).has_table(_TABLE):
op.drop_table(_TABLE)
@@ -0,0 +1,118 @@
"""Add the integrator-supplied type partition to task deep-link details.
Revision ID: 20261022_02
Revises: 20261024_01
Create Date: 2026-10-22
The same task id may now carry one snapshot per business type (the deep-link
``?type=``), so ``sub_type`` joins the primary key. Existing rows backfill to
``''`` — the legacy no-type partition — and callers that omit the type keep
reading/writing exactly that partition, so behaviour is unchanged for them.
The key change is a portable table rebuild (create-copy-drop-rename): this
Alembic build has no batch ``drop_primary_key`` and SQLite cannot alter a
primary key in place, so any other route leaves a two-column key that makes
typed upserts fail with a UNIQUE constraint violation.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects import mysql as sa_mysql
revision = "20261022_02"
down_revision = "20261024_01"
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_details"
_PK = ["detail_type", "detail_id", "sub_type"]
_OLD_PK = ["detail_type", "detail_id"]
def _has_table() -> bool:
return _TABLE in inspect(op.get_bind()).get_table_names()
def _pk_columns() -> list[str] | None:
constraint = inspect(op.get_bind()).get_pk_constraint(_TABLE)
return list(constraint.get("constrained_columns") or []) or None
def _column_specs(with_sub_type: bool) -> list[tuple[str, sa.types.TypeEngine, bool]]:
is_mysql = op.get_bind().dialect.name == "mysql"
specs = [
("detail_type", sa.String(32), False),
("detail_id", sa.String(128), False),
("task_name", sa.String(512), False),
("task_content", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
("overview", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
("extra_json", sa.Text() if is_mysql else sa.JSON(), False),
("version", sa.Integer(), False),
("created_by", sa.String(64), True),
("updated_by", sa.String(64), True),
("created_at", sa.DateTime(), False),
("updated_at", sa.DateTime(), False),
]
if with_sub_type:
specs.append(("sub_type", sa.String(64), False))
return specs
def _rebuild_table(pk_columns: list[str], with_sub_type: bool) -> None:
"""Recreate the table (create-copy-drop-rename) with the given primary key."""
bind = op.get_bind()
rebuild = _TABLE + "__rebuild"
if rebuild in inspect(bind).get_table_names():
op.drop_table(rebuild)
specs = _column_specs(with_sub_type)
op.create_table(
rebuild,
*[
sa.Column(
name,
type_,
nullable=nullable,
server_default="" if name == "sub_type" else None,
)
for name, type_, nullable in specs
],
sa.PrimaryKeyConstraint(*pk_columns),
)
names = [name for name, _, _ in specs]
select_expr = ", ".join(
f"COALESCE({name}, '')" if name == "sub_type" else name for name in names
)
bind.execute(
sa.text(f"INSERT INTO {rebuild} ({', '.join(names)}) SELECT {select_expr} FROM {_TABLE}")
)
op.drop_table(_TABLE)
op.rename_table(rebuild, _TABLE)
def upgrade() -> None:
if not _has_table():
# Fresh databases get the three-column key straight from create_all.
return
inspector = inspect(op.get_bind())
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
if "sub_type" in columns and _pk_columns() == _PK:
return
if "sub_type" not in columns:
op.add_column(
_TABLE,
sa.Column("sub_type", sa.String(64), nullable=False, server_default=""),
)
if _pk_columns() != _PK:
_rebuild_table(_PK, with_sub_type=True)
def downgrade() -> None:
if not _has_table():
return
if _pk_columns() == _OLD_PK:
return
_rebuild_table(_OLD_PK, with_sub_type=False)
@@ -0,0 +1,65 @@
"""Admin catalog switch: ``roundtable_chains.hidden``.
Hidden chains vanish from every non-admin listing (我的 / 公共 / 待审核 / 选链
弹窗) while staying fully usable — single-record reads (get_chain), configured
business mappings and in-flight roundtables are unaffected. Only the admin
management view (``admin_view=true``) keeps listing them.
The Gateway's startup ORM-column sweep also adds this column on existing
databases (Alembic auto-upgrade is disabled there); this migration serves
environments that run Alembic manually. Fully guarded and idempotent.
Revision ID: 20261023_01
Revises: 20261022_03
Create Date: 2026-10-23
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261023_01"
down_revision = "20261022_03"
branch_labels = None
depends_on = None
_TABLE = "roundtable_chains"
_COLUMN = "hidden"
_INDEX = "ix_roundtable_chains_hidden"
def _has_table(table_name: str) -> bool:
return table_name in inspect(op.get_bind()).get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
if _has_table(_TABLE) and not _has_column(_TABLE, _COLUMN):
op.add_column(
_TABLE,
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.text("0")),
)
if _has_table(_TABLE) and not _has_index(_TABLE, _INDEX):
op.create_index(_INDEX, _TABLE, [_COLUMN])
def downgrade() -> None:
if _has_table(_TABLE) and _has_index(_TABLE, _INDEX):
op.drop_index(_INDEX, table_name=_TABLE)
if _has_table(_TABLE) and _has_column(_TABLE, _COLUMN):
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,55 @@
"""Index the common deep-link list path that omits sub_type.
Revision ID: 20261023_02
Revises: 20261024_02, 20261022_02
Create Date: 2026-10-22
``GET /api/task-deeplink-threads`` for 3qfx/rwfx/xdfx filters only
``task_id + go_path`` and orders by ``updated_at, thread_id``. The existing
scope index inserts ``sub_type`` before those sort columns, so the planner
cannot use it for newest-first listing. This extra index matches that query.
"""
from __future__ import annotations
from alembic import op
from sqlalchemy import inspect
revision = "20261023_02"
down_revision = ("20261024_02", "20261022_02")
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_threads"
_INDEX = "ix_task_deeplink_threads_task_go_updated"
_COLUMNS = ["task_id", "go_path", "updated_at", "thread_id"]
def _has_table() -> bool:
return _TABLE in inspect(op.get_bind()).get_table_names()
def _indexes() -> dict[str, list[str]]:
if not _has_table():
return {}
return {
str(index["name"]): [str(column) for column in index["column_names"]]
for index in inspect(op.get_bind()).get_indexes(_TABLE)
if index.get("name")
}
def upgrade() -> None:
if not _has_table():
return
if _indexes().get(_INDEX) != _COLUMNS:
if _INDEX in _indexes():
op.drop_index(_INDEX, table_name=_TABLE)
op.create_index(_INDEX, _TABLE, _COLUMNS)
def downgrade() -> None:
if not _has_table():
return
if _INDEX in _indexes():
op.drop_index(_INDEX, table_name=_TABLE)
@@ -0,0 +1,96 @@
"""Repair task_deeplink_details primary keys left two-column by 20261022_02.
Revision ID: 20261023_03
Revises: 20261023_02
Create Date: 2026-10-22
An interim build of ``20261022_02`` added the ``sub_type`` column but could
not rebuild the primary key (the batch API it called does not exist in this
Alembic version, and SQLite cannot alter a key in place). Databases that ran
that build are stamped at head yet still enforce the legacy
``(detail_type, detail_id)`` key, so typed upserts fail with a UNIQUE
constraint violation. This revision detects that state and performs the
portable create-copy-drop-rename rebuild; databases with the correct
three-column key simply no-op.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects import mysql as sa_mysql
revision = "20261023_03"
down_revision = "20261023_02"
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_details"
_PK = ["detail_type", "detail_id", "sub_type"]
def _has_table() -> bool:
return _TABLE in inspect(op.get_bind()).get_table_names()
def _pk_columns() -> list[str] | None:
constraint = inspect(op.get_bind()).get_pk_constraint(_TABLE)
return list(constraint.get("constrained_columns") or []) or None
def _rebuild_table() -> None:
bind = op.get_bind()
rebuild = _TABLE + "__rebuild"
if rebuild in inspect(bind).get_table_names():
op.drop_table(rebuild)
is_mysql = bind.dialect.name == "mysql"
specs: list[tuple[str, sa.types.TypeEngine, bool]] = [
("detail_type", sa.String(32), False),
("detail_id", sa.String(128), False),
("sub_type", sa.String(64), False),
("task_name", sa.String(512), False),
("task_content", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
("overview", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
("extra_json", sa.Text() if is_mysql else sa.JSON(), False),
("version", sa.Integer(), False),
("created_by", sa.String(64), True),
("updated_by", sa.String(64), True),
("created_at", sa.DateTime(), False),
("updated_at", sa.DateTime(), False),
]
op.create_table(
rebuild,
*[
sa.Column(
name,
type_,
nullable=nullable,
server_default="" if name == "sub_type" else None,
)
for name, type_, nullable in specs
],
sa.PrimaryKeyConstraint(*_PK),
)
names = [name for name, _, _ in specs]
select_expr = ", ".join(
f"COALESCE({name}, '')" if name == "sub_type" else name for name in names
)
bind.execute(
sa.text(f"INSERT INTO {rebuild} ({', '.join(names)}) SELECT {select_expr} FROM {_TABLE}")
)
op.drop_table(_TABLE)
op.rename_table(rebuild, _TABLE)
def upgrade() -> None:
if not _has_table():
return
if _pk_columns() == _PK:
return
_rebuild_table()
def downgrade() -> None:
# The repaired key is what 20261022_02 intended; nothing to undo.
return
@@ -0,0 +1,61 @@
"""Add the b5-1 ?type= partition column to the deep-link history index.
Revision ID: 20261024_01
Revises: 20261023_01
Create Date: 2026-10-22
Existing rows keep sharing one partition per (task_id, go_path): the new
``sub_type`` column backfills to ``''`` and queries that omit ``sub_type``
continue to return every sub-type, so 3qfx/rwfx/xdfx behaviour is unchanged.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261024_01"
down_revision = "20261023_01"
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_threads"
_INDEX = "ix_task_deeplink_threads_scope"
_INDEX_COLUMNS = ["task_id", "go_path", "sub_type", "updated_at", "thread_id"]
def _has_table() -> bool:
return _TABLE in inspect(op.get_bind()).get_table_names()
def upgrade() -> None:
if not _has_table():
# Fresh databases get the column straight from create_all.
return
bind = op.get_bind()
inspector = inspect(bind)
if "sub_type" not in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.add_column(
_TABLE,
sa.Column("sub_type", sa.String(64), nullable=False, server_default=""),
)
indexes = {index["name"]: index["column_names"] for index in inspector.get_indexes(_TABLE)}
if indexes.get(_INDEX) != _INDEX_COLUMNS:
if _INDEX in indexes:
op.drop_index(_INDEX, table_name=_TABLE)
op.create_index(_INDEX, _TABLE, _INDEX_COLUMNS)
def downgrade() -> None:
if not _has_table():
return
bind = op.get_bind()
inspector = inspect(bind)
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
indexes = {index["name"]: index["column_names"] for index in inspector.get_indexes(_TABLE)}
if indexes.get(_INDEX) is not None:
op.drop_index(_INDEX, table_name=_TABLE)
if "sub_type" in columns:
op.drop_column(_TABLE, "sub_type")
op.create_index(_INDEX, _TABLE, ["task_id", "go_path", "updated_at", "thread_id"])
@@ -0,0 +1,113 @@
"""Store deep-link list cards without joining ``threads_meta``.
Revision ID: 20261024_02
Revises: 20261024_01
Create Date: 2026-10-22
The dedicated history table used to be a pointer: listing still joined the
generic ``threads_meta`` catalogue for title / status / metadata. That join
is what made the task workspace slow once ordinary Q&A filled the catalogue.
This revision copies those card fields onto ``task_deeplink_threads`` itself
and backfills existing index rows from ``threads_meta``.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from deerflow.persistence.types import PortableJSON
revision = "20261024_02"
down_revision = "20261024_01"
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_threads"
_META_TABLE = "threads_meta"
def _has_table(table_name: str) -> bool:
return table_name in inspect(op.get_bind()).get_table_names()
def _columns() -> set[str]:
if not _has_table(_TABLE):
return set()
return {str(column["name"]) for column in inspect(op.get_bind()).get_columns(_TABLE)}
def _backfill_cards() -> None:
if not _has_table(_META_TABLE):
return
bind = op.get_bind()
dialect = bind.dialect.name
if dialect in {"postgresql", "opengauss"}:
bind.execute(
sa.text(
f"UPDATE {_TABLE} AS t "
"SET display_name = m.display_name, "
"status = COALESCE(m.status, 'idle'), "
"metadata_json = m.metadata_json "
f"FROM {_META_TABLE} AS m "
"WHERE m.thread_id = t.thread_id"
)
)
return
if dialect == "mysql":
bind.execute(
sa.text(
f"UPDATE {_TABLE} AS t "
f"JOIN {_META_TABLE} AS m ON m.thread_id = t.thread_id "
"SET t.display_name = m.display_name, "
"t.status = COALESCE(m.status, 'idle'), "
"t.metadata_json = m.metadata_json"
)
)
return
bind.execute(
sa.text(
f"UPDATE {_TABLE} SET "
"display_name = ("
f"SELECT display_name FROM {_META_TABLE} "
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
"), "
"status = COALESCE(("
f"SELECT status FROM {_META_TABLE} "
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
"), 'idle'), "
"metadata_json = ("
f"SELECT metadata_json FROM {_META_TABLE} "
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
")"
)
)
def upgrade() -> None:
if not _has_table(_TABLE):
return
columns = _columns()
if "display_name" not in columns:
op.add_column(_TABLE, sa.Column("display_name", sa.String(256), nullable=True))
if "status" not in columns:
op.add_column(
_TABLE,
sa.Column("status", sa.String(20), nullable=False, server_default="idle"),
)
if "metadata_json" not in columns:
op.add_column(_TABLE, sa.Column("metadata_json", PortableJSON(), nullable=True))
_backfill_cards()
def downgrade() -> None:
if not _has_table(_TABLE):
return
columns = _columns()
if "metadata_json" in columns:
op.drop_column(_TABLE, "metadata_json")
if "status" in columns:
op.drop_column(_TABLE, "status")
if "display_name" in columns:
op.drop_column(_TABLE, "display_name")
@@ -0,0 +1,54 @@
"""Indexes for the agent gallery category tabs and tag filters.
Revision ID: 20261025_01
Revises: 20261023_03
Create Date: 2026-10-23
The gallery "广场" tab filters ``hidden`` + ``is_private`` + ``square_id``,
and "我的" filters ``user_id`` + ``hidden``. Tag chips join
``tag_assignments`` by ``(target_type, target_id)`` and filter by
``(target_type, tag_id)``. Single-column indexes and the existing unique
key (which leads with ``tag_id``) do not cover those predicates.
Startup also creates missing ORM indexes, so deployments that skip Alembic
still receive these. This revision stays idempotent for the ones that run it.
"""
from __future__ import annotations
from alembic import op
from sqlalchemy import inspect
revision = "20261025_01"
down_revision = "20261023_03"
branch_labels = None
depends_on = None
_INDEXES = (
("agents", "ix_agents_catalog_square", ("hidden", "is_private", "square_id", "created_at")),
("agents", "ix_agents_owner_hidden", ("user_id", "hidden", "created_at")),
("tag_assignments", "ix_tag_assignments_type_target", ("target_type", "target_id")),
("tag_assignments", "ix_tag_assignments_type_tag", ("target_type", "tag_id", "target_id")),
)
def _has_table(table_name: str) -> bool:
return table_name in inspect(op.get_bind()).get_table_names()
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
return any(idx["name"] == index_name for idx in inspect(op.get_bind()).get_indexes(table_name))
def upgrade() -> None:
for table_name, index_name, columns in _INDEXES:
if _has_table(table_name) and not _has_index(table_name, index_name):
op.create_index(index_name, table_name, list(columns))
def downgrade() -> None:
for table_name, index_name, _columns in reversed(_INDEXES):
if _has_table(table_name) and _has_index(table_name, index_name):
op.drop_index(index_name, table_name=table_name)
@@ -0,0 +1,37 @@
"""Add shared nonce storage for scoped service-token exchange.
Revision ID: 20261025_02
Revises: 20261025_01
Create Date: 2026-10-24
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261025_02"
down_revision = "20261025_01"
branch_labels = None
depends_on = None
_TABLE = "auth_service_nonces"
def upgrade() -> None:
if _TABLE in inspect(op.get_bind()).get_table_names():
return
op.create_table(
_TABLE,
sa.Column("nonce_hash", sa.String(64), primary_key=True),
sa.Column("client_id", sa.String(128), nullable=False),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_auth_service_nonces_client_id", _TABLE, ["client_id"])
op.create_index("ix_auth_service_nonces_expires_at", _TABLE, ["expires_at"])
def downgrade() -> None:
if _TABLE in inspect(op.get_bind()).get_table_names():
op.drop_table(_TABLE)
@@ -0,0 +1,35 @@
"""Add the database-backed skill catalogue snapshot for list requests.
Revision ID: 20261025_03
Revises: 20261025_02
Create Date: 2026-10-25
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261025_03"
down_revision = "20261025_02"
branch_labels = None
depends_on = None
_TABLE = "skill_catalog_snapshots"
def upgrade() -> None:
if _TABLE in inspect(op.get_bind()).get_table_names():
return
op.create_table(
_TABLE,
sa.Column("cache_key", sa.String(32), primary_key=True),
sa.Column("catalog_json", sa.Text(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
if _TABLE in inspect(op.get_bind()).get_table_names():
op.drop_table(_TABLE)
@@ -0,0 +1,62 @@
"""Index canonical usernames for fast SSO and username login.
Revision ID: 20261026_01
Revises: 20261025_03
Create Date: 2026-10-26
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261026_01"
down_revision = "20261025_03"
branch_labels = None
depends_on = None
_TABLE = "users"
_COLUMN = "username_key"
_INDEX = "ix_users_username_key_created_at"
def upgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
if _COLUMN not in columns:
op.add_column(_TABLE, sa.Column(_COLUMN, sa.String(128), nullable=True))
indexes = {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
if _INDEX not in indexes:
op.create_index(_INDEX, _TABLE, [_COLUMN, "created_at"])
# Backfill the normalized local part once. Login requests then use a
# bounded indexed equality lookup instead of LOWER(email) LIKE 'name@%'.
bind = op.get_bind()
rows = bind.execute(sa.text("SELECT id, email FROM users")).fetchall()
for user_id, email in rows:
if not email or "@" not in email:
continue
username_key = str(email).split("@", 1)[0].strip().lower()[:128]
bind.execute(
sa.text(
"UPDATE users SET username_key = :username_key "
"WHERE id = :user_id AND (username_key IS NULL OR username_key = '')"
),
{"username_key": username_key, "user_id": user_id},
)
def downgrade() -> None:
if _TABLE not in inspect(op.get_bind()).get_table_names():
return
indexes = {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
if _INDEX in indexes:
op.drop_index(_INDEX, table_name=_TABLE)
columns = {column["name"] for column in inspect(op.get_bind()).get_columns(_TABLE)}
if _COLUMN in columns:
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,44 @@
"""Persist the globally shared default reasoning effort for each model.
Revision ID: 20261026_02
Revises: 20261026_01, 20261022_01
Create Date: 2026-10-26
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261026_02"
down_revision = ("20261026_01", "20261022_01")
branch_labels = None
depends_on = None
_TABLE = "model_capabilities"
_COLUMN = "default_reasoning_effort"
def upgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.add_column(
_TABLE,
sa.Column(
_COLUMN,
sa.String(length=16),
nullable=False,
server_default="xhigh",
),
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,44 @@
"""Persist the administrator-configured Redis cache TTL.
Revision ID: 20261029_01
Revises: 20261026_02
Create Date: 2026-10-29
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261029_01"
down_revision = "20261026_02"
branch_labels = None
depends_on = None
_TABLE = "api_read_cache_settings"
_COLUMN = "ttl_seconds"
def upgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.add_column(
_TABLE,
sa.Column(
_COLUMN,
sa.Integer(),
nullable=False,
server_default=sa.text("300"),
),
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,32 @@
"""Persist per-API cache switches and browser status access control."""
import sqlalchemy as sa
from alembic import op
revision = "20261029_02"
down_revision = "20261029_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "api_read_cache_settings" not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns("api_read_cache_settings")}
if "collection_overrides" not in columns:
op.add_column("api_read_cache_settings", sa.Column("collection_overrides", sa.JSON(), nullable=True))
if "browser_status_enabled" not in columns:
op.add_column("api_read_cache_settings", sa.Column("browser_status_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
if "cache_epoch" not in columns:
op.add_column("api_read_cache_settings", sa.Column("cache_epoch", sa.String(32), nullable=False, server_default="0"))
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "api_read_cache_settings" not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns("api_read_cache_settings")}
for name in ("cache_epoch", "browser_status_enabled", "collection_overrides"):
if name in columns:
op.drop_column("api_read_cache_settings", name)
@@ -0,0 +1,41 @@
"""Add per-item enablement to compact menu overrides.
Existing compact-menu items remain enabled. New databases receive the field
from ``CompactMenuOverrideRow``; this revision and the startup ORM schema sweep
add it safely to deployments with an existing ``compact_menu_overrides`` table.
Revision ID: 20261102_01
Revises: 20261029_02
Create Date: 2026-11-02
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261102_01"
down_revision = "20261029_02"
branch_labels = None
depends_on = None
_TABLE = "compact_menu_overrides"
_COLUMN = "disabled"
def upgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.add_column(
_TABLE,
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.false()),
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,67 @@
"""Allow administrator-defined business-mapping types and login goPaths.
Revision ID: 20261102_02
Revises: 20261102_01
Create Date: 2026-11-02
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261102_02"
down_revision = "20261102_01"
branch_labels = None
depends_on = None
_MAPPING_TABLE = "business_mapping"
_THREAD_TABLE = "task_deeplink_threads"
def upgrade() -> None:
inspector = inspect(op.get_bind())
tables = set(inspector.get_table_names())
if _MAPPING_TABLE in tables:
columns = {column["name"] for column in inspector.get_columns(_MAPPING_TABLE)}
if "mapping_type" not in columns:
op.add_column(
_MAPPING_TABLE,
sa.Column("mapping_type", sa.String(64), nullable=False, server_default="内置业务"),
)
if "is_custom" not in columns:
op.add_column(
_MAPPING_TABLE,
sa.Column("is_custom", sa.Boolean(), nullable=False, server_default=sa.false()),
)
# PostgreSQL enforces VARCHAR length, unlike SQLite. Widen the dedicated
# history partition too so a 32-character custom goPath can be persisted.
if _THREAD_TABLE in tables:
with op.batch_alter_table(_THREAD_TABLE) as batch:
batch.alter_column(
"go_path",
existing_type=sa.String(16),
type_=sa.String(32),
existing_nullable=False,
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
tables = set(inspector.get_table_names())
if _THREAD_TABLE in tables:
with op.batch_alter_table(_THREAD_TABLE) as batch:
batch.alter_column(
"go_path",
existing_type=sa.String(32),
type_=sa.String(16),
existing_nullable=False,
)
if _MAPPING_TABLE in tables:
columns = {column["name"] for column in inspector.get_columns(_MAPPING_TABLE)}
if "is_custom" in columns:
op.drop_column(_MAPPING_TABLE, "is_custom")
if "mapping_type" in columns:
op.drop_column(_MAPPING_TABLE, "mapping_type")
@@ -28,6 +28,7 @@ class ModelCapabilityStore(ABC):
model_name: str,
*,
supports_reasoning_effort: bool,
default_reasoning_effort: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any]:
raise NotImplementedError
@@ -24,11 +24,15 @@ class MemoryModelCapabilityStore(ModelCapabilityStore):
model_name: str,
*,
supports_reasoning_effort: bool,
default_reasoning_effort: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any]:
row = {
"model_name": model_name,
"supports_reasoning_effort": bool(supports_reasoning_effort),
"default_reasoning_effort": default_reasoning_effort
or (self._rows.get(model_name) or {}).get("default_reasoning_effort")
or "xhigh",
"updated_by": updated_by,
"updated_at": datetime.now(UTC).isoformat(),
}
@@ -21,6 +21,12 @@ class ModelCapabilityRow(Base):
default=False,
server_default=false(),
)
default_reasoning_effort: Mapped[str] = mapped_column(
String(16),
nullable=False,
default="xhigh",
server_default="xhigh",
)
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
@@ -17,6 +17,7 @@ def _to_dict(row: ModelCapabilityRow) -> dict[str, Any]:
return {
"model_name": row.model_name,
"supports_reasoning_effort": bool(row.supports_reasoning_effort),
"default_reasoning_effort": row.default_reasoning_effort or "xhigh",
"updated_by": row.updated_by,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
@@ -41,6 +42,7 @@ class ModelCapabilityRepository(ModelCapabilityStore):
model_name: str,
*,
supports_reasoning_effort: bool,
default_reasoning_effort: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any]:
async with self._sf() as session:
@@ -49,6 +51,8 @@ class ModelCapabilityRepository(ModelCapabilityStore):
row = ModelCapabilityRow(model_name=model_name)
session.add(row)
row.supports_reasoning_effort = bool(supports_reasoning_effort)
if default_reasoning_effort is not None:
row.default_reasoning_effort = default_reasoning_effort
row.updated_by = updated_by
row.updated_at = datetime.now(UTC)
await session.commit()
@@ -35,6 +35,7 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.api_read_cache_settings.model import ApiReadCacheSettingRow
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
from deerflow.persistence.analysis_project_memory.model import (
AnalysisProjectMemoryItemRow,
@@ -79,6 +80,7 @@ from deerflow.persistence.force_situation.model import (
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.knowledge_research import KnowledgeResearchSessionRow
from deerflow.persistence.knowledge_retrieval_settings.model import KnowledgeRetrievalSettingsRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
@@ -127,6 +129,7 @@ from deerflow.persistence.roundtable_drafts.model import (
RoundtableRecommendHistoryRow,
)
from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
from deerflow.persistence.roundtable_task_drafts.model import RoundtableTaskDraftRow
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.scheduled_tasks.model import (
@@ -149,7 +152,12 @@ from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
from deerflow.persistence.skills.model import (
SkillCatalogSnapshotRow,
SkillDisplayAdapterRow,
SkillFavoriteRow,
SkillRow,
)
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.task_analysis.model import (
@@ -165,12 +173,14 @@ from deerflow.persistence.task_analysis.model import (
TaskDataSubmissionRow,
)
from deerflow.persistence.task_buttons.model import TaskButtonRow
from deerflow.persistence.task_deeplink_details.model import TaskDeeplinkDetailRow
from deerflow.persistence.task_deeplink_threads.model import TaskDeeplinkThreadRow
from deerflow.persistence.task_reports.model import TaskReportRow
from deerflow.persistence.taskcop_tasks.model import TaskCopTaskRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
from deerflow.persistence.upload_policy.model import UploadPolicyRow
from deerflow.persistence.user.model import UserRow
from deerflow.persistence.user_preferences.model import UserPreferencesRow
@@ -207,6 +217,7 @@ __all__ = [
"AgentExtrasRow",
"AgentFavoriteRow",
"AgentRow",
"ApiReadCacheSettingRow",
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
@@ -241,6 +252,7 @@ __all__ = [
"KnowledgeRelationRow",
"KnowledgeSourceRow",
"KnowledgeTagRow",
"KnowledgeResearchSessionRow",
"KnowledgeRetrievalSettingsRow",
"LightAppRow",
"WorkflowDataSourceRow",
@@ -289,6 +301,7 @@ __all__ = [
"RoundtableDraftShareRow",
"RoundtableDiagnosticRow",
"RoundtableJobRow",
"RoundtableRoomLeaseRow",
"RoundtableRecommendHistoryRow",
"RoundtableTaskDraftRow",
"PositionRoleRow",
@@ -301,6 +314,7 @@ __all__ = [
"ScheduledTaskSubscriptionRow",
"SchedulerThreadRow",
"SensitiveWordRow",
"SkillCatalogSnapshotRow",
"SkillRow",
"SkillDisplayAdapterRow",
"SkillFavoriteRow",
@@ -328,6 +342,7 @@ __all__ = [
"TaskDataBlockRow",
"TaskDataPeriodRow",
"TaskDataSubmissionRow",
"TaskDeeplinkDetailRow",
"TaskDeeplinkThreadRow",
"AnalysisProjectRow",
"AnalysisProjectThreadRow",
@@ -338,6 +353,7 @@ __all__ = [
"ThreadMetaRow",
"ThreadShareRow",
"ToolCallMetricRow",
"StreamTruncationEventRow",
"UploadPolicyRow",
"UserCustomPromptRow",
"UserMemoryConfigRow",
@@ -15,7 +15,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy import Index, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -24,6 +24,14 @@ from deerflow.persistence.types import BeijingDateTime, PortableLongText
class ParallelAgentTaskRow(Base):
__tablename__ = "parallel_agent_tasks"
__table_args__ = (
Index(
"ix_parallel_agent_tasks_user_updated_id",
"user_id",
"updated_at",
"id",
),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
@@ -11,6 +11,7 @@ that belongs to another user is treated as not found.
from __future__ import annotations
import json
from collections.abc import Mapping
from datetime import UTC, datetime
from typing import Any
@@ -44,17 +45,32 @@ class ParallelAgentTaskRepository:
self._sf = session_factory
@staticmethod
def _to_dict(row: ParallelAgentTaskRow, *, with_messages: bool = True) -> dict[str, Any]:
def _to_dict(
row: ParallelAgentTaskRow | Mapping[str, Any],
*,
with_messages: bool = True,
) -> dict[str, Any]:
def value(name: str) -> Any:
if isinstance(row, Mapping):
return row[name]
return getattr(row, name)
created_at = value("created_at")
updated_at = value("updated_at")
data: dict[str, Any] = {
"id": row.id,
"title": row.title,
"brief": row.brief,
"agents": _loads(row.agents) or [],
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
"id": value("id"),
"title": value("title"),
"brief": value("brief"),
"agents": _loads(value("agents")) or [],
"created_at": (
created_at.isoformat() if isinstance(created_at, datetime) else created_at
),
"updated_at": (
updated_at.isoformat() if isinstance(updated_at, datetime) else updated_at
),
}
if with_messages:
data["messages"] = _loads(row.messages) or []
data["messages"] = _loads(value("messages")) or []
return data
async def list_tasks(
@@ -74,9 +90,22 @@ class ParallelAgentTaskRepository:
and ``offset`` allow those projections to page only their own task
namespace without loading unrelated, potentially large transcripts.
"""
stmt = select(ParallelAgentTaskRow).where(ParallelAgentTaskRow.user_id == user_id)
# History/list callers only need the summary columns. In particular,
# never select the LONGTEXT transcript column unless explicitly asked:
# MySQL otherwise reads and transfers it even though _to_dict omits it.
columns = [
ParallelAgentTaskRow.id,
ParallelAgentTaskRow.title,
ParallelAgentTaskRow.brief,
ParallelAgentTaskRow.agents,
ParallelAgentTaskRow.created_at,
ParallelAgentTaskRow.updated_at,
]
if include_messages:
columns.append(ParallelAgentTaskRow.messages)
stmt = select(*columns).where(ParallelAgentTaskRow.user_id == user_id)
if id_prefix:
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix))
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix, autoescape=True))
stmt = (
stmt.order_by(
ParallelAgentTaskRow.updated_at.desc(),
@@ -88,8 +117,8 @@ class ParallelAgentTaskRepository:
async with self._sf() as session:
result = await session.execute(stmt)
return [
self._to_dict(row, with_messages=include_messages)
for row in result.scalars()
self._to_dict(row._mapping, with_messages=include_messages)
for row in result
]
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
@@ -7,6 +7,15 @@ from typing import Any
class PositionStore(ABC):
async def get_effective_position(self, user_id: str) -> dict[str, Any] | None:
"""Return an assigned position or the default without membership counts."""
position_id = await self.get_user_position_id(user_id)
record = await self.get_position(position_id) if position_id else None
if record is None:
default_id = await self.get_default_position_id()
record = await self.get_position(default_id) if default_id else None
return record
# -------------------------------------------------------------- positions
@abstractmethod
@@ -7,6 +7,7 @@ from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, select, update
from sqlalchemy.orm import aliased
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -60,6 +61,25 @@ class PositionRepository(PositionStore):
counts = await self._member_counts(session, [position_id])
return _position_to_dict(row, counts.get(position_id, 0))
async def get_effective_position(self, user_id: str) -> dict[str, Any] | None:
"""Resolve the assigned/default position with one narrow indexed query."""
assigned = aliased(PositionRow)
default = aliased(PositionRow)
stmt = (
select(assigned, default)
.select_from(UserRow)
.outerjoin(assigned, UserRow.position_id == assigned.id)
.outerjoin(default, default.is_default.is_(True))
.where(UserRow.id == user_id)
.limit(1)
)
async with self._sf() as session:
row = (await session.execute(stmt)).first()
if row is None:
return None
position = row[0] or row[1]
return _position_to_dict(position, 0) if position is not None else None
async def create_position(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
is_default = bool(data.get("is_default"))
@@ -75,6 +75,13 @@ class RoundtableChainRow(Base):
pending_review: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default="0", index=True
)
# 管理员目录开关:True = 从所有普通列表(我的/公共/待审核/选链弹窗)隐藏,仅管理员
# 管理视图(``admin_view=true`` 的列表请求)可见。纯目录开关:单条读取(get_chain)
# 不过滤,已配置的业务映射 / 进行中的研讨不受影响。``server_default`` 显式声明以支持
# 启动期 ``_ensure_orm_columns_sync`` 在既有表上补列(NOT NULL 需带默认)。
hidden: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default="0", index=True
)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -66,6 +66,8 @@ class RoundtableChainRepository:
"enabled": bool(getattr(row, "enabled", True)),
# 「待审核」标记:True = owner 已提交审核、等待管理员发布;发布时清回 False。
"pending_review": bool(getattr(row, "pending_review", False)),
# 管理员目录开关:True = 从普通列表隐藏(仅管理视图可见);缺省 False。
"hidden": bool(getattr(row, "hidden", False)),
"is_mine": (current_user_id is None) or (row.user_id == current_user_id),
# 配置者(owner)的 user_id;路由层据此解析成可读用户名(owner_name)。
"owner_id": row.user_id,
@@ -74,7 +76,14 @@ class RoundtableChainRepository:
}
async def list_chains(
self, user_id: str, *, scope: str = "mine", limit: int = 200, is_admin: bool = False
self,
user_id: str,
*,
scope: str = "mine",
limit: int = 200,
is_admin: bool = False,
include_hidden: bool = False,
hidden: bool | None = None,
) -> list[dict[str, Any]]:
"""列出链条。
@@ -86,6 +95,10 @@ class RoundtableChainRepository:
- ``"all"``:所有账号的全部链条(**仅管理员**,由路由层校验后传 ``is_admin=True``;
非管理员误传则按 ``"mine"`` 退化,绝不越权)。
其余值按 ``"mine"`` 处理。``is_mine`` 标记每条是否本人所有,供前端判断可否编辑。
隐藏开关(``hidden=True`` 的行):默认从**所有**列表排除(含 admin 的普通视图,如
选链弹窗);``include_hidden=True``(管理员管理视图)时不排除,并可再用 ``hidden``
参数筛选「只看隐藏 / 只看显示中」。非管理员的 ``include_hidden`` 一律忽略。
"""
if scope == "public":
cond = RoundtableChainRow.is_public.is_(True)
@@ -98,6 +111,11 @@ class RoundtableChainRepository:
stmt = select(RoundtableChainRow)
if cond is not None:
stmt = stmt.where(cond)
if include_hidden and is_admin:
if hidden is not None:
stmt = stmt.where(RoundtableChainRow.hidden.is_(hidden))
else:
stmt = stmt.where(RoundtableChainRow.hidden.is_(False))
stmt = stmt.order_by(RoundtableChainRow.updated_at.desc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
@@ -157,6 +175,7 @@ class RoundtableChainRepository:
is_public=_UNSET,
enabled=_UNSET,
pending_review=_UNSET,
hidden=_UNSET,
is_admin: bool = False,
) -> dict[str, Any] | None:
async with self._sf() as session:
@@ -186,6 +205,8 @@ class RoundtableChainRepository:
row.enabled = bool(enabled)
if pending_review is not _UNSET and pending_review is not None:
row.pending_review = bool(pending_review)
if hidden is not _UNSET and hidden is not None:
row.hidden = bool(hidden)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -68,7 +68,7 @@ class RoundtableJobRow(Base):
# Fine-grained phase (§2.2): initializing / leader_thinking / dispatching /
# consensus / report_drawing / report_done.
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
# Current orchestration cycle (0..MAX_CYCLES).
# Current orchestration cycle (0..configured roundtable max_cycles).
cycle: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# Dispatch chain nodes (§2.3): JSON array of
# { agentId, name, avatarType, order, state, lastSpokeCycle? }.
@@ -119,6 +119,7 @@ class RoundtableJobRepository:
request_hash: str | None = None,
active_dedupe_key: str | None = None,
input_snapshot: dict[str, Any] | None = None,
room_started_at: datetime | None = None,
) -> RoundtableJobRow:
now = datetime.now(UTC)
return RoundtableJobRow(
@@ -145,7 +146,7 @@ class RoundtableJobRepository:
error=None,
attempt=0,
version=0,
created_at=now,
created_at=room_started_at or now,
updated_at=now,
)
@@ -204,6 +205,7 @@ class RoundtableJobRepository:
request_id: str | None = None,
request_hash: str | None = None,
input_snapshot: dict[str, Any] | None = None,
room_started_at: datetime | None = None,
) -> tuple[dict[str, Any], bool]:
"""原子地「创建活跃作业 → 冲突则幂等返回已有活跃作业」(Phase 3 唯一活跃键版)。
@@ -240,6 +242,7 @@ class RoundtableJobRepository:
request_hash=request_hash,
active_dedupe_key=dedupe_key,
input_snapshot=input_snapshot,
room_started_at=room_started_at,
)
# 无 draft_id → 无法去重,直接创建(保持原有行为)。
@@ -462,6 +465,30 @@ class RoundtableJobRepository:
result = await session.execute(stmt)
return [r[0] for r in result.all()]
async def request_cancel_expired_step2(
self,
*,
created_before: datetime,
non_occupying_phases: tuple[str, ...],
) -> int:
"""Cancel unfinished Step-2 jobs that exceeded the room hard lifetime."""
async with self._sf() as session:
result = await session.execute(
update(RoundtableJobRow)
.where(
RoundtableJobRow.status.in_(list(CANCELLABLE_STATUSES)),
RoundtableJobRow.created_at <= created_before,
RoundtableJobRow.phase.not_in(list(non_occupying_phases)),
)
.values(
status="cancel_requested",
version=RoundtableJobRow.version + 1,
updated_at=datetime.now(UTC),
)
)
await session.commit()
return int(result.rowcount or 0)
async def list_cancel_reapable(self, *, now: datetime, limit: int = 20) -> list[str]:
"""列出可以由 dispatcher 收口的取消请求。
@@ -0,0 +1,19 @@
"""Roundtable step-2 room leases."""
from __future__ import annotations
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
from deerflow.persistence.roundtable_rooms.sql import RoundtableRoomRepository
def make_roundtable_room_store(session_factory) -> RoundtableRoomRepository | None:
if session_factory is None:
return None
return RoundtableRoomRepository(session_factory)
__all__ = [
"RoundtableRoomLeaseRow",
"RoundtableRoomRepository",
"make_roundtable_room_store",
]
@@ -0,0 +1,33 @@
"""Foreground roundtable room leases.
One row is one browser session that has entered step 2 and is still heartbeating.
Background jobs are not stored here: they already occupy a room through
``roundtable_jobs`` while their status is non-terminal. Both sides share the
job active-dedupe key so a foreground session and its later background job
count as one room.
"""
from __future__ import annotations
from datetime import datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class RoundtableRoomLeaseRow(Base):
__tablename__ = "roundtable_room_leases"
room_key: Mapped[str] = mapped_column(String(64), primary_key=True)
draft_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
task_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# Last account to heartbeat. Release only succeeds for this account, so one
# viewer leaving a shared task room does not drop another viewer's lease.
holder_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
@@ -0,0 +1,584 @@
"""Deployment-wide roundtable room leases and background-job occupancy.
A room is one live roundtable execution, not one saved draft, not one account,
and not one seat. Different pages running the same draft have distinct hidden
session ids; every seat and step of one execution shares one id. The active set
is the union of every user's:
- a durable foreground/session lease renewed by the browser or a run entry
- live ``extra_keys`` supplied by the Gateway as a reconciliation fallback
only when no durable lease/job already represents that persisted scope
- a background job that is executing or waiting for the user's Step-2
clarification (``queued`` / ``running`` / ``awaiting_input`` /
``cancel_requested``)
Historical jobs older than the hard room lifetime and merely viewing chat
history are not rooms. There is no per-user pool.
Admission and lease creation happen while holding one database-backed gate
row. Updating that row serializes contenders on SQLite, MySQL and PostgreSQL,
so four simultaneous requests cannot all observe the same stale count.
"""
from __future__ import annotations
import asyncio
import hashlib
from collections.abc import Mapping
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
from deerflow.persistence.roundtable_jobs.sql import compute_active_dedupe_key
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
LEASE_SECONDS = 60
JOB_LIVE_SECONDS = 15 * 60
OCCUPYING_JOB_STATUSES = ("queued", "running", "awaiting_input", "cancel_requested")
# A background job keeps running while Step 3 generates its artifacts, but the
# concurrency cap is intentionally scoped to Step 2. ``consensus`` is written
# before Step 3 begins, which releases the slot without waiting for reporting.
NON_OCCUPYING_JOB_PHASES = frozenset({
"consensus",
"summary_writing",
"report_drawing",
"report_done",
})
GATE_ROW_KEY = "__roundtable_room_gate__"
def compute_room_session_key(room_session_id: str | None) -> str | None:
"""Stable key for one live roundtable execution (not one saved draft)."""
clean = (room_session_id or "").strip()
if not clean:
return None
return hashlib.sha256(f"roundtable-session:{clean}".encode()).hexdigest()
def _aware(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value
def _job_is_live(
lease_until: datetime | None,
updated_at: datetime | None,
now: datetime,
live_after: datetime,
) -> bool:
until = _aware(lease_until)
if until is not None and until > now:
return True
updated = _aware(updated_at)
return updated is not None and updated >= live_after
def _hard_deadline_reached(
created_at: datetime | None,
now: datetime,
max_hold_minutes: int,
) -> bool:
created = _aware(created_at)
return created is not None and created + timedelta(minutes=max_hold_minutes) <= now
class RoundtableRoomRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
self._lock = asyncio.Lock()
async def admit(
self,
*,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
room_session_id: str | None = None,
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Atomically reserve/renew a room, or reject when the pool is full."""
key = compute_room_session_key(room_session_id) or compute_active_dedupe_key(
draft_id, user_id, task_id
)
return await self.admit_room_key(
room_key=key,
draft_id=draft_id,
user_id=user_id,
task_id=task_id,
holder_user_id=user_id,
enabled=enabled,
max_rooms=max_rooms,
max_hold_minutes=max_hold_minutes,
extra_keys=extra_keys,
)
async def admit_room_key(
self,
*,
room_key: str | None,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
holder_user_id: str | None,
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Atomically admit a pre-resolved room key.
``room_key`` is used by ``/run/stream`` when it resolves the room from
thread metadata. ``user_id`` remains the room owner while
``holder_user_id`` is the account currently renewing a shared room.
"""
async with self._lock:
await self._ensure_gate_row()
now = datetime.now(UTC)
async with self._sf() as session:
# This no-op-ish write is intentional: it is the portable DB
# mutex protecting the subsequent count + lease upsert.
await session.execute(
update(RoundtableRoomLeaseRow)
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
.values(updated_at=now)
)
keys, durable_scopes, hard_expired_keys = await self._active_snapshot(
session,
now,
max_hold_minutes=max_hold_minutes,
)
self._merge_extra_keys(keys, durable_scopes, extra_keys)
if room_key is None:
await session.commit()
blocked = bool(enabled)
return self._decision(
ok=not blocked,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=False,
code="DRAFT_REQUIRED" if blocked else "",
message="" if not blocked else "进入第二步需要先保存会商草稿",
)
if room_key in hard_expired_keys:
await session.commit()
return self._decision(
ok=False,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=False,
code="ROOM_MAX_HOLD_EXCEEDED",
message="本次会商已达到最长占用时间,房间已自动释放。请返回第一步后重新发起。",
)
owns = room_key in keys
if enabled and not owns and len(keys) >= max_rooms:
await session.commit()
return self._decision(
ok=False,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=False,
code="ROUNDTABLE_ROOM_FULL",
message="当前会商房间暂时已满,请稍候片刻后再试。",
)
await self._upsert_lease(
session,
room_key=room_key,
draft_id=draft_id,
user_id=user_id,
task_id=task_id,
holder_user_id=holder_user_id,
now=now,
)
await session.commit()
return self._decision(
ok=True,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys) if owns else len(keys) + 1,
renewed=owns,
code="",
message="",
)
async def release(
self,
*,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
room_session_id: str | None = None,
) -> bool:
"""Drop a leftover foreground lease row if one still exists."""
key = compute_room_session_key(room_session_id) or compute_active_dedupe_key(
draft_id, user_id, task_id
)
if not key:
return False
async with self._lock:
await self._ensure_gate_row()
async with self._sf() as session:
await session.execute(
update(RoundtableRoomLeaseRow)
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
.values(updated_at=datetime.now(UTC))
)
row = await session.get(RoundtableRoomLeaseRow, key)
if row is None or (user_id and row.holder_user_id not in (None, user_id)):
await session.commit()
return False
await session.delete(row)
await session.commit()
return True
async def started_at(
self,
*,
room_session_id: str | None,
holder_user_id: str | None,
) -> datetime | None:
"""Return the original admission time for a foreground-to-job handoff."""
key = compute_room_session_key(room_session_id)
if key is None:
return None
async with self._sf() as session:
row = await session.get(RoundtableRoomLeaseRow, key)
if row is None or (
holder_user_id
and row.holder_user_id not in (None, holder_user_id)
):
return None
return _aware(row.created_at)
async def renew(
self,
*,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
room_session_id: str | None,
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Renew one live foreground lease without ever creating it.
Heartbeats are deliberately weaker than admission. A delayed browser
callback must not resurrect a room that was released after a pause,
network disconnect or completed Step 2.
"""
key = compute_room_session_key(room_session_id)
if key is None:
return self._decision(
ok=not enabled,
enabled=enabled,
max_rooms=max_rooms,
active_count=0,
renewed=False,
code="ROOM_SESSION_REQUIRED" if enabled else "",
message="会商会话标识缺失,请刷新页面后重试。" if enabled else "",
)
return await self.renew_room_key(
room_key=key,
holder_user_id=user_id,
enabled=enabled,
max_rooms=max_rooms,
max_hold_minutes=max_hold_minutes,
extra_keys=extra_keys,
)
async def renew_room_key(
self,
*,
room_key: str,
holder_user_id: str | None,
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Renew a resolved persisted key without creating a missing lease."""
async with self._lock:
await self._ensure_gate_row()
now = datetime.now(UTC)
async with self._sf() as session:
await session.execute(
update(RoundtableRoomLeaseRow)
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
.values(updated_at=now)
)
keys, durable_scopes, hard_expired_keys = await self._active_snapshot(
session,
now,
max_hold_minutes=max_hold_minutes,
)
self._merge_extra_keys(keys, durable_scopes, extra_keys)
row = await session.get(RoundtableRoomLeaseRow, room_key)
if row is not None and room_key in hard_expired_keys:
await session.commit()
return self._decision(
ok=False,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=False,
code="ROOM_MAX_HOLD_EXCEEDED",
message="本次会商已达到最长占用时间,房间已自动释放。请返回第一步后重新发起。",
)
owns = row is not None and room_key in keys and (
not holder_user_id or row.holder_user_id in (None, holder_user_id)
)
if not owns:
await session.commit()
return self._decision(
ok=not enabled,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=False,
code="ROOM_LEASE_EXPIRED" if enabled else "",
message="会商连接已断开,请重新恢复研讨。" if enabled else "",
)
row.lease_until = now + timedelta(seconds=LEASE_SECONDS)
row.updated_at = now
await session.commit()
return self._decision(
ok=True,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=True,
code="",
message="",
)
async def summary(
self,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
*,
max_hold_minutes: int = 30,
) -> int:
"""Return the number of distinct occupied rooms across every user."""
async with self._lock:
await self._ensure_gate_row()
now = datetime.now(UTC)
async with self._sf() as session:
keys, durable_scopes, _hard_expired_keys = await self._active_snapshot(
session,
now,
max_hold_minutes=max_hold_minutes,
)
self._merge_extra_keys(keys, durable_scopes, extra_keys)
await session.commit()
return len(keys)
@staticmethod
def _merge_extra_keys(
keys: set[str],
durable_scopes: set[str],
extra_keys: set[str] | Mapping[str, str | None] | None,
) -> None:
"""Add run-manager fallbacks without double-counting durable occupancy.
The map value is the legacy draft/user-or-task scope. If a live page
lease or background job already owns that scope, its stamped run key is
merely evidence for the same room and must not become another room.
"""
if not extra_keys:
return
items = extra_keys.items() if isinstance(extra_keys, Mapping) else ((key, None) for key in extra_keys)
for key, scope_key in items:
if not key or (scope_key and scope_key in durable_scopes):
continue
keys.add(key)
async def _active_snapshot(
self,
session: AsyncSession,
now: datetime,
*,
max_hold_minutes: int,
) -> tuple[set[str], set[str], set[str]]:
lease_rows = (
await session.execute(
select(RoundtableRoomLeaseRow).where(
RoundtableRoomLeaseRow.room_key != GATE_ROW_KEY
)
)
).scalars().all()
expired: list[str] = []
keys: set[str] = set()
durable_scopes: set[str] = set()
hard_expired_keys: set[str] = set()
for row in lease_rows:
until = _aware(row.lease_until)
if until is None or until <= now:
expired.append(row.room_key)
continue
scope_key = compute_active_dedupe_key(row.draft_id, row.user_id, row.task_id)
if _hard_deadline_reached(row.created_at, now, max_hold_minutes):
# Keep the still-live row briefly as a non-occupying tombstone.
# Its scope suppresses a lagging RunManager fallback until the
# browser observes the rejected heartbeat and cancels the run.
hard_expired_keys.add(row.room_key)
if scope_key:
durable_scopes.add(scope_key)
continue
keys.add(row.room_key)
if scope_key:
durable_scopes.add(scope_key)
if expired:
await session.execute(
delete(RoundtableRoomLeaseRow).where(
RoundtableRoomLeaseRow.room_key.in_(expired)
)
)
live_after = now - timedelta(seconds=JOB_LIVE_SECONDS)
job_rows = (
await session.execute(
select(
RoundtableJobRow.id,
RoundtableJobRow.draft_id,
RoundtableJobRow.user_id,
RoundtableJobRow.task_id,
RoundtableJobRow.status,
RoundtableJobRow.phase,
RoundtableJobRow.lease_until,
RoundtableJobRow.updated_at,
RoundtableJobRow.created_at,
).where(RoundtableJobRow.status.in_(OCCUPYING_JOB_STATUSES))
)
).all()
for job_id, draft_id, user_id, task_id, status, phase, lease_until, updated_at, created_at in job_rows:
if phase in NON_OCCUPYING_JOB_PHASES:
continue
if _hard_deadline_reached(created_at, now, max_hold_minutes):
# The worker cancellation watcher will finalize this state.
# Marking the intent here also covers queued jobs and workers
# that stopped renewing but have not yet been reaped.
await session.execute(
update(RoundtableJobRow)
.where(
RoundtableJobRow.id == job_id,
RoundtableJobRow.status.in_(["queued", "running", "awaiting_input"]),
)
.values(
status="cancel_requested",
version=RoundtableJobRow.version + 1,
updated_at=now,
)
)
continue
if status != "awaiting_input" and not _job_is_live(
lease_until,
updated_at,
now,
live_after,
):
continue
job_key = compute_active_dedupe_key(draft_id, user_id, task_id)
if job_key and job_key in durable_scopes:
# Foreground → background handoff can overlap for a few
# milliseconds. Both represent one room for the same scope.
continue
if not job_key:
job_key = f"job:{job_id}"
keys.add(job_key)
scope_key = compute_active_dedupe_key(draft_id, user_id, task_id)
if scope_key:
durable_scopes.add(scope_key)
return keys, durable_scopes, hard_expired_keys
async def _ensure_gate_row(self) -> None:
"""Create the portable database mutex row once, tolerating races."""
now = datetime.now(UTC)
async with self._sf() as session:
if await session.get(RoundtableRoomLeaseRow, GATE_ROW_KEY) is not None:
return
session.add(
RoundtableRoomLeaseRow(
room_key=GATE_ROW_KEY,
draft_id=None,
user_id=None,
task_id=None,
holder_user_id=None,
lease_until=None,
created_at=now,
updated_at=now,
)
)
try:
await session.commit()
except IntegrityError:
await session.rollback()
async def _upsert_lease(
self,
session: AsyncSession,
*,
room_key: str,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
holder_user_id: str | None,
now: datetime,
) -> None:
row = await session.get(RoundtableRoomLeaseRow, room_key)
lease_until = now + timedelta(seconds=LEASE_SECONDS)
if row is None:
session.add(
RoundtableRoomLeaseRow(
room_key=room_key,
draft_id=draft_id,
user_id=user_id,
task_id=task_id,
holder_user_id=holder_user_id,
lease_until=lease_until,
created_at=now,
updated_at=now,
)
)
return
row.draft_id = draft_id
row.user_id = user_id
row.task_id = task_id
row.holder_user_id = holder_user_id
row.lease_until = lease_until
row.updated_at = now
@staticmethod
def _decision(
*,
ok: bool,
enabled: bool,
max_rooms: int,
active_count: int,
renewed: bool,
code: str,
message: str,
) -> dict[str, Any]:
return {
"ok": ok,
"enabled": enabled,
"max_rooms": max_rooms,
"active_count": active_count,
"renewed": renewed,
"code": code,
"message": message,
}
@@ -11,13 +11,15 @@ import json
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy import delete, func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.run.model import RunRow
from deerflow.runtime.runs.store.base import RunStore
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
_TERMINAL_RUN_STATUSES = ("success", "error", "timeout", "interrupted")
class RunRepository(RunStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
@@ -137,6 +139,14 @@ class RunRepository(RunStore):
await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(**values))
await session.commit()
async def update_snapshot(self, run_id, *, metadata, kwargs):
# Internal RunManager retention path; the run identity comes from its
# live registry, never a client-supplied mutation endpoint.
async with self._sf() as session:
result = await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(metadata_json=self._safe_json(metadata), kwargs_json=self._safe_json(kwargs)))
await session.commit()
return bool(result.rowcount)
async def delete(
self,
run_id,
@@ -205,21 +215,70 @@ class RunRepository(RunStore):
await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(**values))
await session.commit()
async def count_by_user(self) -> dict[str, int]:
"""Return ``{user_id: run_count}`` via a single GROUP BY query.
async def count_by_user(self, *, since: datetime | None = None) -> dict[str, int]:
"""Return ``{user_id: run_count}`` via one bounded GROUP BY query.
Each run is one user question, so this is the source for the admin
user center's 问答次数 column. Runs without a ``user_id`` are skipped.
"""
filters = [RunRow.user_id.is_not(None)]
if since is not None:
since_dt = since if since.tzinfo else since.replace(tzinfo=UTC)
filters.append(RunRow.created_at >= since_dt)
stmt = (
select(RunRow.user_id, func.count().label("cnt"))
.where(RunRow.user_id.is_not(None))
.where(*filters)
.group_by(RunRow.user_id)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
return {r.user_id: r.cnt for r in rows}
async def purge_terminal_before(
self,
cutoff: datetime,
*,
batch_size: int = 500,
) -> int:
"""Delete one portable, primary-key-bounded batch of old run audits.
Pending and running records are intentionally excluded, so retention
never removes work that could still be resumed or observed. A run must
also have been *updated* before the cutoff: an unusually long-running
job created months ago but completed today must remain visible. The
compact run row is cleaned, while the conversation checkpoint stays
under its separate user-history lifecycle.
"""
size = max(1, min(int(batch_size), 500))
cutoff_dt = cutoff if cutoff.tzinfo else cutoff.replace(tzinfo=UTC)
async with self._sf() as session:
ids = list(
(
await session.scalars(
select(RunRow.run_id)
.where(
RunRow.created_at < cutoff_dt,
RunRow.updated_at < cutoff_dt,
RunRow.status.in_(_TERMINAL_RUN_STATUSES),
)
.order_by(RunRow.created_at.asc(), RunRow.run_id.asc())
.limit(size)
)
).all()
)
if not ids:
return 0
result = await session.execute(
delete(RunRow).where(
RunRow.run_id.in_(ids),
RunRow.created_at < cutoff_dt,
RunRow.updated_at < cutoff_dt,
RunRow.status.in_(_TERMINAL_RUN_STATUSES),
)
)
await session.commit()
return int(result.rowcount or 0)
async def aggregate_tokens_by_thread(self, thread_id: str) -> dict[str, Any]:
"""Aggregate token usage via a single SQL GROUP BY query."""
_completed = RunRow.status.in_(("success", "error"))
@@ -95,6 +95,15 @@ class SkillStore(ABC):
"""
raise NotImplementedError
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
"""Map skill names to their configured Chinese display names.
Only names that have a non-empty ``name_zh`` are returned. The default
is empty so stores that have not implemented the lookup stay safe.
"""
del names
return {}
@abstractmethod
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
"""Return the skill names in ``user_id``'s "我的技能" collection.
@@ -165,6 +165,17 @@ class MemorySkillStore(SkillStore):
async def list_always_on_names(self) -> set[str]:
return {name for name, record in self._skills.items() if record.get("always_on")}
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
labels: dict[str, str] = {}
for name in names:
record = self._skills.get(name)
if record is None:
continue
label = str(record.get("name_zh") or "").strip()
if label:
labels[name] = label
return labels
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
return set(self._favorites.get(user_id, set()))
@@ -10,7 +10,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, String, Text, UniqueConstraint
from sqlalchemy import BigInteger, Boolean, DateTime, Index, Integer, String, Text, UniqueConstraint, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -19,10 +19,30 @@ from deerflow.persistence.types import BeijingDateTime
class SkillRow(Base):
__tablename__ = "skills"
__table_args__ = (
Index("ix_skills_catalog_available_category_created", "available", "category", "created_at"),
)
# Global unique name — must match the on-disk folder name under skills/custom/.
# Hyphen-case or snake_case allowed; see deerflow.skills.validation.
name: Mapped[str] = mapped_column(String(64), primary_key=True)
# Compact catalog projection fields. The list API reads these columns from
# MySQL and deliberately excludes the long-form ``detail`` text column.
category: Mapped[str] = mapped_column(String(16), nullable=False, default="custom", server_default="custom")
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="", server_default="")
license: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
available: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"), index=True)
use_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
view_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
patch_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
last_activity_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, default=None)
state: Mapped[str] = mapped_column(String(16), nullable=False, default="active", server_default="active")
pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
featured: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
source: Mapped[str] = mapped_column(String(16), nullable=False, default="upload", server_default="upload")
source_mtime_ns: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default=text("0"))
detail_preview: Mapped[str | None] = mapped_column(String(1024), nullable=True, default=None)
owner_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# Human-authored long-form detail/introduction, editable by the skill owner
@@ -83,3 +103,18 @@ class SkillDisplayAdapterRow(Base):
sample_payload: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class SkillCatalogSnapshotRow(Base):
"""MySQL-backed projection of filesystem skill metadata for fast list APIs.
Skill source files remain the authoring format. The Gateway refreshes this
one-row projection during startup so homepage listing requests do not scan
every SKILL.md or read per-skill usage sidecars.
"""
__tablename__ = "skill_catalog_snapshots"
cache_key: Mapped[str] = mapped_column(String(32), primary_key=True)
catalog_json: Mapped[str] = mapped_column(Text, nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -109,6 +109,7 @@ class SkillRepository(SkillStore):
row.published = bool(data["published"])
if "detail" in data:
row.detail = data["detail"]
row.detail_preview = (str(data["detail"] or "")[:1024] or None)
if "name_zh" in data:
row.name_zh = data["name_zh"]
if "always_on" in data:
@@ -131,6 +132,7 @@ class SkillRepository(SkillStore):
row.owner_user_id = data["owner_user_id"]
if "detail" in data:
row.detail = data["detail"]
row.detail_preview = (str(data["detail"] or "")[:1024] or None)
if "name_zh" in data:
row.name_zh = data["name_zh"]
if "always_on" in data:
@@ -183,6 +185,20 @@ class SkillRepository(SkillStore):
result = await session.execute(stmt)
return {row[0] for row in result.all()}
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
wanted = list(dict.fromkeys(name for name in names if name))
if not wanted:
return {}
stmt = select(SkillRow.name, SkillRow.name_zh).where(SkillRow.name.in_(wanted))
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
labels: dict[str, str] = {}
for name, name_zh in rows:
label = (name_zh or "").strip()
if label:
labels[name] = label
return labels
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
stmt = select(SkillFavoriteRow.skill_name).where(SkillFavoriteRow.user_id == user_id)
async with self._sf() as session:
@@ -0,0 +1,104 @@
"""Persistence for model-stream truncation events (模型截断记录).
The patched OpenAI-compatible streaming layer calls
:func:`record_stream_truncation_event` whenever a provider stream ends
abnormally. The write is fire-and-forget: it is routed onto the engine's home
loop via :func:`fire_and_forget_db_write` and must never disturb the model
stream itself. Rows are capped at one month by the SQL repository.
"""
from __future__ import annotations
import logging
import uuid
from datetime import UTC, datetime
from deerflow.persistence.stream_truncation.base import (
RETENTION_DAYS,
StreamTruncationEvent,
StreamTruncationQuery,
StreamTruncationStore,
)
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
from deerflow.persistence.stream_truncation.sql import SqlStreamTruncationStore
__all__ = [
"RETENTION_DAYS",
"SqlStreamTruncationStore",
"StreamTruncationEvent",
"StreamTruncationEventRow",
"StreamTruncationQuery",
"StreamTruncationStore",
"make_stream_truncation_store",
"record_stream_truncation_event",
]
logger = logging.getLogger(__name__)
_store_cache: SqlStreamTruncationStore | None = None
_store_cache_failed = False
def make_stream_truncation_store(session_factory) -> StreamTruncationStore | None:
"""Return a SQL-backed truncation store, or ``None`` without a DB."""
if session_factory is None:
return None
return SqlStreamTruncationStore(session_factory)
def record_stream_truncation_event(
*,
model_name: str | None,
event_type: str,
strict_mode: bool = False,
finish_reason: str = "none",
status: str = "success",
error_type: str | None = None,
chunk_count: int = 0,
duration_ms: int = 0,
first_chunk_ms: int = -1,
provider_completion_id: str | None = None,
run_id: str | None = None,
detail: str | None = None,
) -> None:
"""Best-effort truncation-event write from any loop/thread. Never raises."""
global _store_cache, _store_cache_failed
if _store_cache is None:
if _store_cache_failed:
return
try:
from deerflow.persistence.engine import get_session_factory
session_factory = get_session_factory()
except Exception:
session_factory = None
if session_factory is None:
# No DB configured (memory backend); do not retry on every stream.
_store_cache_failed = True
return
_store_cache = SqlStreamTruncationStore(session_factory)
run_id_value = run_id if run_id and run_id not in {"none", ""} else None
event = StreamTruncationEvent(
id=uuid.uuid4().hex,
created_at=datetime.now(UTC),
model_name=model_name,
provider_completion_id=provider_completion_id
if provider_completion_id and provider_completion_id != "none"
else None,
run_id=run_id_value,
event_type=event_type,
strict_mode=strict_mode,
finish_reason=finish_reason,
status=status,
error_type=error_type,
chunk_count=chunk_count,
duration_ms=duration_ms,
first_chunk_ms=first_chunk_ms,
detail=detail,
)
try:
from deerflow.persistence.engine import fire_and_forget_db_write
fire_and_forget_db_write(_store_cache.record(event))
except Exception:
logger.debug("stream truncation event drop", exc_info=True)
@@ -0,0 +1,76 @@
"""Repository contract for model-stream truncation events."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any
# Hard retention cap. The user-facing promise is "最多存一个月" of truncation
# records; the SQL repository enforces it on every insert so no scheduler is
# required for the cap to hold.
RETENTION_DAYS = 30
@dataclass
class StreamTruncationEvent:
"""In-memory shape of a truncation event before persistence."""
id: str
created_at: datetime
model_name: str | None = None
provider_completion_id: str | None = None
run_id: str | None = None
# silent_no_terminal / incomplete_stream_error / length_truncated /
# content_filter / midstream_error
event_type: str = "silent_no_terminal"
strict_mode: bool = False
finish_reason: str = "none"
status: str = "success"
error_type: str | None = None
chunk_count: int = 0
duration_ms: int = 0
first_chunk_ms: int = -1
detail: str | None = None
@dataclass
class StreamTruncationQuery:
"""Filter set for the admin list endpoint."""
model_name: str | None = None
event_type: str | None = None
since: datetime | None = None
until: datetime | None = None
limit: int = 100
offset: int = 0
class StreamTruncationStore(ABC):
"""Persistence contract for the model-layer recorder + admin API."""
@abstractmethod
async def record(self, event: StreamTruncationEvent) -> None:
"""Persist one event and enforce the retention cap. Must never raise."""
raise NotImplementedError
@abstractmethod
async def list(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
"""Return matching rows (newest first), paginated by ``query``."""
raise NotImplementedError
@abstractmethod
async def count(self, query: StreamTruncationQuery) -> int:
"""Return the unpaginated row count for ``query``."""
raise NotImplementedError
@abstractmethod
async def model_breakdown(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
"""Per-model aggregate: ``[{model_name, event_type counts..., total, last_at}]``."""
raise NotImplementedError
@abstractmethod
async def purge_before(self, cutoff: datetime) -> int:
"""Delete rows older than ``cutoff``; return the deleted row count."""
raise NotImplementedError
@@ -0,0 +1,80 @@
"""ORM model for model-stream truncation events."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Index, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class StreamTruncationEventRow(Base):
"""One row per detected model-stream truncation.
Written from the patched OpenAI-compatible streaming layer when a provider
stream ends abnormally: without a terminal frame (silently committed half
answer), via ``IncompleteChatStreamError`` (strict integrity mode, retried),
on an output-length cap, or interrupted mid-stream by a transport error.
Rows are retained for at most ``RETENTION_DAYS`` by the SQL repository.
"""
__tablename__ = "stream_truncation_events"
__table_args__ = (
Index("ix_stream_truncation_created_model", "created_at", "model_name"),
Index("ix_stream_truncation_created_event", "created_at", "event_type"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), index=True
)
# Configured model name (the config.yaml entry name), sanitized by the model layer.
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
# Provider-side completion id when the gateway supplied one; joins a row to
# the upstream gateway's own request log.
provider_completion_id: Mapped[str | None] = mapped_column(String(128))
# LangGraph run id when the model layer could see one ("none" is dropped).
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
# silent_no_terminal / incomplete_stream_error / length_truncated /
# content_filter / midstream_error
event_type: Mapped[str] = mapped_column(String(32), index=True)
# Whether the model ran with strict_stream_integrity enabled.
strict_mode: Mapped[bool] = mapped_column(Boolean, default=False)
# Raw provider finish reason bucket: none / stop / length / tool_calls /
# function_call / content_filter / end_turn / other.
finish_reason: Mapped[str] = mapped_column(String(16), default="none")
# Stream outcome from the model layer: success / error / cancelled.
status: Mapped[str] = mapped_column(String(16), default="success")
error_type: Mapped[str | None] = mapped_column(String(64))
chunk_count: Mapped[int] = mapped_column(Integer, default=0)
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
first_chunk_ms: Mapped[int] = mapped_column(Integer, default=-1)
detail: Mapped[str | None] = mapped_column(Text)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": self.created_at.isoformat()
if isinstance(self.created_at, datetime)
else self.created_at,
"model_name": self.model_name,
"provider_completion_id": self.provider_completion_id,
"run_id": self.run_id,
"event_type": self.event_type,
"strict_mode": bool(self.strict_mode),
"finish_reason": self.finish_reason,
"status": self.status,
"error_type": self.error_type,
"chunk_count": self.chunk_count,
"duration_ms": self.duration_ms,
"first_chunk_ms": self.first_chunk_ms,
"detail": self.detail,
}
@@ -0,0 +1,175 @@
"""SQLAlchemy-backed stream-truncation event repository."""
from __future__ import annotations
import logging
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, desc, func, select
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.stream_truncation.base import (
RETENTION_DAYS,
StreamTruncationEvent,
StreamTruncationQuery,
StreamTruncationStore,
)
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
logger = logging.getLogger(__name__)
class SqlStreamTruncationStore(StreamTruncationStore):
"""Persist truncation events with a hard one-month retention cap."""
def __init__(self, session_factory: async_sessionmaker) -> None:
self._session_factory = session_factory
async def _ensure_table(self, session) -> None:
# Table creation follows the repo's create_all-on-first-use convention
# (see e.g. task_buttons); safe to call repeatedly.
try:
await session.run_sync(
lambda sync_session: StreamTruncationEventRow.__table__.create(
sync_session.bind, checkfirst=True
)
)
except Exception:
logger.debug("stream_truncation table create skipped", exc_info=True)
async def record(self, event: StreamTruncationEvent) -> None:
try:
async with self._session_factory() as session:
await self._ensure_table(session)
session.add(
StreamTruncationEventRow(
id=event.id,
created_at=event.created_at,
model_name=event.model_name,
provider_completion_id=event.provider_completion_id,
run_id=event.run_id,
event_type=event.event_type,
strict_mode=event.strict_mode,
finish_reason=event.finish_reason,
status=event.status,
error_type=event.error_type,
chunk_count=event.chunk_count,
duration_ms=event.duration_ms,
first_chunk_ms=event.first_chunk_ms,
detail=event.detail,
)
)
await session.commit()
# Same session/transaction scope: enforce the retention cap on
# every insert. Truncations are rare, so this stays cheap and
# keeps the one-month promise without any scheduler.
await self._enforce_retention(session)
except Exception:
logger.warning(
"Failed to persist stream truncation event model=%s type=%s",
event.model_name,
event.event_type,
exc_info=True,
)
async def _enforce_retention(self, session) -> None:
"""Delete rows beyond the retention cap. Best-effort, never raises."""
try:
cutoff = datetime.now(UTC) - timedelta(days=RETENTION_DAYS)
await session.execute(
delete(StreamTruncationEventRow).where(
StreamTruncationEventRow.created_at < cutoff
)
)
await session.commit()
except Exception:
logger.debug("stream truncation retention sweep failed", exc_info=True)
def _apply_filters(self, query: StreamTruncationQuery):
stmt = select(StreamTruncationEventRow)
if query.model_name:
stmt = stmt.where(StreamTruncationEventRow.model_name == query.model_name)
if query.event_type:
stmt = stmt.where(StreamTruncationEventRow.event_type == query.event_type)
if query.since is not None:
stmt = stmt.where(StreamTruncationEventRow.created_at >= query.since)
if query.until is not None:
stmt = stmt.where(StreamTruncationEventRow.created_at <= query.until)
return stmt
async def list(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
try:
async with self._session_factory() as session:
await self._ensure_table(session)
stmt = self._apply_filters(query).order_by(
desc(StreamTruncationEventRow.created_at)
).limit(query.limit).offset(query.offset)
rows = (await session.execute(stmt)).scalars().all()
return [row.to_dict() for row in rows]
except Exception:
logger.warning("Failed to list stream truncation events", exc_info=True)
return []
async def count(self, query: StreamTruncationQuery) -> int:
try:
async with self._session_factory() as session:
await self._ensure_table(session)
stmt = self._apply_filters(query).with_only_columns(
func.count(StreamTruncationEventRow.id)
)
return int((await session.execute(stmt)).scalar() or 0)
except Exception:
logger.warning("Failed to count stream truncation events", exc_info=True)
return 0
async def model_breakdown(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
try:
async with self._session_factory() as session:
await self._ensure_table(session)
stmt = self._apply_filters(query).with_only_columns(
StreamTruncationEventRow.model_name,
StreamTruncationEventRow.event_type,
func.count(StreamTruncationEventRow.id).label("total"),
func.max(StreamTruncationEventRow.created_at).label("last_at"),
).group_by(StreamTruncationEventRow.model_name, StreamTruncationEventRow.event_type)
rows = (await session.execute(stmt)).all()
merged: dict[str, dict[str, Any]] = {}
for row in rows:
name = row.model_name or "(unknown)"
bucket = merged.setdefault(
name, {"model_name": name, "total": 0, "last_at": None}
)
bucket[row.event_type] = int(row.total or 0)
bucket["total"] += int(row.total or 0)
row_last = row.last_at
if row_last is not None and (
bucket["last_at"] is None or row_last > bucket["last_at"]
):
bucket["last_at"] = row_last
result = sorted(
merged.values(), key=lambda item: item["last_at"] or datetime.min.replace(tzinfo=UTC), reverse=True
)
for item in result:
if isinstance(item["last_at"], datetime):
item["last_at"] = item["last_at"].isoformat()
return result
except Exception:
logger.warning("Failed to aggregate stream truncation events", exc_info=True)
return []
async def purge_before(self, cutoff: datetime) -> int:
try:
async with self._session_factory() as session:
await self._ensure_table(session)
result = await session.execute(
delete(StreamTruncationEventRow).where(
StreamTruncationEventRow.created_at < cutoff
)
)
await session.commit()
rowcount = result.rowcount
return 0 if rowcount is None else int(rowcount)
except Exception:
logger.warning("Failed to purge stream truncation events", exc_info=True)
return 0
@@ -14,7 +14,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, UniqueConstraint
from sqlalchemy import Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -43,7 +43,14 @@ class TagRow(Base):
class TagAssignmentRow(Base):
__tablename__ = "tag_assignments"
__table_args__ = (UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),)
__table_args__ = (
UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),
# Gallery list joins assignments by (type, target); the tag filter
# looks up (type, tag). The unique key leads with tag_id, so neither
# query can use it as a prefix.
Index("ix_tag_assignments_type_target", "target_type", "target_id"),
Index("ix_tag_assignments_type_tag", "target_type", "tag_id", "target_id"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
@@ -0,0 +1,22 @@
"""External task deep-link detail persistence."""
from __future__ import annotations
from deerflow.persistence.task_deeplink_details.base import TaskDeeplinkDetailStore
from deerflow.persistence.task_deeplink_details.memory import MemoryTaskDeeplinkDetailStore
from deerflow.persistence.task_deeplink_details.model import TaskDeeplinkDetailRow
from deerflow.persistence.task_deeplink_details.sql import TaskDeeplinkDetailRepository
__all__ = [
"MemoryTaskDeeplinkDetailStore",
"TaskDeeplinkDetailRepository",
"TaskDeeplinkDetailRow",
"TaskDeeplinkDetailStore",
"make_task_deeplink_detail_store",
]
def make_task_deeplink_detail_store(session_factory):
if session_factory is None:
return MemoryTaskDeeplinkDetailStore()
return TaskDeeplinkDetailRepository(session_factory)
@@ -0,0 +1,47 @@
"""Persistence contract for externally supplied task deep-link details."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class TaskDeeplinkDetailStore(ABC):
"""Shared external task snapshots keyed by ``(detail_type, detail_id, sub_type)``.
``sub_type`` is the integrator-supplied business type (the deep-link
``?type=``): the same task id may hold one snapshot per type, each with its
own version counter. The empty string is the legacy no-type partition.
"""
@abstractmethod
async def get_detail(
self,
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
) -> dict[str, Any] | None:
"""Return one persisted task snapshot, or ``None`` when it is unknown."""
raise NotImplementedError
@abstractmethod
async def upsert_detail(
self,
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
task_name: str,
task_content: str,
overview: str,
extra: dict[str, Any],
updated_by: str | None,
) -> tuple[dict[str, Any], bool, bool]:
"""Return ``(row, created, changed)`` for a complete task snapshot.
Implementations must not issue a write when the stored snapshot has
exactly the same task fields. This keeps caller-side retries cheap
and makes a repeated ``PUT`` idempotent in both state and version.
"""
raise NotImplementedError
@@ -0,0 +1,76 @@
"""In-memory implementation for external task deep-link details."""
from __future__ import annotations
from copy import deepcopy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.task_deeplink_details.base import TaskDeeplinkDetailStore
class MemoryTaskDeeplinkDetailStore(TaskDeeplinkDetailStore):
def __init__(self) -> None:
self._rows: dict[tuple[str, str, str], dict[str, Any]] = {}
async def get_detail(
self,
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
) -> dict[str, Any] | None:
row = self._rows.get((detail_type, detail_id, sub_type))
return deepcopy(row) if row is not None else None
async def upsert_detail(
self,
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
task_name: str,
task_content: str,
overview: str,
extra: dict[str, Any],
updated_by: str | None,
) -> tuple[dict[str, Any], bool, bool]:
key = (detail_type, detail_id, sub_type)
row = self._rows.get(key)
created = row is None
if row is None:
now = datetime.now(UTC).isoformat()
row = {
"detail_type": detail_type,
"detail_id": detail_id,
"sub_type": sub_type,
"version": 1,
"created_by": updated_by,
"created_at": now,
}
self._rows[key] = row
changed = True
elif (
row["task_name"] == task_name
and row["task_content"] == task_content
and row["overview"] == overview
and row["extra"] == extra
):
return deepcopy(row), False, False
else:
now = datetime.now(UTC).isoformat()
row["version"] = int(row.get("version") or 0) + 1
changed = True
if changed:
row.update(
{
"task_name": task_name,
"task_content": task_content,
"overview": overview,
"extra": deepcopy(extra),
"updated_by": updated_by,
"updated_at": now,
}
)
return deepcopy(row), created, changed
@@ -0,0 +1,39 @@
"""ORM model for externally supplied task deep-link details."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class TaskDeeplinkDetailRow(Base):
"""Canonical task snapshot received from an external deep-link provider."""
__tablename__ = "task_deeplink_details"
detail_type: Mapped[str] = mapped_column(String(32), primary_key=True)
detail_id: Mapped[str] = mapped_column(String(128), primary_key=True)
# Integrator-supplied business type (the deep-link ``?type=``): one task id
# may carry several typed snapshots; "" is the legacy no-type partition.
sub_type: Mapped[str] = mapped_column(String(64), primary_key=True, default="")
task_name: Mapped[str] = mapped_column(String(512), nullable=False, default="")
task_content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
overview: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
extra_json: Mapped[dict[str, object]] = mapped_column(PortableJSON, nullable=False, default=dict)
version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,102 @@
"""SQLAlchemy-backed external task deep-link detail store."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.task_deeplink_details.base import TaskDeeplinkDetailStore
from deerflow.persistence.task_deeplink_details.model import TaskDeeplinkDetailRow
def _to_dict(row: TaskDeeplinkDetailRow) -> dict[str, Any]:
return {
"detail_type": row.detail_type,
"detail_id": row.detail_id,
"sub_type": row.sub_type or "",
"task_name": row.task_name,
"task_content": row.task_content,
"overview": row.overview,
"extra": dict(row.extra_json or {}),
"version": row.version,
"created_by": row.created_by,
"updated_by": row.updated_by,
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
class TaskDeeplinkDetailRepository(TaskDeeplinkDetailStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get_detail(
self,
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(
TaskDeeplinkDetailRow,
{"detail_type": detail_type, "detail_id": detail_id, "sub_type": sub_type},
)
return _to_dict(row) if row is not None else None
async def upsert_detail(
self,
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
task_name: str,
task_content: str,
overview: str,
extra: dict[str, Any],
updated_by: str | None,
) -> tuple[dict[str, Any], bool, bool]:
async with self._sf() as session:
row = await session.get(
TaskDeeplinkDetailRow,
{"detail_type": detail_type, "detail_id": detail_id, "sub_type": sub_type},
)
created = row is None
now = datetime.now(UTC)
if row is None:
row = TaskDeeplinkDetailRow(
detail_type=detail_type,
detail_id=detail_id,
sub_type=sub_type,
task_name=task_name,
task_content=task_content,
overview=overview,
extra_json=extra,
version=1,
created_by=updated_by,
updated_by=updated_by,
created_at=now,
updated_at=now,
)
session.add(row)
changed = True
elif (
row.task_name == task_name
and row.task_content == task_content
and row.overview == overview
and dict(row.extra_json or {}) == extra
):
return _to_dict(row), False, False
else:
row.task_name = task_name
row.task_content = task_content
row.overview = overview
row.extra_json = extra
row.version += 1
row.updated_by = updated_by
row.updated_at = now
changed = True
await session.commit()
return _to_dict(row), created, changed
@@ -1,4 +1,4 @@
"""Dedicated taskId + goPath conversation-history index."""
"""Dedicated taskId + goPath conversation-history cards."""
from __future__ import annotations
@@ -1,15 +1,22 @@
"""Shared helpers for task deep-link conversation indexing.
"""Shared helpers for task deep-link conversation history.
The LangGraph thread/checkpoint remains the source of truth for messages and
execution state. This package only owns a small, dedicated lookup index so the
task workspaces never have to search/filter the global ``threads_meta`` table.
execution state. This package owns a dedicated history card so the task
workspaces never have to search, filter, or join the global ``threads_meta``
catalogue when listing a task's conversations.
"""
from __future__ import annotations
from typing import Any, TypedDict
TASK_DEEPLINK_GO_PATHS = frozenset({"3qfx", "rwfx", "xdfx"})
TASK_DEEPLINK_GO_PATHS = frozenset({"3qfx", "rwfx", "xdfx", "b5-1"})
# Custom business mappings use their administrator-defined name as goPath.
# Keep this in lock-step with ``business_mapping.business_code``.
MAX_TASK_DEEPLINK_GO_PATH = 32
# Matches the ``task_deeplink_threads.sub_type`` column width.
MAX_TASK_DEEPLINK_SUB_TYPE = 64
class TaskDeeplinkBinding(TypedDict):
@@ -17,6 +24,21 @@ class TaskDeeplinkBinding(TypedDict):
source_task_id: str
go_path: str
agent_id: str | None
# b5-1 integrator-supplied ?type= partition; "" when the link had none.
sub_type: str
def is_valid_task_deeplink_go_path(value: str) -> bool:
"""Accept built-ins and validated custom mapping names.
The harness cannot import the app-level business-mapping store (the import
firewall deliberately forbids that), so it validates the safe wire format
here. Creation itself remains admin-only in the Gateway router.
"""
go_path = value.strip().lower()
if go_path in TASK_DEEPLINK_GO_PATHS:
return True
return bool(go_path) and len(go_path) <= MAX_TASK_DEEPLINK_GO_PATH and not go_path.endswith("-a")
def task_deeplink_binding(metadata: dict[str, Any] | None) -> TaskDeeplinkBinding | None:
@@ -26,21 +48,25 @@ def task_deeplink_binding(metadata: dict[str, Any] | None) -> TaskDeeplinkBindin
actual task id for 3qfx/rwfx and the action id for xdfx. Older callers may
omit it, in which case ``taskId`` is a safe fallback. A ``taskGoPath`` is
mandatory, deliberately preventing legacy task rows from being migrated or
accidentally mixed into the new store.
accidentally mixed into the new store. ``taskType`` (the b5-1 ``?type=``
business type) further partitions one task's history so integrator-supplied
types never mix; it is capped to the column width here.
"""
md = metadata or {}
go_path = str(md.get("taskGoPath") or "").strip().lower()
if go_path not in TASK_DEEPLINK_GO_PATHS:
if not is_valid_task_deeplink_go_path(go_path):
return None
source_task_id = str(md.get("taskId") or "").strip()
task_id = str(md.get("taskRecordId") or source_task_id).strip()
if not task_id or not source_task_id:
return None
agent_id = str(md.get("agent_id") or "").strip() or None
sub_type = str(md.get("taskType") or "").strip()[:MAX_TASK_DEEPLINK_SUB_TYPE]
return {
"task_id": task_id,
"source_task_id": source_task_id,
"go_path": go_path,
"agent_id": agent_id,
"sub_type": sub_type,
}
@@ -1,4 +1,4 @@
"""In-memory task deep-link index backed by LangGraph BaseStore."""
"""In-memory task deep-link history cards backed by LangGraph BaseStore."""
from __future__ import annotations
@@ -16,6 +16,19 @@ if TYPE_CHECKING:
TASK_DEEPLINK_THREADS_NS: tuple[str, ...] = ("task_deeplink_threads",)
def _card_record(thread: dict[str, Any], binding: dict[str, Any]) -> dict[str, Any]:
return {
"thread_id": str(thread.get("thread_id") or ""),
**binding,
"display_name": thread.get("display_name"),
"status": thread.get("status") or "idle",
"metadata": thread.get("metadata") or {},
"created_by": thread.get("user_id"),
"created_at": thread.get("created_at") or now_iso(),
"updated_at": thread.get("updated_at") or now_iso(),
}
async def sync_task_deeplink_index(store: BaseStore, thread: dict[str, Any]) -> None:
metadata = thread.get("metadata") or {}
binding = task_deeplink_binding(metadata)
@@ -26,17 +39,7 @@ async def sync_task_deeplink_index(store: BaseStore, thread: dict[str, Any]) ->
if "taskGoPath" in metadata or "taskRecordId" in metadata:
await store.adelete(TASK_DEEPLINK_THREADS_NS, thread_id)
return
await store.aput(
TASK_DEEPLINK_THREADS_NS,
thread_id,
{
"thread_id": thread_id,
**binding,
"created_by": thread.get("user_id"),
"created_at": thread.get("created_at") or now_iso(),
"updated_at": thread.get("updated_at") or now_iso(),
},
)
await store.aput(TASK_DEEPLINK_THREADS_NS, thread_id, _card_record(thread, binding))
async def delete_task_deeplink_index(store: BaseStore, thread_id: str) -> None:
@@ -46,6 +49,8 @@ async def delete_task_deeplink_index(store: BaseStore, thread_id: str) -> None:
class MemoryTaskDeeplinkThreadStore:
def __init__(self, store: BaseStore, thread_store: ThreadMetaStore) -> None:
self._store = store
# Kept for constructor compatibility with the factory. Listing no
# longer reads the generic thread catalogue.
self._thread_store = thread_store
async def list_threads(
@@ -54,28 +59,33 @@ class MemoryTaskDeeplinkThreadStore:
go_path: str,
*,
limit: int = 100,
sub_type: str | None = None,
) -> list[dict[str, Any]]:
# ``sub_type`` semantics mirror the SQL repository: provided value (even
# "") narrows to that b5-1 ?type= partition; None returns all sub-types.
conditions = {"task_id": task_id, "go_path": go_path}
if sub_type is not None:
conditions["sub_type"] = sub_type
items = await self._store.asearch(
TASK_DEEPLINK_THREADS_NS,
filter={"task_id": task_id, "go_path": go_path},
filter=conditions,
limit=limit,
)
rows: list[dict[str, Any]] = []
for item in items:
thread_id = str(item.value.get("thread_id") or "")
value = item.value or {}
thread_id = str(value.get("thread_id") or "")
if not thread_id:
continue
thread = await self._thread_store.get(thread_id, user_id=None)
if thread is None:
continue
display_name = value.get("display_name")
rows.append(
{
"thread_id": thread_id,
"status": thread.get("status", "idle"),
"created_at": thread.get("created_at", ""),
"updated_at": thread.get("updated_at", ""),
"metadata": thread.get("metadata") or {},
"values": ({"title": thread.get("display_name")} if thread.get("display_name") else {}),
"status": value.get("status") or "idle",
"created_at": value.get("created_at", ""),
"updated_at": value.get("updated_at", ""),
"metadata": value.get("metadata") or {},
"values": ({"title": display_name} if display_name else {}),
"interrupts": {},
}
)
@@ -1,4 +1,4 @@
"""Dedicated lookup rows for single-agent task deep-link conversations."""
"""Dedicated history cards for single-agent task deep-link conversations."""
from __future__ import annotations
@@ -8,15 +8,15 @@ from sqlalchemy import Index, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class TaskDeeplinkThreadRow(Base):
"""One task-workspace history entry pointing at one LangGraph thread.
"""One task-workspace history card pointing at one LangGraph thread.
The row intentionally stores no messages or title. Those remain in the
canonical checkpoint / ``threads_meta`` record and are joined by primary
key when listing a task's history.
List queries read only this table. Title / status / metadata are copied
here so the huge generic ``threads_meta`` catalogue is never joined.
Messages stay in the LangGraph checkpoint.
"""
__tablename__ = "task_deeplink_threads"
@@ -25,6 +25,17 @@ class TaskDeeplinkThreadRow(Base):
"ix_task_deeplink_threads_scope",
"task_id",
"go_path",
"sub_type",
"updated_at",
"thread_id",
),
# 3qfx/rwfx/xdfx omit sub_type. The scope index is ordered by
# sub_type before updated_at, so those listings cannot ride it for
# "newest first" without a filesort. This prefix matches that query.
Index(
"ix_task_deeplink_threads_task_go_updated",
"task_id",
"go_path",
"updated_at",
"thread_id",
),
@@ -35,8 +46,14 @@ class TaskDeeplinkThreadRow(Base):
task_id: Mapped[str] = mapped_column(String(128), nullable=False)
# Resolved real task id used by model/run context (equals task_id except xdfx).
source_task_id: Mapped[str] = mapped_column(String(128), nullable=False)
go_path: Mapped[str] = mapped_column(String(16), nullable=False)
go_path: Mapped[str] = mapped_column(String(32), nullable=False)
# b5-1 ?type= business type; "" for links without one (and for the other
# go paths, which have no type dimension).
sub_type: Mapped[str] = mapped_column(String(64), nullable=False, default="")
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
display_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="idle")
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
# Audit only. Task deep-link histories remain shared across authenticated users.
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -1,11 +1,11 @@
"""SQL implementation of the dedicated task deep-link conversation index."""
"""SQL implementation of the dedicated task deep-link conversation cards."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from sqlalchemy import delete, select
from sqlalchemy import delete, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.task_deeplink_threads.base import task_deeplink_binding
@@ -15,8 +15,28 @@ if TYPE_CHECKING:
from deerflow.persistence.thread_meta.model import ThreadMetaRow
def _card_payload(thread: TaskDeeplinkThreadRow) -> dict[str, Any]:
return {
"thread_id": thread.thread_id,
"status": thread.status or "idle",
"created_at": thread.created_at,
"updated_at": thread.updated_at,
"metadata": thread.metadata_json or {},
"values": ({"title": thread.display_name} if thread.display_name else {}),
"interrupts": {},
}
def _apply_card_fields(row: TaskDeeplinkThreadRow, thread: ThreadMetaRow, now: datetime) -> None:
row.display_name = thread.display_name
row.status = thread.status or "idle"
row.metadata_json = dict(thread.metadata_json or {})
row.created_by = thread.user_id
row.updated_at = now
async def sync_task_deeplink_index(session: AsyncSession, thread: ThreadMetaRow) -> None:
"""Synchronize one index row inside the caller's thread-meta transaction."""
"""Synchronize one history card inside the caller's thread-meta transaction."""
metadata = thread.metadata_json or {}
binding = task_deeplink_binding(metadata)
@@ -30,25 +50,39 @@ async def sync_task_deeplink_index(session: AsyncSession, thread: ThreadMetaRow)
existing = await session.get(TaskDeeplinkThreadRow, thread.thread_id)
now = thread.updated_at if isinstance(thread.updated_at, datetime) else datetime.now(UTC)
if existing is None:
session.add(
TaskDeeplinkThreadRow(
thread_id=thread.thread_id,
task_id=binding["task_id"],
source_task_id=binding["source_task_id"],
go_path=binding["go_path"],
agent_id=binding["agent_id"],
created_by=thread.user_id,
created_at=(thread.created_at if isinstance(thread.created_at, datetime) else now),
updated_at=now,
)
row = TaskDeeplinkThreadRow(
thread_id=thread.thread_id,
task_id=binding["task_id"],
source_task_id=binding["source_task_id"],
go_path=binding["go_path"],
sub_type=binding["sub_type"],
agent_id=binding["agent_id"],
created_by=thread.user_id,
created_at=(thread.created_at if isinstance(thread.created_at, datetime) else now),
updated_at=now,
)
_apply_card_fields(row, thread, now)
session.add(row)
return
existing.task_id = binding["task_id"]
existing.source_task_id = binding["source_task_id"]
existing.go_path = binding["go_path"]
existing.sub_type = binding["sub_type"]
existing.agent_id = binding["agent_id"]
existing.updated_at = now
_apply_card_fields(existing, thread, now)
async def update_task_deeplink_card(session: AsyncSession, thread_id: str, **values: Any) -> None:
"""Patch a history card without reading ``threads_meta``.
Title / status / ``updated_at`` changes on ordinary chats no-op here
because those ``thread_id`` values are absent from this table.
"""
if not values:
return
await session.execute(update(TaskDeeplinkThreadRow).where(TaskDeeplinkThreadRow.thread_id == thread_id).values(**values))
async def delete_task_deeplink_index(session: AsyncSession, thread_id: str) -> None:
@@ -65,41 +99,30 @@ class TaskDeeplinkThreadRepository:
go_path: str,
*,
limit: int = 100,
sub_type: str | None = None,
) -> list[dict[str, Any]]:
"""List one task/goPath partition without scanning thread metadata JSON."""
"""List one task/goPath partition from the dedicated card table only.
# Local import avoids a package-initialization cycle: thread_meta.sql
# imports the sync helpers above for atomic dual writes.
from deerflow.persistence.thread_meta.model import ThreadMetaRow
``sub_type`` narrows the partition to one b5-1 ``?type=`` value when
provided (an explicit "" selects the no-type partition); ``None`` keeps
the untyped legacy behaviour of returning every sub-type at once.
"""
conditions = [
TaskDeeplinkThreadRow.task_id == task_id,
TaskDeeplinkThreadRow.go_path == go_path,
]
if sub_type is not None:
conditions.append(TaskDeeplinkThreadRow.sub_type == sub_type)
stmt = (
select(TaskDeeplinkThreadRow, ThreadMetaRow)
.join(
ThreadMetaRow,
ThreadMetaRow.thread_id == TaskDeeplinkThreadRow.thread_id,
)
.where(
TaskDeeplinkThreadRow.task_id == task_id,
TaskDeeplinkThreadRow.go_path == go_path,
)
select(TaskDeeplinkThreadRow)
.where(*conditions)
.order_by(
ThreadMetaRow.updated_at.desc(),
ThreadMetaRow.thread_id.desc(),
TaskDeeplinkThreadRow.updated_at.desc(),
TaskDeeplinkThreadRow.thread_id.desc(),
)
.limit(limit)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
return [
{
"thread_id": thread.thread_id,
"status": thread.status,
"created_at": thread.created_at,
"updated_at": thread.updated_at,
"metadata": thread.metadata_json or {},
"values": ({"title": thread.display_name} if thread.display_name else {}),
"interrupts": {},
}
for _index, thread in rows
]
rows = (await session.execute(stmt)).scalars().all()
return [_card_payload(row) for row in rows]
@@ -54,9 +54,22 @@ def is_excluded_system_thread(metadata: dict[str, Any] | None) -> bool:
carry ``task_analysis_seat``. Those rows stay in the execution record and
out of recent chats.
"""
md = metadata or {}
if md.get("system"):
return True
md = metadata or {}
if md.get("system"):
return True
# Task-analysis keeps its user-facing history in task_analysis_runs /
# task_analysis_run_messages. The underlying LangGraph threads are runtime
# implementation details; recognise their task markers as a defensive
# fallback in case a legacy caller omitted ``system`` or ``thread_type``.
if any(
isinstance(md.get(key), str) and md[key].strip()
for key in (
"task_analysis_task_id",
"task_analysis_source_task_id",
"task_analysis_run_id",
)
):
return True
seat = md.get("task_analysis_seat")
if isinstance(seat, str) and seat.strip():
return True
@@ -192,10 +192,14 @@ class MemoryThreadMetaStore(ThreadMetaStore):
item = await self._store.aget(THREADS_NS, thread_id)
if item is None:
return not require_existing
record_user_id = item.value.get("user_id")
if record_user_id:
from deerflow.runtime.thread_paths import remember_thread_owner
remember_thread_owner(thread_id, str(record_user_id))
# 任务深链 / 圆桌 / 任务分析系统线程:全局共享(与 SQL store 同口径)。
if is_shared_access_thread(item.value.get("metadata") or {}):
return True
record_user_id = item.value.get("user_id")
if record_user_id is None:
return True
return record_user_id == user_id
@@ -207,6 +211,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
record["display_name"] = display_name
record["updated_at"] = now_iso()
await self._store.aput(THREADS_NS, thread_id, record)
await sync_task_deeplink_index(self._store, record)
async def update_status(self, thread_id: str, status: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_status")
@@ -215,6 +220,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
record["status"] = status
record["updated_at"] = now_iso()
await self._store.aput(THREADS_NS, thread_id, record)
await sync_task_deeplink_index(self._store, record)
async def update_assistant_id(self, thread_id: str, assistant_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_assistant_id")
@@ -223,6 +229,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
record["assistant_id"] = assistant_id
record["updated_at"] = now_iso()
await self._store.aput(THREADS_NS, thread_id, record)
await sync_task_deeplink_index(self._store, record)
async def update_metadata(self, thread_id: str, metadata: dict, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_metadata")
@@ -5,72 +5,60 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy import func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.json_compat import JsonPredicate, supports_json_filter
from deerflow.persistence.task_deeplink_threads.sql import (
delete_task_deeplink_index,
sync_task_deeplink_index,
update_task_deeplink_card,
)
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
from deerflow.persistence.thread_meta.base import EXCLUDED_SYSTEM_THREAD_TYPES, ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
_INDEXED_METADATA_COLUMNS = {
"taskId": ThreadMetaRow.task_id,
"agent_id": ThreadMetaRow.agent_id,
"actionId": ThreadMetaRow.action_id,
}
_INDEXED_METADATA_ATTRIBUTES = {
"taskId": "task_id",
"agent_id": "agent_id",
"actionId": "action_id",
}
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
"""Return metadata predicates that have a maintained SQL projection.
Task deep-link callers use string ids. Restrict the SQL shortcut to
strings so legacy callers that intentionally store another JSON type keep
the exact Python-side equality semantics of the generic metadata filter.
"""
return {
key: value
for key, value in (metadata or {}).items()
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
}
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
"""Build ORM column values from authoritative thread metadata.
Always include every projection so a metadata update that changes a value
from a string to another JSON type cannot leave a stale indexed value.
"""
values = _indexed_metadata_values(metadata)
return {
attribute: values.get(metadata_key)
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
}
def _apply_indexed_metadata_filters(stmt: Any, metadata: dict[str, Any] | None) -> tuple[Any, set[str]]:
"""Apply known metadata predicates in SQL and return their keys.
The caller still verifies the full metadata object in Python whenever a
generic filter or system-thread exclusion is involved. This keeps the
public API's exact-match contract while reducing that verification set to
rows for the requested task/agent/action instead of the entire table.
"""
values = _indexed_metadata_values(metadata)
for key, value in values.items():
stmt = stmt.where(_INDEXED_METADATA_COLUMNS[key] == value)
return stmt, set(values)
class ThreadMetaRepository(ThreadMetaStore):
_INDEXED_METADATA_COLUMNS = {
"taskId": ThreadMetaRow.task_id,
"agent_id": ThreadMetaRow.agent_id,
"actionId": ThreadMetaRow.action_id,
}
_INDEXED_METADATA_ATTRIBUTES = {
"taskId": "task_id",
"agent_id": "agent_id",
"actionId": "action_id",
}
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
"""Return metadata predicates that have a maintained SQL projection.
Task deep-link callers use string ids. Restrict the SQL shortcut to
strings so legacy callers that intentionally store another JSON type keep
the exact Python-side equality semantics of the generic metadata filter.
"""
return {
key: value
for key, value in (metadata or {}).items()
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
}
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
"""Build ORM column values from authoritative thread metadata.
Always include every projection so a metadata update that changes a value
from a string to another JSON type cannot leave a stale indexed value.
"""
values = _indexed_metadata_values(metadata)
return {
attribute: values.get(metadata_key)
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
}
class ThreadMetaRepository(ThreadMetaStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -97,14 +85,14 @@ class ThreadMetaRepository(ThreadMetaStore):
# creates an orphan row (used by migration scripts).
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.create")
now = datetime.now(UTC)
row = ThreadMetaRow(
thread_id=thread_id,
assistant_id=assistant_id,
user_id=resolved_user_id,
**_indexed_thread_metadata(metadata),
display_name=display_name,
status="idle",
metadata_json=metadata or {},
row = ThreadMetaRow(
thread_id=thread_id,
assistant_id=assistant_id,
user_id=resolved_user_id,
**_indexed_thread_metadata(metadata),
display_name=display_name,
status="idle",
metadata_json=metadata or {},
created_at=now,
updated_at=now,
)
@@ -164,6 +152,10 @@ class ThreadMetaRepository(ThreadMetaStore):
row = await session.get(ThreadMetaRow, thread_id)
if row is None:
return not require_existing
if row.user_id:
from deerflow.runtime.thread_paths import remember_thread_owner
remember_thread_owner(thread_id, row.user_id)
md = row.metadata_json or {}
# 全局共享、不区分用户(任何登录用户都可读写/续聊):
# ① 任务深链对话(metadata.taskId);
@@ -176,6 +168,43 @@ class ThreadMetaRepository(ThreadMetaStore):
return True
return row.user_id == user_id
@staticmethod
def _query_conditions(*, dialect, metadata, status, user_id, exclude_system, query):
"""Share exact search/count predicates without changing owner resolution.
Complex metadata and unsupported dialects retain their exact Python
filter. No rejected key/value is silently ignored. Server numeric JSON
comparisons retain that path: DECIMAL and DOUBLE cannot reproduce all
Python float/int comparisons (including large numeric task ids).
"""
conditions = []
if user_id is not None:
conditions.append(ThreadMetaRow.user_id == user_id)
if status:
conditions.append(ThreadMetaRow.status == status)
if query and query.strip():
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
# Retain the master branch's indexed task/agent/action narrowing.
# JSON predicates below still enforce exact case/type semantics.
for key, value in _indexed_metadata_values(metadata).items():
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
supported = dialect in {"sqlite", "postgresql", "mysql"}
residual = {}
for key, value in (metadata or {}).items():
numeric_server = dialect in {"mysql", "postgresql"} and isinstance(value, (bool, int, float))
if supported and supports_json_filter(key, value) and not numeric_server:
conditions.append(JsonPredicate(ThreadMetaRow.metadata_json, key, value=value))
else:
residual[key] = value
if exclude_system and supported:
column = ThreadMetaRow.metadata_json
conditions.append(~or_(
JsonPredicate(column, "system", "truthy"),
JsonPredicate(column, "task_analysis_seat", "nonblank"),
*(JsonPredicate(column, "thread_type", value=value) for value in sorted(EXCLUDED_SYSTEM_THREAD_TYPES)),
))
return conditions, residual, exclude_system and not supported
async def search(
self,
*,
@@ -187,71 +216,46 @@ class ThreadMetaRepository(ThreadMetaStore):
exclude_system: bool = False,
query: str | None = None,
) -> list[dict]:
"""Search threads with optional metadata and status filters.
Owner filter is enforced by default: caller must be in a user
context. Pass ``user_id=None`` to bypass (migration/CLI).
"""
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.search")
# 任务深链对话(按 metadata.taskId 过滤)全局共享、不区分用户:跳过 owner 过滤。
# Preserve task-scoped sharing, still applying EVERY metadata filter
# (taskId + agent_id + actionId) before the pagination window.
if metadata and metadata.get("taskId"):
resolved_user_id = None
# thread_id tie-breaker keeps OFFSET pagination stable when many rows
# share the same updated_at (bulk imports, scheduler bursts).
stmt = select(ThreadMetaRow).order_by(ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc())
if resolved_user_id is not None:
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
if status:
stmt = stmt.where(ThreadMetaRow.status == status)
if query and query.strip():
# Case-insensitive title search. display_name is a real column, so
# this filter runs in SQL even when the JSON filters below fall
# back to Python-side scanning.
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
# ``metadata_json`` is deliberately portable, so arbitrary filters
# still receive Python-side equality checks below. These three task
# identifiers are maintained projections and can safely narrow the
# SQL result first. A deep-link query that used to scan every thread
# now reads only that task's rows.
stmt, indexed_metadata_keys = _apply_indexed_metadata_filters(stmt, metadata)
remaining_metadata_keys = set(metadata or {}) - indexed_metadata_keys
if not remaining_metadata_keys and not exclude_system:
stmt = stmt.limit(limit).offset(offset)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
# Metadata filters live inside the JSON blob, so rows are scanned in
# batches and filtered in Python until the requested window is full.
# A fixed over-fetch window would under-fill pages when most rows are
# filtered out (e.g. heavy scheduled-task usage), making callers that
# paginate believe the list is exhausted. TODO(Phase 2): use JSON DB
# operators (Postgres @>, SQLite json_extract) for server-side filtering.
def _matches(row: dict) -> bool:
md = row.get("metadata") or {}
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
return False
if exclude_system and is_excluded_system_thread(md):
return False
return True
needed = offset + limit
batch_size = max(500, needed)
matched: list[dict] = []
scan_offset = 0
async with self._sf() as session:
conditions, residual, filter_system = self._query_conditions(
dialect=session.get_bind().dialect.name,
metadata=metadata, status=status, user_id=resolved_user_id,
exclude_system=exclude_system, query=query,
)
stmt = select(ThreadMetaRow).where(*conditions).order_by(
ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc(),
)
if not residual and not filter_system:
result = await session.execute(stmt.limit(limit).offset(offset))
return [self._row_to_dict(row) for row in result.scalars()]
# Compatibility path for nested values, unusual literal keys,
# numeric server filters, or the separate openGauss dialect.
needed = offset + limit
batch_size = max(500, needed)
matched = []
scan_offset = 0
while len(matched) < needed:
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
rows = result.scalars().all()
if not rows:
break
matched.extend(d for d in (self._row_to_dict(r) for r in rows) if _matches(d))
for row in rows:
record = self._row_to_dict(row)
md = record.get("metadata") or {}
if all(md.get(key) == value for key, value in residual.items()) and not (
filter_system and is_excluded_system_thread(md)
):
matched.append(record)
if len(rows) < batch_size:
break
scan_offset += len(rows)
return matched[offset:needed]
return matched[offset:needed]
async def count(
self,
@@ -262,56 +266,37 @@ class ThreadMetaRepository(ThreadMetaStore):
exclude_system: bool = False,
query: str | None = None,
) -> int:
"""Count threads matching the same filters as :meth:`search`."""
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.count")
# 任务深链对话(按 metadata.taskId 过滤)全局共享、不区分用户:跳过 owner 过滤。
if metadata and metadata.get("taskId"):
resolved_user_id = None
conditions = []
if resolved_user_id is not None:
conditions.append(ThreadMetaRow.user_id == resolved_user_id)
if status:
conditions.append(ThreadMetaRow.status == status)
if query and query.strip():
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
indexed_metadata = _indexed_metadata_values(metadata)
for key, value in indexed_metadata.items():
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
remaining_metadata_keys = set(metadata or {}) - set(indexed_metadata)
if not remaining_metadata_keys and not exclude_system:
stmt = select(func.count()).select_from(ThreadMetaRow)
for condition in conditions:
stmt = stmt.where(condition)
async with self._sf() as session:
async with self._sf() as session:
conditions, residual, filter_system = self._query_conditions(
dialect=session.get_bind().dialect.name,
metadata=metadata, status=status, user_id=resolved_user_id,
exclude_system=exclude_system, query=query,
)
if not residual and not filter_system:
stmt = select(func.count()).select_from(ThreadMetaRow).where(*conditions)
return int((await session.execute(stmt)).scalar_one())
# JSON filters can't run in portable SQL — scan just the metadata
# column in batches and count in Python (mirrors search()).
stmt = select(ThreadMetaRow.metadata_json).order_by(ThreadMetaRow.thread_id)
for condition in conditions:
stmt = stmt.where(condition)
total = 0
batch_size = 1000
scan_offset = 0
async with self._sf() as session:
stmt = select(ThreadMetaRow.metadata_json).where(*conditions).order_by(ThreadMetaRow.thread_id)
total = 0
scan_offset = 0
batch_size = 1000
while True:
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
rows = result.scalars().all()
rows = (await session.execute(stmt.limit(batch_size).offset(scan_offset))).scalars().all()
if not rows:
break
for metadata_json in rows:
md = metadata_json or {}
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
continue
if exclude_system and is_excluded_system_thread(md):
continue
total += 1
for md in rows:
md = md or {}
if all(md.get(key) == value for key, value in residual.items()) and not (
filter_system and is_excluded_system_thread(md)
):
total += 1
if len(rows) < batch_size:
break
scan_offset += len(rows)
return total
return total
async def _check_ownership(self, session: AsyncSession, thread_id: str, resolved_user_id: str | None) -> bool:
"""Return True if the row exists and is owned (or filter bypassed)."""
@@ -337,7 +322,9 @@ class ThreadMetaRepository(ThreadMetaStore):
async with self._sf() as session:
if not await self._check_ownership(session, thread_id, resolved_user_id):
return
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=datetime.now(UTC)))
now = datetime.now(UTC)
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=now))
await update_task_deeplink_card(session, thread_id, display_name=display_name, updated_at=now)
await session.commit()
async def update_status(
@@ -351,7 +338,9 @@ class ThreadMetaRepository(ThreadMetaStore):
async with self._sf() as session:
if not await self._check_ownership(session, thread_id, resolved_user_id):
return
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=datetime.now(UTC)))
now = datetime.now(UTC)
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=now))
await update_task_deeplink_card(session, thread_id, status=status, updated_at=now)
await session.commit()
async def update_assistant_id(
@@ -365,7 +354,9 @@ class ThreadMetaRepository(ThreadMetaStore):
async with self._sf() as session:
if not await self._check_ownership(session, thread_id, resolved_user_id):
return
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=datetime.now(UTC)))
now = datetime.now(UTC)
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=now))
await update_task_deeplink_card(session, thread_id, updated_at=now)
await session.commit()
async def update_metadata(
@@ -390,14 +381,14 @@ class ThreadMetaRepository(ThreadMetaStore):
task_shared = bool((row.metadata_json or {}).get("taskId"))
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
return
merged = dict(row.metadata_json or {})
merged.update(metadata)
row.metadata_json = merged
for key, value in _indexed_thread_metadata(merged).items():
setattr(row, key, value)
row.updated_at = datetime.now(UTC)
merged = dict(row.metadata_json or {})
merged.update(metadata)
row.metadata_json = merged
for key, value in _indexed_thread_metadata(merged).items():
setattr(row, key, value)
row.updated_at = datetime.now(UTC)
await sync_task_deeplink_index(session, row)
await session.commit()
await session.commit()
async def delete(
self,
@@ -82,3 +82,11 @@ class ToolMetricsStore(ABC):
"which skills were invoked, and how often did each succeed/fail".
"""
raise NotImplementedError
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
"""Delete one bounded batch of audit rows older than ``cutoff``.
The default keeps alternate/in-memory implementations compatible; the
SQL repository supplies the real retention operation.
"""
return 0
@@ -8,7 +8,7 @@ from collections.abc import AsyncIterator
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import desc, func, select
from sqlalchemy import delete, desc, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
@@ -245,3 +245,34 @@ class SqlToolMetricsStore(ToolMetricsStore):
{"skill_name": name, **counts}
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
]
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
"""Delete one small, oldest-first retention batch.
Key selection followed by a primary-key delete works consistently on
every supported SQL dialect and prevents a three-month cleanup from
holding a large audit-table write lock.
"""
# Keep below SQLite's traditional 999 bind-variable limit. The
# retention worker requests batches repeatedly, so portability is more
# valuable than one giant delete statement.
size = max(1, min(int(batch_size), 500))
normalized_cutoff = _normalize_created_at(cutoff)
async with self._sf() as session:
ids = list(
(
await session.scalars(
select(ToolCallMetricRow.id)
.where(ToolCallMetricRow.created_at < normalized_cutoff)
.order_by(ToolCallMetricRow.created_at.asc(), ToolCallMetricRow.id.asc())
.limit(size)
)
).all()
)
if not ids:
return 0
result = await session.execute(
delete(ToolCallMetricRow).where(ToolCallMetricRow.id.in_(ids))
)
await session.commit()
return int(result.rowcount or 0)
@@ -1,23 +1,23 @@
"""ORM model for the users table.
Lives in the harness persistence package so it is picked up by
``Base.metadata.create_all()`` alongside ``threads_meta``, ``runs``,
``run_events``, and ``feedback``. Using the shared engine means:
- One SQLite/Postgres database, one connection pool
- One schema initialisation codepath
- Consistent async sessions across auth and persistence reads
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Index, String, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
"""ORM model for the users table.
Lives in the harness persistence package so it is picked up by
``Base.metadata.create_all()`` alongside ``threads_meta``, ``runs``,
``run_events``, and ``feedback``. Using the shared engine means:
- One SQLite/Postgres database, one connection pool
- One schema initialisation codepath
- Consistent async sessions across auth and persistence reads
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, Index, String, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class AuthInitializationRow(Base):
@@ -32,60 +32,73 @@ class AuthUsernameRow(Base):
__tablename__ = "auth_usernames"
username: Mapped[str] = mapped_column(String(128), primary_key=True)
class UserRow(Base):
__tablename__ = "users"
# UUIDs are stored as 36-char strings for cross-backend portability.
id: Mapped[str] = mapped_column(String(36), primary_key=True)
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
# when new roles are introduced.
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
)
# OAuth linkage (optional). A partial unique index enforces one
# account per (provider, oauth_id) pair, leaving NULL/NULL rows
# unconstrained so plain password accounts can coexist.
oauth_provider: Mapped[str | None] = mapped_column(String(32), nullable=True)
oauth_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# Auth lifecycle flags
needs_setup: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
token_version: Mapped[int] = mapped_column(nullable=False, default=0)
# 登录白名单(放行)标记 —— **黑名单模式**。仅当系统设置 ``login_whitelist.enabled``
# 打开时生效:关 → 忽略此列,人人可登录;开 → 仅 approved=False(被管理员显式
# 「取消放行」)的用户被拦,其余(含从未登录过的新账号)放行。新账号默认 True
# (首次登录即放行);存量用户由迁移回填 True。server_default=1 让迁移 ADD COLUMN
# 时把已存在行置为已放行。
approved: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
class AuthServiceNonceRow(Base):
"""One-time service exchange nonce shared by every gateway worker."""
__tablename__ = "auth_service_nonces"
nonce_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
client_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
class UserRow(Base):
__tablename__ = "users"
# UUIDs are stored as 36-char strings for cross-backend portability.
id: Mapped[str] = mapped_column(String(36), primary_key=True)
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
# Canonical local part used by SSO / username login. Keeping it separately
# avoids applying LOWER()/LIKE to every users.email row on each login.
username_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
# when new roles are introduced.
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
)
# OAuth linkage (optional). A partial unique index enforces one
# account per (provider, oauth_id) pair, leaving NULL/NULL rows
# unconstrained so plain password accounts can coexist.
oauth_provider: Mapped[str | None] = mapped_column(String(32), nullable=True)
oauth_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# Auth lifecycle flags
needs_setup: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
token_version: Mapped[int] = mapped_column(nullable=False, default=0)
# 登录白名单(放行)标记 —— **黑名单模式**。仅当系统设置 ``login_whitelist.enabled``
# 打开时生效:关 → 忽略此列,人人可登录;开 → 仅 approved=False(被管理员显式
# 「取消放行」)的用户被拦,其余(含从未登录过的新账号)放行。新账号默认 True
# (首次登录即放行);存量用户由迁移回填 True。server_default=1 让迁移 ADD COLUMN
# 时把已存在行置为已放行。
approved: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
# Unified password/passphrase. New writes store the same versioned hash as
# password_hash; historical plaintext remains accepted until the next reset.
# Never serialize hashes into user-list or login responses.
login_password: Mapped[str | None] = mapped_column(String(255), nullable=True)
# 岗位 (position) the user is assigned to (1:1). NULL = no explicit
# position → the user inherits the ``is_default`` position's grants. Drives
# the subscription sync engine (see ``deerflow.services.position_sync``).
position_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
__table_args__ = (
Index(
"idx_users_oauth_identity",
"oauth_provider",
"oauth_id",
unique=True,
sqlite_where=text("oauth_provider IS NOT NULL AND oauth_id IS NOT NULL"),
),
)
login_password: Mapped[str | None] = mapped_column(String(255), nullable=True)
# 岗位 (position) the user is assigned to (1:1). NULL = no explicit
# position → the user inherits the ``is_default`` position's grants. Drives
# the subscription sync engine (see ``deerflow.services.position_sync``).
position_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
__table_args__ = (
Index("ix_users_username_key_created_at", "username_key", "created_at"),
Index(
"idx_users_oauth_identity",
"oauth_provider",
"oauth_id",
unique=True,
sqlite_where=text("oauth_provider IS NOT NULL AND oauth_id IS NOT NULL"),
),
)