sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -77,6 +77,11 @@ class AgentStore(ABC):
"""Return featured + still-visible-to-``user_id`` agents, ordered by ``featured_order`` asc."""
raise NotImplementedError
@abstractmethod
async def list_sidebar(self, user_id: str, *, limit: int = 3) -> list[dict[str, Any]]:
"""Return only the id/name fields needed by the workspace sidebar."""
raise NotImplementedError
@abstractmethod
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
@@ -133,6 +138,7 @@ class AgentStore(ABC):
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
hidden: bool | None = None,
) -> tuple[list[dict[str, Any]], int]:
"""Return one page of agents plus the total matching count.
@@ -143,6 +149,9 @@ class AgentStore(ABC):
- ``None`` — the default listing visibility (``list_visible``).
``admin=True`` bypasses the visibility/scope filter entirely (every
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
Admin-hidden agents (``hidden=True`` rows) are excluded from every
non-admin listing; under ``admin=True`` the ``hidden`` param selects
only hidden / only visible rows (``None`` = both).
``search`` matches the raw name OR its desensitized display form.
``tag_ids`` keeps agents carrying ANY of the tags (OR). ``owner`` keeps
@@ -167,8 +176,20 @@ class AgentStore(ABC):
raise NotImplementedError
@abstractmethod
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``."""
async def set_extras(
self,
agent_id: str,
*,
tool_groups: list[str] | None,
model: str | None,
chat_link: Any = None,
update_chat_link: bool = False,
) -> None:
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``.
``chat_link`` is written only when ``update_chat_link`` is true, so a
skills/model refresh does not clear a previously mirrored jump link.
"""
raise NotImplementedError
@abstractmethod
@@ -1,325 +1,398 @@
"""In-memory custom agent store for memory persistence mode."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.config.agent_name_desensitize import name_matches
from deerflow.persistence.agents.base import AgentStore
def _now() -> str:
return datetime.now(UTC).isoformat()
def _copy(data: dict[str, Any]) -> dict[str, Any]:
copied = dict(data)
copied["builtin"] = copied.get("user_id") is None
return copied
class MemoryAgentStore(AgentStore):
def __init__(self) -> None:
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._pins: set[tuple[str, str]] = set()
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
self._skills: dict[str, list[str]] = {}
self._extras: dict[str, dict[str, Any]] = {}
@staticmethod
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
"""Single-record visibility (``get_visible``) — the union of every
listing surface, so a list-visible agent always opens. Mirrors
``AgentRepository._visible_filter``: baseline visibility OR
private-square (password is the discovery gate, reads are not
re-gated) OR favorited (the "我的" listing keeps favorites even after
the owner unpublishes)."""
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
rows = [_copy(agent) for agent in self._agents.values() if self._is_visible(agent, user_id)]
rows.sort(key=lambda agent: agent.get("created_at") or "")
return rows
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or not self._is_fetchable(agent, user_id):
return None
return _copy(agent)
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
return _copy(agent) if agent is not None else None
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or agent.get("user_id") != user_id:
return None
return _copy(agent)
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
now = _now()
row = {
"id": data["id"],
"name": data["name"],
"description": data.get("description") or "",
"user_id": data.get("user_id"),
"published": bool(data.get("published", False)),
"is_private": bool(data.get("is_private", False)),
"square_id": str(data.get("square_id") or ""),
"featured_order": None,
"created_at": now,
"updated_at": now,
}
self._agents[row["id"]] = row
return _copy(row)
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
row = self._agents.get(data["id"])
if row is None:
row = {
"id": data["id"],
"name": data["name"],
"description": data.get("description") or "",
"user_id": None,
"published": bool(data.get("published", True)),
"featured_order": None,
"created_at": _now(),
"updated_at": _now(),
}
self._agents[row["id"]] = row
elif row.get("user_id") is None:
row["name"] = data["name"]
row["description"] = data.get("description") or ""
row["published"] = bool(data.get("published", True))
row["updated_at"] = _now()
return _copy(row)
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete(self, agent_id: str, user_id: str) -> bool:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return False
self._agents.pop(agent_id, None)
return True
async def delete_builtin(self, agent_id: str) -> bool:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return False
self._agents.pop(agent_id, None)
return True
async def count_owned(self, user_id: str) -> int:
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
row["featured_order"] = order
row["updated_at"] = _now()
return _copy(row)
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
candidates = [a for a in self._agents.values() if a.get("featured_order") is not None and self._is_visible(a, user_id)]
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
return [_copy(a) for a in candidates]
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
rows.sort(key=lambda a: order_map.get(a["id"], 0))
return [_copy(a) for a in rows]
async def list_private(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private"))]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def list_all(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values()]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private", "square_id"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete_any(self, agent_id: str) -> bool:
if agent_id not in self._agents:
return False
self._agents.pop(agent_id, None)
return True
# --- Paginated listing ---
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
if admin:
return True
if scope == "square":
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
if scope == "mine":
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
favorited = (user_id, agent["id"]) in self._favorites
return owned or favorited
if scope == "private":
return bool(agent.get("is_private"))
if scope == "builtin":
# 内置智能体 = user_id IS NULL(仅管理员;路由层 admin 闸门)。
return agent.get("user_id") is None
return self._is_visible(agent, user_id)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
) -> tuple[list[dict[str, Any]], int]:
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
owner_key = owner.strip() if owner and owner.strip() else None
square_key = square_id.strip() if square_id and square_id.strip() else None
def _square_ok(a: dict[str, Any]) -> bool:
if square_key is None:
return True
sid = str(a.get("square_id") or "")
return sid == square_key or (square_is_default and sid == "")
matched = [
a
for a in self._agents.values()
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
and (owner_key is None or a.get("user_id") == owner_key)
and _square_ok(a)
]
if sort == "time_asc":
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
elif sort == "updated_desc":
matched.sort(
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
reverse=True,
)
elif sort == "updated_asc":
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
elif sort == "name_asc":
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
elif sort == "name_desc":
matched.sort(
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
reverse=True,
)
else:
matched.sort(
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
reverse=True,
)
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
total = len(matched)
start = max(page - 1, 0) * page_size
page_rows = matched[start : start + page_size]
return [_copy(a) for a in page_rows], total
# --- Listing cache (skills / tool_groups / model) ---
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
self._extras[agent_id] = {"tool_groups": tool_groups, "model": model}
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
out: dict[str, dict[str, Any]] = {}
for agent_id in agent_ids:
has_extras = agent_id in self._extras
has_skills = agent_id in self._skills
if not has_extras and not has_skills:
continue
extras = self._extras.get(agent_id, {})
out[agent_id] = {
"skills": list(self._skills.get(agent_id, [])),
"tool_groups": extras.get("tool_groups"),
"model": extras.get("model"),
}
return out
async def delete_extras_for_agent(self, agent_id: str) -> None:
self._skills.pop(agent_id, None)
self._extras.pop(agent_id, None)
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
self._favorites.add((user_id, agent_id))
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
if (user_id, agent_id) in self._favorites:
self._favorites.discard((user_id, agent_id))
return True
return False
async def list_favorite_ids(self, user_id: str) -> set[str]:
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
self._pins.add((user_id, agent_id))
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
key = (user_id, agent_id)
if key not in self._pins:
return False
self._pins.remove(key)
return True
async def list_pin_ids(self, user_id: str) -> set[str]:
return {agent_id for uid, agent_id in self._pins if uid == user_id}
async def delete_pins_for_agent(self, agent_id: str) -> None:
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
"""In-memory custom agent store for memory persistence mode."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.config.agent_name_desensitize import name_matches
from deerflow.persistence.agents.base import AgentStore
def _now() -> str:
return datetime.now(UTC).isoformat()
def _copy(data: dict[str, Any]) -> dict[str, Any]:
copied = dict(data)
copied["builtin"] = copied.get("user_id") is None
return copied
class MemoryAgentStore(AgentStore):
def __init__(self) -> None:
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._pins: set[tuple[str, str]] = set()
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
self._skills: dict[str, list[str]] = {}
self._extras: dict[str, dict[str, Any]] = {}
@staticmethod
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
"""Single-record visibility (``get_visible``) — the union of every
listing surface, so a list-visible agent always opens. Mirrors
``AgentRepository._visible_filter``: baseline visibility OR
private-square (password is the discovery gate, reads are not
re-gated) OR favorited (the "我的" listing keeps favorites even after
the owner unpublishes)."""
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
rows = [
_copy(agent)
for agent in self._agents.values()
if self._is_visible(agent, user_id) and not agent.get("hidden")
]
rows.sort(key=lambda agent: agent.get("created_at") or "")
return rows
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or not self._is_fetchable(agent, user_id):
return None
return _copy(agent)
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
return _copy(agent) if agent is not None else None
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or agent.get("user_id") != user_id:
return None
return _copy(agent)
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
now = _now()
row = {
"id": data["id"],
"name": data["name"],
"description": data.get("description") or "",
"user_id": data.get("user_id"),
"published": bool(data.get("published", False)),
"is_private": bool(data.get("is_private", False)),
"square_id": str(data.get("square_id") or ""),
"hidden": bool(data.get("hidden", False)),
"featured_order": None,
"created_at": now,
"updated_at": now,
}
self._agents[row["id"]] = row
return _copy(row)
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
row = self._agents.get(data["id"])
if row is None:
row = {
"id": data["id"],
"name": data["name"],
"description": data.get("description") or "",
"user_id": None,
"published": bool(data.get("published", True)),
"hidden": bool(data.get("hidden", False)),
"featured_order": None,
"created_at": _now(),
"updated_at": _now(),
}
self._agents[row["id"]] = row
elif row.get("user_id") is None:
row["name"] = data["name"]
row["description"] = data.get("description") or ""
row["published"] = bool(data.get("published", True))
row["updated_at"] = _now()
return _copy(row)
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return None
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return None
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete(self, agent_id: str, user_id: str) -> bool:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") != user_id:
return False
self._agents.pop(agent_id, None)
return True
async def delete_builtin(self, agent_id: str) -> bool:
row = self._agents.get(agent_id)
if row is None or row.get("user_id") is not None:
return False
self._agents.pop(agent_id, None)
return True
async def count_owned(self, user_id: str) -> int:
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
row["featured_order"] = order
row["updated_at"] = _now()
return _copy(row)
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
candidates = [
a
for a in self._agents.values()
if a.get("featured_order") is not None and self._is_visible(a, user_id) and not a.get("hidden")
]
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
return [_copy(a) for a in candidates]
async def list_sidebar(self, user_id: str, *, limit: int = 3) -> list[dict[str, Any]]:
mine = [
a for a in self._agents.values()
if not a.get("hidden")
and ((a.get("user_id") == user_id) or (user_id, a["id"]) in self._favorites)
]
square = [
a for a in self._agents.values()
if not a.get("hidden")
and (a.get("user_id") is None or a.get("published"))
and not a.get("is_private")
]
for rows in (mine, square):
rows.sort(
key=lambda a: (str(a.get("created_at") or ""), str(a.get("id") or "")),
reverse=True,
)
rows.sort(key=lambda a: (user_id, a["id"]) in self._pins, reverse=True)
out: list[dict[str, Any]] = []
seen: set[str] = set()
def add(agent: dict[str, Any]) -> None:
if len(out) >= limit or agent["id"] in seen:
return
seen.add(agent["id"])
out.append({"id": agent["id"], "name": agent.get("name") or ""})
for agent in [*mine, *square]:
if (user_id, agent["id"]) in self._pins:
add(agent)
for agent in mine:
add(agent)
for agent in square:
add(agent)
return out
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
rows.sort(key=lambda a: order_map.get(a["id"], 0))
return [_copy(a) for a in rows]
async def list_private(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private")) and not a.get("hidden")]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def list_all(self) -> list[dict[str, Any]]:
rows = [_copy(a) for a in self._agents.values()]
rows.sort(key=lambda a: a.get("created_at") or "")
return rows
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._agents.get(agent_id)
if row is None:
return None
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
if key in data:
row[key] = data[key]
row["updated_at"] = _now()
return _copy(row)
async def delete_any(self, agent_id: str) -> bool:
if agent_id not in self._agents:
return False
self._agents.pop(agent_id, None)
return True
# --- Paginated listing ---
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
if admin:
return True # every agent — the hidden filter is a separate param
if agent.get("hidden"):
return False # admin-hidden agents vanish from every non-admin scope
if scope == "square":
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
if scope == "mine":
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
favorited = (user_id, agent["id"]) in self._favorites
return owned or favorited
if scope == "private":
return bool(agent.get("is_private"))
if scope == "builtin":
# 内置智能体 = user_id IS NULL(仅管理员;路由层 admin 闸门)。
return agent.get("user_id") is None
return self._is_visible(agent, user_id)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
square_id: str | None = None,
square_is_default: bool = False,
hidden: bool | None = None,
) -> tuple[list[dict[str, Any]], int]:
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
owner_key = owner.strip() if owner and owner.strip() else None
square_key = square_id.strip() if square_id and square_id.strip() else None
def _square_ok(a: dict[str, Any]) -> bool:
if square_key is None:
return True
sid = str(a.get("square_id") or "")
return sid == square_key or (square_is_default and sid == "")
def _hidden_ok(a: dict[str, Any]) -> bool:
if admin:
# Admin console filter: keep only hidden / only visible agents.
return hidden is None or bool(a.get("hidden")) == hidden
return not a.get("hidden")
matched = [
a
for a in self._agents.values()
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
and _hidden_ok(a)
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
and (owner_key is None or a.get("user_id") == owner_key)
and _square_ok(a)
]
if sort == "time_asc":
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
elif sort == "updated_desc":
matched.sort(
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
reverse=True,
)
elif sort == "updated_asc":
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
elif sort == "name_asc":
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
elif sort == "name_desc":
matched.sort(
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
reverse=True,
)
else:
matched.sort(
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
reverse=True,
)
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
total = len(matched)
start = max(page - 1, 0) * page_size
page_rows = matched[start : start + page_size]
return [_copy(a) for a in page_rows], total
# --- Listing cache (skills / tool_groups / model) ---
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
async def set_extras(
self,
agent_id: str,
*,
tool_groups: list[str] | None,
model: str | None,
chat_link: Any = None,
update_chat_link: bool = False,
) -> None:
previous = self._extras.get(agent_id, {})
stored = {
"tool_groups": tool_groups,
"model": model,
"chat_link": previous.get("chat_link"),
}
if update_chat_link:
stored["chat_link"] = chat_link
self._extras[agent_id] = stored
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
out: dict[str, dict[str, Any]] = {}
for agent_id in agent_ids:
has_extras = agent_id in self._extras
has_skills = agent_id in self._skills
if not has_extras and not has_skills:
continue
extras = self._extras.get(agent_id, {})
out[agent_id] = {
"skills": list(self._skills.get(agent_id, [])),
"tool_groups": extras.get("tool_groups"),
"model": extras.get("model"),
"chat_link": extras.get("chat_link"),
}
return out
async def delete_extras_for_agent(self, agent_id: str) -> None:
self._skills.pop(agent_id, None)
self._extras.pop(agent_id, None)
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
self._favorites.add((user_id, agent_id))
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
if (user_id, agent_id) in self._favorites:
self._favorites.discard((user_id, agent_id))
return True
return False
async def list_favorite_ids(self, user_id: str) -> set[str]:
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
self._pins.add((user_id, agent_id))
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
key = (user_id, agent_id)
if key not in self._pins:
return False
self._pins.remove(key)
return True
async def list_pin_ids(self, user_id: str) -> set[str]:
return {agent_id for uid, agent_id in self._pins if uid == user_id}
async def delete_pins_for_agent(self, agent_id: str) -> None:
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -13,6 +13,13 @@ from deerflow.persistence.types import BeijingDateTime, PortableJSON
class AgentRow(Base):
__tablename__ = "agents"
# Composite indexes for the gallery tabs. Single-column indexes cannot
# serve "广场" (hidden + private + square) or "我的" (owner + hidden)
# without a wide scan once the catalog grows.
__table_args__ = (
Index("ix_agents_catalog_square", "hidden", "is_private", "square_id", "created_at"),
Index("ix_agents_owner_hidden", "user_id", "hidden", "created_at"),
)
# Stored as a plain string so the same id can be used as the filesystem
# directory name under {base_dir}/agents/{id}/.
@@ -27,6 +34,14 @@ class AgentRow(Base):
# visible to its owner in the "我的" tab, and visible to anyone in the
# private-square listing once the password has been verified.
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# When True the agent is hidden from every non-admin listing surface
# (gallery / 我的 / 广场 / homepage selector / private square). Only the
# admin agents console (?admin=true) still lists it. Purely a catalog
# switch: single-record access (detail / chat / run) keeps working, so
# existing conversations with a hidden agent are unaffected.
# server_default keeps the startup ORM-column sweep able to ADD COLUMN
# on existing deployments (NOT NULL needs a default).
hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"), index=True)
# Publish square (发布广场) this agent is filed under. A pure organizational
# category with no access control — independent of ``published`` /
# ``is_private``. Empty string = "not assigned" → the listing layer treats
@@ -124,4 +139,8 @@ class AgentExtrasRow(Base):
# list[str] | None — the agent's tool_groups whitelist from config.yaml.
tool_groups: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True, default=None)
model: Mapped[str | None] = mapped_column(String(191), nullable=True, default=None)
# Gallery 「对话」 jump target mirrored from config.yaml so the list endpoint
# can redirect without reading every agent file. Nullable JSON so existing
# databases pick the column up on the startup ORM sweep.
chat_link: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True, default=None)
skills_synced_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,23 @@
"""MySQL-persisted administrator switch for the optional API Redis cache."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, Boolean, Integer, String, text, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ApiReadCacheSettingRow(Base):
__tablename__ = "api_read_cache_settings"
key: Mapped[str] = mapped_column(String(32), primary_key=True, default="default")
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
ttl_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=300, server_default=text("300"))
collection_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
cache_epoch: Mapped[str] = mapped_column(String(32), nullable=False, default="0", server_default="0")
browser_status_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true())
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -1204,7 +1204,7 @@ class AssistantKnowledgeRepository:
stats[base_id][stat_name] = int(stat_value or 0)
return stats
async def get_base(self, base_id: str | None = None) -> dict[str, Any] | None:
async def get_base(self, base_id: str | None = None, *, include_stats: bool = True) -> dict[str, Any] | None:
async with self._sf() as session:
if base_id:
row = (
@@ -1215,7 +1215,9 @@ class AssistantKnowledgeRepository:
)
)
).scalar_one_or_none()
return await self._base_with_stats(session, row) if row else None
if row is None:
return None
return await self._base_with_stats(session, row) if include_stats else _dict(row)
row = (
await session.execute(
@@ -1240,15 +1242,17 @@ class AssistantKnowledgeRepository:
row.updated_at = _now()
await session.flush()
await session.commit()
return await self._base_with_stats(session, row) if row else None
if row is None:
return None
return await self._base_with_stats(session, row) if include_stats else _dict(row)
async def list_bases(self) -> list[dict[str, Any]]:
# Keep the library switcher a metadata-only read. Global alignment can
# scan and rewrite the complete Wiki/vector corpus after a process
# restart, so doing it here makes merely opening the assistant-library
# landing page scale with the size of the largest library. Content
# reads/searches still call ``_ensure_base_aligned`` before returning
# data, and imports align atomically before committing their snapshot.
# searches/exports still call ``_ensure_base_aligned`` for legacy data,
# and imports align atomically before committing their snapshot.
async with self._sf() as session:
rows = list(
(
@@ -2367,10 +2371,12 @@ class AssistantKnowledgeRepository:
include_chunks: bool = False,
include_content: bool = True,
) -> list[dict[str, Any]]:
base = await self.get_base(base_id)
base = await self.get_base(base_id, include_stats=False)
if base is None:
return []
await self._ensure_base_aligned(str(base["id"]))
# Browse the last committed snapshot. Imports already align it before
# commit; waiting for the global import/alignment locks here would make
# opening any library hang behind an unrelated long-running import.
stmt = select(AssistantKnowledgeWikiPageRow).where(
AssistantKnowledgeWikiPageRow.base_id == base["id"],
AssistantKnowledgeWikiPageRow.status == "active",
@@ -3020,10 +3026,9 @@ class AssistantKnowledgeRepository:
return results
async def get_page(self, slug: str, *, base_id: str | None = None) -> dict[str, Any] | None:
base = await self.get_base(base_id)
base = await self.get_base(base_id, include_stats=False)
if base is None:
return None
await self._ensure_base_aligned(str(base["id"]))
stmt = select(AssistantKnowledgeWikiPageRow).where(
AssistantKnowledgeWikiPageRow.base_id == base["id"],
AssistantKnowledgeWikiPageRow.slug == slug.strip("/"),
@@ -3122,7 +3127,7 @@ class AssistantKnowledgeRepository:
return data
async def list_sources(self, *, base_id: str | None = None) -> list[dict[str, Any]]:
base = await self.get_base(base_id)
base = await self.get_base(base_id, include_stats=False)
if base is None:
return []
async with self._sf() as session:
@@ -3162,7 +3167,7 @@ class AssistantKnowledgeRepository:
return result
async def list_jobs(self, limit: int = 100, *, base_id: str | None = None) -> list[dict[str, Any]]:
base = await self.get_base(base_id)
base = await self.get_base(base_id, include_stats=False)
if base is None:
return []
async with self._sf() as session:
@@ -4,7 +4,7 @@ A "business mapping" row binds a fixed business code (3Q / 6BF / 3q-0…3q-6 /
6-1…6-4 / 7BF / 7-1…7-4 / 8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
account can resolve it). Used to auto-pick the business chain when a task page
launches the multi-agent roundtable.
launches the multi-agent roundtable. ``3q-1…3q-6`` are seeded as custom-type rows.
Unlike ``roundtable_chains`` (per-user), this mapping is **global / shared** —
there is no ``user_id`` column; all accounts see and use the same configuration.
@@ -20,7 +20,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String
from sqlalchemy import Boolean, Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -34,6 +34,16 @@ class BusinessMappingRow(Base):
business_code: Mapped[str] = mapped_column(String(32), primary_key=True)
# Display label shown in the 业务 column (e.g. "任务FX").
label: Mapped[str] = mapped_column(String(128), nullable=False, default="")
# A user-visible category. Built-in rows keep ``内置业务``; rows created from
# the configuration page consistently use ``自定义类型``.
mapping_type: Mapped[str] = mapped_column(
String(64), nullable=False, default="内置业务", server_default="内置业务"
)
# Built-in rows are seeded and cannot be redefined. Custom rows are created
# by an administrator and their business_code doubles as a login goPath.
is_custom: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default="0"
)
# Picked 智能体 (暂存不用,自动流程当前不消费此字段). NULL = 未配置.
agent_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
agent_name: Mapped[str | None] = mapped_column(String(191), nullable=True)
@@ -11,7 +11,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.business_mapping.model import BusinessMappingRow
@@ -23,6 +23,8 @@ _UNSET = object()
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
# b5-1 = 外部写入本系统详情后,由本系统读取的任务分析单智能体深链。
# 3q-0…3q-6 / 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
# 3q-1…3q-6 默认按「自定义类型」落库(goPath 走自定义深链);配置管理页默认不展示这 6 行。
_CUSTOM_SLOT = {"is_custom": True, "mapping_type": "自定义类型"}
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
{"business_code": "3Q", "label": "3QFX"},
{"business_code": "6BF", "label": "任务FX"},
@@ -32,12 +34,12 @@ DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
{"business_code": "b5-1", "label": "任务分析"},
# 独立配置位,与上方业务码无关;固定排在表尾。
{"business_code": "3q-0", "label": "3q-0"},
{"business_code": "3q-1", "label": "3q-1"},
{"business_code": "3q-2", "label": "3q-2"},
{"business_code": "3q-3", "label": "3q-3"},
{"business_code": "3q-4", "label": "3q-4"},
{"business_code": "3q-5", "label": "3q-5"},
{"business_code": "3q-6", "label": "3q-6"},
{"business_code": "3q-1", "label": "3q-1", **_CUSTOM_SLOT},
{"business_code": "3q-2", "label": "3q-2", **_CUSTOM_SLOT},
{"business_code": "3q-3", "label": "3q-3", **_CUSTOM_SLOT},
{"business_code": "3q-4", "label": "3q-4", **_CUSTOM_SLOT},
{"business_code": "3q-5", "label": "3q-5", **_CUSTOM_SLOT},
{"business_code": "3q-6", "label": "3q-6", **_CUSTOM_SLOT},
{"business_code": "6-1", "label": "6-1"},
{"business_code": "6-2", "label": "6-2"},
{"business_code": "6-3", "label": "6-3"},
@@ -49,6 +51,14 @@ DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
]
def _seed_type(seed: dict[str, Any]) -> tuple[bool, str]:
is_custom = bool(seed.get("is_custom", False))
mapping_type = str(seed.get("mapping_type") or "").strip()
if not mapping_type:
mapping_type = "自定义类型" if is_custom else "内置业务"
return is_custom, mapping_type
class BusinessMappingRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -58,6 +68,8 @@ class BusinessMappingRepository:
return {
"business_code": row.business_code,
"label": row.label,
"mapping_type": row.mapping_type or "内置业务",
"is_custom": bool(row.is_custom),
"agent_id": row.agent_id,
"agent_name": row.agent_name,
"chain_id": row.chain_id,
@@ -110,18 +122,62 @@ class BusinessMappingRepository:
await session.refresh(row)
return self._to_dict(row)
async def create_custom_mapping(
self,
*,
business_code: str,
mapping_type: str,
) -> dict[str, Any] | None:
"""Create one administrator-defined mapping.
``business_code`` is deliberately the externally supplied goPath. The
case-insensitive uniqueness check mirrors frontend deep-link matching,
preventing two rows that could resolve from the same login URL.
"""
normalized_code = business_code.strip()
normalized_type = mapping_type.strip()
async with self._sf() as session:
existing = await session.scalar(
select(BusinessMappingRow).where(
func.lower(BusinessMappingRow.business_code) == normalized_code.lower()
)
)
if existing is not None:
return None
max_sort_order = await session.scalar(select(func.max(BusinessMappingRow.sort_order)))
now = datetime.now(UTC)
row = BusinessMappingRow(
business_code=normalized_code,
label=normalized_code,
mapping_type=normalized_type,
is_custom=True,
agent_id=None,
agent_name=None,
chain_id=None,
chain_title=None,
sort_order=(int(max_sort_order) if max_sort_order is not None else -1) + 1,
created_at=now,
updated_at=now,
)
session.add(row)
await session.commit()
await session.refresh(row)
return self._to_dict(row)
async def ensure_seed_defaults(self) -> int:
"""Idempotently insert the fixed business rows if missing. Returns inserted count.
Never overwrites an existing row's agent/chain (respects user config).
Re-syncs ``sort_order`` (and empty ``label``) so newly inserted codes keep
the intended table order next to older seeded rows.
Re-syncs ``sort_order``, empty ``label``, and seed ``is_custom`` /
``mapping_type`` so newly inserted codes keep the intended table order
and 3q-1…3q-6 stay classified as custom type.
"""
inserted = 0
dirty = False
now = datetime.now(UTC)
async with self._sf() as session:
for idx, seed in enumerate(DEFAULT_BUSINESS_ROWS):
is_custom, mapping_type = _seed_type(seed)
existing = await session.get(BusinessMappingRow, seed["business_code"])
if existing is not None:
if existing.sort_order != idx:
@@ -131,11 +187,19 @@ class BusinessMappingRepository:
if seed_label and not (existing.label or "").strip():
existing.label = seed_label
dirty = True
if bool(existing.is_custom) != is_custom:
existing.is_custom = is_custom
dirty = True
if (existing.mapping_type or "") != mapping_type:
existing.mapping_type = mapping_type
dirty = True
continue
session.add(
BusinessMappingRow(
business_code=seed["business_code"],
label=seed.get("label", ""),
mapping_type=mapping_type,
is_custom=is_custom,
agent_id=None,
agent_name=None,
chain_id=None,
@@ -22,6 +22,7 @@ def _normalize(data: dict[str, Any], menu_type: str, updated_by: str | None) ->
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
"sort_order": int(data.get("sort_order", 0)),
"label": (data.get("label") or None),
"disabled": bool(data.get("disabled", False)),
"updated_by": updated_by,
"updated_at": datetime.now(UTC).isoformat(),
}
@@ -1,14 +1,15 @@
"""ORM model for compact/sidebar-summary menu overrides.
This table only controls the compact left menu presentation. The full sidebar
menu layout continues to use ``menu_overrides`` and is unaffected.
This table only controls compact menu presentation: lane, order, label and
enablement for the left menu, settings menu and toolbox. The full sidebar menu
layout continues to use ``menu_overrides`` and is unaffected.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, Integer, String
from sqlalchemy import Boolean, Index, Integer, String, false
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -24,6 +25,7 @@ class CompactMenuOverrideRow(Base):
placement: Mapped[str] = mapped_column(String(32), default="sidebar")
sort_order: Mapped[int] = mapped_column(Integer, default=0)
label: Mapped[str | None] = mapped_column(String(255))
disabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
updated_by: Mapped[str | None] = mapped_column(String(64))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
@@ -41,6 +41,7 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C
placement=placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
sort_order=int(data.get("sort_order", 0)),
label=(data.get("label") or None),
disabled=bool(data.get("disabled", False)),
updated_by=updated_by,
)
@@ -53,6 +54,7 @@ def _normalize_user_row(data: dict[str, Any], menu_type: str) -> dict[str, Any]:
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
"sort_order": int(data.get("sort_order", 0)),
"label": (data.get("label") or None),
"disabled": bool(data.get("disabled", False)),
}
@@ -11,14 +11,20 @@ None and fall back to in-memory implementations.
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import os
import re
import sys
import time
import weakref
from contextvars import ContextVar, Token
from importlib.util import find_spec
from pathlib import Path
from typing import Any
from sqlalchemy import text
from sqlalchemy import event, text
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
@@ -38,21 +44,167 @@ _session_factory: async_sessionmaker[AsyncSession] | None = None
# "got Future attached to a different loop". We capture the engine's home loop here so
# best-effort writes coming from worker threads / other loops can be routed back to it
# (see ``fire_and_forget_db_write``).
_engine_loop: asyncio.AbstractEventLoop | None = None
# These are the complete workflow-studio persistence tables owned by DeerFlow.
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
# wildcard: another application might share a schema and have its own table
# with that prefix.
_WORKFLOW_SCHEMA_TABLE_NAMES = (
"workflow_definitions",
"workflow_versions",
"workflow_runs",
"workflow_node_runs",
"workflow_run_events",
"workflow_run_artifacts",
"workflow_data_sources",
)
_engine_loop: asyncio.AbstractEventLoop | None = None
_performance_request_context: ContextVar[tuple[str, Any] | None] = ContextVar("deerflow_performance_request", default=None)
def bind_performance_request(request_id: str) -> Token:
"""Bind a request ID to this task so slow SQL logs can be correlated safely."""
try:
task = asyncio.current_task()
except RuntimeError:
task = None
return _performance_request_context.set((request_id, weakref.ref(task) if task is not None else None))
def reset_performance_request(token: Token) -> None:
_performance_request_context.reset(token)
def _current_performance_request_id() -> str:
context = _performance_request_context.get()
if context is None:
return "-"
try:
task = asyncio.current_task()
except RuntimeError:
task = None
return context[0] if task is not None and context[1] is not None and context[1]() is task else "-"
def _slow_sql_threshold_ms() -> float:
try:
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_SQL_LOG_MS", "500")), 60_000.0))
except (TypeError, ValueError):
return 500.0
def _sql_operation(statement: str) -> str:
parts = statement.lstrip().split(None, 1)
candidate = parts[0].upper() if parts else "EMPTY"
return candidate if re.fullmatch(r"[A-Z_]+", candidate) else "OTHER"
def _sql_table(statement: str) -> str:
"""Return the target/source table only, without query text or values."""
match = re.search(
r"\b(?:INSERT\s+INTO|UPDATE|DELETE\s+FROM)\s+([`\"\w.]+)",
statement,
re.IGNORECASE,
)
if match is None:
match = re.search(r"\bFROM\s+([`\"\w.]+)", statement, re.IGNORECASE)
if match is None:
return "unknown"
return match.group(1).strip("`\".")[-128:] or "unknown"
def _sql_fingerprint(statement: str) -> str:
return hashlib.sha256(statement.encode("utf-8", "replace")).hexdigest()[:12]
def _database_error_metadata(exception: BaseException | None) -> tuple[str, str, str]:
"""Extract safe driver diagnostics without logging SQL or bound values."""
if exception is None:
return "unknown", "-", "-"
code = getattr(exception, "errno", None) or getattr(exception, "code", None)
args = getattr(exception, "args", ())
if code is None and args and isinstance(args[0], (int, str)):
candidate = str(args[0])
if candidate.isdigit():
code = candidate
sqlstate = getattr(exception, "sqlstate", None) or getattr(exception, "pgcode", None)
diagnostic = getattr(exception, "diag", None)
constraint = getattr(diagnostic, "constraint_name", None)
def _safe_identifier(value: Any) -> str:
text_value = str(value or "")
return text_value[:128] if re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", text_value) else "-"
return _safe_identifier(code), _safe_identifier(sqlstate), _safe_identifier(constraint)
def _install_slow_sql_logging(engine: AsyncEngine, backend: str) -> None:
"""Log slow SQL timings without logging query text or bound values."""
sync_engine = engine.sync_engine
def before_cursor_execute(_conn, _cursor, statement, _parameters, context, _executemany):
if context is not None:
context._deerflow_perf_started = time.perf_counter()
context._deerflow_perf_operation = _sql_operation(statement)
context._deerflow_perf_statement = statement
def after_cursor_execute(_conn, _cursor, _statement, _parameters, context, executemany):
if context is None:
return
started = getattr(context, "_deerflow_perf_started", None)
if started is None:
return
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
if elapsed_ms >= _slow_sql_threshold_ms():
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", ""))
logger.warning(
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%.2f executemany=%s fingerprint=%s outcome=ok",
_current_performance_request_id(),
backend,
getattr(context, "_deerflow_perf_operation", "OTHER"),
_sql_table(getattr(context, "_deerflow_perf_statement", "")),
elapsed_ms,
bool(executemany),
fingerprint,
)
def handle_error(exception_context):
context = exception_context.execution_context
started = getattr(context, "_deerflow_perf_started", None) if context is not None else None
elapsed_ms = round((time.perf_counter() - started) * 1000, 2) if started is not None else None
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
original_exception = exception_context.original_exception
error_code, sqlstate, constraint = _database_error_metadata(original_exception)
table = _sql_table(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
duplicate_key = error_code == "1062" or sqlstate == "23505" or (
table == "scheduled_task_runs"
and "UNIQUE constraint failed" in str(original_exception or "")
)
log = logger.info if table == "scheduled_task_runs" and duplicate_key else logger.error
outcome = "duplicate_ignored" if table == "scheduled_task_runs" and duplicate_key else "error"
log(
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%s executemany=%s fingerprint=%s outcome=%s error_type=%s error_code=%s sqlstate=%s constraint=%s",
_current_performance_request_id(),
backend,
getattr(context, "_deerflow_perf_operation", "OTHER") if context is not None else "OTHER",
table,
elapsed_ms,
bool(getattr(context, "executemany", False)) if context is not None else False,
fingerprint,
outcome,
type(original_exception).__name__ if original_exception is not None else "unknown",
error_code,
sqlstate,
constraint,
)
event.listen(sync_engine, "before_cursor_execute", before_cursor_execute)
event.listen(sync_engine, "after_cursor_execute", after_cursor_execute)
event.listen(sync_engine, "handle_error", handle_error)
# These are the complete workflow-studio persistence tables owned by DeerFlow.
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
# wildcard: another application might share a schema and have its own table
# with that prefix.
_WORKFLOW_SCHEMA_TABLE_NAMES = (
"workflow_definitions",
"workflow_versions",
"workflow_runs",
"workflow_node_runs",
"workflow_run_events",
"workflow_run_artifacts",
"workflow_data_sources",
)
def _add_runtime_vendor_path() -> None:
@@ -81,6 +233,42 @@ def _mysql_driver_name(url: str) -> str:
return "asyncmy"
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
kwargs = dict(
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
# These are not MySQL/psycopg connection arguments.
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
kwargs["connect_args"] = {"command_timeout": command_timeout}
return kwargs
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
kwargs = dict(
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
# These are not MySQL/psycopg connection arguments.
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
kwargs["connect_args"] = {"command_timeout": command_timeout}
return kwargs
async def _run_alembic_upgrade(url: str) -> None:
"""Apply pending Alembic migrations against ``url`` (``alembic upgrade head``).
@@ -176,7 +364,7 @@ async def _auto_create_postgres_db(url: str) -> None:
try:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
finally:
await maint_engine.dispose()
@@ -215,7 +403,7 @@ async def _auto_create_mysql_db(url: str) -> None:
await maint_engine.dispose()
def _ensure_orm_columns_sync(sync_conn) -> None:
def _ensure_orm_columns_sync(sync_conn) -> None:
"""Add ORM-declared columns that are missing from already-existing tables.
``Base.metadata.create_all`` creates brand-new tables but never patches
@@ -226,14 +414,14 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
This sweep compares each existing table against the ORM metadata and issues
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
safely (nullable, or carrying a server_default). Missing ORM indexes are
created as well. Table-level ``UniqueConstraint`` objects
safely (nullable, or carrying a server_default). Missing ORM indexes are
created as well. Table-level ``UniqueConstraint`` objects
that are declared on the model but missing from the table are also created
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
on an existing database, weakening dedupe.
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
``BeijingDateTime``) resolve to the right backend type. Everything is
per-column best-effort: a failure is logged and startup continues.
@@ -258,10 +446,8 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
continue
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
missing = [c for c in table.columns if c.name not in existing_cols]
if not missing:
continue
for col in missing:
for col in missing:
if not col.nullable and col.server_default is None:
logger.warning(
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
@@ -270,21 +456,21 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
)
continue
try:
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
logger.info("Auto-added missing column %s.%s", table.name, col.name)
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
logger.info("Auto-added missing column %s.%s", table.name, col.name)
except Exception:
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if idx.name in existing_idx:
continue
try:
idx.create(sync_conn)
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if not idx.name or idx.name in existing_idx:
continue
try:
idx.create(sync_conn)
logger.info("Auto-created index %s on %s", idx.name, table.name)
except Exception:
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
@@ -328,71 +514,111 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
f"ON {preparer.quote(table.name)} ({cols})"
)
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
Alembic auto-upgrade is intentionally disabled in deployed Gateway
processes, while the ORM column synchronizer is always run. The latter
can add the task lookup columns and indexes but cannot derive values from
JSON metadata. Backfill task rows once at startup so an upgrade never
temporarily hides every historical task conversation from ``/threads/search``.
"""
from sqlalchemy import inspect as sa_inspect
table_name = "threads_meta"
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
try:
inspector = sa_inspect(sync_conn)
if table_name not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns(table_name)}
if not required_columns <= columns:
return
dialect = sync_conn.dialect.name
if dialect == "sqlite":
task_id = "json_extract(metadata_json, '$.taskId')"
agent_id = "json_extract(metadata_json, '$.agent_id')"
action_id = "json_extract(metadata_json, '$.actionId')"
valid_json = "json_valid(metadata_json)"
elif dialect == "mysql":
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
valid_json = "JSON_VALID(metadata_json)"
elif dialect in {"postgresql", "opengauss"}:
task_id = "metadata_json ->> 'taskId'"
agent_id = "metadata_json ->> 'agent_id'"
action_id = "metadata_json ->> 'actionId'"
valid_json = "TRUE"
else:
return
result = sync_conn.execute(
text(
"UPDATE threads_meta "
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
)
)
if result.rowcount:
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
except Exception:
# A malformed legacy JSON row or a backend-specific JSON function must
# not prevent startup. The generic metadata search remains available,
# and the failure is visible to the operator for remediation.
logger.exception("Failed to backfill thread task lookup projections")
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
Alembic auto-upgrade is intentionally disabled in deployed Gateway
processes, while the ORM column synchronizer is always run. The latter
can add the task lookup columns and indexes but cannot derive values from
JSON metadata. Backfill task rows once at startup so an upgrade never
temporarily hides every historical task conversation from ``/threads/search``.
"""
from sqlalchemy import inspect as sa_inspect
table_name = "threads_meta"
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
try:
inspector = sa_inspect(sync_conn)
if table_name not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns(table_name)}
if not required_columns <= columns:
return
dialect = sync_conn.dialect.name
if dialect == "sqlite":
task_id = "json_extract(metadata_json, '$.taskId')"
agent_id = "json_extract(metadata_json, '$.agent_id')"
action_id = "json_extract(metadata_json, '$.actionId')"
valid_json = "json_valid(metadata_json)"
elif dialect == "mysql":
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
valid_json = "JSON_VALID(metadata_json)"
elif dialect in {"postgresql", "opengauss"}:
task_id = "metadata_json ->> 'taskId'"
agent_id = "metadata_json ->> 'agent_id'"
action_id = "metadata_json ->> 'actionId'"
valid_json = "TRUE"
else:
return
result = sync_conn.execute(
text(
"UPDATE threads_meta "
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
)
)
if result.rowcount:
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
except Exception:
# A malformed legacy JSON row or a backend-specific JSON function must
# not prevent startup. The generic metadata search remains available,
# and the failure is visible to the operator for remediation.
logger.exception("Failed to backfill thread task lookup projections")
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
def _acquire_gauss_mysql_schema_bootstrap_lock():
"""Serialize Dolphin schema bootstrap across workers on this host."""
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
if not is_gauss_mysql_product():
return None
try:
import fcntl
except ImportError:
# Production offline deployments are Linux. Keeping a no-lock fallback
# makes imports and unit tests work on Windows without changing MySQL.
logger.warning("fcntl is unavailable; openGauss schema bootstrap is not cross-process locked")
return None
try:
from deerflow.config.runtime_paths import runtime_home
home = runtime_home()
except Exception:
home = Path(os.getenv("DEER_FLOW_HOME", ".deer-flow")).resolve()
lock_path = home / "data" / "gauss-schema-bootstrap.lock"
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = lock_path.open("a+", encoding="utf-8")
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
logger.info("Acquired openGauss schema bootstrap lock: %s", lock_path)
return handle
def _release_gauss_mysql_schema_bootstrap_lock(handle) -> None: # noqa: ANN001
if handle is None:
return
import fcntl
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
finally:
handle.close()
async def init_engine(
backend: str,
*,
@@ -401,28 +627,40 @@ async def init_engine(
pool_size: int = 5,
max_overflow: int = 10,
pool_timeout: int = 30,
pool_recycle: int = 1800,
pool_recycle: int | None = None,
command_timeout: float | None = 30,
sqlite_dir: str = "",
) -> None:
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
echo: Echo SQL to log.
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
command_timeout: asyncpg command deadline, including pool pre-ping (None disables).
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
global _engine, _session_factory, _engine_loop
if pool_recycle is None:
pool_recycle = 300 if backend == "postgres" else 1800
if pool_recycle is None:
pool_recycle = 300 if backend == "postgres" else 1800
if backend == "memory":
logger.info("Persistence backend=memory -- ORM engine not initialized")
return
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
gauss_mysql = backend == "mysql" and is_gauss_mysql_product()
# Remember the loop that owns the engine's connection pool, so off-loop
# best-effort writes (metrics from worker threads, etc.) can be routed here
# instead of reusing pooled connections cross-loop.
@@ -431,25 +669,25 @@ async def init_engine(
except RuntimeError:
_engine_loop = None
if backend in {"postgres", "gauss"}:
if backend in {"postgres", "gauss"}:
try:
import asyncpg # noqa: F401
except ImportError:
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "gauss":
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
install_gauss_dialect_compatibility()
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "gauss":
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
install_gauss_dialect_compatibility()
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
@@ -459,12 +697,12 @@ async def init_engine(
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
)
if gauss_mysql:
from deerflow.persistence.gauss_mysql_compat import install_gauss_mysql_dialect_compatibility
install_gauss_mysql_dialect_compatibility(driver)
if backend == "sqlite":
import os
from sqlalchemy import event
os.makedirs(sqlite_dir or ".", exist_ok=True)
_engine = create_async_engine(url, echo=echo, json_serializer=_json_serializer)
@@ -489,31 +727,28 @@ async def init_engine(
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend in {"postgres", "gauss"}:
elif backend in {"postgres", "gauss"}:
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
elif backend == "mysql":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
else:
raise ValueError(f"Unknown persistence backend: {backend!r}")
_install_slow_sql_logging(_engine, backend)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
# Auto-create tables for fresh deployments, then let Alembic apply any
@@ -530,16 +765,32 @@ async def init_engine(
# This is expected during initial scaffolding or minimal installs.
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
async def _create_all_with_product_integrity(conn) -> None: # noqa: ANN001
await conn.run_sync(Base.metadata.create_all)
if gauss_mysql:
from deerflow.persistence.gauss_schema import ensure_gauss_schema_sync
await conn.run_sync(ensure_gauss_schema_sync, Base.metadata)
# Schema bootstrap is deliberately best-effort. A single incompatible or
# permission-restricted table must not make the whole Gateway unavailable:
# routes backed by healthy existing tables can still work, while a route
# whose table is missing will report its own request-time error. Keep the
# auto-create-database retries, but contain failures from both the initial
# create_all and the retry path.
schema_bootstrap_lock = None
if gauss_mysql:
try:
schema_bootstrap_lock = _acquire_gauss_mysql_schema_bootstrap_lock()
except Exception:
# A lock-file permission problem is operationally important, but
# the global startup invariant still requires the Gateway to come
# up and report schema errors at the affected routes.
logger.exception("Could not acquire openGauss schema bootstrap lock; continuing without the host lock")
try:
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
await _create_all_with_product_integrity(conn)
except Exception as exc:
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
@@ -548,35 +799,33 @@ async def init_engine(
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
_install_slow_sql_logging(_engine, backend)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
await _create_all_with_product_integrity(conn)
elif backend == "mysql" and not gauss_mysql and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
_install_slow_sql_logging(_engine, backend)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
await _create_all_with_product_integrity(conn)
else:
raise
except Exception:
@@ -584,22 +833,34 @@ async def init_engine(
"Automatic ORM schema creation failed; continuing Gateway startup "
"with the existing database schema"
)
finally:
try:
_release_gauss_mysql_schema_bootstrap_lock(schema_bootstrap_lock)
except Exception:
logger.exception("Could not release openGauss schema bootstrap lock")
# Patch existing tables with ORM-declared columns that ``create_all``
# cannot add (it only creates brand-new tables). Best-effort: a failure is
# logged and startup continues on the current schema.
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
await conn.run_sync(_backfill_thread_meta_task_projections_sync)
except Exception:
logger.exception("Auto-updating ORM columns or task lookup projections failed; continuing with current schema")
if gauss_mysql:
# Dolphin returns PostgreSQL-style DDL from SHOW CREATE TABLE. The
# stock MySQL inspector misreads it and may issue duplicate ALTERs,
# aborting the transaction with error 322. The native-catalog helper
# above is the only schema repair path for this product.
logger.info("Skipping MySQL reflection and JSON backfill for openGauss Dolphin")
else:
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
await conn.run_sync(_backfill_thread_meta_task_projections_sync)
except Exception:
logger.exception("Auto-updating ORM columns or task lookup projections failed; continuing with current schema")
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
async def init_engine_from_config(config) -> None:
@@ -614,14 +875,17 @@ async def init_engine_from_config(config) -> None:
pool_size=config.pool_size,
max_overflow=getattr(config, "max_overflow", 10),
pool_timeout=getattr(config, "pool_timeout", 30),
pool_recycle=getattr(config, "pool_recycle", 1800),
pool_recycle=getattr(config, "pool_recycle", None),
command_timeout=getattr(config, "command_timeout", 30),
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
)
def get_session_factory() -> async_sessionmaker[AsyncSession] | None:
"""Return the async session factory, or None if backend=memory."""
return _session_factory
from deerflow.persistence.loop_isolation import isolated_session_factory
return isolated_session_factory() or _session_factory
def get_engine() -> AsyncEngine | None:
@@ -629,73 +893,73 @@ def get_engine() -> AsyncEngine | None:
return _engine
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
"""Return the event loop that owns the engine's connection pool (or None)."""
return _engine_loop
async def rebuild_workflow_schema_for_mysql() -> list[str]:
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
This is a deliberately narrow recovery operation for an unused Workflow
Studio installation. It never uses a prefix wildcard, never changes the
selected database, and refuses to run unless the engine is connected to
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
Returns the allow-listed workflow tables that existed before the rebuild.
"""
engine = get_engine()
if engine is None:
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
if engine.dialect.name != "mysql":
raise RuntimeError(
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
)
expected_database = engine.url.database
if not expected_database:
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
# Make the operation safe when this helper is invoked independently from
# normal app startup as well.
import deerflow.persistence.models # noqa: F401 — register ORM models
from deerflow.persistence.base import Base
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
if missing_models:
raise RuntimeError(
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
)
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
def _rebuild(sync_conn) -> list[str]:
from sqlalchemy import inspect as sa_inspect
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
if current_database != expected_database:
raise RuntimeError(
"Refusing workflow schema rebuild: connected database "
f"{current_database!r} does not match configured database {expected_database!r}"
)
existing = set(sa_inspect(sync_conn).get_table_names())
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
# SQLAlchemy orders parent/child tables around the foreign key from
# workflow_versions to workflow_definitions. Only tables in the exact
# allow-list above are ever passed to drop_all/create_all.
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
return present
async with engine.begin() as conn:
rebuilt = await conn.run_sync(_rebuild)
logger.warning(
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
expected_database,
", ".join(rebuilt) if rebuilt else "none (created fresh)",
)
return rebuilt
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
"""Return the event loop that owns the engine's connection pool (or None)."""
return _engine_loop
async def rebuild_workflow_schema_for_mysql() -> list[str]:
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
This is a deliberately narrow recovery operation for an unused Workflow
Studio installation. It never uses a prefix wildcard, never changes the
selected database, and refuses to run unless the engine is connected to
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
Returns the allow-listed workflow tables that existed before the rebuild.
"""
engine = get_engine()
if engine is None:
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
if engine.dialect.name != "mysql":
raise RuntimeError(
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
)
expected_database = engine.url.database
if not expected_database:
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
# Make the operation safe when this helper is invoked independently from
# normal app startup as well.
import deerflow.persistence.models # noqa: F401 — register ORM models
from deerflow.persistence.base import Base
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
if missing_models:
raise RuntimeError(
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
)
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
def _rebuild(sync_conn) -> list[str]:
from sqlalchemy import inspect as sa_inspect
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
if current_database != expected_database:
raise RuntimeError(
"Refusing workflow schema rebuild: connected database "
f"{current_database!r} does not match configured database {expected_database!r}"
)
existing = set(sa_inspect(sync_conn).get_table_names())
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
# SQLAlchemy orders parent/child tables around the foreign key from
# workflow_versions to workflow_definitions. Only tables in the exact
# allow-list above are ever passed to drop_all/create_all.
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
return present
async with engine.begin() as conn:
rebuilt = await conn.run_sync(_rebuild)
logger.warning(
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
expected_database,
", ".join(rebuilt) if rebuilt else "none (created fresh)",
)
return rebuilt
def fire_and_forget_db_write(coro) -> None:
@@ -744,10 +1008,13 @@ def _run_coro_on_throwaway_loop(coro):
Only reached when there is no usable engine home loop, or when the caller is
*synchronously* inside the home loop (routing back would deadlock). This
touches the shared connection pool from a throwaway loop, so it can still leak
a dead-loop-bound connection — but the common worker-thread path no longer
gets here (see ``run_db_blocking``).
uses an isolated, non-pooled engine for compatibility reads. The coroutine
must obtain its session factory inside its async body so the isolated
context applies; no connection is returned to the Gateway pool.
"""
from deerflow.persistence.loop_isolation import run_isolated_db_read
coro = run_isolated_db_read(coro)
try:
asyncio.get_running_loop()
in_loop = True
@@ -0,0 +1,218 @@
"""Compatibility helpers for openGauss/Dolphin's MySQL protocol.
The Dolphin endpoint deliberately presents itself as MySQL, but a few pieces
of server metadata and DDL reflection differ from real MySQL. These patches
are installed only when the deployment explicitly selects the ``gauss_mysql``
database product. A normal MySQL deployment never imports or installs them.
"""
from __future__ import annotations
import ast
import json
import logging
import math
import os
from typing import Any
logger = logging.getLogger(__name__)
GAUSS_MYSQL_PRODUCT = "gauss_mysql"
GAUSS_MYSQL_SQLALCHEMY_VERSION = (5, 7, 20)
_DOLPHIN_INIT_COMMAND = "SET standard_conforming_strings TO off"
def is_gauss_mysql_product() -> bool:
"""Return whether this process was started for the Dolphin endpoint."""
return os.getenv("DEERFLOW_DATABASE_PRODUCT", "").strip().lower() == GAUSS_MYSQL_PRODUCT
def is_gauss_mysql_dialect(dialect: object) -> bool:
"""Return whether the selected SQLAlchemy dialect has the Dolphin hook."""
return bool(getattr(dialect, "_deerflow_gauss_mysql_compat_installed", False))
def _decode_scalar(value: Any) -> str:
if isinstance(value, bytes):
return value.decode(errors="replace")
return str(value)
def _gauss_mysql_default_schema_name(self, connection): # noqa: ANN001, ARG001
"""Return Dolphin's current schema rather than its physical database."""
schema = connection.exec_driver_sql("SELECT current_schema()").scalar()
if schema is None:
raise RuntimeError("GaussDB did not return current_schema()")
return _decode_scalar(schema)
def _gauss_mysql_server_version_info(self, connection): # noqa: ANN001
"""Expose a stable MySQL capability version for a Dolphin connection."""
dbapi_connection = connection.connection
cursor = dbapi_connection.cursor()
try:
cursor.execute("SELECT VERSION()")
row = cursor.fetchone()
finally:
cursor.close()
raw_version = "" if row is None or row[0] is None else _decode_scalar(row[0])
version = GAUSS_MYSQL_SQLALCHEMY_VERSION
self._deerflow_gauss_server_version = raw_version
self._set_mariadb(False, version)
self._mariadb_normalized_version_info = version
self.server_version_info = version
logger.info(
"GaussDB MySQL compatibility enabled: server=%r, SQLAlchemy capability=%s",
raw_version,
".".join(map(str, version)),
)
return version
def _gauss_mysql_isolation_level(self, dbapi_connection): # noqa: ANN001, ARG001
"""Read isolation without comparing an unparseable vendor version."""
cursor = dbapi_connection.cursor()
try:
cursor.execute("SELECT @@transaction_isolation")
row = cursor.fetchone()
finally:
cursor.close()
if row is None or row[0] is None:
raise NotImplementedError("GaussDB did not return a transaction isolation level")
return _decode_scalar(row[0]).upper().replace("-", " ")
def _gauss_mysql_has_table(self, connection, table_name, schema=None, **kw): # noqa: ANN001, ARG001
"""Check relation existence without a failing ``DESCRIBE`` transaction."""
self._ensure_has_table_connection(connection)
if schema is None:
schema = self.default_schema_name
if schema is None:
raise ValueError("GaussDB MySQL table check requires a database name")
statement = "SELECT 1 FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE n.nspname = %s AND c.relname = %s AND c.relkind IN ('r', 'p', 'v', 'm', 'f') LIMIT 1"
with connection.exec_driver_sql(
statement,
(schema, table_name),
execution_options={"skip_user_error_events": True},
) as result:
return result.fetchone() is not None
def _gauss_mysql_create_connect_args(self, url): # noqa: ANN001
"""Add the Dolphin session setting required by PyMySQL-compatible drivers."""
args, kwargs = self._deerflow_gauss_mysql_original_create_connect_args(url)
existing = kwargs.get("init_command")
if existing and existing.strip().rstrip(";").lower() != _DOLPHIN_INIT_COMMAND.lower():
raise ValueError(
"gauss_mysql requires init_command='SET standard_conforming_strings TO off'; "
"remove the conflicting init_command from the database URL"
)
kwargs["init_command"] = _DOLPHIN_INIT_COMMAND
return args, kwargs
def _unescape_gauss_mysql_json_container(value: str) -> Any:
"""Undo exactly one complete MySQL SQL-string escape layer."""
text = value.strip()
if not text.startswith(("[", "{")) or "\\" not in text:
raise ValueError("Not an escaped JSON container")
escapes = {
"0": "\x00",
"n": "\n",
"r": "\r",
"Z": "\x1a",
"\\": "\\",
"'": "'",
'"': '"',
}
output: list[str] = []
position = 0
while position < len(text):
character = text[position]
if character == "\\":
position += 1
if position == len(text) or text[position] not in escapes:
raise ValueError("Unknown or incomplete MySQL escape")
output.append(escapes[text[position]])
elif character in {'"', "'"}:
raise ValueError("Mixed escaped and unescaped quotes")
else:
output.append(character)
position += 1
decoded = json.loads("".join(output))
if not isinstance(decoded, (dict, list)):
raise ValueError("Escaped legacy JSON must be an object or array")
return decoded
def _is_json_domain(value: Any) -> bool:
"""Recursively reject Python-only values accepted by ``literal_eval``."""
if value is None or isinstance(value, (str, bool, int)):
return True
if isinstance(value, float):
return math.isfinite(value)
if isinstance(value, list):
return all(_is_json_domain(item) for item in value)
if isinstance(value, dict):
return all(isinstance(key, str) and _is_json_domain(item) for key, item in value.items())
return False
def decode_gauss_mysql_json(value: Any) -> Any:
"""Decode strict JSON plus the two legacy Dolphin encodings we shipped."""
if isinstance(value, memoryview):
value = value.tobytes()
if isinstance(value, bytes):
value = value.decode("utf-8")
if not isinstance(value, str):
return value
try:
return json.loads(value)
except json.JSONDecodeError:
pass
try:
decoded = ast.literal_eval(value)
except (SyntaxError, ValueError):
try:
return _unescape_gauss_mysql_json_container(value)
except (TypeError, ValueError):
preview = value[:160].replace("\n", "\\n")
raise ValueError(f"GaussDB MySQL JSON value is neither valid JSON, a supported legacy literal, nor a single SQL-escaped JSON container: {preview!r}") from None
if not _is_json_domain(decoded):
raise ValueError(f"GaussDB MySQL JSON legacy literal decoded to an unsupported type: {type(decoded).__name__}")
return decoded
def install_gauss_mysql_dialect_compatibility(driver: str = "aiomysql") -> None:
"""Patch only the selected async MySQL dialect class, once per process."""
if driver == "aiomysql":
from sqlalchemy.dialects.mysql.aiomysql import MySQLDialect_aiomysql as dialect_class
elif driver == "asyncmy":
from sqlalchemy.dialects.mysql.asyncmy import MySQLDialect_asyncmy as dialect_class
else:
raise ValueError(f"Unsupported GaussDB MySQL driver: {driver}")
if getattr(dialect_class, "_deerflow_gauss_mysql_compat_installed", False):
return
dialect_class._deerflow_gauss_mysql_original_create_connect_args = dialect_class.create_connect_args
dialect_class.create_connect_args = _gauss_mysql_create_connect_args
dialect_class._get_server_version_info = _gauss_mysql_server_version_info
dialect_class._get_default_schema_name = _gauss_mysql_default_schema_name
dialect_class.get_isolation_level = _gauss_mysql_isolation_level
dialect_class.has_table = _gauss_mysql_has_table
dialect_class._deerflow_gauss_mysql_compat_installed = True
@@ -0,0 +1,125 @@
"""Conservatively extend ORM tables through openGauss native catalogs.
This is used only through Dolphin's MySQL protocol. The stock MySQL inspector
cannot parse Dolphin's PostgreSQL-style ``SHOW CREATE TABLE`` result and can
therefore report existing columns as missing.
"""
from __future__ import annotations
import logging
from sqlalchemy import CheckConstraint, ForeignKeyConstraint, MetaData, UniqueConstraint
from sqlalchemy.engine import Connection
from sqlalchemy.schema import CreateColumn, CreateIndex
logger = logging.getLogger(__name__)
def _string(value: object) -> str:
return value.decode("utf-8") if isinstance(value, bytes) else str(value)
def ensure_gauss_schema_sync(connection: Connection, metadata: MetaData) -> None:
"""Add only safe missing columns and indexes involving those columns.
Existing columns and unrelated indexes are never altered. A missing
column without a nullable definition or database-side default cannot be
backfilled reliably and stops this repair before any DDL is issued.
"""
current_schema_value = connection.exec_driver_sql("SELECT current_schema()").scalar_one_or_none()
if not current_schema_value:
raise RuntimeError("openGauss schema integrity check: current_schema() is empty")
schema = _string(current_schema_value)
expected_schema = connection.engine.url.database
if expected_schema and expected_schema != schema:
raise RuntimeError(f"openGauss schema integrity check: URL schema {expected_schema!r} does not match current_schema() {schema!r}")
catalog_rows = connection.exec_driver_sql(
"SELECT c.relname, c.relkind, a.attname "
"FROM pg_catalog.pg_class AS c "
"JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace "
"LEFT JOIN pg_catalog.pg_attribute AS a ON a.attrelid = c.oid "
"AND a.attnum > 0 AND NOT a.attisdropped "
"WHERE n.nspname = %s AND c.relkind IN ('r', 'p', 'v', 'm', 'f')",
(schema,),
).fetchall()
columns_by_table: dict[str, set[str]] = {}
kinds_by_table: dict[str, str] = {}
for table_name, kind, column_name in catalog_rows:
table_name = _string(table_name)
kinds_by_table[table_name] = _string(kind)
columns_by_table.setdefault(table_name, set())
if column_name is not None:
columns_by_table[table_name].add(_string(column_name))
plan = []
failures: list[str] = []
for table in metadata.sorted_tables:
if table.schema is not None and table.schema != schema:
failures.append(f"{table.fullname}: explicit schema differs from {schema!r}")
continue
existing = columns_by_table.get(table.name)
if existing is None:
failures.append(f"{schema}.{table.name}: table is still missing after create_all")
continue
if kinds_by_table[table.name] not in {"r", "p"}:
failures.append(f"{schema}.{table.name}: existing relation is not an ordinary table")
continue
missing = [column for column in table.columns if column.name not in existing]
if not missing:
continue
missing_names = {column.name for column in missing}
for column in missing:
label = f"{schema}.{table.name}.{column.name}"
if not column.nullable and column.server_default is None:
failures.append(f"{label}: NOT NULL column has no server_default; explicit migration required")
elif column.primary_key or column.identity is not None or column.computed is not None:
failures.append(f"{label}: primary-key/identity/computed column requires an explicit migration")
for constraint in table.constraints:
if isinstance(constraint, (ForeignKeyConstraint, UniqueConstraint)):
constrained = {column.name for column in constraint.columns}
if constrained & missing_names:
failures.append(f"{schema}.{table.name}: missing columns participate in {type(constraint).__name__}; explicit constraint migration required")
elif isinstance(constraint, CheckConstraint):
constrained = {column.name for column in constraint.columns}
if not constrained or constrained & missing_names:
failures.append(f"{schema}.{table.name}: CHECK-constrained schema needs an explicit migration")
indexes = [index for index in table.indexes if {column.name for column in index.columns} & missing_names]
for index in indexes:
if not index.name:
failures.append(f"{schema}.{table.name}: index involving new columns has no stable name")
plan.append((table, missing, indexes))
if failures:
raise RuntimeError("openGauss schema is incomplete; startup repair stopped:\n- " + "\n- ".join(failures))
if not plan:
logger.info("openGauss schema integrity check passed: %s ORM tables", len(metadata.tables))
return
existing_indexes = {
(_string(table_name), _string(index_name))
for table_name, index_name in connection.exec_driver_sql(
"SELECT t.relname, i.relname FROM pg_catalog.pg_index AS x "
"JOIN pg_catalog.pg_class AS t ON t.oid = x.indrelid "
"JOIN pg_catalog.pg_class AS i ON i.oid = x.indexrelid "
"JOIN pg_catalog.pg_namespace AS n ON n.oid = t.relnamespace "
"WHERE n.nspname = %s",
(schema,),
).fetchall()
}
preparer = connection.dialect.identifier_preparer
added_columns = 0
for table, missing, indexes in plan:
qualified = f"{preparer.quote_schema(schema)}.{preparer.quote(table.name)}"
for column in missing:
definition = str(CreateColumn(column).compile(dialect=connection.dialect))
connection.exec_driver_sql(f"ALTER TABLE {qualified} ADD COLUMN {definition}")
added_columns += 1
logger.info("Added safe missing openGauss column: %s.%s", qualified, column.name)
for index in sorted(indexes, key=lambda item: str(item.name)):
if (table.name, str(index.name)) not in existing_indexes:
connection.execute(CreateIndex(index))
logger.info("Created openGauss index involving new columns: %s", index.name)
logger.info("openGauss schema integrity repair completed: %s columns", added_columns)
@@ -0,0 +1,160 @@
"""Bound JSON predicates for SQLite, PostgreSQL and MySQL.
Adapted from upstream #2865. MySQL business tables store JSON in TEXT, so
these predicates use JSON functions rather than assuming a native JSON column.
Unsupported keys/values must stay in the repository's residual filter; never
silently discard a predicate (in particular a shared task's taskId filter).
"""
from __future__ import annotations
import math
import re
from typing import Any
from sqlalchemy import BigInteger, Float, String, bindparam
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.visitors import InternalTraversal
from sqlalchemy.types import Boolean
_SAFE_KEY = re.compile(r"[A-Za-z0-9_-]+\Z")
_STRIP_CHARS = "".join(chr(i) for i in range(0x3001) if chr(i).isspace())
def supports_json_filter(key: Any, value: Any) -> bool:
if not isinstance(key, str) or not _SAFE_KEY.fullmatch(key):
return False
if value is None or isinstance(value, (str, bool)):
return True
if isinstance(value, int):
return -(2**63) <= value < 2**63
return isinstance(value, float) and math.isfinite(value)
class JsonPredicate(ColumnElement):
"""A scalar equality, Python truthiness, or nonblank-string predicate."""
inherit_cache = True
type = Boolean()
_is_implicitly_boolean = True
_traverse_internals = [
("column", InternalTraversal.dp_clauseelement),
("key", InternalTraversal.dp_string),
("operation", InternalTraversal.dp_string),
("value", InternalTraversal.dp_plain_obj),
("value_kind", InternalTraversal.dp_string),
]
def __init__(self, column, key: str, operation: str = "equal", value: Any = None):
if not supports_json_filter(key, value):
raise ValueError("Unsupported JSON filter key or value")
if operation not in {"equal", "truthy", "nonblank"}:
raise ValueError("Unsupported JSON predicate operation")
self.column = column
self.key = key
self.operation = operation
self.value = value
# Keep Python value types separate in SQLAlchemy's compiled cache.
self.value_kind = type(value).__name__
super().__init__()
def _compile(element, compiler, family, **kw):
if not _SAFE_KEY.fullmatch(element.key):
raise ValueError("Unsafe JSON key")
column = compiler.process(element.column, **kw)
def bound(value, type_=String()):
return compiler.process(bindparam(None, value, type_=type_), **kw)
if family == "postgres":
raw = f"({column} -> '{element.key}')"
kind = f"json_typeof({raw})"
text = f"({column} ->> '{element.key}')"
string_kind, number_kinds = "string", "'number'"
else:
path = f"'$.\"{element.key}\"'"
raw = f"json_extract({column}, {path})"
if family == "mysql":
kind, text = f"lower(json_type({raw}))", f"json_unquote({raw})"
string_kind, number_kinds = "string", "'integer', 'double', 'decimal'"
else:
kind, text = f"json_type({column}, {path})", raw
string_kind, number_kinds = "text", "'integer', 'real'"
if element.operation == "nonblank":
if family == "mysql":
# MySQL 5.7 regexes are byte-oriented; a Unicode whitespace regex
# can misclassify CJK text. Removing each complete whitespace
# character preserves str.strip()'s nonempty test on 5.7 and 8.x.
stripped = text
for char in _STRIP_CHARS:
stripped = f"replace({stripped}, {bound(char)}, '')"
has_text = f"length({stripped}) > 0"
else:
trim = "btrim" if family == "postgres" else "trim"
# SQLite length(TEXT) stops at NUL, unlike Python str.strip().
has_text = f"{trim}({text}, {bound(_STRIP_CHARS)}) <> ''"
clause = f"({kind} = '{string_kind}' AND {has_text})"
elif element.operation == "truthy":
if family == "sqlite":
containers = f"json({raw}) NOT IN ('[]', '{{}}')"
boolean = f"{kind} = 'true'"
elif family == "mysql":
containers = f"json_length({raw}) > 0"
boolean = f"{text} = 'true'"
else:
containers = f"CAST({raw} AS jsonb) NOT IN ('[]'::jsonb, '{{}}'::jsonb)"
boolean = f"{text} = 'true'"
cast = "NUMERIC" if family == "postgres" else "DOUBLE PRECISION"
number = f"({text} + 0e0)" if family == "mysql" else f"CAST({text} AS {cast})"
nonempty = f"{text} <> ''" if family == "sqlite" else f"length({text}) > 0"
clause = (
f"CASE WHEN {kind} IN ({number_kinds}) THEN {number} <> 0 "
f"WHEN {kind} = '{string_kind}' THEN {nonempty} "
f"WHEN {kind} IN ('array', 'object') THEN {containers} "
f"WHEN {kind} IN ('boolean', 'true', 'false') THEN {boolean} ELSE FALSE END"
)
elif element.value is None:
# Preserve this project's md.get(key) == None semantics for old rows.
clause = f"({kind} IS NULL OR {kind} = 'null')"
elif isinstance(element.value, (bool, int, float)):
if family != "sqlite":
raise NotImplementedError("Server numeric metadata requires the exact residual filter")
value = int(element.value) if isinstance(element.value, bool) else element.value
# Preserve md.get(key) == value: Python considers True == 1 == 1.0.
# Text ids remain distinct from numbers. SQLite compares its extracted
# integers/reals natively; server numeric filters use the residual path.
param = bound(value, Float() if isinstance(value, float) else BigInteger())
clause = f"({kind} IN ({number_kinds}, 'true', 'false') AND {text} = {param})"
else:
# Binary comparison is essential for task/agent ids: MySQL TEXT can
# inherit a case-insensitive collation and ignores trailing spaces.
extract = f"CAST({text} AS BINARY)" if family == "mysql" else text
param = bound(element.value)
if family == "mysql":
param = f"CAST({param} AS BINARY)"
clause = f"({kind} = '{string_kind}' AND {extract} = {param})"
# Missing/null values must remain false under NOT (system exclusions).
return f"COALESCE(({clause}), FALSE)"
@compiles(JsonPredicate, "sqlite")
def _sqlite(element, compiler, **kw):
return _compile(element, compiler, "sqlite", **kw)
@compiles(JsonPredicate, "postgresql")
def _postgres(element, compiler, **kw):
return _compile(element, compiler, "postgres", **kw)
@compiles(JsonPredicate, "mysql")
def _mysql(element, compiler, **kw):
return _compile(element, compiler, "mysql", **kw)
@compiles(JsonPredicate)
def _unsupported(element, compiler, **kw):
raise NotImplementedError(f"JSON predicates unsupported for {compiler.dialect.name}")
@@ -0,0 +1,437 @@
"""Private knowledge-base Q&A sessions with shared Markdown publications.
This persistence unit deliberately contains no Gateway/application imports. A
session and its LangGraph thread belong to exactly one user. Once published,
only the saved Markdown snapshot is shared with readers of the bound knowledge
base; the private draft and thread identifier never become publication data.
"""
from __future__ import annotations
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from copy import deepcopy
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import String, delete, select, update
from sqlalchemy.exc import OperationalError, ProgrammingError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class KnowledgeResearchSessionRow(Base):
__tablename__ = "knowledge_research_sessions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
# A separate opaque id is the only identity exposed on the shared bookshelf.
publication_id: Mapped[str] = mapped_column(String(36), unique=True, index=True)
thread_id: Mapped[str] = mapped_column(String(128), unique=True, index=True)
knowledge_base_id: Mapped[str] = mapped_column(String(128), index=True)
user_id: Mapped[str] = mapped_column(String(255), index=True)
title: Mapped[str] = mapped_column(String(255))
draft_markdown: Mapped[str] = mapped_column(PortableLongText, default="")
wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
wiki_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
wiki_folder_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
saved_title: Mapped[str | None] = mapped_column(String(255), nullable=True)
saved_markdown: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
saved_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
saved_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
nullable=False,
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
nullable=False,
)
def _is_missing_table(exc: Exception) -> bool:
message = str(exc).lower()
return (
"no such table" in message
or "doesn't exist" in message
or "does not exist" in message
or "unknown table" in message
)
def _iso(value: Any) -> Any:
return value.isoformat() if isinstance(value, datetime) else value
def _row_dict(row: KnowledgeResearchSessionRow | dict[str, Any]) -> dict[str, Any]:
if isinstance(row, dict):
return deepcopy(row)
return {
"id": row.id,
"publication_id": row.publication_id,
"thread_id": row.thread_id,
"knowledge_base_id": row.knowledge_base_id,
"user_id": row.user_id,
"title": row.title,
"draft_markdown": row.draft_markdown or "",
"wiki_slug": row.wiki_slug,
"wiki_page_id": row.wiki_page_id,
"wiki_folder_id": row.wiki_folder_id,
"saved_title": row.saved_title,
"saved_markdown": row.saved_markdown,
"saved_hash": row.saved_hash,
"saved_at": _iso(row.saved_at),
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
class _PublicationLease:
"""One transaction-scoped mutable publication row."""
def __init__(self, row: KnowledgeResearchSessionRow | dict[str, Any]) -> None:
self._row = row
def snapshot(self) -> dict[str, Any]:
return _row_dict(self._row)
def save(
self,
*,
title: str,
markdown: str,
saved_hash: str,
wiki_slug: str,
wiki_page_id: str | None,
wiki_folder_id: str | None,
) -> dict[str, Any]:
now = datetime.now(UTC)
values = {
"wiki_slug": wiki_slug,
"wiki_page_id": wiki_page_id,
"wiki_folder_id": wiki_folder_id,
"saved_title": title,
"saved_markdown": markdown,
"saved_hash": saved_hash,
"saved_at": now,
"updated_at": now,
}
if isinstance(self._row, dict):
self._row.update(
{
**values,
"saved_at": now.isoformat(),
"updated_at": now.isoformat(),
}
)
else:
for key, value in values.items():
setattr(self._row, key, value)
return self.snapshot()
class KnowledgeResearchStore:
"""SQL store with a behaviorally equivalent memory fallback."""
def __init__(
self,
session_factory: async_sessionmaker[AsyncSession] | None,
) -> None:
self._sf = session_factory
self._rows: dict[str, dict[str, Any]] = {}
self._publication_locks: dict[str, asyncio.Lock] = {}
@asynccontextmanager
async def locked_publication(
self,
session_id: str,
user_id: str,
) -> AsyncIterator[_PublicationLease | None]:
"""Serialize version-check + remote write + local publication commit.
SQL deployments hold a row lock for the context lifetime, so separate
application workers cannot both pass the same expected-hash check.
SQLite has no ``FOR UPDATE`` support, so a no-op update first acquires
its database write lock. The memory fallback uses one asyncio lock per
research session and commits a copy only on successful exit.
"""
if self._sf is None:
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
async with lock:
original = self._rows.get(session_id)
if original is None or original["user_id"] != user_id:
yield None
return
working = deepcopy(original)
lease = _PublicationLease(working)
yield lease
self._rows[session_id] = working
return
async with self._sf() as session:
async with session.begin():
bind = session.get_bind()
if bind.dialect.name == "sqlite":
await session.execute(
update(KnowledgeResearchSessionRow)
.where(
KnowledgeResearchSessionRow.id == session_id,
KnowledgeResearchSessionRow.user_id == user_id,
)
.values(
updated_at=KnowledgeResearchSessionRow.updated_at
)
)
row = (
await session.execute(
select(KnowledgeResearchSessionRow)
.where(
KnowledgeResearchSessionRow.id == session_id,
KnowledgeResearchSessionRow.user_id == user_id,
)
.with_for_update()
)
).scalar_one_or_none()
yield _PublicationLease(row) if row is not None else None
async def create(
self,
*,
knowledge_base_id: str,
user_id: str,
title: str,
session_id: str | None = None,
publication_id: str | None = None,
thread_id: str | None = None,
) -> dict[str, Any]:
now = datetime.now(UTC)
row = KnowledgeResearchSessionRow(
id=session_id or str(uuid4()),
publication_id=publication_id or str(uuid4()),
thread_id=thread_id or str(uuid4()),
knowledge_base_id=knowledge_base_id,
user_id=user_id,
title=title,
draft_markdown="",
created_at=now,
updated_at=now,
)
if self._sf is None:
data = _row_dict(row)
self._rows[row.id] = data
return deepcopy(data)
async with self._sf() as session:
session.add(row)
await session.commit()
return _row_dict(row)
async def get(self, session_id: str) -> dict[str, Any] | None:
if self._sf is None:
row = self._rows.get(session_id)
return deepcopy(row) if row else None
async with self._sf() as session:
row = await session.get(KnowledgeResearchSessionRow, session_id)
return _row_dict(row) if row else None
async def get_owned(self, session_id: str, user_id: str) -> dict[str, Any] | None:
row = await self.get(session_id)
return row if row and row["user_id"] == user_id else None
async def get_by_thread(self, thread_id: str) -> dict[str, Any] | None:
if self._sf is None:
for row in self._rows.values():
if row["thread_id"] == thread_id:
return deepcopy(row)
return None
async with self._sf() as session:
try:
row = (
await session.execute(
select(KnowledgeResearchSessionRow).where(
KnowledgeResearchSessionRow.thread_id == thread_id
)
)
).scalar_one_or_none()
except (OperationalError, ProgrammingError) as exc:
# Ordinary Q&A hits this lookup on every run. A missing table
# (older DB before create_all registered the model) must not
# abort /runs/stream; treat it as "not a research thread".
if _is_missing_table(exc):
return None
raise
return _row_dict(row) if row else None
async def get_publication(self, publication_id: str) -> dict[str, Any] | None:
if self._sf is None:
for row in self._rows.values():
if row["publication_id"] == publication_id and row.get("saved_hash"):
return deepcopy(row)
return None
async with self._sf() as session:
row = (
await session.execute(
select(KnowledgeResearchSessionRow).where(
KnowledgeResearchSessionRow.publication_id == publication_id,
KnowledgeResearchSessionRow.saved_hash.is_not(None),
)
)
).scalar_one_or_none()
return _row_dict(row) if row else None
async def list_owned(
self,
knowledge_base_id: str,
user_id: str,
) -> list[dict[str, Any]]:
if self._sf is None:
rows = [
deepcopy(row)
for row in self._rows.values()
if row["knowledge_base_id"] == knowledge_base_id
and row["user_id"] == user_id
]
return sorted(rows, key=lambda item: str(item["updated_at"]), reverse=True)
async with self._sf() as session:
rows = (
await session.execute(
select(KnowledgeResearchSessionRow)
.where(
KnowledgeResearchSessionRow.knowledge_base_id
== knowledge_base_id,
KnowledgeResearchSessionRow.user_id == user_id,
)
.order_by(KnowledgeResearchSessionRow.updated_at.desc())
)
).scalars()
return [_row_dict(row) for row in rows]
async def list_published(
self,
knowledge_base_id: str,
) -> list[dict[str, Any]]:
if self._sf is None:
rows = [
deepcopy(row)
for row in self._rows.values()
if row["knowledge_base_id"] == knowledge_base_id
and row.get("saved_hash")
]
return sorted(
rows,
key=lambda item: str(item.get("saved_at") or item["updated_at"]),
reverse=True,
)
async with self._sf() as session:
rows = (
await session.execute(
select(KnowledgeResearchSessionRow)
.where(
KnowledgeResearchSessionRow.knowledge_base_id
== knowledge_base_id,
KnowledgeResearchSessionRow.saved_hash.is_not(None),
)
.order_by(KnowledgeResearchSessionRow.saved_at.desc())
)
).scalars()
return [_row_dict(row) for row in rows]
async def update_owned(
self,
session_id: str,
user_id: str,
*,
title: str | None = None,
draft_markdown: str | None = None,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
if self._sf is None:
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
async with lock:
row = self._rows.get(session_id)
if row is None or row["user_id"] != user_id:
return None
if title is not None:
row["title"] = title
if draft_markdown is not None:
row["draft_markdown"] = draft_markdown
row["updated_at"] = now.isoformat()
return deepcopy(row)
async with self._sf() as session:
row = (
await session.execute(
select(KnowledgeResearchSessionRow)
.where(
KnowledgeResearchSessionRow.id == session_id,
KnowledgeResearchSessionRow.user_id == user_id,
)
.with_for_update()
)
).scalar_one_or_none()
if row is None:
return None
if title is not None:
row.title = title
if draft_markdown is not None:
row.draft_markdown = draft_markdown
row.updated_at = now
await session.commit()
return _row_dict(row)
async def save_publication(
self,
session_id: str,
user_id: str,
*,
title: str,
markdown: str,
saved_hash: str,
wiki_slug: str,
wiki_page_id: str | None,
wiki_folder_id: str | None,
) -> dict[str, Any] | None:
async with self.locked_publication(session_id, user_id) as publication:
if publication is None:
return None
return publication.save(
title=title,
markdown=markdown,
saved_hash=saved_hash,
wiki_slug=wiki_slug,
wiki_page_id=wiki_page_id,
wiki_folder_id=wiki_folder_id,
)
async def delete(self, session_id: str) -> None:
if self._sf is None:
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
async with lock:
self._rows.pop(session_id, None)
return
async with self._sf() as session:
await session.execute(
delete(KnowledgeResearchSessionRow).where(
KnowledgeResearchSessionRow.id == session_id
)
)
await session.commit()
def make_knowledge_research_store(session_factory):
return KnowledgeResearchStore(session_factory)
__all__ = [
"KnowledgeResearchSessionRow",
"KnowledgeResearchStore",
"make_knowledge_research_store",
]
@@ -0,0 +1,30 @@
"""Database-backed global ordinary-knowledge retrieval settings."""
from __future__ import annotations
from deerflow.persistence.knowledge_retrieval_settings.base import (
KnowledgeRetrievalSettingsStore,
)
from deerflow.persistence.knowledge_retrieval_settings.memory import (
MemoryKnowledgeRetrievalSettingsStore,
)
from deerflow.persistence.knowledge_retrieval_settings.model import (
KnowledgeRetrievalSettingsRow,
)
from deerflow.persistence.knowledge_retrieval_settings.sql import (
KnowledgeRetrievalSettingsRepository,
)
__all__ = [
"KnowledgeRetrievalSettingsRepository",
"KnowledgeRetrievalSettingsRow",
"KnowledgeRetrievalSettingsStore",
"MemoryKnowledgeRetrievalSettingsStore",
"make_knowledge_retrieval_settings_store",
]
def make_knowledge_retrieval_settings_store(session_factory):
if session_factory is None:
return MemoryKnowledgeRetrievalSettingsStore()
return KnowledgeRetrievalSettingsRepository(session_factory)
@@ -0,0 +1,38 @@
"""Persistence contract for the global ordinary-knowledge retrieval switch."""
from __future__ import annotations
from abc import ABC, abstractmethod
class KnowledgeRetrievalSettingsStore(ABC):
"""Store the deployment-wide smart-reasoning availability flag."""
@abstractmethod
async def get(self) -> bool:
"""Return whether smart reasoning is enabled; missing state is disabled."""
raise NotImplementedError
@abstractmethod
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
"""Atomically replace the global flag and return the saved value."""
raise NotImplementedError
@abstractmethod
async def get_advanced_management_enabled(self) -> bool:
"""Return whether advanced knowledge-management entries are exposed."""
raise NotImplementedError
@abstractmethod
async def set_advanced_management_enabled(
self,
enabled: bool,
*,
updated_by: str | None = None,
) -> bool:
"""Atomically replace the advanced-management flag."""
raise NotImplementedError
@@ -0,0 +1,41 @@
"""In-memory ordinary-knowledge retrieval settings store."""
from __future__ import annotations
import asyncio
from deerflow.persistence.knowledge_retrieval_settings.base import (
KnowledgeRetrievalSettingsStore,
)
class MemoryKnowledgeRetrievalSettingsStore(KnowledgeRetrievalSettingsStore):
def __init__(self) -> None:
self._enabled = False
self._advanced_management_enabled = False
self._lock = asyncio.Lock()
async def get(self) -> bool:
async with self._lock:
return self._enabled
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
del updated_by
async with self._lock:
self._enabled = bool(enabled)
return self._enabled
async def get_advanced_management_enabled(self) -> bool:
async with self._lock:
return self._advanced_management_enabled
async def set_advanced_management_enabled(
self,
enabled: bool,
*,
updated_by: str | None = None,
) -> bool:
del updated_by
async with self._lock:
self._advanced_management_enabled = bool(enabled)
return self._advanced_management_enabled
@@ -0,0 +1,37 @@
"""ORM row for the database-backed ordinary-knowledge retrieval switch."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, false
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class KnowledgeRetrievalSettingsRow(Base):
__tablename__ = "knowledge_retrieval_settings"
scope: Mapped[str] = mapped_column(String(64), primary_key=True, default="global")
smart_reasoning_enabled: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
default=False,
server_default=false(),
)
advanced_management_enabled: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
default=False,
server_default=false(),
)
revision: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,117 @@
"""SQLAlchemy-backed ordinary-knowledge retrieval settings store."""
from __future__ import annotations
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.knowledge_retrieval_settings.base import (
KnowledgeRetrievalSettingsStore,
)
from deerflow.persistence.knowledge_retrieval_settings.model import (
KnowledgeRetrievalSettingsRow,
)
_GLOBAL_SCOPE = "global"
class KnowledgeRetrievalSettingsRepository(KnowledgeRetrievalSettingsStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get(self) -> bool:
async with self._sf() as session:
row = await session.get(KnowledgeRetrievalSettingsRow, _GLOBAL_SCOPE)
return bool(row.smart_reasoning_enabled) if row is not None else False
async def get_advanced_management_enabled(self) -> bool:
async with self._sf() as session:
row = await session.get(KnowledgeRetrievalSettingsRow, _GLOBAL_SCOPE)
return bool(row.advanced_management_enabled) if row is not None else False
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
saved = bool(enabled)
async with self._sf() as session:
result = await session.execute(
select(KnowledgeRetrievalSettingsRow)
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
.with_for_update()
)
row = result.scalar_one_or_none()
if row is None:
row = KnowledgeRetrievalSettingsRow(
scope=_GLOBAL_SCOPE,
smart_reasoning_enabled=saved,
revision=1,
updated_by=updated_by,
)
session.add(row)
else:
row.smart_reasoning_enabled = saved
row.updated_by = updated_by
row.revision = int(row.revision or 0) + 1
try:
await session.commit()
except IntegrityError:
# Two workers may both observe the initially absent singleton.
# The loser retries as an update instead of surfacing a 500.
await session.rollback()
result = await session.execute(
select(KnowledgeRetrievalSettingsRow)
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
.with_for_update()
)
row = result.scalar_one_or_none()
if row is None:
raise
row.smart_reasoning_enabled = saved
row.updated_by = updated_by
row.revision = int(row.revision or 0) + 1
await session.commit()
return saved
async def set_advanced_management_enabled(
self,
enabled: bool,
*,
updated_by: str | None = None,
) -> bool:
saved = bool(enabled)
async with self._sf() as session:
result = await session.execute(
select(KnowledgeRetrievalSettingsRow)
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
.with_for_update()
)
row = result.scalar_one_or_none()
if row is None:
row = KnowledgeRetrievalSettingsRow(
scope=_GLOBAL_SCOPE,
smart_reasoning_enabled=False,
advanced_management_enabled=saved,
revision=1,
updated_by=updated_by,
)
session.add(row)
else:
row.advanced_management_enabled = saved
row.updated_by = updated_by
row.revision = int(row.revision or 0) + 1
try:
await session.commit()
except IntegrityError:
await session.rollback()
result = await session.execute(
select(KnowledgeRetrievalSettingsRow)
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
.with_for_update()
)
row = result.scalar_one_or_none()
if row is None:
raise
row.advanced_management_enabled = saved
row.updated_by = updated_by
row.revision = int(row.revision or 0) + 1
await session.commit()
return saved
@@ -15,6 +15,19 @@ class DuplicateLightAppNameError(ValueError):
class LightAppStore(ABC):
async def list_sidebar_apps(self) -> list[dict[str, Any]]:
"""Return only the columns needed to render iframe app menu entries."""
rows = await self.list_apps(app_type="iframe", enabled_only=True)
return [
{
"app_id": row.get("app_id"),
"name": row.get("name"),
"mount_location": row.get("mount_location"),
"sort_order": row.get("sort_order", 0),
}
for row in rows
]
@abstractmethod
async def list_apps(
self,
@@ -65,6 +65,22 @@ class LightAppRepository(LightAppStore):
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def list_sidebar_apps(self) -> list[dict[str, Any]]:
"""Avoid loading descriptions, icons, route JSON, tags, and audit data."""
stmt = (
select(
LightAppRow.app_id,
LightAppRow.name,
LightAppRow.mount_location,
LightAppRow.sort_order,
)
.where(LightAppRow.app_type == "iframe", LightAppRow.enabled.is_(True))
.order_by(LightAppRow.sort_order.asc(), LightAppRow.created_at.desc())
)
async with self._sf() as session:
result = await session.execute(stmt)
return [dict(row._mapping) for row in result]
async def get_app(self, app_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LightAppRow, app_id)
@@ -67,3 +67,12 @@ class LlmMetricsStore(ABC):
async def iter_all(self, query: LlmMetricsQuery):
"""Yield rows matching ``query`` without pagination — used by export."""
raise NotImplementedError
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
"""Delete one bounded batch of audit rows older than ``cutoff``.
The default keeps lightweight/in-memory test implementations compatible.
SQL-backed stores override it for the scheduled admin-audit retention
sweep.
"""
return 0
@@ -7,7 +7,7 @@ from collections.abc import AsyncIterator
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import desc, func, select
from sqlalchemy import delete, desc, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery, LlmMetricsStore
@@ -110,3 +110,34 @@ class SqlLlmMetricsStore(LlmMetricsStore):
if len(page) < page_size:
return
offset += page_size
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
"""Delete one small, oldest-first retention batch.
Selecting primary keys first keeps this portable across SQLite, MySQL,
and PostgreSQL (which do not share ``DELETE ... LIMIT`` syntax) and
avoids a long-running delete transaction on a busy audit table.
"""
# Keep below SQLite's traditional 999 bind-variable limit. The
# retention worker requests batches repeatedly, so portability is more
# valuable than one giant delete statement.
size = max(1, min(int(batch_size), 500))
normalized_cutoff = _normalize_created_at(cutoff)
async with self._sf() as session:
ids = list(
(
await session.scalars(
select(LlmCallMetricRow.id)
.where(LlmCallMetricRow.created_at < normalized_cutoff)
.order_by(LlmCallMetricRow.created_at.asc(), LlmCallMetricRow.id.asc())
.limit(size)
)
).all()
)
if not ids:
return 0
result = await session.execute(
delete(LlmCallMetricRow).where(LlmCallMetricRow.id.in_(ids))
)
await session.commit()
return int(result.rowcount or 0)
@@ -6,11 +6,47 @@ from abc import ABC, abstractmethod
from typing import Any, Literal
LlmWikiScope = Literal["personal", "public", "all"]
UNSET = object()
def serialize_wiki_parent_directories(
value: list[str] | None,
) -> list[str] | None:
return None if value is None else [str(item) for item in value]
def serialize_wiki_directory_paths(
value: list[list[str]] | None,
) -> list[list[str]] | None:
return (
None
if value is None
else [[str(segment) for segment in path] for path in value]
)
class LlmWikiStore(ABC):
@abstractmethod
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]: ...
async def create_mapping(
self,
*,
weknora_id: str,
owner_user_id: str,
name: str,
description: str,
kb_type: str,
retrieval_settings: dict[str, Any] | None = None,
wiki_parent_directories: list[str] | None = None,
wiki_directory_paths: list[list[str]] | None = None,
wiki_focus_instructions: str | None = None,
wiki_content_instructions_raw: str | None = None,
wiki_extraction_instructions_raw: str | None = None,
wiki_generation_managed: bool | None = None,
wiki_config_schema_version: int | None = None,
wiki_config_revision: int | None = None,
wiki_config_apply_status: str | None = None,
wiki_config_apply_error: str | None = None,
) -> dict[str, Any]: ...
@abstractmethod
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]: ...
@@ -33,6 +69,16 @@ class LlmWikiStore(ABC):
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
wiki_parent_directories: list[str] | None | object = UNSET,
wiki_directory_paths: list[list[str]] | None | object = UNSET,
wiki_focus_instructions: str | None | object = UNSET,
wiki_content_instructions_raw: str | None | object = UNSET,
wiki_extraction_instructions_raw: str | None | object = UNSET,
wiki_generation_managed: bool | None | object = UNSET,
wiki_config_schema_version: int | None | object = UNSET,
wiki_config_revision: int | None | object = UNSET,
wiki_config_apply_status: str | None | object = UNSET,
wiki_config_apply_error: str | None | object = UNSET,
) -> dict[str, Any] | None: ...
@abstractmethod
@@ -3,22 +3,58 @@
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from typing import Any, cast
from uuid import uuid4
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.base import (
UNSET,
LlmWikiScope,
LlmWikiStore,
serialize_wiki_directory_paths,
serialize_wiki_parent_directories,
)
def _now() -> str:
return datetime.now(UTC).isoformat()
def _mapping_to_dict(row: dict[str, Any]) -> dict[str, Any]:
data = dict(row)
data["wiki_parent_directories"] = serialize_wiki_parent_directories(
data.get("wiki_parent_directories")
)
data["wiki_directory_paths"] = serialize_wiki_directory_paths(
data.get("wiki_directory_paths")
)
return data
class MemoryLlmWikiStore(LlmWikiStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._conversation_deposits: dict[str, dict[str, Any]] = {}
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
async def create_mapping(
self,
*,
weknora_id: str,
owner_user_id: str,
name: str,
description: str,
kb_type: str,
retrieval_settings: dict[str, Any] | None = None,
wiki_parent_directories: list[str] | None = None,
wiki_directory_paths: list[list[str]] | None = None,
wiki_focus_instructions: str | None = None,
wiki_content_instructions_raw: str | None = None,
wiki_extraction_instructions_raw: str | None = None,
wiki_generation_managed: bool | None = None,
wiki_config_schema_version: int | None = None,
wiki_config_revision: int | None = None,
wiki_config_apply_status: str | None = None,
wiki_config_apply_error: str | None = None,
) -> dict[str, Any]:
if any(row["weknora_id"] == weknora_id for row in self._rows.values()):
raise ValueError("WeKnora knowledge base is already mapped")
now = _now()
@@ -29,7 +65,25 @@ class MemoryLlmWikiStore(LlmWikiStore):
"name": name,
"description": description,
"test_questions": None,
"retrieval_settings": None,
"retrieval_settings": (
dict(retrieval_settings)
if retrieval_settings is not None
else None
),
"wiki_parent_directories": serialize_wiki_parent_directories(
wiki_parent_directories
),
"wiki_directory_paths": serialize_wiki_directory_paths(
wiki_directory_paths
),
"wiki_focus_instructions": wiki_focus_instructions,
"wiki_content_instructions_raw": wiki_content_instructions_raw,
"wiki_extraction_instructions_raw": wiki_extraction_instructions_raw,
"wiki_generation_managed": wiki_generation_managed,
"wiki_config_schema_version": wiki_config_schema_version,
"wiki_config_revision": wiki_config_revision,
"wiki_config_apply_status": wiki_config_apply_status,
"wiki_config_apply_error": wiki_config_apply_error,
"kb_type": kb_type,
"publication_status": "private",
"published_by": None,
@@ -42,7 +96,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
"updated_at": now,
}
self._rows[row["id"]] = row
return dict(row)
return _mapping_to_dict(row)
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
rows = list(self._rows.values())
@@ -52,22 +106,22 @@ class MemoryLlmWikiStore(LlmWikiStore):
rows = [row for row in rows if row["publication_status"] == "published"]
else:
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id or row["publication_status"] == "published"]
return [dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
return [_mapping_to_dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
if is_admin or row["owner_user_id"] == user_id:
return dict(row)
return _mapping_to_dict(row)
if not write and row["publication_status"] == "published":
return dict(row)
return _mapping_to_dict(row)
return None
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
for row in self._rows.values():
if row["weknora_id"] == weknora_id:
return dict(row)
return _mapping_to_dict(row)
return None
async def update_mapping(
@@ -81,6 +135,16 @@ class MemoryLlmWikiStore(LlmWikiStore):
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
wiki_parent_directories: list[str] | None | object = UNSET,
wiki_directory_paths: list[list[str]] | None | object = UNSET,
wiki_focus_instructions: str | None | object = UNSET,
wiki_content_instructions_raw: str | None | object = UNSET,
wiki_extraction_instructions_raw: str | None | object = UNSET,
wiki_generation_managed: bool | None | object = UNSET,
wiki_config_schema_version: int | None | object = UNSET,
wiki_config_revision: int | None | object = UNSET,
wiki_config_apply_status: str | None | object = UNSET,
wiki_config_apply_error: str | None | object = UNSET,
) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
@@ -98,8 +162,56 @@ class MemoryLlmWikiStore(LlmWikiStore):
if external_search_enabled is not None:
row["external_search_enabled"] = external_search_enabled
row["external_search_updated_at"] = _now()
if wiki_parent_directories is not UNSET:
row["wiki_parent_directories"] = serialize_wiki_parent_directories(
cast(list[str] | None, wiki_parent_directories)
)
if wiki_directory_paths is not UNSET:
row["wiki_directory_paths"] = serialize_wiki_directory_paths(
cast(list[list[str]] | None, wiki_directory_paths)
)
if wiki_focus_instructions is not UNSET:
row["wiki_focus_instructions"] = cast(
str | None,
wiki_focus_instructions,
)
if wiki_content_instructions_raw is not UNSET:
row["wiki_content_instructions_raw"] = cast(
str | None,
wiki_content_instructions_raw,
)
if wiki_extraction_instructions_raw is not UNSET:
row["wiki_extraction_instructions_raw"] = cast(
str | None,
wiki_extraction_instructions_raw,
)
if wiki_generation_managed is not UNSET:
row["wiki_generation_managed"] = cast(
bool | None,
wiki_generation_managed,
)
if wiki_config_schema_version is not UNSET:
row["wiki_config_schema_version"] = cast(
int | None,
wiki_config_schema_version,
)
if wiki_config_revision is not UNSET:
row["wiki_config_revision"] = cast(
int | None,
wiki_config_revision,
)
if wiki_config_apply_status is not UNSET:
row["wiki_config_apply_status"] = cast(
str | None,
wiki_config_apply_status,
)
if wiki_config_apply_error is not UNSET:
row["wiki_config_apply_error"] = cast(
str | None,
wiki_config_apply_error,
)
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def adopt_conversation_deposit_mapping(
self,
@@ -124,7 +236,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
"updated_at": now,
}
)
return dict(row)
return _mapping_to_dict(row)
async def delete_mapping(self, mapping_id: str) -> bool:
return self._rows.pop(mapping_id, None) is not None
@@ -138,7 +250,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["reviewed_by"] = None
row["published_at"] = _now()
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
@@ -148,7 +260,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["reviewed_by"] = reviewer_user_id
row["published_at"] = _now() if approved else None
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
@@ -158,18 +270,18 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["reviewed_by"] = actor_user_id if is_admin else row.get("reviewed_by")
row["published_at"] = None
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
rows = [row for row in self._rows.values() if row["publication_status"] != "private"]
if status:
rows = [row for row in rows if row["publication_status"] == status]
return [dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
return [_mapping_to_dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
for row in self._rows.values():
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
return dict(row)
return _mapping_to_dict(row)
return None
async def get_conversation_deposit(
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -24,6 +24,16 @@ class LlmWikiKnowledgeBaseRow(Base):
# questions for this library.
test_questions: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True)
retrieval_settings: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
wiki_parent_directories: Mapped[list[str] | None] = mapped_column(PortableJSON(), nullable=True)
wiki_directory_paths: Mapped[list[list[str]] | None] = mapped_column(PortableJSON(), nullable=True)
wiki_focus_instructions: Mapped[str | None] = mapped_column(Text, nullable=True)
wiki_content_instructions_raw: Mapped[str | None] = mapped_column(Text, nullable=True)
wiki_extraction_instructions_raw: Mapped[str | None] = mapped_column(Text, nullable=True)
wiki_generation_managed: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
wiki_config_schema_version: Mapped[int | None] = mapped_column(Integer, nullable=True)
wiki_config_revision: Mapped[int | None] = mapped_column(Integer, nullable=True)
wiki_config_apply_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
wiki_config_apply_error: Mapped[str | None] = mapped_column(Text, nullable=True)
kb_type: Mapped[str] = mapped_column(String(32), default="document")
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
@@ -3,13 +3,19 @@
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from typing import Any, cast
from uuid import uuid4
from sqlalchemy import delete, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.base import (
UNSET,
LlmWikiScope,
LlmWikiStore,
serialize_wiki_directory_paths,
serialize_wiki_parent_directories,
)
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
@@ -19,6 +25,14 @@ def _now() -> datetime:
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
data = row.to_dict()
if "wiki_parent_directories" in data:
data["wiki_parent_directories"] = serialize_wiki_parent_directories(
data["wiki_parent_directories"]
)
if "wiki_directory_paths" in data:
data["wiki_directory_paths"] = serialize_wiki_directory_paths(
data["wiki_directory_paths"]
)
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
@@ -30,13 +44,51 @@ class LlmWikiRepository(LlmWikiStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
async def create_mapping(
self,
*,
weknora_id: str,
owner_user_id: str,
name: str,
description: str,
kb_type: str,
retrieval_settings: dict[str, Any] | None = None,
wiki_parent_directories: list[str] | None = None,
wiki_directory_paths: list[list[str]] | None = None,
wiki_focus_instructions: str | None = None,
wiki_content_instructions_raw: str | None = None,
wiki_extraction_instructions_raw: str | None = None,
wiki_generation_managed: bool | None = None,
wiki_config_schema_version: int | None = None,
wiki_config_revision: int | None = None,
wiki_config_apply_status: str | None = None,
wiki_config_apply_error: str | None = None,
) -> dict[str, Any]:
row = LlmWikiKnowledgeBaseRow(
id=str(uuid4()),
weknora_id=weknora_id,
owner_user_id=owner_user_id,
name=name,
description=description,
retrieval_settings=(
dict(retrieval_settings)
if retrieval_settings is not None
else None
),
wiki_parent_directories=serialize_wiki_parent_directories(
wiki_parent_directories
),
wiki_directory_paths=serialize_wiki_directory_paths(
wiki_directory_paths
),
wiki_focus_instructions=wiki_focus_instructions,
wiki_content_instructions_raw=wiki_content_instructions_raw,
wiki_extraction_instructions_raw=wiki_extraction_instructions_raw,
wiki_generation_managed=wiki_generation_managed,
wiki_config_schema_version=wiki_config_schema_version,
wiki_config_revision=wiki_config_revision,
wiki_config_apply_status=wiki_config_apply_status,
wiki_config_apply_error=wiki_config_apply_error,
kb_type=kb_type,
publication_status="private",
created_at=_now(),
@@ -93,6 +145,16 @@ class LlmWikiRepository(LlmWikiStore):
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
wiki_parent_directories: list[str] | None | object = UNSET,
wiki_directory_paths: list[list[str]] | None | object = UNSET,
wiki_focus_instructions: str | None | object = UNSET,
wiki_content_instructions_raw: str | None | object = UNSET,
wiki_extraction_instructions_raw: str | None | object = UNSET,
wiki_generation_managed: bool | None | object = UNSET,
wiki_config_schema_version: int | None | object = UNSET,
wiki_config_revision: int | None | object = UNSET,
wiki_config_apply_status: str | None | object = UNSET,
wiki_config_apply_error: str | None | object = UNSET,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
@@ -111,6 +173,54 @@ class LlmWikiRepository(LlmWikiStore):
if external_search_enabled is not None:
row.external_search_enabled = external_search_enabled
row.external_search_updated_at = _now()
if wiki_parent_directories is not UNSET:
row.wiki_parent_directories = serialize_wiki_parent_directories(
cast(list[str] | None, wiki_parent_directories)
)
if wiki_directory_paths is not UNSET:
row.wiki_directory_paths = serialize_wiki_directory_paths(
cast(list[list[str]] | None, wiki_directory_paths)
)
if wiki_focus_instructions is not UNSET:
row.wiki_focus_instructions = cast(
str | None,
wiki_focus_instructions,
)
if wiki_content_instructions_raw is not UNSET:
row.wiki_content_instructions_raw = cast(
str | None,
wiki_content_instructions_raw,
)
if wiki_extraction_instructions_raw is not UNSET:
row.wiki_extraction_instructions_raw = cast(
str | None,
wiki_extraction_instructions_raw,
)
if wiki_generation_managed is not UNSET:
row.wiki_generation_managed = cast(
bool | None,
wiki_generation_managed,
)
if wiki_config_schema_version is not UNSET:
row.wiki_config_schema_version = cast(
int | None,
wiki_config_schema_version,
)
if wiki_config_revision is not UNSET:
row.wiki_config_revision = cast(
int | None,
wiki_config_revision,
)
if wiki_config_apply_status is not UNSET:
row.wiki_config_apply_status = cast(
str | None,
wiki_config_apply_status,
)
if wiki_config_apply_error is not UNSET:
row.wiki_config_apply_error = cast(
str | None,
wiki_config_apply_error,
)
row.updated_at = _now()
await session.commit()
return _to_dict(row)
@@ -30,6 +30,23 @@ class LlmWikiIndexStore(ABC):
return []
async def list_pages(
self,
mapping_ids: list[str],
*,
limit: int = 500,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
"""Return a bounded, newest-first Wiki page snapshot.
Smart-reasoning regex search applies its expression in Python so the
behavior stays identical on SQLite, MySQL and GaussDB. Third-party
stores may keep the empty default; the tool then uses the remote Wiki
API fallback.
"""
return []
async def replace_graph_edges(
self,
mapping_id: str,
@@ -72,6 +72,28 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
break
return rows
async def list_pages(
self,
mapping_ids: list[str],
*,
limit: int = 500,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
wanted = {str(item).strip() for item in mapping_ids if str(item).strip()}
cap = max(1, min(int(limit), 2000))
rows = [
dict(page)
for page in self.pages.values()
if page.get("knowledge_base_mapping_id") in wanted
and not page.get("is_remote_deleted")
and page.get("status") in ({"published", "draft"} if include_drafts else {"published"})
]
rows.sort(
key=lambda page: str(page.get("remote_updated_at") or page.get("updated_at") or ""),
reverse=True,
)
return rows[:cap]
async def replace_graph_edges(
self,
mapping_id: str,
@@ -123,6 +123,35 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def list_pages(
self,
mapping_ids: list[str],
*,
limit: int = 500,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
normalized_ids = list(
dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip())
)
if not normalized_ids:
return []
statuses = ["published", "draft"] if include_drafts else ["published"]
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(statuses),
)
.order_by(
LlmWikiWikiPageRow.remote_updated_at.desc(),
LlmWikiWikiPageRow.updated_at.desc(),
)
.limit(max(1, min(int(limit), 2000)))
)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def replace_graph_edges(
self,
mapping_id: str,
@@ -0,0 +1,29 @@
"""Keep synchronous compatibility reads off the Gateway's async connection pool."""
import asyncio
from contextvars import ContextVar
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.pool import NullPool
_isolated_factory = ContextVar("isolated_db_factory", default=None)
def isolated_session_factory():
return _isolated_factory.get()
async def run_isolated_db_read(coro):
from deerflow.persistence.engine import get_engine
source = get_engine()
if source is None:
return await coro
# Never return a connection created on this temporary loop to the main pool.
engine = create_async_engine(source.url, poolclass=NullPool)
token = _isolated_factory.set(async_sessionmaker(engine, expire_on_commit=False))
try:
return await asyncio.wait_for(coro, 30)
finally:
_isolated_factory.reset(token)
await engine.dispose()
@@ -0,0 +1,28 @@
"""Helpers shared by the online Alembic environment and its tests."""
from __future__ import annotations
import os
from typing import Any
from sqlalchemy import text
def configure_mysql_migration_lock_waits(connection: Any) -> None:
"""Set bounded MySQL DDL lock waits without leaving a transaction open.
SQLAlchemy starts an implicit transaction for ``SET SESSION`` on MySQL.
Alembic's migration transaction then reuses it, and closing the connection
rolls back the ``alembic_version`` updates even though MySQL DDL itself has
already committed. Commit the session-local settings before Alembic starts
its transaction so successful migrations are recorded durably.
"""
try:
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
except ValueError:
lock_wait = 15
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
connection.commit()
@@ -9,14 +9,13 @@ from __future__ import annotations
import asyncio
import logging
import os
from logging.config import fileConfig
from alembic import context
from sqlalchemy import text
from sqlalchemy.ext.asyncio import create_async_engine
from deerflow.persistence.base import Base
from deerflow.persistence.migration_utils import configure_mysql_migration_lock_waits
# Import all models so metadata is populated.
try:
@@ -54,13 +53,10 @@ def do_run_migrations(connection):
# = 1 year; innodb_lock_wait_timeout = 50s). Without this, an idle
# transaction on another connection can freeze startup indefinitely.
# Tunable via DEER_FLOW_ALEMBIC_LOCK_WAIT (seconds, default 15).
if connection.dialect.name == "mysql":
try:
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
except ValueError:
lock_wait = 15
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
if connection.dialect.name == "mysql" and not is_gauss_mysql_product():
configure_mysql_migration_lock_waits(connection)
context.configure(
connection=connection,
@@ -0,0 +1,39 @@
"""Persist DeerFlow-owned fixed parent directories for LLMWiki mappings.
Revision ID: 20261006_01
Revises: 20260920_02
Create Date: 2026-10-06
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20261006_01"
down_revision = "20260920_02"
branch_labels = None
depends_on = None
_TABLE = "llmwiki_knowledge_bases"
_COLUMN = "wiki_parent_directories"
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TABLE):
return set()
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
def upgrade() -> None:
columns = _columns()
if columns and _COLUMN not in columns:
op.add_column(_TABLE, sa.Column(_COLUMN, PortableJSON(), nullable=True))
def downgrade() -> None:
if _COLUMN in _columns():
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,40 @@
"""Persist administrator-managed per-model runtime capabilities.
Revision ID: 20261009_01
Revises: 20261006_01
Create Date: 2026-10-09
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20261009_01"
down_revision = "20261006_01"
branch_labels = None
depends_on = None
_TABLE = "model_capabilities"
def upgrade() -> None:
if sa.inspect(op.get_bind()).has_table(_TABLE):
return
op.create_table(
_TABLE,
sa.Column("model_name", sa.String(length=160), primary_key=True),
sa.Column(
"supports_reasoning_effort",
sa.Boolean(),
nullable=False,
server_default=sa.false(),
),
sa.Column("updated_by", sa.String(length=64), nullable=True),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
if sa.inspect(op.get_bind()).has_table(_TABLE):
op.drop_table(_TABLE)
@@ -0,0 +1,41 @@
"""Persist the global ordinary-knowledge smart-reasoning switch.
Revision ID: 20261014_01
Revises: 20261009_01
Create Date: 2026-10-14
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20261014_01"
down_revision = "20261009_01"
branch_labels = None
depends_on = None
_TABLE = "knowledge_retrieval_settings"
def upgrade() -> None:
if sa.inspect(op.get_bind()).has_table(_TABLE):
return
op.create_table(
_TABLE,
sa.Column("scope", sa.String(length=64), primary_key=True),
sa.Column(
"smart_reasoning_enabled",
sa.Boolean(),
nullable=False,
server_default=sa.false(),
),
sa.Column("revision", sa.Integer(), nullable=False, server_default="1"),
sa.Column("updated_by", sa.String(length=64), nullable=True),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
if sa.inspect(op.get_bind()).has_table(_TABLE):
op.drop_table(_TABLE)
@@ -0,0 +1,54 @@
"""Persist hierarchical Wiki generation configuration.
Revision ID: 20261014_02
Revises: 20261014_01
Create Date: 2026-10-14
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20261014_02"
down_revision = "20261014_01"
branch_labels = None
depends_on = None
_TABLE = "llmwiki_knowledge_bases"
_COLUMNS: tuple[sa.Column, ...] = (
sa.Column("wiki_directory_paths", PortableJSON(), nullable=True),
sa.Column("wiki_focus_instructions", sa.Text(), nullable=True),
sa.Column("wiki_content_instructions_raw", sa.Text(), nullable=True),
sa.Column("wiki_extraction_instructions_raw", sa.Text(), nullable=True),
sa.Column("wiki_generation_managed", sa.Boolean(), nullable=True),
sa.Column("wiki_config_schema_version", sa.Integer(), nullable=True),
sa.Column("wiki_config_revision", sa.Integer(), nullable=True),
sa.Column("wiki_config_apply_status", sa.String(length=32), nullable=True),
sa.Column("wiki_config_apply_error", sa.Text(), nullable=True),
)
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TABLE):
return set()
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
def upgrade() -> None:
existing = _columns()
if not existing:
return
for column in _COLUMNS:
if column.name not in existing:
op.add_column(_TABLE, column)
def downgrade() -> None:
existing = _columns()
for column in reversed(_COLUMNS):
if column.name in existing:
op.drop_column(_TABLE, column.name)
@@ -0,0 +1,90 @@
"""Persist the knowledge-management advanced-entry switch.
Revision ID: 20261020_01
Revises: 20261019_01, 20261014_02
Create Date: 2026-10-20
An earlier release accidentally reused revision ``20261019_01`` for this
column and for the task deep-link table. A database that ran the column
migration can therefore report the task migration as applied even though its
table is absent. The upgrade reconciles that table and its scope index before
ensuring the advanced-management column, so both fresh and affected databases
converge on the same schema. The task objects belong to the parent revision and
are intentionally left intact by this revision's downgrade.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20261020_01"
down_revision = ("20261019_01", "20261014_02")
branch_labels = None
depends_on = None
_TABLE = "knowledge_retrieval_settings"
_COLUMN = "advanced_management_enabled"
_TASK_TABLE = "task_deeplink_threads"
_TASK_INDEX = "ix_task_deeplink_threads_scope"
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TABLE):
return set()
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
def _ensure_task_deeplink_schema() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TASK_TABLE):
op.create_table(
_TASK_TABLE,
sa.Column("thread_id", sa.String(64), primary_key=True),
sa.Column("task_id", sa.String(128), nullable=False),
sa.Column("source_task_id", sa.String(128), nullable=False),
sa.Column("go_path", sa.String(16), nullable=False),
sa.Column("agent_id", sa.String(128), nullable=True),
sa.Column("created_by", sa.String(64), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
)
op.create_index(
_TASK_INDEX,
_TASK_TABLE,
["task_id", "go_path", "updated_at", "thread_id"],
)
return
indexes = {
str(index["name"])
for index in inspector.get_indexes(_TASK_TABLE)
if index.get("name")
}
if _TASK_INDEX not in indexes:
op.create_index(
_TASK_INDEX,
_TASK_TABLE,
["task_id", "go_path", "updated_at", "thread_id"],
)
def upgrade() -> None:
_ensure_task_deeplink_schema()
if _COLUMN in _columns():
return
op.add_column(
_TABLE,
sa.Column(
_COLUMN,
sa.Boolean(),
nullable=False,
server_default=sa.false(),
),
)
def downgrade() -> None:
if _COLUMN in _columns():
op.drop_column(_TABLE, _COLUMN)
@@ -1,8 +1,15 @@
"""Add snapshots supplied to the b5-1 task-analysis deep link.
Revision ID: 20261020_01
Revises: 20261019_01
Revision ID: 20261021_01
Revises: 20261020_01
Create Date: 2026-10-20
This file originally shipped with revision id ``20261020_01`` — the same id as
``20261020_01_knowledge_management_advanced_tools`` — which left the migration
graph with a duplicate revision and no single head. Re-issued as
``20261021_01`` chained after the canonical ``20261020_01``; its upgrade is
fully guarded (``if _has_table(): return``) so databases that already created
the table under the old id re-run it as a no-op.
"""
from __future__ import annotations
@@ -11,8 +18,8 @@ import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261020_01"
down_revision = "20261019_01"
revision = "20261021_01"
down_revision = "20261020_01"
branch_labels = None
depends_on = None
@@ -0,0 +1,66 @@
"""Admin catalog switch: ``agents.hidden``.
Hidden agents disappear from every non-admin listing surface (gallery / 我的 /
广场 / homepage selector / private square) while staying fully usable — only
the admin agents console (?admin=true) keeps listing them.
The Gateway's startup ORM-column sweep also adds this column on existing
databases (Alembic auto-upgrade is disabled there), so this migration mostly
serves environments that run Alembic manually. Fully guarded and idempotent.
Revision ID: 20261022_03
Revises: 20261021_01
Create Date: 2026-10-22
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261022_03"
down_revision = "20261021_01"
branch_labels = None
depends_on = None
_TABLE = "agents"
_COLUMN = "hidden"
_INDEX = "ix_agents_hidden"
def _has_table(table_name: str) -> bool:
return table_name in inspect(op.get_bind()).get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
if _has_table(_TABLE) and not _has_column(_TABLE, _COLUMN):
# server_default mirrors the ORM declaration so the sweep on live
# databases and this migration produce the same NOT NULL column.
op.add_column(
_TABLE,
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.text("0")),
)
if _has_table(_TABLE) and not _has_index(_TABLE, _INDEX):
op.create_index(_INDEX, _TABLE, [_COLUMN])
def downgrade() -> None:
if _has_table(_TABLE) and _has_index(_TABLE, _INDEX):
op.drop_index(_INDEX, table_name=_TABLE)
if _has_table(_TABLE) and _has_column(_TABLE, _COLUMN):
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,72 @@
"""Persist private knowledge-research sessions and shared publications.
Revision ID: 20261022_01
Revises: 20261014_02
Create Date: 2026-10-22
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableLongText
revision = "20261022_01"
down_revision = "20261014_02"
branch_labels = None
depends_on = None
_TABLE = "knowledge_research_sessions"
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if inspector.has_table(_TABLE):
return
op.create_table(
_TABLE,
sa.Column("id", sa.String(length=36), primary_key=True),
sa.Column("publication_id", sa.String(length=36), nullable=False),
sa.Column("thread_id", sa.String(length=128), nullable=False),
sa.Column("knowledge_base_id", sa.String(length=128), nullable=False),
sa.Column("user_id", sa.String(length=255), nullable=False),
sa.Column("title", sa.String(length=255), nullable=False),
sa.Column("draft_markdown", PortableLongText(), nullable=False),
sa.Column("wiki_slug", sa.String(length=512), nullable=True),
sa.Column("wiki_page_id", sa.String(length=128), nullable=True),
sa.Column("wiki_folder_id", sa.String(length=128), nullable=True),
sa.Column("saved_title", sa.String(length=255), nullable=True),
sa.Column("saved_markdown", PortableLongText(), nullable=True),
sa.Column("saved_hash", sa.String(length=64), nullable=True),
sa.Column("saved_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("publication_id", name="uq_knowledge_research_publication_id"),
sa.UniqueConstraint("thread_id", name="uq_knowledge_research_thread_id"),
)
op.create_index(
"ix_knowledge_research_sessions_publication_id",
_TABLE,
["publication_id"],
)
op.create_index(
"ix_knowledge_research_sessions_thread_id",
_TABLE,
["thread_id"],
)
op.create_index(
"ix_knowledge_research_sessions_knowledge_base_id",
_TABLE,
["knowledge_base_id"],
)
op.create_index(
"ix_knowledge_research_sessions_user_id",
_TABLE,
["user_id"],
)
def downgrade() -> None:
if sa.inspect(op.get_bind()).has_table(_TABLE):
op.drop_table(_TABLE)
@@ -0,0 +1,118 @@
"""Add the integrator-supplied type partition to task deep-link details.
Revision ID: 20261022_02
Revises: 20261024_01
Create Date: 2026-10-22
The same task id may now carry one snapshot per business type (the deep-link
``?type=``), so ``sub_type`` joins the primary key. Existing rows backfill to
``''`` — the legacy no-type partition — and callers that omit the type keep
reading/writing exactly that partition, so behaviour is unchanged for them.
The key change is a portable table rebuild (create-copy-drop-rename): this
Alembic build has no batch ``drop_primary_key`` and SQLite cannot alter a
primary key in place, so any other route leaves a two-column key that makes
typed upserts fail with a UNIQUE constraint violation.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects import mysql as sa_mysql
revision = "20261022_02"
down_revision = "20261024_01"
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_details"
_PK = ["detail_type", "detail_id", "sub_type"]
_OLD_PK = ["detail_type", "detail_id"]
def _has_table() -> bool:
return _TABLE in inspect(op.get_bind()).get_table_names()
def _pk_columns() -> list[str] | None:
constraint = inspect(op.get_bind()).get_pk_constraint(_TABLE)
return list(constraint.get("constrained_columns") or []) or None
def _column_specs(with_sub_type: bool) -> list[tuple[str, sa.types.TypeEngine, bool]]:
is_mysql = op.get_bind().dialect.name == "mysql"
specs = [
("detail_type", sa.String(32), False),
("detail_id", sa.String(128), False),
("task_name", sa.String(512), False),
("task_content", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
("overview", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
("extra_json", sa.Text() if is_mysql else sa.JSON(), False),
("version", sa.Integer(), False),
("created_by", sa.String(64), True),
("updated_by", sa.String(64), True),
("created_at", sa.DateTime(), False),
("updated_at", sa.DateTime(), False),
]
if with_sub_type:
specs.append(("sub_type", sa.String(64), False))
return specs
def _rebuild_table(pk_columns: list[str], with_sub_type: bool) -> None:
"""Recreate the table (create-copy-drop-rename) with the given primary key."""
bind = op.get_bind()
rebuild = _TABLE + "__rebuild"
if rebuild in inspect(bind).get_table_names():
op.drop_table(rebuild)
specs = _column_specs(with_sub_type)
op.create_table(
rebuild,
*[
sa.Column(
name,
type_,
nullable=nullable,
server_default="" if name == "sub_type" else None,
)
for name, type_, nullable in specs
],
sa.PrimaryKeyConstraint(*pk_columns),
)
names = [name for name, _, _ in specs]
select_expr = ", ".join(
f"COALESCE({name}, '')" if name == "sub_type" else name for name in names
)
bind.execute(
sa.text(f"INSERT INTO {rebuild} ({', '.join(names)}) SELECT {select_expr} FROM {_TABLE}")
)
op.drop_table(_TABLE)
op.rename_table(rebuild, _TABLE)
def upgrade() -> None:
if not _has_table():
# Fresh databases get the three-column key straight from create_all.
return
inspector = inspect(op.get_bind())
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
if "sub_type" in columns and _pk_columns() == _PK:
return
if "sub_type" not in columns:
op.add_column(
_TABLE,
sa.Column("sub_type", sa.String(64), nullable=False, server_default=""),
)
if _pk_columns() != _PK:
_rebuild_table(_PK, with_sub_type=True)
def downgrade() -> None:
if not _has_table():
return
if _pk_columns() == _OLD_PK:
return
_rebuild_table(_OLD_PK, with_sub_type=False)
@@ -0,0 +1,65 @@
"""Admin catalog switch: ``roundtable_chains.hidden``.
Hidden chains vanish from every non-admin listing (我的 / 公共 / 待审核 / 选链
弹窗) while staying fully usable — single-record reads (get_chain), configured
business mappings and in-flight roundtables are unaffected. Only the admin
management view (``admin_view=true``) keeps listing them.
The Gateway's startup ORM-column sweep also adds this column on existing
databases (Alembic auto-upgrade is disabled there); this migration serves
environments that run Alembic manually. Fully guarded and idempotent.
Revision ID: 20261023_01
Revises: 20261022_03
Create Date: 2026-10-23
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261023_01"
down_revision = "20261022_03"
branch_labels = None
depends_on = None
_TABLE = "roundtable_chains"
_COLUMN = "hidden"
_INDEX = "ix_roundtable_chains_hidden"
def _has_table(table_name: str) -> bool:
return table_name in inspect(op.get_bind()).get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
if _has_table(_TABLE) and not _has_column(_TABLE, _COLUMN):
op.add_column(
_TABLE,
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.text("0")),
)
if _has_table(_TABLE) and not _has_index(_TABLE, _INDEX):
op.create_index(_INDEX, _TABLE, [_COLUMN])
def downgrade() -> None:
if _has_table(_TABLE) and _has_index(_TABLE, _INDEX):
op.drop_index(_INDEX, table_name=_TABLE)
if _has_table(_TABLE) and _has_column(_TABLE, _COLUMN):
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,55 @@
"""Index the common deep-link list path that omits sub_type.
Revision ID: 20261023_02
Revises: 20261024_02, 20261022_02
Create Date: 2026-10-22
``GET /api/task-deeplink-threads`` for 3qfx/rwfx/xdfx filters only
``task_id + go_path`` and orders by ``updated_at, thread_id``. The existing
scope index inserts ``sub_type`` before those sort columns, so the planner
cannot use it for newest-first listing. This extra index matches that query.
"""
from __future__ import annotations
from alembic import op
from sqlalchemy import inspect
revision = "20261023_02"
down_revision = ("20261024_02", "20261022_02")
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_threads"
_INDEX = "ix_task_deeplink_threads_task_go_updated"
_COLUMNS = ["task_id", "go_path", "updated_at", "thread_id"]
def _has_table() -> bool:
return _TABLE in inspect(op.get_bind()).get_table_names()
def _indexes() -> dict[str, list[str]]:
if not _has_table():
return {}
return {
str(index["name"]): [str(column) for column in index["column_names"]]
for index in inspect(op.get_bind()).get_indexes(_TABLE)
if index.get("name")
}
def upgrade() -> None:
if not _has_table():
return
if _indexes().get(_INDEX) != _COLUMNS:
if _INDEX in _indexes():
op.drop_index(_INDEX, table_name=_TABLE)
op.create_index(_INDEX, _TABLE, _COLUMNS)
def downgrade() -> None:
if not _has_table():
return
if _INDEX in _indexes():
op.drop_index(_INDEX, table_name=_TABLE)
@@ -0,0 +1,96 @@
"""Repair task_deeplink_details primary keys left two-column by 20261022_02.
Revision ID: 20261023_03
Revises: 20261023_02
Create Date: 2026-10-22
An interim build of ``20261022_02`` added the ``sub_type`` column but could
not rebuild the primary key (the batch API it called does not exist in this
Alembic version, and SQLite cannot alter a key in place). Databases that ran
that build are stamped at head yet still enforce the legacy
``(detail_type, detail_id)`` key, so typed upserts fail with a UNIQUE
constraint violation. This revision detects that state and performs the
portable create-copy-drop-rename rebuild; databases with the correct
three-column key simply no-op.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects import mysql as sa_mysql
revision = "20261023_03"
down_revision = "20261023_02"
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_details"
_PK = ["detail_type", "detail_id", "sub_type"]
def _has_table() -> bool:
return _TABLE in inspect(op.get_bind()).get_table_names()
def _pk_columns() -> list[str] | None:
constraint = inspect(op.get_bind()).get_pk_constraint(_TABLE)
return list(constraint.get("constrained_columns") or []) or None
def _rebuild_table() -> None:
bind = op.get_bind()
rebuild = _TABLE + "__rebuild"
if rebuild in inspect(bind).get_table_names():
op.drop_table(rebuild)
is_mysql = bind.dialect.name == "mysql"
specs: list[tuple[str, sa.types.TypeEngine, bool]] = [
("detail_type", sa.String(32), False),
("detail_id", sa.String(128), False),
("sub_type", sa.String(64), False),
("task_name", sa.String(512), False),
("task_content", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
("overview", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
("extra_json", sa.Text() if is_mysql else sa.JSON(), False),
("version", sa.Integer(), False),
("created_by", sa.String(64), True),
("updated_by", sa.String(64), True),
("created_at", sa.DateTime(), False),
("updated_at", sa.DateTime(), False),
]
op.create_table(
rebuild,
*[
sa.Column(
name,
type_,
nullable=nullable,
server_default="" if name == "sub_type" else None,
)
for name, type_, nullable in specs
],
sa.PrimaryKeyConstraint(*_PK),
)
names = [name for name, _, _ in specs]
select_expr = ", ".join(
f"COALESCE({name}, '')" if name == "sub_type" else name for name in names
)
bind.execute(
sa.text(f"INSERT INTO {rebuild} ({', '.join(names)}) SELECT {select_expr} FROM {_TABLE}")
)
op.drop_table(_TABLE)
op.rename_table(rebuild, _TABLE)
def upgrade() -> None:
if not _has_table():
return
if _pk_columns() == _PK:
return
_rebuild_table()
def downgrade() -> None:
# The repaired key is what 20261022_02 intended; nothing to undo.
return
@@ -0,0 +1,61 @@
"""Add the b5-1 ?type= partition column to the deep-link history index.
Revision ID: 20261024_01
Revises: 20261023_01
Create Date: 2026-10-22
Existing rows keep sharing one partition per (task_id, go_path): the new
``sub_type`` column backfills to ``''`` and queries that omit ``sub_type``
continue to return every sub-type, so 3qfx/rwfx/xdfx behaviour is unchanged.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261024_01"
down_revision = "20261023_01"
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_threads"
_INDEX = "ix_task_deeplink_threads_scope"
_INDEX_COLUMNS = ["task_id", "go_path", "sub_type", "updated_at", "thread_id"]
def _has_table() -> bool:
return _TABLE in inspect(op.get_bind()).get_table_names()
def upgrade() -> None:
if not _has_table():
# Fresh databases get the column straight from create_all.
return
bind = op.get_bind()
inspector = inspect(bind)
if "sub_type" not in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.add_column(
_TABLE,
sa.Column("sub_type", sa.String(64), nullable=False, server_default=""),
)
indexes = {index["name"]: index["column_names"] for index in inspector.get_indexes(_TABLE)}
if indexes.get(_INDEX) != _INDEX_COLUMNS:
if _INDEX in indexes:
op.drop_index(_INDEX, table_name=_TABLE)
op.create_index(_INDEX, _TABLE, _INDEX_COLUMNS)
def downgrade() -> None:
if not _has_table():
return
bind = op.get_bind()
inspector = inspect(bind)
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
indexes = {index["name"]: index["column_names"] for index in inspector.get_indexes(_TABLE)}
if indexes.get(_INDEX) is not None:
op.drop_index(_INDEX, table_name=_TABLE)
if "sub_type" in columns:
op.drop_column(_TABLE, "sub_type")
op.create_index(_INDEX, _TABLE, ["task_id", "go_path", "updated_at", "thread_id"])
@@ -0,0 +1,113 @@
"""Store deep-link list cards without joining ``threads_meta``.
Revision ID: 20261024_02
Revises: 20261024_01
Create Date: 2026-10-22
The dedicated history table used to be a pointer: listing still joined the
generic ``threads_meta`` catalogue for title / status / metadata. That join
is what made the task workspace slow once ordinary Q&A filled the catalogue.
This revision copies those card fields onto ``task_deeplink_threads`` itself
and backfills existing index rows from ``threads_meta``.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from deerflow.persistence.types import PortableJSON
revision = "20261024_02"
down_revision = "20261024_01"
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_threads"
_META_TABLE = "threads_meta"
def _has_table(table_name: str) -> bool:
return table_name in inspect(op.get_bind()).get_table_names()
def _columns() -> set[str]:
if not _has_table(_TABLE):
return set()
return {str(column["name"]) for column in inspect(op.get_bind()).get_columns(_TABLE)}
def _backfill_cards() -> None:
if not _has_table(_META_TABLE):
return
bind = op.get_bind()
dialect = bind.dialect.name
if dialect in {"postgresql", "opengauss"}:
bind.execute(
sa.text(
f"UPDATE {_TABLE} AS t "
"SET display_name = m.display_name, "
"status = COALESCE(m.status, 'idle'), "
"metadata_json = m.metadata_json "
f"FROM {_META_TABLE} AS m "
"WHERE m.thread_id = t.thread_id"
)
)
return
if dialect == "mysql":
bind.execute(
sa.text(
f"UPDATE {_TABLE} AS t "
f"JOIN {_META_TABLE} AS m ON m.thread_id = t.thread_id "
"SET t.display_name = m.display_name, "
"t.status = COALESCE(m.status, 'idle'), "
"t.metadata_json = m.metadata_json"
)
)
return
bind.execute(
sa.text(
f"UPDATE {_TABLE} SET "
"display_name = ("
f"SELECT display_name FROM {_META_TABLE} "
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
"), "
"status = COALESCE(("
f"SELECT status FROM {_META_TABLE} "
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
"), 'idle'), "
"metadata_json = ("
f"SELECT metadata_json FROM {_META_TABLE} "
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
")"
)
)
def upgrade() -> None:
if not _has_table(_TABLE):
return
columns = _columns()
if "display_name" not in columns:
op.add_column(_TABLE, sa.Column("display_name", sa.String(256), nullable=True))
if "status" not in columns:
op.add_column(
_TABLE,
sa.Column("status", sa.String(20), nullable=False, server_default="idle"),
)
if "metadata_json" not in columns:
op.add_column(_TABLE, sa.Column("metadata_json", PortableJSON(), nullable=True))
_backfill_cards()
def downgrade() -> None:
if not _has_table(_TABLE):
return
columns = _columns()
if "metadata_json" in columns:
op.drop_column(_TABLE, "metadata_json")
if "status" in columns:
op.drop_column(_TABLE, "status")
if "display_name" in columns:
op.drop_column(_TABLE, "display_name")
@@ -0,0 +1,54 @@
"""Indexes for the agent gallery category tabs and tag filters.
Revision ID: 20261025_01
Revises: 20261023_03
Create Date: 2026-10-23
The gallery "广场" tab filters ``hidden`` + ``is_private`` + ``square_id``,
and "我的" filters ``user_id`` + ``hidden``. Tag chips join
``tag_assignments`` by ``(target_type, target_id)`` and filter by
``(target_type, tag_id)``. Single-column indexes and the existing unique
key (which leads with ``tag_id``) do not cover those predicates.
Startup also creates missing ORM indexes, so deployments that skip Alembic
still receive these. This revision stays idempotent for the ones that run it.
"""
from __future__ import annotations
from alembic import op
from sqlalchemy import inspect
revision = "20261025_01"
down_revision = "20261023_03"
branch_labels = None
depends_on = None
_INDEXES = (
("agents", "ix_agents_catalog_square", ("hidden", "is_private", "square_id", "created_at")),
("agents", "ix_agents_owner_hidden", ("user_id", "hidden", "created_at")),
("tag_assignments", "ix_tag_assignments_type_target", ("target_type", "target_id")),
("tag_assignments", "ix_tag_assignments_type_tag", ("target_type", "tag_id", "target_id")),
)
def _has_table(table_name: str) -> bool:
return table_name in inspect(op.get_bind()).get_table_names()
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
return any(idx["name"] == index_name for idx in inspect(op.get_bind()).get_indexes(table_name))
def upgrade() -> None:
for table_name, index_name, columns in _INDEXES:
if _has_table(table_name) and not _has_index(table_name, index_name):
op.create_index(index_name, table_name, list(columns))
def downgrade() -> None:
for table_name, index_name, _columns in reversed(_INDEXES):
if _has_table(table_name) and _has_index(table_name, index_name):
op.drop_index(index_name, table_name=table_name)
@@ -0,0 +1,37 @@
"""Add shared nonce storage for scoped service-token exchange.
Revision ID: 20261025_02
Revises: 20261025_01
Create Date: 2026-10-24
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261025_02"
down_revision = "20261025_01"
branch_labels = None
depends_on = None
_TABLE = "auth_service_nonces"
def upgrade() -> None:
if _TABLE in inspect(op.get_bind()).get_table_names():
return
op.create_table(
_TABLE,
sa.Column("nonce_hash", sa.String(64), primary_key=True),
sa.Column("client_id", sa.String(128), nullable=False),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_auth_service_nonces_client_id", _TABLE, ["client_id"])
op.create_index("ix_auth_service_nonces_expires_at", _TABLE, ["expires_at"])
def downgrade() -> None:
if _TABLE in inspect(op.get_bind()).get_table_names():
op.drop_table(_TABLE)
@@ -0,0 +1,35 @@
"""Add the database-backed skill catalogue snapshot for list requests.
Revision ID: 20261025_03
Revises: 20261025_02
Create Date: 2026-10-25
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261025_03"
down_revision = "20261025_02"
branch_labels = None
depends_on = None
_TABLE = "skill_catalog_snapshots"
def upgrade() -> None:
if _TABLE in inspect(op.get_bind()).get_table_names():
return
op.create_table(
_TABLE,
sa.Column("cache_key", sa.String(32), primary_key=True),
sa.Column("catalog_json", sa.Text(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
if _TABLE in inspect(op.get_bind()).get_table_names():
op.drop_table(_TABLE)
@@ -0,0 +1,62 @@
"""Index canonical usernames for fast SSO and username login.
Revision ID: 20261026_01
Revises: 20261025_03
Create Date: 2026-10-26
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261026_01"
down_revision = "20261025_03"
branch_labels = None
depends_on = None
_TABLE = "users"
_COLUMN = "username_key"
_INDEX = "ix_users_username_key_created_at"
def upgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
if _COLUMN not in columns:
op.add_column(_TABLE, sa.Column(_COLUMN, sa.String(128), nullable=True))
indexes = {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
if _INDEX not in indexes:
op.create_index(_INDEX, _TABLE, [_COLUMN, "created_at"])
# Backfill the normalized local part once. Login requests then use a
# bounded indexed equality lookup instead of LOWER(email) LIKE 'name@%'.
bind = op.get_bind()
rows = bind.execute(sa.text("SELECT id, email FROM users")).fetchall()
for user_id, email in rows:
if not email or "@" not in email:
continue
username_key = str(email).split("@", 1)[0].strip().lower()[:128]
bind.execute(
sa.text(
"UPDATE users SET username_key = :username_key "
"WHERE id = :user_id AND (username_key IS NULL OR username_key = '')"
),
{"username_key": username_key, "user_id": user_id},
)
def downgrade() -> None:
if _TABLE not in inspect(op.get_bind()).get_table_names():
return
indexes = {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
if _INDEX in indexes:
op.drop_index(_INDEX, table_name=_TABLE)
columns = {column["name"] for column in inspect(op.get_bind()).get_columns(_TABLE)}
if _COLUMN in columns:
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,44 @@
"""Persist the globally shared default reasoning effort for each model.
Revision ID: 20261026_02
Revises: 20261026_01, 20261022_01
Create Date: 2026-10-26
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261026_02"
down_revision = ("20261026_01", "20261022_01")
branch_labels = None
depends_on = None
_TABLE = "model_capabilities"
_COLUMN = "default_reasoning_effort"
def upgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.add_column(
_TABLE,
sa.Column(
_COLUMN,
sa.String(length=16),
nullable=False,
server_default="xhigh",
),
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,44 @@
"""Persist the administrator-configured Redis cache TTL.
Revision ID: 20261029_01
Revises: 20261026_02
Create Date: 2026-10-29
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261029_01"
down_revision = "20261026_02"
branch_labels = None
depends_on = None
_TABLE = "api_read_cache_settings"
_COLUMN = "ttl_seconds"
def upgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.add_column(
_TABLE,
sa.Column(
_COLUMN,
sa.Integer(),
nullable=False,
server_default=sa.text("300"),
),
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,32 @@
"""Persist per-API cache switches and browser status access control."""
import sqlalchemy as sa
from alembic import op
revision = "20261029_02"
down_revision = "20261029_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "api_read_cache_settings" not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns("api_read_cache_settings")}
if "collection_overrides" not in columns:
op.add_column("api_read_cache_settings", sa.Column("collection_overrides", sa.JSON(), nullable=True))
if "browser_status_enabled" not in columns:
op.add_column("api_read_cache_settings", sa.Column("browser_status_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
if "cache_epoch" not in columns:
op.add_column("api_read_cache_settings", sa.Column("cache_epoch", sa.String(32), nullable=False, server_default="0"))
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "api_read_cache_settings" not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns("api_read_cache_settings")}
for name in ("cache_epoch", "browser_status_enabled", "collection_overrides"):
if name in columns:
op.drop_column("api_read_cache_settings", name)
@@ -0,0 +1,41 @@
"""Add per-item enablement to compact menu overrides.
Existing compact-menu items remain enabled. New databases receive the field
from ``CompactMenuOverrideRow``; this revision and the startup ORM schema sweep
add it safely to deployments with an existing ``compact_menu_overrides`` table.
Revision ID: 20261102_01
Revises: 20261029_02
Create Date: 2026-11-02
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261102_01"
down_revision = "20261029_02"
branch_labels = None
depends_on = None
_TABLE = "compact_menu_overrides"
_COLUMN = "disabled"
def upgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.add_column(
_TABLE,
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.false()),
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
if _TABLE not in inspector.get_table_names():
return
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,67 @@
"""Allow administrator-defined business-mapping types and login goPaths.
Revision ID: 20261102_02
Revises: 20261102_01
Create Date: 2026-11-02
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261102_02"
down_revision = "20261102_01"
branch_labels = None
depends_on = None
_MAPPING_TABLE = "business_mapping"
_THREAD_TABLE = "task_deeplink_threads"
def upgrade() -> None:
inspector = inspect(op.get_bind())
tables = set(inspector.get_table_names())
if _MAPPING_TABLE in tables:
columns = {column["name"] for column in inspector.get_columns(_MAPPING_TABLE)}
if "mapping_type" not in columns:
op.add_column(
_MAPPING_TABLE,
sa.Column("mapping_type", sa.String(64), nullable=False, server_default="内置业务"),
)
if "is_custom" not in columns:
op.add_column(
_MAPPING_TABLE,
sa.Column("is_custom", sa.Boolean(), nullable=False, server_default=sa.false()),
)
# PostgreSQL enforces VARCHAR length, unlike SQLite. Widen the dedicated
# history partition too so a 32-character custom goPath can be persisted.
if _THREAD_TABLE in tables:
with op.batch_alter_table(_THREAD_TABLE) as batch:
batch.alter_column(
"go_path",
existing_type=sa.String(16),
type_=sa.String(32),
existing_nullable=False,
)
def downgrade() -> None:
inspector = inspect(op.get_bind())
tables = set(inspector.get_table_names())
if _THREAD_TABLE in tables:
with op.batch_alter_table(_THREAD_TABLE) as batch:
batch.alter_column(
"go_path",
existing_type=sa.String(32),
type_=sa.String(16),
existing_nullable=False,
)
if _MAPPING_TABLE in tables:
columns = {column["name"] for column in inspector.get_columns(_MAPPING_TABLE)}
if "is_custom" in columns:
op.drop_column(_MAPPING_TABLE, "is_custom")
if "mapping_type" in columns:
op.drop_column(_MAPPING_TABLE, "mapping_type")
@@ -0,0 +1,49 @@
"""Administrator-managed per-model runtime capabilities."""
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
from deerflow.persistence.model_capabilities.memory import MemoryModelCapabilityStore
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
from deerflow.persistence.model_capabilities.sql import ModelCapabilityRepository
__all__ = [
"MemoryModelCapabilityStore",
"ModelCapabilityRepository",
"ModelCapabilityRow",
"ModelCapabilityStore",
"import_legacy_reasoning_effort_capabilities",
"make_model_capability_store",
]
async def import_legacy_reasoning_effort_capabilities(
store: ModelCapabilityStore,
models: Iterable[Any],
) -> list[str]:
"""One-time compatibility import for legacy config-backed ``true`` flags.
A configured ``false`` never creates a row. Once any database row exists,
its value is authoritative and cannot be overwritten by legacy YAML.
"""
imported: list[str] = []
for model in models:
if not bool(getattr(model, "supports_reasoning_effort", False)):
continue
model_name = str(getattr(model, "name", "") or "").strip()
if model_name and await store.seed_reasoning_effort_support(
model_name,
updated_by="legacy-config-import",
):
imported.append(model_name)
return imported
def make_model_capability_store(session_factory):
if session_factory is None:
return MemoryModelCapabilityStore()
return ModelCapabilityRepository(session_factory)
@@ -0,0 +1,49 @@
"""Persistence contract for administrator-managed model capabilities.
The provider connection details still live in ``config.yaml``. Runtime
capabilities that administrators change from the web UI live here instead so
they are shared by every Gateway worker and survive application restarts.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class ModelCapabilityStore(ABC):
"""Global capability settings keyed by the stable configured model name."""
@abstractmethod
async def list_capabilities(self) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def set_reasoning_effort_support(
self,
model_name: str,
*,
supports_reasoning_effort: bool,
default_reasoning_effort: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def seed_reasoning_effort_support(
self,
model_name: str,
*,
updated_by: str | None = None,
) -> bool:
"""Insert legacy ``true`` only when no authoritative row exists."""
raise NotImplementedError
@abstractmethod
async def delete_capability(self, model_name: str) -> None:
raise NotImplementedError
@@ -0,0 +1,58 @@
"""In-memory model-capability store used by memory-mode deployments/tests."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
class MemoryModelCapabilityStore(ModelCapabilityStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
async def list_capabilities(self) -> list[dict[str, Any]]:
return [dict(self._rows[name]) for name in sorted(self._rows)]
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
row = self._rows.get(model_name)
return dict(row) if row is not None else None
async def set_reasoning_effort_support(
self,
model_name: str,
*,
supports_reasoning_effort: bool,
default_reasoning_effort: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any]:
row = {
"model_name": model_name,
"supports_reasoning_effort": bool(supports_reasoning_effort),
"default_reasoning_effort": default_reasoning_effort
or (self._rows.get(model_name) or {}).get("default_reasoning_effort")
or "xhigh",
"updated_by": updated_by,
"updated_at": datetime.now(UTC).isoformat(),
}
self._rows[model_name] = row
return dict(row)
async def seed_reasoning_effort_support(
self,
model_name: str,
*,
updated_by: str | None = None,
) -> bool:
if model_name in self._rows:
return False
await self.set_reasoning_effort_support(
model_name,
supports_reasoning_effort=True,
updated_by=updated_by,
)
return True
async def delete_capability(self, model_name: str) -> None:
self._rows.pop(model_name, None)
@@ -0,0 +1,36 @@
"""ORM row for administrator-managed model capabilities."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String, false
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ModelCapabilityRow(Base):
__tablename__ = "model_capabilities"
model_name: Mapped[str] = mapped_column(String(160), primary_key=True)
supports_reasoning_effort: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
default=False,
server_default=false(),
)
default_reasoning_effort: Mapped[str] = mapped_column(
String(16),
nullable=False,
default="xhigh",
server_default="xhigh",
)
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,96 @@
"""SQLAlchemy-backed model-capability store."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
def _to_dict(row: ModelCapabilityRow) -> dict[str, Any]:
return {
"model_name": row.model_name,
"supports_reasoning_effort": bool(row.supports_reasoning_effort),
"default_reasoning_effort": row.default_reasoning_effort or "xhigh",
"updated_by": row.updated_by,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
class ModelCapabilityRepository(ModelCapabilityStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_capabilities(self) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(select(ModelCapabilityRow).order_by(ModelCapabilityRow.model_name.asc()))
return [_to_dict(row) for row in result.scalars()]
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ModelCapabilityRow, model_name)
return _to_dict(row) if row is not None else None
async def set_reasoning_effort_support(
self,
model_name: str,
*,
supports_reasoning_effort: bool,
default_reasoning_effort: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(ModelCapabilityRow, model_name)
if row is None:
row = ModelCapabilityRow(model_name=model_name)
session.add(row)
row.supports_reasoning_effort = bool(supports_reasoning_effort)
if default_reasoning_effort is not None:
row.default_reasoning_effort = default_reasoning_effort
row.updated_by = updated_by
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def seed_reasoning_effort_support(
self,
model_name: str,
*,
updated_by: str | None = None,
) -> bool:
"""Insert a legacy ``true`` without ever overwriting an existing row.
The primary-key conflict handles concurrent Gateway startup safely: if
another worker or an administrator creates the row after our read, the
losing seed transaction rolls back and preserves that authoritative
value, including ``false``.
"""
async with self._sf() as session:
if await session.get(ModelCapabilityRow, model_name) is not None:
return False
session.add(
ModelCapabilityRow(
model_name=model_name,
supports_reasoning_effort=True,
updated_by=updated_by,
)
)
try:
await session.commit()
except IntegrityError:
await session.rollback()
return False
return True
async def delete_capability(self, model_name: str) -> None:
async with self._sf() as session:
await session.execute(delete(ModelCapabilityRow).where(ModelCapabilityRow.model_name == model_name))
await session.commit()
@@ -35,12 +35,15 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.api_read_cache_settings.model import ApiReadCacheSettingRow
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
from deerflow.persistence.analysis_project_memory.model import (
AnalysisProjectMemoryItemRow,
AnalysisProjectMemoryJobRow,
AnalysisProjectMemoryStateRow,
)
from deerflow.persistence.analysis_projects import AnalysisProjectConfigRow, AnalysisProjectRow, AnalysisProjectThreadRow
from deerflow.persistence.article_types.model import ArticleTypeRow
from deerflow.persistence.artifact_library.model import (
ArtifactLibraryItemRow,
ArtifactLibrarySyncStateRow,
@@ -75,7 +78,10 @@ from deerflow.persistence.force_situation.model import (
ForceSituationDetailVersionRow,
)
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.knowledge_research import KnowledgeResearchSessionRow
from deerflow.persistence.knowledge_retrieval_settings.model import KnowledgeRetrievalSettingsRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
@@ -87,6 +93,7 @@ from deerflow.persistence.llmwiki_index.model import (
)
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.notifications.model import NotificationRow
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
@@ -122,6 +129,7 @@ from deerflow.persistence.roundtable_drafts.model import (
RoundtableRecommendHistoryRow,
)
from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
from deerflow.persistence.roundtable_task_drafts.model import RoundtableTaskDraftRow
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.scheduled_tasks.model import (
@@ -144,9 +152,26 @@ from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
from deerflow.persistence.skills.model import (
SkillCatalogSnapshotRow,
SkillDisplayAdapterRow,
SkillFavoriteRow,
SkillRow,
)
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.task_analysis.model import (
TaskAnalysisBindingRow,
TaskAnalysisContentVersionRow,
TaskAnalysisRunMessageRow,
TaskAnalysisRunRow,
TaskAnalysisSettingRow,
TaskAnalysisTaskRow,
TaskDataBlockDefRow,
TaskDataBlockRow,
TaskDataPeriodRow,
TaskDataSubmissionRow,
)
from deerflow.persistence.task_buttons.model import TaskButtonRow
from deerflow.persistence.task_deeplink_details.model import TaskDeeplinkDetailRow
from deerflow.persistence.task_deeplink_threads.model import TaskDeeplinkThreadRow
@@ -155,6 +180,7 @@ from deerflow.persistence.taskcop_tasks.model import TaskCopTaskRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
from deerflow.persistence.upload_policy.model import UploadPolicyRow
from deerflow.persistence.user.model import UserRow
from deerflow.persistence.user_preferences.model import UserPreferencesRow
@@ -173,21 +199,25 @@ from deerflow.persistence.workflow_runs.model import (
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
__all__ = [
"AIWritingSessionRow",
"AnalysisProjectConfigRow",
"AnalysisProjectMemoryItemRow",
"AnalysisProjectMemoryJobRow",
"AnalysisProjectMemoryStateRow",
"ArtifactLibraryItemRow",
"ArtifactLibrarySyncStateRow",
"ArticleTypeRow",
"FeedbackRow",
"FixedQuestionRow",
"ForceSituationDetailRow",
"ForceSituationDetailVersionRow",
"GongwenDocumentRow",
"GongwenDocumentVersionRow",
"HtmlPageFavoriteRow",
"AgentExtrasRow",
"AgentFavoriteRow",
"AgentRow",
"ApiReadCacheSettingRow",
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
@@ -222,6 +252,8 @@ __all__ = [
"KnowledgeRelationRow",
"KnowledgeSourceRow",
"KnowledgeTagRow",
"KnowledgeResearchSessionRow",
"KnowledgeRetrievalSettingsRow",
"LightAppRow",
"WorkflowDataSourceRow",
"WorkflowDefinitionRow",
@@ -240,6 +272,7 @@ __all__ = [
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"MenuOverrideRow",
"ModelCapabilityRow",
"UserMenuOverrideConfigRow",
"NotificationRow",
"ParallelAgentTaskRow",
@@ -268,17 +301,20 @@ __all__ = [
"RoundtableDraftShareRow",
"RoundtableDiagnosticRow",
"RoundtableJobRow",
"RoundtableRoomLeaseRow",
"RoundtableRecommendHistoryRow",
"RoundtableTaskDraftRow",
"PositionRoleRow",
"RunEventRow",
"RunRow",
"ScheduledTaskDeliveryProfileRow",
"ScheduledHtmlPageRow",
"ScheduledTaskRow",
"ScheduledTaskRunRow",
"ScheduledTaskSubscriptionRow",
"SchedulerThreadRow",
"SensitiveWordRow",
"SkillCatalogSnapshotRow",
"SkillRow",
"SkillDisplayAdapterRow",
"SkillFavoriteRow",
@@ -296,6 +332,16 @@ __all__ = [
"TagAssignmentRow",
"TagRow",
"TaskButtonRow",
"TaskAnalysisBindingRow",
"TaskAnalysisContentVersionRow",
"TaskAnalysisRunMessageRow",
"TaskAnalysisRunRow",
"TaskAnalysisSettingRow",
"TaskAnalysisTaskRow",
"TaskDataBlockDefRow",
"TaskDataBlockRow",
"TaskDataPeriodRow",
"TaskDataSubmissionRow",
"TaskDeeplinkDetailRow",
"TaskDeeplinkThreadRow",
"AnalysisProjectRow",
@@ -307,6 +353,7 @@ __all__ = [
"ThreadMetaRow",
"ThreadShareRow",
"ToolCallMetricRow",
"StreamTruncationEventRow",
"UploadPolicyRow",
"UserCustomPromptRow",
"UserMemoryConfigRow",
@@ -15,7 +15,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy import Index, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -24,6 +24,14 @@ from deerflow.persistence.types import BeijingDateTime, PortableLongText
class ParallelAgentTaskRow(Base):
__tablename__ = "parallel_agent_tasks"
__table_args__ = (
Index(
"ix_parallel_agent_tasks_user_updated_id",
"user_id",
"updated_at",
"id",
),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
@@ -11,6 +11,7 @@ that belongs to another user is treated as not found.
from __future__ import annotations
import json
from collections.abc import Mapping
from datetime import UTC, datetime
from typing import Any
@@ -44,17 +45,32 @@ class ParallelAgentTaskRepository:
self._sf = session_factory
@staticmethod
def _to_dict(row: ParallelAgentTaskRow, *, with_messages: bool = True) -> dict[str, Any]:
def _to_dict(
row: ParallelAgentTaskRow | Mapping[str, Any],
*,
with_messages: bool = True,
) -> dict[str, Any]:
def value(name: str) -> Any:
if isinstance(row, Mapping):
return row[name]
return getattr(row, name)
created_at = value("created_at")
updated_at = value("updated_at")
data: dict[str, Any] = {
"id": row.id,
"title": row.title,
"brief": row.brief,
"agents": _loads(row.agents) or [],
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
"id": value("id"),
"title": value("title"),
"brief": value("brief"),
"agents": _loads(value("agents")) or [],
"created_at": (
created_at.isoformat() if isinstance(created_at, datetime) else created_at
),
"updated_at": (
updated_at.isoformat() if isinstance(updated_at, datetime) else updated_at
),
}
if with_messages:
data["messages"] = _loads(row.messages) or []
data["messages"] = _loads(value("messages")) or []
return data
async def list_tasks(
@@ -74,9 +90,22 @@ class ParallelAgentTaskRepository:
and ``offset`` allow those projections to page only their own task
namespace without loading unrelated, potentially large transcripts.
"""
stmt = select(ParallelAgentTaskRow).where(ParallelAgentTaskRow.user_id == user_id)
# History/list callers only need the summary columns. In particular,
# never select the LONGTEXT transcript column unless explicitly asked:
# MySQL otherwise reads and transfers it even though _to_dict omits it.
columns = [
ParallelAgentTaskRow.id,
ParallelAgentTaskRow.title,
ParallelAgentTaskRow.brief,
ParallelAgentTaskRow.agents,
ParallelAgentTaskRow.created_at,
ParallelAgentTaskRow.updated_at,
]
if include_messages:
columns.append(ParallelAgentTaskRow.messages)
stmt = select(*columns).where(ParallelAgentTaskRow.user_id == user_id)
if id_prefix:
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix))
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix, autoescape=True))
stmt = (
stmt.order_by(
ParallelAgentTaskRow.updated_at.desc(),
@@ -88,8 +117,8 @@ class ParallelAgentTaskRepository:
async with self._sf() as session:
result = await session.execute(stmt)
return [
self._to_dict(row, with_messages=include_messages)
for row in result.scalars()
self._to_dict(row._mapping, with_messages=include_messages)
for row in result
]
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
@@ -7,6 +7,15 @@ from typing import Any
class PositionStore(ABC):
async def get_effective_position(self, user_id: str) -> dict[str, Any] | None:
"""Return an assigned position or the default without membership counts."""
position_id = await self.get_user_position_id(user_id)
record = await self.get_position(position_id) if position_id else None
if record is None:
default_id = await self.get_default_position_id()
record = await self.get_position(default_id) if default_id else None
return record
# -------------------------------------------------------------- positions
@abstractmethod
@@ -7,6 +7,7 @@ from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, select, update
from sqlalchemy.orm import aliased
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -60,6 +61,25 @@ class PositionRepository(PositionStore):
counts = await self._member_counts(session, [position_id])
return _position_to_dict(row, counts.get(position_id, 0))
async def get_effective_position(self, user_id: str) -> dict[str, Any] | None:
"""Resolve the assigned/default position with one narrow indexed query."""
assigned = aliased(PositionRow)
default = aliased(PositionRow)
stmt = (
select(assigned, default)
.select_from(UserRow)
.outerjoin(assigned, UserRow.position_id == assigned.id)
.outerjoin(default, default.is_default.is_(True))
.where(UserRow.id == user_id)
.limit(1)
)
async with self._sf() as session:
row = (await session.execute(stmt)).first()
if row is None:
return None
position = row[0] or row[1]
return _position_to_dict(position, 0) if position is not None else None
async def create_position(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
is_default = bool(data.get("is_default"))
@@ -75,6 +75,13 @@ class RoundtableChainRow(Base):
pending_review: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default="0", index=True
)
# 管理员目录开关:True = 从所有普通列表(我的/公共/待审核/选链弹窗)隐藏,仅管理员
# 管理视图(``admin_view=true`` 的列表请求)可见。纯目录开关:单条读取(get_chain)
# 不过滤,已配置的业务映射 / 进行中的研讨不受影响。``server_default`` 显式声明以支持
# 启动期 ``_ensure_orm_columns_sync`` 在既有表上补列(NOT NULL 需带默认)。
hidden: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default="0", index=True
)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -66,6 +66,8 @@ class RoundtableChainRepository:
"enabled": bool(getattr(row, "enabled", True)),
# 「待审核」标记:True = owner 已提交审核、等待管理员发布;发布时清回 False。
"pending_review": bool(getattr(row, "pending_review", False)),
# 管理员目录开关:True = 从普通列表隐藏(仅管理视图可见);缺省 False。
"hidden": bool(getattr(row, "hidden", False)),
"is_mine": (current_user_id is None) or (row.user_id == current_user_id),
# 配置者(owner)的 user_id;路由层据此解析成可读用户名(owner_name)。
"owner_id": row.user_id,
@@ -74,7 +76,14 @@ class RoundtableChainRepository:
}
async def list_chains(
self, user_id: str, *, scope: str = "mine", limit: int = 200, is_admin: bool = False
self,
user_id: str,
*,
scope: str = "mine",
limit: int = 200,
is_admin: bool = False,
include_hidden: bool = False,
hidden: bool | None = None,
) -> list[dict[str, Any]]:
"""列出链条。
@@ -86,6 +95,10 @@ class RoundtableChainRepository:
- ``"all"``:所有账号的全部链条(**仅管理员**,由路由层校验后传 ``is_admin=True``;
非管理员误传则按 ``"mine"`` 退化,绝不越权)。
其余值按 ``"mine"`` 处理。``is_mine`` 标记每条是否本人所有,供前端判断可否编辑。
隐藏开关(``hidden=True`` 的行):默认从**所有**列表排除(含 admin 的普通视图,如
选链弹窗);``include_hidden=True``(管理员管理视图)时不排除,并可再用 ``hidden``
参数筛选「只看隐藏 / 只看显示中」。非管理员的 ``include_hidden`` 一律忽略。
"""
if scope == "public":
cond = RoundtableChainRow.is_public.is_(True)
@@ -98,6 +111,11 @@ class RoundtableChainRepository:
stmt = select(RoundtableChainRow)
if cond is not None:
stmt = stmt.where(cond)
if include_hidden and is_admin:
if hidden is not None:
stmt = stmt.where(RoundtableChainRow.hidden.is_(hidden))
else:
stmt = stmt.where(RoundtableChainRow.hidden.is_(False))
stmt = stmt.order_by(RoundtableChainRow.updated_at.desc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
@@ -157,6 +175,7 @@ class RoundtableChainRepository:
is_public=_UNSET,
enabled=_UNSET,
pending_review=_UNSET,
hidden=_UNSET,
is_admin: bool = False,
) -> dict[str, Any] | None:
async with self._sf() as session:
@@ -186,6 +205,8 @@ class RoundtableChainRepository:
row.enabled = bool(enabled)
if pending_review is not _UNSET and pending_review is not None:
row.pending_review = bool(pending_review)
if hidden is not _UNSET and hidden is not None:
row.hidden = bool(hidden)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -68,7 +68,7 @@ class RoundtableJobRow(Base):
# Fine-grained phase (§2.2): initializing / leader_thinking / dispatching /
# consensus / report_drawing / report_done.
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
# Current orchestration cycle (0..MAX_CYCLES).
# Current orchestration cycle (0..configured roundtable max_cycles).
cycle: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# Dispatch chain nodes (§2.3): JSON array of
# { agentId, name, avatarType, order, state, lastSpokeCycle? }.
@@ -119,6 +119,7 @@ class RoundtableJobRepository:
request_hash: str | None = None,
active_dedupe_key: str | None = None,
input_snapshot: dict[str, Any] | None = None,
room_started_at: datetime | None = None,
) -> RoundtableJobRow:
now = datetime.now(UTC)
return RoundtableJobRow(
@@ -145,7 +146,7 @@ class RoundtableJobRepository:
error=None,
attempt=0,
version=0,
created_at=now,
created_at=room_started_at or now,
updated_at=now,
)
@@ -204,6 +205,7 @@ class RoundtableJobRepository:
request_id: str | None = None,
request_hash: str | None = None,
input_snapshot: dict[str, Any] | None = None,
room_started_at: datetime | None = None,
) -> tuple[dict[str, Any], bool]:
"""原子地「创建活跃作业 → 冲突则幂等返回已有活跃作业」(Phase 3 唯一活跃键版)。
@@ -240,6 +242,7 @@ class RoundtableJobRepository:
request_hash=request_hash,
active_dedupe_key=dedupe_key,
input_snapshot=input_snapshot,
room_started_at=room_started_at,
)
# 无 draft_id → 无法去重,直接创建(保持原有行为)。
@@ -462,6 +465,30 @@ class RoundtableJobRepository:
result = await session.execute(stmt)
return [r[0] for r in result.all()]
async def request_cancel_expired_step2(
self,
*,
created_before: datetime,
non_occupying_phases: tuple[str, ...],
) -> int:
"""Cancel unfinished Step-2 jobs that exceeded the room hard lifetime."""
async with self._sf() as session:
result = await session.execute(
update(RoundtableJobRow)
.where(
RoundtableJobRow.status.in_(list(CANCELLABLE_STATUSES)),
RoundtableJobRow.created_at <= created_before,
RoundtableJobRow.phase.not_in(list(non_occupying_phases)),
)
.values(
status="cancel_requested",
version=RoundtableJobRow.version + 1,
updated_at=datetime.now(UTC),
)
)
await session.commit()
return int(result.rowcount or 0)
async def list_cancel_reapable(self, *, now: datetime, limit: int = 20) -> list[str]:
"""列出可以由 dispatcher 收口的取消请求。
@@ -0,0 +1,19 @@
"""Roundtable step-2 room leases."""
from __future__ import annotations
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
from deerflow.persistence.roundtable_rooms.sql import RoundtableRoomRepository
def make_roundtable_room_store(session_factory) -> RoundtableRoomRepository | None:
if session_factory is None:
return None
return RoundtableRoomRepository(session_factory)
__all__ = [
"RoundtableRoomLeaseRow",
"RoundtableRoomRepository",
"make_roundtable_room_store",
]
@@ -0,0 +1,33 @@
"""Foreground roundtable room leases.
One row is one browser session that has entered step 2 and is still heartbeating.
Background jobs are not stored here: they already occupy a room through
``roundtable_jobs`` while their status is non-terminal. Both sides share the
job active-dedupe key so a foreground session and its later background job
count as one room.
"""
from __future__ import annotations
from datetime import datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class RoundtableRoomLeaseRow(Base):
__tablename__ = "roundtable_room_leases"
room_key: Mapped[str] = mapped_column(String(64), primary_key=True)
draft_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
task_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# Last account to heartbeat. Release only succeeds for this account, so one
# viewer leaving a shared task room does not drop another viewer's lease.
holder_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
@@ -0,0 +1,584 @@
"""Deployment-wide roundtable room leases and background-job occupancy.
A room is one live roundtable execution, not one saved draft, not one account,
and not one seat. Different pages running the same draft have distinct hidden
session ids; every seat and step of one execution shares one id. The active set
is the union of every user's:
- a durable foreground/session lease renewed by the browser or a run entry
- live ``extra_keys`` supplied by the Gateway as a reconciliation fallback
only when no durable lease/job already represents that persisted scope
- a background job that is executing or waiting for the user's Step-2
clarification (``queued`` / ``running`` / ``awaiting_input`` /
``cancel_requested``)
Historical jobs older than the hard room lifetime and merely viewing chat
history are not rooms. There is no per-user pool.
Admission and lease creation happen while holding one database-backed gate
row. Updating that row serializes contenders on SQLite, MySQL and PostgreSQL,
so four simultaneous requests cannot all observe the same stale count.
"""
from __future__ import annotations
import asyncio
import hashlib
from collections.abc import Mapping
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
from deerflow.persistence.roundtable_jobs.sql import compute_active_dedupe_key
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
LEASE_SECONDS = 60
JOB_LIVE_SECONDS = 15 * 60
OCCUPYING_JOB_STATUSES = ("queued", "running", "awaiting_input", "cancel_requested")
# A background job keeps running while Step 3 generates its artifacts, but the
# concurrency cap is intentionally scoped to Step 2. ``consensus`` is written
# before Step 3 begins, which releases the slot without waiting for reporting.
NON_OCCUPYING_JOB_PHASES = frozenset({
"consensus",
"summary_writing",
"report_drawing",
"report_done",
})
GATE_ROW_KEY = "__roundtable_room_gate__"
def compute_room_session_key(room_session_id: str | None) -> str | None:
"""Stable key for one live roundtable execution (not one saved draft)."""
clean = (room_session_id or "").strip()
if not clean:
return None
return hashlib.sha256(f"roundtable-session:{clean}".encode()).hexdigest()
def _aware(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value
def _job_is_live(
lease_until: datetime | None,
updated_at: datetime | None,
now: datetime,
live_after: datetime,
) -> bool:
until = _aware(lease_until)
if until is not None and until > now:
return True
updated = _aware(updated_at)
return updated is not None and updated >= live_after
def _hard_deadline_reached(
created_at: datetime | None,
now: datetime,
max_hold_minutes: int,
) -> bool:
created = _aware(created_at)
return created is not None and created + timedelta(minutes=max_hold_minutes) <= now
class RoundtableRoomRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
self._lock = asyncio.Lock()
async def admit(
self,
*,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
room_session_id: str | None = None,
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Atomically reserve/renew a room, or reject when the pool is full."""
key = compute_room_session_key(room_session_id) or compute_active_dedupe_key(
draft_id, user_id, task_id
)
return await self.admit_room_key(
room_key=key,
draft_id=draft_id,
user_id=user_id,
task_id=task_id,
holder_user_id=user_id,
enabled=enabled,
max_rooms=max_rooms,
max_hold_minutes=max_hold_minutes,
extra_keys=extra_keys,
)
async def admit_room_key(
self,
*,
room_key: str | None,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
holder_user_id: str | None,
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Atomically admit a pre-resolved room key.
``room_key`` is used by ``/run/stream`` when it resolves the room from
thread metadata. ``user_id`` remains the room owner while
``holder_user_id`` is the account currently renewing a shared room.
"""
async with self._lock:
await self._ensure_gate_row()
now = datetime.now(UTC)
async with self._sf() as session:
# This no-op-ish write is intentional: it is the portable DB
# mutex protecting the subsequent count + lease upsert.
await session.execute(
update(RoundtableRoomLeaseRow)
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
.values(updated_at=now)
)
keys, durable_scopes, hard_expired_keys = await self._active_snapshot(
session,
now,
max_hold_minutes=max_hold_minutes,
)
self._merge_extra_keys(keys, durable_scopes, extra_keys)
if room_key is None:
await session.commit()
blocked = bool(enabled)
return self._decision(
ok=not blocked,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=False,
code="DRAFT_REQUIRED" if blocked else "",
message="" if not blocked else "进入第二步需要先保存会商草稿",
)
if room_key in hard_expired_keys:
await session.commit()
return self._decision(
ok=False,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=False,
code="ROOM_MAX_HOLD_EXCEEDED",
message="本次会商已达到最长占用时间,房间已自动释放。请返回第一步后重新发起。",
)
owns = room_key in keys
if enabled and not owns and len(keys) >= max_rooms:
await session.commit()
return self._decision(
ok=False,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=False,
code="ROUNDTABLE_ROOM_FULL",
message="当前会商房间暂时已满,请稍候片刻后再试。",
)
await self._upsert_lease(
session,
room_key=room_key,
draft_id=draft_id,
user_id=user_id,
task_id=task_id,
holder_user_id=holder_user_id,
now=now,
)
await session.commit()
return self._decision(
ok=True,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys) if owns else len(keys) + 1,
renewed=owns,
code="",
message="",
)
async def release(
self,
*,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
room_session_id: str | None = None,
) -> bool:
"""Drop a leftover foreground lease row if one still exists."""
key = compute_room_session_key(room_session_id) or compute_active_dedupe_key(
draft_id, user_id, task_id
)
if not key:
return False
async with self._lock:
await self._ensure_gate_row()
async with self._sf() as session:
await session.execute(
update(RoundtableRoomLeaseRow)
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
.values(updated_at=datetime.now(UTC))
)
row = await session.get(RoundtableRoomLeaseRow, key)
if row is None or (user_id and row.holder_user_id not in (None, user_id)):
await session.commit()
return False
await session.delete(row)
await session.commit()
return True
async def started_at(
self,
*,
room_session_id: str | None,
holder_user_id: str | None,
) -> datetime | None:
"""Return the original admission time for a foreground-to-job handoff."""
key = compute_room_session_key(room_session_id)
if key is None:
return None
async with self._sf() as session:
row = await session.get(RoundtableRoomLeaseRow, key)
if row is None or (
holder_user_id
and row.holder_user_id not in (None, holder_user_id)
):
return None
return _aware(row.created_at)
async def renew(
self,
*,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
room_session_id: str | None,
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Renew one live foreground lease without ever creating it.
Heartbeats are deliberately weaker than admission. A delayed browser
callback must not resurrect a room that was released after a pause,
network disconnect or completed Step 2.
"""
key = compute_room_session_key(room_session_id)
if key is None:
return self._decision(
ok=not enabled,
enabled=enabled,
max_rooms=max_rooms,
active_count=0,
renewed=False,
code="ROOM_SESSION_REQUIRED" if enabled else "",
message="会商会话标识缺失,请刷新页面后重试。" if enabled else "",
)
return await self.renew_room_key(
room_key=key,
holder_user_id=user_id,
enabled=enabled,
max_rooms=max_rooms,
max_hold_minutes=max_hold_minutes,
extra_keys=extra_keys,
)
async def renew_room_key(
self,
*,
room_key: str,
holder_user_id: str | None,
enabled: bool,
max_rooms: int,
max_hold_minutes: int = 30,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
) -> dict[str, Any]:
"""Renew a resolved persisted key without creating a missing lease."""
async with self._lock:
await self._ensure_gate_row()
now = datetime.now(UTC)
async with self._sf() as session:
await session.execute(
update(RoundtableRoomLeaseRow)
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
.values(updated_at=now)
)
keys, durable_scopes, hard_expired_keys = await self._active_snapshot(
session,
now,
max_hold_minutes=max_hold_minutes,
)
self._merge_extra_keys(keys, durable_scopes, extra_keys)
row = await session.get(RoundtableRoomLeaseRow, room_key)
if row is not None and room_key in hard_expired_keys:
await session.commit()
return self._decision(
ok=False,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=False,
code="ROOM_MAX_HOLD_EXCEEDED",
message="本次会商已达到最长占用时间,房间已自动释放。请返回第一步后重新发起。",
)
owns = row is not None and room_key in keys and (
not holder_user_id or row.holder_user_id in (None, holder_user_id)
)
if not owns:
await session.commit()
return self._decision(
ok=not enabled,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=False,
code="ROOM_LEASE_EXPIRED" if enabled else "",
message="会商连接已断开,请重新恢复研讨。" if enabled else "",
)
row.lease_until = now + timedelta(seconds=LEASE_SECONDS)
row.updated_at = now
await session.commit()
return self._decision(
ok=True,
enabled=enabled,
max_rooms=max_rooms,
active_count=len(keys),
renewed=True,
code="",
message="",
)
async def summary(
self,
extra_keys: set[str] | Mapping[str, str | None] | None = None,
*,
max_hold_minutes: int = 30,
) -> int:
"""Return the number of distinct occupied rooms across every user."""
async with self._lock:
await self._ensure_gate_row()
now = datetime.now(UTC)
async with self._sf() as session:
keys, durable_scopes, _hard_expired_keys = await self._active_snapshot(
session,
now,
max_hold_minutes=max_hold_minutes,
)
self._merge_extra_keys(keys, durable_scopes, extra_keys)
await session.commit()
return len(keys)
@staticmethod
def _merge_extra_keys(
keys: set[str],
durable_scopes: set[str],
extra_keys: set[str] | Mapping[str, str | None] | None,
) -> None:
"""Add run-manager fallbacks without double-counting durable occupancy.
The map value is the legacy draft/user-or-task scope. If a live page
lease or background job already owns that scope, its stamped run key is
merely evidence for the same room and must not become another room.
"""
if not extra_keys:
return
items = extra_keys.items() if isinstance(extra_keys, Mapping) else ((key, None) for key in extra_keys)
for key, scope_key in items:
if not key or (scope_key and scope_key in durable_scopes):
continue
keys.add(key)
async def _active_snapshot(
self,
session: AsyncSession,
now: datetime,
*,
max_hold_minutes: int,
) -> tuple[set[str], set[str], set[str]]:
lease_rows = (
await session.execute(
select(RoundtableRoomLeaseRow).where(
RoundtableRoomLeaseRow.room_key != GATE_ROW_KEY
)
)
).scalars().all()
expired: list[str] = []
keys: set[str] = set()
durable_scopes: set[str] = set()
hard_expired_keys: set[str] = set()
for row in lease_rows:
until = _aware(row.lease_until)
if until is None or until <= now:
expired.append(row.room_key)
continue
scope_key = compute_active_dedupe_key(row.draft_id, row.user_id, row.task_id)
if _hard_deadline_reached(row.created_at, now, max_hold_minutes):
# Keep the still-live row briefly as a non-occupying tombstone.
# Its scope suppresses a lagging RunManager fallback until the
# browser observes the rejected heartbeat and cancels the run.
hard_expired_keys.add(row.room_key)
if scope_key:
durable_scopes.add(scope_key)
continue
keys.add(row.room_key)
if scope_key:
durable_scopes.add(scope_key)
if expired:
await session.execute(
delete(RoundtableRoomLeaseRow).where(
RoundtableRoomLeaseRow.room_key.in_(expired)
)
)
live_after = now - timedelta(seconds=JOB_LIVE_SECONDS)
job_rows = (
await session.execute(
select(
RoundtableJobRow.id,
RoundtableJobRow.draft_id,
RoundtableJobRow.user_id,
RoundtableJobRow.task_id,
RoundtableJobRow.status,
RoundtableJobRow.phase,
RoundtableJobRow.lease_until,
RoundtableJobRow.updated_at,
RoundtableJobRow.created_at,
).where(RoundtableJobRow.status.in_(OCCUPYING_JOB_STATUSES))
)
).all()
for job_id, draft_id, user_id, task_id, status, phase, lease_until, updated_at, created_at in job_rows:
if phase in NON_OCCUPYING_JOB_PHASES:
continue
if _hard_deadline_reached(created_at, now, max_hold_minutes):
# The worker cancellation watcher will finalize this state.
# Marking the intent here also covers queued jobs and workers
# that stopped renewing but have not yet been reaped.
await session.execute(
update(RoundtableJobRow)
.where(
RoundtableJobRow.id == job_id,
RoundtableJobRow.status.in_(["queued", "running", "awaiting_input"]),
)
.values(
status="cancel_requested",
version=RoundtableJobRow.version + 1,
updated_at=now,
)
)
continue
if status != "awaiting_input" and not _job_is_live(
lease_until,
updated_at,
now,
live_after,
):
continue
job_key = compute_active_dedupe_key(draft_id, user_id, task_id)
if job_key and job_key in durable_scopes:
# Foreground → background handoff can overlap for a few
# milliseconds. Both represent one room for the same scope.
continue
if not job_key:
job_key = f"job:{job_id}"
keys.add(job_key)
scope_key = compute_active_dedupe_key(draft_id, user_id, task_id)
if scope_key:
durable_scopes.add(scope_key)
return keys, durable_scopes, hard_expired_keys
async def _ensure_gate_row(self) -> None:
"""Create the portable database mutex row once, tolerating races."""
now = datetime.now(UTC)
async with self._sf() as session:
if await session.get(RoundtableRoomLeaseRow, GATE_ROW_KEY) is not None:
return
session.add(
RoundtableRoomLeaseRow(
room_key=GATE_ROW_KEY,
draft_id=None,
user_id=None,
task_id=None,
holder_user_id=None,
lease_until=None,
created_at=now,
updated_at=now,
)
)
try:
await session.commit()
except IntegrityError:
await session.rollback()
async def _upsert_lease(
self,
session: AsyncSession,
*,
room_key: str,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
holder_user_id: str | None,
now: datetime,
) -> None:
row = await session.get(RoundtableRoomLeaseRow, room_key)
lease_until = now + timedelta(seconds=LEASE_SECONDS)
if row is None:
session.add(
RoundtableRoomLeaseRow(
room_key=room_key,
draft_id=draft_id,
user_id=user_id,
task_id=task_id,
holder_user_id=holder_user_id,
lease_until=lease_until,
created_at=now,
updated_at=now,
)
)
return
row.draft_id = draft_id
row.user_id = user_id
row.task_id = task_id
row.holder_user_id = holder_user_id
row.lease_until = lease_until
row.updated_at = now
@staticmethod
def _decision(
*,
ok: bool,
enabled: bool,
max_rooms: int,
active_count: int,
renewed: bool,
code: str,
message: str,
) -> dict[str, Any]:
return {
"ok": ok,
"enabled": enabled,
"max_rooms": max_rooms,
"active_count": active_count,
"renewed": renewed,
"code": code,
"message": message,
}
@@ -11,13 +11,15 @@ import json
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy import delete, func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.run.model import RunRow
from deerflow.runtime.runs.store.base import RunStore
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
_TERMINAL_RUN_STATUSES = ("success", "error", "timeout", "interrupted")
class RunRepository(RunStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
@@ -137,6 +139,14 @@ class RunRepository(RunStore):
await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(**values))
await session.commit()
async def update_snapshot(self, run_id, *, metadata, kwargs):
# Internal RunManager retention path; the run identity comes from its
# live registry, never a client-supplied mutation endpoint.
async with self._sf() as session:
result = await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(metadata_json=self._safe_json(metadata), kwargs_json=self._safe_json(kwargs)))
await session.commit()
return bool(result.rowcount)
async def delete(
self,
run_id,
@@ -205,21 +215,70 @@ class RunRepository(RunStore):
await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(**values))
await session.commit()
async def count_by_user(self) -> dict[str, int]:
"""Return ``{user_id: run_count}`` via a single GROUP BY query.
async def count_by_user(self, *, since: datetime | None = None) -> dict[str, int]:
"""Return ``{user_id: run_count}`` via one bounded GROUP BY query.
Each run is one user question, so this is the source for the admin
user center's 问答次数 column. Runs without a ``user_id`` are skipped.
"""
filters = [RunRow.user_id.is_not(None)]
if since is not None:
since_dt = since if since.tzinfo else since.replace(tzinfo=UTC)
filters.append(RunRow.created_at >= since_dt)
stmt = (
select(RunRow.user_id, func.count().label("cnt"))
.where(RunRow.user_id.is_not(None))
.where(*filters)
.group_by(RunRow.user_id)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
return {r.user_id: r.cnt for r in rows}
async def purge_terminal_before(
self,
cutoff: datetime,
*,
batch_size: int = 500,
) -> int:
"""Delete one portable, primary-key-bounded batch of old run audits.
Pending and running records are intentionally excluded, so retention
never removes work that could still be resumed or observed. A run must
also have been *updated* before the cutoff: an unusually long-running
job created months ago but completed today must remain visible. The
compact run row is cleaned, while the conversation checkpoint stays
under its separate user-history lifecycle.
"""
size = max(1, min(int(batch_size), 500))
cutoff_dt = cutoff if cutoff.tzinfo else cutoff.replace(tzinfo=UTC)
async with self._sf() as session:
ids = list(
(
await session.scalars(
select(RunRow.run_id)
.where(
RunRow.created_at < cutoff_dt,
RunRow.updated_at < cutoff_dt,
RunRow.status.in_(_TERMINAL_RUN_STATUSES),
)
.order_by(RunRow.created_at.asc(), RunRow.run_id.asc())
.limit(size)
)
).all()
)
if not ids:
return 0
result = await session.execute(
delete(RunRow).where(
RunRow.run_id.in_(ids),
RunRow.created_at < cutoff_dt,
RunRow.updated_at < cutoff_dt,
RunRow.status.in_(_TERMINAL_RUN_STATUSES),
)
)
await session.commit()
return int(result.rowcount or 0)
async def aggregate_tokens_by_thread(self, thread_id: str) -> dict[str, Any]:
"""Aggregate token usage via a single SQL GROUP BY query."""
_completed = RunRow.status.in_(("success", "error"))
@@ -95,6 +95,15 @@ class SkillStore(ABC):
"""
raise NotImplementedError
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
"""Map skill names to their configured Chinese display names.
Only names that have a non-empty ``name_zh`` are returned. The default
is empty so stores that have not implemented the lookup stay safe.
"""
del names
return {}
@abstractmethod
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
"""Return the skill names in ``user_id``'s "我的技能" collection.
@@ -165,6 +165,17 @@ class MemorySkillStore(SkillStore):
async def list_always_on_names(self) -> set[str]:
return {name for name, record in self._skills.items() if record.get("always_on")}
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
labels: dict[str, str] = {}
for name in names:
record = self._skills.get(name)
if record is None:
continue
label = str(record.get("name_zh") or "").strip()
if label:
labels[name] = label
return labels
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
return set(self._favorites.get(user_id, set()))
@@ -10,7 +10,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, String, Text, UniqueConstraint
from sqlalchemy import BigInteger, Boolean, DateTime, Index, Integer, String, Text, UniqueConstraint, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -19,10 +19,30 @@ from deerflow.persistence.types import BeijingDateTime
class SkillRow(Base):
__tablename__ = "skills"
__table_args__ = (
Index("ix_skills_catalog_available_category_created", "available", "category", "created_at"),
)
# Global unique name — must match the on-disk folder name under skills/custom/.
# Hyphen-case or snake_case allowed; see deerflow.skills.validation.
name: Mapped[str] = mapped_column(String(64), primary_key=True)
# Compact catalog projection fields. The list API reads these columns from
# MySQL and deliberately excludes the long-form ``detail`` text column.
category: Mapped[str] = mapped_column(String(16), nullable=False, default="custom", server_default="custom")
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="", server_default="")
license: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
available: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"), index=True)
use_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
view_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
patch_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
last_activity_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, default=None)
state: Mapped[str] = mapped_column(String(16), nullable=False, default="active", server_default="active")
pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
featured: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
source: Mapped[str] = mapped_column(String(16), nullable=False, default="upload", server_default="upload")
source_mtime_ns: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default=text("0"))
detail_preview: Mapped[str | None] = mapped_column(String(1024), nullable=True, default=None)
owner_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
# Human-authored long-form detail/introduction, editable by the skill owner
@@ -83,3 +103,18 @@ class SkillDisplayAdapterRow(Base):
sample_payload: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class SkillCatalogSnapshotRow(Base):
"""MySQL-backed projection of filesystem skill metadata for fast list APIs.
Skill source files remain the authoring format. The Gateway refreshes this
one-row projection during startup so homepage listing requests do not scan
every SKILL.md or read per-skill usage sidecars.
"""
__tablename__ = "skill_catalog_snapshots"
cache_key: Mapped[str] = mapped_column(String(32), primary_key=True)
catalog_json: Mapped[str] = mapped_column(Text, nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -109,6 +109,7 @@ class SkillRepository(SkillStore):
row.published = bool(data["published"])
if "detail" in data:
row.detail = data["detail"]
row.detail_preview = (str(data["detail"] or "")[:1024] or None)
if "name_zh" in data:
row.name_zh = data["name_zh"]
if "always_on" in data:
@@ -131,6 +132,7 @@ class SkillRepository(SkillStore):
row.owner_user_id = data["owner_user_id"]
if "detail" in data:
row.detail = data["detail"]
row.detail_preview = (str(data["detail"] or "")[:1024] or None)
if "name_zh" in data:
row.name_zh = data["name_zh"]
if "always_on" in data:
@@ -183,6 +185,20 @@ class SkillRepository(SkillStore):
result = await session.execute(stmt)
return {row[0] for row in result.all()}
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
wanted = list(dict.fromkeys(name for name in names if name))
if not wanted:
return {}
stmt = select(SkillRow.name, SkillRow.name_zh).where(SkillRow.name.in_(wanted))
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
labels: dict[str, str] = {}
for name, name_zh in rows:
label = (name_zh or "").strip()
if label:
labels[name] = label
return labels
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
stmt = select(SkillFavoriteRow.skill_name).where(SkillFavoriteRow.user_id == user_id)
async with self._sf() as session:
@@ -0,0 +1,104 @@
"""Persistence for model-stream truncation events (模型截断记录).
The patched OpenAI-compatible streaming layer calls
:func:`record_stream_truncation_event` whenever a provider stream ends
abnormally. The write is fire-and-forget: it is routed onto the engine's home
loop via :func:`fire_and_forget_db_write` and must never disturb the model
stream itself. Rows are capped at one month by the SQL repository.
"""
from __future__ import annotations
import logging
import uuid
from datetime import UTC, datetime
from deerflow.persistence.stream_truncation.base import (
RETENTION_DAYS,
StreamTruncationEvent,
StreamTruncationQuery,
StreamTruncationStore,
)
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
from deerflow.persistence.stream_truncation.sql import SqlStreamTruncationStore
__all__ = [
"RETENTION_DAYS",
"SqlStreamTruncationStore",
"StreamTruncationEvent",
"StreamTruncationEventRow",
"StreamTruncationQuery",
"StreamTruncationStore",
"make_stream_truncation_store",
"record_stream_truncation_event",
]
logger = logging.getLogger(__name__)
_store_cache: SqlStreamTruncationStore | None = None
_store_cache_failed = False
def make_stream_truncation_store(session_factory) -> StreamTruncationStore | None:
"""Return a SQL-backed truncation store, or ``None`` without a DB."""
if session_factory is None:
return None
return SqlStreamTruncationStore(session_factory)
def record_stream_truncation_event(
*,
model_name: str | None,
event_type: str,
strict_mode: bool = False,
finish_reason: str = "none",
status: str = "success",
error_type: str | None = None,
chunk_count: int = 0,
duration_ms: int = 0,
first_chunk_ms: int = -1,
provider_completion_id: str | None = None,
run_id: str | None = None,
detail: str | None = None,
) -> None:
"""Best-effort truncation-event write from any loop/thread. Never raises."""
global _store_cache, _store_cache_failed
if _store_cache is None:
if _store_cache_failed:
return
try:
from deerflow.persistence.engine import get_session_factory
session_factory = get_session_factory()
except Exception:
session_factory = None
if session_factory is None:
# No DB configured (memory backend); do not retry on every stream.
_store_cache_failed = True
return
_store_cache = SqlStreamTruncationStore(session_factory)
run_id_value = run_id if run_id and run_id not in {"none", ""} else None
event = StreamTruncationEvent(
id=uuid.uuid4().hex,
created_at=datetime.now(UTC),
model_name=model_name,
provider_completion_id=provider_completion_id
if provider_completion_id and provider_completion_id != "none"
else None,
run_id=run_id_value,
event_type=event_type,
strict_mode=strict_mode,
finish_reason=finish_reason,
status=status,
error_type=error_type,
chunk_count=chunk_count,
duration_ms=duration_ms,
first_chunk_ms=first_chunk_ms,
detail=detail,
)
try:
from deerflow.persistence.engine import fire_and_forget_db_write
fire_and_forget_db_write(_store_cache.record(event))
except Exception:
logger.debug("stream truncation event drop", exc_info=True)
@@ -0,0 +1,76 @@
"""Repository contract for model-stream truncation events."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any
# Hard retention cap. The user-facing promise is "最多存一个月" of truncation
# records; the SQL repository enforces it on every insert so no scheduler is
# required for the cap to hold.
RETENTION_DAYS = 30
@dataclass
class StreamTruncationEvent:
"""In-memory shape of a truncation event before persistence."""
id: str
created_at: datetime
model_name: str | None = None
provider_completion_id: str | None = None
run_id: str | None = None
# silent_no_terminal / incomplete_stream_error / length_truncated /
# content_filter / midstream_error
event_type: str = "silent_no_terminal"
strict_mode: bool = False
finish_reason: str = "none"
status: str = "success"
error_type: str | None = None
chunk_count: int = 0
duration_ms: int = 0
first_chunk_ms: int = -1
detail: str | None = None
@dataclass
class StreamTruncationQuery:
"""Filter set for the admin list endpoint."""
model_name: str | None = None
event_type: str | None = None
since: datetime | None = None
until: datetime | None = None
limit: int = 100
offset: int = 0
class StreamTruncationStore(ABC):
"""Persistence contract for the model-layer recorder + admin API."""
@abstractmethod
async def record(self, event: StreamTruncationEvent) -> None:
"""Persist one event and enforce the retention cap. Must never raise."""
raise NotImplementedError
@abstractmethod
async def list(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
"""Return matching rows (newest first), paginated by ``query``."""
raise NotImplementedError
@abstractmethod
async def count(self, query: StreamTruncationQuery) -> int:
"""Return the unpaginated row count for ``query``."""
raise NotImplementedError
@abstractmethod
async def model_breakdown(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
"""Per-model aggregate: ``[{model_name, event_type counts..., total, last_at}]``."""
raise NotImplementedError
@abstractmethod
async def purge_before(self, cutoff: datetime) -> int:
"""Delete rows older than ``cutoff``; return the deleted row count."""
raise NotImplementedError
@@ -0,0 +1,80 @@
"""ORM model for model-stream truncation events."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Index, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class StreamTruncationEventRow(Base):
"""One row per detected model-stream truncation.
Written from the patched OpenAI-compatible streaming layer when a provider
stream ends abnormally: without a terminal frame (silently committed half
answer), via ``IncompleteChatStreamError`` (strict integrity mode, retried),
on an output-length cap, or interrupted mid-stream by a transport error.
Rows are retained for at most ``RETENTION_DAYS`` by the SQL repository.
"""
__tablename__ = "stream_truncation_events"
__table_args__ = (
Index("ix_stream_truncation_created_model", "created_at", "model_name"),
Index("ix_stream_truncation_created_event", "created_at", "event_type"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), index=True
)
# Configured model name (the config.yaml entry name), sanitized by the model layer.
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
# Provider-side completion id when the gateway supplied one; joins a row to
# the upstream gateway's own request log.
provider_completion_id: Mapped[str | None] = mapped_column(String(128))
# LangGraph run id when the model layer could see one ("none" is dropped).
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
# silent_no_terminal / incomplete_stream_error / length_truncated /
# content_filter / midstream_error
event_type: Mapped[str] = mapped_column(String(32), index=True)
# Whether the model ran with strict_stream_integrity enabled.
strict_mode: Mapped[bool] = mapped_column(Boolean, default=False)
# Raw provider finish reason bucket: none / stop / length / tool_calls /
# function_call / content_filter / end_turn / other.
finish_reason: Mapped[str] = mapped_column(String(16), default="none")
# Stream outcome from the model layer: success / error / cancelled.
status: Mapped[str] = mapped_column(String(16), default="success")
error_type: Mapped[str | None] = mapped_column(String(64))
chunk_count: Mapped[int] = mapped_column(Integer, default=0)
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
first_chunk_ms: Mapped[int] = mapped_column(Integer, default=-1)
detail: Mapped[str | None] = mapped_column(Text)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": self.created_at.isoformat()
if isinstance(self.created_at, datetime)
else self.created_at,
"model_name": self.model_name,
"provider_completion_id": self.provider_completion_id,
"run_id": self.run_id,
"event_type": self.event_type,
"strict_mode": bool(self.strict_mode),
"finish_reason": self.finish_reason,
"status": self.status,
"error_type": self.error_type,
"chunk_count": self.chunk_count,
"duration_ms": self.duration_ms,
"first_chunk_ms": self.first_chunk_ms,
"detail": self.detail,
}
@@ -0,0 +1,175 @@
"""SQLAlchemy-backed stream-truncation event repository."""
from __future__ import annotations
import logging
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, desc, func, select
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.stream_truncation.base import (
RETENTION_DAYS,
StreamTruncationEvent,
StreamTruncationQuery,
StreamTruncationStore,
)
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
logger = logging.getLogger(__name__)
class SqlStreamTruncationStore(StreamTruncationStore):
"""Persist truncation events with a hard one-month retention cap."""
def __init__(self, session_factory: async_sessionmaker) -> None:
self._session_factory = session_factory
async def _ensure_table(self, session) -> None:
# Table creation follows the repo's create_all-on-first-use convention
# (see e.g. task_buttons); safe to call repeatedly.
try:
await session.run_sync(
lambda sync_session: StreamTruncationEventRow.__table__.create(
sync_session.bind, checkfirst=True
)
)
except Exception:
logger.debug("stream_truncation table create skipped", exc_info=True)
async def record(self, event: StreamTruncationEvent) -> None:
try:
async with self._session_factory() as session:
await self._ensure_table(session)
session.add(
StreamTruncationEventRow(
id=event.id,
created_at=event.created_at,
model_name=event.model_name,
provider_completion_id=event.provider_completion_id,
run_id=event.run_id,
event_type=event.event_type,
strict_mode=event.strict_mode,
finish_reason=event.finish_reason,
status=event.status,
error_type=event.error_type,
chunk_count=event.chunk_count,
duration_ms=event.duration_ms,
first_chunk_ms=event.first_chunk_ms,
detail=event.detail,
)
)
await session.commit()
# Same session/transaction scope: enforce the retention cap on
# every insert. Truncations are rare, so this stays cheap and
# keeps the one-month promise without any scheduler.
await self._enforce_retention(session)
except Exception:
logger.warning(
"Failed to persist stream truncation event model=%s type=%s",
event.model_name,
event.event_type,
exc_info=True,
)
async def _enforce_retention(self, session) -> None:
"""Delete rows beyond the retention cap. Best-effort, never raises."""
try:
cutoff = datetime.now(UTC) - timedelta(days=RETENTION_DAYS)
await session.execute(
delete(StreamTruncationEventRow).where(
StreamTruncationEventRow.created_at < cutoff
)
)
await session.commit()
except Exception:
logger.debug("stream truncation retention sweep failed", exc_info=True)
def _apply_filters(self, query: StreamTruncationQuery):
stmt = select(StreamTruncationEventRow)
if query.model_name:
stmt = stmt.where(StreamTruncationEventRow.model_name == query.model_name)
if query.event_type:
stmt = stmt.where(StreamTruncationEventRow.event_type == query.event_type)
if query.since is not None:
stmt = stmt.where(StreamTruncationEventRow.created_at >= query.since)
if query.until is not None:
stmt = stmt.where(StreamTruncationEventRow.created_at <= query.until)
return stmt
async def list(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
try:
async with self._session_factory() as session:
await self._ensure_table(session)
stmt = self._apply_filters(query).order_by(
desc(StreamTruncationEventRow.created_at)
).limit(query.limit).offset(query.offset)
rows = (await session.execute(stmt)).scalars().all()
return [row.to_dict() for row in rows]
except Exception:
logger.warning("Failed to list stream truncation events", exc_info=True)
return []
async def count(self, query: StreamTruncationQuery) -> int:
try:
async with self._session_factory() as session:
await self._ensure_table(session)
stmt = self._apply_filters(query).with_only_columns(
func.count(StreamTruncationEventRow.id)
)
return int((await session.execute(stmt)).scalar() or 0)
except Exception:
logger.warning("Failed to count stream truncation events", exc_info=True)
return 0
async def model_breakdown(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
try:
async with self._session_factory() as session:
await self._ensure_table(session)
stmt = self._apply_filters(query).with_only_columns(
StreamTruncationEventRow.model_name,
StreamTruncationEventRow.event_type,
func.count(StreamTruncationEventRow.id).label("total"),
func.max(StreamTruncationEventRow.created_at).label("last_at"),
).group_by(StreamTruncationEventRow.model_name, StreamTruncationEventRow.event_type)
rows = (await session.execute(stmt)).all()
merged: dict[str, dict[str, Any]] = {}
for row in rows:
name = row.model_name or "(unknown)"
bucket = merged.setdefault(
name, {"model_name": name, "total": 0, "last_at": None}
)
bucket[row.event_type] = int(row.total or 0)
bucket["total"] += int(row.total or 0)
row_last = row.last_at
if row_last is not None and (
bucket["last_at"] is None or row_last > bucket["last_at"]
):
bucket["last_at"] = row_last
result = sorted(
merged.values(), key=lambda item: item["last_at"] or datetime.min.replace(tzinfo=UTC), reverse=True
)
for item in result:
if isinstance(item["last_at"], datetime):
item["last_at"] = item["last_at"].isoformat()
return result
except Exception:
logger.warning("Failed to aggregate stream truncation events", exc_info=True)
return []
async def purge_before(self, cutoff: datetime) -> int:
try:
async with self._session_factory() as session:
await self._ensure_table(session)
result = await session.execute(
delete(StreamTruncationEventRow).where(
StreamTruncationEventRow.created_at < cutoff
)
)
await session.commit()
rowcount = result.rowcount
return 0 if rowcount is None else int(rowcount)
except Exception:
logger.warning("Failed to purge stream truncation events", exc_info=True)
return 0
@@ -14,7 +14,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, UniqueConstraint
from sqlalchemy import Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -43,7 +43,14 @@ class TagRow(Base):
class TagAssignmentRow(Base):
__tablename__ = "tag_assignments"
__table_args__ = (UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),)
__table_args__ = (
UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),
# Gallery list joins assignments by (type, target); the tag filter
# looks up (type, tag). The unique key leads with tag_id, so neither
# query can use it as a prefix.
Index("ix_tag_assignments_type_target", "target_type", "target_id"),
Index("ix_tag_assignments_type_tag", "target_type", "tag_id", "target_id"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
@@ -7,10 +7,21 @@ from typing import Any
class TaskDeeplinkDetailStore(ABC):
"""Shared external task snapshots keyed by ``(detail_type, detail_id)``."""
"""Shared external task snapshots keyed by ``(detail_type, detail_id, sub_type)``.
``sub_type`` is the integrator-supplied business type (the deep-link
``?type=``): the same task id may hold one snapshot per type, each with its
own version counter. The empty string is the legacy no-type partition.
"""
@abstractmethod
async def get_detail(self, detail_type: str, detail_id: str) -> dict[str, Any] | None:
async def get_detail(
self,
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
) -> dict[str, Any] | None:
"""Return one persisted task snapshot, or ``None`` when it is unknown."""
raise NotImplementedError
@@ -20,6 +31,7 @@ class TaskDeeplinkDetailStore(ABC):
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
task_name: str,
task_content: str,
overview: str,
@@ -11,10 +11,16 @@ from deerflow.persistence.task_deeplink_details.base import TaskDeeplinkDetailSt
class MemoryTaskDeeplinkDetailStore(TaskDeeplinkDetailStore):
def __init__(self) -> None:
self._rows: dict[tuple[str, str], dict[str, Any]] = {}
self._rows: dict[tuple[str, str, str], dict[str, Any]] = {}
async def get_detail(self, detail_type: str, detail_id: str) -> dict[str, Any] | None:
row = self._rows.get((detail_type, detail_id))
async def get_detail(
self,
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
) -> dict[str, Any] | None:
row = self._rows.get((detail_type, detail_id, sub_type))
return deepcopy(row) if row is not None else None
async def upsert_detail(
@@ -22,13 +28,14 @@ class MemoryTaskDeeplinkDetailStore(TaskDeeplinkDetailStore):
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
task_name: str,
task_content: str,
overview: str,
extra: dict[str, Any],
updated_by: str | None,
) -> tuple[dict[str, Any], bool, bool]:
key = (detail_type, detail_id)
key = (detail_type, detail_id, sub_type)
row = self._rows.get(key)
created = row is None
if row is None:
@@ -36,6 +43,7 @@ class MemoryTaskDeeplinkDetailStore(TaskDeeplinkDetailStore):
row = {
"detail_type": detail_type,
"detail_id": detail_id,
"sub_type": sub_type,
"version": 1,
"created_by": updated_by,
"created_at": now,
@@ -18,6 +18,9 @@ class TaskDeeplinkDetailRow(Base):
detail_type: Mapped[str] = mapped_column(String(32), primary_key=True)
detail_id: Mapped[str] = mapped_column(String(128), primary_key=True)
# Integrator-supplied business type (the deep-link ``?type=``): one task id
# may carry several typed snapshots; "" is the legacy no-type partition.
sub_type: Mapped[str] = mapped_column(String(64), primary_key=True, default="")
task_name: Mapped[str] = mapped_column(String(512), nullable=False, default="")
task_content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
overview: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
@@ -15,6 +15,7 @@ def _to_dict(row: TaskDeeplinkDetailRow) -> dict[str, Any]:
return {
"detail_type": row.detail_type,
"detail_id": row.detail_id,
"sub_type": row.sub_type or "",
"task_name": row.task_name,
"task_content": row.task_content,
"overview": row.overview,
@@ -31,11 +32,17 @@ class TaskDeeplinkDetailRepository(TaskDeeplinkDetailStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get_detail(self, detail_type: str, detail_id: str) -> dict[str, Any] | None:
async def get_detail(
self,
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(
TaskDeeplinkDetailRow,
{"detail_type": detail_type, "detail_id": detail_id},
{"detail_type": detail_type, "detail_id": detail_id, "sub_type": sub_type},
)
return _to_dict(row) if row is not None else None
@@ -44,6 +51,7 @@ class TaskDeeplinkDetailRepository(TaskDeeplinkDetailStore):
detail_type: str,
detail_id: str,
*,
sub_type: str = "",
task_name: str,
task_content: str,
overview: str,
@@ -53,7 +61,7 @@ class TaskDeeplinkDetailRepository(TaskDeeplinkDetailStore):
async with self._sf() as session:
row = await session.get(
TaskDeeplinkDetailRow,
{"detail_type": detail_type, "detail_id": detail_id},
{"detail_type": detail_type, "detail_id": detail_id, "sub_type": sub_type},
)
created = row is None
now = datetime.now(UTC)
@@ -61,6 +69,7 @@ class TaskDeeplinkDetailRepository(TaskDeeplinkDetailStore):
row = TaskDeeplinkDetailRow(
detail_type=detail_type,
detail_id=detail_id,
sub_type=sub_type,
task_name=task_name,
task_content=task_content,
overview=overview,
@@ -1,4 +1,4 @@
"""Dedicated taskId + goPath conversation-history index."""
"""Dedicated taskId + goPath conversation-history cards."""
from __future__ import annotations
@@ -1,8 +1,9 @@
"""Shared helpers for task deep-link conversation indexing.
"""Shared helpers for task deep-link conversation history.
The LangGraph thread/checkpoint remains the source of truth for messages and
execution state. This package only owns a small, dedicated lookup index so the
task workspaces never have to search/filter the global ``threads_meta`` table.
execution state. This package owns a dedicated history card so the task
workspaces never have to search, filter, or join the global ``threads_meta``
catalogue when listing a task's conversations.
"""
from __future__ import annotations
@@ -10,6 +11,12 @@ from __future__ import annotations
from typing import Any, TypedDict
TASK_DEEPLINK_GO_PATHS = frozenset({"3qfx", "rwfx", "xdfx", "b5-1"})
# Custom business mappings use their administrator-defined name as goPath.
# Keep this in lock-step with ``business_mapping.business_code``.
MAX_TASK_DEEPLINK_GO_PATH = 32
# Matches the ``task_deeplink_threads.sub_type`` column width.
MAX_TASK_DEEPLINK_SUB_TYPE = 64
class TaskDeeplinkBinding(TypedDict):
@@ -17,6 +24,21 @@ class TaskDeeplinkBinding(TypedDict):
source_task_id: str
go_path: str
agent_id: str | None
# b5-1 integrator-supplied ?type= partition; "" when the link had none.
sub_type: str
def is_valid_task_deeplink_go_path(value: str) -> bool:
"""Accept built-ins and validated custom mapping names.
The harness cannot import the app-level business-mapping store (the import
firewall deliberately forbids that), so it validates the safe wire format
here. Creation itself remains admin-only in the Gateway router.
"""
go_path = value.strip().lower()
if go_path in TASK_DEEPLINK_GO_PATHS:
return True
return bool(go_path) and len(go_path) <= MAX_TASK_DEEPLINK_GO_PATH and not go_path.endswith("-a")
def task_deeplink_binding(metadata: dict[str, Any] | None) -> TaskDeeplinkBinding | None:
@@ -26,21 +48,25 @@ def task_deeplink_binding(metadata: dict[str, Any] | None) -> TaskDeeplinkBindin
actual task id for 3qfx/rwfx and the action id for xdfx. Older callers may
omit it, in which case ``taskId`` is a safe fallback. A ``taskGoPath`` is
mandatory, deliberately preventing legacy task rows from being migrated or
accidentally mixed into the new store.
accidentally mixed into the new store. ``taskType`` (the b5-1 ``?type=``
business type) further partitions one task's history so integrator-supplied
types never mix; it is capped to the column width here.
"""
md = metadata or {}
go_path = str(md.get("taskGoPath") or "").strip().lower()
if go_path not in TASK_DEEPLINK_GO_PATHS:
if not is_valid_task_deeplink_go_path(go_path):
return None
source_task_id = str(md.get("taskId") or "").strip()
task_id = str(md.get("taskRecordId") or source_task_id).strip()
if not task_id or not source_task_id:
return None
agent_id = str(md.get("agent_id") or "").strip() or None
sub_type = str(md.get("taskType") or "").strip()[:MAX_TASK_DEEPLINK_SUB_TYPE]
return {
"task_id": task_id,
"source_task_id": source_task_id,
"go_path": go_path,
"agent_id": agent_id,
"sub_type": sub_type,
}
@@ -1,4 +1,4 @@
"""In-memory task deep-link index backed by LangGraph BaseStore."""
"""In-memory task deep-link history cards backed by LangGraph BaseStore."""
from __future__ import annotations
@@ -16,6 +16,19 @@ if TYPE_CHECKING:
TASK_DEEPLINK_THREADS_NS: tuple[str, ...] = ("task_deeplink_threads",)
def _card_record(thread: dict[str, Any], binding: dict[str, Any]) -> dict[str, Any]:
return {
"thread_id": str(thread.get("thread_id") or ""),
**binding,
"display_name": thread.get("display_name"),
"status": thread.get("status") or "idle",
"metadata": thread.get("metadata") or {},
"created_by": thread.get("user_id"),
"created_at": thread.get("created_at") or now_iso(),
"updated_at": thread.get("updated_at") or now_iso(),
}
async def sync_task_deeplink_index(store: BaseStore, thread: dict[str, Any]) -> None:
metadata = thread.get("metadata") or {}
binding = task_deeplink_binding(metadata)
@@ -26,17 +39,7 @@ async def sync_task_deeplink_index(store: BaseStore, thread: dict[str, Any]) ->
if "taskGoPath" in metadata or "taskRecordId" in metadata:
await store.adelete(TASK_DEEPLINK_THREADS_NS, thread_id)
return
await store.aput(
TASK_DEEPLINK_THREADS_NS,
thread_id,
{
"thread_id": thread_id,
**binding,
"created_by": thread.get("user_id"),
"created_at": thread.get("created_at") or now_iso(),
"updated_at": thread.get("updated_at") or now_iso(),
},
)
await store.aput(TASK_DEEPLINK_THREADS_NS, thread_id, _card_record(thread, binding))
async def delete_task_deeplink_index(store: BaseStore, thread_id: str) -> None:
@@ -46,6 +49,8 @@ async def delete_task_deeplink_index(store: BaseStore, thread_id: str) -> None:
class MemoryTaskDeeplinkThreadStore:
def __init__(self, store: BaseStore, thread_store: ThreadMetaStore) -> None:
self._store = store
# Kept for constructor compatibility with the factory. Listing no
# longer reads the generic thread catalogue.
self._thread_store = thread_store
async def list_threads(
@@ -54,28 +59,33 @@ class MemoryTaskDeeplinkThreadStore:
go_path: str,
*,
limit: int = 100,
sub_type: str | None = None,
) -> list[dict[str, Any]]:
# ``sub_type`` semantics mirror the SQL repository: provided value (even
# "") narrows to that b5-1 ?type= partition; None returns all sub-types.
conditions = {"task_id": task_id, "go_path": go_path}
if sub_type is not None:
conditions["sub_type"] = sub_type
items = await self._store.asearch(
TASK_DEEPLINK_THREADS_NS,
filter={"task_id": task_id, "go_path": go_path},
filter=conditions,
limit=limit,
)
rows: list[dict[str, Any]] = []
for item in items:
thread_id = str(item.value.get("thread_id") or "")
value = item.value or {}
thread_id = str(value.get("thread_id") or "")
if not thread_id:
continue
thread = await self._thread_store.get(thread_id, user_id=None)
if thread is None:
continue
display_name = value.get("display_name")
rows.append(
{
"thread_id": thread_id,
"status": thread.get("status", "idle"),
"created_at": thread.get("created_at", ""),
"updated_at": thread.get("updated_at", ""),
"metadata": thread.get("metadata") or {},
"values": ({"title": thread.get("display_name")} if thread.get("display_name") else {}),
"status": value.get("status") or "idle",
"created_at": value.get("created_at", ""),
"updated_at": value.get("updated_at", ""),
"metadata": value.get("metadata") or {},
"values": ({"title": display_name} if display_name else {}),
"interrupts": {},
}
)
@@ -1,4 +1,4 @@
"""Dedicated lookup rows for single-agent task deep-link conversations."""
"""Dedicated history cards for single-agent task deep-link conversations."""
from __future__ import annotations
@@ -8,15 +8,15 @@ from sqlalchemy import Index, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class TaskDeeplinkThreadRow(Base):
"""One task-workspace history entry pointing at one LangGraph thread.
"""One task-workspace history card pointing at one LangGraph thread.
The row intentionally stores no messages or title. Those remain in the
canonical checkpoint / ``threads_meta`` record and are joined by primary
key when listing a task's history.
List queries read only this table. Title / status / metadata are copied
here so the huge generic ``threads_meta`` catalogue is never joined.
Messages stay in the LangGraph checkpoint.
"""
__tablename__ = "task_deeplink_threads"
@@ -25,6 +25,17 @@ class TaskDeeplinkThreadRow(Base):
"ix_task_deeplink_threads_scope",
"task_id",
"go_path",
"sub_type",
"updated_at",
"thread_id",
),
# 3qfx/rwfx/xdfx omit sub_type. The scope index is ordered by
# sub_type before updated_at, so those listings cannot ride it for
# "newest first" without a filesort. This prefix matches that query.
Index(
"ix_task_deeplink_threads_task_go_updated",
"task_id",
"go_path",
"updated_at",
"thread_id",
),
@@ -35,8 +46,14 @@ class TaskDeeplinkThreadRow(Base):
task_id: Mapped[str] = mapped_column(String(128), nullable=False)
# Resolved real task id used by model/run context (equals task_id except xdfx).
source_task_id: Mapped[str] = mapped_column(String(128), nullable=False)
go_path: Mapped[str] = mapped_column(String(16), nullable=False)
go_path: Mapped[str] = mapped_column(String(32), nullable=False)
# b5-1 ?type= business type; "" for links without one (and for the other
# go paths, which have no type dimension).
sub_type: Mapped[str] = mapped_column(String(64), nullable=False, default="")
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
display_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="idle")
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
# Audit only. Task deep-link histories remain shared across authenticated users.
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -1,11 +1,11 @@
"""SQL implementation of the dedicated task deep-link conversation index."""
"""SQL implementation of the dedicated task deep-link conversation cards."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from sqlalchemy import delete, select
from sqlalchemy import delete, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.task_deeplink_threads.base import task_deeplink_binding
@@ -15,8 +15,28 @@ if TYPE_CHECKING:
from deerflow.persistence.thread_meta.model import ThreadMetaRow
def _card_payload(thread: TaskDeeplinkThreadRow) -> dict[str, Any]:
return {
"thread_id": thread.thread_id,
"status": thread.status or "idle",
"created_at": thread.created_at,
"updated_at": thread.updated_at,
"metadata": thread.metadata_json or {},
"values": ({"title": thread.display_name} if thread.display_name else {}),
"interrupts": {},
}
def _apply_card_fields(row: TaskDeeplinkThreadRow, thread: ThreadMetaRow, now: datetime) -> None:
row.display_name = thread.display_name
row.status = thread.status or "idle"
row.metadata_json = dict(thread.metadata_json or {})
row.created_by = thread.user_id
row.updated_at = now
async def sync_task_deeplink_index(session: AsyncSession, thread: ThreadMetaRow) -> None:
"""Synchronize one index row inside the caller's thread-meta transaction."""
"""Synchronize one history card inside the caller's thread-meta transaction."""
metadata = thread.metadata_json or {}
binding = task_deeplink_binding(metadata)
@@ -30,25 +50,39 @@ async def sync_task_deeplink_index(session: AsyncSession, thread: ThreadMetaRow)
existing = await session.get(TaskDeeplinkThreadRow, thread.thread_id)
now = thread.updated_at if isinstance(thread.updated_at, datetime) else datetime.now(UTC)
if existing is None:
session.add(
TaskDeeplinkThreadRow(
thread_id=thread.thread_id,
task_id=binding["task_id"],
source_task_id=binding["source_task_id"],
go_path=binding["go_path"],
agent_id=binding["agent_id"],
created_by=thread.user_id,
created_at=(thread.created_at if isinstance(thread.created_at, datetime) else now),
updated_at=now,
)
row = TaskDeeplinkThreadRow(
thread_id=thread.thread_id,
task_id=binding["task_id"],
source_task_id=binding["source_task_id"],
go_path=binding["go_path"],
sub_type=binding["sub_type"],
agent_id=binding["agent_id"],
created_by=thread.user_id,
created_at=(thread.created_at if isinstance(thread.created_at, datetime) else now),
updated_at=now,
)
_apply_card_fields(row, thread, now)
session.add(row)
return
existing.task_id = binding["task_id"]
existing.source_task_id = binding["source_task_id"]
existing.go_path = binding["go_path"]
existing.sub_type = binding["sub_type"]
existing.agent_id = binding["agent_id"]
existing.updated_at = now
_apply_card_fields(existing, thread, now)
async def update_task_deeplink_card(session: AsyncSession, thread_id: str, **values: Any) -> None:
"""Patch a history card without reading ``threads_meta``.
Title / status / ``updated_at`` changes on ordinary chats no-op here
because those ``thread_id`` values are absent from this table.
"""
if not values:
return
await session.execute(update(TaskDeeplinkThreadRow).where(TaskDeeplinkThreadRow.thread_id == thread_id).values(**values))
async def delete_task_deeplink_index(session: AsyncSession, thread_id: str) -> None:
@@ -65,41 +99,30 @@ class TaskDeeplinkThreadRepository:
go_path: str,
*,
limit: int = 100,
sub_type: str | None = None,
) -> list[dict[str, Any]]:
"""List one task/goPath partition without scanning thread metadata JSON."""
"""List one task/goPath partition from the dedicated card table only.
# Local import avoids a package-initialization cycle: thread_meta.sql
# imports the sync helpers above for atomic dual writes.
from deerflow.persistence.thread_meta.model import ThreadMetaRow
``sub_type`` narrows the partition to one b5-1 ``?type=`` value when
provided (an explicit "" selects the no-type partition); ``None`` keeps
the untyped legacy behaviour of returning every sub-type at once.
"""
conditions = [
TaskDeeplinkThreadRow.task_id == task_id,
TaskDeeplinkThreadRow.go_path == go_path,
]
if sub_type is not None:
conditions.append(TaskDeeplinkThreadRow.sub_type == sub_type)
stmt = (
select(TaskDeeplinkThreadRow, ThreadMetaRow)
.join(
ThreadMetaRow,
ThreadMetaRow.thread_id == TaskDeeplinkThreadRow.thread_id,
)
.where(
TaskDeeplinkThreadRow.task_id == task_id,
TaskDeeplinkThreadRow.go_path == go_path,
)
select(TaskDeeplinkThreadRow)
.where(*conditions)
.order_by(
ThreadMetaRow.updated_at.desc(),
ThreadMetaRow.thread_id.desc(),
TaskDeeplinkThreadRow.updated_at.desc(),
TaskDeeplinkThreadRow.thread_id.desc(),
)
.limit(limit)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
return [
{
"thread_id": thread.thread_id,
"status": thread.status,
"created_at": thread.created_at,
"updated_at": thread.updated_at,
"metadata": thread.metadata_json or {},
"values": ({"title": thread.display_name} if thread.display_name else {}),
"interrupts": {},
}
for _index, thread in rows
]
rows = (await session.execute(stmt)).scalars().all()
return [_card_payload(row) for row in rows]
@@ -192,10 +192,14 @@ class MemoryThreadMetaStore(ThreadMetaStore):
item = await self._store.aget(THREADS_NS, thread_id)
if item is None:
return not require_existing
record_user_id = item.value.get("user_id")
if record_user_id:
from deerflow.runtime.thread_paths import remember_thread_owner
remember_thread_owner(thread_id, str(record_user_id))
# 任务深链 / 圆桌 / 任务分析系统线程:全局共享(与 SQL store 同口径)。
if is_shared_access_thread(item.value.get("metadata") or {}):
return True
record_user_id = item.value.get("user_id")
if record_user_id is None:
return True
return record_user_id == user_id
@@ -207,6 +211,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
record["display_name"] = display_name
record["updated_at"] = now_iso()
await self._store.aput(THREADS_NS, thread_id, record)
await sync_task_deeplink_index(self._store, record)
async def update_status(self, thread_id: str, status: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_status")
@@ -215,6 +220,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
record["status"] = status
record["updated_at"] = now_iso()
await self._store.aput(THREADS_NS, thread_id, record)
await sync_task_deeplink_index(self._store, record)
async def update_assistant_id(self, thread_id: str, assistant_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_assistant_id")
@@ -223,6 +229,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
record["assistant_id"] = assistant_id
record["updated_at"] = now_iso()
await self._store.aput(THREADS_NS, thread_id, record)
await sync_task_deeplink_index(self._store, record)
async def update_metadata(self, thread_id: str, metadata: dict, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_metadata")
@@ -5,72 +5,60 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy import func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.json_compat import JsonPredicate, supports_json_filter
from deerflow.persistence.task_deeplink_threads.sql import (
delete_task_deeplink_index,
sync_task_deeplink_index,
update_task_deeplink_card,
)
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
from deerflow.persistence.thread_meta.base import EXCLUDED_SYSTEM_THREAD_TYPES, ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
_INDEXED_METADATA_COLUMNS = {
"taskId": ThreadMetaRow.task_id,
"agent_id": ThreadMetaRow.agent_id,
"actionId": ThreadMetaRow.action_id,
}
_INDEXED_METADATA_ATTRIBUTES = {
"taskId": "task_id",
"agent_id": "agent_id",
"actionId": "action_id",
}
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
"""Return metadata predicates that have a maintained SQL projection.
Task deep-link callers use string ids. Restrict the SQL shortcut to
strings so legacy callers that intentionally store another JSON type keep
the exact Python-side equality semantics of the generic metadata filter.
"""
return {
key: value
for key, value in (metadata or {}).items()
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
}
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
"""Build ORM column values from authoritative thread metadata.
Always include every projection so a metadata update that changes a value
from a string to another JSON type cannot leave a stale indexed value.
"""
values = _indexed_metadata_values(metadata)
return {
attribute: values.get(metadata_key)
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
}
def _apply_indexed_metadata_filters(stmt: Any, metadata: dict[str, Any] | None) -> tuple[Any, set[str]]:
"""Apply known metadata predicates in SQL and return their keys.
The caller still verifies the full metadata object in Python whenever a
generic filter or system-thread exclusion is involved. This keeps the
public API's exact-match contract while reducing that verification set to
rows for the requested task/agent/action instead of the entire table.
"""
values = _indexed_metadata_values(metadata)
for key, value in values.items():
stmt = stmt.where(_INDEXED_METADATA_COLUMNS[key] == value)
return stmt, set(values)
class ThreadMetaRepository(ThreadMetaStore):
_INDEXED_METADATA_COLUMNS = {
"taskId": ThreadMetaRow.task_id,
"agent_id": ThreadMetaRow.agent_id,
"actionId": ThreadMetaRow.action_id,
}
_INDEXED_METADATA_ATTRIBUTES = {
"taskId": "task_id",
"agent_id": "agent_id",
"actionId": "action_id",
}
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
"""Return metadata predicates that have a maintained SQL projection.
Task deep-link callers use string ids. Restrict the SQL shortcut to
strings so legacy callers that intentionally store another JSON type keep
the exact Python-side equality semantics of the generic metadata filter.
"""
return {
key: value
for key, value in (metadata or {}).items()
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
}
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
"""Build ORM column values from authoritative thread metadata.
Always include every projection so a metadata update that changes a value
from a string to another JSON type cannot leave a stale indexed value.
"""
values = _indexed_metadata_values(metadata)
return {
attribute: values.get(metadata_key)
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
}
class ThreadMetaRepository(ThreadMetaStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -97,14 +85,14 @@ class ThreadMetaRepository(ThreadMetaStore):
# creates an orphan row (used by migration scripts).
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.create")
now = datetime.now(UTC)
row = ThreadMetaRow(
thread_id=thread_id,
assistant_id=assistant_id,
user_id=resolved_user_id,
**_indexed_thread_metadata(metadata),
display_name=display_name,
status="idle",
metadata_json=metadata or {},
row = ThreadMetaRow(
thread_id=thread_id,
assistant_id=assistant_id,
user_id=resolved_user_id,
**_indexed_thread_metadata(metadata),
display_name=display_name,
status="idle",
metadata_json=metadata or {},
created_at=now,
updated_at=now,
)
@@ -164,6 +152,10 @@ class ThreadMetaRepository(ThreadMetaStore):
row = await session.get(ThreadMetaRow, thread_id)
if row is None:
return not require_existing
if row.user_id:
from deerflow.runtime.thread_paths import remember_thread_owner
remember_thread_owner(thread_id, row.user_id)
md = row.metadata_json or {}
# 全局共享、不区分用户(任何登录用户都可读写/续聊):
# ① 任务深链对话(metadata.taskId);
@@ -176,6 +168,43 @@ class ThreadMetaRepository(ThreadMetaStore):
return True
return row.user_id == user_id
@staticmethod
def _query_conditions(*, dialect, metadata, status, user_id, exclude_system, query):
"""Share exact search/count predicates without changing owner resolution.
Complex metadata and unsupported dialects retain their exact Python
filter. No rejected key/value is silently ignored. Server numeric JSON
comparisons retain that path: DECIMAL and DOUBLE cannot reproduce all
Python float/int comparisons (including large numeric task ids).
"""
conditions = []
if user_id is not None:
conditions.append(ThreadMetaRow.user_id == user_id)
if status:
conditions.append(ThreadMetaRow.status == status)
if query and query.strip():
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
# Retain the master branch's indexed task/agent/action narrowing.
# JSON predicates below still enforce exact case/type semantics.
for key, value in _indexed_metadata_values(metadata).items():
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
supported = dialect in {"sqlite", "postgresql", "mysql"}
residual = {}
for key, value in (metadata or {}).items():
numeric_server = dialect in {"mysql", "postgresql"} and isinstance(value, (bool, int, float))
if supported and supports_json_filter(key, value) and not numeric_server:
conditions.append(JsonPredicate(ThreadMetaRow.metadata_json, key, value=value))
else:
residual[key] = value
if exclude_system and supported:
column = ThreadMetaRow.metadata_json
conditions.append(~or_(
JsonPredicate(column, "system", "truthy"),
JsonPredicate(column, "task_analysis_seat", "nonblank"),
*(JsonPredicate(column, "thread_type", value=value) for value in sorted(EXCLUDED_SYSTEM_THREAD_TYPES)),
))
return conditions, residual, exclude_system and not supported
async def search(
self,
*,
@@ -187,71 +216,46 @@ class ThreadMetaRepository(ThreadMetaStore):
exclude_system: bool = False,
query: str | None = None,
) -> list[dict]:
"""Search threads with optional metadata and status filters.
Owner filter is enforced by default: caller must be in a user
context. Pass ``user_id=None`` to bypass (migration/CLI).
"""
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.search")
# 任务深链对话(按 metadata.taskId 过滤)全局共享、不区分用户:跳过 owner 过滤。
# Preserve task-scoped sharing, still applying EVERY metadata filter
# (taskId + agent_id + actionId) before the pagination window.
if metadata and metadata.get("taskId"):
resolved_user_id = None
# thread_id tie-breaker keeps OFFSET pagination stable when many rows
# share the same updated_at (bulk imports, scheduler bursts).
stmt = select(ThreadMetaRow).order_by(ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc())
if resolved_user_id is not None:
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
if status:
stmt = stmt.where(ThreadMetaRow.status == status)
if query and query.strip():
# Case-insensitive title search. display_name is a real column, so
# this filter runs in SQL even when the JSON filters below fall
# back to Python-side scanning.
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
# ``metadata_json`` is deliberately portable, so arbitrary filters
# still receive Python-side equality checks below. These three task
# identifiers are maintained projections and can safely narrow the
# SQL result first. A deep-link query that used to scan every thread
# now reads only that task's rows.
stmt, indexed_metadata_keys = _apply_indexed_metadata_filters(stmt, metadata)
remaining_metadata_keys = set(metadata or {}) - indexed_metadata_keys
if not remaining_metadata_keys and not exclude_system:
stmt = stmt.limit(limit).offset(offset)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
# Metadata filters live inside the JSON blob, so rows are scanned in
# batches and filtered in Python until the requested window is full.
# A fixed over-fetch window would under-fill pages when most rows are
# filtered out (e.g. heavy scheduled-task usage), making callers that
# paginate believe the list is exhausted. TODO(Phase 2): use JSON DB
# operators (Postgres @>, SQLite json_extract) for server-side filtering.
def _matches(row: dict) -> bool:
md = row.get("metadata") or {}
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
return False
if exclude_system and is_excluded_system_thread(md):
return False
return True
needed = offset + limit
batch_size = max(500, needed)
matched: list[dict] = []
scan_offset = 0
async with self._sf() as session:
conditions, residual, filter_system = self._query_conditions(
dialect=session.get_bind().dialect.name,
metadata=metadata, status=status, user_id=resolved_user_id,
exclude_system=exclude_system, query=query,
)
stmt = select(ThreadMetaRow).where(*conditions).order_by(
ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc(),
)
if not residual and not filter_system:
result = await session.execute(stmt.limit(limit).offset(offset))
return [self._row_to_dict(row) for row in result.scalars()]
# Compatibility path for nested values, unusual literal keys,
# numeric server filters, or the separate openGauss dialect.
needed = offset + limit
batch_size = max(500, needed)
matched = []
scan_offset = 0
while len(matched) < needed:
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
rows = result.scalars().all()
if not rows:
break
matched.extend(d for d in (self._row_to_dict(r) for r in rows) if _matches(d))
for row in rows:
record = self._row_to_dict(row)
md = record.get("metadata") or {}
if all(md.get(key) == value for key, value in residual.items()) and not (
filter_system and is_excluded_system_thread(md)
):
matched.append(record)
if len(rows) < batch_size:
break
scan_offset += len(rows)
return matched[offset:needed]
return matched[offset:needed]
async def count(
self,
@@ -262,56 +266,37 @@ class ThreadMetaRepository(ThreadMetaStore):
exclude_system: bool = False,
query: str | None = None,
) -> int:
"""Count threads matching the same filters as :meth:`search`."""
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.count")
# 任务深链对话(按 metadata.taskId 过滤)全局共享、不区分用户:跳过 owner 过滤。
if metadata and metadata.get("taskId"):
resolved_user_id = None
conditions = []
if resolved_user_id is not None:
conditions.append(ThreadMetaRow.user_id == resolved_user_id)
if status:
conditions.append(ThreadMetaRow.status == status)
if query and query.strip():
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
indexed_metadata = _indexed_metadata_values(metadata)
for key, value in indexed_metadata.items():
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
remaining_metadata_keys = set(metadata or {}) - set(indexed_metadata)
if not remaining_metadata_keys and not exclude_system:
stmt = select(func.count()).select_from(ThreadMetaRow)
for condition in conditions:
stmt = stmt.where(condition)
async with self._sf() as session:
async with self._sf() as session:
conditions, residual, filter_system = self._query_conditions(
dialect=session.get_bind().dialect.name,
metadata=metadata, status=status, user_id=resolved_user_id,
exclude_system=exclude_system, query=query,
)
if not residual and not filter_system:
stmt = select(func.count()).select_from(ThreadMetaRow).where(*conditions)
return int((await session.execute(stmt)).scalar_one())
# JSON filters can't run in portable SQL — scan just the metadata
# column in batches and count in Python (mirrors search()).
stmt = select(ThreadMetaRow.metadata_json).order_by(ThreadMetaRow.thread_id)
for condition in conditions:
stmt = stmt.where(condition)
total = 0
batch_size = 1000
scan_offset = 0
async with self._sf() as session:
stmt = select(ThreadMetaRow.metadata_json).where(*conditions).order_by(ThreadMetaRow.thread_id)
total = 0
scan_offset = 0
batch_size = 1000
while True:
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
rows = result.scalars().all()
rows = (await session.execute(stmt.limit(batch_size).offset(scan_offset))).scalars().all()
if not rows:
break
for metadata_json in rows:
md = metadata_json or {}
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
continue
if exclude_system and is_excluded_system_thread(md):
continue
total += 1
for md in rows:
md = md or {}
if all(md.get(key) == value for key, value in residual.items()) and not (
filter_system and is_excluded_system_thread(md)
):
total += 1
if len(rows) < batch_size:
break
scan_offset += len(rows)
return total
return total
async def _check_ownership(self, session: AsyncSession, thread_id: str, resolved_user_id: str | None) -> bool:
"""Return True if the row exists and is owned (or filter bypassed)."""
@@ -337,7 +322,9 @@ class ThreadMetaRepository(ThreadMetaStore):
async with self._sf() as session:
if not await self._check_ownership(session, thread_id, resolved_user_id):
return
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=datetime.now(UTC)))
now = datetime.now(UTC)
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=now))
await update_task_deeplink_card(session, thread_id, display_name=display_name, updated_at=now)
await session.commit()
async def update_status(
@@ -351,7 +338,9 @@ class ThreadMetaRepository(ThreadMetaStore):
async with self._sf() as session:
if not await self._check_ownership(session, thread_id, resolved_user_id):
return
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=datetime.now(UTC)))
now = datetime.now(UTC)
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=now))
await update_task_deeplink_card(session, thread_id, status=status, updated_at=now)
await session.commit()
async def update_assistant_id(
@@ -365,7 +354,9 @@ class ThreadMetaRepository(ThreadMetaStore):
async with self._sf() as session:
if not await self._check_ownership(session, thread_id, resolved_user_id):
return
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=datetime.now(UTC)))
now = datetime.now(UTC)
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=now))
await update_task_deeplink_card(session, thread_id, updated_at=now)
await session.commit()
async def update_metadata(
@@ -390,14 +381,14 @@ class ThreadMetaRepository(ThreadMetaStore):
task_shared = bool((row.metadata_json or {}).get("taskId"))
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
return
merged = dict(row.metadata_json or {})
merged.update(metadata)
row.metadata_json = merged
for key, value in _indexed_thread_metadata(merged).items():
setattr(row, key, value)
row.updated_at = datetime.now(UTC)
merged = dict(row.metadata_json or {})
merged.update(metadata)
row.metadata_json = merged
for key, value in _indexed_thread_metadata(merged).items():
setattr(row, key, value)
row.updated_at = datetime.now(UTC)
await sync_task_deeplink_index(session, row)
await session.commit()
await session.commit()
async def delete(
self,
@@ -82,3 +82,11 @@ class ToolMetricsStore(ABC):
"which skills were invoked, and how often did each succeed/fail".
"""
raise NotImplementedError
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
"""Delete one bounded batch of audit rows older than ``cutoff``.
The default keeps alternate/in-memory implementations compatible; the
SQL repository supplies the real retention operation.
"""
return 0
@@ -8,7 +8,7 @@ from collections.abc import AsyncIterator
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import desc, func, select
from sqlalchemy import delete, desc, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
@@ -245,3 +245,34 @@ class SqlToolMetricsStore(ToolMetricsStore):
{"skill_name": name, **counts}
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
]
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
"""Delete one small, oldest-first retention batch.
Key selection followed by a primary-key delete works consistently on
every supported SQL dialect and prevents a three-month cleanup from
holding a large audit-table write lock.
"""
# Keep below SQLite's traditional 999 bind-variable limit. The
# retention worker requests batches repeatedly, so portability is more
# valuable than one giant delete statement.
size = max(1, min(int(batch_size), 500))
normalized_cutoff = _normalize_created_at(cutoff)
async with self._sf() as session:
ids = list(
(
await session.scalars(
select(ToolCallMetricRow.id)
.where(ToolCallMetricRow.created_at < normalized_cutoff)
.order_by(ToolCallMetricRow.created_at.asc(), ToolCallMetricRow.id.asc())
.limit(size)
)
).all()
)
if not ids:
return 0
result = await session.execute(
delete(ToolCallMetricRow).where(ToolCallMetricRow.id.in_(ids))
)
await session.commit()
return int(result.rowcount or 0)
@@ -43,7 +43,17 @@ class PortableJSON(TypeDecorator):
return value
if not value:
return None
return json.loads(value)
try:
return json.loads(value)
except json.JSONDecodeError:
from deerflow.persistence.gauss_mysql_compat import (
decode_gauss_mysql_json,
is_gauss_mysql_dialect,
)
if not is_gauss_mysql_dialect(dialect):
raise
return decode_gauss_mysql_json(value)
class PortableLongText(TypeDecorator):
@@ -1,79 +1,104 @@
"""ORM model for the users table.
Lives in the harness persistence package so it is picked up by
``Base.metadata.create_all()`` alongside ``threads_meta``, ``runs``,
``run_events``, and ``feedback``. Using the shared engine means:
- One SQLite/Postgres database, one connection pool
- One schema initialisation codepath
- Consistent async sessions across auth and persistence reads
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Index, String, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class UserRow(Base):
__tablename__ = "users"
# UUIDs are stored as 36-char strings for cross-backend portability.
id: Mapped[str] = mapped_column(String(36), primary_key=True)
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
# when new roles are introduced.
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
)
# OAuth linkage (optional). A partial unique index enforces one
# account per (provider, oauth_id) pair, leaving NULL/NULL rows
# unconstrained so plain password accounts can coexist.
oauth_provider: Mapped[str | None] = mapped_column(String(32), nullable=True)
oauth_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# Auth lifecycle flags
needs_setup: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
token_version: Mapped[int] = mapped_column(nullable=False, default=0)
# 登录白名单(放行)标记 —— **黑名单模式**。仅当系统设置 ``login_whitelist.enabled``
# 打开时生效:关 → 忽略此列,人人可登录;开 → 仅 approved=False(被管理员显式
# 「取消放行」)的用户被拦,其余(含从未登录过的新账号)放行。新账号默认 True
# (首次登录即放行);存量用户由迁移回填 True。server_default=1 让迁移 ADD COLUMN
# 时把已存在行置为已放行。
approved: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
# 管理员在「白名单管理」页给个别账号设置的「登录口令」(**明文**)。非空 → 该账号的
# 用户名直登(``POST /api/v1/auth/login/username``)必须带 ``?password=`` 且与此值
# 完全一致才放行(优先于全局共享口令门);为空/NULL → 走原有逻辑(全局共享口令门
# 或免口令)。明文存储是有意为之:口令本就以 URL query 明文传输,存明文才能让管理员
# 随时复制完整登录链接 ``…/login/<用户名>?password=xxxx``。仅 admin 可读写。
login_password: Mapped[str | None] = mapped_column(String(255), nullable=True)
# 岗位 (position) the user is assigned to (1:1). NULL = no explicit
# position → the user inherits the ``is_default`` position's grants. Drives
# the subscription sync engine (see ``deerflow.services.position_sync``).
position_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
__table_args__ = (
Index(
"idx_users_oauth_identity",
"oauth_provider",
"oauth_id",
unique=True,
sqlite_where=text("oauth_provider IS NOT NULL AND oauth_id IS NOT NULL"),
),
)
"""ORM model for the users table.
Lives in the harness persistence package so it is picked up by
``Base.metadata.create_all()`` alongside ``threads_meta``, ``runs``,
``run_events``, and ``feedback``. Using the shared engine means:
- One SQLite/Postgres database, one connection pool
- One schema initialisation codepath
- Consistent async sessions across auth and persistence reads
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, DateTime, Index, String, text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class AuthInitializationRow(Base):
"""Singleton transaction claim for the public first-admin setup endpoint."""
__tablename__ = "auth_initialization"
id: Mapped[str] = mapped_column(String(32), primary_key=True)
class AuthUsernameRow(Base):
"""Reserve new account local-parts across concurrent email-domain variants."""
__tablename__ = "auth_usernames"
username: Mapped[str] = mapped_column(String(128), primary_key=True)
class AuthServiceNonceRow(Base):
"""One-time service exchange nonce shared by every gateway worker."""
__tablename__ = "auth_service_nonces"
nonce_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
client_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
class UserRow(Base):
__tablename__ = "users"
# UUIDs are stored as 36-char strings for cross-backend portability.
id: Mapped[str] = mapped_column(String(36), primary_key=True)
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
# Canonical local part used by SSO / username login. Keeping it separately
# avoids applying LOWER()/LIKE to every users.email row on each login.
username_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
# when new roles are introduced.
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
)
# OAuth linkage (optional). A partial unique index enforces one
# account per (provider, oauth_id) pair, leaving NULL/NULL rows
# unconstrained so plain password accounts can coexist.
oauth_provider: Mapped[str | None] = mapped_column(String(32), nullable=True)
oauth_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# Auth lifecycle flags
needs_setup: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
token_version: Mapped[int] = mapped_column(nullable=False, default=0)
# 登录白名单(放行)标记 —— **黑名单模式**。仅当系统设置 ``login_whitelist.enabled``
# 打开时生效:关 → 忽略此列,人人可登录;开 → 仅 approved=False(被管理员显式
# 「取消放行」)的用户被拦,其余(含从未登录过的新账号)放行。新账号默认 True
# (首次登录即放行);存量用户由迁移回填 True。server_default=1 让迁移 ADD COLUMN
# 时把已存在行置为已放行。
approved: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
# Unified password/passphrase. New writes store the same versioned hash as
# password_hash; historical plaintext remains accepted until the next reset.
# Never serialize hashes into user-list or login responses.
login_password: Mapped[str | None] = mapped_column(String(255), nullable=True)
# 岗位 (position) the user is assigned to (1:1). NULL = no explicit
# position → the user inherits the ``is_default`` position's grants. Drives
# the subscription sync engine (see ``deerflow.services.position_sync``).
position_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
__table_args__ = (
Index("ix_users_username_key_created_at", "username_key", "created_at"),
Index(
"idx_users_oauth_identity",
"oauth_provider",
"oauth_id",
unique=True,
sqlite_where=text("oauth_provider IS NOT NULL AND oauth_id IS NOT NULL"),
),
)