sync update from LAN repo (source master @ ca48c4ec)
This commit is contained in:
1 parent
8bc8b0e082
commit
ea9dcff9c0
873 files changed
+107866
-14011
No files matched your search
+23
-2
@@ -77,6 +77,11 @@ class AgentStore(ABC):
|
||||
"""Return featured + still-visible-to-``user_id`` agents, ordered by ``featured_order`` asc."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_sidebar(self, user_id: str, *, limit: int = 3) -> list[dict[str, Any]]:
|
||||
"""Return only the id/name fields needed by the workspace sidebar."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
|
||||
@@ -133,6 +138,7 @@ class AgentStore(ABC):
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
hidden: bool | None = None,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Return one page of agents plus the total matching count.
|
||||
|
||||
@@ -143,6 +149,9 @@ class AgentStore(ABC):
|
||||
- ``None`` — the default listing visibility (``list_visible``).
|
||||
``admin=True`` bypasses the visibility/scope filter entirely (every
|
||||
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
|
||||
Admin-hidden agents (``hidden=True`` rows) are excluded from every
|
||||
non-admin listing; under ``admin=True`` the ``hidden`` param selects
|
||||
only hidden / only visible rows (``None`` = both).
|
||||
|
||||
``search`` matches the raw name OR its desensitized display form.
|
||||
``tag_ids`` keeps agents carrying ANY of the tags (OR). ``owner`` keeps
|
||||
@@ -167,8 +176,20 @@ class AgentStore(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
|
||||
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``."""
|
||||
async def set_extras(
|
||||
self,
|
||||
agent_id: str,
|
||||
*,
|
||||
tool_groups: list[str] | None,
|
||||
model: str | None,
|
||||
chat_link: Any = None,
|
||||
update_chat_link: bool = False,
|
||||
) -> None:
|
||||
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``.
|
||||
|
||||
``chat_link`` is written only when ``update_chat_link`` is true, so a
|
||||
skills/model refresh does not clear a previously mirrored jump link.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
|
||||
+398
-325
@@ -1,325 +1,398 @@
|
||||
"""In-memory custom agent store for memory persistence mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.agent_name_desensitize import name_matches
|
||||
from deerflow.persistence.agents.base import AgentStore
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _copy(data: dict[str, Any]) -> dict[str, Any]:
|
||||
copied = dict(data)
|
||||
copied["builtin"] = copied.get("user_id") is None
|
||||
return copied
|
||||
|
||||
|
||||
class MemoryAgentStore(AgentStore):
|
||||
def __init__(self) -> None:
|
||||
self._agents: dict[str, dict[str, Any]] = {}
|
||||
self._favorites: set[tuple[str, str]] = set()
|
||||
self._pins: set[tuple[str, str]] = set()
|
||||
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
|
||||
self._skills: dict[str, list[str]] = {}
|
||||
self._extras: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
|
||||
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
|
||||
|
||||
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
|
||||
"""Single-record visibility (``get_visible``) — the union of every
|
||||
listing surface, so a list-visible agent always opens. Mirrors
|
||||
``AgentRepository._visible_filter``: baseline visibility OR
|
||||
private-square (password is the discovery gate, reads are not
|
||||
re-gated) OR favorited (the "我的" listing keeps favorites even after
|
||||
the owner unpublishes)."""
|
||||
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
|
||||
|
||||
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
|
||||
rows = [_copy(agent) for agent in self._agents.values() if self._is_visible(agent, user_id)]
|
||||
rows.sort(key=lambda agent: agent.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or not self._is_fetchable(agent, user_id):
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
return _copy(agent) if agent is not None else None
|
||||
|
||||
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or agent.get("user_id") != user_id:
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = _now()
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"name": data["name"],
|
||||
"description": data.get("description") or "",
|
||||
"user_id": data.get("user_id"),
|
||||
"published": bool(data.get("published", False)),
|
||||
"is_private": bool(data.get("is_private", False)),
|
||||
"square_id": str(data.get("square_id") or ""),
|
||||
"featured_order": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._agents[row["id"]] = row
|
||||
return _copy(row)
|
||||
|
||||
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = self._agents.get(data["id"])
|
||||
if row is None:
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"name": data["name"],
|
||||
"description": data.get("description") or "",
|
||||
"user_id": None,
|
||||
"published": bool(data.get("published", True)),
|
||||
"featured_order": None,
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
self._agents[row["id"]] = row
|
||||
elif row.get("user_id") is None:
|
||||
row["name"] = data["name"]
|
||||
row["description"] = data.get("description") or ""
|
||||
row["published"] = bool(data.get("published", True))
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete(self, agent_id: str, user_id: str) -> bool:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
async def delete_builtin(self, agent_id: str) -> bool:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
|
||||
|
||||
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
row["featured_order"] = order
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
candidates = [a for a in self._agents.values() if a.get("featured_order") is not None and self._is_visible(a, user_id)]
|
||||
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
|
||||
return [_copy(a) for a in candidates]
|
||||
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
|
||||
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
|
||||
rows.sort(key=lambda a: order_map.get(a["id"], 0))
|
||||
return [_copy(a) for a in rows]
|
||||
|
||||
async def list_private(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private"))]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values()]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete_any(self, agent_id: str) -> bool:
|
||||
if agent_id not in self._agents:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
# --- Paginated listing ---
|
||||
|
||||
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
|
||||
if admin:
|
||||
return True
|
||||
if scope == "square":
|
||||
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
|
||||
if scope == "mine":
|
||||
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
|
||||
favorited = (user_id, agent["id"]) in self._favorites
|
||||
return owned or favorited
|
||||
if scope == "private":
|
||||
return bool(agent.get("is_private"))
|
||||
if scope == "builtin":
|
||||
# 内置智能体 = user_id IS NULL(仅管理员;路由层 admin 闸门)。
|
||||
return agent.get("user_id") is None
|
||||
return self._is_visible(agent, user_id)
|
||||
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
|
||||
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
|
||||
owner_key = owner.strip() if owner and owner.strip() else None
|
||||
square_key = square_id.strip() if square_id and square_id.strip() else None
|
||||
|
||||
def _square_ok(a: dict[str, Any]) -> bool:
|
||||
if square_key is None:
|
||||
return True
|
||||
sid = str(a.get("square_id") or "")
|
||||
return sid == square_key or (square_is_default and sid == "")
|
||||
|
||||
matched = [
|
||||
a
|
||||
for a in self._agents.values()
|
||||
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
|
||||
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
|
||||
and (owner_key is None or a.get("user_id") == owner_key)
|
||||
and _square_ok(a)
|
||||
]
|
||||
if sort == "time_asc":
|
||||
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
|
||||
elif sort == "updated_desc":
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
elif sort == "updated_asc":
|
||||
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
|
||||
elif sort == "name_asc":
|
||||
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
|
||||
elif sort == "name_desc":
|
||||
matched.sort(
|
||||
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
else:
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
|
||||
total = len(matched)
|
||||
start = max(page - 1, 0) * page_size
|
||||
page_rows = matched[start : start + page_size]
|
||||
return [_copy(a) for a in page_rows], total
|
||||
|
||||
# --- Listing cache (skills / tool_groups / model) ---
|
||||
|
||||
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
|
||||
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
|
||||
|
||||
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
|
||||
self._extras[agent_id] = {"tool_groups": tool_groups, "model": model}
|
||||
|
||||
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for agent_id in agent_ids:
|
||||
has_extras = agent_id in self._extras
|
||||
has_skills = agent_id in self._skills
|
||||
if not has_extras and not has_skills:
|
||||
continue
|
||||
extras = self._extras.get(agent_id, {})
|
||||
out[agent_id] = {
|
||||
"skills": list(self._skills.get(agent_id, [])),
|
||||
"tool_groups": extras.get("tool_groups"),
|
||||
"model": extras.get("model"),
|
||||
}
|
||||
return out
|
||||
|
||||
async def delete_extras_for_agent(self, agent_id: str) -> None:
|
||||
self._skills.pop(agent_id, None)
|
||||
self._extras.pop(agent_id, None)
|
||||
|
||||
# --- Favorites ("收藏") ---
|
||||
|
||||
async def add_favorite(self, user_id: str, agent_id: str) -> None:
|
||||
self._favorites.add((user_id, agent_id))
|
||||
|
||||
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
|
||||
if (user_id, agent_id) in self._favorites:
|
||||
self._favorites.discard((user_id, agent_id))
|
||||
return True
|
||||
return False
|
||||
|
||||
async def list_favorite_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
|
||||
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
|
||||
|
||||
# --- Pins ("置顶") ---
|
||||
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
self._pins.add((user_id, agent_id))
|
||||
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
key = (user_id, agent_id)
|
||||
if key not in self._pins:
|
||||
return False
|
||||
self._pins.remove(key)
|
||||
return True
|
||||
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
|
||||
"""In-memory custom agent store for memory persistence mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.agent_name_desensitize import name_matches
|
||||
from deerflow.persistence.agents.base import AgentStore
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _copy(data: dict[str, Any]) -> dict[str, Any]:
|
||||
copied = dict(data)
|
||||
copied["builtin"] = copied.get("user_id") is None
|
||||
return copied
|
||||
|
||||
|
||||
class MemoryAgentStore(AgentStore):
|
||||
def __init__(self) -> None:
|
||||
self._agents: dict[str, dict[str, Any]] = {}
|
||||
self._favorites: set[tuple[str, str]] = set()
|
||||
self._pins: set[tuple[str, str]] = set()
|
||||
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
|
||||
self._skills: dict[str, list[str]] = {}
|
||||
self._extras: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
|
||||
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
|
||||
|
||||
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
|
||||
"""Single-record visibility (``get_visible``) — the union of every
|
||||
listing surface, so a list-visible agent always opens. Mirrors
|
||||
``AgentRepository._visible_filter``: baseline visibility OR
|
||||
private-square (password is the discovery gate, reads are not
|
||||
re-gated) OR favorited (the "我的" listing keeps favorites even after
|
||||
the owner unpublishes)."""
|
||||
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
|
||||
|
||||
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
|
||||
rows = [
|
||||
_copy(agent)
|
||||
for agent in self._agents.values()
|
||||
if self._is_visible(agent, user_id) and not agent.get("hidden")
|
||||
]
|
||||
rows.sort(key=lambda agent: agent.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or not self._is_fetchable(agent, user_id):
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
return _copy(agent) if agent is not None else None
|
||||
|
||||
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or agent.get("user_id") != user_id:
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = _now()
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"name": data["name"],
|
||||
"description": data.get("description") or "",
|
||||
"user_id": data.get("user_id"),
|
||||
"published": bool(data.get("published", False)),
|
||||
"is_private": bool(data.get("is_private", False)),
|
||||
"square_id": str(data.get("square_id") or ""),
|
||||
"hidden": bool(data.get("hidden", False)),
|
||||
"featured_order": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._agents[row["id"]] = row
|
||||
return _copy(row)
|
||||
|
||||
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = self._agents.get(data["id"])
|
||||
if row is None:
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"name": data["name"],
|
||||
"description": data.get("description") or "",
|
||||
"user_id": None,
|
||||
"published": bool(data.get("published", True)),
|
||||
"hidden": bool(data.get("hidden", False)),
|
||||
"featured_order": None,
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
self._agents[row["id"]] = row
|
||||
elif row.get("user_id") is None:
|
||||
row["name"] = data["name"]
|
||||
row["description"] = data.get("description") or ""
|
||||
row["published"] = bool(data.get("published", True))
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete(self, agent_id: str, user_id: str) -> bool:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
async def delete_builtin(self, agent_id: str) -> bool:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
|
||||
|
||||
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
row["featured_order"] = order
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
candidates = [
|
||||
a
|
||||
for a in self._agents.values()
|
||||
if a.get("featured_order") is not None and self._is_visible(a, user_id) and not a.get("hidden")
|
||||
]
|
||||
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
|
||||
return [_copy(a) for a in candidates]
|
||||
|
||||
async def list_sidebar(self, user_id: str, *, limit: int = 3) -> list[dict[str, Any]]:
|
||||
mine = [
|
||||
a for a in self._agents.values()
|
||||
if not a.get("hidden")
|
||||
and ((a.get("user_id") == user_id) or (user_id, a["id"]) in self._favorites)
|
||||
]
|
||||
square = [
|
||||
a for a in self._agents.values()
|
||||
if not a.get("hidden")
|
||||
and (a.get("user_id") is None or a.get("published"))
|
||||
and not a.get("is_private")
|
||||
]
|
||||
for rows in (mine, square):
|
||||
rows.sort(
|
||||
key=lambda a: (str(a.get("created_at") or ""), str(a.get("id") or "")),
|
||||
reverse=True,
|
||||
)
|
||||
rows.sort(key=lambda a: (user_id, a["id"]) in self._pins, reverse=True)
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(agent: dict[str, Any]) -> None:
|
||||
if len(out) >= limit or agent["id"] in seen:
|
||||
return
|
||||
seen.add(agent["id"])
|
||||
out.append({"id": agent["id"], "name": agent.get("name") or ""})
|
||||
|
||||
for agent in [*mine, *square]:
|
||||
if (user_id, agent["id"]) in self._pins:
|
||||
add(agent)
|
||||
for agent in mine:
|
||||
add(agent)
|
||||
for agent in square:
|
||||
add(agent)
|
||||
return out
|
||||
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
|
||||
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
|
||||
rows.sort(key=lambda a: order_map.get(a["id"], 0))
|
||||
return [_copy(a) for a in rows]
|
||||
|
||||
async def list_private(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private")) and not a.get("hidden")]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values()]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete_any(self, agent_id: str) -> bool:
|
||||
if agent_id not in self._agents:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
# --- Paginated listing ---
|
||||
|
||||
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
|
||||
if admin:
|
||||
return True # every agent — the hidden filter is a separate param
|
||||
if agent.get("hidden"):
|
||||
return False # admin-hidden agents vanish from every non-admin scope
|
||||
if scope == "square":
|
||||
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
|
||||
if scope == "mine":
|
||||
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
|
||||
favorited = (user_id, agent["id"]) in self._favorites
|
||||
return owned or favorited
|
||||
if scope == "private":
|
||||
return bool(agent.get("is_private"))
|
||||
if scope == "builtin":
|
||||
# 内置智能体 = user_id IS NULL(仅管理员;路由层 admin 闸门)。
|
||||
return agent.get("user_id") is None
|
||||
return self._is_visible(agent, user_id)
|
||||
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
hidden: bool | None = None,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
|
||||
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
|
||||
owner_key = owner.strip() if owner and owner.strip() else None
|
||||
square_key = square_id.strip() if square_id and square_id.strip() else None
|
||||
|
||||
def _square_ok(a: dict[str, Any]) -> bool:
|
||||
if square_key is None:
|
||||
return True
|
||||
sid = str(a.get("square_id") or "")
|
||||
return sid == square_key or (square_is_default and sid == "")
|
||||
|
||||
def _hidden_ok(a: dict[str, Any]) -> bool:
|
||||
if admin:
|
||||
# Admin console filter: keep only hidden / only visible agents.
|
||||
return hidden is None or bool(a.get("hidden")) == hidden
|
||||
return not a.get("hidden")
|
||||
|
||||
matched = [
|
||||
a
|
||||
for a in self._agents.values()
|
||||
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
|
||||
and _hidden_ok(a)
|
||||
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
|
||||
and (owner_key is None or a.get("user_id") == owner_key)
|
||||
and _square_ok(a)
|
||||
]
|
||||
if sort == "time_asc":
|
||||
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
|
||||
elif sort == "updated_desc":
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
elif sort == "updated_asc":
|
||||
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
|
||||
elif sort == "name_asc":
|
||||
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
|
||||
elif sort == "name_desc":
|
||||
matched.sort(
|
||||
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
else:
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
|
||||
total = len(matched)
|
||||
start = max(page - 1, 0) * page_size
|
||||
page_rows = matched[start : start + page_size]
|
||||
return [_copy(a) for a in page_rows], total
|
||||
|
||||
# --- Listing cache (skills / tool_groups / model) ---
|
||||
|
||||
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
|
||||
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
|
||||
|
||||
async def set_extras(
|
||||
self,
|
||||
agent_id: str,
|
||||
*,
|
||||
tool_groups: list[str] | None,
|
||||
model: str | None,
|
||||
chat_link: Any = None,
|
||||
update_chat_link: bool = False,
|
||||
) -> None:
|
||||
previous = self._extras.get(agent_id, {})
|
||||
stored = {
|
||||
"tool_groups": tool_groups,
|
||||
"model": model,
|
||||
"chat_link": previous.get("chat_link"),
|
||||
}
|
||||
if update_chat_link:
|
||||
stored["chat_link"] = chat_link
|
||||
self._extras[agent_id] = stored
|
||||
|
||||
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for agent_id in agent_ids:
|
||||
has_extras = agent_id in self._extras
|
||||
has_skills = agent_id in self._skills
|
||||
if not has_extras and not has_skills:
|
||||
continue
|
||||
extras = self._extras.get(agent_id, {})
|
||||
out[agent_id] = {
|
||||
"skills": list(self._skills.get(agent_id, [])),
|
||||
"tool_groups": extras.get("tool_groups"),
|
||||
"model": extras.get("model"),
|
||||
"chat_link": extras.get("chat_link"),
|
||||
}
|
||||
return out
|
||||
|
||||
async def delete_extras_for_agent(self, agent_id: str) -> None:
|
||||
self._skills.pop(agent_id, None)
|
||||
self._extras.pop(agent_id, None)
|
||||
|
||||
# --- Favorites ("收藏") ---
|
||||
|
||||
async def add_favorite(self, user_id: str, agent_id: str) -> None:
|
||||
self._favorites.add((user_id, agent_id))
|
||||
|
||||
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
|
||||
if (user_id, agent_id) in self._favorites:
|
||||
self._favorites.discard((user_id, agent_id))
|
||||
return True
|
||||
return False
|
||||
|
||||
async def list_favorite_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
|
||||
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
|
||||
|
||||
# --- Pins ("置顶") ---
|
||||
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
self._pins.add((user_id, agent_id))
|
||||
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
key = (user_id, agent_id)
|
||||
if key not in self._pins:
|
||||
return False
|
||||
self._pins.remove(key)
|
||||
return True
|
||||
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
|
||||
+20
-1
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -13,6 +13,13 @@ from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
class AgentRow(Base):
|
||||
__tablename__ = "agents"
|
||||
# Composite indexes for the gallery tabs. Single-column indexes cannot
|
||||
# serve "广场" (hidden + private + square) or "我的" (owner + hidden)
|
||||
# without a wide scan once the catalog grows.
|
||||
__table_args__ = (
|
||||
Index("ix_agents_catalog_square", "hidden", "is_private", "square_id", "created_at"),
|
||||
Index("ix_agents_owner_hidden", "user_id", "hidden", "created_at"),
|
||||
)
|
||||
|
||||
# Stored as a plain string so the same id can be used as the filesystem
|
||||
# directory name under {base_dir}/agents/{id}/.
|
||||
@@ -27,6 +34,14 @@ class AgentRow(Base):
|
||||
# visible to its owner in the "我的" tab, and visible to anyone in the
|
||||
# private-square listing once the password has been verified.
|
||||
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# When True the agent is hidden from every non-admin listing surface
|
||||
# (gallery / 我的 / 广场 / homepage selector / private square). Only the
|
||||
# admin agents console (?admin=true) still lists it. Purely a catalog
|
||||
# switch: single-record access (detail / chat / run) keeps working, so
|
||||
# existing conversations with a hidden agent are unaffected.
|
||||
# server_default keeps the startup ORM-column sweep able to ADD COLUMN
|
||||
# on existing deployments (NOT NULL needs a default).
|
||||
hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"), index=True)
|
||||
# Publish square (发布广场) this agent is filed under. A pure organizational
|
||||
# category with no access control — independent of ``published`` /
|
||||
# ``is_private``. Empty string = "not assigned" → the listing layer treats
|
||||
@@ -124,4 +139,8 @@ class AgentExtrasRow(Base):
|
||||
# list[str] | None — the agent's tool_groups whitelist from config.yaml.
|
||||
tool_groups: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True, default=None)
|
||||
model: Mapped[str | None] = mapped_column(String(191), nullable=True, default=None)
|
||||
# Gallery 「对话」 jump target mirrored from config.yaml so the list endpoint
|
||||
# can redirect without reading every agent file. Nullable JSON so existing
|
||||
# databases pick the column up on the startup ORM sweep.
|
||||
chat_link: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True, default=None)
|
||||
skills_synced_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+596
-504
File diff suppressed because it is too large.
Load diff
+23
@@ -0,0 +1,23 @@
|
||||
"""MySQL-persisted administrator switch for the optional API Redis cache."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, Boolean, Integer, String, text, true
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ApiReadCacheSettingRow(Base):
|
||||
__tablename__ = "api_read_cache_settings"
|
||||
|
||||
key: Mapped[str] = mapped_column(String(32), primary_key=True, default="default")
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
|
||||
ttl_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=300, server_default=text("300"))
|
||||
collection_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
cache_epoch: Mapped[str] = mapped_column(String(32), nullable=False, default="0", server_default="0")
|
||||
browser_status_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true())
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+12
-2
@@ -4,7 +4,7 @@ A "business mapping" row binds a fixed business code (3Q / 6BF / 3q-0…3q-6 /
|
||||
6-1…6-4 / 7BF / 7-1…7-4 / 8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
|
||||
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
|
||||
account can resolve it). Used to auto-pick the business chain when a task page
|
||||
launches the multi-agent roundtable.
|
||||
launches the multi-agent roundtable. ``3q-1…3q-6`` are seeded as custom-type rows.
|
||||
|
||||
Unlike ``roundtable_chains`` (per-user), this mapping is **global / shared** —
|
||||
there is no ``user_id`` column; all accounts see and use the same configuration.
|
||||
@@ -20,7 +20,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy import Boolean, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -34,6 +34,16 @@ class BusinessMappingRow(Base):
|
||||
business_code: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
# Display label shown in the 业务 column (e.g. "任务FX").
|
||||
label: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
# A user-visible category. Built-in rows keep ``内置业务``; rows created from
|
||||
# the configuration page consistently use ``自定义类型``.
|
||||
mapping_type: Mapped[str] = mapped_column(
|
||||
String(64), nullable=False, default="内置业务", server_default="内置业务"
|
||||
)
|
||||
# Built-in rows are seeded and cannot be redefined. Custom rows are created
|
||||
# by an administrator and their business_code doubles as a login goPath.
|
||||
is_custom: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0"
|
||||
)
|
||||
# Picked 智能体 (暂存不用,自动流程当前不消费此字段). NULL = 未配置.
|
||||
agent_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(191), nullable=True)
|
||||
|
||||
+75
-9
@@ -11,7 +11,7 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.business_mapping.model import BusinessMappingRow
|
||||
@@ -21,21 +21,25 @@ _UNSET = object()
|
||||
# Fixed business rows (业务码 → 展示名). Seeded on first boot; agent/chain left empty.
|
||||
# 8BF 页面尚未建好,先占位保留行。顺序即管理表的展示顺序。
|
||||
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
|
||||
# b5-1 = 外部写入本系统详情后,由本系统读取的任务分析单智能体深链。
|
||||
# 3q-0…3q-6 / 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
|
||||
# 3q-1…3q-6 默认按「自定义类型」落库(goPath 走自定义深链);配置管理页默认不展示这 6 行。
|
||||
_CUSTOM_SLOT = {"is_custom": True, "mapping_type": "自定义类型"}
|
||||
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
{"business_code": "3Q", "label": "3QFX"},
|
||||
{"business_code": "6BF", "label": "任务FX"},
|
||||
{"business_code": "7BF", "label": "XDFX"},
|
||||
{"business_code": "8BF", "label": "8BF"},
|
||||
{"business_code": "AFX", "label": "AGENTFX"},
|
||||
{"business_code": "b5-1", "label": "任务分析"},
|
||||
# 独立配置位,与上方业务码无关;固定排在表尾。
|
||||
{"business_code": "3q-0", "label": "3q-0"},
|
||||
{"business_code": "3q-1", "label": "3q-1"},
|
||||
{"business_code": "3q-2", "label": "3q-2"},
|
||||
{"business_code": "3q-3", "label": "3q-3"},
|
||||
{"business_code": "3q-4", "label": "3q-4"},
|
||||
{"business_code": "3q-5", "label": "3q-5"},
|
||||
{"business_code": "3q-6", "label": "3q-6"},
|
||||
{"business_code": "3q-1", "label": "3q-1", **_CUSTOM_SLOT},
|
||||
{"business_code": "3q-2", "label": "3q-2", **_CUSTOM_SLOT},
|
||||
{"business_code": "3q-3", "label": "3q-3", **_CUSTOM_SLOT},
|
||||
{"business_code": "3q-4", "label": "3q-4", **_CUSTOM_SLOT},
|
||||
{"business_code": "3q-5", "label": "3q-5", **_CUSTOM_SLOT},
|
||||
{"business_code": "3q-6", "label": "3q-6", **_CUSTOM_SLOT},
|
||||
{"business_code": "6-1", "label": "6-1"},
|
||||
{"business_code": "6-2", "label": "6-2"},
|
||||
{"business_code": "6-3", "label": "6-3"},
|
||||
@@ -47,6 +51,14 @@ DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
]
|
||||
|
||||
|
||||
def _seed_type(seed: dict[str, Any]) -> tuple[bool, str]:
|
||||
is_custom = bool(seed.get("is_custom", False))
|
||||
mapping_type = str(seed.get("mapping_type") or "").strip()
|
||||
if not mapping_type:
|
||||
mapping_type = "自定义类型" if is_custom else "内置业务"
|
||||
return is_custom, mapping_type
|
||||
|
||||
|
||||
class BusinessMappingRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
@@ -56,6 +68,8 @@ class BusinessMappingRepository:
|
||||
return {
|
||||
"business_code": row.business_code,
|
||||
"label": row.label,
|
||||
"mapping_type": row.mapping_type or "内置业务",
|
||||
"is_custom": bool(row.is_custom),
|
||||
"agent_id": row.agent_id,
|
||||
"agent_name": row.agent_name,
|
||||
"chain_id": row.chain_id,
|
||||
@@ -108,18 +122,62 @@ class BusinessMappingRepository:
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
|
||||
async def create_custom_mapping(
|
||||
self,
|
||||
*,
|
||||
business_code: str,
|
||||
mapping_type: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Create one administrator-defined mapping.
|
||||
|
||||
``business_code`` is deliberately the externally supplied goPath. The
|
||||
case-insensitive uniqueness check mirrors frontend deep-link matching,
|
||||
preventing two rows that could resolve from the same login URL.
|
||||
"""
|
||||
normalized_code = business_code.strip()
|
||||
normalized_type = mapping_type.strip()
|
||||
async with self._sf() as session:
|
||||
existing = await session.scalar(
|
||||
select(BusinessMappingRow).where(
|
||||
func.lower(BusinessMappingRow.business_code) == normalized_code.lower()
|
||||
)
|
||||
)
|
||||
if existing is not None:
|
||||
return None
|
||||
max_sort_order = await session.scalar(select(func.max(BusinessMappingRow.sort_order)))
|
||||
now = datetime.now(UTC)
|
||||
row = BusinessMappingRow(
|
||||
business_code=normalized_code,
|
||||
label=normalized_code,
|
||||
mapping_type=normalized_type,
|
||||
is_custom=True,
|
||||
agent_id=None,
|
||||
agent_name=None,
|
||||
chain_id=None,
|
||||
chain_title=None,
|
||||
sort_order=(int(max_sort_order) if max_sort_order is not None else -1) + 1,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
|
||||
async def ensure_seed_defaults(self) -> int:
|
||||
"""Idempotently insert the fixed business rows if missing. Returns inserted count.
|
||||
|
||||
Never overwrites an existing row's agent/chain (respects user config).
|
||||
Re-syncs ``sort_order`` (and empty ``label``) so newly inserted codes keep
|
||||
the intended table order next to older seeded rows.
|
||||
Re-syncs ``sort_order``, empty ``label``, and seed ``is_custom`` /
|
||||
``mapping_type`` so newly inserted codes keep the intended table order
|
||||
and 3q-1…3q-6 stay classified as custom type.
|
||||
"""
|
||||
inserted = 0
|
||||
dirty = False
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
for idx, seed in enumerate(DEFAULT_BUSINESS_ROWS):
|
||||
is_custom, mapping_type = _seed_type(seed)
|
||||
existing = await session.get(BusinessMappingRow, seed["business_code"])
|
||||
if existing is not None:
|
||||
if existing.sort_order != idx:
|
||||
@@ -129,11 +187,19 @@ class BusinessMappingRepository:
|
||||
if seed_label and not (existing.label or "").strip():
|
||||
existing.label = seed_label
|
||||
dirty = True
|
||||
if bool(existing.is_custom) != is_custom:
|
||||
existing.is_custom = is_custom
|
||||
dirty = True
|
||||
if (existing.mapping_type or "") != mapping_type:
|
||||
existing.mapping_type = mapping_type
|
||||
dirty = True
|
||||
continue
|
||||
session.add(
|
||||
BusinessMappingRow(
|
||||
business_code=seed["business_code"],
|
||||
label=seed.get("label", ""),
|
||||
mapping_type=mapping_type,
|
||||
is_custom=is_custom,
|
||||
agent_id=None,
|
||||
agent_name=None,
|
||||
chain_id=None,
|
||||
|
||||
+1
@@ -22,6 +22,7 @@ def _normalize(data: dict[str, Any], menu_type: str, updated_by: str | None) ->
|
||||
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"label": (data.get("label") or None),
|
||||
"disabled": bool(data.get("disabled", False)),
|
||||
"updated_by": updated_by,
|
||||
"updated_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
+5
-3
@@ -1,14 +1,15 @@
|
||||
"""ORM model for compact/sidebar-summary menu overrides.
|
||||
|
||||
This table only controls the compact left menu presentation. The full sidebar
|
||||
menu layout continues to use ``menu_overrides`` and is unaffected.
|
||||
This table only controls compact menu presentation: lane, order, label and
|
||||
enablement for the left menu, settings menu and toolbox. The full sidebar menu
|
||||
layout continues to use ``menu_overrides`` and is unaffected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, Integer, String
|
||||
from sqlalchemy import Boolean, Index, Integer, String, false
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -24,6 +25,7 @@ class CompactMenuOverrideRow(Base):
|
||||
placement: Mapped[str] = mapped_column(String(32), default="sidebar")
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0)
|
||||
label: Mapped[str | None] = mapped_column(String(255))
|
||||
disabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
|
||||
+2
@@ -41,6 +41,7 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C
|
||||
placement=placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
sort_order=int(data.get("sort_order", 0)),
|
||||
label=(data.get("label") or None),
|
||||
disabled=bool(data.get("disabled", False)),
|
||||
updated_by=updated_by,
|
||||
)
|
||||
|
||||
@@ -53,6 +54,7 @@ def _normalize_user_row(data: dict[str, Any], menu_type: str) -> dict[str, Any]:
|
||||
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"label": (data.get("label") or None),
|
||||
"disabled": bool(data.get("disabled", False)),
|
||||
}
|
||||
|
||||
|
||||
|
||||
+418
-227
@@ -11,14 +11,20 @@ None and fall back to in-memory implementations.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import weakref
|
||||
from contextvars import ContextVar, Token
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy import event, text
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
@@ -38,21 +44,167 @@ _session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
# "got Future attached to a different loop". We capture the engine's home loop here so
|
||||
# best-effort writes coming from worker threads / other loops can be routed back to it
|
||||
# (see ``fire_and_forget_db_write``).
|
||||
_engine_loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
# These are the complete workflow-studio persistence tables owned by DeerFlow.
|
||||
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
|
||||
# wildcard: another application might share a schema and have its own table
|
||||
# with that prefix.
|
||||
_WORKFLOW_SCHEMA_TABLE_NAMES = (
|
||||
"workflow_definitions",
|
||||
"workflow_versions",
|
||||
"workflow_runs",
|
||||
"workflow_node_runs",
|
||||
"workflow_run_events",
|
||||
"workflow_run_artifacts",
|
||||
"workflow_data_sources",
|
||||
)
|
||||
_engine_loop: asyncio.AbstractEventLoop | None = None
|
||||
_performance_request_context: ContextVar[tuple[str, Any] | None] = ContextVar("deerflow_performance_request", default=None)
|
||||
|
||||
|
||||
def bind_performance_request(request_id: str) -> Token:
|
||||
"""Bind a request ID to this task so slow SQL logs can be correlated safely."""
|
||||
|
||||
try:
|
||||
task = asyncio.current_task()
|
||||
except RuntimeError:
|
||||
task = None
|
||||
return _performance_request_context.set((request_id, weakref.ref(task) if task is not None else None))
|
||||
|
||||
|
||||
def reset_performance_request(token: Token) -> None:
|
||||
_performance_request_context.reset(token)
|
||||
|
||||
|
||||
def _current_performance_request_id() -> str:
|
||||
context = _performance_request_context.get()
|
||||
if context is None:
|
||||
return "-"
|
||||
try:
|
||||
task = asyncio.current_task()
|
||||
except RuntimeError:
|
||||
task = None
|
||||
return context[0] if task is not None and context[1] is not None and context[1]() is task else "-"
|
||||
|
||||
|
||||
def _slow_sql_threshold_ms() -> float:
|
||||
try:
|
||||
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_SQL_LOG_MS", "500")), 60_000.0))
|
||||
except (TypeError, ValueError):
|
||||
return 500.0
|
||||
|
||||
|
||||
def _sql_operation(statement: str) -> str:
|
||||
parts = statement.lstrip().split(None, 1)
|
||||
candidate = parts[0].upper() if parts else "EMPTY"
|
||||
return candidate if re.fullmatch(r"[A-Z_]+", candidate) else "OTHER"
|
||||
|
||||
|
||||
def _sql_table(statement: str) -> str:
|
||||
"""Return the target/source table only, without query text or values."""
|
||||
match = re.search(
|
||||
r"\b(?:INSERT\s+INTO|UPDATE|DELETE\s+FROM)\s+([`\"\w.]+)",
|
||||
statement,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if match is None:
|
||||
match = re.search(r"\bFROM\s+([`\"\w.]+)", statement, re.IGNORECASE)
|
||||
if match is None:
|
||||
return "unknown"
|
||||
return match.group(1).strip("`\".")[-128:] or "unknown"
|
||||
|
||||
|
||||
def _sql_fingerprint(statement: str) -> str:
|
||||
return hashlib.sha256(statement.encode("utf-8", "replace")).hexdigest()[:12]
|
||||
|
||||
|
||||
def _database_error_metadata(exception: BaseException | None) -> tuple[str, str, str]:
|
||||
"""Extract safe driver diagnostics without logging SQL or bound values."""
|
||||
if exception is None:
|
||||
return "unknown", "-", "-"
|
||||
|
||||
code = getattr(exception, "errno", None) or getattr(exception, "code", None)
|
||||
args = getattr(exception, "args", ())
|
||||
if code is None and args and isinstance(args[0], (int, str)):
|
||||
candidate = str(args[0])
|
||||
if candidate.isdigit():
|
||||
code = candidate
|
||||
|
||||
sqlstate = getattr(exception, "sqlstate", None) or getattr(exception, "pgcode", None)
|
||||
diagnostic = getattr(exception, "diag", None)
|
||||
constraint = getattr(diagnostic, "constraint_name", None)
|
||||
|
||||
def _safe_identifier(value: Any) -> str:
|
||||
text_value = str(value or "")
|
||||
return text_value[:128] if re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", text_value) else "-"
|
||||
|
||||
return _safe_identifier(code), _safe_identifier(sqlstate), _safe_identifier(constraint)
|
||||
|
||||
|
||||
def _install_slow_sql_logging(engine: AsyncEngine, backend: str) -> None:
|
||||
"""Log slow SQL timings without logging query text or bound values."""
|
||||
|
||||
sync_engine = engine.sync_engine
|
||||
|
||||
def before_cursor_execute(_conn, _cursor, statement, _parameters, context, _executemany):
|
||||
if context is not None:
|
||||
context._deerflow_perf_started = time.perf_counter()
|
||||
context._deerflow_perf_operation = _sql_operation(statement)
|
||||
context._deerflow_perf_statement = statement
|
||||
|
||||
def after_cursor_execute(_conn, _cursor, _statement, _parameters, context, executemany):
|
||||
if context is None:
|
||||
return
|
||||
started = getattr(context, "_deerflow_perf_started", None)
|
||||
if started is None:
|
||||
return
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
if elapsed_ms >= _slow_sql_threshold_ms():
|
||||
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", ""))
|
||||
logger.warning(
|
||||
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%.2f executemany=%s fingerprint=%s outcome=ok",
|
||||
_current_performance_request_id(),
|
||||
backend,
|
||||
getattr(context, "_deerflow_perf_operation", "OTHER"),
|
||||
_sql_table(getattr(context, "_deerflow_perf_statement", "")),
|
||||
elapsed_ms,
|
||||
bool(executemany),
|
||||
fingerprint,
|
||||
)
|
||||
|
||||
def handle_error(exception_context):
|
||||
context = exception_context.execution_context
|
||||
started = getattr(context, "_deerflow_perf_started", None) if context is not None else None
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2) if started is not None else None
|
||||
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
|
||||
original_exception = exception_context.original_exception
|
||||
error_code, sqlstate, constraint = _database_error_metadata(original_exception)
|
||||
table = _sql_table(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
|
||||
duplicate_key = error_code == "1062" or sqlstate == "23505" or (
|
||||
table == "scheduled_task_runs"
|
||||
and "UNIQUE constraint failed" in str(original_exception or "")
|
||||
)
|
||||
log = logger.info if table == "scheduled_task_runs" and duplicate_key else logger.error
|
||||
outcome = "duplicate_ignored" if table == "scheduled_task_runs" and duplicate_key else "error"
|
||||
log(
|
||||
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%s executemany=%s fingerprint=%s outcome=%s error_type=%s error_code=%s sqlstate=%s constraint=%s",
|
||||
_current_performance_request_id(),
|
||||
backend,
|
||||
getattr(context, "_deerflow_perf_operation", "OTHER") if context is not None else "OTHER",
|
||||
table,
|
||||
elapsed_ms,
|
||||
bool(getattr(context, "executemany", False)) if context is not None else False,
|
||||
fingerprint,
|
||||
outcome,
|
||||
type(original_exception).__name__ if original_exception is not None else "unknown",
|
||||
error_code,
|
||||
sqlstate,
|
||||
constraint,
|
||||
)
|
||||
|
||||
event.listen(sync_engine, "before_cursor_execute", before_cursor_execute)
|
||||
event.listen(sync_engine, "after_cursor_execute", after_cursor_execute)
|
||||
event.listen(sync_engine, "handle_error", handle_error)
|
||||
|
||||
# These are the complete workflow-studio persistence tables owned by DeerFlow.
|
||||
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
|
||||
# wildcard: another application might share a schema and have its own table
|
||||
# with that prefix.
|
||||
_WORKFLOW_SCHEMA_TABLE_NAMES = (
|
||||
"workflow_definitions",
|
||||
"workflow_versions",
|
||||
"workflow_runs",
|
||||
"workflow_node_runs",
|
||||
"workflow_run_events",
|
||||
"workflow_run_artifacts",
|
||||
"workflow_data_sources",
|
||||
)
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
@@ -81,6 +233,42 @@ def _mysql_driver_name(url: str) -> str:
|
||||
return "asyncmy"
|
||||
|
||||
|
||||
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
|
||||
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
|
||||
kwargs = dict(
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
|
||||
# These are not MySQL/psycopg connection arguments.
|
||||
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
|
||||
kwargs["connect_args"] = {"command_timeout": command_timeout}
|
||||
return kwargs
|
||||
|
||||
|
||||
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
|
||||
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
|
||||
kwargs = dict(
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
|
||||
# These are not MySQL/psycopg connection arguments.
|
||||
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
|
||||
kwargs["connect_args"] = {"command_timeout": command_timeout}
|
||||
return kwargs
|
||||
|
||||
|
||||
async def _run_alembic_upgrade(url: str) -> None:
|
||||
"""Apply pending Alembic migrations against ``url`` (``alembic upgrade head``).
|
||||
|
||||
@@ -176,7 +364,7 @@ async def _auto_create_postgres_db(url: str) -> None:
|
||||
try:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
|
||||
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
|
||||
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
@@ -215,7 +403,7 @@ async def _auto_create_mysql_db(url: str) -> None:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
"""Add ORM-declared columns that are missing from already-existing tables.
|
||||
|
||||
``Base.metadata.create_all`` creates brand-new tables but never patches
|
||||
@@ -226,14 +414,14 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
|
||||
This sweep compares each existing table against the ORM metadata and issues
|
||||
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
|
||||
safely (nullable, or carrying a server_default). Missing ORM indexes are
|
||||
created as well. Table-level ``UniqueConstraint`` objects
|
||||
safely (nullable, or carrying a server_default). Missing ORM indexes are
|
||||
created as well. Table-level ``UniqueConstraint`` objects
|
||||
that are declared on the model but missing from the table are also created
|
||||
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
|
||||
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
|
||||
on an existing database, weakening dedupe.
|
||||
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
|
||||
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
|
||||
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
|
||||
``BeijingDateTime``) resolve to the right backend type. Everything is
|
||||
per-column best-effort: a failure is logged and startup continues.
|
||||
@@ -258,10 +446,8 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
continue
|
||||
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
|
||||
missing = [c for c in table.columns if c.name not in existing_cols]
|
||||
if not missing:
|
||||
continue
|
||||
|
||||
for col in missing:
|
||||
for col in missing:
|
||||
if not col.nullable and col.server_default is None:
|
||||
logger.warning(
|
||||
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
|
||||
@@ -270,21 +456,21 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
)
|
||||
continue
|
||||
try:
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
|
||||
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx:
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if not idx.name or idx.name in existing_idx:
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
logger.info("Auto-created index %s on %s", idx.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
|
||||
@@ -328,67 +514,67 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
f"ON {preparer.quote(table.name)} ({cols})"
|
||||
)
|
||||
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
|
||||
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
|
||||
|
||||
Alembic auto-upgrade is intentionally disabled in deployed Gateway
|
||||
processes, while the ORM column synchronizer is always run. The latter
|
||||
can add the task lookup columns and indexes but cannot derive values from
|
||||
JSON metadata. Backfill task rows once at startup so an upgrade never
|
||||
temporarily hides every historical task conversation from ``/threads/search``.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
table_name = "threads_meta"
|
||||
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
|
||||
try:
|
||||
inspector = sa_inspect(sync_conn)
|
||||
if table_name not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns(table_name)}
|
||||
if not required_columns <= columns:
|
||||
return
|
||||
|
||||
dialect = sync_conn.dialect.name
|
||||
if dialect == "sqlite":
|
||||
task_id = "json_extract(metadata_json, '$.taskId')"
|
||||
agent_id = "json_extract(metadata_json, '$.agent_id')"
|
||||
action_id = "json_extract(metadata_json, '$.actionId')"
|
||||
valid_json = "json_valid(metadata_json)"
|
||||
elif dialect == "mysql":
|
||||
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
|
||||
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
|
||||
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
|
||||
valid_json = "JSON_VALID(metadata_json)"
|
||||
elif dialect in {"postgresql", "opengauss"}:
|
||||
task_id = "metadata_json ->> 'taskId'"
|
||||
agent_id = "metadata_json ->> 'agent_id'"
|
||||
action_id = "metadata_json ->> 'actionId'"
|
||||
valid_json = "TRUE"
|
||||
else:
|
||||
return
|
||||
|
||||
result = sync_conn.execute(
|
||||
text(
|
||||
"UPDATE threads_meta "
|
||||
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
|
||||
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
|
||||
)
|
||||
)
|
||||
if result.rowcount:
|
||||
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
|
||||
except Exception:
|
||||
# A malformed legacy JSON row or a backend-specific JSON function must
|
||||
# not prevent startup. The generic metadata search remains available,
|
||||
# and the failure is visible to the operator for remediation.
|
||||
logger.exception("Failed to backfill thread task lookup projections")
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
|
||||
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
|
||||
|
||||
Alembic auto-upgrade is intentionally disabled in deployed Gateway
|
||||
processes, while the ORM column synchronizer is always run. The latter
|
||||
can add the task lookup columns and indexes but cannot derive values from
|
||||
JSON metadata. Backfill task rows once at startup so an upgrade never
|
||||
temporarily hides every historical task conversation from ``/threads/search``.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
table_name = "threads_meta"
|
||||
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
|
||||
try:
|
||||
inspector = sa_inspect(sync_conn)
|
||||
if table_name not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns(table_name)}
|
||||
if not required_columns <= columns:
|
||||
return
|
||||
|
||||
dialect = sync_conn.dialect.name
|
||||
if dialect == "sqlite":
|
||||
task_id = "json_extract(metadata_json, '$.taskId')"
|
||||
agent_id = "json_extract(metadata_json, '$.agent_id')"
|
||||
action_id = "json_extract(metadata_json, '$.actionId')"
|
||||
valid_json = "json_valid(metadata_json)"
|
||||
elif dialect == "mysql":
|
||||
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
|
||||
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
|
||||
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
|
||||
valid_json = "JSON_VALID(metadata_json)"
|
||||
elif dialect in {"postgresql", "opengauss"}:
|
||||
task_id = "metadata_json ->> 'taskId'"
|
||||
agent_id = "metadata_json ->> 'agent_id'"
|
||||
action_id = "metadata_json ->> 'actionId'"
|
||||
valid_json = "TRUE"
|
||||
else:
|
||||
return
|
||||
|
||||
result = sync_conn.execute(
|
||||
text(
|
||||
"UPDATE threads_meta "
|
||||
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
|
||||
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
|
||||
)
|
||||
)
|
||||
if result.rowcount:
|
||||
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
|
||||
except Exception:
|
||||
# A malformed legacy JSON row or a backend-specific JSON function must
|
||||
# not prevent startup. The generic metadata search remains available,
|
||||
# and the failure is visible to the operator for remediation.
|
||||
logger.exception("Failed to backfill thread task lookup projections")
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
|
||||
@@ -441,24 +627,32 @@ async def init_engine(
|
||||
pool_size: int = 5,
|
||||
max_overflow: int = 10,
|
||||
pool_timeout: int = 30,
|
||||
pool_recycle: int = 1800,
|
||||
pool_recycle: int | None = None,
|
||||
command_timeout: float | None = 30,
|
||||
sqlite_dir: str = "",
|
||||
) -> None:
|
||||
"""Create the async engine and session factory, then auto-create tables.
|
||||
|
||||
Args:
|
||||
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
|
||||
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
|
||||
echo: Echo SQL to log.
|
||||
pool_size: Base connection pool size (postgres/gauss/mysql).
|
||||
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
|
||||
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
|
||||
pool_size: Base connection pool size (postgres/gauss/mysql).
|
||||
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
|
||||
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
|
||||
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
|
||||
command_timeout: asyncpg command deadline, including pool pre-ping (None disables).
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
"""
|
||||
global _engine, _session_factory, _engine_loop
|
||||
|
||||
if pool_recycle is None:
|
||||
pool_recycle = 300 if backend == "postgres" else 1800
|
||||
|
||||
if pool_recycle is None:
|
||||
pool_recycle = 300 if backend == "postgres" else 1800
|
||||
|
||||
if backend == "memory":
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
@@ -475,25 +669,25 @@ async def init_engine(
|
||||
except RuntimeError:
|
||||
_engine_loop = None
|
||||
|
||||
if backend in {"postgres", "gauss"}:
|
||||
if backend in {"postgres", "gauss"}:
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
extra = "gauss" if backend == "gauss" else "postgres"
|
||||
raise ImportError(
|
||||
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
|
||||
f"Install it with:\n uv sync --extra {extra}\n"
|
||||
"Or switch to backend: sqlite in config.yaml for single-node deployment."
|
||||
) from None
|
||||
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
|
||||
"is not installed.\nInstall it with:\n uv sync --extra gauss"
|
||||
)
|
||||
if backend == "gauss":
|
||||
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
|
||||
|
||||
install_gauss_dialect_compatibility()
|
||||
extra = "gauss" if backend == "gauss" else "postgres"
|
||||
raise ImportError(
|
||||
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
|
||||
f"Install it with:\n uv sync --extra {extra}\n"
|
||||
"Or switch to backend: sqlite in config.yaml for single-node deployment."
|
||||
) from None
|
||||
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
|
||||
"is not installed.\nInstall it with:\n uv sync --extra gauss"
|
||||
)
|
||||
if backend == "gauss":
|
||||
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
|
||||
|
||||
install_gauss_dialect_compatibility()
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
@@ -509,10 +703,6 @@ async def init_engine(
|
||||
install_gauss_mysql_dialect_compatibility(driver)
|
||||
|
||||
if backend == "sqlite":
|
||||
import os
|
||||
|
||||
from sqlalchemy import event
|
||||
|
||||
os.makedirs(sqlite_dir or ".", exist_ok=True)
|
||||
_engine = create_async_engine(url, echo=echo, json_serializer=_json_serializer)
|
||||
|
||||
@@ -537,31 +727,28 @@ async def init_engine(
|
||||
cursor.execute("PRAGMA foreign_keys=ON;")
|
||||
finally:
|
||||
cursor.close()
|
||||
elif backend in {"postgres", "gauss"}:
|
||||
elif backend in {"postgres", "gauss"}:
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
)
|
||||
elif backend == "mysql":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
|
||||
_install_slow_sql_logging(_engine, backend)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
# Auto-create tables for fresh deployments, then let Alembic apply any
|
||||
@@ -612,14 +799,13 @@ async def init_engine(
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
)
|
||||
_install_slow_sql_logging(_engine, backend)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await _create_all_with_product_integrity(conn)
|
||||
@@ -630,14 +816,13 @@ async def init_engine(
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
)
|
||||
_install_slow_sql_logging(_engine, backend)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await _create_all_with_product_integrity(conn)
|
||||
@@ -675,7 +860,7 @@ async def init_engine(
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
# await _run_alembic_upgrade(url)
|
||||
|
||||
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
|
||||
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
|
||||
|
||||
|
||||
async def init_engine_from_config(config) -> None:
|
||||
@@ -690,14 +875,17 @@ async def init_engine_from_config(config) -> None:
|
||||
pool_size=config.pool_size,
|
||||
max_overflow=getattr(config, "max_overflow", 10),
|
||||
pool_timeout=getattr(config, "pool_timeout", 30),
|
||||
pool_recycle=getattr(config, "pool_recycle", 1800),
|
||||
pool_recycle=getattr(config, "pool_recycle", None),
|
||||
command_timeout=getattr(config, "command_timeout", 30),
|
||||
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
|
||||
)
|
||||
|
||||
|
||||
def get_session_factory() -> async_sessionmaker[AsyncSession] | None:
|
||||
"""Return the async session factory, or None if backend=memory."""
|
||||
return _session_factory
|
||||
from deerflow.persistence.loop_isolation import isolated_session_factory
|
||||
|
||||
return isolated_session_factory() or _session_factory
|
||||
|
||||
|
||||
def get_engine() -> AsyncEngine | None:
|
||||
@@ -705,73 +893,73 @@ def get_engine() -> AsyncEngine | None:
|
||||
return _engine
|
||||
|
||||
|
||||
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
|
||||
"""Return the event loop that owns the engine's connection pool (or None)."""
|
||||
return _engine_loop
|
||||
|
||||
|
||||
async def rebuild_workflow_schema_for_mysql() -> list[str]:
|
||||
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
|
||||
|
||||
This is a deliberately narrow recovery operation for an unused Workflow
|
||||
Studio installation. It never uses a prefix wildcard, never changes the
|
||||
selected database, and refuses to run unless the engine is connected to
|
||||
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
|
||||
|
||||
Returns the allow-listed workflow tables that existed before the rebuild.
|
||||
"""
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
|
||||
if engine.dialect.name != "mysql":
|
||||
raise RuntimeError(
|
||||
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
|
||||
)
|
||||
|
||||
expected_database = engine.url.database
|
||||
if not expected_database:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
|
||||
|
||||
# Make the operation safe when this helper is invoked independently from
|
||||
# normal app startup as well.
|
||||
import deerflow.persistence.models # noqa: F401 — register ORM models
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
|
||||
if missing_models:
|
||||
raise RuntimeError(
|
||||
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
|
||||
)
|
||||
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
|
||||
|
||||
def _rebuild(sync_conn) -> list[str]:
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
|
||||
if current_database != expected_database:
|
||||
raise RuntimeError(
|
||||
"Refusing workflow schema rebuild: connected database "
|
||||
f"{current_database!r} does not match configured database {expected_database!r}"
|
||||
)
|
||||
|
||||
existing = set(sa_inspect(sync_conn).get_table_names())
|
||||
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
|
||||
# SQLAlchemy orders parent/child tables around the foreign key from
|
||||
# workflow_versions to workflow_definitions. Only tables in the exact
|
||||
# allow-list above are ever passed to drop_all/create_all.
|
||||
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
return present
|
||||
|
||||
async with engine.begin() as conn:
|
||||
rebuilt = await conn.run_sync(_rebuild)
|
||||
|
||||
logger.warning(
|
||||
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
|
||||
expected_database,
|
||||
", ".join(rebuilt) if rebuilt else "none (created fresh)",
|
||||
)
|
||||
return rebuilt
|
||||
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
|
||||
"""Return the event loop that owns the engine's connection pool (or None)."""
|
||||
return _engine_loop
|
||||
|
||||
|
||||
async def rebuild_workflow_schema_for_mysql() -> list[str]:
|
||||
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
|
||||
|
||||
This is a deliberately narrow recovery operation for an unused Workflow
|
||||
Studio installation. It never uses a prefix wildcard, never changes the
|
||||
selected database, and refuses to run unless the engine is connected to
|
||||
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
|
||||
|
||||
Returns the allow-listed workflow tables that existed before the rebuild.
|
||||
"""
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
|
||||
if engine.dialect.name != "mysql":
|
||||
raise RuntimeError(
|
||||
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
|
||||
)
|
||||
|
||||
expected_database = engine.url.database
|
||||
if not expected_database:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
|
||||
|
||||
# Make the operation safe when this helper is invoked independently from
|
||||
# normal app startup as well.
|
||||
import deerflow.persistence.models # noqa: F401 — register ORM models
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
|
||||
if missing_models:
|
||||
raise RuntimeError(
|
||||
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
|
||||
)
|
||||
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
|
||||
|
||||
def _rebuild(sync_conn) -> list[str]:
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
|
||||
if current_database != expected_database:
|
||||
raise RuntimeError(
|
||||
"Refusing workflow schema rebuild: connected database "
|
||||
f"{current_database!r} does not match configured database {expected_database!r}"
|
||||
)
|
||||
|
||||
existing = set(sa_inspect(sync_conn).get_table_names())
|
||||
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
|
||||
# SQLAlchemy orders parent/child tables around the foreign key from
|
||||
# workflow_versions to workflow_definitions. Only tables in the exact
|
||||
# allow-list above are ever passed to drop_all/create_all.
|
||||
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
return present
|
||||
|
||||
async with engine.begin() as conn:
|
||||
rebuilt = await conn.run_sync(_rebuild)
|
||||
|
||||
logger.warning(
|
||||
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
|
||||
expected_database,
|
||||
", ".join(rebuilt) if rebuilt else "none (created fresh)",
|
||||
)
|
||||
return rebuilt
|
||||
|
||||
|
||||
def fire_and_forget_db_write(coro) -> None:
|
||||
@@ -820,10 +1008,13 @@ def _run_coro_on_throwaway_loop(coro):
|
||||
|
||||
Only reached when there is no usable engine home loop, or when the caller is
|
||||
*synchronously* inside the home loop (routing back would deadlock). This
|
||||
touches the shared connection pool from a throwaway loop, so it can still leak
|
||||
a dead-loop-bound connection — but the common worker-thread path no longer
|
||||
gets here (see ``run_db_blocking``).
|
||||
uses an isolated, non-pooled engine for compatibility reads. The coroutine
|
||||
must obtain its session factory inside its async body so the isolated
|
||||
context applies; no connection is returned to the Gateway pool.
|
||||
"""
|
||||
from deerflow.persistence.loop_isolation import run_isolated_db_read
|
||||
|
||||
coro = run_isolated_db_read(coro)
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
"""Bound JSON predicates for SQLite, PostgreSQL and MySQL.
|
||||
|
||||
Adapted from upstream #2865. MySQL business tables store JSON in TEXT, so
|
||||
these predicates use JSON functions rather than assuming a native JSON column.
|
||||
Unsupported keys/values must stay in the repository's residual filter; never
|
||||
silently discard a predicate (in particular a shared task's taskId filter).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import BigInteger, Float, String, bindparam
|
||||
from sqlalchemy.ext.compiler import compiles
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.visitors import InternalTraversal
|
||||
from sqlalchemy.types import Boolean
|
||||
|
||||
_SAFE_KEY = re.compile(r"[A-Za-z0-9_-]+\Z")
|
||||
_STRIP_CHARS = "".join(chr(i) for i in range(0x3001) if chr(i).isspace())
|
||||
|
||||
|
||||
def supports_json_filter(key: Any, value: Any) -> bool:
|
||||
if not isinstance(key, str) or not _SAFE_KEY.fullmatch(key):
|
||||
return False
|
||||
if value is None or isinstance(value, (str, bool)):
|
||||
return True
|
||||
if isinstance(value, int):
|
||||
return -(2**63) <= value < 2**63
|
||||
return isinstance(value, float) and math.isfinite(value)
|
||||
|
||||
|
||||
class JsonPredicate(ColumnElement):
|
||||
"""A scalar equality, Python truthiness, or nonblank-string predicate."""
|
||||
|
||||
inherit_cache = True
|
||||
type = Boolean()
|
||||
_is_implicitly_boolean = True
|
||||
_traverse_internals = [
|
||||
("column", InternalTraversal.dp_clauseelement),
|
||||
("key", InternalTraversal.dp_string),
|
||||
("operation", InternalTraversal.dp_string),
|
||||
("value", InternalTraversal.dp_plain_obj),
|
||||
("value_kind", InternalTraversal.dp_string),
|
||||
]
|
||||
|
||||
def __init__(self, column, key: str, operation: str = "equal", value: Any = None):
|
||||
if not supports_json_filter(key, value):
|
||||
raise ValueError("Unsupported JSON filter key or value")
|
||||
if operation not in {"equal", "truthy", "nonblank"}:
|
||||
raise ValueError("Unsupported JSON predicate operation")
|
||||
self.column = column
|
||||
self.key = key
|
||||
self.operation = operation
|
||||
self.value = value
|
||||
# Keep Python value types separate in SQLAlchemy's compiled cache.
|
||||
self.value_kind = type(value).__name__
|
||||
super().__init__()
|
||||
|
||||
|
||||
def _compile(element, compiler, family, **kw):
|
||||
if not _SAFE_KEY.fullmatch(element.key):
|
||||
raise ValueError("Unsafe JSON key")
|
||||
column = compiler.process(element.column, **kw)
|
||||
|
||||
def bound(value, type_=String()):
|
||||
return compiler.process(bindparam(None, value, type_=type_), **kw)
|
||||
|
||||
if family == "postgres":
|
||||
raw = f"({column} -> '{element.key}')"
|
||||
kind = f"json_typeof({raw})"
|
||||
text = f"({column} ->> '{element.key}')"
|
||||
string_kind, number_kinds = "string", "'number'"
|
||||
else:
|
||||
path = f"'$.\"{element.key}\"'"
|
||||
raw = f"json_extract({column}, {path})"
|
||||
if family == "mysql":
|
||||
kind, text = f"lower(json_type({raw}))", f"json_unquote({raw})"
|
||||
string_kind, number_kinds = "string", "'integer', 'double', 'decimal'"
|
||||
else:
|
||||
kind, text = f"json_type({column}, {path})", raw
|
||||
string_kind, number_kinds = "text", "'integer', 'real'"
|
||||
|
||||
if element.operation == "nonblank":
|
||||
if family == "mysql":
|
||||
# MySQL 5.7 regexes are byte-oriented; a Unicode whitespace regex
|
||||
# can misclassify CJK text. Removing each complete whitespace
|
||||
# character preserves str.strip()'s nonempty test on 5.7 and 8.x.
|
||||
stripped = text
|
||||
for char in _STRIP_CHARS:
|
||||
stripped = f"replace({stripped}, {bound(char)}, '')"
|
||||
has_text = f"length({stripped}) > 0"
|
||||
else:
|
||||
trim = "btrim" if family == "postgres" else "trim"
|
||||
# SQLite length(TEXT) stops at NUL, unlike Python str.strip().
|
||||
has_text = f"{trim}({text}, {bound(_STRIP_CHARS)}) <> ''"
|
||||
clause = f"({kind} = '{string_kind}' AND {has_text})"
|
||||
elif element.operation == "truthy":
|
||||
if family == "sqlite":
|
||||
containers = f"json({raw}) NOT IN ('[]', '{{}}')"
|
||||
boolean = f"{kind} = 'true'"
|
||||
elif family == "mysql":
|
||||
containers = f"json_length({raw}) > 0"
|
||||
boolean = f"{text} = 'true'"
|
||||
else:
|
||||
containers = f"CAST({raw} AS jsonb) NOT IN ('[]'::jsonb, '{{}}'::jsonb)"
|
||||
boolean = f"{text} = 'true'"
|
||||
cast = "NUMERIC" if family == "postgres" else "DOUBLE PRECISION"
|
||||
number = f"({text} + 0e0)" if family == "mysql" else f"CAST({text} AS {cast})"
|
||||
nonempty = f"{text} <> ''" if family == "sqlite" else f"length({text}) > 0"
|
||||
clause = (
|
||||
f"CASE WHEN {kind} IN ({number_kinds}) THEN {number} <> 0 "
|
||||
f"WHEN {kind} = '{string_kind}' THEN {nonempty} "
|
||||
f"WHEN {kind} IN ('array', 'object') THEN {containers} "
|
||||
f"WHEN {kind} IN ('boolean', 'true', 'false') THEN {boolean} ELSE FALSE END"
|
||||
)
|
||||
elif element.value is None:
|
||||
# Preserve this project's md.get(key) == None semantics for old rows.
|
||||
clause = f"({kind} IS NULL OR {kind} = 'null')"
|
||||
elif isinstance(element.value, (bool, int, float)):
|
||||
if family != "sqlite":
|
||||
raise NotImplementedError("Server numeric metadata requires the exact residual filter")
|
||||
value = int(element.value) if isinstance(element.value, bool) else element.value
|
||||
# Preserve md.get(key) == value: Python considers True == 1 == 1.0.
|
||||
# Text ids remain distinct from numbers. SQLite compares its extracted
|
||||
# integers/reals natively; server numeric filters use the residual path.
|
||||
param = bound(value, Float() if isinstance(value, float) else BigInteger())
|
||||
clause = f"({kind} IN ({number_kinds}, 'true', 'false') AND {text} = {param})"
|
||||
else:
|
||||
# Binary comparison is essential for task/agent ids: MySQL TEXT can
|
||||
# inherit a case-insensitive collation and ignores trailing spaces.
|
||||
extract = f"CAST({text} AS BINARY)" if family == "mysql" else text
|
||||
param = bound(element.value)
|
||||
if family == "mysql":
|
||||
param = f"CAST({param} AS BINARY)"
|
||||
clause = f"({kind} = '{string_kind}' AND {extract} = {param})"
|
||||
# Missing/null values must remain false under NOT (system exclusions).
|
||||
return f"COALESCE(({clause}), FALSE)"
|
||||
|
||||
|
||||
@compiles(JsonPredicate, "sqlite")
|
||||
def _sqlite(element, compiler, **kw):
|
||||
return _compile(element, compiler, "sqlite", **kw)
|
||||
|
||||
|
||||
@compiles(JsonPredicate, "postgresql")
|
||||
def _postgres(element, compiler, **kw):
|
||||
return _compile(element, compiler, "postgres", **kw)
|
||||
|
||||
|
||||
@compiles(JsonPredicate, "mysql")
|
||||
def _mysql(element, compiler, **kw):
|
||||
return _compile(element, compiler, "mysql", **kw)
|
||||
|
||||
|
||||
@compiles(JsonPredicate)
|
||||
def _unsupported(element, compiler, **kw):
|
||||
raise NotImplementedError(f"JSON predicates unsupported for {compiler.dialect.name}")
|
||||
+437
@@ -0,0 +1,437 @@
|
||||
"""Private knowledge-base Q&A sessions with shared Markdown publications.
|
||||
|
||||
This persistence unit deliberately contains no Gateway/application imports. A
|
||||
session and its LangGraph thread belong to exactly one user. Once published,
|
||||
only the saved Markdown snapshot is shared with readers of the bound knowledge
|
||||
base; the private draft and thread identifier never become publication data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import String, delete, select, update
|
||||
from sqlalchemy.exc import OperationalError, ProgrammingError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class KnowledgeResearchSessionRow(Base):
|
||||
__tablename__ = "knowledge_research_sessions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
# A separate opaque id is the only identity exposed on the shared bookshelf.
|
||||
publication_id: Mapped[str] = mapped_column(String(36), unique=True, index=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(128), unique=True, index=True)
|
||||
knowledge_base_id: Mapped[str] = mapped_column(String(128), index=True)
|
||||
user_id: Mapped[str] = mapped_column(String(255), index=True)
|
||||
title: Mapped[str] = mapped_column(String(255))
|
||||
draft_markdown: Mapped[str] = mapped_column(PortableLongText, default="")
|
||||
|
||||
wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
wiki_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
wiki_folder_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
saved_title: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
saved_markdown: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
saved_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
saved_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
nullable=False,
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
|
||||
def _is_missing_table(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return (
|
||||
"no such table" in message
|
||||
or "doesn't exist" in message
|
||||
or "does not exist" in message
|
||||
or "unknown table" in message
|
||||
)
|
||||
|
||||
|
||||
def _iso(value: Any) -> Any:
|
||||
return value.isoformat() if isinstance(value, datetime) else value
|
||||
|
||||
|
||||
def _row_dict(row: KnowledgeResearchSessionRow | dict[str, Any]) -> dict[str, Any]:
|
||||
if isinstance(row, dict):
|
||||
return deepcopy(row)
|
||||
return {
|
||||
"id": row.id,
|
||||
"publication_id": row.publication_id,
|
||||
"thread_id": row.thread_id,
|
||||
"knowledge_base_id": row.knowledge_base_id,
|
||||
"user_id": row.user_id,
|
||||
"title": row.title,
|
||||
"draft_markdown": row.draft_markdown or "",
|
||||
"wiki_slug": row.wiki_slug,
|
||||
"wiki_page_id": row.wiki_page_id,
|
||||
"wiki_folder_id": row.wiki_folder_id,
|
||||
"saved_title": row.saved_title,
|
||||
"saved_markdown": row.saved_markdown,
|
||||
"saved_hash": row.saved_hash,
|
||||
"saved_at": _iso(row.saved_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
class _PublicationLease:
|
||||
"""One transaction-scoped mutable publication row."""
|
||||
|
||||
def __init__(self, row: KnowledgeResearchSessionRow | dict[str, Any]) -> None:
|
||||
self._row = row
|
||||
|
||||
def snapshot(self) -> dict[str, Any]:
|
||||
return _row_dict(self._row)
|
||||
|
||||
def save(
|
||||
self,
|
||||
*,
|
||||
title: str,
|
||||
markdown: str,
|
||||
saved_hash: str,
|
||||
wiki_slug: str,
|
||||
wiki_page_id: str | None,
|
||||
wiki_folder_id: str | None,
|
||||
) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
values = {
|
||||
"wiki_slug": wiki_slug,
|
||||
"wiki_page_id": wiki_page_id,
|
||||
"wiki_folder_id": wiki_folder_id,
|
||||
"saved_title": title,
|
||||
"saved_markdown": markdown,
|
||||
"saved_hash": saved_hash,
|
||||
"saved_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
if isinstance(self._row, dict):
|
||||
self._row.update(
|
||||
{
|
||||
**values,
|
||||
"saved_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
}
|
||||
)
|
||||
else:
|
||||
for key, value in values.items():
|
||||
setattr(self._row, key, value)
|
||||
return self.snapshot()
|
||||
|
||||
|
||||
class KnowledgeResearchStore:
|
||||
"""SQL store with a behaviorally equivalent memory fallback."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: async_sessionmaker[AsyncSession] | None,
|
||||
) -> None:
|
||||
self._sf = session_factory
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._publication_locks: dict[str, asyncio.Lock] = {}
|
||||
|
||||
@asynccontextmanager
|
||||
async def locked_publication(
|
||||
self,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
) -> AsyncIterator[_PublicationLease | None]:
|
||||
"""Serialize version-check + remote write + local publication commit.
|
||||
|
||||
SQL deployments hold a row lock for the context lifetime, so separate
|
||||
application workers cannot both pass the same expected-hash check.
|
||||
SQLite has no ``FOR UPDATE`` support, so a no-op update first acquires
|
||||
its database write lock. The memory fallback uses one asyncio lock per
|
||||
research session and commits a copy only on successful exit.
|
||||
"""
|
||||
|
||||
if self._sf is None:
|
||||
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
|
||||
async with lock:
|
||||
original = self._rows.get(session_id)
|
||||
if original is None or original["user_id"] != user_id:
|
||||
yield None
|
||||
return
|
||||
working = deepcopy(original)
|
||||
lease = _PublicationLease(working)
|
||||
yield lease
|
||||
self._rows[session_id] = working
|
||||
return
|
||||
|
||||
async with self._sf() as session:
|
||||
async with session.begin():
|
||||
bind = session.get_bind()
|
||||
if bind.dialect.name == "sqlite":
|
||||
await session.execute(
|
||||
update(KnowledgeResearchSessionRow)
|
||||
.where(
|
||||
KnowledgeResearchSessionRow.id == session_id,
|
||||
KnowledgeResearchSessionRow.user_id == user_id,
|
||||
)
|
||||
.values(
|
||||
updated_at=KnowledgeResearchSessionRow.updated_at
|
||||
)
|
||||
)
|
||||
row = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow)
|
||||
.where(
|
||||
KnowledgeResearchSessionRow.id == session_id,
|
||||
KnowledgeResearchSessionRow.user_id == user_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
yield _PublicationLease(row) if row is not None else None
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
knowledge_base_id: str,
|
||||
user_id: str,
|
||||
title: str,
|
||||
session_id: str | None = None,
|
||||
publication_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = KnowledgeResearchSessionRow(
|
||||
id=session_id or str(uuid4()),
|
||||
publication_id=publication_id or str(uuid4()),
|
||||
thread_id=thread_id or str(uuid4()),
|
||||
knowledge_base_id=knowledge_base_id,
|
||||
user_id=user_id,
|
||||
title=title,
|
||||
draft_markdown="",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
if self._sf is None:
|
||||
data = _row_dict(row)
|
||||
self._rows[row.id] = data
|
||||
return deepcopy(data)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
return _row_dict(row)
|
||||
|
||||
async def get(self, session_id: str) -> dict[str, Any] | None:
|
||||
if self._sf is None:
|
||||
row = self._rows.get(session_id)
|
||||
return deepcopy(row) if row else None
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeResearchSessionRow, session_id)
|
||||
return _row_dict(row) if row else None
|
||||
|
||||
async def get_owned(self, session_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
row = await self.get(session_id)
|
||||
return row if row and row["user_id"] == user_id else None
|
||||
|
||||
async def get_by_thread(self, thread_id: str) -> dict[str, Any] | None:
|
||||
if self._sf is None:
|
||||
for row in self._rows.values():
|
||||
if row["thread_id"] == thread_id:
|
||||
return deepcopy(row)
|
||||
return None
|
||||
async with self._sf() as session:
|
||||
try:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow).where(
|
||||
KnowledgeResearchSessionRow.thread_id == thread_id
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
except (OperationalError, ProgrammingError) as exc:
|
||||
# Ordinary Q&A hits this lookup on every run. A missing table
|
||||
# (older DB before create_all registered the model) must not
|
||||
# abort /runs/stream; treat it as "not a research thread".
|
||||
if _is_missing_table(exc):
|
||||
return None
|
||||
raise
|
||||
return _row_dict(row) if row else None
|
||||
|
||||
async def get_publication(self, publication_id: str) -> dict[str, Any] | None:
|
||||
if self._sf is None:
|
||||
for row in self._rows.values():
|
||||
if row["publication_id"] == publication_id and row.get("saved_hash"):
|
||||
return deepcopy(row)
|
||||
return None
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow).where(
|
||||
KnowledgeResearchSessionRow.publication_id == publication_id,
|
||||
KnowledgeResearchSessionRow.saved_hash.is_not(None),
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return _row_dict(row) if row else None
|
||||
|
||||
async def list_owned(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
user_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
if self._sf is None:
|
||||
rows = [
|
||||
deepcopy(row)
|
||||
for row in self._rows.values()
|
||||
if row["knowledge_base_id"] == knowledge_base_id
|
||||
and row["user_id"] == user_id
|
||||
]
|
||||
return sorted(rows, key=lambda item: str(item["updated_at"]), reverse=True)
|
||||
async with self._sf() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow)
|
||||
.where(
|
||||
KnowledgeResearchSessionRow.knowledge_base_id
|
||||
== knowledge_base_id,
|
||||
KnowledgeResearchSessionRow.user_id == user_id,
|
||||
)
|
||||
.order_by(KnowledgeResearchSessionRow.updated_at.desc())
|
||||
)
|
||||
).scalars()
|
||||
return [_row_dict(row) for row in rows]
|
||||
|
||||
async def list_published(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
if self._sf is None:
|
||||
rows = [
|
||||
deepcopy(row)
|
||||
for row in self._rows.values()
|
||||
if row["knowledge_base_id"] == knowledge_base_id
|
||||
and row.get("saved_hash")
|
||||
]
|
||||
return sorted(
|
||||
rows,
|
||||
key=lambda item: str(item.get("saved_at") or item["updated_at"]),
|
||||
reverse=True,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow)
|
||||
.where(
|
||||
KnowledgeResearchSessionRow.knowledge_base_id
|
||||
== knowledge_base_id,
|
||||
KnowledgeResearchSessionRow.saved_hash.is_not(None),
|
||||
)
|
||||
.order_by(KnowledgeResearchSessionRow.saved_at.desc())
|
||||
)
|
||||
).scalars()
|
||||
return [_row_dict(row) for row in rows]
|
||||
|
||||
async def update_owned(
|
||||
self,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
title: str | None = None,
|
||||
draft_markdown: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
if self._sf is None:
|
||||
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
|
||||
async with lock:
|
||||
row = self._rows.get(session_id)
|
||||
if row is None or row["user_id"] != user_id:
|
||||
return None
|
||||
if title is not None:
|
||||
row["title"] = title
|
||||
if draft_markdown is not None:
|
||||
row["draft_markdown"] = draft_markdown
|
||||
row["updated_at"] = now.isoformat()
|
||||
return deepcopy(row)
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow)
|
||||
.where(
|
||||
KnowledgeResearchSessionRow.id == session_id,
|
||||
KnowledgeResearchSessionRow.user_id == user_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
if title is not None:
|
||||
row.title = title
|
||||
if draft_markdown is not None:
|
||||
row.draft_markdown = draft_markdown
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
return _row_dict(row)
|
||||
|
||||
async def save_publication(
|
||||
self,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
title: str,
|
||||
markdown: str,
|
||||
saved_hash: str,
|
||||
wiki_slug: str,
|
||||
wiki_page_id: str | None,
|
||||
wiki_folder_id: str | None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self.locked_publication(session_id, user_id) as publication:
|
||||
if publication is None:
|
||||
return None
|
||||
return publication.save(
|
||||
title=title,
|
||||
markdown=markdown,
|
||||
saved_hash=saved_hash,
|
||||
wiki_slug=wiki_slug,
|
||||
wiki_page_id=wiki_page_id,
|
||||
wiki_folder_id=wiki_folder_id,
|
||||
)
|
||||
|
||||
async def delete(self, session_id: str) -> None:
|
||||
if self._sf is None:
|
||||
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
|
||||
async with lock:
|
||||
self._rows.pop(session_id, None)
|
||||
return
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
delete(KnowledgeResearchSessionRow).where(
|
||||
KnowledgeResearchSessionRow.id == session_id
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
def make_knowledge_research_store(session_factory):
|
||||
return KnowledgeResearchStore(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeResearchSessionRow",
|
||||
"KnowledgeResearchStore",
|
||||
"make_knowledge_research_store",
|
||||
]
|
||||
+13
@@ -15,6 +15,19 @@ class DuplicateLightAppNameError(ValueError):
|
||||
|
||||
|
||||
class LightAppStore(ABC):
|
||||
async def list_sidebar_apps(self) -> list[dict[str, Any]]:
|
||||
"""Return only the columns needed to render iframe app menu entries."""
|
||||
rows = await self.list_apps(app_type="iframe", enabled_only=True)
|
||||
return [
|
||||
{
|
||||
"app_id": row.get("app_id"),
|
||||
"name": row.get("name"),
|
||||
"mount_location": row.get("mount_location"),
|
||||
"sort_order": row.get("sort_order", 0),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
@abstractmethod
|
||||
async def list_apps(
|
||||
self,
|
||||
|
||||
+16
@@ -65,6 +65,22 @@ class LightAppRepository(LightAppStore):
|
||||
result = await session.execute(stmt)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_sidebar_apps(self) -> list[dict[str, Any]]:
|
||||
"""Avoid loading descriptions, icons, route JSON, tags, and audit data."""
|
||||
stmt = (
|
||||
select(
|
||||
LightAppRow.app_id,
|
||||
LightAppRow.name,
|
||||
LightAppRow.mount_location,
|
||||
LightAppRow.sort_order,
|
||||
)
|
||||
.where(LightAppRow.app_type == "iframe", LightAppRow.enabled.is_(True))
|
||||
.order_by(LightAppRow.sort_order.asc(), LightAppRow.created_at.desc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [dict(row._mapping) for row in result]
|
||||
|
||||
async def get_app(self, app_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LightAppRow, app_id)
|
||||
|
||||
+9
@@ -67,3 +67,12 @@ class LlmMetricsStore(ABC):
|
||||
async def iter_all(self, query: LlmMetricsQuery):
|
||||
"""Yield rows matching ``query`` without pagination — used by export."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
|
||||
"""Delete one bounded batch of audit rows older than ``cutoff``.
|
||||
|
||||
The default keeps lightweight/in-memory test implementations compatible.
|
||||
SQL-backed stores override it for the scheduled admin-audit retention
|
||||
sweep.
|
||||
"""
|
||||
return 0
|
||||
+32
-1
@@ -7,7 +7,7 @@ from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy import delete, desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery, LlmMetricsStore
|
||||
@@ -110,3 +110,34 @@ class SqlLlmMetricsStore(LlmMetricsStore):
|
||||
if len(page) < page_size:
|
||||
return
|
||||
offset += page_size
|
||||
|
||||
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
|
||||
"""Delete one small, oldest-first retention batch.
|
||||
|
||||
Selecting primary keys first keeps this portable across SQLite, MySQL,
|
||||
and PostgreSQL (which do not share ``DELETE ... LIMIT`` syntax) and
|
||||
avoids a long-running delete transaction on a busy audit table.
|
||||
"""
|
||||
# Keep below SQLite's traditional 999 bind-variable limit. The
|
||||
# retention worker requests batches repeatedly, so portability is more
|
||||
# valuable than one giant delete statement.
|
||||
size = max(1, min(int(batch_size), 500))
|
||||
normalized_cutoff = _normalize_created_at(cutoff)
|
||||
async with self._sf() as session:
|
||||
ids = list(
|
||||
(
|
||||
await session.scalars(
|
||||
select(LlmCallMetricRow.id)
|
||||
.where(LlmCallMetricRow.created_at < normalized_cutoff)
|
||||
.order_by(LlmCallMetricRow.created_at.asc(), LlmCallMetricRow.id.asc())
|
||||
.limit(size)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not ids:
|
||||
return 0
|
||||
result = await session.execute(
|
||||
delete(LlmCallMetricRow).where(LlmCallMetricRow.id.in_(ids))
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
"""Keep synchronous compatibility reads off the Gateway's async connection pool."""
|
||||
|
||||
import asyncio
|
||||
from contextvars import ContextVar
|
||||
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
_isolated_factory = ContextVar("isolated_db_factory", default=None)
|
||||
|
||||
|
||||
def isolated_session_factory():
|
||||
return _isolated_factory.get()
|
||||
|
||||
|
||||
async def run_isolated_db_read(coro):
|
||||
from deerflow.persistence.engine import get_engine
|
||||
|
||||
source = get_engine()
|
||||
if source is None:
|
||||
return await coro
|
||||
# Never return a connection created on this temporary loop to the main pool.
|
||||
engine = create_async_engine(source.url, poolclass=NullPool)
|
||||
token = _isolated_factory.set(async_sessionmaker(engine, expire_on_commit=False))
|
||||
try:
|
||||
return await asyncio.wait_for(coro, 30)
|
||||
finally:
|
||||
_isolated_factory.reset(token)
|
||||
await engine.dispose()
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
"""Helpers shared by the online Alembic environment and its tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
def configure_mysql_migration_lock_waits(connection: Any) -> None:
|
||||
"""Set bounded MySQL DDL lock waits without leaving a transaction open.
|
||||
|
||||
SQLAlchemy starts an implicit transaction for ``SET SESSION`` on MySQL.
|
||||
Alembic's migration transaction then reuses it, and closing the connection
|
||||
rolls back the ``alembic_version`` updates even though MySQL DDL itself has
|
||||
already committed. Commit the session-local settings before Alembic starts
|
||||
its transaction so successful migrations are recorded durably.
|
||||
"""
|
||||
|
||||
try:
|
||||
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
|
||||
except ValueError:
|
||||
lock_wait = 15
|
||||
|
||||
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
|
||||
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
|
||||
connection.commit()
|
||||
+2
-8
@@ -9,14 +9,13 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.migration_utils import configure_mysql_migration_lock_waits
|
||||
|
||||
# Import all models so metadata is populated.
|
||||
try:
|
||||
@@ -57,12 +56,7 @@ def do_run_migrations(connection):
|
||||
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
|
||||
|
||||
if connection.dialect.name == "mysql" and not is_gauss_mysql_product():
|
||||
try:
|
||||
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
|
||||
except ValueError:
|
||||
lock_wait = 15
|
||||
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
|
||||
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
|
||||
configure_mysql_migration_lock_waits(connection)
|
||||
|
||||
context.configure(
|
||||
connection=connection,
|
||||
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
"""Add snapshots supplied to the b5-1 task-analysis deep link.
|
||||
|
||||
Revision ID: 20261021_01
|
||||
Revises: 20261020_01
|
||||
Create Date: 2026-10-20
|
||||
|
||||
This file originally shipped with revision id ``20261020_01`` — the same id as
|
||||
``20261020_01_knowledge_management_advanced_tools`` — which left the migration
|
||||
graph with a duplicate revision and no single head. Re-issued as
|
||||
``20261021_01`` chained after the canonical ``20261020_01``; its upgrade is
|
||||
fully guarded (``if _has_table(): return``) so databases that already created
|
||||
the table under the old id re-run it as a no-op.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261021_01"
|
||||
down_revision = "20261020_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_details"
|
||||
|
||||
|
||||
def _has_table() -> bool:
|
||||
return _TABLE in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _json_type() -> sa.types.TypeEngine:
|
||||
# Keep the migration consistent with PortableJSON, whose MySQL form is
|
||||
# TEXT for compatibility with older deployments.
|
||||
return sa.Text() if op.get_bind().dialect.name == "mysql" else sa.JSON()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table():
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("detail_type", sa.String(32), primary_key=True),
|
||||
sa.Column("detail_id", sa.String(128), primary_key=True),
|
||||
sa.Column("task_name", sa.String(512), nullable=False),
|
||||
sa.Column("task_content", sa.Text(), nullable=False),
|
||||
sa.Column("overview", sa.Text(), nullable=False),
|
||||
sa.Column("extra_json", _json_type(), nullable=False),
|
||||
sa.Column("version", sa.Integer(), nullable=False),
|
||||
sa.Column("created_by", sa.String(64), nullable=True),
|
||||
sa.Column("updated_by", sa.String(64), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table():
|
||||
op.drop_table(_TABLE)
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""Admin catalog switch: ``agents.hidden``.
|
||||
|
||||
Hidden agents disappear from every non-admin listing surface (gallery / 我的 /
|
||||
广场 / homepage selector / private square) while staying fully usable — only
|
||||
the admin agents console (?admin=true) keeps listing them.
|
||||
|
||||
The Gateway's startup ORM-column sweep also adds this column on existing
|
||||
databases (Alembic auto-upgrade is disabled there), so this migration mostly
|
||||
serves environments that run Alembic manually. Fully guarded and idempotent.
|
||||
|
||||
Revision ID: 20261022_03
|
||||
Revises: 20261021_01
|
||||
Create Date: 2026-10-22
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261022_03"
|
||||
down_revision = "20261021_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "agents"
|
||||
_COLUMN = "hidden"
|
||||
_INDEX = "ix_agents_hidden"
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
return table_name in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table(_TABLE) and not _has_column(_TABLE, _COLUMN):
|
||||
# server_default mirrors the ORM declaration so the sweep on live
|
||||
# databases and this migration produce the same NOT NULL column.
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.text("0")),
|
||||
)
|
||||
if _has_table(_TABLE) and not _has_index(_TABLE, _INDEX):
|
||||
op.create_index(_INDEX, _TABLE, [_COLUMN])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table(_TABLE) and _has_index(_TABLE, _INDEX):
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
if _has_table(_TABLE) and _has_column(_TABLE, _COLUMN):
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
"""Persist private knowledge-research sessions and shared publications.
|
||||
|
||||
Revision ID: 20261022_01
|
||||
Revises: 20261014_02
|
||||
Create Date: 2026-10-22
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableLongText
|
||||
|
||||
revision = "20261022_01"
|
||||
down_revision = "20261014_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "knowledge_research_sessions"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if inspector.has_table(_TABLE):
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("id", sa.String(length=36), primary_key=True),
|
||||
sa.Column("publication_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("thread_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("knowledge_base_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("user_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("title", sa.String(length=255), nullable=False),
|
||||
sa.Column("draft_markdown", PortableLongText(), nullable=False),
|
||||
sa.Column("wiki_slug", sa.String(length=512), nullable=True),
|
||||
sa.Column("wiki_page_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("wiki_folder_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("saved_title", sa.String(length=255), nullable=True),
|
||||
sa.Column("saved_markdown", PortableLongText(), nullable=True),
|
||||
sa.Column("saved_hash", sa.String(length=64), nullable=True),
|
||||
sa.Column("saved_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("publication_id", name="uq_knowledge_research_publication_id"),
|
||||
sa.UniqueConstraint("thread_id", name="uq_knowledge_research_thread_id"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_knowledge_research_sessions_publication_id",
|
||||
_TABLE,
|
||||
["publication_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_knowledge_research_sessions_thread_id",
|
||||
_TABLE,
|
||||
["thread_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_knowledge_research_sessions_knowledge_base_id",
|
||||
_TABLE,
|
||||
["knowledge_base_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_knowledge_research_sessions_user_id",
|
||||
_TABLE,
|
||||
["user_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
op.drop_table(_TABLE)
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
"""Add the integrator-supplied type partition to task deep-link details.
|
||||
|
||||
Revision ID: 20261022_02
|
||||
Revises: 20261024_01
|
||||
Create Date: 2026-10-22
|
||||
|
||||
The same task id may now carry one snapshot per business type (the deep-link
|
||||
``?type=``), so ``sub_type`` joins the primary key. Existing rows backfill to
|
||||
``''`` — the legacy no-type partition — and callers that omit the type keep
|
||||
reading/writing exactly that partition, so behaviour is unchanged for them.
|
||||
|
||||
The key change is a portable table rebuild (create-copy-drop-rename): this
|
||||
Alembic build has no batch ``drop_primary_key`` and SQLite cannot alter a
|
||||
primary key in place, so any other route leaves a two-column key that makes
|
||||
typed upserts fail with a UNIQUE constraint violation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.dialects import mysql as sa_mysql
|
||||
|
||||
revision = "20261022_02"
|
||||
down_revision = "20261024_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_details"
|
||||
_PK = ["detail_type", "detail_id", "sub_type"]
|
||||
_OLD_PK = ["detail_type", "detail_id"]
|
||||
|
||||
|
||||
def _has_table() -> bool:
|
||||
return _TABLE in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _pk_columns() -> list[str] | None:
|
||||
constraint = inspect(op.get_bind()).get_pk_constraint(_TABLE)
|
||||
return list(constraint.get("constrained_columns") or []) or None
|
||||
|
||||
|
||||
def _column_specs(with_sub_type: bool) -> list[tuple[str, sa.types.TypeEngine, bool]]:
|
||||
is_mysql = op.get_bind().dialect.name == "mysql"
|
||||
specs = [
|
||||
("detail_type", sa.String(32), False),
|
||||
("detail_id", sa.String(128), False),
|
||||
("task_name", sa.String(512), False),
|
||||
("task_content", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
|
||||
("overview", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
|
||||
("extra_json", sa.Text() if is_mysql else sa.JSON(), False),
|
||||
("version", sa.Integer(), False),
|
||||
("created_by", sa.String(64), True),
|
||||
("updated_by", sa.String(64), True),
|
||||
("created_at", sa.DateTime(), False),
|
||||
("updated_at", sa.DateTime(), False),
|
||||
]
|
||||
if with_sub_type:
|
||||
specs.append(("sub_type", sa.String(64), False))
|
||||
return specs
|
||||
|
||||
|
||||
def _rebuild_table(pk_columns: list[str], with_sub_type: bool) -> None:
|
||||
"""Recreate the table (create-copy-drop-rename) with the given primary key."""
|
||||
|
||||
bind = op.get_bind()
|
||||
rebuild = _TABLE + "__rebuild"
|
||||
if rebuild in inspect(bind).get_table_names():
|
||||
op.drop_table(rebuild)
|
||||
specs = _column_specs(with_sub_type)
|
||||
op.create_table(
|
||||
rebuild,
|
||||
*[
|
||||
sa.Column(
|
||||
name,
|
||||
type_,
|
||||
nullable=nullable,
|
||||
server_default="" if name == "sub_type" else None,
|
||||
)
|
||||
for name, type_, nullable in specs
|
||||
],
|
||||
sa.PrimaryKeyConstraint(*pk_columns),
|
||||
)
|
||||
names = [name for name, _, _ in specs]
|
||||
select_expr = ", ".join(
|
||||
f"COALESCE({name}, '')" if name == "sub_type" else name for name in names
|
||||
)
|
||||
bind.execute(
|
||||
sa.text(f"INSERT INTO {rebuild} ({', '.join(names)}) SELECT {select_expr} FROM {_TABLE}")
|
||||
)
|
||||
op.drop_table(_TABLE)
|
||||
op.rename_table(rebuild, _TABLE)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table():
|
||||
# Fresh databases get the three-column key straight from create_all.
|
||||
return
|
||||
inspector = inspect(op.get_bind())
|
||||
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
|
||||
if "sub_type" in columns and _pk_columns() == _PK:
|
||||
return
|
||||
if "sub_type" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("sub_type", sa.String(64), nullable=False, server_default=""),
|
||||
)
|
||||
if _pk_columns() != _PK:
|
||||
_rebuild_table(_PK, with_sub_type=True)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
if _pk_columns() == _OLD_PK:
|
||||
return
|
||||
_rebuild_table(_OLD_PK, with_sub_type=False)
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
"""Admin catalog switch: ``roundtable_chains.hidden``.
|
||||
|
||||
Hidden chains vanish from every non-admin listing (我的 / 公共 / 待审核 / 选链
|
||||
弹窗) while staying fully usable — single-record reads (get_chain), configured
|
||||
business mappings and in-flight roundtables are unaffected. Only the admin
|
||||
management view (``admin_view=true``) keeps listing them.
|
||||
|
||||
The Gateway's startup ORM-column sweep also adds this column on existing
|
||||
databases (Alembic auto-upgrade is disabled there); this migration serves
|
||||
environments that run Alembic manually. Fully guarded and idempotent.
|
||||
|
||||
Revision ID: 20261023_01
|
||||
Revises: 20261022_03
|
||||
Create Date: 2026-10-23
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261023_01"
|
||||
down_revision = "20261022_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "roundtable_chains"
|
||||
_COLUMN = "hidden"
|
||||
_INDEX = "ix_roundtable_chains_hidden"
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
return table_name in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table(_TABLE) and not _has_column(_TABLE, _COLUMN):
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.text("0")),
|
||||
)
|
||||
if _has_table(_TABLE) and not _has_index(_TABLE, _INDEX):
|
||||
op.create_index(_INDEX, _TABLE, [_COLUMN])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table(_TABLE) and _has_index(_TABLE, _INDEX):
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
if _has_table(_TABLE) and _has_column(_TABLE, _COLUMN):
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
"""Index the common deep-link list path that omits sub_type.
|
||||
|
||||
Revision ID: 20261023_02
|
||||
Revises: 20261024_02, 20261022_02
|
||||
Create Date: 2026-10-22
|
||||
|
||||
``GET /api/task-deeplink-threads`` for 3qfx/rwfx/xdfx filters only
|
||||
``task_id + go_path`` and orders by ``updated_at, thread_id``. The existing
|
||||
scope index inserts ``sub_type`` before those sort columns, so the planner
|
||||
cannot use it for newest-first listing. This extra index matches that query.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261023_02"
|
||||
down_revision = ("20261024_02", "20261022_02")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_threads"
|
||||
_INDEX = "ix_task_deeplink_threads_task_go_updated"
|
||||
_COLUMNS = ["task_id", "go_path", "updated_at", "thread_id"]
|
||||
|
||||
|
||||
def _has_table() -> bool:
|
||||
return _TABLE in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _indexes() -> dict[str, list[str]]:
|
||||
if not _has_table():
|
||||
return {}
|
||||
return {
|
||||
str(index["name"]): [str(column) for column in index["column_names"]]
|
||||
for index in inspect(op.get_bind()).get_indexes(_TABLE)
|
||||
if index.get("name")
|
||||
}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
if _indexes().get(_INDEX) != _COLUMNS:
|
||||
if _INDEX in _indexes():
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
op.create_index(_INDEX, _TABLE, _COLUMNS)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
if _INDEX in _indexes():
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
"""Repair task_deeplink_details primary keys left two-column by 20261022_02.
|
||||
|
||||
Revision ID: 20261023_03
|
||||
Revises: 20261023_02
|
||||
Create Date: 2026-10-22
|
||||
|
||||
An interim build of ``20261022_02`` added the ``sub_type`` column but could
|
||||
not rebuild the primary key (the batch API it called does not exist in this
|
||||
Alembic version, and SQLite cannot alter a key in place). Databases that ran
|
||||
that build are stamped at head yet still enforce the legacy
|
||||
``(detail_type, detail_id)`` key, so typed upserts fail with a UNIQUE
|
||||
constraint violation. This revision detects that state and performs the
|
||||
portable create-copy-drop-rename rebuild; databases with the correct
|
||||
three-column key simply no-op.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.dialects import mysql as sa_mysql
|
||||
|
||||
revision = "20261023_03"
|
||||
down_revision = "20261023_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_details"
|
||||
_PK = ["detail_type", "detail_id", "sub_type"]
|
||||
|
||||
|
||||
def _has_table() -> bool:
|
||||
return _TABLE in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _pk_columns() -> list[str] | None:
|
||||
constraint = inspect(op.get_bind()).get_pk_constraint(_TABLE)
|
||||
return list(constraint.get("constrained_columns") or []) or None
|
||||
|
||||
|
||||
def _rebuild_table() -> None:
|
||||
bind = op.get_bind()
|
||||
rebuild = _TABLE + "__rebuild"
|
||||
if rebuild in inspect(bind).get_table_names():
|
||||
op.drop_table(rebuild)
|
||||
is_mysql = bind.dialect.name == "mysql"
|
||||
specs: list[tuple[str, sa.types.TypeEngine, bool]] = [
|
||||
("detail_type", sa.String(32), False),
|
||||
("detail_id", sa.String(128), False),
|
||||
("sub_type", sa.String(64), False),
|
||||
("task_name", sa.String(512), False),
|
||||
("task_content", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
|
||||
("overview", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
|
||||
("extra_json", sa.Text() if is_mysql else sa.JSON(), False),
|
||||
("version", sa.Integer(), False),
|
||||
("created_by", sa.String(64), True),
|
||||
("updated_by", sa.String(64), True),
|
||||
("created_at", sa.DateTime(), False),
|
||||
("updated_at", sa.DateTime(), False),
|
||||
]
|
||||
op.create_table(
|
||||
rebuild,
|
||||
*[
|
||||
sa.Column(
|
||||
name,
|
||||
type_,
|
||||
nullable=nullable,
|
||||
server_default="" if name == "sub_type" else None,
|
||||
)
|
||||
for name, type_, nullable in specs
|
||||
],
|
||||
sa.PrimaryKeyConstraint(*_PK),
|
||||
)
|
||||
names = [name for name, _, _ in specs]
|
||||
select_expr = ", ".join(
|
||||
f"COALESCE({name}, '')" if name == "sub_type" else name for name in names
|
||||
)
|
||||
bind.execute(
|
||||
sa.text(f"INSERT INTO {rebuild} ({', '.join(names)}) SELECT {select_expr} FROM {_TABLE}")
|
||||
)
|
||||
op.drop_table(_TABLE)
|
||||
op.rename_table(rebuild, _TABLE)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
if _pk_columns() == _PK:
|
||||
return
|
||||
_rebuild_table()
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# The repaired key is what 20261022_02 intended; nothing to undo.
|
||||
return
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
"""Add the b5-1 ?type= partition column to the deep-link history index.
|
||||
|
||||
Revision ID: 20261024_01
|
||||
Revises: 20261023_01
|
||||
Create Date: 2026-10-22
|
||||
|
||||
Existing rows keep sharing one partition per (task_id, go_path): the new
|
||||
``sub_type`` column backfills to ``''`` and queries that omit ``sub_type``
|
||||
continue to return every sub-type, so 3qfx/rwfx/xdfx behaviour is unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261024_01"
|
||||
down_revision = "20261023_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_threads"
|
||||
_INDEX = "ix_task_deeplink_threads_scope"
|
||||
_INDEX_COLUMNS = ["task_id", "go_path", "sub_type", "updated_at", "thread_id"]
|
||||
|
||||
|
||||
def _has_table() -> bool:
|
||||
return _TABLE in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table():
|
||||
# Fresh databases get the column straight from create_all.
|
||||
return
|
||||
bind = op.get_bind()
|
||||
inspector = inspect(bind)
|
||||
if "sub_type" not in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("sub_type", sa.String(64), nullable=False, server_default=""),
|
||||
)
|
||||
indexes = {index["name"]: index["column_names"] for index in inspector.get_indexes(_TABLE)}
|
||||
if indexes.get(_INDEX) != _INDEX_COLUMNS:
|
||||
if _INDEX in indexes:
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
op.create_index(_INDEX, _TABLE, _INDEX_COLUMNS)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
bind = op.get_bind()
|
||||
inspector = inspect(bind)
|
||||
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
|
||||
indexes = {index["name"]: index["column_names"] for index in inspector.get_indexes(_TABLE)}
|
||||
if indexes.get(_INDEX) is not None:
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
if "sub_type" in columns:
|
||||
op.drop_column(_TABLE, "sub_type")
|
||||
op.create_index(_INDEX, _TABLE, ["task_id", "go_path", "updated_at", "thread_id"])
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
"""Store deep-link list cards without joining ``threads_meta``.
|
||||
|
||||
Revision ID: 20261024_02
|
||||
Revises: 20261024_01
|
||||
Create Date: 2026-10-22
|
||||
|
||||
The dedicated history table used to be a pointer: listing still joined the
|
||||
generic ``threads_meta`` catalogue for title / status / metadata. That join
|
||||
is what made the task workspace slow once ordinary Q&A filled the catalogue.
|
||||
This revision copies those card fields onto ``task_deeplink_threads`` itself
|
||||
and backfills existing index rows from ``threads_meta``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20261024_02"
|
||||
down_revision = "20261024_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_threads"
|
||||
_META_TABLE = "threads_meta"
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
return table_name in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
if not _has_table(_TABLE):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspect(op.get_bind()).get_columns(_TABLE)}
|
||||
|
||||
|
||||
def _backfill_cards() -> None:
|
||||
if not _has_table(_META_TABLE):
|
||||
return
|
||||
bind = op.get_bind()
|
||||
dialect = bind.dialect.name
|
||||
if dialect in {"postgresql", "opengauss"}:
|
||||
bind.execute(
|
||||
sa.text(
|
||||
f"UPDATE {_TABLE} AS t "
|
||||
"SET display_name = m.display_name, "
|
||||
"status = COALESCE(m.status, 'idle'), "
|
||||
"metadata_json = m.metadata_json "
|
||||
f"FROM {_META_TABLE} AS m "
|
||||
"WHERE m.thread_id = t.thread_id"
|
||||
)
|
||||
)
|
||||
return
|
||||
if dialect == "mysql":
|
||||
bind.execute(
|
||||
sa.text(
|
||||
f"UPDATE {_TABLE} AS t "
|
||||
f"JOIN {_META_TABLE} AS m ON m.thread_id = t.thread_id "
|
||||
"SET t.display_name = m.display_name, "
|
||||
"t.status = COALESCE(m.status, 'idle'), "
|
||||
"t.metadata_json = m.metadata_json"
|
||||
)
|
||||
)
|
||||
return
|
||||
bind.execute(
|
||||
sa.text(
|
||||
f"UPDATE {_TABLE} SET "
|
||||
"display_name = ("
|
||||
f"SELECT display_name FROM {_META_TABLE} "
|
||||
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
|
||||
"), "
|
||||
"status = COALESCE(("
|
||||
f"SELECT status FROM {_META_TABLE} "
|
||||
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
|
||||
"), 'idle'), "
|
||||
"metadata_json = ("
|
||||
f"SELECT metadata_json FROM {_META_TABLE} "
|
||||
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
|
||||
")"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table(_TABLE):
|
||||
return
|
||||
columns = _columns()
|
||||
if "display_name" not in columns:
|
||||
op.add_column(_TABLE, sa.Column("display_name", sa.String(256), nullable=True))
|
||||
if "status" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("status", sa.String(20), nullable=False, server_default="idle"),
|
||||
)
|
||||
if "metadata_json" not in columns:
|
||||
op.add_column(_TABLE, sa.Column("metadata_json", PortableJSON(), nullable=True))
|
||||
_backfill_cards()
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table(_TABLE):
|
||||
return
|
||||
columns = _columns()
|
||||
if "metadata_json" in columns:
|
||||
op.drop_column(_TABLE, "metadata_json")
|
||||
if "status" in columns:
|
||||
op.drop_column(_TABLE, "status")
|
||||
if "display_name" in columns:
|
||||
op.drop_column(_TABLE, "display_name")
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""Indexes for the agent gallery category tabs and tag filters.
|
||||
|
||||
Revision ID: 20261025_01
|
||||
Revises: 20261023_03
|
||||
Create Date: 2026-10-23
|
||||
|
||||
The gallery "广场" tab filters ``hidden`` + ``is_private`` + ``square_id``,
|
||||
and "我的" filters ``user_id`` + ``hidden``. Tag chips join
|
||||
``tag_assignments`` by ``(target_type, target_id)`` and filter by
|
||||
``(target_type, tag_id)``. Single-column indexes and the existing unique
|
||||
key (which leads with ``tag_id``) do not cover those predicates.
|
||||
|
||||
Startup also creates missing ORM indexes, so deployments that skip Alembic
|
||||
still receive these. This revision stays idempotent for the ones that run it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261025_01"
|
||||
down_revision = "20261023_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_INDEXES = (
|
||||
("agents", "ix_agents_catalog_square", ("hidden", "is_private", "square_id", "created_at")),
|
||||
("agents", "ix_agents_owner_hidden", ("user_id", "hidden", "created_at")),
|
||||
("tag_assignments", "ix_tag_assignments_type_target", ("target_type", "target_id")),
|
||||
("tag_assignments", "ix_tag_assignments_type_tag", ("target_type", "tag_id", "target_id")),
|
||||
)
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
return table_name in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
return any(idx["name"] == index_name for idx in inspect(op.get_bind()).get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for table_name, index_name, columns in _INDEXES:
|
||||
if _has_table(table_name) and not _has_index(table_name, index_name):
|
||||
op.create_index(index_name, table_name, list(columns))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table_name, index_name, _columns in reversed(_INDEXES):
|
||||
if _has_table(table_name) and _has_index(table_name, index_name):
|
||||
op.drop_index(index_name, table_name=table_name)
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"""Add shared nonce storage for scoped service-token exchange.
|
||||
|
||||
Revision ID: 20261025_02
|
||||
Revises: 20261025_01
|
||||
Create Date: 2026-10-24
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261025_02"
|
||||
down_revision = "20261025_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "auth_service_nonces"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _TABLE in inspect(op.get_bind()).get_table_names():
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("nonce_hash", sa.String(64), primary_key=True),
|
||||
sa.Column("client_id", sa.String(128), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_auth_service_nonces_client_id", _TABLE, ["client_id"])
|
||||
op.create_index("ix_auth_service_nonces_expires_at", _TABLE, ["expires_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _TABLE in inspect(op.get_bind()).get_table_names():
|
||||
op.drop_table(_TABLE)
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
"""Add the database-backed skill catalogue snapshot for list requests.
|
||||
|
||||
Revision ID: 20261025_03
|
||||
Revises: 20261025_02
|
||||
Create Date: 2026-10-25
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261025_03"
|
||||
down_revision = "20261025_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "skill_catalog_snapshots"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _TABLE in inspect(op.get_bind()).get_table_names():
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("cache_key", sa.String(32), primary_key=True),
|
||||
sa.Column("catalog_json", sa.Text(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _TABLE in inspect(op.get_bind()).get_table_names():
|
||||
op.drop_table(_TABLE)
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""Index canonical usernames for fast SSO and username login.
|
||||
|
||||
Revision ID: 20261026_01
|
||||
Revises: 20261025_03
|
||||
Create Date: 2026-10-26
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261026_01"
|
||||
down_revision = "20261025_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "users"
|
||||
_COLUMN = "username_key"
|
||||
_INDEX = "ix_users_username_key_created_at"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
|
||||
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
|
||||
if _COLUMN not in columns:
|
||||
op.add_column(_TABLE, sa.Column(_COLUMN, sa.String(128), nullable=True))
|
||||
|
||||
indexes = {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
|
||||
if _INDEX not in indexes:
|
||||
op.create_index(_INDEX, _TABLE, [_COLUMN, "created_at"])
|
||||
|
||||
# Backfill the normalized local part once. Login requests then use a
|
||||
# bounded indexed equality lookup instead of LOWER(email) LIKE 'name@%'.
|
||||
bind = op.get_bind()
|
||||
rows = bind.execute(sa.text("SELECT id, email FROM users")).fetchall()
|
||||
for user_id, email in rows:
|
||||
if not email or "@" not in email:
|
||||
continue
|
||||
username_key = str(email).split("@", 1)[0].strip().lower()[:128]
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"UPDATE users SET username_key = :username_key "
|
||||
"WHERE id = :user_id AND (username_key IS NULL OR username_key = '')"
|
||||
),
|
||||
{"username_key": username_key, "user_id": user_id},
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _TABLE not in inspect(op.get_bind()).get_table_names():
|
||||
return
|
||||
indexes = {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
|
||||
if _INDEX in indexes:
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
columns = {column["name"] for column in inspect(op.get_bind()).get_columns(_TABLE)}
|
||||
if _COLUMN in columns:
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""Persist the globally shared default reasoning effort for each model.
|
||||
|
||||
Revision ID: 20261026_02
|
||||
Revises: 20261026_01, 20261022_01
|
||||
Create Date: 2026-10-26
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261026_02"
|
||||
down_revision = ("20261026_01", "20261022_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "model_capabilities"
|
||||
_COLUMN = "default_reasoning_effort"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(
|
||||
_COLUMN,
|
||||
sa.String(length=16),
|
||||
nullable=False,
|
||||
server_default="xhigh",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""Persist the administrator-configured Redis cache TTL.
|
||||
|
||||
Revision ID: 20261029_01
|
||||
Revises: 20261026_02
|
||||
Create Date: 2026-10-29
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261029_01"
|
||||
down_revision = "20261026_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "api_read_cache_settings"
|
||||
_COLUMN = "ttl_seconds"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(
|
||||
_COLUMN,
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
server_default=sa.text("300"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
"""Persist per-API cache switches and browser status access control."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20261029_02"
|
||||
down_revision = "20261029_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if "api_read_cache_settings" not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns("api_read_cache_settings")}
|
||||
if "collection_overrides" not in columns:
|
||||
op.add_column("api_read_cache_settings", sa.Column("collection_overrides", sa.JSON(), nullable=True))
|
||||
if "browser_status_enabled" not in columns:
|
||||
op.add_column("api_read_cache_settings", sa.Column("browser_status_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
|
||||
if "cache_epoch" not in columns:
|
||||
op.add_column("api_read_cache_settings", sa.Column("cache_epoch", sa.String(32), nullable=False, server_default="0"))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if "api_read_cache_settings" not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns("api_read_cache_settings")}
|
||||
for name in ("cache_epoch", "browser_status_enabled", "collection_overrides"):
|
||||
if name in columns:
|
||||
op.drop_column("api_read_cache_settings", name)
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
"""Add per-item enablement to compact menu overrides.
|
||||
|
||||
Existing compact-menu items remain enabled. New databases receive the field
|
||||
from ``CompactMenuOverrideRow``; this revision and the startup ORM schema sweep
|
||||
add it safely to deployments with an existing ``compact_menu_overrides`` table.
|
||||
|
||||
Revision ID: 20261102_01
|
||||
Revises: 20261029_02
|
||||
Create Date: 2026-11-02
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261102_01"
|
||||
down_revision = "20261029_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "compact_menu_overrides"
|
||||
_COLUMN = "disabled"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
"""Allow administrator-defined business-mapping types and login goPaths.
|
||||
|
||||
Revision ID: 20261102_02
|
||||
Revises: 20261102_01
|
||||
Create Date: 2026-11-02
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261102_02"
|
||||
down_revision = "20261102_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_MAPPING_TABLE = "business_mapping"
|
||||
_THREAD_TABLE = "task_deeplink_threads"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
tables = set(inspector.get_table_names())
|
||||
if _MAPPING_TABLE in tables:
|
||||
columns = {column["name"] for column in inspector.get_columns(_MAPPING_TABLE)}
|
||||
if "mapping_type" not in columns:
|
||||
op.add_column(
|
||||
_MAPPING_TABLE,
|
||||
sa.Column("mapping_type", sa.String(64), nullable=False, server_default="内置业务"),
|
||||
)
|
||||
if "is_custom" not in columns:
|
||||
op.add_column(
|
||||
_MAPPING_TABLE,
|
||||
sa.Column("is_custom", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
)
|
||||
|
||||
# PostgreSQL enforces VARCHAR length, unlike SQLite. Widen the dedicated
|
||||
# history partition too so a 32-character custom goPath can be persisted.
|
||||
if _THREAD_TABLE in tables:
|
||||
with op.batch_alter_table(_THREAD_TABLE) as batch:
|
||||
batch.alter_column(
|
||||
"go_path",
|
||||
existing_type=sa.String(16),
|
||||
type_=sa.String(32),
|
||||
existing_nullable=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
tables = set(inspector.get_table_names())
|
||||
if _THREAD_TABLE in tables:
|
||||
with op.batch_alter_table(_THREAD_TABLE) as batch:
|
||||
batch.alter_column(
|
||||
"go_path",
|
||||
existing_type=sa.String(32),
|
||||
type_=sa.String(16),
|
||||
existing_nullable=False,
|
||||
)
|
||||
if _MAPPING_TABLE in tables:
|
||||
columns = {column["name"] for column in inspector.get_columns(_MAPPING_TABLE)}
|
||||
if "is_custom" in columns:
|
||||
op.drop_column(_MAPPING_TABLE, "is_custom")
|
||||
if "mapping_type" in columns:
|
||||
op.drop_column(_MAPPING_TABLE, "mapping_type")
|
||||
+1
@@ -28,6 +28,7 @@ class ModelCapabilityStore(ABC):
|
||||
model_name: str,
|
||||
*,
|
||||
supports_reasoning_effort: bool,
|
||||
default_reasoning_effort: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
+4
@@ -24,11 +24,15 @@ class MemoryModelCapabilityStore(ModelCapabilityStore):
|
||||
model_name: str,
|
||||
*,
|
||||
supports_reasoning_effort: bool,
|
||||
default_reasoning_effort: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
row = {
|
||||
"model_name": model_name,
|
||||
"supports_reasoning_effort": bool(supports_reasoning_effort),
|
||||
"default_reasoning_effort": default_reasoning_effort
|
||||
or (self._rows.get(model_name) or {}).get("default_reasoning_effort")
|
||||
or "xhigh",
|
||||
"updated_by": updated_by,
|
||||
"updated_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
+6
@@ -21,6 +21,12 @@ class ModelCapabilityRow(Base):
|
||||
default=False,
|
||||
server_default=false(),
|
||||
)
|
||||
default_reasoning_effort: Mapped[str] = mapped_column(
|
||||
String(16),
|
||||
nullable=False,
|
||||
default="xhigh",
|
||||
server_default="xhigh",
|
||||
)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
|
||||
+4
@@ -17,6 +17,7 @@ def _to_dict(row: ModelCapabilityRow) -> dict[str, Any]:
|
||||
return {
|
||||
"model_name": row.model_name,
|
||||
"supports_reasoning_effort": bool(row.supports_reasoning_effort),
|
||||
"default_reasoning_effort": row.default_reasoning_effort or "xhigh",
|
||||
"updated_by": row.updated_by,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
}
|
||||
@@ -41,6 +42,7 @@ class ModelCapabilityRepository(ModelCapabilityStore):
|
||||
model_name: str,
|
||||
*,
|
||||
supports_reasoning_effort: bool,
|
||||
default_reasoning_effort: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
@@ -49,6 +51,8 @@ class ModelCapabilityRepository(ModelCapabilityStore):
|
||||
row = ModelCapabilityRow(model_name=model_name)
|
||||
session.add(row)
|
||||
row.supports_reasoning_effort = bool(supports_reasoning_effort)
|
||||
if default_reasoning_effort is not None:
|
||||
row.default_reasoning_effort = default_reasoning_effort
|
||||
row.updated_by = updated_by
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
|
||||
+17
-1
@@ -35,6 +35,7 @@ from deerflow.persistence.agents.model import (
|
||||
AgentRow,
|
||||
AgentSkillRow,
|
||||
)
|
||||
from deerflow.persistence.api_read_cache_settings.model import ApiReadCacheSettingRow
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
from deerflow.persistence.analysis_project_memory.model import (
|
||||
AnalysisProjectMemoryItemRow,
|
||||
@@ -79,6 +80,7 @@ from deerflow.persistence.force_situation.model import (
|
||||
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
|
||||
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
from deerflow.persistence.knowledge_research import KnowledgeResearchSessionRow
|
||||
from deerflow.persistence.knowledge_retrieval_settings.model import KnowledgeRetrievalSettingsRow
|
||||
from deerflow.persistence.light_apps.model import LightAppRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
@@ -127,6 +129,7 @@ from deerflow.persistence.roundtable_drafts.model import (
|
||||
RoundtableRecommendHistoryRow,
|
||||
)
|
||||
from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
|
||||
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
|
||||
from deerflow.persistence.roundtable_task_drafts.model import RoundtableTaskDraftRow
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.scheduled_tasks.model import (
|
||||
@@ -149,7 +152,12 @@ from deerflow.persistence.skill_knowledge.model import (
|
||||
SkillKnowledgeSyncItemRow,
|
||||
SkillKnowledgeSyncJobRow,
|
||||
)
|
||||
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
|
||||
from deerflow.persistence.skills.model import (
|
||||
SkillCatalogSnapshotRow,
|
||||
SkillDisplayAdapterRow,
|
||||
SkillFavoriteRow,
|
||||
SkillRow,
|
||||
)
|
||||
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.task_analysis.model import (
|
||||
@@ -165,12 +173,14 @@ from deerflow.persistence.task_analysis.model import (
|
||||
TaskDataSubmissionRow,
|
||||
)
|
||||
from deerflow.persistence.task_buttons.model import TaskButtonRow
|
||||
from deerflow.persistence.task_deeplink_details.model import TaskDeeplinkDetailRow
|
||||
from deerflow.persistence.task_deeplink_threads.model import TaskDeeplinkThreadRow
|
||||
from deerflow.persistence.task_reports.model import TaskReportRow
|
||||
from deerflow.persistence.taskcop_tasks.model import TaskCopTaskRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
|
||||
from deerflow.persistence.upload_policy.model import UploadPolicyRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.user_preferences.model import UserPreferencesRow
|
||||
@@ -207,6 +217,7 @@ __all__ = [
|
||||
"AgentExtrasRow",
|
||||
"AgentFavoriteRow",
|
||||
"AgentRow",
|
||||
"ApiReadCacheSettingRow",
|
||||
"AgentSkillRow",
|
||||
"AdminLeaderboardDailyStatRow",
|
||||
"BusinessMappingRow",
|
||||
@@ -241,6 +252,7 @@ __all__ = [
|
||||
"KnowledgeRelationRow",
|
||||
"KnowledgeSourceRow",
|
||||
"KnowledgeTagRow",
|
||||
"KnowledgeResearchSessionRow",
|
||||
"KnowledgeRetrievalSettingsRow",
|
||||
"LightAppRow",
|
||||
"WorkflowDataSourceRow",
|
||||
@@ -289,6 +301,7 @@ __all__ = [
|
||||
"RoundtableDraftShareRow",
|
||||
"RoundtableDiagnosticRow",
|
||||
"RoundtableJobRow",
|
||||
"RoundtableRoomLeaseRow",
|
||||
"RoundtableRecommendHistoryRow",
|
||||
"RoundtableTaskDraftRow",
|
||||
"PositionRoleRow",
|
||||
@@ -301,6 +314,7 @@ __all__ = [
|
||||
"ScheduledTaskSubscriptionRow",
|
||||
"SchedulerThreadRow",
|
||||
"SensitiveWordRow",
|
||||
"SkillCatalogSnapshotRow",
|
||||
"SkillRow",
|
||||
"SkillDisplayAdapterRow",
|
||||
"SkillFavoriteRow",
|
||||
@@ -328,6 +342,7 @@ __all__ = [
|
||||
"TaskDataBlockRow",
|
||||
"TaskDataPeriodRow",
|
||||
"TaskDataSubmissionRow",
|
||||
"TaskDeeplinkDetailRow",
|
||||
"TaskDeeplinkThreadRow",
|
||||
"AnalysisProjectRow",
|
||||
"AnalysisProjectThreadRow",
|
||||
@@ -338,6 +353,7 @@ __all__ = [
|
||||
"ThreadMetaRow",
|
||||
"ThreadShareRow",
|
||||
"ToolCallMetricRow",
|
||||
"StreamTruncationEventRow",
|
||||
"UploadPolicyRow",
|
||||
"UserCustomPromptRow",
|
||||
"UserMemoryConfigRow",
|
||||
|
||||
+9
-1
@@ -15,7 +15,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy import Index, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -24,6 +24,14 @@ from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
class ParallelAgentTaskRow(Base):
|
||||
__tablename__ = "parallel_agent_tasks"
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_parallel_agent_tasks_user_updated_id",
|
||||
"user_id",
|
||||
"updated_at",
|
||||
"id",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
|
||||
+41
-12
@@ -11,6 +11,7 @@ that belongs to another user is treated as not found.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
@@ -44,17 +45,32 @@ class ParallelAgentTaskRepository:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _to_dict(row: ParallelAgentTaskRow, *, with_messages: bool = True) -> dict[str, Any]:
|
||||
def _to_dict(
|
||||
row: ParallelAgentTaskRow | Mapping[str, Any],
|
||||
*,
|
||||
with_messages: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
def value(name: str) -> Any:
|
||||
if isinstance(row, Mapping):
|
||||
return row[name]
|
||||
return getattr(row, name)
|
||||
|
||||
created_at = value("created_at")
|
||||
updated_at = value("updated_at")
|
||||
data: dict[str, Any] = {
|
||||
"id": row.id,
|
||||
"title": row.title,
|
||||
"brief": row.brief,
|
||||
"agents": _loads(row.agents) or [],
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
"id": value("id"),
|
||||
"title": value("title"),
|
||||
"brief": value("brief"),
|
||||
"agents": _loads(value("agents")) or [],
|
||||
"created_at": (
|
||||
created_at.isoformat() if isinstance(created_at, datetime) else created_at
|
||||
),
|
||||
"updated_at": (
|
||||
updated_at.isoformat() if isinstance(updated_at, datetime) else updated_at
|
||||
),
|
||||
}
|
||||
if with_messages:
|
||||
data["messages"] = _loads(row.messages) or []
|
||||
data["messages"] = _loads(value("messages")) or []
|
||||
return data
|
||||
|
||||
async def list_tasks(
|
||||
@@ -74,9 +90,22 @@ class ParallelAgentTaskRepository:
|
||||
and ``offset`` allow those projections to page only their own task
|
||||
namespace without loading unrelated, potentially large transcripts.
|
||||
"""
|
||||
stmt = select(ParallelAgentTaskRow).where(ParallelAgentTaskRow.user_id == user_id)
|
||||
# History/list callers only need the summary columns. In particular,
|
||||
# never select the LONGTEXT transcript column unless explicitly asked:
|
||||
# MySQL otherwise reads and transfers it even though _to_dict omits it.
|
||||
columns = [
|
||||
ParallelAgentTaskRow.id,
|
||||
ParallelAgentTaskRow.title,
|
||||
ParallelAgentTaskRow.brief,
|
||||
ParallelAgentTaskRow.agents,
|
||||
ParallelAgentTaskRow.created_at,
|
||||
ParallelAgentTaskRow.updated_at,
|
||||
]
|
||||
if include_messages:
|
||||
columns.append(ParallelAgentTaskRow.messages)
|
||||
stmt = select(*columns).where(ParallelAgentTaskRow.user_id == user_id)
|
||||
if id_prefix:
|
||||
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix))
|
||||
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix, autoescape=True))
|
||||
stmt = (
|
||||
stmt.order_by(
|
||||
ParallelAgentTaskRow.updated_at.desc(),
|
||||
@@ -88,8 +117,8 @@ class ParallelAgentTaskRepository:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [
|
||||
self._to_dict(row, with_messages=include_messages)
|
||||
for row in result.scalars()
|
||||
self._to_dict(row._mapping, with_messages=include_messages)
|
||||
for row in result
|
||||
]
|
||||
|
||||
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
|
||||
+9
@@ -7,6 +7,15 @@ from typing import Any
|
||||
|
||||
|
||||
class PositionStore(ABC):
|
||||
async def get_effective_position(self, user_id: str) -> dict[str, Any] | None:
|
||||
"""Return an assigned position or the default without membership counts."""
|
||||
position_id = await self.get_user_position_id(user_id)
|
||||
record = await self.get_position(position_id) if position_id else None
|
||||
if record is None:
|
||||
default_id = await self.get_default_position_id()
|
||||
record = await self.get_position(default_id) if default_id else None
|
||||
return record
|
||||
|
||||
# -------------------------------------------------------------- positions
|
||||
|
||||
@abstractmethod
|
||||
|
||||
+20
@@ -7,6 +7,7 @@ from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.orm import aliased
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
@@ -60,6 +61,25 @@ class PositionRepository(PositionStore):
|
||||
counts = await self._member_counts(session, [position_id])
|
||||
return _position_to_dict(row, counts.get(position_id, 0))
|
||||
|
||||
async def get_effective_position(self, user_id: str) -> dict[str, Any] | None:
|
||||
"""Resolve the assigned/default position with one narrow indexed query."""
|
||||
assigned = aliased(PositionRow)
|
||||
default = aliased(PositionRow)
|
||||
stmt = (
|
||||
select(assigned, default)
|
||||
.select_from(UserRow)
|
||||
.outerjoin(assigned, UserRow.position_id == assigned.id)
|
||||
.outerjoin(default, default.is_default.is_(True))
|
||||
.where(UserRow.id == user_id)
|
||||
.limit(1)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).first()
|
||||
if row is None:
|
||||
return None
|
||||
position = row[0] or row[1]
|
||||
return _position_to_dict(position, 0) if position is not None else None
|
||||
|
||||
async def create_position(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
is_default = bool(data.get("is_default"))
|
||||
|
||||
+7
@@ -75,6 +75,13 @@ class RoundtableChainRow(Base):
|
||||
pending_review: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0", index=True
|
||||
)
|
||||
# 管理员目录开关:True = 从所有普通列表(我的/公共/待审核/选链弹窗)隐藏,仅管理员
|
||||
# 管理视图(``admin_view=true`` 的列表请求)可见。纯目录开关:单条读取(get_chain)
|
||||
# 不过滤,已配置的业务映射 / 进行中的研讨不受影响。``server_default`` 显式声明以支持
|
||||
# 启动期 ``_ensure_orm_columns_sync`` 在既有表上补列(NOT NULL 需带默认)。
|
||||
hidden: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0", index=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
+22
-1
@@ -66,6 +66,8 @@ class RoundtableChainRepository:
|
||||
"enabled": bool(getattr(row, "enabled", True)),
|
||||
# 「待审核」标记:True = owner 已提交审核、等待管理员发布;发布时清回 False。
|
||||
"pending_review": bool(getattr(row, "pending_review", False)),
|
||||
# 管理员目录开关:True = 从普通列表隐藏(仅管理视图可见);缺省 False。
|
||||
"hidden": bool(getattr(row, "hidden", False)),
|
||||
"is_mine": (current_user_id is None) or (row.user_id == current_user_id),
|
||||
# 配置者(owner)的 user_id;路由层据此解析成可读用户名(owner_name)。
|
||||
"owner_id": row.user_id,
|
||||
@@ -74,7 +76,14 @@ class RoundtableChainRepository:
|
||||
}
|
||||
|
||||
async def list_chains(
|
||||
self, user_id: str, *, scope: str = "mine", limit: int = 200, is_admin: bool = False
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
scope: str = "mine",
|
||||
limit: int = 200,
|
||||
is_admin: bool = False,
|
||||
include_hidden: bool = False,
|
||||
hidden: bool | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""列出链条。
|
||||
|
||||
@@ -86,6 +95,10 @@ class RoundtableChainRepository:
|
||||
- ``"all"``:所有账号的全部链条(**仅管理员**,由路由层校验后传 ``is_admin=True``;
|
||||
非管理员误传则按 ``"mine"`` 退化,绝不越权)。
|
||||
其余值按 ``"mine"`` 处理。``is_mine`` 标记每条是否本人所有,供前端判断可否编辑。
|
||||
|
||||
隐藏开关(``hidden=True`` 的行):默认从**所有**列表排除(含 admin 的普通视图,如
|
||||
选链弹窗);``include_hidden=True``(管理员管理视图)时不排除,并可再用 ``hidden``
|
||||
参数筛选「只看隐藏 / 只看显示中」。非管理员的 ``include_hidden`` 一律忽略。
|
||||
"""
|
||||
if scope == "public":
|
||||
cond = RoundtableChainRow.is_public.is_(True)
|
||||
@@ -98,6 +111,11 @@ class RoundtableChainRepository:
|
||||
stmt = select(RoundtableChainRow)
|
||||
if cond is not None:
|
||||
stmt = stmt.where(cond)
|
||||
if include_hidden and is_admin:
|
||||
if hidden is not None:
|
||||
stmt = stmt.where(RoundtableChainRow.hidden.is_(hidden))
|
||||
else:
|
||||
stmt = stmt.where(RoundtableChainRow.hidden.is_(False))
|
||||
stmt = stmt.order_by(RoundtableChainRow.updated_at.desc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
@@ -157,6 +175,7 @@ class RoundtableChainRepository:
|
||||
is_public=_UNSET,
|
||||
enabled=_UNSET,
|
||||
pending_review=_UNSET,
|
||||
hidden=_UNSET,
|
||||
is_admin: bool = False,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
@@ -186,6 +205,8 @@ class RoundtableChainRepository:
|
||||
row.enabled = bool(enabled)
|
||||
if pending_review is not _UNSET and pending_review is not None:
|
||||
row.pending_review = bool(pending_review)
|
||||
if hidden is not _UNSET and hidden is not None:
|
||||
row.hidden = bool(hidden)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
|
||||
+1
-1
@@ -68,7 +68,7 @@ class RoundtableJobRow(Base):
|
||||
# Fine-grained phase (§2.2): initializing / leader_thinking / dispatching /
|
||||
# consensus / report_drawing / report_done.
|
||||
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
|
||||
# Current orchestration cycle (0..MAX_CYCLES).
|
||||
# Current orchestration cycle (0..configured roundtable max_cycles).
|
||||
cycle: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# Dispatch chain nodes (§2.3): JSON array of
|
||||
# { agentId, name, avatarType, order, state, lastSpokeCycle? }.
|
||||
|
||||
+28
-1
@@ -119,6 +119,7 @@ class RoundtableJobRepository:
|
||||
request_hash: str | None = None,
|
||||
active_dedupe_key: str | None = None,
|
||||
input_snapshot: dict[str, Any] | None = None,
|
||||
room_started_at: datetime | None = None,
|
||||
) -> RoundtableJobRow:
|
||||
now = datetime.now(UTC)
|
||||
return RoundtableJobRow(
|
||||
@@ -145,7 +146,7 @@ class RoundtableJobRepository:
|
||||
error=None,
|
||||
attempt=0,
|
||||
version=0,
|
||||
created_at=now,
|
||||
created_at=room_started_at or now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
@@ -204,6 +205,7 @@ class RoundtableJobRepository:
|
||||
request_id: str | None = None,
|
||||
request_hash: str | None = None,
|
||||
input_snapshot: dict[str, Any] | None = None,
|
||||
room_started_at: datetime | None = None,
|
||||
) -> tuple[dict[str, Any], bool]:
|
||||
"""原子地「创建活跃作业 → 冲突则幂等返回已有活跃作业」(Phase 3 唯一活跃键版)。
|
||||
|
||||
@@ -240,6 +242,7 @@ class RoundtableJobRepository:
|
||||
request_hash=request_hash,
|
||||
active_dedupe_key=dedupe_key,
|
||||
input_snapshot=input_snapshot,
|
||||
room_started_at=room_started_at,
|
||||
)
|
||||
|
||||
# 无 draft_id → 无法去重,直接创建(保持原有行为)。
|
||||
@@ -462,6 +465,30 @@ class RoundtableJobRepository:
|
||||
result = await session.execute(stmt)
|
||||
return [r[0] for r in result.all()]
|
||||
|
||||
async def request_cancel_expired_step2(
|
||||
self,
|
||||
*,
|
||||
created_before: datetime,
|
||||
non_occupying_phases: tuple[str, ...],
|
||||
) -> int:
|
||||
"""Cancel unfinished Step-2 jobs that exceeded the room hard lifetime."""
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(RoundtableJobRow)
|
||||
.where(
|
||||
RoundtableJobRow.status.in_(list(CANCELLABLE_STATUSES)),
|
||||
RoundtableJobRow.created_at <= created_before,
|
||||
RoundtableJobRow.phase.not_in(list(non_occupying_phases)),
|
||||
)
|
||||
.values(
|
||||
status="cancel_requested",
|
||||
version=RoundtableJobRow.version + 1,
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def list_cancel_reapable(self, *, now: datetime, limit: int = 20) -> list[str]:
|
||||
"""列出可以由 dispatcher 收口的取消请求。
|
||||
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"""Roundtable step-2 room leases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
|
||||
from deerflow.persistence.roundtable_rooms.sql import RoundtableRoomRepository
|
||||
|
||||
|
||||
def make_roundtable_room_store(session_factory) -> RoundtableRoomRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return RoundtableRoomRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RoundtableRoomLeaseRow",
|
||||
"RoundtableRoomRepository",
|
||||
"make_roundtable_room_store",
|
||||
]
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
"""Foreground roundtable room leases.
|
||||
|
||||
One row is one browser session that has entered step 2 and is still heartbeating.
|
||||
Background jobs are not stored here: they already occupy a room through
|
||||
``roundtable_jobs`` while their status is non-terminal. Both sides share the
|
||||
job active-dedupe key so a foreground session and its later background job
|
||||
count as one room.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class RoundtableRoomLeaseRow(Base):
|
||||
__tablename__ = "roundtable_room_leases"
|
||||
|
||||
room_key: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
draft_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
task_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
# Last account to heartbeat. Release only succeeds for this account, so one
|
||||
# viewer leaving a shared task room does not drop another viewer's lease.
|
||||
holder_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
+584
@@ -0,0 +1,584 @@
|
||||
"""Deployment-wide roundtable room leases and background-job occupancy.
|
||||
|
||||
A room is one live roundtable execution, not one saved draft, not one account,
|
||||
and not one seat. Different pages running the same draft have distinct hidden
|
||||
session ids; every seat and step of one execution shares one id. The active set
|
||||
is the union of every user's:
|
||||
|
||||
- a durable foreground/session lease renewed by the browser or a run entry
|
||||
- live ``extra_keys`` supplied by the Gateway as a reconciliation fallback
|
||||
only when no durable lease/job already represents that persisted scope
|
||||
- a background job that is executing or waiting for the user's Step-2
|
||||
clarification (``queued`` / ``running`` / ``awaiting_input`` /
|
||||
``cancel_requested``)
|
||||
|
||||
Historical jobs older than the hard room lifetime and merely viewing chat
|
||||
history are not rooms. There is no per-user pool.
|
||||
|
||||
Admission and lease creation happen while holding one database-backed gate
|
||||
row. Updating that row serializes contenders on SQLite, MySQL and PostgreSQL,
|
||||
so four simultaneous requests cannot all observe the same stale count.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
|
||||
from deerflow.persistence.roundtable_jobs.sql import compute_active_dedupe_key
|
||||
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
|
||||
|
||||
LEASE_SECONDS = 60
|
||||
JOB_LIVE_SECONDS = 15 * 60
|
||||
OCCUPYING_JOB_STATUSES = ("queued", "running", "awaiting_input", "cancel_requested")
|
||||
# A background job keeps running while Step 3 generates its artifacts, but the
|
||||
# concurrency cap is intentionally scoped to Step 2. ``consensus`` is written
|
||||
# before Step 3 begins, which releases the slot without waiting for reporting.
|
||||
NON_OCCUPYING_JOB_PHASES = frozenset({
|
||||
"consensus",
|
||||
"summary_writing",
|
||||
"report_drawing",
|
||||
"report_done",
|
||||
})
|
||||
GATE_ROW_KEY = "__roundtable_room_gate__"
|
||||
|
||||
|
||||
def compute_room_session_key(room_session_id: str | None) -> str | None:
|
||||
"""Stable key for one live roundtable execution (not one saved draft)."""
|
||||
clean = (room_session_id or "").strip()
|
||||
if not clean:
|
||||
return None
|
||||
return hashlib.sha256(f"roundtable-session:{clean}".encode()).hexdigest()
|
||||
|
||||
|
||||
def _aware(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=UTC)
|
||||
return value
|
||||
|
||||
|
||||
def _job_is_live(
|
||||
lease_until: datetime | None,
|
||||
updated_at: datetime | None,
|
||||
now: datetime,
|
||||
live_after: datetime,
|
||||
) -> bool:
|
||||
until = _aware(lease_until)
|
||||
if until is not None and until > now:
|
||||
return True
|
||||
updated = _aware(updated_at)
|
||||
return updated is not None and updated >= live_after
|
||||
|
||||
|
||||
def _hard_deadline_reached(
|
||||
created_at: datetime | None,
|
||||
now: datetime,
|
||||
max_hold_minutes: int,
|
||||
) -> bool:
|
||||
created = _aware(created_at)
|
||||
return created is not None and created + timedelta(minutes=max_hold_minutes) <= now
|
||||
|
||||
|
||||
class RoundtableRoomRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def admit(
|
||||
self,
|
||||
*,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
room_session_id: str | None = None,
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically reserve/renew a room, or reject when the pool is full."""
|
||||
key = compute_room_session_key(room_session_id) or compute_active_dedupe_key(
|
||||
draft_id, user_id, task_id
|
||||
)
|
||||
return await self.admit_room_key(
|
||||
room_key=key,
|
||||
draft_id=draft_id,
|
||||
user_id=user_id,
|
||||
task_id=task_id,
|
||||
holder_user_id=user_id,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
extra_keys=extra_keys,
|
||||
)
|
||||
|
||||
async def admit_room_key(
|
||||
self,
|
||||
*,
|
||||
room_key: str | None,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
holder_user_id: str | None,
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically admit a pre-resolved room key.
|
||||
|
||||
``room_key`` is used by ``/run/stream`` when it resolves the room from
|
||||
thread metadata. ``user_id`` remains the room owner while
|
||||
``holder_user_id`` is the account currently renewing a shared room.
|
||||
"""
|
||||
async with self._lock:
|
||||
await self._ensure_gate_row()
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
# This no-op-ish write is intentional: it is the portable DB
|
||||
# mutex protecting the subsequent count + lease upsert.
|
||||
await session.execute(
|
||||
update(RoundtableRoomLeaseRow)
|
||||
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
|
||||
.values(updated_at=now)
|
||||
)
|
||||
keys, durable_scopes, hard_expired_keys = await self._active_snapshot(
|
||||
session,
|
||||
now,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
)
|
||||
self._merge_extra_keys(keys, durable_scopes, extra_keys)
|
||||
if room_key is None:
|
||||
await session.commit()
|
||||
blocked = bool(enabled)
|
||||
return self._decision(
|
||||
ok=not blocked,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=False,
|
||||
code="DRAFT_REQUIRED" if blocked else "",
|
||||
message="" if not blocked else "进入第二步需要先保存会商草稿",
|
||||
)
|
||||
if room_key in hard_expired_keys:
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=False,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=False,
|
||||
code="ROOM_MAX_HOLD_EXCEEDED",
|
||||
message="本次会商已达到最长占用时间,房间已自动释放。请返回第一步后重新发起。",
|
||||
)
|
||||
owns = room_key in keys
|
||||
if enabled and not owns and len(keys) >= max_rooms:
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=False,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=False,
|
||||
code="ROUNDTABLE_ROOM_FULL",
|
||||
message="当前会商房间暂时已满,请稍候片刻后再试。",
|
||||
)
|
||||
await self._upsert_lease(
|
||||
session,
|
||||
room_key=room_key,
|
||||
draft_id=draft_id,
|
||||
user_id=user_id,
|
||||
task_id=task_id,
|
||||
holder_user_id=holder_user_id,
|
||||
now=now,
|
||||
)
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=True,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys) if owns else len(keys) + 1,
|
||||
renewed=owns,
|
||||
code="",
|
||||
message="",
|
||||
)
|
||||
|
||||
async def release(
|
||||
self,
|
||||
*,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
room_session_id: str | None = None,
|
||||
) -> bool:
|
||||
"""Drop a leftover foreground lease row if one still exists."""
|
||||
key = compute_room_session_key(room_session_id) or compute_active_dedupe_key(
|
||||
draft_id, user_id, task_id
|
||||
)
|
||||
if not key:
|
||||
return False
|
||||
async with self._lock:
|
||||
await self._ensure_gate_row()
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(RoundtableRoomLeaseRow)
|
||||
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
|
||||
.values(updated_at=datetime.now(UTC))
|
||||
)
|
||||
row = await session.get(RoundtableRoomLeaseRow, key)
|
||||
if row is None or (user_id and row.holder_user_id not in (None, user_id)):
|
||||
await session.commit()
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def started_at(
|
||||
self,
|
||||
*,
|
||||
room_session_id: str | None,
|
||||
holder_user_id: str | None,
|
||||
) -> datetime | None:
|
||||
"""Return the original admission time for a foreground-to-job handoff."""
|
||||
key = compute_room_session_key(room_session_id)
|
||||
if key is None:
|
||||
return None
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableRoomLeaseRow, key)
|
||||
if row is None or (
|
||||
holder_user_id
|
||||
and row.holder_user_id not in (None, holder_user_id)
|
||||
):
|
||||
return None
|
||||
return _aware(row.created_at)
|
||||
|
||||
async def renew(
|
||||
self,
|
||||
*,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
room_session_id: str | None,
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Renew one live foreground lease without ever creating it.
|
||||
|
||||
Heartbeats are deliberately weaker than admission. A delayed browser
|
||||
callback must not resurrect a room that was released after a pause,
|
||||
network disconnect or completed Step 2.
|
||||
"""
|
||||
key = compute_room_session_key(room_session_id)
|
||||
if key is None:
|
||||
return self._decision(
|
||||
ok=not enabled,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=0,
|
||||
renewed=False,
|
||||
code="ROOM_SESSION_REQUIRED" if enabled else "",
|
||||
message="会商会话标识缺失,请刷新页面后重试。" if enabled else "",
|
||||
)
|
||||
|
||||
return await self.renew_room_key(
|
||||
room_key=key,
|
||||
holder_user_id=user_id,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
extra_keys=extra_keys,
|
||||
)
|
||||
|
||||
async def renew_room_key(
|
||||
self,
|
||||
*,
|
||||
room_key: str,
|
||||
holder_user_id: str | None,
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Renew a resolved persisted key without creating a missing lease."""
|
||||
async with self._lock:
|
||||
await self._ensure_gate_row()
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(RoundtableRoomLeaseRow)
|
||||
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
|
||||
.values(updated_at=now)
|
||||
)
|
||||
keys, durable_scopes, hard_expired_keys = await self._active_snapshot(
|
||||
session,
|
||||
now,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
)
|
||||
self._merge_extra_keys(keys, durable_scopes, extra_keys)
|
||||
row = await session.get(RoundtableRoomLeaseRow, room_key)
|
||||
if row is not None and room_key in hard_expired_keys:
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=False,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=False,
|
||||
code="ROOM_MAX_HOLD_EXCEEDED",
|
||||
message="本次会商已达到最长占用时间,房间已自动释放。请返回第一步后重新发起。",
|
||||
)
|
||||
owns = row is not None and room_key in keys and (
|
||||
not holder_user_id or row.holder_user_id in (None, holder_user_id)
|
||||
)
|
||||
if not owns:
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=not enabled,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=False,
|
||||
code="ROOM_LEASE_EXPIRED" if enabled else "",
|
||||
message="会商连接已断开,请重新恢复研讨。" if enabled else "",
|
||||
)
|
||||
|
||||
row.lease_until = now + timedelta(seconds=LEASE_SECONDS)
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=True,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=True,
|
||||
code="",
|
||||
message="",
|
||||
)
|
||||
|
||||
async def summary(
|
||||
self,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
*,
|
||||
max_hold_minutes: int = 30,
|
||||
) -> int:
|
||||
"""Return the number of distinct occupied rooms across every user."""
|
||||
async with self._lock:
|
||||
await self._ensure_gate_row()
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
keys, durable_scopes, _hard_expired_keys = await self._active_snapshot(
|
||||
session,
|
||||
now,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
)
|
||||
self._merge_extra_keys(keys, durable_scopes, extra_keys)
|
||||
await session.commit()
|
||||
return len(keys)
|
||||
|
||||
@staticmethod
|
||||
def _merge_extra_keys(
|
||||
keys: set[str],
|
||||
durable_scopes: set[str],
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None,
|
||||
) -> None:
|
||||
"""Add run-manager fallbacks without double-counting durable occupancy.
|
||||
|
||||
The map value is the legacy draft/user-or-task scope. If a live page
|
||||
lease or background job already owns that scope, its stamped run key is
|
||||
merely evidence for the same room and must not become another room.
|
||||
"""
|
||||
if not extra_keys:
|
||||
return
|
||||
items = extra_keys.items() if isinstance(extra_keys, Mapping) else ((key, None) for key in extra_keys)
|
||||
for key, scope_key in items:
|
||||
if not key or (scope_key and scope_key in durable_scopes):
|
||||
continue
|
||||
keys.add(key)
|
||||
|
||||
async def _active_snapshot(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
now: datetime,
|
||||
*,
|
||||
max_hold_minutes: int,
|
||||
) -> tuple[set[str], set[str], set[str]]:
|
||||
lease_rows = (
|
||||
await session.execute(
|
||||
select(RoundtableRoomLeaseRow).where(
|
||||
RoundtableRoomLeaseRow.room_key != GATE_ROW_KEY
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
expired: list[str] = []
|
||||
keys: set[str] = set()
|
||||
durable_scopes: set[str] = set()
|
||||
hard_expired_keys: set[str] = set()
|
||||
for row in lease_rows:
|
||||
until = _aware(row.lease_until)
|
||||
if until is None or until <= now:
|
||||
expired.append(row.room_key)
|
||||
continue
|
||||
scope_key = compute_active_dedupe_key(row.draft_id, row.user_id, row.task_id)
|
||||
if _hard_deadline_reached(row.created_at, now, max_hold_minutes):
|
||||
# Keep the still-live row briefly as a non-occupying tombstone.
|
||||
# Its scope suppresses a lagging RunManager fallback until the
|
||||
# browser observes the rejected heartbeat and cancels the run.
|
||||
hard_expired_keys.add(row.room_key)
|
||||
if scope_key:
|
||||
durable_scopes.add(scope_key)
|
||||
continue
|
||||
keys.add(row.room_key)
|
||||
if scope_key:
|
||||
durable_scopes.add(scope_key)
|
||||
if expired:
|
||||
await session.execute(
|
||||
delete(RoundtableRoomLeaseRow).where(
|
||||
RoundtableRoomLeaseRow.room_key.in_(expired)
|
||||
)
|
||||
)
|
||||
|
||||
live_after = now - timedelta(seconds=JOB_LIVE_SECONDS)
|
||||
job_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
RoundtableJobRow.id,
|
||||
RoundtableJobRow.draft_id,
|
||||
RoundtableJobRow.user_id,
|
||||
RoundtableJobRow.task_id,
|
||||
RoundtableJobRow.status,
|
||||
RoundtableJobRow.phase,
|
||||
RoundtableJobRow.lease_until,
|
||||
RoundtableJobRow.updated_at,
|
||||
RoundtableJobRow.created_at,
|
||||
).where(RoundtableJobRow.status.in_(OCCUPYING_JOB_STATUSES))
|
||||
)
|
||||
).all()
|
||||
for job_id, draft_id, user_id, task_id, status, phase, lease_until, updated_at, created_at in job_rows:
|
||||
if phase in NON_OCCUPYING_JOB_PHASES:
|
||||
continue
|
||||
if _hard_deadline_reached(created_at, now, max_hold_minutes):
|
||||
# The worker cancellation watcher will finalize this state.
|
||||
# Marking the intent here also covers queued jobs and workers
|
||||
# that stopped renewing but have not yet been reaped.
|
||||
await session.execute(
|
||||
update(RoundtableJobRow)
|
||||
.where(
|
||||
RoundtableJobRow.id == job_id,
|
||||
RoundtableJobRow.status.in_(["queued", "running", "awaiting_input"]),
|
||||
)
|
||||
.values(
|
||||
status="cancel_requested",
|
||||
version=RoundtableJobRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
continue
|
||||
if status != "awaiting_input" and not _job_is_live(
|
||||
lease_until,
|
||||
updated_at,
|
||||
now,
|
||||
live_after,
|
||||
):
|
||||
continue
|
||||
job_key = compute_active_dedupe_key(draft_id, user_id, task_id)
|
||||
if job_key and job_key in durable_scopes:
|
||||
# Foreground → background handoff can overlap for a few
|
||||
# milliseconds. Both represent one room for the same scope.
|
||||
continue
|
||||
if not job_key:
|
||||
job_key = f"job:{job_id}"
|
||||
keys.add(job_key)
|
||||
scope_key = compute_active_dedupe_key(draft_id, user_id, task_id)
|
||||
if scope_key:
|
||||
durable_scopes.add(scope_key)
|
||||
return keys, durable_scopes, hard_expired_keys
|
||||
|
||||
async def _ensure_gate_row(self) -> None:
|
||||
"""Create the portable database mutex row once, tolerating races."""
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
if await session.get(RoundtableRoomLeaseRow, GATE_ROW_KEY) is not None:
|
||||
return
|
||||
session.add(
|
||||
RoundtableRoomLeaseRow(
|
||||
room_key=GATE_ROW_KEY,
|
||||
draft_id=None,
|
||||
user_id=None,
|
||||
task_id=None,
|
||||
holder_user_id=None,
|
||||
lease_until=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
|
||||
async def _upsert_lease(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
room_key: str,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
holder_user_id: str | None,
|
||||
now: datetime,
|
||||
) -> None:
|
||||
row = await session.get(RoundtableRoomLeaseRow, room_key)
|
||||
lease_until = now + timedelta(seconds=LEASE_SECONDS)
|
||||
if row is None:
|
||||
session.add(
|
||||
RoundtableRoomLeaseRow(
|
||||
room_key=room_key,
|
||||
draft_id=draft_id,
|
||||
user_id=user_id,
|
||||
task_id=task_id,
|
||||
holder_user_id=holder_user_id,
|
||||
lease_until=lease_until,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
return
|
||||
row.draft_id = draft_id
|
||||
row.user_id = user_id
|
||||
row.task_id = task_id
|
||||
row.holder_user_id = holder_user_id
|
||||
row.lease_until = lease_until
|
||||
row.updated_at = now
|
||||
|
||||
@staticmethod
|
||||
def _decision(
|
||||
*,
|
||||
ok: bool,
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
active_count: int,
|
||||
renewed: bool,
|
||||
code: str,
|
||||
message: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"ok": ok,
|
||||
"enabled": enabled,
|
||||
"max_rooms": max_rooms,
|
||||
"active_count": active_count,
|
||||
"renewed": renewed,
|
||||
"code": code,
|
||||
"message": message,
|
||||
}
|
||||
@@ -11,13 +11,15 @@ import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.runtime.runs.store.base import RunStore
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
_TERMINAL_RUN_STATUSES = ("success", "error", "timeout", "interrupted")
|
||||
|
||||
|
||||
class RunRepository(RunStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
@@ -137,6 +139,14 @@ class RunRepository(RunStore):
|
||||
await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(**values))
|
||||
await session.commit()
|
||||
|
||||
async def update_snapshot(self, run_id, *, metadata, kwargs):
|
||||
# Internal RunManager retention path; the run identity comes from its
|
||||
# live registry, never a client-supplied mutation endpoint.
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(metadata_json=self._safe_json(metadata), kwargs_json=self._safe_json(kwargs)))
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
run_id,
|
||||
@@ -205,21 +215,70 @@ class RunRepository(RunStore):
|
||||
await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(**values))
|
||||
await session.commit()
|
||||
|
||||
async def count_by_user(self) -> dict[str, int]:
|
||||
"""Return ``{user_id: run_count}`` via a single GROUP BY query.
|
||||
async def count_by_user(self, *, since: datetime | None = None) -> dict[str, int]:
|
||||
"""Return ``{user_id: run_count}`` via one bounded GROUP BY query.
|
||||
|
||||
Each run is one user question, so this is the source for the admin
|
||||
user center's 问答次数 column. Runs without a ``user_id`` are skipped.
|
||||
"""
|
||||
filters = [RunRow.user_id.is_not(None)]
|
||||
if since is not None:
|
||||
since_dt = since if since.tzinfo else since.replace(tzinfo=UTC)
|
||||
filters.append(RunRow.created_at >= since_dt)
|
||||
stmt = (
|
||||
select(RunRow.user_id, func.count().label("cnt"))
|
||||
.where(RunRow.user_id.is_not(None))
|
||||
.where(*filters)
|
||||
.group_by(RunRow.user_id)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return {r.user_id: r.cnt for r in rows}
|
||||
|
||||
async def purge_terminal_before(
|
||||
self,
|
||||
cutoff: datetime,
|
||||
*,
|
||||
batch_size: int = 500,
|
||||
) -> int:
|
||||
"""Delete one portable, primary-key-bounded batch of old run audits.
|
||||
|
||||
Pending and running records are intentionally excluded, so retention
|
||||
never removes work that could still be resumed or observed. A run must
|
||||
also have been *updated* before the cutoff: an unusually long-running
|
||||
job created months ago but completed today must remain visible. The
|
||||
compact run row is cleaned, while the conversation checkpoint stays
|
||||
under its separate user-history lifecycle.
|
||||
"""
|
||||
size = max(1, min(int(batch_size), 500))
|
||||
cutoff_dt = cutoff if cutoff.tzinfo else cutoff.replace(tzinfo=UTC)
|
||||
async with self._sf() as session:
|
||||
ids = list(
|
||||
(
|
||||
await session.scalars(
|
||||
select(RunRow.run_id)
|
||||
.where(
|
||||
RunRow.created_at < cutoff_dt,
|
||||
RunRow.updated_at < cutoff_dt,
|
||||
RunRow.status.in_(_TERMINAL_RUN_STATUSES),
|
||||
)
|
||||
.order_by(RunRow.created_at.asc(), RunRow.run_id.asc())
|
||||
.limit(size)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not ids:
|
||||
return 0
|
||||
result = await session.execute(
|
||||
delete(RunRow).where(
|
||||
RunRow.run_id.in_(ids),
|
||||
RunRow.created_at < cutoff_dt,
|
||||
RunRow.updated_at < cutoff_dt,
|
||||
RunRow.status.in_(_TERMINAL_RUN_STATUSES),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def aggregate_tokens_by_thread(self, thread_id: str) -> dict[str, Any]:
|
||||
"""Aggregate token usage via a single SQL GROUP BY query."""
|
||||
_completed = RunRow.status.in_(("success", "error"))
|
||||
|
||||
@@ -95,6 +95,15 @@ class SkillStore(ABC):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
|
||||
"""Map skill names to their configured Chinese display names.
|
||||
|
||||
Only names that have a non-empty ``name_zh`` are returned. The default
|
||||
is empty so stores that have not implemented the lookup stay safe.
|
||||
"""
|
||||
del names
|
||||
return {}
|
||||
|
||||
@abstractmethod
|
||||
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
|
||||
"""Return the skill names in ``user_id``'s "我的技能" collection.
|
||||
|
||||
+11
@@ -165,6 +165,17 @@ class MemorySkillStore(SkillStore):
|
||||
async def list_always_on_names(self) -> set[str]:
|
||||
return {name for name, record in self._skills.items() if record.get("always_on")}
|
||||
|
||||
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
|
||||
labels: dict[str, str] = {}
|
||||
for name in names:
|
||||
record = self._skills.get(name)
|
||||
if record is None:
|
||||
continue
|
||||
label = str(record.get("name_zh") or "").strip()
|
||||
if label:
|
||||
labels[name] = label
|
||||
return labels
|
||||
|
||||
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
|
||||
return set(self._favorites.get(user_id, set()))
|
||||
|
||||
|
||||
+36
-1
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, String, Text, UniqueConstraint
|
||||
from sqlalchemy import BigInteger, Boolean, DateTime, Index, Integer, String, Text, UniqueConstraint, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -19,10 +19,30 @@ from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
class SkillRow(Base):
|
||||
__tablename__ = "skills"
|
||||
__table_args__ = (
|
||||
Index("ix_skills_catalog_available_category_created", "available", "category", "created_at"),
|
||||
)
|
||||
|
||||
# Global unique name — must match the on-disk folder name under skills/custom/.
|
||||
# Hyphen-case or snake_case allowed; see deerflow.skills.validation.
|
||||
name: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Compact catalog projection fields. The list API reads these columns from
|
||||
# MySQL and deliberately excludes the long-form ``detail`` text column.
|
||||
category: Mapped[str] = mapped_column(String(16), nullable=False, default="custom", server_default="custom")
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="", server_default="")
|
||||
license: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
|
||||
available: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"), index=True)
|
||||
use_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
view_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
patch_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
last_activity_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, default=None)
|
||||
state: Mapped[str] = mapped_column(String(16), nullable=False, default="active", server_default="active")
|
||||
pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
|
||||
featured: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
|
||||
source: Mapped[str] = mapped_column(String(16), nullable=False, default="upload", server_default="upload")
|
||||
source_mtime_ns: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default=text("0"))
|
||||
detail_preview: Mapped[str | None] = mapped_column(String(1024), nullable=True, default=None)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# Human-authored long-form detail/introduction, editable by the skill owner
|
||||
@@ -83,3 +103,18 @@ class SkillDisplayAdapterRow(Base):
|
||||
sample_payload: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class SkillCatalogSnapshotRow(Base):
|
||||
"""MySQL-backed projection of filesystem skill metadata for fast list APIs.
|
||||
|
||||
Skill source files remain the authoring format. The Gateway refreshes this
|
||||
one-row projection during startup so homepage listing requests do not scan
|
||||
every SKILL.md or read per-skill usage sidecars.
|
||||
"""
|
||||
|
||||
__tablename__ = "skill_catalog_snapshots"
|
||||
|
||||
cache_key: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
catalog_json: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -109,6 +109,7 @@ class SkillRepository(SkillStore):
|
||||
row.published = bool(data["published"])
|
||||
if "detail" in data:
|
||||
row.detail = data["detail"]
|
||||
row.detail_preview = (str(data["detail"] or "")[:1024] or None)
|
||||
if "name_zh" in data:
|
||||
row.name_zh = data["name_zh"]
|
||||
if "always_on" in data:
|
||||
@@ -131,6 +132,7 @@ class SkillRepository(SkillStore):
|
||||
row.owner_user_id = data["owner_user_id"]
|
||||
if "detail" in data:
|
||||
row.detail = data["detail"]
|
||||
row.detail_preview = (str(data["detail"] or "")[:1024] or None)
|
||||
if "name_zh" in data:
|
||||
row.name_zh = data["name_zh"]
|
||||
if "always_on" in data:
|
||||
@@ -183,6 +185,20 @@ class SkillRepository(SkillStore):
|
||||
result = await session.execute(stmt)
|
||||
return {row[0] for row in result.all()}
|
||||
|
||||
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
|
||||
wanted = list(dict.fromkeys(name for name in names if name))
|
||||
if not wanted:
|
||||
return {}
|
||||
stmt = select(SkillRow.name, SkillRow.name_zh).where(SkillRow.name.in_(wanted))
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
labels: dict[str, str] = {}
|
||||
for name, name_zh in rows:
|
||||
label = (name_zh or "").strip()
|
||||
if label:
|
||||
labels[name] = label
|
||||
return labels
|
||||
|
||||
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
|
||||
stmt = select(SkillFavoriteRow.skill_name).where(SkillFavoriteRow.user_id == user_id)
|
||||
async with self._sf() as session:
|
||||
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
"""Persistence for model-stream truncation events (模型截断记录).
|
||||
|
||||
The patched OpenAI-compatible streaming layer calls
|
||||
:func:`record_stream_truncation_event` whenever a provider stream ends
|
||||
abnormally. The write is fire-and-forget: it is routed onto the engine's home
|
||||
loop via :func:`fire_and_forget_db_write` and must never disturb the model
|
||||
stream itself. Rows are capped at one month by the SQL repository.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from deerflow.persistence.stream_truncation.base import (
|
||||
RETENTION_DAYS,
|
||||
StreamTruncationEvent,
|
||||
StreamTruncationQuery,
|
||||
StreamTruncationStore,
|
||||
)
|
||||
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
|
||||
from deerflow.persistence.stream_truncation.sql import SqlStreamTruncationStore
|
||||
|
||||
__all__ = [
|
||||
"RETENTION_DAYS",
|
||||
"SqlStreamTruncationStore",
|
||||
"StreamTruncationEvent",
|
||||
"StreamTruncationEventRow",
|
||||
"StreamTruncationQuery",
|
||||
"StreamTruncationStore",
|
||||
"make_stream_truncation_store",
|
||||
"record_stream_truncation_event",
|
||||
]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_store_cache: SqlStreamTruncationStore | None = None
|
||||
_store_cache_failed = False
|
||||
|
||||
|
||||
def make_stream_truncation_store(session_factory) -> StreamTruncationStore | None:
|
||||
"""Return a SQL-backed truncation store, or ``None`` without a DB."""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return SqlStreamTruncationStore(session_factory)
|
||||
|
||||
|
||||
def record_stream_truncation_event(
|
||||
*,
|
||||
model_name: str | None,
|
||||
event_type: str,
|
||||
strict_mode: bool = False,
|
||||
finish_reason: str = "none",
|
||||
status: str = "success",
|
||||
error_type: str | None = None,
|
||||
chunk_count: int = 0,
|
||||
duration_ms: int = 0,
|
||||
first_chunk_ms: int = -1,
|
||||
provider_completion_id: str | None = None,
|
||||
run_id: str | None = None,
|
||||
detail: str | None = None,
|
||||
) -> None:
|
||||
"""Best-effort truncation-event write from any loop/thread. Never raises."""
|
||||
global _store_cache, _store_cache_failed
|
||||
if _store_cache is None:
|
||||
if _store_cache_failed:
|
||||
return
|
||||
try:
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
session_factory = get_session_factory()
|
||||
except Exception:
|
||||
session_factory = None
|
||||
if session_factory is None:
|
||||
# No DB configured (memory backend); do not retry on every stream.
|
||||
_store_cache_failed = True
|
||||
return
|
||||
_store_cache = SqlStreamTruncationStore(session_factory)
|
||||
run_id_value = run_id if run_id and run_id not in {"none", ""} else None
|
||||
event = StreamTruncationEvent(
|
||||
id=uuid.uuid4().hex,
|
||||
created_at=datetime.now(UTC),
|
||||
model_name=model_name,
|
||||
provider_completion_id=provider_completion_id
|
||||
if provider_completion_id and provider_completion_id != "none"
|
||||
else None,
|
||||
run_id=run_id_value,
|
||||
event_type=event_type,
|
||||
strict_mode=strict_mode,
|
||||
finish_reason=finish_reason,
|
||||
status=status,
|
||||
error_type=error_type,
|
||||
chunk_count=chunk_count,
|
||||
duration_ms=duration_ms,
|
||||
first_chunk_ms=first_chunk_ms,
|
||||
detail=detail,
|
||||
)
|
||||
try:
|
||||
from deerflow.persistence.engine import fire_and_forget_db_write
|
||||
|
||||
fire_and_forget_db_write(_store_cache.record(event))
|
||||
except Exception:
|
||||
logger.debug("stream truncation event drop", exc_info=True)
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
"""Repository contract for model-stream truncation events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
# Hard retention cap. The user-facing promise is "最多存一个月" of truncation
|
||||
# records; the SQL repository enforces it on every insert so no scheduler is
|
||||
# required for the cap to hold.
|
||||
RETENTION_DAYS = 30
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamTruncationEvent:
|
||||
"""In-memory shape of a truncation event before persistence."""
|
||||
|
||||
id: str
|
||||
created_at: datetime
|
||||
model_name: str | None = None
|
||||
provider_completion_id: str | None = None
|
||||
run_id: str | None = None
|
||||
# silent_no_terminal / incomplete_stream_error / length_truncated /
|
||||
# content_filter / midstream_error
|
||||
event_type: str = "silent_no_terminal"
|
||||
strict_mode: bool = False
|
||||
finish_reason: str = "none"
|
||||
status: str = "success"
|
||||
error_type: str | None = None
|
||||
chunk_count: int = 0
|
||||
duration_ms: int = 0
|
||||
first_chunk_ms: int = -1
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamTruncationQuery:
|
||||
"""Filter set for the admin list endpoint."""
|
||||
|
||||
model_name: str | None = None
|
||||
event_type: str | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
limit: int = 100
|
||||
offset: int = 0
|
||||
|
||||
|
||||
class StreamTruncationStore(ABC):
|
||||
"""Persistence contract for the model-layer recorder + admin API."""
|
||||
|
||||
@abstractmethod
|
||||
async def record(self, event: StreamTruncationEvent) -> None:
|
||||
"""Persist one event and enforce the retention cap. Must never raise."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
"""Return matching rows (newest first), paginated by ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count(self, query: StreamTruncationQuery) -> int:
|
||||
"""Return the unpaginated row count for ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def model_breakdown(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
"""Per-model aggregate: ``[{model_name, event_type counts..., total, last_at}]``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_before(self, cutoff: datetime) -> int:
|
||||
"""Delete rows older than ``cutoff``; return the deleted row count."""
|
||||
raise NotImplementedError
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
"""ORM model for model-stream truncation events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Index, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class StreamTruncationEventRow(Base):
|
||||
"""One row per detected model-stream truncation.
|
||||
|
||||
Written from the patched OpenAI-compatible streaming layer when a provider
|
||||
stream ends abnormally: without a terminal frame (silently committed half
|
||||
answer), via ``IncompleteChatStreamError`` (strict integrity mode, retried),
|
||||
on an output-length cap, or interrupted mid-stream by a transport error.
|
||||
Rows are retained for at most ``RETENTION_DAYS`` by the SQL repository.
|
||||
"""
|
||||
|
||||
__tablename__ = "stream_truncation_events"
|
||||
__table_args__ = (
|
||||
Index("ix_stream_truncation_created_model", "created_at", "model_name"),
|
||||
Index("ix_stream_truncation_created_event", "created_at", "event_type"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), index=True
|
||||
)
|
||||
|
||||
# Configured model name (the config.yaml entry name), sanitized by the model layer.
|
||||
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
# Provider-side completion id when the gateway supplied one; joins a row to
|
||||
# the upstream gateway's own request log.
|
||||
provider_completion_id: Mapped[str | None] = mapped_column(String(128))
|
||||
# LangGraph run id when the model layer could see one ("none" is dropped).
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
|
||||
# silent_no_terminal / incomplete_stream_error / length_truncated /
|
||||
# content_filter / midstream_error
|
||||
event_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
# Whether the model ran with strict_stream_integrity enabled.
|
||||
strict_mode: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
# Raw provider finish reason bucket: none / stop / length / tool_calls /
|
||||
# function_call / content_filter / end_turn / other.
|
||||
finish_reason: Mapped[str] = mapped_column(String(16), default="none")
|
||||
# Stream outcome from the model layer: success / error / cancelled.
|
||||
status: Mapped[str] = mapped_column(String(16), default="success")
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
chunk_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
first_chunk_ms: Mapped[int] = mapped_column(Integer, default=-1)
|
||||
|
||||
detail: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat()
|
||||
if isinstance(self.created_at, datetime)
|
||||
else self.created_at,
|
||||
"model_name": self.model_name,
|
||||
"provider_completion_id": self.provider_completion_id,
|
||||
"run_id": self.run_id,
|
||||
"event_type": self.event_type,
|
||||
"strict_mode": bool(self.strict_mode),
|
||||
"finish_reason": self.finish_reason,
|
||||
"status": self.status,
|
||||
"error_type": self.error_type,
|
||||
"chunk_count": self.chunk_count,
|
||||
"duration_ms": self.duration_ms,
|
||||
"first_chunk_ms": self.first_chunk_ms,
|
||||
"detail": self.detail,
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
"""SQLAlchemy-backed stream-truncation event repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, desc, func, select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.stream_truncation.base import (
|
||||
RETENTION_DAYS,
|
||||
StreamTruncationEvent,
|
||||
StreamTruncationQuery,
|
||||
StreamTruncationStore,
|
||||
)
|
||||
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SqlStreamTruncationStore(StreamTruncationStore):
|
||||
"""Persist truncation events with a hard one-month retention cap."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
async def _ensure_table(self, session) -> None:
|
||||
# Table creation follows the repo's create_all-on-first-use convention
|
||||
# (see e.g. task_buttons); safe to call repeatedly.
|
||||
try:
|
||||
await session.run_sync(
|
||||
lambda sync_session: StreamTruncationEventRow.__table__.create(
|
||||
sync_session.bind, checkfirst=True
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("stream_truncation table create skipped", exc_info=True)
|
||||
|
||||
async def record(self, event: StreamTruncationEvent) -> None:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
session.add(
|
||||
StreamTruncationEventRow(
|
||||
id=event.id,
|
||||
created_at=event.created_at,
|
||||
model_name=event.model_name,
|
||||
provider_completion_id=event.provider_completion_id,
|
||||
run_id=event.run_id,
|
||||
event_type=event.event_type,
|
||||
strict_mode=event.strict_mode,
|
||||
finish_reason=event.finish_reason,
|
||||
status=event.status,
|
||||
error_type=event.error_type,
|
||||
chunk_count=event.chunk_count,
|
||||
duration_ms=event.duration_ms,
|
||||
first_chunk_ms=event.first_chunk_ms,
|
||||
detail=event.detail,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
# Same session/transaction scope: enforce the retention cap on
|
||||
# every insert. Truncations are rare, so this stays cheap and
|
||||
# keeps the one-month promise without any scheduler.
|
||||
await self._enforce_retention(session)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to persist stream truncation event model=%s type=%s",
|
||||
event.model_name,
|
||||
event.event_type,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
async def _enforce_retention(self, session) -> None:
|
||||
"""Delete rows beyond the retention cap. Best-effort, never raises."""
|
||||
try:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=RETENTION_DAYS)
|
||||
await session.execute(
|
||||
delete(StreamTruncationEventRow).where(
|
||||
StreamTruncationEventRow.created_at < cutoff
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.debug("stream truncation retention sweep failed", exc_info=True)
|
||||
|
||||
def _apply_filters(self, query: StreamTruncationQuery):
|
||||
stmt = select(StreamTruncationEventRow)
|
||||
if query.model_name:
|
||||
stmt = stmt.where(StreamTruncationEventRow.model_name == query.model_name)
|
||||
if query.event_type:
|
||||
stmt = stmt.where(StreamTruncationEventRow.event_type == query.event_type)
|
||||
if query.since is not None:
|
||||
stmt = stmt.where(StreamTruncationEventRow.created_at >= query.since)
|
||||
if query.until is not None:
|
||||
stmt = stmt.where(StreamTruncationEventRow.created_at <= query.until)
|
||||
return stmt
|
||||
|
||||
async def list(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
stmt = self._apply_filters(query).order_by(
|
||||
desc(StreamTruncationEventRow.created_at)
|
||||
).limit(query.limit).offset(query.offset)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [row.to_dict() for row in rows]
|
||||
except Exception:
|
||||
logger.warning("Failed to list stream truncation events", exc_info=True)
|
||||
return []
|
||||
|
||||
async def count(self, query: StreamTruncationQuery) -> int:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
stmt = self._apply_filters(query).with_only_columns(
|
||||
func.count(StreamTruncationEventRow.id)
|
||||
)
|
||||
return int((await session.execute(stmt)).scalar() or 0)
|
||||
except Exception:
|
||||
logger.warning("Failed to count stream truncation events", exc_info=True)
|
||||
return 0
|
||||
|
||||
async def model_breakdown(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
stmt = self._apply_filters(query).with_only_columns(
|
||||
StreamTruncationEventRow.model_name,
|
||||
StreamTruncationEventRow.event_type,
|
||||
func.count(StreamTruncationEventRow.id).label("total"),
|
||||
func.max(StreamTruncationEventRow.created_at).label("last_at"),
|
||||
).group_by(StreamTruncationEventRow.model_name, StreamTruncationEventRow.event_type)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
merged: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
name = row.model_name or "(unknown)"
|
||||
bucket = merged.setdefault(
|
||||
name, {"model_name": name, "total": 0, "last_at": None}
|
||||
)
|
||||
bucket[row.event_type] = int(row.total or 0)
|
||||
bucket["total"] += int(row.total or 0)
|
||||
row_last = row.last_at
|
||||
if row_last is not None and (
|
||||
bucket["last_at"] is None or row_last > bucket["last_at"]
|
||||
):
|
||||
bucket["last_at"] = row_last
|
||||
result = sorted(
|
||||
merged.values(), key=lambda item: item["last_at"] or datetime.min.replace(tzinfo=UTC), reverse=True
|
||||
)
|
||||
for item in result:
|
||||
if isinstance(item["last_at"], datetime):
|
||||
item["last_at"] = item["last_at"].isoformat()
|
||||
return result
|
||||
except Exception:
|
||||
logger.warning("Failed to aggregate stream truncation events", exc_info=True)
|
||||
return []
|
||||
|
||||
async def purge_before(self, cutoff: datetime) -> int:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
result = await session.execute(
|
||||
delete(StreamTruncationEventRow).where(
|
||||
StreamTruncationEventRow.created_at < cutoff
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
rowcount = result.rowcount
|
||||
return 0 if rowcount is None else int(rowcount)
|
||||
except Exception:
|
||||
logger.warning("Failed to purge stream truncation events", exc_info=True)
|
||||
return 0
|
||||
+9
-2
@@ -14,7 +14,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, UniqueConstraint
|
||||
from sqlalchemy import Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -43,7 +43,14 @@ class TagRow(Base):
|
||||
|
||||
class TagAssignmentRow(Base):
|
||||
__tablename__ = "tag_assignments"
|
||||
__table_args__ = (UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),)
|
||||
__table_args__ = (
|
||||
UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),
|
||||
# Gallery list joins assignments by (type, target); the tag filter
|
||||
# looks up (type, tag). The unique key leads with tag_id, so neither
|
||||
# query can use it as a prefix.
|
||||
Index("ix_tag_assignments_type_target", "target_type", "target_id"),
|
||||
Index("ix_tag_assignments_type_tag", "target_type", "tag_id", "target_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""External task deep-link detail persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.task_deeplink_details.base import TaskDeeplinkDetailStore
|
||||
from deerflow.persistence.task_deeplink_details.memory import MemoryTaskDeeplinkDetailStore
|
||||
from deerflow.persistence.task_deeplink_details.model import TaskDeeplinkDetailRow
|
||||
from deerflow.persistence.task_deeplink_details.sql import TaskDeeplinkDetailRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryTaskDeeplinkDetailStore",
|
||||
"TaskDeeplinkDetailRepository",
|
||||
"TaskDeeplinkDetailRow",
|
||||
"TaskDeeplinkDetailStore",
|
||||
"make_task_deeplink_detail_store",
|
||||
]
|
||||
|
||||
|
||||
def make_task_deeplink_detail_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryTaskDeeplinkDetailStore()
|
||||
return TaskDeeplinkDetailRepository(session_factory)
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
"""Persistence contract for externally supplied task deep-link details."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class TaskDeeplinkDetailStore(ABC):
|
||||
"""Shared external task snapshots keyed by ``(detail_type, detail_id, sub_type)``.
|
||||
|
||||
``sub_type`` is the integrator-supplied business type (the deep-link
|
||||
``?type=``): the same task id may hold one snapshot per type, each with its
|
||||
own version counter. The empty string is the legacy no-type partition.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def get_detail(
|
||||
self,
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
"""Return one persisted task snapshot, or ``None`` when it is unknown."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def upsert_detail(
|
||||
self,
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
task_name: str,
|
||||
task_content: str,
|
||||
overview: str,
|
||||
extra: dict[str, Any],
|
||||
updated_by: str | None,
|
||||
) -> tuple[dict[str, Any], bool, bool]:
|
||||
"""Return ``(row, created, changed)`` for a complete task snapshot.
|
||||
|
||||
Implementations must not issue a write when the stored snapshot has
|
||||
exactly the same task fields. This keeps caller-side retries cheap
|
||||
and makes a repeated ``PUT`` idempotent in both state and version.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
"""In-memory implementation for external task deep-link details."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.task_deeplink_details.base import TaskDeeplinkDetailStore
|
||||
|
||||
|
||||
class MemoryTaskDeeplinkDetailStore(TaskDeeplinkDetailStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
|
||||
async def get_detail(
|
||||
self,
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
row = self._rows.get((detail_type, detail_id, sub_type))
|
||||
return deepcopy(row) if row is not None else None
|
||||
|
||||
async def upsert_detail(
|
||||
self,
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
task_name: str,
|
||||
task_content: str,
|
||||
overview: str,
|
||||
extra: dict[str, Any],
|
||||
updated_by: str | None,
|
||||
) -> tuple[dict[str, Any], bool, bool]:
|
||||
key = (detail_type, detail_id, sub_type)
|
||||
row = self._rows.get(key)
|
||||
created = row is None
|
||||
if row is None:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row = {
|
||||
"detail_type": detail_type,
|
||||
"detail_id": detail_id,
|
||||
"sub_type": sub_type,
|
||||
"version": 1,
|
||||
"created_by": updated_by,
|
||||
"created_at": now,
|
||||
}
|
||||
self._rows[key] = row
|
||||
changed = True
|
||||
elif (
|
||||
row["task_name"] == task_name
|
||||
and row["task_content"] == task_content
|
||||
and row["overview"] == overview
|
||||
and row["extra"] == extra
|
||||
):
|
||||
return deepcopy(row), False, False
|
||||
else:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row["version"] = int(row.get("version") or 0) + 1
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
row.update(
|
||||
{
|
||||
"task_name": task_name,
|
||||
"task_content": task_content,
|
||||
"overview": overview,
|
||||
"extra": deepcopy(extra),
|
||||
"updated_by": updated_by,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
return deepcopy(row), created, changed
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""ORM model for externally supplied task deep-link details."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class TaskDeeplinkDetailRow(Base):
|
||||
"""Canonical task snapshot received from an external deep-link provider."""
|
||||
|
||||
__tablename__ = "task_deeplink_details"
|
||||
|
||||
detail_type: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
detail_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
# Integrator-supplied business type (the deep-link ``?type=``): one task id
|
||||
# may carry several typed snapshots; "" is the legacy no-type partition.
|
||||
sub_type: Mapped[str] = mapped_column(String(64), primary_key=True, default="")
|
||||
task_name: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
task_content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
|
||||
overview: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
|
||||
extra_json: Mapped[dict[str, object]] = mapped_column(PortableJSON, nullable=False, default=dict)
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
"""SQLAlchemy-backed external task deep-link detail store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.task_deeplink_details.base import TaskDeeplinkDetailStore
|
||||
from deerflow.persistence.task_deeplink_details.model import TaskDeeplinkDetailRow
|
||||
|
||||
|
||||
def _to_dict(row: TaskDeeplinkDetailRow) -> dict[str, Any]:
|
||||
return {
|
||||
"detail_type": row.detail_type,
|
||||
"detail_id": row.detail_id,
|
||||
"sub_type": row.sub_type or "",
|
||||
"task_name": row.task_name,
|
||||
"task_content": row.task_content,
|
||||
"overview": row.overview,
|
||||
"extra": dict(row.extra_json or {}),
|
||||
"version": row.version,
|
||||
"created_by": row.created_by,
|
||||
"updated_by": row.updated_by,
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
}
|
||||
|
||||
|
||||
class TaskDeeplinkDetailRepository(TaskDeeplinkDetailStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def get_detail(
|
||||
self,
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(
|
||||
TaskDeeplinkDetailRow,
|
||||
{"detail_type": detail_type, "detail_id": detail_id, "sub_type": sub_type},
|
||||
)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def upsert_detail(
|
||||
self,
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
task_name: str,
|
||||
task_content: str,
|
||||
overview: str,
|
||||
extra: dict[str, Any],
|
||||
updated_by: str | None,
|
||||
) -> tuple[dict[str, Any], bool, bool]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(
|
||||
TaskDeeplinkDetailRow,
|
||||
{"detail_type": detail_type, "detail_id": detail_id, "sub_type": sub_type},
|
||||
)
|
||||
created = row is None
|
||||
now = datetime.now(UTC)
|
||||
if row is None:
|
||||
row = TaskDeeplinkDetailRow(
|
||||
detail_type=detail_type,
|
||||
detail_id=detail_id,
|
||||
sub_type=sub_type,
|
||||
task_name=task_name,
|
||||
task_content=task_content,
|
||||
overview=overview,
|
||||
extra_json=extra,
|
||||
version=1,
|
||||
created_by=updated_by,
|
||||
updated_by=updated_by,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
changed = True
|
||||
elif (
|
||||
row.task_name == task_name
|
||||
and row.task_content == task_content
|
||||
and row.overview == overview
|
||||
and dict(row.extra_json or {}) == extra
|
||||
):
|
||||
return _to_dict(row), False, False
|
||||
else:
|
||||
row.task_name = task_name
|
||||
row.task_content = task_content
|
||||
row.overview = overview
|
||||
row.extra_json = extra
|
||||
row.version += 1
|
||||
row.updated_by = updated_by
|
||||
row.updated_at = now
|
||||
changed = True
|
||||
await session.commit()
|
||||
return _to_dict(row), created, changed
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""Dedicated taskId + goPath conversation-history index."""
|
||||
"""Dedicated taskId + goPath conversation-history cards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
+32
-6
@@ -1,15 +1,22 @@
|
||||
"""Shared helpers for task deep-link conversation indexing.
|
||||
"""Shared helpers for task deep-link conversation history.
|
||||
|
||||
The LangGraph thread/checkpoint remains the source of truth for messages and
|
||||
execution state. This package only owns a small, dedicated lookup index so the
|
||||
task workspaces never have to search/filter the global ``threads_meta`` table.
|
||||
execution state. This package owns a dedicated history card so the task
|
||||
workspaces never have to search, filter, or join the global ``threads_meta``
|
||||
catalogue when listing a task's conversations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, TypedDict
|
||||
|
||||
TASK_DEEPLINK_GO_PATHS = frozenset({"3qfx", "rwfx", "xdfx"})
|
||||
TASK_DEEPLINK_GO_PATHS = frozenset({"3qfx", "rwfx", "xdfx", "b5-1"})
|
||||
# Custom business mappings use their administrator-defined name as goPath.
|
||||
# Keep this in lock-step with ``business_mapping.business_code``.
|
||||
MAX_TASK_DEEPLINK_GO_PATH = 32
|
||||
|
||||
# Matches the ``task_deeplink_threads.sub_type`` column width.
|
||||
MAX_TASK_DEEPLINK_SUB_TYPE = 64
|
||||
|
||||
|
||||
class TaskDeeplinkBinding(TypedDict):
|
||||
@@ -17,6 +24,21 @@ class TaskDeeplinkBinding(TypedDict):
|
||||
source_task_id: str
|
||||
go_path: str
|
||||
agent_id: str | None
|
||||
# b5-1 integrator-supplied ?type= partition; "" when the link had none.
|
||||
sub_type: str
|
||||
|
||||
|
||||
def is_valid_task_deeplink_go_path(value: str) -> bool:
|
||||
"""Accept built-ins and validated custom mapping names.
|
||||
|
||||
The harness cannot import the app-level business-mapping store (the import
|
||||
firewall deliberately forbids that), so it validates the safe wire format
|
||||
here. Creation itself remains admin-only in the Gateway router.
|
||||
"""
|
||||
go_path = value.strip().lower()
|
||||
if go_path in TASK_DEEPLINK_GO_PATHS:
|
||||
return True
|
||||
return bool(go_path) and len(go_path) <= MAX_TASK_DEEPLINK_GO_PATH and not go_path.endswith("-a")
|
||||
|
||||
|
||||
def task_deeplink_binding(metadata: dict[str, Any] | None) -> TaskDeeplinkBinding | None:
|
||||
@@ -26,21 +48,25 @@ def task_deeplink_binding(metadata: dict[str, Any] | None) -> TaskDeeplinkBindin
|
||||
actual task id for 3qfx/rwfx and the action id for xdfx. Older callers may
|
||||
omit it, in which case ``taskId`` is a safe fallback. A ``taskGoPath`` is
|
||||
mandatory, deliberately preventing legacy task rows from being migrated or
|
||||
accidentally mixed into the new store.
|
||||
accidentally mixed into the new store. ``taskType`` (the b5-1 ``?type=``
|
||||
business type) further partitions one task's history so integrator-supplied
|
||||
types never mix; it is capped to the column width here.
|
||||
"""
|
||||
|
||||
md = metadata or {}
|
||||
go_path = str(md.get("taskGoPath") or "").strip().lower()
|
||||
if go_path not in TASK_DEEPLINK_GO_PATHS:
|
||||
if not is_valid_task_deeplink_go_path(go_path):
|
||||
return None
|
||||
source_task_id = str(md.get("taskId") or "").strip()
|
||||
task_id = str(md.get("taskRecordId") or source_task_id).strip()
|
||||
if not task_id or not source_task_id:
|
||||
return None
|
||||
agent_id = str(md.get("agent_id") or "").strip() or None
|
||||
sub_type = str(md.get("taskType") or "").strip()[:MAX_TASK_DEEPLINK_SUB_TYPE]
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"source_task_id": source_task_id,
|
||||
"go_path": go_path,
|
||||
"agent_id": agent_id,
|
||||
"sub_type": sub_type,
|
||||
}
|
||||
+32
-22
@@ -1,4 +1,4 @@
|
||||
"""In-memory task deep-link index backed by LangGraph BaseStore."""
|
||||
"""In-memory task deep-link history cards backed by LangGraph BaseStore."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -16,6 +16,19 @@ if TYPE_CHECKING:
|
||||
TASK_DEEPLINK_THREADS_NS: tuple[str, ...] = ("task_deeplink_threads",)
|
||||
|
||||
|
||||
def _card_record(thread: dict[str, Any], binding: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"thread_id": str(thread.get("thread_id") or ""),
|
||||
**binding,
|
||||
"display_name": thread.get("display_name"),
|
||||
"status": thread.get("status") or "idle",
|
||||
"metadata": thread.get("metadata") or {},
|
||||
"created_by": thread.get("user_id"),
|
||||
"created_at": thread.get("created_at") or now_iso(),
|
||||
"updated_at": thread.get("updated_at") or now_iso(),
|
||||
}
|
||||
|
||||
|
||||
async def sync_task_deeplink_index(store: BaseStore, thread: dict[str, Any]) -> None:
|
||||
metadata = thread.get("metadata") or {}
|
||||
binding = task_deeplink_binding(metadata)
|
||||
@@ -26,17 +39,7 @@ async def sync_task_deeplink_index(store: BaseStore, thread: dict[str, Any]) ->
|
||||
if "taskGoPath" in metadata or "taskRecordId" in metadata:
|
||||
await store.adelete(TASK_DEEPLINK_THREADS_NS, thread_id)
|
||||
return
|
||||
await store.aput(
|
||||
TASK_DEEPLINK_THREADS_NS,
|
||||
thread_id,
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
**binding,
|
||||
"created_by": thread.get("user_id"),
|
||||
"created_at": thread.get("created_at") or now_iso(),
|
||||
"updated_at": thread.get("updated_at") or now_iso(),
|
||||
},
|
||||
)
|
||||
await store.aput(TASK_DEEPLINK_THREADS_NS, thread_id, _card_record(thread, binding))
|
||||
|
||||
|
||||
async def delete_task_deeplink_index(store: BaseStore, thread_id: str) -> None:
|
||||
@@ -46,6 +49,8 @@ async def delete_task_deeplink_index(store: BaseStore, thread_id: str) -> None:
|
||||
class MemoryTaskDeeplinkThreadStore:
|
||||
def __init__(self, store: BaseStore, thread_store: ThreadMetaStore) -> None:
|
||||
self._store = store
|
||||
# Kept for constructor compatibility with the factory. Listing no
|
||||
# longer reads the generic thread catalogue.
|
||||
self._thread_store = thread_store
|
||||
|
||||
async def list_threads(
|
||||
@@ -54,28 +59,33 @@ class MemoryTaskDeeplinkThreadStore:
|
||||
go_path: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
sub_type: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
# ``sub_type`` semantics mirror the SQL repository: provided value (even
|
||||
# "") narrows to that b5-1 ?type= partition; None returns all sub-types.
|
||||
conditions = {"task_id": task_id, "go_path": go_path}
|
||||
if sub_type is not None:
|
||||
conditions["sub_type"] = sub_type
|
||||
items = await self._store.asearch(
|
||||
TASK_DEEPLINK_THREADS_NS,
|
||||
filter={"task_id": task_id, "go_path": go_path},
|
||||
filter=conditions,
|
||||
limit=limit,
|
||||
)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
thread_id = str(item.value.get("thread_id") or "")
|
||||
value = item.value or {}
|
||||
thread_id = str(value.get("thread_id") or "")
|
||||
if not thread_id:
|
||||
continue
|
||||
thread = await self._thread_store.get(thread_id, user_id=None)
|
||||
if thread is None:
|
||||
continue
|
||||
display_name = value.get("display_name")
|
||||
rows.append(
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"status": thread.get("status", "idle"),
|
||||
"created_at": thread.get("created_at", ""),
|
||||
"updated_at": thread.get("updated_at", ""),
|
||||
"metadata": thread.get("metadata") or {},
|
||||
"values": ({"title": thread.get("display_name")} if thread.get("display_name") else {}),
|
||||
"status": value.get("status") or "idle",
|
||||
"created_at": value.get("created_at", ""),
|
||||
"updated_at": value.get("updated_at", ""),
|
||||
"metadata": value.get("metadata") or {},
|
||||
"values": ({"title": display_name} if display_name else {}),
|
||||
"interrupts": {},
|
||||
}
|
||||
)
|
||||
|
||||
+24
-7
@@ -1,4 +1,4 @@
|
||||
"""Dedicated lookup rows for single-agent task deep-link conversations."""
|
||||
"""Dedicated history cards for single-agent task deep-link conversations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -8,15 +8,15 @@ from sqlalchemy import Index, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class TaskDeeplinkThreadRow(Base):
|
||||
"""One task-workspace history entry pointing at one LangGraph thread.
|
||||
"""One task-workspace history card pointing at one LangGraph thread.
|
||||
|
||||
The row intentionally stores no messages or title. Those remain in the
|
||||
canonical checkpoint / ``threads_meta`` record and are joined by primary
|
||||
key when listing a task's history.
|
||||
List queries read only this table. Title / status / metadata are copied
|
||||
here so the huge generic ``threads_meta`` catalogue is never joined.
|
||||
Messages stay in the LangGraph checkpoint.
|
||||
"""
|
||||
|
||||
__tablename__ = "task_deeplink_threads"
|
||||
@@ -25,6 +25,17 @@ class TaskDeeplinkThreadRow(Base):
|
||||
"ix_task_deeplink_threads_scope",
|
||||
"task_id",
|
||||
"go_path",
|
||||
"sub_type",
|
||||
"updated_at",
|
||||
"thread_id",
|
||||
),
|
||||
# 3qfx/rwfx/xdfx omit sub_type. The scope index is ordered by
|
||||
# sub_type before updated_at, so those listings cannot ride it for
|
||||
# "newest first" without a filesort. This prefix matches that query.
|
||||
Index(
|
||||
"ix_task_deeplink_threads_task_go_updated",
|
||||
"task_id",
|
||||
"go_path",
|
||||
"updated_at",
|
||||
"thread_id",
|
||||
),
|
||||
@@ -35,8 +46,14 @@ class TaskDeeplinkThreadRow(Base):
|
||||
task_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
# Resolved real task id used by model/run context (equals task_id except xdfx).
|
||||
source_task_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
go_path: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
go_path: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# b5-1 ?type= business type; "" for links without one (and for the other
|
||||
# go paths, which have no type dimension).
|
||||
sub_type: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
display_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(20), nullable=False, default="idle")
|
||||
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
|
||||
# Audit only. Task deep-link histories remain shared across authenticated users.
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
+67
-44
@@ -1,11 +1,11 @@
|
||||
"""SQL implementation of the dedicated task deep-link conversation index."""
|
||||
"""SQL implementation of the dedicated task deep-link conversation cards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.task_deeplink_threads.base import task_deeplink_binding
|
||||
@@ -15,8 +15,28 @@ if TYPE_CHECKING:
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
|
||||
|
||||
def _card_payload(thread: TaskDeeplinkThreadRow) -> dict[str, Any]:
|
||||
return {
|
||||
"thread_id": thread.thread_id,
|
||||
"status": thread.status or "idle",
|
||||
"created_at": thread.created_at,
|
||||
"updated_at": thread.updated_at,
|
||||
"metadata": thread.metadata_json or {},
|
||||
"values": ({"title": thread.display_name} if thread.display_name else {}),
|
||||
"interrupts": {},
|
||||
}
|
||||
|
||||
|
||||
def _apply_card_fields(row: TaskDeeplinkThreadRow, thread: ThreadMetaRow, now: datetime) -> None:
|
||||
row.display_name = thread.display_name
|
||||
row.status = thread.status or "idle"
|
||||
row.metadata_json = dict(thread.metadata_json or {})
|
||||
row.created_by = thread.user_id
|
||||
row.updated_at = now
|
||||
|
||||
|
||||
async def sync_task_deeplink_index(session: AsyncSession, thread: ThreadMetaRow) -> None:
|
||||
"""Synchronize one index row inside the caller's thread-meta transaction."""
|
||||
"""Synchronize one history card inside the caller's thread-meta transaction."""
|
||||
|
||||
metadata = thread.metadata_json or {}
|
||||
binding = task_deeplink_binding(metadata)
|
||||
@@ -30,25 +50,39 @@ async def sync_task_deeplink_index(session: AsyncSession, thread: ThreadMetaRow)
|
||||
existing = await session.get(TaskDeeplinkThreadRow, thread.thread_id)
|
||||
now = thread.updated_at if isinstance(thread.updated_at, datetime) else datetime.now(UTC)
|
||||
if existing is None:
|
||||
session.add(
|
||||
TaskDeeplinkThreadRow(
|
||||
thread_id=thread.thread_id,
|
||||
task_id=binding["task_id"],
|
||||
source_task_id=binding["source_task_id"],
|
||||
go_path=binding["go_path"],
|
||||
agent_id=binding["agent_id"],
|
||||
created_by=thread.user_id,
|
||||
created_at=(thread.created_at if isinstance(thread.created_at, datetime) else now),
|
||||
updated_at=now,
|
||||
)
|
||||
row = TaskDeeplinkThreadRow(
|
||||
thread_id=thread.thread_id,
|
||||
task_id=binding["task_id"],
|
||||
source_task_id=binding["source_task_id"],
|
||||
go_path=binding["go_path"],
|
||||
sub_type=binding["sub_type"],
|
||||
agent_id=binding["agent_id"],
|
||||
created_by=thread.user_id,
|
||||
created_at=(thread.created_at if isinstance(thread.created_at, datetime) else now),
|
||||
updated_at=now,
|
||||
)
|
||||
_apply_card_fields(row, thread, now)
|
||||
session.add(row)
|
||||
return
|
||||
|
||||
existing.task_id = binding["task_id"]
|
||||
existing.source_task_id = binding["source_task_id"]
|
||||
existing.go_path = binding["go_path"]
|
||||
existing.sub_type = binding["sub_type"]
|
||||
existing.agent_id = binding["agent_id"]
|
||||
existing.updated_at = now
|
||||
_apply_card_fields(existing, thread, now)
|
||||
|
||||
|
||||
async def update_task_deeplink_card(session: AsyncSession, thread_id: str, **values: Any) -> None:
|
||||
"""Patch a history card without reading ``threads_meta``.
|
||||
|
||||
Title / status / ``updated_at`` changes on ordinary chats no-op here
|
||||
because those ``thread_id`` values are absent from this table.
|
||||
"""
|
||||
|
||||
if not values:
|
||||
return
|
||||
await session.execute(update(TaskDeeplinkThreadRow).where(TaskDeeplinkThreadRow.thread_id == thread_id).values(**values))
|
||||
|
||||
|
||||
async def delete_task_deeplink_index(session: AsyncSession, thread_id: str) -> None:
|
||||
@@ -65,41 +99,30 @@ class TaskDeeplinkThreadRepository:
|
||||
go_path: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
sub_type: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List one task/goPath partition without scanning thread metadata JSON."""
|
||||
"""List one task/goPath partition from the dedicated card table only.
|
||||
|
||||
# Local import avoids a package-initialization cycle: thread_meta.sql
|
||||
# imports the sync helpers above for atomic dual writes.
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
``sub_type`` narrows the partition to one b5-1 ``?type=`` value when
|
||||
provided (an explicit "" selects the no-type partition); ``None`` keeps
|
||||
the untyped legacy behaviour of returning every sub-type at once.
|
||||
"""
|
||||
|
||||
conditions = [
|
||||
TaskDeeplinkThreadRow.task_id == task_id,
|
||||
TaskDeeplinkThreadRow.go_path == go_path,
|
||||
]
|
||||
if sub_type is not None:
|
||||
conditions.append(TaskDeeplinkThreadRow.sub_type == sub_type)
|
||||
stmt = (
|
||||
select(TaskDeeplinkThreadRow, ThreadMetaRow)
|
||||
.join(
|
||||
ThreadMetaRow,
|
||||
ThreadMetaRow.thread_id == TaskDeeplinkThreadRow.thread_id,
|
||||
)
|
||||
.where(
|
||||
TaskDeeplinkThreadRow.task_id == task_id,
|
||||
TaskDeeplinkThreadRow.go_path == go_path,
|
||||
)
|
||||
select(TaskDeeplinkThreadRow)
|
||||
.where(*conditions)
|
||||
.order_by(
|
||||
ThreadMetaRow.updated_at.desc(),
|
||||
ThreadMetaRow.thread_id.desc(),
|
||||
TaskDeeplinkThreadRow.updated_at.desc(),
|
||||
TaskDeeplinkThreadRow.thread_id.desc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
|
||||
return [
|
||||
{
|
||||
"thread_id": thread.thread_id,
|
||||
"status": thread.status,
|
||||
"created_at": thread.created_at,
|
||||
"updated_at": thread.updated_at,
|
||||
"metadata": thread.metadata_json or {},
|
||||
"values": ({"title": thread.display_name} if thread.display_name else {}),
|
||||
"interrupts": {},
|
||||
}
|
||||
for _index, thread in rows
|
||||
]
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [_card_payload(row) for row in rows]
|
||||
+16
-3
@@ -54,9 +54,22 @@ def is_excluded_system_thread(metadata: dict[str, Any] | None) -> bool:
|
||||
carry ``task_analysis_seat``. Those rows stay in the execution record and
|
||||
out of recent chats.
|
||||
"""
|
||||
md = metadata or {}
|
||||
if md.get("system"):
|
||||
return True
|
||||
md = metadata or {}
|
||||
if md.get("system"):
|
||||
return True
|
||||
# Task-analysis keeps its user-facing history in task_analysis_runs /
|
||||
# task_analysis_run_messages. The underlying LangGraph threads are runtime
|
||||
# implementation details; recognise their task markers as a defensive
|
||||
# fallback in case a legacy caller omitted ``system`` or ``thread_type``.
|
||||
if any(
|
||||
isinstance(md.get(key), str) and md[key].strip()
|
||||
for key in (
|
||||
"task_analysis_task_id",
|
||||
"task_analysis_source_task_id",
|
||||
"task_analysis_run_id",
|
||||
)
|
||||
):
|
||||
return True
|
||||
seat = md.get("task_analysis_seat")
|
||||
if isinstance(seat, str) and seat.strip():
|
||||
return True
|
||||
|
||||
+8
-1
@@ -192,10 +192,14 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
item = await self._store.aget(THREADS_NS, thread_id)
|
||||
if item is None:
|
||||
return not require_existing
|
||||
record_user_id = item.value.get("user_id")
|
||||
if record_user_id:
|
||||
from deerflow.runtime.thread_paths import remember_thread_owner
|
||||
|
||||
remember_thread_owner(thread_id, str(record_user_id))
|
||||
# 任务深链 / 圆桌 / 任务分析系统线程:全局共享(与 SQL store 同口径)。
|
||||
if is_shared_access_thread(item.value.get("metadata") or {}):
|
||||
return True
|
||||
record_user_id = item.value.get("user_id")
|
||||
if record_user_id is None:
|
||||
return True
|
||||
return record_user_id == user_id
|
||||
@@ -207,6 +211,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
record["display_name"] = display_name
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
await sync_task_deeplink_index(self._store, record)
|
||||
|
||||
async def update_status(self, thread_id: str, status: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_status")
|
||||
@@ -215,6 +220,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
record["status"] = status
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
await sync_task_deeplink_index(self._store, record)
|
||||
|
||||
async def update_assistant_id(self, thread_id: str, assistant_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_assistant_id")
|
||||
@@ -223,6 +229,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
record["assistant_id"] = assistant_id
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
await sync_task_deeplink_index(self._store, record)
|
||||
|
||||
async def update_metadata(self, thread_id: str, metadata: dict, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_metadata")
|
||||
|
||||
+158
-167
@@ -5,72 +5,60 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.json_compat import JsonPredicate, supports_json_filter
|
||||
from deerflow.persistence.task_deeplink_threads.sql import (
|
||||
delete_task_deeplink_index,
|
||||
sync_task_deeplink_index,
|
||||
update_task_deeplink_card,
|
||||
)
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
|
||||
from deerflow.persistence.thread_meta.base import EXCLUDED_SYSTEM_THREAD_TYPES, ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
_INDEXED_METADATA_COLUMNS = {
|
||||
"taskId": ThreadMetaRow.task_id,
|
||||
"agent_id": ThreadMetaRow.agent_id,
|
||||
"actionId": ThreadMetaRow.action_id,
|
||||
}
|
||||
|
||||
_INDEXED_METADATA_ATTRIBUTES = {
|
||||
"taskId": "task_id",
|
||||
"agent_id": "agent_id",
|
||||
"actionId": "action_id",
|
||||
}
|
||||
|
||||
|
||||
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
|
||||
"""Return metadata predicates that have a maintained SQL projection.
|
||||
|
||||
Task deep-link callers use string ids. Restrict the SQL shortcut to
|
||||
strings so legacy callers that intentionally store another JSON type keep
|
||||
the exact Python-side equality semantics of the generic metadata filter.
|
||||
"""
|
||||
return {
|
||||
key: value
|
||||
for key, value in (metadata or {}).items()
|
||||
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
|
||||
}
|
||||
|
||||
|
||||
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
|
||||
"""Build ORM column values from authoritative thread metadata.
|
||||
|
||||
Always include every projection so a metadata update that changes a value
|
||||
from a string to another JSON type cannot leave a stale indexed value.
|
||||
"""
|
||||
values = _indexed_metadata_values(metadata)
|
||||
return {
|
||||
attribute: values.get(metadata_key)
|
||||
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
|
||||
}
|
||||
|
||||
|
||||
def _apply_indexed_metadata_filters(stmt: Any, metadata: dict[str, Any] | None) -> tuple[Any, set[str]]:
|
||||
"""Apply known metadata predicates in SQL and return their keys.
|
||||
|
||||
The caller still verifies the full metadata object in Python whenever a
|
||||
generic filter or system-thread exclusion is involved. This keeps the
|
||||
public API's exact-match contract while reducing that verification set to
|
||||
rows for the requested task/agent/action instead of the entire table.
|
||||
"""
|
||||
values = _indexed_metadata_values(metadata)
|
||||
for key, value in values.items():
|
||||
stmt = stmt.where(_INDEXED_METADATA_COLUMNS[key] == value)
|
||||
return stmt, set(values)
|
||||
|
||||
|
||||
class ThreadMetaRepository(ThreadMetaStore):
|
||||
_INDEXED_METADATA_COLUMNS = {
|
||||
"taskId": ThreadMetaRow.task_id,
|
||||
"agent_id": ThreadMetaRow.agent_id,
|
||||
"actionId": ThreadMetaRow.action_id,
|
||||
}
|
||||
|
||||
_INDEXED_METADATA_ATTRIBUTES = {
|
||||
"taskId": "task_id",
|
||||
"agent_id": "agent_id",
|
||||
"actionId": "action_id",
|
||||
}
|
||||
|
||||
|
||||
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
|
||||
"""Return metadata predicates that have a maintained SQL projection.
|
||||
|
||||
Task deep-link callers use string ids. Restrict the SQL shortcut to
|
||||
strings so legacy callers that intentionally store another JSON type keep
|
||||
the exact Python-side equality semantics of the generic metadata filter.
|
||||
"""
|
||||
return {
|
||||
key: value
|
||||
for key, value in (metadata or {}).items()
|
||||
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
|
||||
}
|
||||
|
||||
|
||||
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
|
||||
"""Build ORM column values from authoritative thread metadata.
|
||||
|
||||
Always include every projection so a metadata update that changes a value
|
||||
from a string to another JSON type cannot leave a stale indexed value.
|
||||
"""
|
||||
values = _indexed_metadata_values(metadata)
|
||||
return {
|
||||
attribute: values.get(metadata_key)
|
||||
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
|
||||
}
|
||||
|
||||
|
||||
class ThreadMetaRepository(ThreadMetaStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@@ -97,14 +85,14 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
# creates an orphan row (used by migration scripts).
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.create")
|
||||
now = datetime.now(UTC)
|
||||
row = ThreadMetaRow(
|
||||
thread_id=thread_id,
|
||||
assistant_id=assistant_id,
|
||||
user_id=resolved_user_id,
|
||||
**_indexed_thread_metadata(metadata),
|
||||
display_name=display_name,
|
||||
status="idle",
|
||||
metadata_json=metadata or {},
|
||||
row = ThreadMetaRow(
|
||||
thread_id=thread_id,
|
||||
assistant_id=assistant_id,
|
||||
user_id=resolved_user_id,
|
||||
**_indexed_thread_metadata(metadata),
|
||||
display_name=display_name,
|
||||
status="idle",
|
||||
metadata_json=metadata or {},
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -164,6 +152,10 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
row = await session.get(ThreadMetaRow, thread_id)
|
||||
if row is None:
|
||||
return not require_existing
|
||||
if row.user_id:
|
||||
from deerflow.runtime.thread_paths import remember_thread_owner
|
||||
|
||||
remember_thread_owner(thread_id, row.user_id)
|
||||
md = row.metadata_json or {}
|
||||
# 全局共享、不区分用户(任何登录用户都可读写/续聊):
|
||||
# ① 任务深链对话(metadata.taskId);
|
||||
@@ -176,6 +168,43 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
return True
|
||||
return row.user_id == user_id
|
||||
|
||||
@staticmethod
|
||||
def _query_conditions(*, dialect, metadata, status, user_id, exclude_system, query):
|
||||
"""Share exact search/count predicates without changing owner resolution.
|
||||
|
||||
Complex metadata and unsupported dialects retain their exact Python
|
||||
filter. No rejected key/value is silently ignored. Server numeric JSON
|
||||
comparisons retain that path: DECIMAL and DOUBLE cannot reproduce all
|
||||
Python float/int comparisons (including large numeric task ids).
|
||||
"""
|
||||
conditions = []
|
||||
if user_id is not None:
|
||||
conditions.append(ThreadMetaRow.user_id == user_id)
|
||||
if status:
|
||||
conditions.append(ThreadMetaRow.status == status)
|
||||
if query and query.strip():
|
||||
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
# Retain the master branch's indexed task/agent/action narrowing.
|
||||
# JSON predicates below still enforce exact case/type semantics.
|
||||
for key, value in _indexed_metadata_values(metadata).items():
|
||||
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
|
||||
supported = dialect in {"sqlite", "postgresql", "mysql"}
|
||||
residual = {}
|
||||
for key, value in (metadata or {}).items():
|
||||
numeric_server = dialect in {"mysql", "postgresql"} and isinstance(value, (bool, int, float))
|
||||
if supported and supports_json_filter(key, value) and not numeric_server:
|
||||
conditions.append(JsonPredicate(ThreadMetaRow.metadata_json, key, value=value))
|
||||
else:
|
||||
residual[key] = value
|
||||
if exclude_system and supported:
|
||||
column = ThreadMetaRow.metadata_json
|
||||
conditions.append(~or_(
|
||||
JsonPredicate(column, "system", "truthy"),
|
||||
JsonPredicate(column, "task_analysis_seat", "nonblank"),
|
||||
*(JsonPredicate(column, "thread_type", value=value) for value in sorted(EXCLUDED_SYSTEM_THREAD_TYPES)),
|
||||
))
|
||||
return conditions, residual, exclude_system and not supported
|
||||
|
||||
async def search(
|
||||
self,
|
||||
*,
|
||||
@@ -187,71 +216,46 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
exclude_system: bool = False,
|
||||
query: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Search threads with optional metadata and status filters.
|
||||
|
||||
Owner filter is enforced by default: caller must be in a user
|
||||
context. Pass ``user_id=None`` to bypass (migration/CLI).
|
||||
"""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.search")
|
||||
# 任务深链对话(按 metadata.taskId 过滤)全局共享、不区分用户:跳过 owner 过滤。
|
||||
# Preserve task-scoped sharing, still applying EVERY metadata filter
|
||||
# (taskId + agent_id + actionId) before the pagination window.
|
||||
if metadata and metadata.get("taskId"):
|
||||
resolved_user_id = None
|
||||
# thread_id tie-breaker keeps OFFSET pagination stable when many rows
|
||||
# share the same updated_at (bulk imports, scheduler bursts).
|
||||
stmt = select(ThreadMetaRow).order_by(ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc())
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
|
||||
if status:
|
||||
stmt = stmt.where(ThreadMetaRow.status == status)
|
||||
if query and query.strip():
|
||||
# Case-insensitive title search. display_name is a real column, so
|
||||
# this filter runs in SQL even when the JSON filters below fall
|
||||
# back to Python-side scanning.
|
||||
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
|
||||
# ``metadata_json`` is deliberately portable, so arbitrary filters
|
||||
# still receive Python-side equality checks below. These three task
|
||||
# identifiers are maintained projections and can safely narrow the
|
||||
# SQL result first. A deep-link query that used to scan every thread
|
||||
# now reads only that task's rows.
|
||||
stmt, indexed_metadata_keys = _apply_indexed_metadata_filters(stmt, metadata)
|
||||
remaining_metadata_keys = set(metadata or {}) - indexed_metadata_keys
|
||||
|
||||
if not remaining_metadata_keys and not exclude_system:
|
||||
stmt = stmt.limit(limit).offset(offset)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
# Metadata filters live inside the JSON blob, so rows are scanned in
|
||||
# batches and filtered in Python until the requested window is full.
|
||||
# A fixed over-fetch window would under-fill pages when most rows are
|
||||
# filtered out (e.g. heavy scheduled-task usage), making callers that
|
||||
# paginate believe the list is exhausted. TODO(Phase 2): use JSON DB
|
||||
# operators (Postgres @>, SQLite json_extract) for server-side filtering.
|
||||
def _matches(row: dict) -> bool:
|
||||
md = row.get("metadata") or {}
|
||||
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
|
||||
return False
|
||||
if exclude_system and is_excluded_system_thread(md):
|
||||
return False
|
||||
return True
|
||||
|
||||
needed = offset + limit
|
||||
batch_size = max(500, needed)
|
||||
matched: list[dict] = []
|
||||
scan_offset = 0
|
||||
async with self._sf() as session:
|
||||
conditions, residual, filter_system = self._query_conditions(
|
||||
dialect=session.get_bind().dialect.name,
|
||||
metadata=metadata, status=status, user_id=resolved_user_id,
|
||||
exclude_system=exclude_system, query=query,
|
||||
)
|
||||
stmt = select(ThreadMetaRow).where(*conditions).order_by(
|
||||
ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc(),
|
||||
)
|
||||
if not residual and not filter_system:
|
||||
result = await session.execute(stmt.limit(limit).offset(offset))
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
# Compatibility path for nested values, unusual literal keys,
|
||||
# numeric server filters, or the separate openGauss dialect.
|
||||
needed = offset + limit
|
||||
batch_size = max(500, needed)
|
||||
matched = []
|
||||
scan_offset = 0
|
||||
while len(matched) < needed:
|
||||
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
|
||||
rows = result.scalars().all()
|
||||
if not rows:
|
||||
break
|
||||
matched.extend(d for d in (self._row_to_dict(r) for r in rows) if _matches(d))
|
||||
for row in rows:
|
||||
record = self._row_to_dict(row)
|
||||
md = record.get("metadata") or {}
|
||||
if all(md.get(key) == value for key, value in residual.items()) and not (
|
||||
filter_system and is_excluded_system_thread(md)
|
||||
):
|
||||
matched.append(record)
|
||||
if len(rows) < batch_size:
|
||||
break
|
||||
scan_offset += len(rows)
|
||||
return matched[offset:needed]
|
||||
return matched[offset:needed]
|
||||
|
||||
async def count(
|
||||
self,
|
||||
@@ -262,56 +266,37 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
exclude_system: bool = False,
|
||||
query: str | None = None,
|
||||
) -> int:
|
||||
"""Count threads matching the same filters as :meth:`search`."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.count")
|
||||
# 任务深链对话(按 metadata.taskId 过滤)全局共享、不区分用户:跳过 owner 过滤。
|
||||
if metadata and metadata.get("taskId"):
|
||||
resolved_user_id = None
|
||||
conditions = []
|
||||
if resolved_user_id is not None:
|
||||
conditions.append(ThreadMetaRow.user_id == resolved_user_id)
|
||||
if status:
|
||||
conditions.append(ThreadMetaRow.status == status)
|
||||
if query and query.strip():
|
||||
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
|
||||
indexed_metadata = _indexed_metadata_values(metadata)
|
||||
for key, value in indexed_metadata.items():
|
||||
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
|
||||
remaining_metadata_keys = set(metadata or {}) - set(indexed_metadata)
|
||||
|
||||
if not remaining_metadata_keys and not exclude_system:
|
||||
stmt = select(func.count()).select_from(ThreadMetaRow)
|
||||
for condition in conditions:
|
||||
stmt = stmt.where(condition)
|
||||
async with self._sf() as session:
|
||||
async with self._sf() as session:
|
||||
conditions, residual, filter_system = self._query_conditions(
|
||||
dialect=session.get_bind().dialect.name,
|
||||
metadata=metadata, status=status, user_id=resolved_user_id,
|
||||
exclude_system=exclude_system, query=query,
|
||||
)
|
||||
if not residual and not filter_system:
|
||||
stmt = select(func.count()).select_from(ThreadMetaRow).where(*conditions)
|
||||
return int((await session.execute(stmt)).scalar_one())
|
||||
|
||||
# JSON filters can't run in portable SQL — scan just the metadata
|
||||
# column in batches and count in Python (mirrors search()).
|
||||
stmt = select(ThreadMetaRow.metadata_json).order_by(ThreadMetaRow.thread_id)
|
||||
for condition in conditions:
|
||||
stmt = stmt.where(condition)
|
||||
total = 0
|
||||
batch_size = 1000
|
||||
scan_offset = 0
|
||||
async with self._sf() as session:
|
||||
stmt = select(ThreadMetaRow.metadata_json).where(*conditions).order_by(ThreadMetaRow.thread_id)
|
||||
total = 0
|
||||
scan_offset = 0
|
||||
batch_size = 1000
|
||||
while True:
|
||||
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
|
||||
rows = result.scalars().all()
|
||||
rows = (await session.execute(stmt.limit(batch_size).offset(scan_offset))).scalars().all()
|
||||
if not rows:
|
||||
break
|
||||
for metadata_json in rows:
|
||||
md = metadata_json or {}
|
||||
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
|
||||
continue
|
||||
if exclude_system and is_excluded_system_thread(md):
|
||||
continue
|
||||
total += 1
|
||||
for md in rows:
|
||||
md = md or {}
|
||||
if all(md.get(key) == value for key, value in residual.items()) and not (
|
||||
filter_system and is_excluded_system_thread(md)
|
||||
):
|
||||
total += 1
|
||||
if len(rows) < batch_size:
|
||||
break
|
||||
scan_offset += len(rows)
|
||||
return total
|
||||
return total
|
||||
|
||||
async def _check_ownership(self, session: AsyncSession, thread_id: str, resolved_user_id: str | None) -> bool:
|
||||
"""Return True if the row exists and is owned (or filter bypassed)."""
|
||||
@@ -337,7 +322,9 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
async with self._sf() as session:
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id):
|
||||
return
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=datetime.now(UTC)))
|
||||
now = datetime.now(UTC)
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=now))
|
||||
await update_task_deeplink_card(session, thread_id, display_name=display_name, updated_at=now)
|
||||
await session.commit()
|
||||
|
||||
async def update_status(
|
||||
@@ -351,7 +338,9 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
async with self._sf() as session:
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id):
|
||||
return
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=datetime.now(UTC)))
|
||||
now = datetime.now(UTC)
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=now))
|
||||
await update_task_deeplink_card(session, thread_id, status=status, updated_at=now)
|
||||
await session.commit()
|
||||
|
||||
async def update_assistant_id(
|
||||
@@ -365,7 +354,9 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
async with self._sf() as session:
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id):
|
||||
return
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=datetime.now(UTC)))
|
||||
now = datetime.now(UTC)
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=now))
|
||||
await update_task_deeplink_card(session, thread_id, updated_at=now)
|
||||
await session.commit()
|
||||
|
||||
async def update_metadata(
|
||||
@@ -390,14 +381,14 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
task_shared = bool((row.metadata_json or {}).get("taskId"))
|
||||
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
|
||||
return
|
||||
merged = dict(row.metadata_json or {})
|
||||
merged.update(metadata)
|
||||
row.metadata_json = merged
|
||||
for key, value in _indexed_thread_metadata(merged).items():
|
||||
setattr(row, key, value)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
merged = dict(row.metadata_json or {})
|
||||
merged.update(metadata)
|
||||
row.metadata_json = merged
|
||||
for key, value in _indexed_thread_metadata(merged).items():
|
||||
setattr(row, key, value)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await sync_task_deeplink_index(session, row)
|
||||
await session.commit()
|
||||
await session.commit()
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
|
||||
+8
@@ -82,3 +82,11 @@ class ToolMetricsStore(ABC):
|
||||
"which skills were invoked, and how often did each succeed/fail".
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
|
||||
"""Delete one bounded batch of audit rows older than ``cutoff``.
|
||||
|
||||
The default keeps alternate/in-memory implementations compatible; the
|
||||
SQL repository supplies the real retention operation.
|
||||
"""
|
||||
return 0
|
||||
+32
-1
@@ -8,7 +8,7 @@ from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy import delete, desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
|
||||
@@ -245,3 +245,34 @@ class SqlToolMetricsStore(ToolMetricsStore):
|
||||
{"skill_name": name, **counts}
|
||||
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
|
||||
]
|
||||
|
||||
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
|
||||
"""Delete one small, oldest-first retention batch.
|
||||
|
||||
Key selection followed by a primary-key delete works consistently on
|
||||
every supported SQL dialect and prevents a three-month cleanup from
|
||||
holding a large audit-table write lock.
|
||||
"""
|
||||
# Keep below SQLite's traditional 999 bind-variable limit. The
|
||||
# retention worker requests batches repeatedly, so portability is more
|
||||
# valuable than one giant delete statement.
|
||||
size = max(1, min(int(batch_size), 500))
|
||||
normalized_cutoff = _normalize_created_at(cutoff)
|
||||
async with self._sf() as session:
|
||||
ids = list(
|
||||
(
|
||||
await session.scalars(
|
||||
select(ToolCallMetricRow.id)
|
||||
.where(ToolCallMetricRow.created_at < normalized_cutoff)
|
||||
.order_by(ToolCallMetricRow.created_at.asc(), ToolCallMetricRow.id.asc())
|
||||
.limit(size)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not ids:
|
||||
return 0
|
||||
result = await session.execute(
|
||||
delete(ToolCallMetricRow).where(ToolCallMetricRow.id.in_(ids))
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
+87
-74
@@ -1,23 +1,23 @@
|
||||
"""ORM model for the users table.
|
||||
|
||||
Lives in the harness persistence package so it is picked up by
|
||||
``Base.metadata.create_all()`` alongside ``threads_meta``, ``runs``,
|
||||
``run_events``, and ``feedback``. Using the shared engine means:
|
||||
|
||||
- One SQLite/Postgres database, one connection pool
|
||||
- One schema initialisation codepath
|
||||
- Consistent async sessions across auth and persistence reads
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Index, String, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
"""ORM model for the users table.
|
||||
|
||||
Lives in the harness persistence package so it is picked up by
|
||||
``Base.metadata.create_all()`` alongside ``threads_meta``, ``runs``,
|
||||
``run_events``, and ``feedback``. Using the shared engine means:
|
||||
|
||||
- One SQLite/Postgres database, one connection pool
|
||||
- One schema initialisation codepath
|
||||
- Consistent async sessions across auth and persistence reads
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Index, String, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class AuthInitializationRow(Base):
|
||||
@@ -32,60 +32,73 @@ class AuthUsernameRow(Base):
|
||||
|
||||
__tablename__ = "auth_usernames"
|
||||
username: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
|
||||
|
||||
class UserRow(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
# UUIDs are stored as 36-char strings for cross-backend portability.
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
|
||||
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
|
||||
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
|
||||
# when new roles are introduced.
|
||||
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
# OAuth linkage (optional). A partial unique index enforces one
|
||||
# account per (provider, oauth_id) pair, leaving NULL/NULL rows
|
||||
# unconstrained so plain password accounts can coexist.
|
||||
oauth_provider: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
oauth_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# Auth lifecycle flags
|
||||
needs_setup: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
token_version: Mapped[int] = mapped_column(nullable=False, default=0)
|
||||
|
||||
# 登录白名单(放行)标记 —— **黑名单模式**。仅当系统设置 ``login_whitelist.enabled``
|
||||
# 打开时生效:关 → 忽略此列,人人可登录;开 → 仅 approved=False(被管理员显式
|
||||
# 「取消放行」)的用户被拦,其余(含从未登录过的新账号)放行。新账号默认 True
|
||||
# (首次登录即放行);存量用户由迁移回填 True。server_default=1 让迁移 ADD COLUMN
|
||||
# 时把已存在行置为已放行。
|
||||
approved: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
|
||||
|
||||
|
||||
|
||||
class AuthServiceNonceRow(Base):
|
||||
"""One-time service exchange nonce shared by every gateway worker."""
|
||||
|
||||
__tablename__ = "auth_service_nonces"
|
||||
nonce_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
client_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
|
||||
|
||||
|
||||
class UserRow(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
# UUIDs are stored as 36-char strings for cross-backend portability.
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
|
||||
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
|
||||
# Canonical local part used by SSO / username login. Keeping it separately
|
||||
# avoids applying LOWER()/LIKE to every users.email row on each login.
|
||||
username_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
|
||||
# when new roles are introduced.
|
||||
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
# OAuth linkage (optional). A partial unique index enforces one
|
||||
# account per (provider, oauth_id) pair, leaving NULL/NULL rows
|
||||
# unconstrained so plain password accounts can coexist.
|
||||
oauth_provider: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
oauth_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# Auth lifecycle flags
|
||||
needs_setup: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
token_version: Mapped[int] = mapped_column(nullable=False, default=0)
|
||||
|
||||
# 登录白名单(放行)标记 —— **黑名单模式**。仅当系统设置 ``login_whitelist.enabled``
|
||||
# 打开时生效:关 → 忽略此列,人人可登录;开 → 仅 approved=False(被管理员显式
|
||||
# 「取消放行」)的用户被拦,其余(含从未登录过的新账号)放行。新账号默认 True
|
||||
# (首次登录即放行);存量用户由迁移回填 True。server_default=1 让迁移 ADD COLUMN
|
||||
# 时把已存在行置为已放行。
|
||||
approved: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
|
||||
|
||||
# Unified password/passphrase. New writes store the same versioned hash as
|
||||
# password_hash; historical plaintext remains accepted until the next reset.
|
||||
# Never serialize hashes into user-list or login responses.
|
||||
login_password: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
|
||||
# 岗位 (position) the user is assigned to (1:1). NULL = no explicit
|
||||
# position → the user inherits the ``is_default`` position's grants. Drives
|
||||
# the subscription sync engine (see ``deerflow.services.position_sync``).
|
||||
position_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"idx_users_oauth_identity",
|
||||
"oauth_provider",
|
||||
"oauth_id",
|
||||
unique=True,
|
||||
sqlite_where=text("oauth_provider IS NOT NULL AND oauth_id IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
login_password: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
|
||||
# 岗位 (position) the user is assigned to (1:1). NULL = no explicit
|
||||
# position → the user inherits the ``is_default`` position's grants. Drives
|
||||
# the subscription sync engine (see ``deerflow.services.position_sync``).
|
||||
position_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_users_username_key_created_at", "username_key", "created_at"),
|
||||
Index(
|
||||
"idx_users_oauth_identity",
|
||||
"oauth_provider",
|
||||
"oauth_id",
|
||||
unique=True,
|
||||
sqlite_where=text("oauth_provider IS NOT NULL AND oauth_id IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
Reference in new issue
Block a user