sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
+23
-2
@@ -77,6 +77,11 @@ class AgentStore(ABC):
|
||||
"""Return featured + still-visible-to-``user_id`` agents, ordered by ``featured_order`` asc."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_sidebar(self, user_id: str, *, limit: int = 3) -> list[dict[str, Any]]:
|
||||
"""Return only the id/name fields needed by the workspace sidebar."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
"""Return rows for ``agent_ids`` visible to ``user_id``, preserving the input order."""
|
||||
@@ -133,6 +138,7 @@ class AgentStore(ABC):
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
hidden: bool | None = None,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Return one page of agents plus the total matching count.
|
||||
|
||||
@@ -143,6 +149,9 @@ class AgentStore(ABC):
|
||||
- ``None`` — the default listing visibility (``list_visible``).
|
||||
``admin=True`` bypasses the visibility/scope filter entirely (every
|
||||
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
|
||||
Admin-hidden agents (``hidden=True`` rows) are excluded from every
|
||||
non-admin listing; under ``admin=True`` the ``hidden`` param selects
|
||||
only hidden / only visible rows (``None`` = both).
|
||||
|
||||
``search`` matches the raw name OR its desensitized display form.
|
||||
``tag_ids`` keeps agents carrying ANY of the tags (OR). ``owner`` keeps
|
||||
@@ -167,8 +176,20 @@ class AgentStore(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
|
||||
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``."""
|
||||
async def set_extras(
|
||||
self,
|
||||
agent_id: str,
|
||||
*,
|
||||
tool_groups: list[str] | None,
|
||||
model: str | None,
|
||||
chat_link: Any = None,
|
||||
update_chat_link: bool = False,
|
||||
) -> None:
|
||||
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``.
|
||||
|
||||
``chat_link`` is written only when ``update_chat_link`` is true, so a
|
||||
skills/model refresh does not clear a previously mirrored jump link.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
|
||||
+398
-325
@@ -1,325 +1,398 @@
|
||||
"""In-memory custom agent store for memory persistence mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.agent_name_desensitize import name_matches
|
||||
from deerflow.persistence.agents.base import AgentStore
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _copy(data: dict[str, Any]) -> dict[str, Any]:
|
||||
copied = dict(data)
|
||||
copied["builtin"] = copied.get("user_id") is None
|
||||
return copied
|
||||
|
||||
|
||||
class MemoryAgentStore(AgentStore):
|
||||
def __init__(self) -> None:
|
||||
self._agents: dict[str, dict[str, Any]] = {}
|
||||
self._favorites: set[tuple[str, str]] = set()
|
||||
self._pins: set[tuple[str, str]] = set()
|
||||
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
|
||||
self._skills: dict[str, list[str]] = {}
|
||||
self._extras: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
|
||||
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
|
||||
|
||||
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
|
||||
"""Single-record visibility (``get_visible``) — the union of every
|
||||
listing surface, so a list-visible agent always opens. Mirrors
|
||||
``AgentRepository._visible_filter``: baseline visibility OR
|
||||
private-square (password is the discovery gate, reads are not
|
||||
re-gated) OR favorited (the "我的" listing keeps favorites even after
|
||||
the owner unpublishes)."""
|
||||
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
|
||||
|
||||
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
|
||||
rows = [_copy(agent) for agent in self._agents.values() if self._is_visible(agent, user_id)]
|
||||
rows.sort(key=lambda agent: agent.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or not self._is_fetchable(agent, user_id):
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
return _copy(agent) if agent is not None else None
|
||||
|
||||
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or agent.get("user_id") != user_id:
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = _now()
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"name": data["name"],
|
||||
"description": data.get("description") or "",
|
||||
"user_id": data.get("user_id"),
|
||||
"published": bool(data.get("published", False)),
|
||||
"is_private": bool(data.get("is_private", False)),
|
||||
"square_id": str(data.get("square_id") or ""),
|
||||
"featured_order": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._agents[row["id"]] = row
|
||||
return _copy(row)
|
||||
|
||||
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = self._agents.get(data["id"])
|
||||
if row is None:
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"name": data["name"],
|
||||
"description": data.get("description") or "",
|
||||
"user_id": None,
|
||||
"published": bool(data.get("published", True)),
|
||||
"featured_order": None,
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
self._agents[row["id"]] = row
|
||||
elif row.get("user_id") is None:
|
||||
row["name"] = data["name"]
|
||||
row["description"] = data.get("description") or ""
|
||||
row["published"] = bool(data.get("published", True))
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete(self, agent_id: str, user_id: str) -> bool:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
async def delete_builtin(self, agent_id: str) -> bool:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
|
||||
|
||||
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
row["featured_order"] = order
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
candidates = [a for a in self._agents.values() if a.get("featured_order") is not None and self._is_visible(a, user_id)]
|
||||
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
|
||||
return [_copy(a) for a in candidates]
|
||||
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
|
||||
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
|
||||
rows.sort(key=lambda a: order_map.get(a["id"], 0))
|
||||
return [_copy(a) for a in rows]
|
||||
|
||||
async def list_private(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private"))]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values()]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete_any(self, agent_id: str) -> bool:
|
||||
if agent_id not in self._agents:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
# --- Paginated listing ---
|
||||
|
||||
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
|
||||
if admin:
|
||||
return True
|
||||
if scope == "square":
|
||||
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
|
||||
if scope == "mine":
|
||||
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
|
||||
favorited = (user_id, agent["id"]) in self._favorites
|
||||
return owned or favorited
|
||||
if scope == "private":
|
||||
return bool(agent.get("is_private"))
|
||||
if scope == "builtin":
|
||||
# 内置智能体 = user_id IS NULL(仅管理员;路由层 admin 闸门)。
|
||||
return agent.get("user_id") is None
|
||||
return self._is_visible(agent, user_id)
|
||||
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
|
||||
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
|
||||
owner_key = owner.strip() if owner and owner.strip() else None
|
||||
square_key = square_id.strip() if square_id and square_id.strip() else None
|
||||
|
||||
def _square_ok(a: dict[str, Any]) -> bool:
|
||||
if square_key is None:
|
||||
return True
|
||||
sid = str(a.get("square_id") or "")
|
||||
return sid == square_key or (square_is_default and sid == "")
|
||||
|
||||
matched = [
|
||||
a
|
||||
for a in self._agents.values()
|
||||
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
|
||||
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
|
||||
and (owner_key is None or a.get("user_id") == owner_key)
|
||||
and _square_ok(a)
|
||||
]
|
||||
if sort == "time_asc":
|
||||
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
|
||||
elif sort == "updated_desc":
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
elif sort == "updated_asc":
|
||||
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
|
||||
elif sort == "name_asc":
|
||||
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
|
||||
elif sort == "name_desc":
|
||||
matched.sort(
|
||||
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
else:
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
|
||||
total = len(matched)
|
||||
start = max(page - 1, 0) * page_size
|
||||
page_rows = matched[start : start + page_size]
|
||||
return [_copy(a) for a in page_rows], total
|
||||
|
||||
# --- Listing cache (skills / tool_groups / model) ---
|
||||
|
||||
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
|
||||
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
|
||||
|
||||
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
|
||||
self._extras[agent_id] = {"tool_groups": tool_groups, "model": model}
|
||||
|
||||
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for agent_id in agent_ids:
|
||||
has_extras = agent_id in self._extras
|
||||
has_skills = agent_id in self._skills
|
||||
if not has_extras and not has_skills:
|
||||
continue
|
||||
extras = self._extras.get(agent_id, {})
|
||||
out[agent_id] = {
|
||||
"skills": list(self._skills.get(agent_id, [])),
|
||||
"tool_groups": extras.get("tool_groups"),
|
||||
"model": extras.get("model"),
|
||||
}
|
||||
return out
|
||||
|
||||
async def delete_extras_for_agent(self, agent_id: str) -> None:
|
||||
self._skills.pop(agent_id, None)
|
||||
self._extras.pop(agent_id, None)
|
||||
|
||||
# --- Favorites ("收藏") ---
|
||||
|
||||
async def add_favorite(self, user_id: str, agent_id: str) -> None:
|
||||
self._favorites.add((user_id, agent_id))
|
||||
|
||||
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
|
||||
if (user_id, agent_id) in self._favorites:
|
||||
self._favorites.discard((user_id, agent_id))
|
||||
return True
|
||||
return False
|
||||
|
||||
async def list_favorite_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
|
||||
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
|
||||
|
||||
# --- Pins ("置顶") ---
|
||||
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
self._pins.add((user_id, agent_id))
|
||||
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
key = (user_id, agent_id)
|
||||
if key not in self._pins:
|
||||
return False
|
||||
self._pins.remove(key)
|
||||
return True
|
||||
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
|
||||
"""In-memory custom agent store for memory persistence mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.agent_name_desensitize import name_matches
|
||||
from deerflow.persistence.agents.base import AgentStore
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _copy(data: dict[str, Any]) -> dict[str, Any]:
|
||||
copied = dict(data)
|
||||
copied["builtin"] = copied.get("user_id") is None
|
||||
return copied
|
||||
|
||||
|
||||
class MemoryAgentStore(AgentStore):
|
||||
def __init__(self) -> None:
|
||||
self._agents: dict[str, dict[str, Any]] = {}
|
||||
self._favorites: set[tuple[str, str]] = set()
|
||||
self._pins: set[tuple[str, str]] = set()
|
||||
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
|
||||
self._skills: dict[str, list[str]] = {}
|
||||
self._extras: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
|
||||
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
|
||||
|
||||
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
|
||||
"""Single-record visibility (``get_visible``) — the union of every
|
||||
listing surface, so a list-visible agent always opens. Mirrors
|
||||
``AgentRepository._visible_filter``: baseline visibility OR
|
||||
private-square (password is the discovery gate, reads are not
|
||||
re-gated) OR favorited (the "我的" listing keeps favorites even after
|
||||
the owner unpublishes)."""
|
||||
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
|
||||
|
||||
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
|
||||
rows = [
|
||||
_copy(agent)
|
||||
for agent in self._agents.values()
|
||||
if self._is_visible(agent, user_id) and not agent.get("hidden")
|
||||
]
|
||||
rows.sort(key=lambda agent: agent.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or not self._is_fetchable(agent, user_id):
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
async def get_any(self, agent_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
return _copy(agent) if agent is not None else None
|
||||
|
||||
async def get_owned(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or agent.get("user_id") != user_id:
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
async def create(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = _now()
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"name": data["name"],
|
||||
"description": data.get("description") or "",
|
||||
"user_id": data.get("user_id"),
|
||||
"published": bool(data.get("published", False)),
|
||||
"is_private": bool(data.get("is_private", False)),
|
||||
"square_id": str(data.get("square_id") or ""),
|
||||
"hidden": bool(data.get("hidden", False)),
|
||||
"featured_order": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._agents[row["id"]] = row
|
||||
return _copy(row)
|
||||
|
||||
async def ensure_builtin(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = self._agents.get(data["id"])
|
||||
if row is None:
|
||||
row = {
|
||||
"id": data["id"],
|
||||
"name": data["name"],
|
||||
"description": data.get("description") or "",
|
||||
"user_id": None,
|
||||
"published": bool(data.get("published", True)),
|
||||
"hidden": bool(data.get("hidden", False)),
|
||||
"featured_order": None,
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
self._agents[row["id"]] = row
|
||||
elif row.get("user_id") is None:
|
||||
row["name"] = data["name"]
|
||||
row["description"] = data.get("description") or ""
|
||||
row["published"] = bool(data.get("published", True))
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def update(self, agent_id: str, user_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def update_builtin(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete(self, agent_id: str, user_id: str) -> bool:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
async def delete_builtin(self, agent_id: str) -> bool:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
async def count_owned(self, user_id: str) -> int:
|
||||
return sum(1 for a in self._agents.values() if a.get("user_id") == user_id)
|
||||
|
||||
async def set_featured(self, agent_id: str, order: int | None) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
row["featured_order"] = order
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def list_featured(self, user_id: str) -> list[dict[str, Any]]:
|
||||
candidates = [
|
||||
a
|
||||
for a in self._agents.values()
|
||||
if a.get("featured_order") is not None and self._is_visible(a, user_id) and not a.get("hidden")
|
||||
]
|
||||
candidates.sort(key=lambda a: (a.get("featured_order"), a.get("created_at") or ""))
|
||||
return [_copy(a) for a in candidates]
|
||||
|
||||
async def list_sidebar(self, user_id: str, *, limit: int = 3) -> list[dict[str, Any]]:
|
||||
mine = [
|
||||
a for a in self._agents.values()
|
||||
if not a.get("hidden")
|
||||
and ((a.get("user_id") == user_id) or (user_id, a["id"]) in self._favorites)
|
||||
]
|
||||
square = [
|
||||
a for a in self._agents.values()
|
||||
if not a.get("hidden")
|
||||
and (a.get("user_id") is None or a.get("published"))
|
||||
and not a.get("is_private")
|
||||
]
|
||||
for rows in (mine, square):
|
||||
rows.sort(
|
||||
key=lambda a: (str(a.get("created_at") or ""), str(a.get("id") or "")),
|
||||
reverse=True,
|
||||
)
|
||||
rows.sort(key=lambda a: (user_id, a["id"]) in self._pins, reverse=True)
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(agent: dict[str, Any]) -> None:
|
||||
if len(out) >= limit or agent["id"] in seen:
|
||||
return
|
||||
seen.add(agent["id"])
|
||||
out.append({"id": agent["id"], "name": agent.get("name") or ""})
|
||||
|
||||
for agent in [*mine, *square]:
|
||||
if (user_id, agent["id"]) in self._pins:
|
||||
add(agent)
|
||||
for agent in mine:
|
||||
add(agent)
|
||||
for agent in square:
|
||||
add(agent)
|
||||
return out
|
||||
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict[str, Any]]:
|
||||
order_map = {aid: idx for idx, aid in enumerate(agent_ids)}
|
||||
rows = [self._agents[aid] for aid in agent_ids if aid in self._agents and self._is_visible(self._agents[aid], user_id)]
|
||||
rows.sort(key=lambda a: order_map.get(a["id"], 0))
|
||||
return [_copy(a) for a in rows]
|
||||
|
||||
async def list_private(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values() if bool(a.get("is_private")) and not a.get("hidden")]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def list_all(self) -> list[dict[str, Any]]:
|
||||
rows = [_copy(a) for a in self._agents.values()]
|
||||
rows.sort(key=lambda a: a.get("created_at") or "")
|
||||
return rows
|
||||
|
||||
async def update_any(self, agent_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private", "square_id", "hidden"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete_any(self, agent_id: str) -> bool:
|
||||
if agent_id not in self._agents:
|
||||
return False
|
||||
self._agents.pop(agent_id, None)
|
||||
return True
|
||||
|
||||
# --- Paginated listing ---
|
||||
|
||||
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
|
||||
if admin:
|
||||
return True # every agent — the hidden filter is a separate param
|
||||
if agent.get("hidden"):
|
||||
return False # admin-hidden agents vanish from every non-admin scope
|
||||
if scope == "square":
|
||||
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
|
||||
if scope == "mine":
|
||||
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
|
||||
favorited = (user_id, agent["id"]) in self._favorites
|
||||
return owned or favorited
|
||||
if scope == "private":
|
||||
return bool(agent.get("is_private"))
|
||||
if scope == "builtin":
|
||||
# 内置智能体 = user_id IS NULL(仅管理员;路由层 admin 闸门)。
|
||||
return agent.get("user_id") is None
|
||||
return self._is_visible(agent, user_id)
|
||||
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
hidden: bool | None = None,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
|
||||
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
|
||||
owner_key = owner.strip() if owner and owner.strip() else None
|
||||
square_key = square_id.strip() if square_id and square_id.strip() else None
|
||||
|
||||
def _square_ok(a: dict[str, Any]) -> bool:
|
||||
if square_key is None:
|
||||
return True
|
||||
sid = str(a.get("square_id") or "")
|
||||
return sid == square_key or (square_is_default and sid == "")
|
||||
|
||||
def _hidden_ok(a: dict[str, Any]) -> bool:
|
||||
if admin:
|
||||
# Admin console filter: keep only hidden / only visible agents.
|
||||
return hidden is None or bool(a.get("hidden")) == hidden
|
||||
return not a.get("hidden")
|
||||
|
||||
matched = [
|
||||
a
|
||||
for a in self._agents.values()
|
||||
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
|
||||
and _hidden_ok(a)
|
||||
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
|
||||
and (owner_key is None or a.get("user_id") == owner_key)
|
||||
and _square_ok(a)
|
||||
]
|
||||
if sort == "time_asc":
|
||||
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
|
||||
elif sort == "updated_desc":
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
elif sort == "updated_asc":
|
||||
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
|
||||
elif sort == "name_asc":
|
||||
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
|
||||
elif sort == "name_desc":
|
||||
matched.sort(
|
||||
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
else:
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
|
||||
total = len(matched)
|
||||
start = max(page - 1, 0) * page_size
|
||||
page_rows = matched[start : start + page_size]
|
||||
return [_copy(a) for a in page_rows], total
|
||||
|
||||
# --- Listing cache (skills / tool_groups / model) ---
|
||||
|
||||
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
|
||||
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
|
||||
|
||||
async def set_extras(
|
||||
self,
|
||||
agent_id: str,
|
||||
*,
|
||||
tool_groups: list[str] | None,
|
||||
model: str | None,
|
||||
chat_link: Any = None,
|
||||
update_chat_link: bool = False,
|
||||
) -> None:
|
||||
previous = self._extras.get(agent_id, {})
|
||||
stored = {
|
||||
"tool_groups": tool_groups,
|
||||
"model": model,
|
||||
"chat_link": previous.get("chat_link"),
|
||||
}
|
||||
if update_chat_link:
|
||||
stored["chat_link"] = chat_link
|
||||
self._extras[agent_id] = stored
|
||||
|
||||
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for agent_id in agent_ids:
|
||||
has_extras = agent_id in self._extras
|
||||
has_skills = agent_id in self._skills
|
||||
if not has_extras and not has_skills:
|
||||
continue
|
||||
extras = self._extras.get(agent_id, {})
|
||||
out[agent_id] = {
|
||||
"skills": list(self._skills.get(agent_id, [])),
|
||||
"tool_groups": extras.get("tool_groups"),
|
||||
"model": extras.get("model"),
|
||||
"chat_link": extras.get("chat_link"),
|
||||
}
|
||||
return out
|
||||
|
||||
async def delete_extras_for_agent(self, agent_id: str) -> None:
|
||||
self._skills.pop(agent_id, None)
|
||||
self._extras.pop(agent_id, None)
|
||||
|
||||
# --- Favorites ("收藏") ---
|
||||
|
||||
async def add_favorite(self, user_id: str, agent_id: str) -> None:
|
||||
self._favorites.add((user_id, agent_id))
|
||||
|
||||
async def remove_favorite(self, user_id: str, agent_id: str) -> bool:
|
||||
if (user_id, agent_id) in self._favorites:
|
||||
self._favorites.discard((user_id, agent_id))
|
||||
return True
|
||||
return False
|
||||
|
||||
async def list_favorite_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
|
||||
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
|
||||
|
||||
# --- Pins ("置顶") ---
|
||||
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
self._pins.add((user_id, agent_id))
|
||||
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
key = (user_id, agent_id)
|
||||
if key not in self._pins:
|
||||
return False
|
||||
self._pins.remove(key)
|
||||
return True
|
||||
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
|
||||
+20
-1
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -13,6 +13,13 @@ from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
class AgentRow(Base):
|
||||
__tablename__ = "agents"
|
||||
# Composite indexes for the gallery tabs. Single-column indexes cannot
|
||||
# serve "广场" (hidden + private + square) or "我的" (owner + hidden)
|
||||
# without a wide scan once the catalog grows.
|
||||
__table_args__ = (
|
||||
Index("ix_agents_catalog_square", "hidden", "is_private", "square_id", "created_at"),
|
||||
Index("ix_agents_owner_hidden", "user_id", "hidden", "created_at"),
|
||||
)
|
||||
|
||||
# Stored as a plain string so the same id can be used as the filesystem
|
||||
# directory name under {base_dir}/agents/{id}/.
|
||||
@@ -27,6 +34,14 @@ class AgentRow(Base):
|
||||
# visible to its owner in the "我的" tab, and visible to anyone in the
|
||||
# private-square listing once the password has been verified.
|
||||
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# When True the agent is hidden from every non-admin listing surface
|
||||
# (gallery / 我的 / 广场 / homepage selector / private square). Only the
|
||||
# admin agents console (?admin=true) still lists it. Purely a catalog
|
||||
# switch: single-record access (detail / chat / run) keeps working, so
|
||||
# existing conversations with a hidden agent are unaffected.
|
||||
# server_default keeps the startup ORM-column sweep able to ADD COLUMN
|
||||
# on existing deployments (NOT NULL needs a default).
|
||||
hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"), index=True)
|
||||
# Publish square (发布广场) this agent is filed under. A pure organizational
|
||||
# category with no access control — independent of ``published`` /
|
||||
# ``is_private``. Empty string = "not assigned" → the listing layer treats
|
||||
@@ -124,4 +139,8 @@ class AgentExtrasRow(Base):
|
||||
# list[str] | None — the agent's tool_groups whitelist from config.yaml.
|
||||
tool_groups: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True, default=None)
|
||||
model: Mapped[str | None] = mapped_column(String(191), nullable=True, default=None)
|
||||
# Gallery 「对话」 jump target mirrored from config.yaml so the list endpoint
|
||||
# can redirect without reading every agent file. Nullable JSON so existing
|
||||
# databases pick the column up on the startup ORM sweep.
|
||||
chat_link: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True, default=None)
|
||||
skills_synced_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+596
-504
File diff suppressed because it is too large.
Load diff
+23
@@ -0,0 +1,23 @@
|
||||
"""MySQL-persisted administrator switch for the optional API Redis cache."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, Boolean, Integer, String, text, true
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ApiReadCacheSettingRow(Base):
|
||||
__tablename__ = "api_read_cache_settings"
|
||||
|
||||
key: Mapped[str] = mapped_column(String(32), primary_key=True, default="default")
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
|
||||
ttl_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=300, server_default=text("300"))
|
||||
collection_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
cache_epoch: Mapped[str] = mapped_column(String(32), nullable=False, default="0", server_default="0")
|
||||
browser_status_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true())
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
+16
-11
@@ -1204,7 +1204,7 @@ class AssistantKnowledgeRepository:
|
||||
stats[base_id][stat_name] = int(stat_value or 0)
|
||||
return stats
|
||||
|
||||
async def get_base(self, base_id: str | None = None) -> dict[str, Any] | None:
|
||||
async def get_base(self, base_id: str | None = None, *, include_stats: bool = True) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
if base_id:
|
||||
row = (
|
||||
@@ -1215,7 +1215,9 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return await self._base_with_stats(session, row) if row else None
|
||||
if row is None:
|
||||
return None
|
||||
return await self._base_with_stats(session, row) if include_stats else _dict(row)
|
||||
|
||||
row = (
|
||||
await session.execute(
|
||||
@@ -1240,15 +1242,17 @@ class AssistantKnowledgeRepository:
|
||||
row.updated_at = _now()
|
||||
await session.flush()
|
||||
await session.commit()
|
||||
return await self._base_with_stats(session, row) if row else None
|
||||
if row is None:
|
||||
return None
|
||||
return await self._base_with_stats(session, row) if include_stats else _dict(row)
|
||||
|
||||
async def list_bases(self) -> list[dict[str, Any]]:
|
||||
# Keep the library switcher a metadata-only read. Global alignment can
|
||||
# scan and rewrite the complete Wiki/vector corpus after a process
|
||||
# restart, so doing it here makes merely opening the assistant-library
|
||||
# landing page scale with the size of the largest library. Content
|
||||
# reads/searches still call ``_ensure_base_aligned`` before returning
|
||||
# data, and imports align atomically before committing their snapshot.
|
||||
# searches/exports still call ``_ensure_base_aligned`` for legacy data,
|
||||
# and imports align atomically before committing their snapshot.
|
||||
async with self._sf() as session:
|
||||
rows = list(
|
||||
(
|
||||
@@ -2367,10 +2371,12 @@ class AssistantKnowledgeRepository:
|
||||
include_chunks: bool = False,
|
||||
include_content: bool = True,
|
||||
) -> list[dict[str, Any]]:
|
||||
base = await self.get_base(base_id)
|
||||
base = await self.get_base(base_id, include_stats=False)
|
||||
if base is None:
|
||||
return []
|
||||
await self._ensure_base_aligned(str(base["id"]))
|
||||
# Browse the last committed snapshot. Imports already align it before
|
||||
# commit; waiting for the global import/alignment locks here would make
|
||||
# opening any library hang behind an unrelated long-running import.
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == base["id"],
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
@@ -3020,10 +3026,9 @@ class AssistantKnowledgeRepository:
|
||||
return results
|
||||
|
||||
async def get_page(self, slug: str, *, base_id: str | None = None) -> dict[str, Any] | None:
|
||||
base = await self.get_base(base_id)
|
||||
base = await self.get_base(base_id, include_stats=False)
|
||||
if base is None:
|
||||
return None
|
||||
await self._ensure_base_aligned(str(base["id"]))
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == base["id"],
|
||||
AssistantKnowledgeWikiPageRow.slug == slug.strip("/"),
|
||||
@@ -3122,7 +3127,7 @@ class AssistantKnowledgeRepository:
|
||||
return data
|
||||
|
||||
async def list_sources(self, *, base_id: str | None = None) -> list[dict[str, Any]]:
|
||||
base = await self.get_base(base_id)
|
||||
base = await self.get_base(base_id, include_stats=False)
|
||||
if base is None:
|
||||
return []
|
||||
async with self._sf() as session:
|
||||
@@ -3162,7 +3167,7 @@ class AssistantKnowledgeRepository:
|
||||
return result
|
||||
|
||||
async def list_jobs(self, limit: int = 100, *, base_id: str | None = None) -> list[dict[str, Any]]:
|
||||
base = await self.get_base(base_id)
|
||||
base = await self.get_base(base_id, include_stats=False)
|
||||
if base is None:
|
||||
return []
|
||||
async with self._sf() as session:
|
||||
|
||||
+12
-2
@@ -4,7 +4,7 @@ A "business mapping" row binds a fixed business code (3Q / 6BF / 3q-0…3q-6 /
|
||||
6-1…6-4 / 7BF / 7-1…7-4 / 8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
|
||||
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
|
||||
account can resolve it). Used to auto-pick the business chain when a task page
|
||||
launches the multi-agent roundtable.
|
||||
launches the multi-agent roundtable. ``3q-1…3q-6`` are seeded as custom-type rows.
|
||||
|
||||
Unlike ``roundtable_chains`` (per-user), this mapping is **global / shared** —
|
||||
there is no ``user_id`` column; all accounts see and use the same configuration.
|
||||
@@ -20,7 +20,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy import Boolean, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -34,6 +34,16 @@ class BusinessMappingRow(Base):
|
||||
business_code: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
# Display label shown in the 业务 column (e.g. "任务FX").
|
||||
label: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
# A user-visible category. Built-in rows keep ``内置业务``; rows created from
|
||||
# the configuration page consistently use ``自定义类型``.
|
||||
mapping_type: Mapped[str] = mapped_column(
|
||||
String(64), nullable=False, default="内置业务", server_default="内置业务"
|
||||
)
|
||||
# Built-in rows are seeded and cannot be redefined. Custom rows are created
|
||||
# by an administrator and their business_code doubles as a login goPath.
|
||||
is_custom: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0"
|
||||
)
|
||||
# Picked 智能体 (暂存不用,自动流程当前不消费此字段). NULL = 未配置.
|
||||
agent_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(191), nullable=True)
|
||||
|
||||
+73
-9
@@ -11,7 +11,7 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.business_mapping.model import BusinessMappingRow
|
||||
@@ -23,6 +23,8 @@ _UNSET = object()
|
||||
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
|
||||
# b5-1 = 外部写入本系统详情后,由本系统读取的任务分析单智能体深链。
|
||||
# 3q-0…3q-6 / 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
|
||||
# 3q-1…3q-6 默认按「自定义类型」落库(goPath 走自定义深链);配置管理页默认不展示这 6 行。
|
||||
_CUSTOM_SLOT = {"is_custom": True, "mapping_type": "自定义类型"}
|
||||
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
{"business_code": "3Q", "label": "3QFX"},
|
||||
{"business_code": "6BF", "label": "任务FX"},
|
||||
@@ -32,12 +34,12 @@ DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
{"business_code": "b5-1", "label": "任务分析"},
|
||||
# 独立配置位,与上方业务码无关;固定排在表尾。
|
||||
{"business_code": "3q-0", "label": "3q-0"},
|
||||
{"business_code": "3q-1", "label": "3q-1"},
|
||||
{"business_code": "3q-2", "label": "3q-2"},
|
||||
{"business_code": "3q-3", "label": "3q-3"},
|
||||
{"business_code": "3q-4", "label": "3q-4"},
|
||||
{"business_code": "3q-5", "label": "3q-5"},
|
||||
{"business_code": "3q-6", "label": "3q-6"},
|
||||
{"business_code": "3q-1", "label": "3q-1", **_CUSTOM_SLOT},
|
||||
{"business_code": "3q-2", "label": "3q-2", **_CUSTOM_SLOT},
|
||||
{"business_code": "3q-3", "label": "3q-3", **_CUSTOM_SLOT},
|
||||
{"business_code": "3q-4", "label": "3q-4", **_CUSTOM_SLOT},
|
||||
{"business_code": "3q-5", "label": "3q-5", **_CUSTOM_SLOT},
|
||||
{"business_code": "3q-6", "label": "3q-6", **_CUSTOM_SLOT},
|
||||
{"business_code": "6-1", "label": "6-1"},
|
||||
{"business_code": "6-2", "label": "6-2"},
|
||||
{"business_code": "6-3", "label": "6-3"},
|
||||
@@ -49,6 +51,14 @@ DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
]
|
||||
|
||||
|
||||
def _seed_type(seed: dict[str, Any]) -> tuple[bool, str]:
|
||||
is_custom = bool(seed.get("is_custom", False))
|
||||
mapping_type = str(seed.get("mapping_type") or "").strip()
|
||||
if not mapping_type:
|
||||
mapping_type = "自定义类型" if is_custom else "内置业务"
|
||||
return is_custom, mapping_type
|
||||
|
||||
|
||||
class BusinessMappingRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
@@ -58,6 +68,8 @@ class BusinessMappingRepository:
|
||||
return {
|
||||
"business_code": row.business_code,
|
||||
"label": row.label,
|
||||
"mapping_type": row.mapping_type or "内置业务",
|
||||
"is_custom": bool(row.is_custom),
|
||||
"agent_id": row.agent_id,
|
||||
"agent_name": row.agent_name,
|
||||
"chain_id": row.chain_id,
|
||||
@@ -110,18 +122,62 @@ class BusinessMappingRepository:
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
|
||||
async def create_custom_mapping(
|
||||
self,
|
||||
*,
|
||||
business_code: str,
|
||||
mapping_type: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Create one administrator-defined mapping.
|
||||
|
||||
``business_code`` is deliberately the externally supplied goPath. The
|
||||
case-insensitive uniqueness check mirrors frontend deep-link matching,
|
||||
preventing two rows that could resolve from the same login URL.
|
||||
"""
|
||||
normalized_code = business_code.strip()
|
||||
normalized_type = mapping_type.strip()
|
||||
async with self._sf() as session:
|
||||
existing = await session.scalar(
|
||||
select(BusinessMappingRow).where(
|
||||
func.lower(BusinessMappingRow.business_code) == normalized_code.lower()
|
||||
)
|
||||
)
|
||||
if existing is not None:
|
||||
return None
|
||||
max_sort_order = await session.scalar(select(func.max(BusinessMappingRow.sort_order)))
|
||||
now = datetime.now(UTC)
|
||||
row = BusinessMappingRow(
|
||||
business_code=normalized_code,
|
||||
label=normalized_code,
|
||||
mapping_type=normalized_type,
|
||||
is_custom=True,
|
||||
agent_id=None,
|
||||
agent_name=None,
|
||||
chain_id=None,
|
||||
chain_title=None,
|
||||
sort_order=(int(max_sort_order) if max_sort_order is not None else -1) + 1,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
|
||||
async def ensure_seed_defaults(self) -> int:
|
||||
"""Idempotently insert the fixed business rows if missing. Returns inserted count.
|
||||
|
||||
Never overwrites an existing row's agent/chain (respects user config).
|
||||
Re-syncs ``sort_order`` (and empty ``label``) so newly inserted codes keep
|
||||
the intended table order next to older seeded rows.
|
||||
Re-syncs ``sort_order``, empty ``label``, and seed ``is_custom`` /
|
||||
``mapping_type`` so newly inserted codes keep the intended table order
|
||||
and 3q-1…3q-6 stay classified as custom type.
|
||||
"""
|
||||
inserted = 0
|
||||
dirty = False
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
for idx, seed in enumerate(DEFAULT_BUSINESS_ROWS):
|
||||
is_custom, mapping_type = _seed_type(seed)
|
||||
existing = await session.get(BusinessMappingRow, seed["business_code"])
|
||||
if existing is not None:
|
||||
if existing.sort_order != idx:
|
||||
@@ -131,11 +187,19 @@ class BusinessMappingRepository:
|
||||
if seed_label and not (existing.label or "").strip():
|
||||
existing.label = seed_label
|
||||
dirty = True
|
||||
if bool(existing.is_custom) != is_custom:
|
||||
existing.is_custom = is_custom
|
||||
dirty = True
|
||||
if (existing.mapping_type or "") != mapping_type:
|
||||
existing.mapping_type = mapping_type
|
||||
dirty = True
|
||||
continue
|
||||
session.add(
|
||||
BusinessMappingRow(
|
||||
business_code=seed["business_code"],
|
||||
label=seed.get("label", ""),
|
||||
mapping_type=mapping_type,
|
||||
is_custom=is_custom,
|
||||
agent_id=None,
|
||||
agent_name=None,
|
||||
chain_id=None,
|
||||
|
||||
+1
@@ -22,6 +22,7 @@ def _normalize(data: dict[str, Any], menu_type: str, updated_by: str | None) ->
|
||||
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"label": (data.get("label") or None),
|
||||
"disabled": bool(data.get("disabled", False)),
|
||||
"updated_by": updated_by,
|
||||
"updated_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
+5
-3
@@ -1,14 +1,15 @@
|
||||
"""ORM model for compact/sidebar-summary menu overrides.
|
||||
|
||||
This table only controls the compact left menu presentation. The full sidebar
|
||||
menu layout continues to use ``menu_overrides`` and is unaffected.
|
||||
This table only controls compact menu presentation: lane, order, label and
|
||||
enablement for the left menu, settings menu and toolbox. The full sidebar menu
|
||||
layout continues to use ``menu_overrides`` and is unaffected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, Integer, String
|
||||
from sqlalchemy import Boolean, Index, Integer, String, false
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -24,6 +25,7 @@ class CompactMenuOverrideRow(Base):
|
||||
placement: Mapped[str] = mapped_column(String(32), default="sidebar")
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0)
|
||||
label: Mapped[str | None] = mapped_column(String(255))
|
||||
disabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
|
||||
+2
@@ -41,6 +41,7 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C
|
||||
placement=placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
sort_order=int(data.get("sort_order", 0)),
|
||||
label=(data.get("label") or None),
|
||||
disabled=bool(data.get("disabled", False)),
|
||||
updated_by=updated_by,
|
||||
)
|
||||
|
||||
@@ -53,6 +54,7 @@ def _normalize_user_row(data: dict[str, Any], menu_type: str) -> dict[str, Any]:
|
||||
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"label": (data.get("label") or None),
|
||||
"disabled": bool(data.get("disabled", False)),
|
||||
}
|
||||
|
||||
|
||||
|
||||
+504
-237
@@ -11,14 +11,20 @@ None and fall back to in-memory implementations.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import weakref
|
||||
from contextvars import ContextVar, Token
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy import event, text
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
@@ -38,21 +44,167 @@ _session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
# "got Future attached to a different loop". We capture the engine's home loop here so
|
||||
# best-effort writes coming from worker threads / other loops can be routed back to it
|
||||
# (see ``fire_and_forget_db_write``).
|
||||
_engine_loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
# These are the complete workflow-studio persistence tables owned by DeerFlow.
|
||||
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
|
||||
# wildcard: another application might share a schema and have its own table
|
||||
# with that prefix.
|
||||
_WORKFLOW_SCHEMA_TABLE_NAMES = (
|
||||
"workflow_definitions",
|
||||
"workflow_versions",
|
||||
"workflow_runs",
|
||||
"workflow_node_runs",
|
||||
"workflow_run_events",
|
||||
"workflow_run_artifacts",
|
||||
"workflow_data_sources",
|
||||
)
|
||||
_engine_loop: asyncio.AbstractEventLoop | None = None
|
||||
_performance_request_context: ContextVar[tuple[str, Any] | None] = ContextVar("deerflow_performance_request", default=None)
|
||||
|
||||
|
||||
def bind_performance_request(request_id: str) -> Token:
|
||||
"""Bind a request ID to this task so slow SQL logs can be correlated safely."""
|
||||
|
||||
try:
|
||||
task = asyncio.current_task()
|
||||
except RuntimeError:
|
||||
task = None
|
||||
return _performance_request_context.set((request_id, weakref.ref(task) if task is not None else None))
|
||||
|
||||
|
||||
def reset_performance_request(token: Token) -> None:
|
||||
_performance_request_context.reset(token)
|
||||
|
||||
|
||||
def _current_performance_request_id() -> str:
|
||||
context = _performance_request_context.get()
|
||||
if context is None:
|
||||
return "-"
|
||||
try:
|
||||
task = asyncio.current_task()
|
||||
except RuntimeError:
|
||||
task = None
|
||||
return context[0] if task is not None and context[1] is not None and context[1]() is task else "-"
|
||||
|
||||
|
||||
def _slow_sql_threshold_ms() -> float:
|
||||
try:
|
||||
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_SQL_LOG_MS", "500")), 60_000.0))
|
||||
except (TypeError, ValueError):
|
||||
return 500.0
|
||||
|
||||
|
||||
def _sql_operation(statement: str) -> str:
|
||||
parts = statement.lstrip().split(None, 1)
|
||||
candidate = parts[0].upper() if parts else "EMPTY"
|
||||
return candidate if re.fullmatch(r"[A-Z_]+", candidate) else "OTHER"
|
||||
|
||||
|
||||
def _sql_table(statement: str) -> str:
|
||||
"""Return the target/source table only, without query text or values."""
|
||||
match = re.search(
|
||||
r"\b(?:INSERT\s+INTO|UPDATE|DELETE\s+FROM)\s+([`\"\w.]+)",
|
||||
statement,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if match is None:
|
||||
match = re.search(r"\bFROM\s+([`\"\w.]+)", statement, re.IGNORECASE)
|
||||
if match is None:
|
||||
return "unknown"
|
||||
return match.group(1).strip("`\".")[-128:] or "unknown"
|
||||
|
||||
|
||||
def _sql_fingerprint(statement: str) -> str:
|
||||
return hashlib.sha256(statement.encode("utf-8", "replace")).hexdigest()[:12]
|
||||
|
||||
|
||||
def _database_error_metadata(exception: BaseException | None) -> tuple[str, str, str]:
|
||||
"""Extract safe driver diagnostics without logging SQL or bound values."""
|
||||
if exception is None:
|
||||
return "unknown", "-", "-"
|
||||
|
||||
code = getattr(exception, "errno", None) or getattr(exception, "code", None)
|
||||
args = getattr(exception, "args", ())
|
||||
if code is None and args and isinstance(args[0], (int, str)):
|
||||
candidate = str(args[0])
|
||||
if candidate.isdigit():
|
||||
code = candidate
|
||||
|
||||
sqlstate = getattr(exception, "sqlstate", None) or getattr(exception, "pgcode", None)
|
||||
diagnostic = getattr(exception, "diag", None)
|
||||
constraint = getattr(diagnostic, "constraint_name", None)
|
||||
|
||||
def _safe_identifier(value: Any) -> str:
|
||||
text_value = str(value or "")
|
||||
return text_value[:128] if re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", text_value) else "-"
|
||||
|
||||
return _safe_identifier(code), _safe_identifier(sqlstate), _safe_identifier(constraint)
|
||||
|
||||
|
||||
def _install_slow_sql_logging(engine: AsyncEngine, backend: str) -> None:
|
||||
"""Log slow SQL timings without logging query text or bound values."""
|
||||
|
||||
sync_engine = engine.sync_engine
|
||||
|
||||
def before_cursor_execute(_conn, _cursor, statement, _parameters, context, _executemany):
|
||||
if context is not None:
|
||||
context._deerflow_perf_started = time.perf_counter()
|
||||
context._deerflow_perf_operation = _sql_operation(statement)
|
||||
context._deerflow_perf_statement = statement
|
||||
|
||||
def after_cursor_execute(_conn, _cursor, _statement, _parameters, context, executemany):
|
||||
if context is None:
|
||||
return
|
||||
started = getattr(context, "_deerflow_perf_started", None)
|
||||
if started is None:
|
||||
return
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
if elapsed_ms >= _slow_sql_threshold_ms():
|
||||
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", ""))
|
||||
logger.warning(
|
||||
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%.2f executemany=%s fingerprint=%s outcome=ok",
|
||||
_current_performance_request_id(),
|
||||
backend,
|
||||
getattr(context, "_deerflow_perf_operation", "OTHER"),
|
||||
_sql_table(getattr(context, "_deerflow_perf_statement", "")),
|
||||
elapsed_ms,
|
||||
bool(executemany),
|
||||
fingerprint,
|
||||
)
|
||||
|
||||
def handle_error(exception_context):
|
||||
context = exception_context.execution_context
|
||||
started = getattr(context, "_deerflow_perf_started", None) if context is not None else None
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2) if started is not None else None
|
||||
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
|
||||
original_exception = exception_context.original_exception
|
||||
error_code, sqlstate, constraint = _database_error_metadata(original_exception)
|
||||
table = _sql_table(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
|
||||
duplicate_key = error_code == "1062" or sqlstate == "23505" or (
|
||||
table == "scheduled_task_runs"
|
||||
and "UNIQUE constraint failed" in str(original_exception or "")
|
||||
)
|
||||
log = logger.info if table == "scheduled_task_runs" and duplicate_key else logger.error
|
||||
outcome = "duplicate_ignored" if table == "scheduled_task_runs" and duplicate_key else "error"
|
||||
log(
|
||||
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%s executemany=%s fingerprint=%s outcome=%s error_type=%s error_code=%s sqlstate=%s constraint=%s",
|
||||
_current_performance_request_id(),
|
||||
backend,
|
||||
getattr(context, "_deerflow_perf_operation", "OTHER") if context is not None else "OTHER",
|
||||
table,
|
||||
elapsed_ms,
|
||||
bool(getattr(context, "executemany", False)) if context is not None else False,
|
||||
fingerprint,
|
||||
outcome,
|
||||
type(original_exception).__name__ if original_exception is not None else "unknown",
|
||||
error_code,
|
||||
sqlstate,
|
||||
constraint,
|
||||
)
|
||||
|
||||
event.listen(sync_engine, "before_cursor_execute", before_cursor_execute)
|
||||
event.listen(sync_engine, "after_cursor_execute", after_cursor_execute)
|
||||
event.listen(sync_engine, "handle_error", handle_error)
|
||||
|
||||
# These are the complete workflow-studio persistence tables owned by DeerFlow.
|
||||
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
|
||||
# wildcard: another application might share a schema and have its own table
|
||||
# with that prefix.
|
||||
_WORKFLOW_SCHEMA_TABLE_NAMES = (
|
||||
"workflow_definitions",
|
||||
"workflow_versions",
|
||||
"workflow_runs",
|
||||
"workflow_node_runs",
|
||||
"workflow_run_events",
|
||||
"workflow_run_artifacts",
|
||||
"workflow_data_sources",
|
||||
)
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
@@ -81,6 +233,42 @@ def _mysql_driver_name(url: str) -> str:
|
||||
return "asyncmy"
|
||||
|
||||
|
||||
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
|
||||
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
|
||||
kwargs = dict(
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
|
||||
# These are not MySQL/psycopg connection arguments.
|
||||
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
|
||||
kwargs["connect_args"] = {"command_timeout": command_timeout}
|
||||
return kwargs
|
||||
|
||||
|
||||
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
|
||||
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
|
||||
kwargs = dict(
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
|
||||
# These are not MySQL/psycopg connection arguments.
|
||||
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
|
||||
kwargs["connect_args"] = {"command_timeout": command_timeout}
|
||||
return kwargs
|
||||
|
||||
|
||||
async def _run_alembic_upgrade(url: str) -> None:
|
||||
"""Apply pending Alembic migrations against ``url`` (``alembic upgrade head``).
|
||||
|
||||
@@ -176,7 +364,7 @@ async def _auto_create_postgres_db(url: str) -> None:
|
||||
try:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
|
||||
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
|
||||
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
@@ -215,7 +403,7 @@ async def _auto_create_mysql_db(url: str) -> None:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
"""Add ORM-declared columns that are missing from already-existing tables.
|
||||
|
||||
``Base.metadata.create_all`` creates brand-new tables but never patches
|
||||
@@ -226,14 +414,14 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
|
||||
This sweep compares each existing table against the ORM metadata and issues
|
||||
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
|
||||
safely (nullable, or carrying a server_default). Missing ORM indexes are
|
||||
created as well. Table-level ``UniqueConstraint`` objects
|
||||
safely (nullable, or carrying a server_default). Missing ORM indexes are
|
||||
created as well. Table-level ``UniqueConstraint`` objects
|
||||
that are declared on the model but missing from the table are also created
|
||||
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
|
||||
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
|
||||
on an existing database, weakening dedupe.
|
||||
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
|
||||
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
|
||||
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
|
||||
``BeijingDateTime``) resolve to the right backend type. Everything is
|
||||
per-column best-effort: a failure is logged and startup continues.
|
||||
@@ -258,10 +446,8 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
continue
|
||||
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
|
||||
missing = [c for c in table.columns if c.name not in existing_cols]
|
||||
if not missing:
|
||||
continue
|
||||
|
||||
for col in missing:
|
||||
for col in missing:
|
||||
if not col.nullable and col.server_default is None:
|
||||
logger.warning(
|
||||
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
|
||||
@@ -270,21 +456,21 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
)
|
||||
continue
|
||||
try:
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
|
||||
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx:
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if not idx.name or idx.name in existing_idx:
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
logger.info("Auto-created index %s on %s", idx.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
|
||||
@@ -328,71 +514,111 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
f"ON {preparer.quote(table.name)} ({cols})"
|
||||
)
|
||||
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
|
||||
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
|
||||
|
||||
Alembic auto-upgrade is intentionally disabled in deployed Gateway
|
||||
processes, while the ORM column synchronizer is always run. The latter
|
||||
can add the task lookup columns and indexes but cannot derive values from
|
||||
JSON metadata. Backfill task rows once at startup so an upgrade never
|
||||
temporarily hides every historical task conversation from ``/threads/search``.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
table_name = "threads_meta"
|
||||
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
|
||||
try:
|
||||
inspector = sa_inspect(sync_conn)
|
||||
if table_name not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns(table_name)}
|
||||
if not required_columns <= columns:
|
||||
return
|
||||
|
||||
dialect = sync_conn.dialect.name
|
||||
if dialect == "sqlite":
|
||||
task_id = "json_extract(metadata_json, '$.taskId')"
|
||||
agent_id = "json_extract(metadata_json, '$.agent_id')"
|
||||
action_id = "json_extract(metadata_json, '$.actionId')"
|
||||
valid_json = "json_valid(metadata_json)"
|
||||
elif dialect == "mysql":
|
||||
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
|
||||
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
|
||||
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
|
||||
valid_json = "JSON_VALID(metadata_json)"
|
||||
elif dialect in {"postgresql", "opengauss"}:
|
||||
task_id = "metadata_json ->> 'taskId'"
|
||||
agent_id = "metadata_json ->> 'agent_id'"
|
||||
action_id = "metadata_json ->> 'actionId'"
|
||||
valid_json = "TRUE"
|
||||
else:
|
||||
return
|
||||
|
||||
result = sync_conn.execute(
|
||||
text(
|
||||
"UPDATE threads_meta "
|
||||
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
|
||||
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
|
||||
)
|
||||
)
|
||||
if result.rowcount:
|
||||
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
|
||||
except Exception:
|
||||
# A malformed legacy JSON row or a backend-specific JSON function must
|
||||
# not prevent startup. The generic metadata search remains available,
|
||||
# and the failure is visible to the operator for remediation.
|
||||
logger.exception("Failed to backfill thread task lookup projections")
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
|
||||
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
|
||||
|
||||
Alembic auto-upgrade is intentionally disabled in deployed Gateway
|
||||
processes, while the ORM column synchronizer is always run. The latter
|
||||
can add the task lookup columns and indexes but cannot derive values from
|
||||
JSON metadata. Backfill task rows once at startup so an upgrade never
|
||||
temporarily hides every historical task conversation from ``/threads/search``.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
table_name = "threads_meta"
|
||||
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
|
||||
try:
|
||||
inspector = sa_inspect(sync_conn)
|
||||
if table_name not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns(table_name)}
|
||||
if not required_columns <= columns:
|
||||
return
|
||||
|
||||
dialect = sync_conn.dialect.name
|
||||
if dialect == "sqlite":
|
||||
task_id = "json_extract(metadata_json, '$.taskId')"
|
||||
agent_id = "json_extract(metadata_json, '$.agent_id')"
|
||||
action_id = "json_extract(metadata_json, '$.actionId')"
|
||||
valid_json = "json_valid(metadata_json)"
|
||||
elif dialect == "mysql":
|
||||
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
|
||||
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
|
||||
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
|
||||
valid_json = "JSON_VALID(metadata_json)"
|
||||
elif dialect in {"postgresql", "opengauss"}:
|
||||
task_id = "metadata_json ->> 'taskId'"
|
||||
agent_id = "metadata_json ->> 'agent_id'"
|
||||
action_id = "metadata_json ->> 'actionId'"
|
||||
valid_json = "TRUE"
|
||||
else:
|
||||
return
|
||||
|
||||
result = sync_conn.execute(
|
||||
text(
|
||||
"UPDATE threads_meta "
|
||||
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
|
||||
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
|
||||
)
|
||||
)
|
||||
if result.rowcount:
|
||||
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
|
||||
except Exception:
|
||||
# A malformed legacy JSON row or a backend-specific JSON function must
|
||||
# not prevent startup. The generic metadata search remains available,
|
||||
# and the failure is visible to the operator for remediation.
|
||||
logger.exception("Failed to backfill thread task lookup projections")
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
|
||||
|
||||
def _acquire_gauss_mysql_schema_bootstrap_lock():
|
||||
"""Serialize Dolphin schema bootstrap across workers on this host."""
|
||||
|
||||
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
|
||||
|
||||
if not is_gauss_mysql_product():
|
||||
return None
|
||||
try:
|
||||
import fcntl
|
||||
except ImportError:
|
||||
# Production offline deployments are Linux. Keeping a no-lock fallback
|
||||
# makes imports and unit tests work on Windows without changing MySQL.
|
||||
logger.warning("fcntl is unavailable; openGauss schema bootstrap is not cross-process locked")
|
||||
return None
|
||||
|
||||
try:
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
home = runtime_home()
|
||||
except Exception:
|
||||
home = Path(os.getenv("DEER_FLOW_HOME", ".deer-flow")).resolve()
|
||||
lock_path = home / "data" / "gauss-schema-bootstrap.lock"
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = lock_path.open("a+", encoding="utf-8")
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||
logger.info("Acquired openGauss schema bootstrap lock: %s", lock_path)
|
||||
return handle
|
||||
|
||||
|
||||
def _release_gauss_mysql_schema_bootstrap_lock(handle) -> None: # noqa: ANN001
|
||||
if handle is None:
|
||||
return
|
||||
import fcntl
|
||||
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
finally:
|
||||
handle.close()
|
||||
|
||||
|
||||
async def init_engine(
|
||||
backend: str,
|
||||
*,
|
||||
@@ -401,28 +627,40 @@ async def init_engine(
|
||||
pool_size: int = 5,
|
||||
max_overflow: int = 10,
|
||||
pool_timeout: int = 30,
|
||||
pool_recycle: int = 1800,
|
||||
pool_recycle: int | None = None,
|
||||
command_timeout: float | None = 30,
|
||||
sqlite_dir: str = "",
|
||||
) -> None:
|
||||
"""Create the async engine and session factory, then auto-create tables.
|
||||
|
||||
Args:
|
||||
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
|
||||
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
|
||||
echo: Echo SQL to log.
|
||||
pool_size: Base connection pool size (postgres/gauss/mysql).
|
||||
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
|
||||
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
|
||||
pool_size: Base connection pool size (postgres/gauss/mysql).
|
||||
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
|
||||
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
|
||||
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
|
||||
command_timeout: asyncpg command deadline, including pool pre-ping (None disables).
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
"""
|
||||
global _engine, _session_factory, _engine_loop
|
||||
|
||||
if pool_recycle is None:
|
||||
pool_recycle = 300 if backend == "postgres" else 1800
|
||||
|
||||
if pool_recycle is None:
|
||||
pool_recycle = 300 if backend == "postgres" else 1800
|
||||
|
||||
if backend == "memory":
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
|
||||
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
|
||||
|
||||
gauss_mysql = backend == "mysql" and is_gauss_mysql_product()
|
||||
|
||||
# Remember the loop that owns the engine's connection pool, so off-loop
|
||||
# best-effort writes (metrics from worker threads, etc.) can be routed here
|
||||
# instead of reusing pooled connections cross-loop.
|
||||
@@ -431,25 +669,25 @@ async def init_engine(
|
||||
except RuntimeError:
|
||||
_engine_loop = None
|
||||
|
||||
if backend in {"postgres", "gauss"}:
|
||||
if backend in {"postgres", "gauss"}:
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
extra = "gauss" if backend == "gauss" else "postgres"
|
||||
raise ImportError(
|
||||
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
|
||||
f"Install it with:\n uv sync --extra {extra}\n"
|
||||
"Or switch to backend: sqlite in config.yaml for single-node deployment."
|
||||
) from None
|
||||
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
|
||||
"is not installed.\nInstall it with:\n uv sync --extra gauss"
|
||||
)
|
||||
if backend == "gauss":
|
||||
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
|
||||
|
||||
install_gauss_dialect_compatibility()
|
||||
extra = "gauss" if backend == "gauss" else "postgres"
|
||||
raise ImportError(
|
||||
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
|
||||
f"Install it with:\n uv sync --extra {extra}\n"
|
||||
"Or switch to backend: sqlite in config.yaml for single-node deployment."
|
||||
) from None
|
||||
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
|
||||
"is not installed.\nInstall it with:\n uv sync --extra gauss"
|
||||
)
|
||||
if backend == "gauss":
|
||||
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
|
||||
|
||||
install_gauss_dialect_compatibility()
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
@@ -459,12 +697,12 @@ async def init_engine(
|
||||
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
|
||||
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
|
||||
)
|
||||
if gauss_mysql:
|
||||
from deerflow.persistence.gauss_mysql_compat import install_gauss_mysql_dialect_compatibility
|
||||
|
||||
install_gauss_mysql_dialect_compatibility(driver)
|
||||
|
||||
if backend == "sqlite":
|
||||
import os
|
||||
|
||||
from sqlalchemy import event
|
||||
|
||||
os.makedirs(sqlite_dir or ".", exist_ok=True)
|
||||
_engine = create_async_engine(url, echo=echo, json_serializer=_json_serializer)
|
||||
|
||||
@@ -489,31 +727,28 @@ async def init_engine(
|
||||
cursor.execute("PRAGMA foreign_keys=ON;")
|
||||
finally:
|
||||
cursor.close()
|
||||
elif backend in {"postgres", "gauss"}:
|
||||
elif backend in {"postgres", "gauss"}:
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
)
|
||||
elif backend == "mysql":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
|
||||
_install_slow_sql_logging(_engine, backend)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
# Auto-create tables for fresh deployments, then let Alembic apply any
|
||||
@@ -530,16 +765,32 @@ async def init_engine(
|
||||
# This is expected during initial scaffolding or minimal installs.
|
||||
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
|
||||
|
||||
async def _create_all_with_product_integrity(conn) -> None: # noqa: ANN001
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
if gauss_mysql:
|
||||
from deerflow.persistence.gauss_schema import ensure_gauss_schema_sync
|
||||
|
||||
await conn.run_sync(ensure_gauss_schema_sync, Base.metadata)
|
||||
|
||||
# Schema bootstrap is deliberately best-effort. A single incompatible or
|
||||
# permission-restricted table must not make the whole Gateway unavailable:
|
||||
# routes backed by healthy existing tables can still work, while a route
|
||||
# whose table is missing will report its own request-time error. Keep the
|
||||
# auto-create-database retries, but contain failures from both the initial
|
||||
# create_all and the retry path.
|
||||
schema_bootstrap_lock = None
|
||||
if gauss_mysql:
|
||||
try:
|
||||
schema_bootstrap_lock = _acquire_gauss_mysql_schema_bootstrap_lock()
|
||||
except Exception:
|
||||
# A lock-file permission problem is operationally important, but
|
||||
# the global startup invariant still requires the Gateway to come
|
||||
# up and report schema errors at the affected routes.
|
||||
logger.exception("Could not acquire openGauss schema bootstrap lock; continuing without the host lock")
|
||||
try:
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
await _create_all_with_product_integrity(conn)
|
||||
except Exception as exc:
|
||||
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
|
||||
# Database not yet created — attempt to auto-create it, then retry.
|
||||
@@ -548,35 +799,33 @@ async def init_engine(
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
)
|
||||
_install_slow_sql_logging(_engine, backend)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
|
||||
await _create_all_with_product_integrity(conn)
|
||||
elif backend == "mysql" and not gauss_mysql and _is_mysql_unknown_database_error(exc):
|
||||
# Database not yet created. Create it, rebuild the app engine
|
||||
# against the target DB, then let SQLAlchemy create all tables.
|
||||
await _auto_create_mysql_db(url)
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
)
|
||||
_install_slow_sql_logging(_engine, backend)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
await _create_all_with_product_integrity(conn)
|
||||
else:
|
||||
raise
|
||||
except Exception:
|
||||
@@ -584,22 +833,34 @@ async def init_engine(
|
||||
"Automatic ORM schema creation failed; continuing Gateway startup "
|
||||
"with the existing database schema"
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
_release_gauss_mysql_schema_bootstrap_lock(schema_bootstrap_lock)
|
||||
except Exception:
|
||||
logger.exception("Could not release openGauss schema bootstrap lock")
|
||||
|
||||
# Patch existing tables with ORM-declared columns that ``create_all``
|
||||
# cannot add (it only creates brand-new tables). Best-effort: a failure is
|
||||
# logged and startup continues on the current schema.
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(_ensure_orm_columns_sync)
|
||||
await conn.run_sync(_backfill_thread_meta_task_projections_sync)
|
||||
except Exception:
|
||||
logger.exception("Auto-updating ORM columns or task lookup projections failed; continuing with current schema")
|
||||
if gauss_mysql:
|
||||
# Dolphin returns PostgreSQL-style DDL from SHOW CREATE TABLE. The
|
||||
# stock MySQL inspector misreads it and may issue duplicate ALTERs,
|
||||
# aborting the transaction with error 322. The native-catalog helper
|
||||
# above is the only schema repair path for this product.
|
||||
logger.info("Skipping MySQL reflection and JSON backfill for openGauss Dolphin")
|
||||
else:
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(_ensure_orm_columns_sync)
|
||||
await conn.run_sync(_backfill_thread_meta_task_projections_sync)
|
||||
except Exception:
|
||||
logger.exception("Auto-updating ORM columns or task lookup projections failed; continuing with current schema")
|
||||
|
||||
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
# await _run_alembic_upgrade(url)
|
||||
|
||||
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
|
||||
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
|
||||
|
||||
|
||||
async def init_engine_from_config(config) -> None:
|
||||
@@ -614,14 +875,17 @@ async def init_engine_from_config(config) -> None:
|
||||
pool_size=config.pool_size,
|
||||
max_overflow=getattr(config, "max_overflow", 10),
|
||||
pool_timeout=getattr(config, "pool_timeout", 30),
|
||||
pool_recycle=getattr(config, "pool_recycle", 1800),
|
||||
pool_recycle=getattr(config, "pool_recycle", None),
|
||||
command_timeout=getattr(config, "command_timeout", 30),
|
||||
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
|
||||
)
|
||||
|
||||
|
||||
def get_session_factory() -> async_sessionmaker[AsyncSession] | None:
|
||||
"""Return the async session factory, or None if backend=memory."""
|
||||
return _session_factory
|
||||
from deerflow.persistence.loop_isolation import isolated_session_factory
|
||||
|
||||
return isolated_session_factory() or _session_factory
|
||||
|
||||
|
||||
def get_engine() -> AsyncEngine | None:
|
||||
@@ -629,73 +893,73 @@ def get_engine() -> AsyncEngine | None:
|
||||
return _engine
|
||||
|
||||
|
||||
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
|
||||
"""Return the event loop that owns the engine's connection pool (or None)."""
|
||||
return _engine_loop
|
||||
|
||||
|
||||
async def rebuild_workflow_schema_for_mysql() -> list[str]:
|
||||
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
|
||||
|
||||
This is a deliberately narrow recovery operation for an unused Workflow
|
||||
Studio installation. It never uses a prefix wildcard, never changes the
|
||||
selected database, and refuses to run unless the engine is connected to
|
||||
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
|
||||
|
||||
Returns the allow-listed workflow tables that existed before the rebuild.
|
||||
"""
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
|
||||
if engine.dialect.name != "mysql":
|
||||
raise RuntimeError(
|
||||
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
|
||||
)
|
||||
|
||||
expected_database = engine.url.database
|
||||
if not expected_database:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
|
||||
|
||||
# Make the operation safe when this helper is invoked independently from
|
||||
# normal app startup as well.
|
||||
import deerflow.persistence.models # noqa: F401 — register ORM models
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
|
||||
if missing_models:
|
||||
raise RuntimeError(
|
||||
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
|
||||
)
|
||||
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
|
||||
|
||||
def _rebuild(sync_conn) -> list[str]:
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
|
||||
if current_database != expected_database:
|
||||
raise RuntimeError(
|
||||
"Refusing workflow schema rebuild: connected database "
|
||||
f"{current_database!r} does not match configured database {expected_database!r}"
|
||||
)
|
||||
|
||||
existing = set(sa_inspect(sync_conn).get_table_names())
|
||||
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
|
||||
# SQLAlchemy orders parent/child tables around the foreign key from
|
||||
# workflow_versions to workflow_definitions. Only tables in the exact
|
||||
# allow-list above are ever passed to drop_all/create_all.
|
||||
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
return present
|
||||
|
||||
async with engine.begin() as conn:
|
||||
rebuilt = await conn.run_sync(_rebuild)
|
||||
|
||||
logger.warning(
|
||||
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
|
||||
expected_database,
|
||||
", ".join(rebuilt) if rebuilt else "none (created fresh)",
|
||||
)
|
||||
return rebuilt
|
||||
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
|
||||
"""Return the event loop that owns the engine's connection pool (or None)."""
|
||||
return _engine_loop
|
||||
|
||||
|
||||
async def rebuild_workflow_schema_for_mysql() -> list[str]:
|
||||
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
|
||||
|
||||
This is a deliberately narrow recovery operation for an unused Workflow
|
||||
Studio installation. It never uses a prefix wildcard, never changes the
|
||||
selected database, and refuses to run unless the engine is connected to
|
||||
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
|
||||
|
||||
Returns the allow-listed workflow tables that existed before the rebuild.
|
||||
"""
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
|
||||
if engine.dialect.name != "mysql":
|
||||
raise RuntimeError(
|
||||
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
|
||||
)
|
||||
|
||||
expected_database = engine.url.database
|
||||
if not expected_database:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
|
||||
|
||||
# Make the operation safe when this helper is invoked independently from
|
||||
# normal app startup as well.
|
||||
import deerflow.persistence.models # noqa: F401 — register ORM models
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
|
||||
if missing_models:
|
||||
raise RuntimeError(
|
||||
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
|
||||
)
|
||||
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
|
||||
|
||||
def _rebuild(sync_conn) -> list[str]:
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
|
||||
if current_database != expected_database:
|
||||
raise RuntimeError(
|
||||
"Refusing workflow schema rebuild: connected database "
|
||||
f"{current_database!r} does not match configured database {expected_database!r}"
|
||||
)
|
||||
|
||||
existing = set(sa_inspect(sync_conn).get_table_names())
|
||||
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
|
||||
# SQLAlchemy orders parent/child tables around the foreign key from
|
||||
# workflow_versions to workflow_definitions. Only tables in the exact
|
||||
# allow-list above are ever passed to drop_all/create_all.
|
||||
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
return present
|
||||
|
||||
async with engine.begin() as conn:
|
||||
rebuilt = await conn.run_sync(_rebuild)
|
||||
|
||||
logger.warning(
|
||||
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
|
||||
expected_database,
|
||||
", ".join(rebuilt) if rebuilt else "none (created fresh)",
|
||||
)
|
||||
return rebuilt
|
||||
|
||||
|
||||
def fire_and_forget_db_write(coro) -> None:
|
||||
@@ -744,10 +1008,13 @@ def _run_coro_on_throwaway_loop(coro):
|
||||
|
||||
Only reached when there is no usable engine home loop, or when the caller is
|
||||
*synchronously* inside the home loop (routing back would deadlock). This
|
||||
touches the shared connection pool from a throwaway loop, so it can still leak
|
||||
a dead-loop-bound connection — but the common worker-thread path no longer
|
||||
gets here (see ``run_db_blocking``).
|
||||
uses an isolated, non-pooled engine for compatibility reads. The coroutine
|
||||
must obtain its session factory inside its async body so the isolated
|
||||
context applies; no connection is returned to the Gateway pool.
|
||||
"""
|
||||
from deerflow.persistence.loop_isolation import run_isolated_db_read
|
||||
|
||||
coro = run_isolated_db_read(coro)
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
"""Compatibility helpers for openGauss/Dolphin's MySQL protocol.
|
||||
|
||||
The Dolphin endpoint deliberately presents itself as MySQL, but a few pieces
|
||||
of server metadata and DDL reflection differ from real MySQL. These patches
|
||||
are installed only when the deployment explicitly selects the ``gauss_mysql``
|
||||
database product. A normal MySQL deployment never imports or installs them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
GAUSS_MYSQL_PRODUCT = "gauss_mysql"
|
||||
GAUSS_MYSQL_SQLALCHEMY_VERSION = (5, 7, 20)
|
||||
_DOLPHIN_INIT_COMMAND = "SET standard_conforming_strings TO off"
|
||||
|
||||
|
||||
def is_gauss_mysql_product() -> bool:
|
||||
"""Return whether this process was started for the Dolphin endpoint."""
|
||||
|
||||
return os.getenv("DEERFLOW_DATABASE_PRODUCT", "").strip().lower() == GAUSS_MYSQL_PRODUCT
|
||||
|
||||
|
||||
def is_gauss_mysql_dialect(dialect: object) -> bool:
|
||||
"""Return whether the selected SQLAlchemy dialect has the Dolphin hook."""
|
||||
|
||||
return bool(getattr(dialect, "_deerflow_gauss_mysql_compat_installed", False))
|
||||
|
||||
|
||||
def _decode_scalar(value: Any) -> str:
|
||||
if isinstance(value, bytes):
|
||||
return value.decode(errors="replace")
|
||||
return str(value)
|
||||
|
||||
|
||||
def _gauss_mysql_default_schema_name(self, connection): # noqa: ANN001, ARG001
|
||||
"""Return Dolphin's current schema rather than its physical database."""
|
||||
|
||||
schema = connection.exec_driver_sql("SELECT current_schema()").scalar()
|
||||
if schema is None:
|
||||
raise RuntimeError("GaussDB did not return current_schema()")
|
||||
return _decode_scalar(schema)
|
||||
|
||||
|
||||
def _gauss_mysql_server_version_info(self, connection): # noqa: ANN001
|
||||
"""Expose a stable MySQL capability version for a Dolphin connection."""
|
||||
|
||||
dbapi_connection = connection.connection
|
||||
cursor = dbapi_connection.cursor()
|
||||
try:
|
||||
cursor.execute("SELECT VERSION()")
|
||||
row = cursor.fetchone()
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
raw_version = "" if row is None or row[0] is None else _decode_scalar(row[0])
|
||||
version = GAUSS_MYSQL_SQLALCHEMY_VERSION
|
||||
self._deerflow_gauss_server_version = raw_version
|
||||
self._set_mariadb(False, version)
|
||||
self._mariadb_normalized_version_info = version
|
||||
self.server_version_info = version
|
||||
logger.info(
|
||||
"GaussDB MySQL compatibility enabled: server=%r, SQLAlchemy capability=%s",
|
||||
raw_version,
|
||||
".".join(map(str, version)),
|
||||
)
|
||||
return version
|
||||
|
||||
|
||||
def _gauss_mysql_isolation_level(self, dbapi_connection): # noqa: ANN001, ARG001
|
||||
"""Read isolation without comparing an unparseable vendor version."""
|
||||
|
||||
cursor = dbapi_connection.cursor()
|
||||
try:
|
||||
cursor.execute("SELECT @@transaction_isolation")
|
||||
row = cursor.fetchone()
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
if row is None or row[0] is None:
|
||||
raise NotImplementedError("GaussDB did not return a transaction isolation level")
|
||||
return _decode_scalar(row[0]).upper().replace("-", " ")
|
||||
|
||||
|
||||
def _gauss_mysql_has_table(self, connection, table_name, schema=None, **kw): # noqa: ANN001, ARG001
|
||||
"""Check relation existence without a failing ``DESCRIBE`` transaction."""
|
||||
|
||||
self._ensure_has_table_connection(connection)
|
||||
if schema is None:
|
||||
schema = self.default_schema_name
|
||||
if schema is None:
|
||||
raise ValueError("GaussDB MySQL table check requires a database name")
|
||||
|
||||
statement = "SELECT 1 FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE n.nspname = %s AND c.relname = %s AND c.relkind IN ('r', 'p', 'v', 'm', 'f') LIMIT 1"
|
||||
with connection.exec_driver_sql(
|
||||
statement,
|
||||
(schema, table_name),
|
||||
execution_options={"skip_user_error_events": True},
|
||||
) as result:
|
||||
return result.fetchone() is not None
|
||||
|
||||
|
||||
def _gauss_mysql_create_connect_args(self, url): # noqa: ANN001
|
||||
"""Add the Dolphin session setting required by PyMySQL-compatible drivers."""
|
||||
|
||||
args, kwargs = self._deerflow_gauss_mysql_original_create_connect_args(url)
|
||||
existing = kwargs.get("init_command")
|
||||
if existing and existing.strip().rstrip(";").lower() != _DOLPHIN_INIT_COMMAND.lower():
|
||||
raise ValueError(
|
||||
"gauss_mysql requires init_command='SET standard_conforming_strings TO off'; "
|
||||
"remove the conflicting init_command from the database URL"
|
||||
)
|
||||
kwargs["init_command"] = _DOLPHIN_INIT_COMMAND
|
||||
return args, kwargs
|
||||
|
||||
|
||||
def _unescape_gauss_mysql_json_container(value: str) -> Any:
|
||||
"""Undo exactly one complete MySQL SQL-string escape layer."""
|
||||
|
||||
text = value.strip()
|
||||
if not text.startswith(("[", "{")) or "\\" not in text:
|
||||
raise ValueError("Not an escaped JSON container")
|
||||
escapes = {
|
||||
"0": "\x00",
|
||||
"n": "\n",
|
||||
"r": "\r",
|
||||
"Z": "\x1a",
|
||||
"\\": "\\",
|
||||
"'": "'",
|
||||
'"': '"',
|
||||
}
|
||||
output: list[str] = []
|
||||
position = 0
|
||||
while position < len(text):
|
||||
character = text[position]
|
||||
if character == "\\":
|
||||
position += 1
|
||||
if position == len(text) or text[position] not in escapes:
|
||||
raise ValueError("Unknown or incomplete MySQL escape")
|
||||
output.append(escapes[text[position]])
|
||||
elif character in {'"', "'"}:
|
||||
raise ValueError("Mixed escaped and unescaped quotes")
|
||||
else:
|
||||
output.append(character)
|
||||
position += 1
|
||||
decoded = json.loads("".join(output))
|
||||
if not isinstance(decoded, (dict, list)):
|
||||
raise ValueError("Escaped legacy JSON must be an object or array")
|
||||
return decoded
|
||||
|
||||
|
||||
def _is_json_domain(value: Any) -> bool:
|
||||
"""Recursively reject Python-only values accepted by ``literal_eval``."""
|
||||
|
||||
if value is None or isinstance(value, (str, bool, int)):
|
||||
return True
|
||||
if isinstance(value, float):
|
||||
return math.isfinite(value)
|
||||
if isinstance(value, list):
|
||||
return all(_is_json_domain(item) for item in value)
|
||||
if isinstance(value, dict):
|
||||
return all(isinstance(key, str) and _is_json_domain(item) for key, item in value.items())
|
||||
return False
|
||||
|
||||
|
||||
def decode_gauss_mysql_json(value: Any) -> Any:
|
||||
"""Decode strict JSON plus the two legacy Dolphin encodings we shipped."""
|
||||
|
||||
if isinstance(value, memoryview):
|
||||
value = value.tobytes()
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8")
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
try:
|
||||
decoded = ast.literal_eval(value)
|
||||
except (SyntaxError, ValueError):
|
||||
try:
|
||||
return _unescape_gauss_mysql_json_container(value)
|
||||
except (TypeError, ValueError):
|
||||
preview = value[:160].replace("\n", "\\n")
|
||||
raise ValueError(f"GaussDB MySQL JSON value is neither valid JSON, a supported legacy literal, nor a single SQL-escaped JSON container: {preview!r}") from None
|
||||
if not _is_json_domain(decoded):
|
||||
raise ValueError(f"GaussDB MySQL JSON legacy literal decoded to an unsupported type: {type(decoded).__name__}")
|
||||
return decoded
|
||||
|
||||
|
||||
def install_gauss_mysql_dialect_compatibility(driver: str = "aiomysql") -> None:
|
||||
"""Patch only the selected async MySQL dialect class, once per process."""
|
||||
|
||||
if driver == "aiomysql":
|
||||
from sqlalchemy.dialects.mysql.aiomysql import MySQLDialect_aiomysql as dialect_class
|
||||
elif driver == "asyncmy":
|
||||
from sqlalchemy.dialects.mysql.asyncmy import MySQLDialect_asyncmy as dialect_class
|
||||
else:
|
||||
raise ValueError(f"Unsupported GaussDB MySQL driver: {driver}")
|
||||
|
||||
if getattr(dialect_class, "_deerflow_gauss_mysql_compat_installed", False):
|
||||
return
|
||||
|
||||
dialect_class._deerflow_gauss_mysql_original_create_connect_args = dialect_class.create_connect_args
|
||||
dialect_class.create_connect_args = _gauss_mysql_create_connect_args
|
||||
dialect_class._get_server_version_info = _gauss_mysql_server_version_info
|
||||
dialect_class._get_default_schema_name = _gauss_mysql_default_schema_name
|
||||
dialect_class.get_isolation_level = _gauss_mysql_isolation_level
|
||||
dialect_class.has_table = _gauss_mysql_has_table
|
||||
dialect_class._deerflow_gauss_mysql_compat_installed = True
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
"""Conservatively extend ORM tables through openGauss native catalogs.
|
||||
|
||||
This is used only through Dolphin's MySQL protocol. The stock MySQL inspector
|
||||
cannot parse Dolphin's PostgreSQL-style ``SHOW CREATE TABLE`` result and can
|
||||
therefore report existing columns as missing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import CheckConstraint, ForeignKeyConstraint, MetaData, UniqueConstraint
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.schema import CreateColumn, CreateIndex
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _string(value: object) -> str:
|
||||
return value.decode("utf-8") if isinstance(value, bytes) else str(value)
|
||||
|
||||
|
||||
def ensure_gauss_schema_sync(connection: Connection, metadata: MetaData) -> None:
|
||||
"""Add only safe missing columns and indexes involving those columns.
|
||||
|
||||
Existing columns and unrelated indexes are never altered. A missing
|
||||
column without a nullable definition or database-side default cannot be
|
||||
backfilled reliably and stops this repair before any DDL is issued.
|
||||
"""
|
||||
|
||||
current_schema_value = connection.exec_driver_sql("SELECT current_schema()").scalar_one_or_none()
|
||||
if not current_schema_value:
|
||||
raise RuntimeError("openGauss schema integrity check: current_schema() is empty")
|
||||
schema = _string(current_schema_value)
|
||||
expected_schema = connection.engine.url.database
|
||||
if expected_schema and expected_schema != schema:
|
||||
raise RuntimeError(f"openGauss schema integrity check: URL schema {expected_schema!r} does not match current_schema() {schema!r}")
|
||||
|
||||
catalog_rows = connection.exec_driver_sql(
|
||||
"SELECT c.relname, c.relkind, a.attname "
|
||||
"FROM pg_catalog.pg_class AS c "
|
||||
"JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace "
|
||||
"LEFT JOIN pg_catalog.pg_attribute AS a ON a.attrelid = c.oid "
|
||||
"AND a.attnum > 0 AND NOT a.attisdropped "
|
||||
"WHERE n.nspname = %s AND c.relkind IN ('r', 'p', 'v', 'm', 'f')",
|
||||
(schema,),
|
||||
).fetchall()
|
||||
columns_by_table: dict[str, set[str]] = {}
|
||||
kinds_by_table: dict[str, str] = {}
|
||||
for table_name, kind, column_name in catalog_rows:
|
||||
table_name = _string(table_name)
|
||||
kinds_by_table[table_name] = _string(kind)
|
||||
columns_by_table.setdefault(table_name, set())
|
||||
if column_name is not None:
|
||||
columns_by_table[table_name].add(_string(column_name))
|
||||
|
||||
plan = []
|
||||
failures: list[str] = []
|
||||
for table in metadata.sorted_tables:
|
||||
if table.schema is not None and table.schema != schema:
|
||||
failures.append(f"{table.fullname}: explicit schema differs from {schema!r}")
|
||||
continue
|
||||
existing = columns_by_table.get(table.name)
|
||||
if existing is None:
|
||||
failures.append(f"{schema}.{table.name}: table is still missing after create_all")
|
||||
continue
|
||||
if kinds_by_table[table.name] not in {"r", "p"}:
|
||||
failures.append(f"{schema}.{table.name}: existing relation is not an ordinary table")
|
||||
continue
|
||||
missing = [column for column in table.columns if column.name not in existing]
|
||||
if not missing:
|
||||
continue
|
||||
missing_names = {column.name for column in missing}
|
||||
for column in missing:
|
||||
label = f"{schema}.{table.name}.{column.name}"
|
||||
if not column.nullable and column.server_default is None:
|
||||
failures.append(f"{label}: NOT NULL column has no server_default; explicit migration required")
|
||||
elif column.primary_key or column.identity is not None or column.computed is not None:
|
||||
failures.append(f"{label}: primary-key/identity/computed column requires an explicit migration")
|
||||
for constraint in table.constraints:
|
||||
if isinstance(constraint, (ForeignKeyConstraint, UniqueConstraint)):
|
||||
constrained = {column.name for column in constraint.columns}
|
||||
if constrained & missing_names:
|
||||
failures.append(f"{schema}.{table.name}: missing columns participate in {type(constraint).__name__}; explicit constraint migration required")
|
||||
elif isinstance(constraint, CheckConstraint):
|
||||
constrained = {column.name for column in constraint.columns}
|
||||
if not constrained or constrained & missing_names:
|
||||
failures.append(f"{schema}.{table.name}: CHECK-constrained schema needs an explicit migration")
|
||||
indexes = [index for index in table.indexes if {column.name for column in index.columns} & missing_names]
|
||||
for index in indexes:
|
||||
if not index.name:
|
||||
failures.append(f"{schema}.{table.name}: index involving new columns has no stable name")
|
||||
plan.append((table, missing, indexes))
|
||||
|
||||
if failures:
|
||||
raise RuntimeError("openGauss schema is incomplete; startup repair stopped:\n- " + "\n- ".join(failures))
|
||||
if not plan:
|
||||
logger.info("openGauss schema integrity check passed: %s ORM tables", len(metadata.tables))
|
||||
return
|
||||
|
||||
existing_indexes = {
|
||||
(_string(table_name), _string(index_name))
|
||||
for table_name, index_name in connection.exec_driver_sql(
|
||||
"SELECT t.relname, i.relname FROM pg_catalog.pg_index AS x "
|
||||
"JOIN pg_catalog.pg_class AS t ON t.oid = x.indrelid "
|
||||
"JOIN pg_catalog.pg_class AS i ON i.oid = x.indexrelid "
|
||||
"JOIN pg_catalog.pg_namespace AS n ON n.oid = t.relnamespace "
|
||||
"WHERE n.nspname = %s",
|
||||
(schema,),
|
||||
).fetchall()
|
||||
}
|
||||
preparer = connection.dialect.identifier_preparer
|
||||
added_columns = 0
|
||||
for table, missing, indexes in plan:
|
||||
qualified = f"{preparer.quote_schema(schema)}.{preparer.quote(table.name)}"
|
||||
for column in missing:
|
||||
definition = str(CreateColumn(column).compile(dialect=connection.dialect))
|
||||
connection.exec_driver_sql(f"ALTER TABLE {qualified} ADD COLUMN {definition}")
|
||||
added_columns += 1
|
||||
logger.info("Added safe missing openGauss column: %s.%s", qualified, column.name)
|
||||
for index in sorted(indexes, key=lambda item: str(item.name)):
|
||||
if (table.name, str(index.name)) not in existing_indexes:
|
||||
connection.execute(CreateIndex(index))
|
||||
logger.info("Created openGauss index involving new columns: %s", index.name)
|
||||
logger.info("openGauss schema integrity repair completed: %s columns", added_columns)
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
"""Bound JSON predicates for SQLite, PostgreSQL and MySQL.
|
||||
|
||||
Adapted from upstream #2865. MySQL business tables store JSON in TEXT, so
|
||||
these predicates use JSON functions rather than assuming a native JSON column.
|
||||
Unsupported keys/values must stay in the repository's residual filter; never
|
||||
silently discard a predicate (in particular a shared task's taskId filter).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import BigInteger, Float, String, bindparam
|
||||
from sqlalchemy.ext.compiler import compiles
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.sql.visitors import InternalTraversal
|
||||
from sqlalchemy.types import Boolean
|
||||
|
||||
_SAFE_KEY = re.compile(r"[A-Za-z0-9_-]+\Z")
|
||||
_STRIP_CHARS = "".join(chr(i) for i in range(0x3001) if chr(i).isspace())
|
||||
|
||||
|
||||
def supports_json_filter(key: Any, value: Any) -> bool:
|
||||
if not isinstance(key, str) or not _SAFE_KEY.fullmatch(key):
|
||||
return False
|
||||
if value is None or isinstance(value, (str, bool)):
|
||||
return True
|
||||
if isinstance(value, int):
|
||||
return -(2**63) <= value < 2**63
|
||||
return isinstance(value, float) and math.isfinite(value)
|
||||
|
||||
|
||||
class JsonPredicate(ColumnElement):
|
||||
"""A scalar equality, Python truthiness, or nonblank-string predicate."""
|
||||
|
||||
inherit_cache = True
|
||||
type = Boolean()
|
||||
_is_implicitly_boolean = True
|
||||
_traverse_internals = [
|
||||
("column", InternalTraversal.dp_clauseelement),
|
||||
("key", InternalTraversal.dp_string),
|
||||
("operation", InternalTraversal.dp_string),
|
||||
("value", InternalTraversal.dp_plain_obj),
|
||||
("value_kind", InternalTraversal.dp_string),
|
||||
]
|
||||
|
||||
def __init__(self, column, key: str, operation: str = "equal", value: Any = None):
|
||||
if not supports_json_filter(key, value):
|
||||
raise ValueError("Unsupported JSON filter key or value")
|
||||
if operation not in {"equal", "truthy", "nonblank"}:
|
||||
raise ValueError("Unsupported JSON predicate operation")
|
||||
self.column = column
|
||||
self.key = key
|
||||
self.operation = operation
|
||||
self.value = value
|
||||
# Keep Python value types separate in SQLAlchemy's compiled cache.
|
||||
self.value_kind = type(value).__name__
|
||||
super().__init__()
|
||||
|
||||
|
||||
def _compile(element, compiler, family, **kw):
|
||||
if not _SAFE_KEY.fullmatch(element.key):
|
||||
raise ValueError("Unsafe JSON key")
|
||||
column = compiler.process(element.column, **kw)
|
||||
|
||||
def bound(value, type_=String()):
|
||||
return compiler.process(bindparam(None, value, type_=type_), **kw)
|
||||
|
||||
if family == "postgres":
|
||||
raw = f"({column} -> '{element.key}')"
|
||||
kind = f"json_typeof({raw})"
|
||||
text = f"({column} ->> '{element.key}')"
|
||||
string_kind, number_kinds = "string", "'number'"
|
||||
else:
|
||||
path = f"'$.\"{element.key}\"'"
|
||||
raw = f"json_extract({column}, {path})"
|
||||
if family == "mysql":
|
||||
kind, text = f"lower(json_type({raw}))", f"json_unquote({raw})"
|
||||
string_kind, number_kinds = "string", "'integer', 'double', 'decimal'"
|
||||
else:
|
||||
kind, text = f"json_type({column}, {path})", raw
|
||||
string_kind, number_kinds = "text", "'integer', 'real'"
|
||||
|
||||
if element.operation == "nonblank":
|
||||
if family == "mysql":
|
||||
# MySQL 5.7 regexes are byte-oriented; a Unicode whitespace regex
|
||||
# can misclassify CJK text. Removing each complete whitespace
|
||||
# character preserves str.strip()'s nonempty test on 5.7 and 8.x.
|
||||
stripped = text
|
||||
for char in _STRIP_CHARS:
|
||||
stripped = f"replace({stripped}, {bound(char)}, '')"
|
||||
has_text = f"length({stripped}) > 0"
|
||||
else:
|
||||
trim = "btrim" if family == "postgres" else "trim"
|
||||
# SQLite length(TEXT) stops at NUL, unlike Python str.strip().
|
||||
has_text = f"{trim}({text}, {bound(_STRIP_CHARS)}) <> ''"
|
||||
clause = f"({kind} = '{string_kind}' AND {has_text})"
|
||||
elif element.operation == "truthy":
|
||||
if family == "sqlite":
|
||||
containers = f"json({raw}) NOT IN ('[]', '{{}}')"
|
||||
boolean = f"{kind} = 'true'"
|
||||
elif family == "mysql":
|
||||
containers = f"json_length({raw}) > 0"
|
||||
boolean = f"{text} = 'true'"
|
||||
else:
|
||||
containers = f"CAST({raw} AS jsonb) NOT IN ('[]'::jsonb, '{{}}'::jsonb)"
|
||||
boolean = f"{text} = 'true'"
|
||||
cast = "NUMERIC" if family == "postgres" else "DOUBLE PRECISION"
|
||||
number = f"({text} + 0e0)" if family == "mysql" else f"CAST({text} AS {cast})"
|
||||
nonempty = f"{text} <> ''" if family == "sqlite" else f"length({text}) > 0"
|
||||
clause = (
|
||||
f"CASE WHEN {kind} IN ({number_kinds}) THEN {number} <> 0 "
|
||||
f"WHEN {kind} = '{string_kind}' THEN {nonempty} "
|
||||
f"WHEN {kind} IN ('array', 'object') THEN {containers} "
|
||||
f"WHEN {kind} IN ('boolean', 'true', 'false') THEN {boolean} ELSE FALSE END"
|
||||
)
|
||||
elif element.value is None:
|
||||
# Preserve this project's md.get(key) == None semantics for old rows.
|
||||
clause = f"({kind} IS NULL OR {kind} = 'null')"
|
||||
elif isinstance(element.value, (bool, int, float)):
|
||||
if family != "sqlite":
|
||||
raise NotImplementedError("Server numeric metadata requires the exact residual filter")
|
||||
value = int(element.value) if isinstance(element.value, bool) else element.value
|
||||
# Preserve md.get(key) == value: Python considers True == 1 == 1.0.
|
||||
# Text ids remain distinct from numbers. SQLite compares its extracted
|
||||
# integers/reals natively; server numeric filters use the residual path.
|
||||
param = bound(value, Float() if isinstance(value, float) else BigInteger())
|
||||
clause = f"({kind} IN ({number_kinds}, 'true', 'false') AND {text} = {param})"
|
||||
else:
|
||||
# Binary comparison is essential for task/agent ids: MySQL TEXT can
|
||||
# inherit a case-insensitive collation and ignores trailing spaces.
|
||||
extract = f"CAST({text} AS BINARY)" if family == "mysql" else text
|
||||
param = bound(element.value)
|
||||
if family == "mysql":
|
||||
param = f"CAST({param} AS BINARY)"
|
||||
clause = f"({kind} = '{string_kind}' AND {extract} = {param})"
|
||||
# Missing/null values must remain false under NOT (system exclusions).
|
||||
return f"COALESCE(({clause}), FALSE)"
|
||||
|
||||
|
||||
@compiles(JsonPredicate, "sqlite")
|
||||
def _sqlite(element, compiler, **kw):
|
||||
return _compile(element, compiler, "sqlite", **kw)
|
||||
|
||||
|
||||
@compiles(JsonPredicate, "postgresql")
|
||||
def _postgres(element, compiler, **kw):
|
||||
return _compile(element, compiler, "postgres", **kw)
|
||||
|
||||
|
||||
@compiles(JsonPredicate, "mysql")
|
||||
def _mysql(element, compiler, **kw):
|
||||
return _compile(element, compiler, "mysql", **kw)
|
||||
|
||||
|
||||
@compiles(JsonPredicate)
|
||||
def _unsupported(element, compiler, **kw):
|
||||
raise NotImplementedError(f"JSON predicates unsupported for {compiler.dialect.name}")
|
||||
+437
@@ -0,0 +1,437 @@
|
||||
"""Private knowledge-base Q&A sessions with shared Markdown publications.
|
||||
|
||||
This persistence unit deliberately contains no Gateway/application imports. A
|
||||
session and its LangGraph thread belong to exactly one user. Once published,
|
||||
only the saved Markdown snapshot is shared with readers of the bound knowledge
|
||||
base; the private draft and thread identifier never become publication data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import String, delete, select, update
|
||||
from sqlalchemy.exc import OperationalError, ProgrammingError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class KnowledgeResearchSessionRow(Base):
|
||||
__tablename__ = "knowledge_research_sessions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
# A separate opaque id is the only identity exposed on the shared bookshelf.
|
||||
publication_id: Mapped[str] = mapped_column(String(36), unique=True, index=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(128), unique=True, index=True)
|
||||
knowledge_base_id: Mapped[str] = mapped_column(String(128), index=True)
|
||||
user_id: Mapped[str] = mapped_column(String(255), index=True)
|
||||
title: Mapped[str] = mapped_column(String(255))
|
||||
draft_markdown: Mapped[str] = mapped_column(PortableLongText, default="")
|
||||
|
||||
wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
wiki_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
wiki_folder_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
saved_title: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
saved_markdown: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
saved_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
saved_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
nullable=False,
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
|
||||
def _is_missing_table(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return (
|
||||
"no such table" in message
|
||||
or "doesn't exist" in message
|
||||
or "does not exist" in message
|
||||
or "unknown table" in message
|
||||
)
|
||||
|
||||
|
||||
def _iso(value: Any) -> Any:
|
||||
return value.isoformat() if isinstance(value, datetime) else value
|
||||
|
||||
|
||||
def _row_dict(row: KnowledgeResearchSessionRow | dict[str, Any]) -> dict[str, Any]:
|
||||
if isinstance(row, dict):
|
||||
return deepcopy(row)
|
||||
return {
|
||||
"id": row.id,
|
||||
"publication_id": row.publication_id,
|
||||
"thread_id": row.thread_id,
|
||||
"knowledge_base_id": row.knowledge_base_id,
|
||||
"user_id": row.user_id,
|
||||
"title": row.title,
|
||||
"draft_markdown": row.draft_markdown or "",
|
||||
"wiki_slug": row.wiki_slug,
|
||||
"wiki_page_id": row.wiki_page_id,
|
||||
"wiki_folder_id": row.wiki_folder_id,
|
||||
"saved_title": row.saved_title,
|
||||
"saved_markdown": row.saved_markdown,
|
||||
"saved_hash": row.saved_hash,
|
||||
"saved_at": _iso(row.saved_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
class _PublicationLease:
|
||||
"""One transaction-scoped mutable publication row."""
|
||||
|
||||
def __init__(self, row: KnowledgeResearchSessionRow | dict[str, Any]) -> None:
|
||||
self._row = row
|
||||
|
||||
def snapshot(self) -> dict[str, Any]:
|
||||
return _row_dict(self._row)
|
||||
|
||||
def save(
|
||||
self,
|
||||
*,
|
||||
title: str,
|
||||
markdown: str,
|
||||
saved_hash: str,
|
||||
wiki_slug: str,
|
||||
wiki_page_id: str | None,
|
||||
wiki_folder_id: str | None,
|
||||
) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
values = {
|
||||
"wiki_slug": wiki_slug,
|
||||
"wiki_page_id": wiki_page_id,
|
||||
"wiki_folder_id": wiki_folder_id,
|
||||
"saved_title": title,
|
||||
"saved_markdown": markdown,
|
||||
"saved_hash": saved_hash,
|
||||
"saved_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
if isinstance(self._row, dict):
|
||||
self._row.update(
|
||||
{
|
||||
**values,
|
||||
"saved_at": now.isoformat(),
|
||||
"updated_at": now.isoformat(),
|
||||
}
|
||||
)
|
||||
else:
|
||||
for key, value in values.items():
|
||||
setattr(self._row, key, value)
|
||||
return self.snapshot()
|
||||
|
||||
|
||||
class KnowledgeResearchStore:
|
||||
"""SQL store with a behaviorally equivalent memory fallback."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: async_sessionmaker[AsyncSession] | None,
|
||||
) -> None:
|
||||
self._sf = session_factory
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._publication_locks: dict[str, asyncio.Lock] = {}
|
||||
|
||||
@asynccontextmanager
|
||||
async def locked_publication(
|
||||
self,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
) -> AsyncIterator[_PublicationLease | None]:
|
||||
"""Serialize version-check + remote write + local publication commit.
|
||||
|
||||
SQL deployments hold a row lock for the context lifetime, so separate
|
||||
application workers cannot both pass the same expected-hash check.
|
||||
SQLite has no ``FOR UPDATE`` support, so a no-op update first acquires
|
||||
its database write lock. The memory fallback uses one asyncio lock per
|
||||
research session and commits a copy only on successful exit.
|
||||
"""
|
||||
|
||||
if self._sf is None:
|
||||
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
|
||||
async with lock:
|
||||
original = self._rows.get(session_id)
|
||||
if original is None or original["user_id"] != user_id:
|
||||
yield None
|
||||
return
|
||||
working = deepcopy(original)
|
||||
lease = _PublicationLease(working)
|
||||
yield lease
|
||||
self._rows[session_id] = working
|
||||
return
|
||||
|
||||
async with self._sf() as session:
|
||||
async with session.begin():
|
||||
bind = session.get_bind()
|
||||
if bind.dialect.name == "sqlite":
|
||||
await session.execute(
|
||||
update(KnowledgeResearchSessionRow)
|
||||
.where(
|
||||
KnowledgeResearchSessionRow.id == session_id,
|
||||
KnowledgeResearchSessionRow.user_id == user_id,
|
||||
)
|
||||
.values(
|
||||
updated_at=KnowledgeResearchSessionRow.updated_at
|
||||
)
|
||||
)
|
||||
row = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow)
|
||||
.where(
|
||||
KnowledgeResearchSessionRow.id == session_id,
|
||||
KnowledgeResearchSessionRow.user_id == user_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
yield _PublicationLease(row) if row is not None else None
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
knowledge_base_id: str,
|
||||
user_id: str,
|
||||
title: str,
|
||||
session_id: str | None = None,
|
||||
publication_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = KnowledgeResearchSessionRow(
|
||||
id=session_id or str(uuid4()),
|
||||
publication_id=publication_id or str(uuid4()),
|
||||
thread_id=thread_id or str(uuid4()),
|
||||
knowledge_base_id=knowledge_base_id,
|
||||
user_id=user_id,
|
||||
title=title,
|
||||
draft_markdown="",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
if self._sf is None:
|
||||
data = _row_dict(row)
|
||||
self._rows[row.id] = data
|
||||
return deepcopy(data)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
return _row_dict(row)
|
||||
|
||||
async def get(self, session_id: str) -> dict[str, Any] | None:
|
||||
if self._sf is None:
|
||||
row = self._rows.get(session_id)
|
||||
return deepcopy(row) if row else None
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeResearchSessionRow, session_id)
|
||||
return _row_dict(row) if row else None
|
||||
|
||||
async def get_owned(self, session_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
row = await self.get(session_id)
|
||||
return row if row and row["user_id"] == user_id else None
|
||||
|
||||
async def get_by_thread(self, thread_id: str) -> dict[str, Any] | None:
|
||||
if self._sf is None:
|
||||
for row in self._rows.values():
|
||||
if row["thread_id"] == thread_id:
|
||||
return deepcopy(row)
|
||||
return None
|
||||
async with self._sf() as session:
|
||||
try:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow).where(
|
||||
KnowledgeResearchSessionRow.thread_id == thread_id
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
except (OperationalError, ProgrammingError) as exc:
|
||||
# Ordinary Q&A hits this lookup on every run. A missing table
|
||||
# (older DB before create_all registered the model) must not
|
||||
# abort /runs/stream; treat it as "not a research thread".
|
||||
if _is_missing_table(exc):
|
||||
return None
|
||||
raise
|
||||
return _row_dict(row) if row else None
|
||||
|
||||
async def get_publication(self, publication_id: str) -> dict[str, Any] | None:
|
||||
if self._sf is None:
|
||||
for row in self._rows.values():
|
||||
if row["publication_id"] == publication_id and row.get("saved_hash"):
|
||||
return deepcopy(row)
|
||||
return None
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow).where(
|
||||
KnowledgeResearchSessionRow.publication_id == publication_id,
|
||||
KnowledgeResearchSessionRow.saved_hash.is_not(None),
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return _row_dict(row) if row else None
|
||||
|
||||
async def list_owned(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
user_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
if self._sf is None:
|
||||
rows = [
|
||||
deepcopy(row)
|
||||
for row in self._rows.values()
|
||||
if row["knowledge_base_id"] == knowledge_base_id
|
||||
and row["user_id"] == user_id
|
||||
]
|
||||
return sorted(rows, key=lambda item: str(item["updated_at"]), reverse=True)
|
||||
async with self._sf() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow)
|
||||
.where(
|
||||
KnowledgeResearchSessionRow.knowledge_base_id
|
||||
== knowledge_base_id,
|
||||
KnowledgeResearchSessionRow.user_id == user_id,
|
||||
)
|
||||
.order_by(KnowledgeResearchSessionRow.updated_at.desc())
|
||||
)
|
||||
).scalars()
|
||||
return [_row_dict(row) for row in rows]
|
||||
|
||||
async def list_published(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
if self._sf is None:
|
||||
rows = [
|
||||
deepcopy(row)
|
||||
for row in self._rows.values()
|
||||
if row["knowledge_base_id"] == knowledge_base_id
|
||||
and row.get("saved_hash")
|
||||
]
|
||||
return sorted(
|
||||
rows,
|
||||
key=lambda item: str(item.get("saved_at") or item["updated_at"]),
|
||||
reverse=True,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow)
|
||||
.where(
|
||||
KnowledgeResearchSessionRow.knowledge_base_id
|
||||
== knowledge_base_id,
|
||||
KnowledgeResearchSessionRow.saved_hash.is_not(None),
|
||||
)
|
||||
.order_by(KnowledgeResearchSessionRow.saved_at.desc())
|
||||
)
|
||||
).scalars()
|
||||
return [_row_dict(row) for row in rows]
|
||||
|
||||
async def update_owned(
|
||||
self,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
title: str | None = None,
|
||||
draft_markdown: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
if self._sf is None:
|
||||
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
|
||||
async with lock:
|
||||
row = self._rows.get(session_id)
|
||||
if row is None or row["user_id"] != user_id:
|
||||
return None
|
||||
if title is not None:
|
||||
row["title"] = title
|
||||
if draft_markdown is not None:
|
||||
row["draft_markdown"] = draft_markdown
|
||||
row["updated_at"] = now.isoformat()
|
||||
return deepcopy(row)
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(KnowledgeResearchSessionRow)
|
||||
.where(
|
||||
KnowledgeResearchSessionRow.id == session_id,
|
||||
KnowledgeResearchSessionRow.user_id == user_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
if title is not None:
|
||||
row.title = title
|
||||
if draft_markdown is not None:
|
||||
row.draft_markdown = draft_markdown
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
return _row_dict(row)
|
||||
|
||||
async def save_publication(
|
||||
self,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
title: str,
|
||||
markdown: str,
|
||||
saved_hash: str,
|
||||
wiki_slug: str,
|
||||
wiki_page_id: str | None,
|
||||
wiki_folder_id: str | None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self.locked_publication(session_id, user_id) as publication:
|
||||
if publication is None:
|
||||
return None
|
||||
return publication.save(
|
||||
title=title,
|
||||
markdown=markdown,
|
||||
saved_hash=saved_hash,
|
||||
wiki_slug=wiki_slug,
|
||||
wiki_page_id=wiki_page_id,
|
||||
wiki_folder_id=wiki_folder_id,
|
||||
)
|
||||
|
||||
async def delete(self, session_id: str) -> None:
|
||||
if self._sf is None:
|
||||
lock = self._publication_locks.setdefault(session_id, asyncio.Lock())
|
||||
async with lock:
|
||||
self._rows.pop(session_id, None)
|
||||
return
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
delete(KnowledgeResearchSessionRow).where(
|
||||
KnowledgeResearchSessionRow.id == session_id
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
def make_knowledge_research_store(session_factory):
|
||||
return KnowledgeResearchStore(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeResearchSessionRow",
|
||||
"KnowledgeResearchStore",
|
||||
"make_knowledge_research_store",
|
||||
]
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""Database-backed global ordinary-knowledge retrieval settings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.knowledge_retrieval_settings.base import (
|
||||
KnowledgeRetrievalSettingsStore,
|
||||
)
|
||||
from deerflow.persistence.knowledge_retrieval_settings.memory import (
|
||||
MemoryKnowledgeRetrievalSettingsStore,
|
||||
)
|
||||
from deerflow.persistence.knowledge_retrieval_settings.model import (
|
||||
KnowledgeRetrievalSettingsRow,
|
||||
)
|
||||
from deerflow.persistence.knowledge_retrieval_settings.sql import (
|
||||
KnowledgeRetrievalSettingsRepository,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeRetrievalSettingsRepository",
|
||||
"KnowledgeRetrievalSettingsRow",
|
||||
"KnowledgeRetrievalSettingsStore",
|
||||
"MemoryKnowledgeRetrievalSettingsStore",
|
||||
"make_knowledge_retrieval_settings_store",
|
||||
]
|
||||
|
||||
|
||||
def make_knowledge_retrieval_settings_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryKnowledgeRetrievalSettingsStore()
|
||||
return KnowledgeRetrievalSettingsRepository(session_factory)
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
"""Persistence contract for the global ordinary-knowledge retrieval switch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class KnowledgeRetrievalSettingsStore(ABC):
|
||||
"""Store the deployment-wide smart-reasoning availability flag."""
|
||||
|
||||
@abstractmethod
|
||||
async def get(self) -> bool:
|
||||
"""Return whether smart reasoning is enabled; missing state is disabled."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
|
||||
"""Atomically replace the global flag and return the saved value."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_advanced_management_enabled(self) -> bool:
|
||||
"""Return whether advanced knowledge-management entries are exposed."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_advanced_management_enabled(
|
||||
self,
|
||||
enabled: bool,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
"""Atomically replace the advanced-management flag."""
|
||||
|
||||
raise NotImplementedError
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
"""In-memory ordinary-knowledge retrieval settings store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from deerflow.persistence.knowledge_retrieval_settings.base import (
|
||||
KnowledgeRetrievalSettingsStore,
|
||||
)
|
||||
|
||||
|
||||
class MemoryKnowledgeRetrievalSettingsStore(KnowledgeRetrievalSettingsStore):
|
||||
def __init__(self) -> None:
|
||||
self._enabled = False
|
||||
self._advanced_management_enabled = False
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def get(self) -> bool:
|
||||
async with self._lock:
|
||||
return self._enabled
|
||||
|
||||
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
|
||||
del updated_by
|
||||
async with self._lock:
|
||||
self._enabled = bool(enabled)
|
||||
return self._enabled
|
||||
|
||||
async def get_advanced_management_enabled(self) -> bool:
|
||||
async with self._lock:
|
||||
return self._advanced_management_enabled
|
||||
|
||||
async def set_advanced_management_enabled(
|
||||
self,
|
||||
enabled: bool,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
del updated_by
|
||||
async with self._lock:
|
||||
self._advanced_management_enabled = bool(enabled)
|
||||
return self._advanced_management_enabled
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"""ORM row for the database-backed ordinary-knowledge retrieval switch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, false
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class KnowledgeRetrievalSettingsRow(Base):
|
||||
__tablename__ = "knowledge_retrieval_settings"
|
||||
|
||||
scope: Mapped[str] = mapped_column(String(64), primary_key=True, default="global")
|
||||
smart_reasoning_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
server_default=false(),
|
||||
)
|
||||
advanced_management_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
server_default=false(),
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
"""SQLAlchemy-backed ordinary-knowledge retrieval settings store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.knowledge_retrieval_settings.base import (
|
||||
KnowledgeRetrievalSettingsStore,
|
||||
)
|
||||
from deerflow.persistence.knowledge_retrieval_settings.model import (
|
||||
KnowledgeRetrievalSettingsRow,
|
||||
)
|
||||
|
||||
_GLOBAL_SCOPE = "global"
|
||||
|
||||
|
||||
class KnowledgeRetrievalSettingsRepository(KnowledgeRetrievalSettingsStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def get(self) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeRetrievalSettingsRow, _GLOBAL_SCOPE)
|
||||
return bool(row.smart_reasoning_enabled) if row is not None else False
|
||||
|
||||
async def get_advanced_management_enabled(self) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeRetrievalSettingsRow, _GLOBAL_SCOPE)
|
||||
return bool(row.advanced_management_enabled) if row is not None else False
|
||||
|
||||
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
|
||||
saved = bool(enabled)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(KnowledgeRetrievalSettingsRow)
|
||||
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
|
||||
.with_for_update()
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
row = KnowledgeRetrievalSettingsRow(
|
||||
scope=_GLOBAL_SCOPE,
|
||||
smart_reasoning_enabled=saved,
|
||||
revision=1,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.smart_reasoning_enabled = saved
|
||||
row.updated_by = updated_by
|
||||
row.revision = int(row.revision or 0) + 1
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Two workers may both observe the initially absent singleton.
|
||||
# The loser retries as an update instead of surfacing a 500.
|
||||
await session.rollback()
|
||||
result = await session.execute(
|
||||
select(KnowledgeRetrievalSettingsRow)
|
||||
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
|
||||
.with_for_update()
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
raise
|
||||
row.smart_reasoning_enabled = saved
|
||||
row.updated_by = updated_by
|
||||
row.revision = int(row.revision or 0) + 1
|
||||
await session.commit()
|
||||
return saved
|
||||
|
||||
async def set_advanced_management_enabled(
|
||||
self,
|
||||
enabled: bool,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
saved = bool(enabled)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(KnowledgeRetrievalSettingsRow)
|
||||
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
|
||||
.with_for_update()
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
row = KnowledgeRetrievalSettingsRow(
|
||||
scope=_GLOBAL_SCOPE,
|
||||
smart_reasoning_enabled=False,
|
||||
advanced_management_enabled=saved,
|
||||
revision=1,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.advanced_management_enabled = saved
|
||||
row.updated_by = updated_by
|
||||
row.revision = int(row.revision or 0) + 1
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
result = await session.execute(
|
||||
select(KnowledgeRetrievalSettingsRow)
|
||||
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
|
||||
.with_for_update()
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
raise
|
||||
row.advanced_management_enabled = saved
|
||||
row.updated_by = updated_by
|
||||
row.revision = int(row.revision or 0) + 1
|
||||
await session.commit()
|
||||
return saved
|
||||
+13
@@ -15,6 +15,19 @@ class DuplicateLightAppNameError(ValueError):
|
||||
|
||||
|
||||
class LightAppStore(ABC):
|
||||
async def list_sidebar_apps(self) -> list[dict[str, Any]]:
|
||||
"""Return only the columns needed to render iframe app menu entries."""
|
||||
rows = await self.list_apps(app_type="iframe", enabled_only=True)
|
||||
return [
|
||||
{
|
||||
"app_id": row.get("app_id"),
|
||||
"name": row.get("name"),
|
||||
"mount_location": row.get("mount_location"),
|
||||
"sort_order": row.get("sort_order", 0),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
@abstractmethod
|
||||
async def list_apps(
|
||||
self,
|
||||
|
||||
+16
@@ -65,6 +65,22 @@ class LightAppRepository(LightAppStore):
|
||||
result = await session.execute(stmt)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_sidebar_apps(self) -> list[dict[str, Any]]:
|
||||
"""Avoid loading descriptions, icons, route JSON, tags, and audit data."""
|
||||
stmt = (
|
||||
select(
|
||||
LightAppRow.app_id,
|
||||
LightAppRow.name,
|
||||
LightAppRow.mount_location,
|
||||
LightAppRow.sort_order,
|
||||
)
|
||||
.where(LightAppRow.app_type == "iframe", LightAppRow.enabled.is_(True))
|
||||
.order_by(LightAppRow.sort_order.asc(), LightAppRow.created_at.desc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [dict(row._mapping) for row in result]
|
||||
|
||||
async def get_app(self, app_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LightAppRow, app_id)
|
||||
|
||||
+9
@@ -67,3 +67,12 @@ class LlmMetricsStore(ABC):
|
||||
async def iter_all(self, query: LlmMetricsQuery):
|
||||
"""Yield rows matching ``query`` without pagination — used by export."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
|
||||
"""Delete one bounded batch of audit rows older than ``cutoff``.
|
||||
|
||||
The default keeps lightweight/in-memory test implementations compatible.
|
||||
SQL-backed stores override it for the scheduled admin-audit retention
|
||||
sweep.
|
||||
"""
|
||||
return 0
|
||||
+32
-1
@@ -7,7 +7,7 @@ from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy import delete, desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery, LlmMetricsStore
|
||||
@@ -110,3 +110,34 @@ class SqlLlmMetricsStore(LlmMetricsStore):
|
||||
if len(page) < page_size:
|
||||
return
|
||||
offset += page_size
|
||||
|
||||
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
|
||||
"""Delete one small, oldest-first retention batch.
|
||||
|
||||
Selecting primary keys first keeps this portable across SQLite, MySQL,
|
||||
and PostgreSQL (which do not share ``DELETE ... LIMIT`` syntax) and
|
||||
avoids a long-running delete transaction on a busy audit table.
|
||||
"""
|
||||
# Keep below SQLite's traditional 999 bind-variable limit. The
|
||||
# retention worker requests batches repeatedly, so portability is more
|
||||
# valuable than one giant delete statement.
|
||||
size = max(1, min(int(batch_size), 500))
|
||||
normalized_cutoff = _normalize_created_at(cutoff)
|
||||
async with self._sf() as session:
|
||||
ids = list(
|
||||
(
|
||||
await session.scalars(
|
||||
select(LlmCallMetricRow.id)
|
||||
.where(LlmCallMetricRow.created_at < normalized_cutoff)
|
||||
.order_by(LlmCallMetricRow.created_at.asc(), LlmCallMetricRow.id.asc())
|
||||
.limit(size)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not ids:
|
||||
return 0
|
||||
result = await session.execute(
|
||||
delete(LlmCallMetricRow).where(LlmCallMetricRow.id.in_(ids))
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
+47
-1
@@ -6,11 +6,47 @@ from abc import ABC, abstractmethod
|
||||
from typing import Any, Literal
|
||||
|
||||
LlmWikiScope = Literal["personal", "public", "all"]
|
||||
UNSET = object()
|
||||
|
||||
|
||||
def serialize_wiki_parent_directories(
|
||||
value: list[str] | None,
|
||||
) -> list[str] | None:
|
||||
return None if value is None else [str(item) for item in value]
|
||||
|
||||
|
||||
def serialize_wiki_directory_paths(
|
||||
value: list[list[str]] | None,
|
||||
) -> list[list[str]] | None:
|
||||
return (
|
||||
None
|
||||
if value is None
|
||||
else [[str(segment) for segment in path] for path in value]
|
||||
)
|
||||
|
||||
|
||||
class LlmWikiStore(ABC):
|
||||
@abstractmethod
|
||||
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]: ...
|
||||
async def create_mapping(
|
||||
self,
|
||||
*,
|
||||
weknora_id: str,
|
||||
owner_user_id: str,
|
||||
name: str,
|
||||
description: str,
|
||||
kb_type: str,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_parent_directories: list[str] | None = None,
|
||||
wiki_directory_paths: list[list[str]] | None = None,
|
||||
wiki_focus_instructions: str | None = None,
|
||||
wiki_content_instructions_raw: str | None = None,
|
||||
wiki_extraction_instructions_raw: str | None = None,
|
||||
wiki_generation_managed: bool | None = None,
|
||||
wiki_config_schema_version: int | None = None,
|
||||
wiki_config_revision: int | None = None,
|
||||
wiki_config_apply_status: str | None = None,
|
||||
wiki_config_apply_error: str | None = None,
|
||||
) -> dict[str, Any]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]: ...
|
||||
@@ -33,6 +69,16 @@ class LlmWikiStore(ABC):
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
wiki_parent_directories: list[str] | None | object = UNSET,
|
||||
wiki_directory_paths: list[list[str]] | None | object = UNSET,
|
||||
wiki_focus_instructions: str | None | object = UNSET,
|
||||
wiki_content_instructions_raw: str | None | object = UNSET,
|
||||
wiki_extraction_instructions_raw: str | None | object = UNSET,
|
||||
wiki_generation_managed: bool | None | object = UNSET,
|
||||
wiki_config_schema_version: int | None | object = UNSET,
|
||||
wiki_config_revision: int | None | object = UNSET,
|
||||
wiki_config_apply_status: str | None | object = UNSET,
|
||||
wiki_config_apply_error: str | None | object = UNSET,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
|
||||
+128
-16
@@ -3,22 +3,58 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
|
||||
from deerflow.persistence.llmwiki.base import (
|
||||
UNSET,
|
||||
LlmWikiScope,
|
||||
LlmWikiStore,
|
||||
serialize_wiki_directory_paths,
|
||||
serialize_wiki_parent_directories,
|
||||
)
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _mapping_to_dict(row: dict[str, Any]) -> dict[str, Any]:
|
||||
data = dict(row)
|
||||
data["wiki_parent_directories"] = serialize_wiki_parent_directories(
|
||||
data.get("wiki_parent_directories")
|
||||
)
|
||||
data["wiki_directory_paths"] = serialize_wiki_directory_paths(
|
||||
data.get("wiki_directory_paths")
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
class MemoryLlmWikiStore(LlmWikiStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._conversation_deposits: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
|
||||
async def create_mapping(
|
||||
self,
|
||||
*,
|
||||
weknora_id: str,
|
||||
owner_user_id: str,
|
||||
name: str,
|
||||
description: str,
|
||||
kb_type: str,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_parent_directories: list[str] | None = None,
|
||||
wiki_directory_paths: list[list[str]] | None = None,
|
||||
wiki_focus_instructions: str | None = None,
|
||||
wiki_content_instructions_raw: str | None = None,
|
||||
wiki_extraction_instructions_raw: str | None = None,
|
||||
wiki_generation_managed: bool | None = None,
|
||||
wiki_config_schema_version: int | None = None,
|
||||
wiki_config_revision: int | None = None,
|
||||
wiki_config_apply_status: str | None = None,
|
||||
wiki_config_apply_error: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if any(row["weknora_id"] == weknora_id for row in self._rows.values()):
|
||||
raise ValueError("WeKnora knowledge base is already mapped")
|
||||
now = _now()
|
||||
@@ -29,7 +65,25 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
"name": name,
|
||||
"description": description,
|
||||
"test_questions": None,
|
||||
"retrieval_settings": None,
|
||||
"retrieval_settings": (
|
||||
dict(retrieval_settings)
|
||||
if retrieval_settings is not None
|
||||
else None
|
||||
),
|
||||
"wiki_parent_directories": serialize_wiki_parent_directories(
|
||||
wiki_parent_directories
|
||||
),
|
||||
"wiki_directory_paths": serialize_wiki_directory_paths(
|
||||
wiki_directory_paths
|
||||
),
|
||||
"wiki_focus_instructions": wiki_focus_instructions,
|
||||
"wiki_content_instructions_raw": wiki_content_instructions_raw,
|
||||
"wiki_extraction_instructions_raw": wiki_extraction_instructions_raw,
|
||||
"wiki_generation_managed": wiki_generation_managed,
|
||||
"wiki_config_schema_version": wiki_config_schema_version,
|
||||
"wiki_config_revision": wiki_config_revision,
|
||||
"wiki_config_apply_status": wiki_config_apply_status,
|
||||
"wiki_config_apply_error": wiki_config_apply_error,
|
||||
"kb_type": kb_type,
|
||||
"publication_status": "private",
|
||||
"published_by": None,
|
||||
@@ -42,7 +96,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
"updated_at": now,
|
||||
}
|
||||
self._rows[row["id"]] = row
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
|
||||
rows = list(self._rows.values())
|
||||
@@ -52,22 +106,22 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
rows = [row for row in rows if row["publication_status"] == "published"]
|
||||
else:
|
||||
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id or row["publication_status"] == "published"]
|
||||
return [dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
|
||||
return [_mapping_to_dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
|
||||
|
||||
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None:
|
||||
return None
|
||||
if is_admin or row["owner_user_id"] == user_id:
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
if not write and row["publication_status"] == "published":
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
return None
|
||||
|
||||
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
|
||||
for row in self._rows.values():
|
||||
if row["weknora_id"] == weknora_id:
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
return None
|
||||
|
||||
async def update_mapping(
|
||||
@@ -81,6 +135,16 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
wiki_parent_directories: list[str] | None | object = UNSET,
|
||||
wiki_directory_paths: list[list[str]] | None | object = UNSET,
|
||||
wiki_focus_instructions: str | None | object = UNSET,
|
||||
wiki_content_instructions_raw: str | None | object = UNSET,
|
||||
wiki_extraction_instructions_raw: str | None | object = UNSET,
|
||||
wiki_generation_managed: bool | None | object = UNSET,
|
||||
wiki_config_schema_version: int | None | object = UNSET,
|
||||
wiki_config_revision: int | None | object = UNSET,
|
||||
wiki_config_apply_status: str | None | object = UNSET,
|
||||
wiki_config_apply_error: str | None | object = UNSET,
|
||||
) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None:
|
||||
@@ -98,8 +162,56 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
if external_search_enabled is not None:
|
||||
row["external_search_enabled"] = external_search_enabled
|
||||
row["external_search_updated_at"] = _now()
|
||||
if wiki_parent_directories is not UNSET:
|
||||
row["wiki_parent_directories"] = serialize_wiki_parent_directories(
|
||||
cast(list[str] | None, wiki_parent_directories)
|
||||
)
|
||||
if wiki_directory_paths is not UNSET:
|
||||
row["wiki_directory_paths"] = serialize_wiki_directory_paths(
|
||||
cast(list[list[str]] | None, wiki_directory_paths)
|
||||
)
|
||||
if wiki_focus_instructions is not UNSET:
|
||||
row["wiki_focus_instructions"] = cast(
|
||||
str | None,
|
||||
wiki_focus_instructions,
|
||||
)
|
||||
if wiki_content_instructions_raw is not UNSET:
|
||||
row["wiki_content_instructions_raw"] = cast(
|
||||
str | None,
|
||||
wiki_content_instructions_raw,
|
||||
)
|
||||
if wiki_extraction_instructions_raw is not UNSET:
|
||||
row["wiki_extraction_instructions_raw"] = cast(
|
||||
str | None,
|
||||
wiki_extraction_instructions_raw,
|
||||
)
|
||||
if wiki_generation_managed is not UNSET:
|
||||
row["wiki_generation_managed"] = cast(
|
||||
bool | None,
|
||||
wiki_generation_managed,
|
||||
)
|
||||
if wiki_config_schema_version is not UNSET:
|
||||
row["wiki_config_schema_version"] = cast(
|
||||
int | None,
|
||||
wiki_config_schema_version,
|
||||
)
|
||||
if wiki_config_revision is not UNSET:
|
||||
row["wiki_config_revision"] = cast(
|
||||
int | None,
|
||||
wiki_config_revision,
|
||||
)
|
||||
if wiki_config_apply_status is not UNSET:
|
||||
row["wiki_config_apply_status"] = cast(
|
||||
str | None,
|
||||
wiki_config_apply_status,
|
||||
)
|
||||
if wiki_config_apply_error is not UNSET:
|
||||
row["wiki_config_apply_error"] = cast(
|
||||
str | None,
|
||||
wiki_config_apply_error,
|
||||
)
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def adopt_conversation_deposit_mapping(
|
||||
self,
|
||||
@@ -124,7 +236,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def delete_mapping(self, mapping_id: str) -> bool:
|
||||
return self._rows.pop(mapping_id, None) is not None
|
||||
@@ -138,7 +250,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
row["reviewed_by"] = None
|
||||
row["published_at"] = _now()
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
@@ -148,7 +260,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
row["reviewed_by"] = reviewer_user_id
|
||||
row["published_at"] = _now() if approved else None
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
@@ -158,18 +270,18 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
row["reviewed_by"] = actor_user_id if is_admin else row.get("reviewed_by")
|
||||
row["published_at"] = None
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
|
||||
rows = [row for row in self._rows.values() if row["publication_status"] != "private"]
|
||||
if status:
|
||||
rows = [row for row in rows if row["publication_status"] == status]
|
||||
return [dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
|
||||
return [_mapping_to_dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
|
||||
|
||||
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
|
||||
for row in self._rows.values():
|
||||
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
return None
|
||||
|
||||
async def get_conversation_deposit(
|
||||
|
||||
+11
-1
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint, false, true
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -24,6 +24,16 @@ class LlmWikiKnowledgeBaseRow(Base):
|
||||
# questions for this library.
|
||||
test_questions: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
retrieval_settings: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
wiki_parent_directories: Mapped[list[str] | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
wiki_directory_paths: Mapped[list[list[str]] | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
wiki_focus_instructions: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
wiki_content_instructions_raw: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
wiki_extraction_instructions_raw: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
wiki_generation_managed: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
||||
wiki_config_schema_version: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
wiki_config_revision: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
wiki_config_apply_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
wiki_config_apply_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
kb_type: Mapped[str] = mapped_column(String(32), default="document")
|
||||
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
|
||||
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
|
||||
+113
-3
@@ -3,13 +3,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
|
||||
from deerflow.persistence.llmwiki.base import (
|
||||
UNSET,
|
||||
LlmWikiScope,
|
||||
LlmWikiStore,
|
||||
serialize_wiki_directory_paths,
|
||||
serialize_wiki_parent_directories,
|
||||
)
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
|
||||
|
||||
@@ -19,6 +25,14 @@ def _now() -> datetime:
|
||||
|
||||
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
if "wiki_parent_directories" in data:
|
||||
data["wiki_parent_directories"] = serialize_wiki_parent_directories(
|
||||
data["wiki_parent_directories"]
|
||||
)
|
||||
if "wiki_directory_paths" in data:
|
||||
data["wiki_directory_paths"] = serialize_wiki_directory_paths(
|
||||
data["wiki_directory_paths"]
|
||||
)
|
||||
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
@@ -30,13 +44,51 @@ class LlmWikiRepository(LlmWikiStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
|
||||
async def create_mapping(
|
||||
self,
|
||||
*,
|
||||
weknora_id: str,
|
||||
owner_user_id: str,
|
||||
name: str,
|
||||
description: str,
|
||||
kb_type: str,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_parent_directories: list[str] | None = None,
|
||||
wiki_directory_paths: list[list[str]] | None = None,
|
||||
wiki_focus_instructions: str | None = None,
|
||||
wiki_content_instructions_raw: str | None = None,
|
||||
wiki_extraction_instructions_raw: str | None = None,
|
||||
wiki_generation_managed: bool | None = None,
|
||||
wiki_config_schema_version: int | None = None,
|
||||
wiki_config_revision: int | None = None,
|
||||
wiki_config_apply_status: str | None = None,
|
||||
wiki_config_apply_error: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
row = LlmWikiKnowledgeBaseRow(
|
||||
id=str(uuid4()),
|
||||
weknora_id=weknora_id,
|
||||
owner_user_id=owner_user_id,
|
||||
name=name,
|
||||
description=description,
|
||||
retrieval_settings=(
|
||||
dict(retrieval_settings)
|
||||
if retrieval_settings is not None
|
||||
else None
|
||||
),
|
||||
wiki_parent_directories=serialize_wiki_parent_directories(
|
||||
wiki_parent_directories
|
||||
),
|
||||
wiki_directory_paths=serialize_wiki_directory_paths(
|
||||
wiki_directory_paths
|
||||
),
|
||||
wiki_focus_instructions=wiki_focus_instructions,
|
||||
wiki_content_instructions_raw=wiki_content_instructions_raw,
|
||||
wiki_extraction_instructions_raw=wiki_extraction_instructions_raw,
|
||||
wiki_generation_managed=wiki_generation_managed,
|
||||
wiki_config_schema_version=wiki_config_schema_version,
|
||||
wiki_config_revision=wiki_config_revision,
|
||||
wiki_config_apply_status=wiki_config_apply_status,
|
||||
wiki_config_apply_error=wiki_config_apply_error,
|
||||
kb_type=kb_type,
|
||||
publication_status="private",
|
||||
created_at=_now(),
|
||||
@@ -93,6 +145,16 @@ class LlmWikiRepository(LlmWikiStore):
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
wiki_parent_directories: list[str] | None | object = UNSET,
|
||||
wiki_directory_paths: list[list[str]] | None | object = UNSET,
|
||||
wiki_focus_instructions: str | None | object = UNSET,
|
||||
wiki_content_instructions_raw: str | None | object = UNSET,
|
||||
wiki_extraction_instructions_raw: str | None | object = UNSET,
|
||||
wiki_generation_managed: bool | None | object = UNSET,
|
||||
wiki_config_schema_version: int | None | object = UNSET,
|
||||
wiki_config_revision: int | None | object = UNSET,
|
||||
wiki_config_apply_status: str | None | object = UNSET,
|
||||
wiki_config_apply_error: str | None | object = UNSET,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
|
||||
@@ -111,6 +173,54 @@ class LlmWikiRepository(LlmWikiStore):
|
||||
if external_search_enabled is not None:
|
||||
row.external_search_enabled = external_search_enabled
|
||||
row.external_search_updated_at = _now()
|
||||
if wiki_parent_directories is not UNSET:
|
||||
row.wiki_parent_directories = serialize_wiki_parent_directories(
|
||||
cast(list[str] | None, wiki_parent_directories)
|
||||
)
|
||||
if wiki_directory_paths is not UNSET:
|
||||
row.wiki_directory_paths = serialize_wiki_directory_paths(
|
||||
cast(list[list[str]] | None, wiki_directory_paths)
|
||||
)
|
||||
if wiki_focus_instructions is not UNSET:
|
||||
row.wiki_focus_instructions = cast(
|
||||
str | None,
|
||||
wiki_focus_instructions,
|
||||
)
|
||||
if wiki_content_instructions_raw is not UNSET:
|
||||
row.wiki_content_instructions_raw = cast(
|
||||
str | None,
|
||||
wiki_content_instructions_raw,
|
||||
)
|
||||
if wiki_extraction_instructions_raw is not UNSET:
|
||||
row.wiki_extraction_instructions_raw = cast(
|
||||
str | None,
|
||||
wiki_extraction_instructions_raw,
|
||||
)
|
||||
if wiki_generation_managed is not UNSET:
|
||||
row.wiki_generation_managed = cast(
|
||||
bool | None,
|
||||
wiki_generation_managed,
|
||||
)
|
||||
if wiki_config_schema_version is not UNSET:
|
||||
row.wiki_config_schema_version = cast(
|
||||
int | None,
|
||||
wiki_config_schema_version,
|
||||
)
|
||||
if wiki_config_revision is not UNSET:
|
||||
row.wiki_config_revision = cast(
|
||||
int | None,
|
||||
wiki_config_revision,
|
||||
)
|
||||
if wiki_config_apply_status is not UNSET:
|
||||
row.wiki_config_apply_status = cast(
|
||||
str | None,
|
||||
wiki_config_apply_status,
|
||||
)
|
||||
if wiki_config_apply_error is not UNSET:
|
||||
row.wiki_config_apply_error = cast(
|
||||
str | None,
|
||||
wiki_config_apply_error,
|
||||
)
|
||||
row.updated_at = _now()
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
+17
@@ -30,6 +30,23 @@ class LlmWikiIndexStore(ABC):
|
||||
|
||||
return []
|
||||
|
||||
async def list_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
*,
|
||||
limit: int = 500,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return a bounded, newest-first Wiki page snapshot.
|
||||
|
||||
Smart-reasoning regex search applies its expression in Python so the
|
||||
behavior stays identical on SQLite, MySQL and GaussDB. Third-party
|
||||
stores may keep the empty default; the tool then uses the remote Wiki
|
||||
API fallback.
|
||||
"""
|
||||
|
||||
return []
|
||||
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
|
||||
+22
@@ -72,6 +72,28 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
break
|
||||
return rows
|
||||
|
||||
async def list_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
*,
|
||||
limit: int = 500,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
wanted = {str(item).strip() for item in mapping_ids if str(item).strip()}
|
||||
cap = max(1, min(int(limit), 2000))
|
||||
rows = [
|
||||
dict(page)
|
||||
for page in self.pages.values()
|
||||
if page.get("knowledge_base_mapping_id") in wanted
|
||||
and not page.get("is_remote_deleted")
|
||||
and page.get("status") in ({"published", "draft"} if include_drafts else {"published"})
|
||||
]
|
||||
rows.sort(
|
||||
key=lambda page: str(page.get("remote_updated_at") or page.get("updated_at") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
return rows[:cap]
|
||||
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
|
||||
+29
@@ -123,6 +123,35 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
*,
|
||||
limit: int = 500,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
normalized_ids = list(
|
||||
dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip())
|
||||
)
|
||||
if not normalized_ids:
|
||||
return []
|
||||
statuses = ["published", "draft"] if include_drafts else ["published"]
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(statuses),
|
||||
)
|
||||
.order_by(
|
||||
LlmWikiWikiPageRow.remote_updated_at.desc(),
|
||||
LlmWikiWikiPageRow.updated_at.desc(),
|
||||
)
|
||||
.limit(max(1, min(int(limit), 2000)))
|
||||
)
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
"""Keep synchronous compatibility reads off the Gateway's async connection pool."""
|
||||
|
||||
import asyncio
|
||||
from contextvars import ContextVar
|
||||
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
_isolated_factory = ContextVar("isolated_db_factory", default=None)
|
||||
|
||||
|
||||
def isolated_session_factory():
|
||||
return _isolated_factory.get()
|
||||
|
||||
|
||||
async def run_isolated_db_read(coro):
|
||||
from deerflow.persistence.engine import get_engine
|
||||
|
||||
source = get_engine()
|
||||
if source is None:
|
||||
return await coro
|
||||
# Never return a connection created on this temporary loop to the main pool.
|
||||
engine = create_async_engine(source.url, poolclass=NullPool)
|
||||
token = _isolated_factory.set(async_sessionmaker(engine, expire_on_commit=False))
|
||||
try:
|
||||
return await asyncio.wait_for(coro, 30)
|
||||
finally:
|
||||
_isolated_factory.reset(token)
|
||||
await engine.dispose()
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
"""Helpers shared by the online Alembic environment and its tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
def configure_mysql_migration_lock_waits(connection: Any) -> None:
|
||||
"""Set bounded MySQL DDL lock waits without leaving a transaction open.
|
||||
|
||||
SQLAlchemy starts an implicit transaction for ``SET SESSION`` on MySQL.
|
||||
Alembic's migration transaction then reuses it, and closing the connection
|
||||
rolls back the ``alembic_version`` updates even though MySQL DDL itself has
|
||||
already committed. Commit the session-local settings before Alembic starts
|
||||
its transaction so successful migrations are recorded durably.
|
||||
"""
|
||||
|
||||
try:
|
||||
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
|
||||
except ValueError:
|
||||
lock_wait = 15
|
||||
|
||||
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
|
||||
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
|
||||
connection.commit()
|
||||
+5
-9
@@ -9,14 +9,13 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.migration_utils import configure_mysql_migration_lock_waits
|
||||
|
||||
# Import all models so metadata is populated.
|
||||
try:
|
||||
@@ -54,13 +53,10 @@ def do_run_migrations(connection):
|
||||
# = 1 year; innodb_lock_wait_timeout = 50s). Without this, an idle
|
||||
# transaction on another connection can freeze startup indefinitely.
|
||||
# Tunable via DEER_FLOW_ALEMBIC_LOCK_WAIT (seconds, default 15).
|
||||
if connection.dialect.name == "mysql":
|
||||
try:
|
||||
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
|
||||
except ValueError:
|
||||
lock_wait = 15
|
||||
connection.execute(text(f"SET SESSION lock_wait_timeout = {lock_wait}"))
|
||||
connection.execute(text(f"SET SESSION innodb_lock_wait_timeout = {lock_wait}"))
|
||||
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
|
||||
|
||||
if connection.dialect.name == "mysql" and not is_gauss_mysql_product():
|
||||
configure_mysql_migration_lock_waits(connection)
|
||||
|
||||
context.configure(
|
||||
connection=connection,
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""Persist DeerFlow-owned fixed parent directories for LLMWiki mappings.
|
||||
|
||||
Revision ID: 20261006_01
|
||||
Revises: 20260920_02
|
||||
Create Date: 2026-10-06
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20261006_01"
|
||||
down_revision = "20260920_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "llmwiki_knowledge_bases"
|
||||
_COLUMN = "wiki_parent_directories"
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TABLE):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
columns = _columns()
|
||||
if columns and _COLUMN not in columns:
|
||||
op.add_column(_TABLE, sa.Column(_COLUMN, PortableJSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _COLUMN in _columns():
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
"""Persist administrator-managed per-model runtime capabilities.
|
||||
|
||||
Revision ID: 20261009_01
|
||||
Revises: 20261006_01
|
||||
Create Date: 2026-10-09
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20261009_01"
|
||||
down_revision = "20261006_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "model_capabilities"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("model_name", sa.String(length=160), primary_key=True),
|
||||
sa.Column(
|
||||
"supports_reasoning_effort",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
sa.Column("updated_by", sa.String(length=64), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
op.drop_table(_TABLE)
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
"""Persist the global ordinary-knowledge smart-reasoning switch.
|
||||
|
||||
Revision ID: 20261014_01
|
||||
Revises: 20261009_01
|
||||
Create Date: 2026-10-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20261014_01"
|
||||
down_revision = "20261009_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "knowledge_retrieval_settings"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("scope", sa.String(length=64), primary_key=True),
|
||||
sa.Column(
|
||||
"smart_reasoning_enabled",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
sa.Column("revision", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("updated_by", sa.String(length=64), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
op.drop_table(_TABLE)
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""Persist hierarchical Wiki generation configuration.
|
||||
|
||||
Revision ID: 20261014_02
|
||||
Revises: 20261014_01
|
||||
Create Date: 2026-10-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20261014_02"
|
||||
down_revision = "20261014_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "llmwiki_knowledge_bases"
|
||||
_COLUMNS: tuple[sa.Column, ...] = (
|
||||
sa.Column("wiki_directory_paths", PortableJSON(), nullable=True),
|
||||
sa.Column("wiki_focus_instructions", sa.Text(), nullable=True),
|
||||
sa.Column("wiki_content_instructions_raw", sa.Text(), nullable=True),
|
||||
sa.Column("wiki_extraction_instructions_raw", sa.Text(), nullable=True),
|
||||
sa.Column("wiki_generation_managed", sa.Boolean(), nullable=True),
|
||||
sa.Column("wiki_config_schema_version", sa.Integer(), nullable=True),
|
||||
sa.Column("wiki_config_revision", sa.Integer(), nullable=True),
|
||||
sa.Column("wiki_config_apply_status", sa.String(length=32), nullable=True),
|
||||
sa.Column("wiki_config_apply_error", sa.Text(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TABLE):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
existing = _columns()
|
||||
if not existing:
|
||||
return
|
||||
for column in _COLUMNS:
|
||||
if column.name not in existing:
|
||||
op.add_column(_TABLE, column)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
existing = _columns()
|
||||
for column in reversed(_COLUMNS):
|
||||
if column.name in existing:
|
||||
op.drop_column(_TABLE, column.name)
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
"""Persist the knowledge-management advanced-entry switch.
|
||||
|
||||
Revision ID: 20261020_01
|
||||
Revises: 20261019_01, 20261014_02
|
||||
Create Date: 2026-10-20
|
||||
|
||||
An earlier release accidentally reused revision ``20261019_01`` for this
|
||||
column and for the task deep-link table. A database that ran the column
|
||||
migration can therefore report the task migration as applied even though its
|
||||
table is absent. The upgrade reconciles that table and its scope index before
|
||||
ensuring the advanced-management column, so both fresh and affected databases
|
||||
converge on the same schema. The task objects belong to the parent revision and
|
||||
are intentionally left intact by this revision's downgrade.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20261020_01"
|
||||
down_revision = ("20261019_01", "20261014_02")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "knowledge_retrieval_settings"
|
||||
_COLUMN = "advanced_management_enabled"
|
||||
_TASK_TABLE = "task_deeplink_threads"
|
||||
_TASK_INDEX = "ix_task_deeplink_threads_scope"
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TABLE):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
|
||||
|
||||
|
||||
def _ensure_task_deeplink_schema() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TASK_TABLE):
|
||||
op.create_table(
|
||||
_TASK_TABLE,
|
||||
sa.Column("thread_id", sa.String(64), primary_key=True),
|
||||
sa.Column("task_id", sa.String(128), nullable=False),
|
||||
sa.Column("source_task_id", sa.String(128), nullable=False),
|
||||
sa.Column("go_path", sa.String(16), nullable=False),
|
||||
sa.Column("agent_id", sa.String(128), nullable=True),
|
||||
sa.Column("created_by", sa.String(64), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
_TASK_INDEX,
|
||||
_TASK_TABLE,
|
||||
["task_id", "go_path", "updated_at", "thread_id"],
|
||||
)
|
||||
return
|
||||
|
||||
indexes = {
|
||||
str(index["name"])
|
||||
for index in inspector.get_indexes(_TASK_TABLE)
|
||||
if index.get("name")
|
||||
}
|
||||
if _TASK_INDEX not in indexes:
|
||||
op.create_index(
|
||||
_TASK_INDEX,
|
||||
_TASK_TABLE,
|
||||
["task_id", "go_path", "updated_at", "thread_id"],
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
_ensure_task_deeplink_schema()
|
||||
if _COLUMN in _columns():
|
||||
return
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(
|
||||
_COLUMN,
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _COLUMN in _columns():
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+11
-4
@@ -1,8 +1,15 @@
|
||||
"""Add snapshots supplied to the b5-1 task-analysis deep link.
|
||||
|
||||
Revision ID: 20261020_01
|
||||
Revises: 20261019_01
|
||||
Revision ID: 20261021_01
|
||||
Revises: 20261020_01
|
||||
Create Date: 2026-10-20
|
||||
|
||||
This file originally shipped with revision id ``20261020_01`` — the same id as
|
||||
``20261020_01_knowledge_management_advanced_tools`` — which left the migration
|
||||
graph with a duplicate revision and no single head. Re-issued as
|
||||
``20261021_01`` chained after the canonical ``20261020_01``; its upgrade is
|
||||
fully guarded (``if _has_table(): return``) so databases that already created
|
||||
the table under the old id re-run it as a no-op.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,8 +18,8 @@ import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261020_01"
|
||||
down_revision = "20261019_01"
|
||||
revision = "20261021_01"
|
||||
down_revision = "20261020_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""Admin catalog switch: ``agents.hidden``.
|
||||
|
||||
Hidden agents disappear from every non-admin listing surface (gallery / 我的 /
|
||||
广场 / homepage selector / private square) while staying fully usable — only
|
||||
the admin agents console (?admin=true) keeps listing them.
|
||||
|
||||
The Gateway's startup ORM-column sweep also adds this column on existing
|
||||
databases (Alembic auto-upgrade is disabled there), so this migration mostly
|
||||
serves environments that run Alembic manually. Fully guarded and idempotent.
|
||||
|
||||
Revision ID: 20261022_03
|
||||
Revises: 20261021_01
|
||||
Create Date: 2026-10-22
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261022_03"
|
||||
down_revision = "20261021_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "agents"
|
||||
_COLUMN = "hidden"
|
||||
_INDEX = "ix_agents_hidden"
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
return table_name in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table(_TABLE) and not _has_column(_TABLE, _COLUMN):
|
||||
# server_default mirrors the ORM declaration so the sweep on live
|
||||
# databases and this migration produce the same NOT NULL column.
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.text("0")),
|
||||
)
|
||||
if _has_table(_TABLE) and not _has_index(_TABLE, _INDEX):
|
||||
op.create_index(_INDEX, _TABLE, [_COLUMN])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table(_TABLE) and _has_index(_TABLE, _INDEX):
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
if _has_table(_TABLE) and _has_column(_TABLE, _COLUMN):
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
"""Persist private knowledge-research sessions and shared publications.
|
||||
|
||||
Revision ID: 20261022_01
|
||||
Revises: 20261014_02
|
||||
Create Date: 2026-10-22
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableLongText
|
||||
|
||||
revision = "20261022_01"
|
||||
down_revision = "20261014_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "knowledge_research_sessions"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if inspector.has_table(_TABLE):
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("id", sa.String(length=36), primary_key=True),
|
||||
sa.Column("publication_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("thread_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("knowledge_base_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("user_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("title", sa.String(length=255), nullable=False),
|
||||
sa.Column("draft_markdown", PortableLongText(), nullable=False),
|
||||
sa.Column("wiki_slug", sa.String(length=512), nullable=True),
|
||||
sa.Column("wiki_page_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("wiki_folder_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("saved_title", sa.String(length=255), nullable=True),
|
||||
sa.Column("saved_markdown", PortableLongText(), nullable=True),
|
||||
sa.Column("saved_hash", sa.String(length=64), nullable=True),
|
||||
sa.Column("saved_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("publication_id", name="uq_knowledge_research_publication_id"),
|
||||
sa.UniqueConstraint("thread_id", name="uq_knowledge_research_thread_id"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_knowledge_research_sessions_publication_id",
|
||||
_TABLE,
|
||||
["publication_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_knowledge_research_sessions_thread_id",
|
||||
_TABLE,
|
||||
["thread_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_knowledge_research_sessions_knowledge_base_id",
|
||||
_TABLE,
|
||||
["knowledge_base_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_knowledge_research_sessions_user_id",
|
||||
_TABLE,
|
||||
["user_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
op.drop_table(_TABLE)
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
"""Add the integrator-supplied type partition to task deep-link details.
|
||||
|
||||
Revision ID: 20261022_02
|
||||
Revises: 20261024_01
|
||||
Create Date: 2026-10-22
|
||||
|
||||
The same task id may now carry one snapshot per business type (the deep-link
|
||||
``?type=``), so ``sub_type`` joins the primary key. Existing rows backfill to
|
||||
``''`` — the legacy no-type partition — and callers that omit the type keep
|
||||
reading/writing exactly that partition, so behaviour is unchanged for them.
|
||||
|
||||
The key change is a portable table rebuild (create-copy-drop-rename): this
|
||||
Alembic build has no batch ``drop_primary_key`` and SQLite cannot alter a
|
||||
primary key in place, so any other route leaves a two-column key that makes
|
||||
typed upserts fail with a UNIQUE constraint violation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.dialects import mysql as sa_mysql
|
||||
|
||||
revision = "20261022_02"
|
||||
down_revision = "20261024_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_details"
|
||||
_PK = ["detail_type", "detail_id", "sub_type"]
|
||||
_OLD_PK = ["detail_type", "detail_id"]
|
||||
|
||||
|
||||
def _has_table() -> bool:
|
||||
return _TABLE in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _pk_columns() -> list[str] | None:
|
||||
constraint = inspect(op.get_bind()).get_pk_constraint(_TABLE)
|
||||
return list(constraint.get("constrained_columns") or []) or None
|
||||
|
||||
|
||||
def _column_specs(with_sub_type: bool) -> list[tuple[str, sa.types.TypeEngine, bool]]:
|
||||
is_mysql = op.get_bind().dialect.name == "mysql"
|
||||
specs = [
|
||||
("detail_type", sa.String(32), False),
|
||||
("detail_id", sa.String(128), False),
|
||||
("task_name", sa.String(512), False),
|
||||
("task_content", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
|
||||
("overview", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
|
||||
("extra_json", sa.Text() if is_mysql else sa.JSON(), False),
|
||||
("version", sa.Integer(), False),
|
||||
("created_by", sa.String(64), True),
|
||||
("updated_by", sa.String(64), True),
|
||||
("created_at", sa.DateTime(), False),
|
||||
("updated_at", sa.DateTime(), False),
|
||||
]
|
||||
if with_sub_type:
|
||||
specs.append(("sub_type", sa.String(64), False))
|
||||
return specs
|
||||
|
||||
|
||||
def _rebuild_table(pk_columns: list[str], with_sub_type: bool) -> None:
|
||||
"""Recreate the table (create-copy-drop-rename) with the given primary key."""
|
||||
|
||||
bind = op.get_bind()
|
||||
rebuild = _TABLE + "__rebuild"
|
||||
if rebuild in inspect(bind).get_table_names():
|
||||
op.drop_table(rebuild)
|
||||
specs = _column_specs(with_sub_type)
|
||||
op.create_table(
|
||||
rebuild,
|
||||
*[
|
||||
sa.Column(
|
||||
name,
|
||||
type_,
|
||||
nullable=nullable,
|
||||
server_default="" if name == "sub_type" else None,
|
||||
)
|
||||
for name, type_, nullable in specs
|
||||
],
|
||||
sa.PrimaryKeyConstraint(*pk_columns),
|
||||
)
|
||||
names = [name for name, _, _ in specs]
|
||||
select_expr = ", ".join(
|
||||
f"COALESCE({name}, '')" if name == "sub_type" else name for name in names
|
||||
)
|
||||
bind.execute(
|
||||
sa.text(f"INSERT INTO {rebuild} ({', '.join(names)}) SELECT {select_expr} FROM {_TABLE}")
|
||||
)
|
||||
op.drop_table(_TABLE)
|
||||
op.rename_table(rebuild, _TABLE)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table():
|
||||
# Fresh databases get the three-column key straight from create_all.
|
||||
return
|
||||
inspector = inspect(op.get_bind())
|
||||
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
|
||||
if "sub_type" in columns and _pk_columns() == _PK:
|
||||
return
|
||||
if "sub_type" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("sub_type", sa.String(64), nullable=False, server_default=""),
|
||||
)
|
||||
if _pk_columns() != _PK:
|
||||
_rebuild_table(_PK, with_sub_type=True)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
if _pk_columns() == _OLD_PK:
|
||||
return
|
||||
_rebuild_table(_OLD_PK, with_sub_type=False)
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
"""Admin catalog switch: ``roundtable_chains.hidden``.
|
||||
|
||||
Hidden chains vanish from every non-admin listing (我的 / 公共 / 待审核 / 选链
|
||||
弹窗) while staying fully usable — single-record reads (get_chain), configured
|
||||
business mappings and in-flight roundtables are unaffected. Only the admin
|
||||
management view (``admin_view=true``) keeps listing them.
|
||||
|
||||
The Gateway's startup ORM-column sweep also adds this column on existing
|
||||
databases (Alembic auto-upgrade is disabled there); this migration serves
|
||||
environments that run Alembic manually. Fully guarded and idempotent.
|
||||
|
||||
Revision ID: 20261023_01
|
||||
Revises: 20261022_03
|
||||
Create Date: 2026-10-23
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261023_01"
|
||||
down_revision = "20261022_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "roundtable_chains"
|
||||
_COLUMN = "hidden"
|
||||
_INDEX = "ix_roundtable_chains_hidden"
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
return table_name in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table(_TABLE) and not _has_column(_TABLE, _COLUMN):
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.text("0")),
|
||||
)
|
||||
if _has_table(_TABLE) and not _has_index(_TABLE, _INDEX):
|
||||
op.create_index(_INDEX, _TABLE, [_COLUMN])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table(_TABLE) and _has_index(_TABLE, _INDEX):
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
if _has_table(_TABLE) and _has_column(_TABLE, _COLUMN):
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
"""Index the common deep-link list path that omits sub_type.
|
||||
|
||||
Revision ID: 20261023_02
|
||||
Revises: 20261024_02, 20261022_02
|
||||
Create Date: 2026-10-22
|
||||
|
||||
``GET /api/task-deeplink-threads`` for 3qfx/rwfx/xdfx filters only
|
||||
``task_id + go_path`` and orders by ``updated_at, thread_id``. The existing
|
||||
scope index inserts ``sub_type`` before those sort columns, so the planner
|
||||
cannot use it for newest-first listing. This extra index matches that query.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261023_02"
|
||||
down_revision = ("20261024_02", "20261022_02")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_threads"
|
||||
_INDEX = "ix_task_deeplink_threads_task_go_updated"
|
||||
_COLUMNS = ["task_id", "go_path", "updated_at", "thread_id"]
|
||||
|
||||
|
||||
def _has_table() -> bool:
|
||||
return _TABLE in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _indexes() -> dict[str, list[str]]:
|
||||
if not _has_table():
|
||||
return {}
|
||||
return {
|
||||
str(index["name"]): [str(column) for column in index["column_names"]]
|
||||
for index in inspect(op.get_bind()).get_indexes(_TABLE)
|
||||
if index.get("name")
|
||||
}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
if _indexes().get(_INDEX) != _COLUMNS:
|
||||
if _INDEX in _indexes():
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
op.create_index(_INDEX, _TABLE, _COLUMNS)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
if _INDEX in _indexes():
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
"""Repair task_deeplink_details primary keys left two-column by 20261022_02.
|
||||
|
||||
Revision ID: 20261023_03
|
||||
Revises: 20261023_02
|
||||
Create Date: 2026-10-22
|
||||
|
||||
An interim build of ``20261022_02`` added the ``sub_type`` column but could
|
||||
not rebuild the primary key (the batch API it called does not exist in this
|
||||
Alembic version, and SQLite cannot alter a key in place). Databases that ran
|
||||
that build are stamped at head yet still enforce the legacy
|
||||
``(detail_type, detail_id)`` key, so typed upserts fail with a UNIQUE
|
||||
constraint violation. This revision detects that state and performs the
|
||||
portable create-copy-drop-rename rebuild; databases with the correct
|
||||
three-column key simply no-op.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.dialects import mysql as sa_mysql
|
||||
|
||||
revision = "20261023_03"
|
||||
down_revision = "20261023_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_details"
|
||||
_PK = ["detail_type", "detail_id", "sub_type"]
|
||||
|
||||
|
||||
def _has_table() -> bool:
|
||||
return _TABLE in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _pk_columns() -> list[str] | None:
|
||||
constraint = inspect(op.get_bind()).get_pk_constraint(_TABLE)
|
||||
return list(constraint.get("constrained_columns") or []) or None
|
||||
|
||||
|
||||
def _rebuild_table() -> None:
|
||||
bind = op.get_bind()
|
||||
rebuild = _TABLE + "__rebuild"
|
||||
if rebuild in inspect(bind).get_table_names():
|
||||
op.drop_table(rebuild)
|
||||
is_mysql = bind.dialect.name == "mysql"
|
||||
specs: list[tuple[str, sa.types.TypeEngine, bool]] = [
|
||||
("detail_type", sa.String(32), False),
|
||||
("detail_id", sa.String(128), False),
|
||||
("sub_type", sa.String(64), False),
|
||||
("task_name", sa.String(512), False),
|
||||
("task_content", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
|
||||
("overview", sa_mysql.LONGTEXT() if is_mysql else sa.Text(), False),
|
||||
("extra_json", sa.Text() if is_mysql else sa.JSON(), False),
|
||||
("version", sa.Integer(), False),
|
||||
("created_by", sa.String(64), True),
|
||||
("updated_by", sa.String(64), True),
|
||||
("created_at", sa.DateTime(), False),
|
||||
("updated_at", sa.DateTime(), False),
|
||||
]
|
||||
op.create_table(
|
||||
rebuild,
|
||||
*[
|
||||
sa.Column(
|
||||
name,
|
||||
type_,
|
||||
nullable=nullable,
|
||||
server_default="" if name == "sub_type" else None,
|
||||
)
|
||||
for name, type_, nullable in specs
|
||||
],
|
||||
sa.PrimaryKeyConstraint(*_PK),
|
||||
)
|
||||
names = [name for name, _, _ in specs]
|
||||
select_expr = ", ".join(
|
||||
f"COALESCE({name}, '')" if name == "sub_type" else name for name in names
|
||||
)
|
||||
bind.execute(
|
||||
sa.text(f"INSERT INTO {rebuild} ({', '.join(names)}) SELECT {select_expr} FROM {_TABLE}")
|
||||
)
|
||||
op.drop_table(_TABLE)
|
||||
op.rename_table(rebuild, _TABLE)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
if _pk_columns() == _PK:
|
||||
return
|
||||
_rebuild_table()
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# The repaired key is what 20261022_02 intended; nothing to undo.
|
||||
return
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
"""Add the b5-1 ?type= partition column to the deep-link history index.
|
||||
|
||||
Revision ID: 20261024_01
|
||||
Revises: 20261023_01
|
||||
Create Date: 2026-10-22
|
||||
|
||||
Existing rows keep sharing one partition per (task_id, go_path): the new
|
||||
``sub_type`` column backfills to ``''`` and queries that omit ``sub_type``
|
||||
continue to return every sub-type, so 3qfx/rwfx/xdfx behaviour is unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261024_01"
|
||||
down_revision = "20261023_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_threads"
|
||||
_INDEX = "ix_task_deeplink_threads_scope"
|
||||
_INDEX_COLUMNS = ["task_id", "go_path", "sub_type", "updated_at", "thread_id"]
|
||||
|
||||
|
||||
def _has_table() -> bool:
|
||||
return _TABLE in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table():
|
||||
# Fresh databases get the column straight from create_all.
|
||||
return
|
||||
bind = op.get_bind()
|
||||
inspector = inspect(bind)
|
||||
if "sub_type" not in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("sub_type", sa.String(64), nullable=False, server_default=""),
|
||||
)
|
||||
indexes = {index["name"]: index["column_names"] for index in inspector.get_indexes(_TABLE)}
|
||||
if indexes.get(_INDEX) != _INDEX_COLUMNS:
|
||||
if _INDEX in indexes:
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
op.create_index(_INDEX, _TABLE, _INDEX_COLUMNS)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
bind = op.get_bind()
|
||||
inspector = inspect(bind)
|
||||
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
|
||||
indexes = {index["name"]: index["column_names"] for index in inspector.get_indexes(_TABLE)}
|
||||
if indexes.get(_INDEX) is not None:
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
if "sub_type" in columns:
|
||||
op.drop_column(_TABLE, "sub_type")
|
||||
op.create_index(_INDEX, _TABLE, ["task_id", "go_path", "updated_at", "thread_id"])
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
"""Store deep-link list cards without joining ``threads_meta``.
|
||||
|
||||
Revision ID: 20261024_02
|
||||
Revises: 20261024_01
|
||||
Create Date: 2026-10-22
|
||||
|
||||
The dedicated history table used to be a pointer: listing still joined the
|
||||
generic ``threads_meta`` catalogue for title / status / metadata. That join
|
||||
is what made the task workspace slow once ordinary Q&A filled the catalogue.
|
||||
This revision copies those card fields onto ``task_deeplink_threads`` itself
|
||||
and backfills existing index rows from ``threads_meta``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20261024_02"
|
||||
down_revision = "20261024_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_threads"
|
||||
_META_TABLE = "threads_meta"
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
return table_name in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
if not _has_table(_TABLE):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspect(op.get_bind()).get_columns(_TABLE)}
|
||||
|
||||
|
||||
def _backfill_cards() -> None:
|
||||
if not _has_table(_META_TABLE):
|
||||
return
|
||||
bind = op.get_bind()
|
||||
dialect = bind.dialect.name
|
||||
if dialect in {"postgresql", "opengauss"}:
|
||||
bind.execute(
|
||||
sa.text(
|
||||
f"UPDATE {_TABLE} AS t "
|
||||
"SET display_name = m.display_name, "
|
||||
"status = COALESCE(m.status, 'idle'), "
|
||||
"metadata_json = m.metadata_json "
|
||||
f"FROM {_META_TABLE} AS m "
|
||||
"WHERE m.thread_id = t.thread_id"
|
||||
)
|
||||
)
|
||||
return
|
||||
if dialect == "mysql":
|
||||
bind.execute(
|
||||
sa.text(
|
||||
f"UPDATE {_TABLE} AS t "
|
||||
f"JOIN {_META_TABLE} AS m ON m.thread_id = t.thread_id "
|
||||
"SET t.display_name = m.display_name, "
|
||||
"t.status = COALESCE(m.status, 'idle'), "
|
||||
"t.metadata_json = m.metadata_json"
|
||||
)
|
||||
)
|
||||
return
|
||||
bind.execute(
|
||||
sa.text(
|
||||
f"UPDATE {_TABLE} SET "
|
||||
"display_name = ("
|
||||
f"SELECT display_name FROM {_META_TABLE} "
|
||||
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
|
||||
"), "
|
||||
"status = COALESCE(("
|
||||
f"SELECT status FROM {_META_TABLE} "
|
||||
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
|
||||
"), 'idle'), "
|
||||
"metadata_json = ("
|
||||
f"SELECT metadata_json FROM {_META_TABLE} "
|
||||
f"WHERE {_META_TABLE}.thread_id = {_TABLE}.thread_id"
|
||||
")"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table(_TABLE):
|
||||
return
|
||||
columns = _columns()
|
||||
if "display_name" not in columns:
|
||||
op.add_column(_TABLE, sa.Column("display_name", sa.String(256), nullable=True))
|
||||
if "status" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("status", sa.String(20), nullable=False, server_default="idle"),
|
||||
)
|
||||
if "metadata_json" not in columns:
|
||||
op.add_column(_TABLE, sa.Column("metadata_json", PortableJSON(), nullable=True))
|
||||
_backfill_cards()
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table(_TABLE):
|
||||
return
|
||||
columns = _columns()
|
||||
if "metadata_json" in columns:
|
||||
op.drop_column(_TABLE, "metadata_json")
|
||||
if "status" in columns:
|
||||
op.drop_column(_TABLE, "status")
|
||||
if "display_name" in columns:
|
||||
op.drop_column(_TABLE, "display_name")
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""Indexes for the agent gallery category tabs and tag filters.
|
||||
|
||||
Revision ID: 20261025_01
|
||||
Revises: 20261023_03
|
||||
Create Date: 2026-10-23
|
||||
|
||||
The gallery "广场" tab filters ``hidden`` + ``is_private`` + ``square_id``,
|
||||
and "我的" filters ``user_id`` + ``hidden``. Tag chips join
|
||||
``tag_assignments`` by ``(target_type, target_id)`` and filter by
|
||||
``(target_type, tag_id)``. Single-column indexes and the existing unique
|
||||
key (which leads with ``tag_id``) do not cover those predicates.
|
||||
|
||||
Startup also creates missing ORM indexes, so deployments that skip Alembic
|
||||
still receive these. This revision stays idempotent for the ones that run it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261025_01"
|
||||
down_revision = "20261023_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_INDEXES = (
|
||||
("agents", "ix_agents_catalog_square", ("hidden", "is_private", "square_id", "created_at")),
|
||||
("agents", "ix_agents_owner_hidden", ("user_id", "hidden", "created_at")),
|
||||
("tag_assignments", "ix_tag_assignments_type_target", ("target_type", "target_id")),
|
||||
("tag_assignments", "ix_tag_assignments_type_tag", ("target_type", "tag_id", "target_id")),
|
||||
)
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
return table_name in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
return any(idx["name"] == index_name for idx in inspect(op.get_bind()).get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for table_name, index_name, columns in _INDEXES:
|
||||
if _has_table(table_name) and not _has_index(table_name, index_name):
|
||||
op.create_index(index_name, table_name, list(columns))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table_name, index_name, _columns in reversed(_INDEXES):
|
||||
if _has_table(table_name) and _has_index(table_name, index_name):
|
||||
op.drop_index(index_name, table_name=table_name)
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"""Add shared nonce storage for scoped service-token exchange.
|
||||
|
||||
Revision ID: 20261025_02
|
||||
Revises: 20261025_01
|
||||
Create Date: 2026-10-24
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261025_02"
|
||||
down_revision = "20261025_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "auth_service_nonces"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _TABLE in inspect(op.get_bind()).get_table_names():
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("nonce_hash", sa.String(64), primary_key=True),
|
||||
sa.Column("client_id", sa.String(128), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_auth_service_nonces_client_id", _TABLE, ["client_id"])
|
||||
op.create_index("ix_auth_service_nonces_expires_at", _TABLE, ["expires_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _TABLE in inspect(op.get_bind()).get_table_names():
|
||||
op.drop_table(_TABLE)
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
"""Add the database-backed skill catalogue snapshot for list requests.
|
||||
|
||||
Revision ID: 20261025_03
|
||||
Revises: 20261025_02
|
||||
Create Date: 2026-10-25
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261025_03"
|
||||
down_revision = "20261025_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "skill_catalog_snapshots"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _TABLE in inspect(op.get_bind()).get_table_names():
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("cache_key", sa.String(32), primary_key=True),
|
||||
sa.Column("catalog_json", sa.Text(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _TABLE in inspect(op.get_bind()).get_table_names():
|
||||
op.drop_table(_TABLE)
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""Index canonical usernames for fast SSO and username login.
|
||||
|
||||
Revision ID: 20261026_01
|
||||
Revises: 20261025_03
|
||||
Create Date: 2026-10-26
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261026_01"
|
||||
down_revision = "20261025_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "users"
|
||||
_COLUMN = "username_key"
|
||||
_INDEX = "ix_users_username_key_created_at"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
|
||||
columns = {column["name"] for column in inspector.get_columns(_TABLE)}
|
||||
if _COLUMN not in columns:
|
||||
op.add_column(_TABLE, sa.Column(_COLUMN, sa.String(128), nullable=True))
|
||||
|
||||
indexes = {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
|
||||
if _INDEX not in indexes:
|
||||
op.create_index(_INDEX, _TABLE, [_COLUMN, "created_at"])
|
||||
|
||||
# Backfill the normalized local part once. Login requests then use a
|
||||
# bounded indexed equality lookup instead of LOWER(email) LIKE 'name@%'.
|
||||
bind = op.get_bind()
|
||||
rows = bind.execute(sa.text("SELECT id, email FROM users")).fetchall()
|
||||
for user_id, email in rows:
|
||||
if not email or "@" not in email:
|
||||
continue
|
||||
username_key = str(email).split("@", 1)[0].strip().lower()[:128]
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"UPDATE users SET username_key = :username_key "
|
||||
"WHERE id = :user_id AND (username_key IS NULL OR username_key = '')"
|
||||
),
|
||||
{"username_key": username_key, "user_id": user_id},
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _TABLE not in inspect(op.get_bind()).get_table_names():
|
||||
return
|
||||
indexes = {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
|
||||
if _INDEX in indexes:
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
columns = {column["name"] for column in inspect(op.get_bind()).get_columns(_TABLE)}
|
||||
if _COLUMN in columns:
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""Persist the globally shared default reasoning effort for each model.
|
||||
|
||||
Revision ID: 20261026_02
|
||||
Revises: 20261026_01, 20261022_01
|
||||
Create Date: 2026-10-26
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261026_02"
|
||||
down_revision = ("20261026_01", "20261022_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "model_capabilities"
|
||||
_COLUMN = "default_reasoning_effort"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(
|
||||
_COLUMN,
|
||||
sa.String(length=16),
|
||||
nullable=False,
|
||||
server_default="xhigh",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""Persist the administrator-configured Redis cache TTL.
|
||||
|
||||
Revision ID: 20261029_01
|
||||
Revises: 20261026_02
|
||||
Create Date: 2026-10-29
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261029_01"
|
||||
down_revision = "20261026_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "api_read_cache_settings"
|
||||
_COLUMN = "ttl_seconds"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(
|
||||
_COLUMN,
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
server_default=sa.text("300"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
"""Persist per-API cache switches and browser status access control."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20261029_02"
|
||||
down_revision = "20261029_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if "api_read_cache_settings" not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns("api_read_cache_settings")}
|
||||
if "collection_overrides" not in columns:
|
||||
op.add_column("api_read_cache_settings", sa.Column("collection_overrides", sa.JSON(), nullable=True))
|
||||
if "browser_status_enabled" not in columns:
|
||||
op.add_column("api_read_cache_settings", sa.Column("browser_status_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
|
||||
if "cache_epoch" not in columns:
|
||||
op.add_column("api_read_cache_settings", sa.Column("cache_epoch", sa.String(32), nullable=False, server_default="0"))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if "api_read_cache_settings" not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns("api_read_cache_settings")}
|
||||
for name in ("cache_epoch", "browser_status_enabled", "collection_overrides"):
|
||||
if name in columns:
|
||||
op.drop_column("api_read_cache_settings", name)
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
"""Add per-item enablement to compact menu overrides.
|
||||
|
||||
Existing compact-menu items remain enabled. New databases receive the field
|
||||
from ``CompactMenuOverrideRow``; this revision and the startup ORM schema sweep
|
||||
add it safely to deployments with an existing ``compact_menu_overrides`` table.
|
||||
|
||||
Revision ID: 20261102_01
|
||||
Revises: 20261029_02
|
||||
Create Date: 2026-11-02
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261102_01"
|
||||
down_revision = "20261029_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "compact_menu_overrides"
|
||||
_COLUMN = "disabled"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN not in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(_COLUMN, sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
if _COLUMN in {column["name"] for column in inspector.get_columns(_TABLE)}:
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
"""Allow administrator-defined business-mapping types and login goPaths.
|
||||
|
||||
Revision ID: 20261102_02
|
||||
Revises: 20261102_01
|
||||
Create Date: 2026-11-02
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261102_02"
|
||||
down_revision = "20261102_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_MAPPING_TABLE = "business_mapping"
|
||||
_THREAD_TABLE = "task_deeplink_threads"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
tables = set(inspector.get_table_names())
|
||||
if _MAPPING_TABLE in tables:
|
||||
columns = {column["name"] for column in inspector.get_columns(_MAPPING_TABLE)}
|
||||
if "mapping_type" not in columns:
|
||||
op.add_column(
|
||||
_MAPPING_TABLE,
|
||||
sa.Column("mapping_type", sa.String(64), nullable=False, server_default="内置业务"),
|
||||
)
|
||||
if "is_custom" not in columns:
|
||||
op.add_column(
|
||||
_MAPPING_TABLE,
|
||||
sa.Column("is_custom", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
)
|
||||
|
||||
# PostgreSQL enforces VARCHAR length, unlike SQLite. Widen the dedicated
|
||||
# history partition too so a 32-character custom goPath can be persisted.
|
||||
if _THREAD_TABLE in tables:
|
||||
with op.batch_alter_table(_THREAD_TABLE) as batch:
|
||||
batch.alter_column(
|
||||
"go_path",
|
||||
existing_type=sa.String(16),
|
||||
type_=sa.String(32),
|
||||
existing_nullable=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = inspect(op.get_bind())
|
||||
tables = set(inspector.get_table_names())
|
||||
if _THREAD_TABLE in tables:
|
||||
with op.batch_alter_table(_THREAD_TABLE) as batch:
|
||||
batch.alter_column(
|
||||
"go_path",
|
||||
existing_type=sa.String(32),
|
||||
type_=sa.String(16),
|
||||
existing_nullable=False,
|
||||
)
|
||||
if _MAPPING_TABLE in tables:
|
||||
columns = {column["name"] for column in inspector.get_columns(_MAPPING_TABLE)}
|
||||
if "is_custom" in columns:
|
||||
op.drop_column(_MAPPING_TABLE, "is_custom")
|
||||
if "mapping_type" in columns:
|
||||
op.drop_column(_MAPPING_TABLE, "mapping_type")
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""Administrator-managed per-model runtime capabilities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
|
||||
from deerflow.persistence.model_capabilities.memory import MemoryModelCapabilityStore
|
||||
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
|
||||
from deerflow.persistence.model_capabilities.sql import ModelCapabilityRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryModelCapabilityStore",
|
||||
"ModelCapabilityRepository",
|
||||
"ModelCapabilityRow",
|
||||
"ModelCapabilityStore",
|
||||
"import_legacy_reasoning_effort_capabilities",
|
||||
"make_model_capability_store",
|
||||
]
|
||||
|
||||
|
||||
async def import_legacy_reasoning_effort_capabilities(
|
||||
store: ModelCapabilityStore,
|
||||
models: Iterable[Any],
|
||||
) -> list[str]:
|
||||
"""One-time compatibility import for legacy config-backed ``true`` flags.
|
||||
|
||||
A configured ``false`` never creates a row. Once any database row exists,
|
||||
its value is authoritative and cannot be overwritten by legacy YAML.
|
||||
"""
|
||||
|
||||
imported: list[str] = []
|
||||
for model in models:
|
||||
if not bool(getattr(model, "supports_reasoning_effort", False)):
|
||||
continue
|
||||
model_name = str(getattr(model, "name", "") or "").strip()
|
||||
if model_name and await store.seed_reasoning_effort_support(
|
||||
model_name,
|
||||
updated_by="legacy-config-import",
|
||||
):
|
||||
imported.append(model_name)
|
||||
return imported
|
||||
|
||||
|
||||
def make_model_capability_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryModelCapabilityStore()
|
||||
return ModelCapabilityRepository(session_factory)
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""Persistence contract for administrator-managed model capabilities.
|
||||
|
||||
The provider connection details still live in ``config.yaml``. Runtime
|
||||
capabilities that administrators change from the web UI live here instead so
|
||||
they are shared by every Gateway worker and survive application restarts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ModelCapabilityStore(ABC):
|
||||
"""Global capability settings keyed by the stable configured model name."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_capabilities(self) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
supports_reasoning_effort: bool,
|
||||
default_reasoning_effort: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def seed_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
"""Insert legacy ``true`` only when no authoritative row exists."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_capability(self, model_name: str) -> None:
|
||||
raise NotImplementedError
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
"""In-memory model-capability store used by memory-mode deployments/tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
|
||||
|
||||
|
||||
class MemoryModelCapabilityStore(ModelCapabilityStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def list_capabilities(self) -> list[dict[str, Any]]:
|
||||
return [dict(self._rows[name]) for name in sorted(self._rows)]
|
||||
|
||||
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(model_name)
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
async def set_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
supports_reasoning_effort: bool,
|
||||
default_reasoning_effort: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
row = {
|
||||
"model_name": model_name,
|
||||
"supports_reasoning_effort": bool(supports_reasoning_effort),
|
||||
"default_reasoning_effort": default_reasoning_effort
|
||||
or (self._rows.get(model_name) or {}).get("default_reasoning_effort")
|
||||
or "xhigh",
|
||||
"updated_by": updated_by,
|
||||
"updated_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
self._rows[model_name] = row
|
||||
return dict(row)
|
||||
|
||||
async def seed_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
if model_name in self._rows:
|
||||
return False
|
||||
await self.set_reasoning_effort_support(
|
||||
model_name,
|
||||
supports_reasoning_effort=True,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
return True
|
||||
|
||||
async def delete_capability(self, model_name: str) -> None:
|
||||
self._rows.pop(model_name, None)
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
"""ORM row for administrator-managed model capabilities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, String, false
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ModelCapabilityRow(Base):
|
||||
__tablename__ = "model_capabilities"
|
||||
|
||||
model_name: Mapped[str] = mapped_column(String(160), primary_key=True)
|
||||
supports_reasoning_effort: Mapped[bool] = mapped_column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
server_default=false(),
|
||||
)
|
||||
default_reasoning_effort: Mapped[str] = mapped_column(
|
||||
String(16),
|
||||
nullable=False,
|
||||
default="xhigh",
|
||||
server_default="xhigh",
|
||||
)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
"""SQLAlchemy-backed model-capability store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
|
||||
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
|
||||
|
||||
|
||||
def _to_dict(row: ModelCapabilityRow) -> dict[str, Any]:
|
||||
return {
|
||||
"model_name": row.model_name,
|
||||
"supports_reasoning_effort": bool(row.supports_reasoning_effort),
|
||||
"default_reasoning_effort": row.default_reasoning_effort or "xhigh",
|
||||
"updated_by": row.updated_by,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
}
|
||||
|
||||
|
||||
class ModelCapabilityRepository(ModelCapabilityStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_capabilities(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(select(ModelCapabilityRow).order_by(ModelCapabilityRow.model_name.asc()))
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ModelCapabilityRow, model_name)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def set_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
supports_reasoning_effort: bool,
|
||||
default_reasoning_effort: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ModelCapabilityRow, model_name)
|
||||
if row is None:
|
||||
row = ModelCapabilityRow(model_name=model_name)
|
||||
session.add(row)
|
||||
row.supports_reasoning_effort = bool(supports_reasoning_effort)
|
||||
if default_reasoning_effort is not None:
|
||||
row.default_reasoning_effort = default_reasoning_effort
|
||||
row.updated_by = updated_by
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def seed_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
"""Insert a legacy ``true`` without ever overwriting an existing row.
|
||||
|
||||
The primary-key conflict handles concurrent Gateway startup safely: if
|
||||
another worker or an administrator creates the row after our read, the
|
||||
losing seed transaction rolls back and preserves that authoritative
|
||||
value, including ``false``.
|
||||
"""
|
||||
|
||||
async with self._sf() as session:
|
||||
if await session.get(ModelCapabilityRow, model_name) is not None:
|
||||
return False
|
||||
session.add(
|
||||
ModelCapabilityRow(
|
||||
model_name=model_name,
|
||||
supports_reasoning_effort=True,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
return False
|
||||
return True
|
||||
|
||||
async def delete_capability(self, model_name: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(ModelCapabilityRow).where(ModelCapabilityRow.model_name == model_name))
|
||||
await session.commit()
|
||||
+48
-1
@@ -35,12 +35,15 @@ from deerflow.persistence.agents.model import (
|
||||
AgentRow,
|
||||
AgentSkillRow,
|
||||
)
|
||||
from deerflow.persistence.api_read_cache_settings.model import ApiReadCacheSettingRow
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
from deerflow.persistence.analysis_project_memory.model import (
|
||||
AnalysisProjectMemoryItemRow,
|
||||
AnalysisProjectMemoryJobRow,
|
||||
AnalysisProjectMemoryStateRow,
|
||||
)
|
||||
from deerflow.persistence.analysis_projects import AnalysisProjectConfigRow, AnalysisProjectRow, AnalysisProjectThreadRow
|
||||
from deerflow.persistence.article_types.model import ArticleTypeRow
|
||||
from deerflow.persistence.artifact_library.model import (
|
||||
ArtifactLibraryItemRow,
|
||||
ArtifactLibrarySyncStateRow,
|
||||
@@ -75,7 +78,10 @@ from deerflow.persistence.force_situation.model import (
|
||||
ForceSituationDetailVersionRow,
|
||||
)
|
||||
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
|
||||
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
from deerflow.persistence.knowledge_research import KnowledgeResearchSessionRow
|
||||
from deerflow.persistence.knowledge_retrieval_settings.model import KnowledgeRetrievalSettingsRow
|
||||
from deerflow.persistence.light_apps.model import LightAppRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
@@ -87,6 +93,7 @@ from deerflow.persistence.llmwiki_index.model import (
|
||||
)
|
||||
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
|
||||
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.notifications.model import NotificationRow
|
||||
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
|
||||
@@ -122,6 +129,7 @@ from deerflow.persistence.roundtable_drafts.model import (
|
||||
RoundtableRecommendHistoryRow,
|
||||
)
|
||||
from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
|
||||
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
|
||||
from deerflow.persistence.roundtable_task_drafts.model import RoundtableTaskDraftRow
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.scheduled_tasks.model import (
|
||||
@@ -144,9 +152,26 @@ from deerflow.persistence.skill_knowledge.model import (
|
||||
SkillKnowledgeSyncItemRow,
|
||||
SkillKnowledgeSyncJobRow,
|
||||
)
|
||||
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
|
||||
from deerflow.persistence.skills.model import (
|
||||
SkillCatalogSnapshotRow,
|
||||
SkillDisplayAdapterRow,
|
||||
SkillFavoriteRow,
|
||||
SkillRow,
|
||||
)
|
||||
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.task_analysis.model import (
|
||||
TaskAnalysisBindingRow,
|
||||
TaskAnalysisContentVersionRow,
|
||||
TaskAnalysisRunMessageRow,
|
||||
TaskAnalysisRunRow,
|
||||
TaskAnalysisSettingRow,
|
||||
TaskAnalysisTaskRow,
|
||||
TaskDataBlockDefRow,
|
||||
TaskDataBlockRow,
|
||||
TaskDataPeriodRow,
|
||||
TaskDataSubmissionRow,
|
||||
)
|
||||
from deerflow.persistence.task_buttons.model import TaskButtonRow
|
||||
from deerflow.persistence.task_deeplink_details.model import TaskDeeplinkDetailRow
|
||||
from deerflow.persistence.task_deeplink_threads.model import TaskDeeplinkThreadRow
|
||||
@@ -155,6 +180,7 @@ from deerflow.persistence.taskcop_tasks.model import TaskCopTaskRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
|
||||
from deerflow.persistence.upload_policy.model import UploadPolicyRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.user_preferences.model import UserPreferencesRow
|
||||
@@ -173,21 +199,25 @@ from deerflow.persistence.workflow_runs.model import (
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
|
||||
__all__ = [
|
||||
"AIWritingSessionRow",
|
||||
"AnalysisProjectConfigRow",
|
||||
"AnalysisProjectMemoryItemRow",
|
||||
"AnalysisProjectMemoryJobRow",
|
||||
"AnalysisProjectMemoryStateRow",
|
||||
"ArtifactLibraryItemRow",
|
||||
"ArtifactLibrarySyncStateRow",
|
||||
"ArticleTypeRow",
|
||||
"FeedbackRow",
|
||||
"FixedQuestionRow",
|
||||
"ForceSituationDetailRow",
|
||||
"ForceSituationDetailVersionRow",
|
||||
"GongwenDocumentRow",
|
||||
"GongwenDocumentVersionRow",
|
||||
"HtmlPageFavoriteRow",
|
||||
"AgentExtrasRow",
|
||||
"AgentFavoriteRow",
|
||||
"AgentRow",
|
||||
"ApiReadCacheSettingRow",
|
||||
"AgentSkillRow",
|
||||
"AdminLeaderboardDailyStatRow",
|
||||
"BusinessMappingRow",
|
||||
@@ -222,6 +252,8 @@ __all__ = [
|
||||
"KnowledgeRelationRow",
|
||||
"KnowledgeSourceRow",
|
||||
"KnowledgeTagRow",
|
||||
"KnowledgeResearchSessionRow",
|
||||
"KnowledgeRetrievalSettingsRow",
|
||||
"LightAppRow",
|
||||
"WorkflowDataSourceRow",
|
||||
"WorkflowDefinitionRow",
|
||||
@@ -240,6 +272,7 @@ __all__ = [
|
||||
"LlmWikiWikiVectorRow",
|
||||
"LlmWikiWikiSyncStateRow",
|
||||
"MenuOverrideRow",
|
||||
"ModelCapabilityRow",
|
||||
"UserMenuOverrideConfigRow",
|
||||
"NotificationRow",
|
||||
"ParallelAgentTaskRow",
|
||||
@@ -268,17 +301,20 @@ __all__ = [
|
||||
"RoundtableDraftShareRow",
|
||||
"RoundtableDiagnosticRow",
|
||||
"RoundtableJobRow",
|
||||
"RoundtableRoomLeaseRow",
|
||||
"RoundtableRecommendHistoryRow",
|
||||
"RoundtableTaskDraftRow",
|
||||
"PositionRoleRow",
|
||||
"RunEventRow",
|
||||
"RunRow",
|
||||
"ScheduledTaskDeliveryProfileRow",
|
||||
"ScheduledHtmlPageRow",
|
||||
"ScheduledTaskRow",
|
||||
"ScheduledTaskRunRow",
|
||||
"ScheduledTaskSubscriptionRow",
|
||||
"SchedulerThreadRow",
|
||||
"SensitiveWordRow",
|
||||
"SkillCatalogSnapshotRow",
|
||||
"SkillRow",
|
||||
"SkillDisplayAdapterRow",
|
||||
"SkillFavoriteRow",
|
||||
@@ -296,6 +332,16 @@ __all__ = [
|
||||
"TagAssignmentRow",
|
||||
"TagRow",
|
||||
"TaskButtonRow",
|
||||
"TaskAnalysisBindingRow",
|
||||
"TaskAnalysisContentVersionRow",
|
||||
"TaskAnalysisRunMessageRow",
|
||||
"TaskAnalysisRunRow",
|
||||
"TaskAnalysisSettingRow",
|
||||
"TaskAnalysisTaskRow",
|
||||
"TaskDataBlockDefRow",
|
||||
"TaskDataBlockRow",
|
||||
"TaskDataPeriodRow",
|
||||
"TaskDataSubmissionRow",
|
||||
"TaskDeeplinkDetailRow",
|
||||
"TaskDeeplinkThreadRow",
|
||||
"AnalysisProjectRow",
|
||||
@@ -307,6 +353,7 @@ __all__ = [
|
||||
"ThreadMetaRow",
|
||||
"ThreadShareRow",
|
||||
"ToolCallMetricRow",
|
||||
"StreamTruncationEventRow",
|
||||
"UploadPolicyRow",
|
||||
"UserCustomPromptRow",
|
||||
"UserMemoryConfigRow",
|
||||
|
||||
+9
-1
@@ -15,7 +15,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy import Index, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -24,6 +24,14 @@ from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
class ParallelAgentTaskRow(Base):
|
||||
__tablename__ = "parallel_agent_tasks"
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_parallel_agent_tasks_user_updated_id",
|
||||
"user_id",
|
||||
"updated_at",
|
||||
"id",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
|
||||
+41
-12
@@ -11,6 +11,7 @@ that belongs to another user is treated as not found.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
@@ -44,17 +45,32 @@ class ParallelAgentTaskRepository:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _to_dict(row: ParallelAgentTaskRow, *, with_messages: bool = True) -> dict[str, Any]:
|
||||
def _to_dict(
|
||||
row: ParallelAgentTaskRow | Mapping[str, Any],
|
||||
*,
|
||||
with_messages: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
def value(name: str) -> Any:
|
||||
if isinstance(row, Mapping):
|
||||
return row[name]
|
||||
return getattr(row, name)
|
||||
|
||||
created_at = value("created_at")
|
||||
updated_at = value("updated_at")
|
||||
data: dict[str, Any] = {
|
||||
"id": row.id,
|
||||
"title": row.title,
|
||||
"brief": row.brief,
|
||||
"agents": _loads(row.agents) or [],
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
"id": value("id"),
|
||||
"title": value("title"),
|
||||
"brief": value("brief"),
|
||||
"agents": _loads(value("agents")) or [],
|
||||
"created_at": (
|
||||
created_at.isoformat() if isinstance(created_at, datetime) else created_at
|
||||
),
|
||||
"updated_at": (
|
||||
updated_at.isoformat() if isinstance(updated_at, datetime) else updated_at
|
||||
),
|
||||
}
|
||||
if with_messages:
|
||||
data["messages"] = _loads(row.messages) or []
|
||||
data["messages"] = _loads(value("messages")) or []
|
||||
return data
|
||||
|
||||
async def list_tasks(
|
||||
@@ -74,9 +90,22 @@ class ParallelAgentTaskRepository:
|
||||
and ``offset`` allow those projections to page only their own task
|
||||
namespace without loading unrelated, potentially large transcripts.
|
||||
"""
|
||||
stmt = select(ParallelAgentTaskRow).where(ParallelAgentTaskRow.user_id == user_id)
|
||||
# History/list callers only need the summary columns. In particular,
|
||||
# never select the LONGTEXT transcript column unless explicitly asked:
|
||||
# MySQL otherwise reads and transfers it even though _to_dict omits it.
|
||||
columns = [
|
||||
ParallelAgentTaskRow.id,
|
||||
ParallelAgentTaskRow.title,
|
||||
ParallelAgentTaskRow.brief,
|
||||
ParallelAgentTaskRow.agents,
|
||||
ParallelAgentTaskRow.created_at,
|
||||
ParallelAgentTaskRow.updated_at,
|
||||
]
|
||||
if include_messages:
|
||||
columns.append(ParallelAgentTaskRow.messages)
|
||||
stmt = select(*columns).where(ParallelAgentTaskRow.user_id == user_id)
|
||||
if id_prefix:
|
||||
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix))
|
||||
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix, autoescape=True))
|
||||
stmt = (
|
||||
stmt.order_by(
|
||||
ParallelAgentTaskRow.updated_at.desc(),
|
||||
@@ -88,8 +117,8 @@ class ParallelAgentTaskRepository:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [
|
||||
self._to_dict(row, with_messages=include_messages)
|
||||
for row in result.scalars()
|
||||
self._to_dict(row._mapping, with_messages=include_messages)
|
||||
for row in result
|
||||
]
|
||||
|
||||
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
|
||||
+9
@@ -7,6 +7,15 @@ from typing import Any
|
||||
|
||||
|
||||
class PositionStore(ABC):
|
||||
async def get_effective_position(self, user_id: str) -> dict[str, Any] | None:
|
||||
"""Return an assigned position or the default without membership counts."""
|
||||
position_id = await self.get_user_position_id(user_id)
|
||||
record = await self.get_position(position_id) if position_id else None
|
||||
if record is None:
|
||||
default_id = await self.get_default_position_id()
|
||||
record = await self.get_position(default_id) if default_id else None
|
||||
return record
|
||||
|
||||
# -------------------------------------------------------------- positions
|
||||
|
||||
@abstractmethod
|
||||
|
||||
+20
@@ -7,6 +7,7 @@ from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.orm import aliased
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
@@ -60,6 +61,25 @@ class PositionRepository(PositionStore):
|
||||
counts = await self._member_counts(session, [position_id])
|
||||
return _position_to_dict(row, counts.get(position_id, 0))
|
||||
|
||||
async def get_effective_position(self, user_id: str) -> dict[str, Any] | None:
|
||||
"""Resolve the assigned/default position with one narrow indexed query."""
|
||||
assigned = aliased(PositionRow)
|
||||
default = aliased(PositionRow)
|
||||
stmt = (
|
||||
select(assigned, default)
|
||||
.select_from(UserRow)
|
||||
.outerjoin(assigned, UserRow.position_id == assigned.id)
|
||||
.outerjoin(default, default.is_default.is_(True))
|
||||
.where(UserRow.id == user_id)
|
||||
.limit(1)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).first()
|
||||
if row is None:
|
||||
return None
|
||||
position = row[0] or row[1]
|
||||
return _position_to_dict(position, 0) if position is not None else None
|
||||
|
||||
async def create_position(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
is_default = bool(data.get("is_default"))
|
||||
|
||||
+7
@@ -75,6 +75,13 @@ class RoundtableChainRow(Base):
|
||||
pending_review: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0", index=True
|
||||
)
|
||||
# 管理员目录开关:True = 从所有普通列表(我的/公共/待审核/选链弹窗)隐藏,仅管理员
|
||||
# 管理视图(``admin_view=true`` 的列表请求)可见。纯目录开关:单条读取(get_chain)
|
||||
# 不过滤,已配置的业务映射 / 进行中的研讨不受影响。``server_default`` 显式声明以支持
|
||||
# 启动期 ``_ensure_orm_columns_sync`` 在既有表上补列(NOT NULL 需带默认)。
|
||||
hidden: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0", index=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
+22
-1
@@ -66,6 +66,8 @@ class RoundtableChainRepository:
|
||||
"enabled": bool(getattr(row, "enabled", True)),
|
||||
# 「待审核」标记:True = owner 已提交审核、等待管理员发布;发布时清回 False。
|
||||
"pending_review": bool(getattr(row, "pending_review", False)),
|
||||
# 管理员目录开关:True = 从普通列表隐藏(仅管理视图可见);缺省 False。
|
||||
"hidden": bool(getattr(row, "hidden", False)),
|
||||
"is_mine": (current_user_id is None) or (row.user_id == current_user_id),
|
||||
# 配置者(owner)的 user_id;路由层据此解析成可读用户名(owner_name)。
|
||||
"owner_id": row.user_id,
|
||||
@@ -74,7 +76,14 @@ class RoundtableChainRepository:
|
||||
}
|
||||
|
||||
async def list_chains(
|
||||
self, user_id: str, *, scope: str = "mine", limit: int = 200, is_admin: bool = False
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
scope: str = "mine",
|
||||
limit: int = 200,
|
||||
is_admin: bool = False,
|
||||
include_hidden: bool = False,
|
||||
hidden: bool | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""列出链条。
|
||||
|
||||
@@ -86,6 +95,10 @@ class RoundtableChainRepository:
|
||||
- ``"all"``:所有账号的全部链条(**仅管理员**,由路由层校验后传 ``is_admin=True``;
|
||||
非管理员误传则按 ``"mine"`` 退化,绝不越权)。
|
||||
其余值按 ``"mine"`` 处理。``is_mine`` 标记每条是否本人所有,供前端判断可否编辑。
|
||||
|
||||
隐藏开关(``hidden=True`` 的行):默认从**所有**列表排除(含 admin 的普通视图,如
|
||||
选链弹窗);``include_hidden=True``(管理员管理视图)时不排除,并可再用 ``hidden``
|
||||
参数筛选「只看隐藏 / 只看显示中」。非管理员的 ``include_hidden`` 一律忽略。
|
||||
"""
|
||||
if scope == "public":
|
||||
cond = RoundtableChainRow.is_public.is_(True)
|
||||
@@ -98,6 +111,11 @@ class RoundtableChainRepository:
|
||||
stmt = select(RoundtableChainRow)
|
||||
if cond is not None:
|
||||
stmt = stmt.where(cond)
|
||||
if include_hidden and is_admin:
|
||||
if hidden is not None:
|
||||
stmt = stmt.where(RoundtableChainRow.hidden.is_(hidden))
|
||||
else:
|
||||
stmt = stmt.where(RoundtableChainRow.hidden.is_(False))
|
||||
stmt = stmt.order_by(RoundtableChainRow.updated_at.desc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
@@ -157,6 +175,7 @@ class RoundtableChainRepository:
|
||||
is_public=_UNSET,
|
||||
enabled=_UNSET,
|
||||
pending_review=_UNSET,
|
||||
hidden=_UNSET,
|
||||
is_admin: bool = False,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
@@ -186,6 +205,8 @@ class RoundtableChainRepository:
|
||||
row.enabled = bool(enabled)
|
||||
if pending_review is not _UNSET and pending_review is not None:
|
||||
row.pending_review = bool(pending_review)
|
||||
if hidden is not _UNSET and hidden is not None:
|
||||
row.hidden = bool(hidden)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
|
||||
+1
-1
@@ -68,7 +68,7 @@ class RoundtableJobRow(Base):
|
||||
# Fine-grained phase (§2.2): initializing / leader_thinking / dispatching /
|
||||
# consensus / report_drawing / report_done.
|
||||
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
|
||||
# Current orchestration cycle (0..MAX_CYCLES).
|
||||
# Current orchestration cycle (0..configured roundtable max_cycles).
|
||||
cycle: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# Dispatch chain nodes (§2.3): JSON array of
|
||||
# { agentId, name, avatarType, order, state, lastSpokeCycle? }.
|
||||
|
||||
+28
-1
@@ -119,6 +119,7 @@ class RoundtableJobRepository:
|
||||
request_hash: str | None = None,
|
||||
active_dedupe_key: str | None = None,
|
||||
input_snapshot: dict[str, Any] | None = None,
|
||||
room_started_at: datetime | None = None,
|
||||
) -> RoundtableJobRow:
|
||||
now = datetime.now(UTC)
|
||||
return RoundtableJobRow(
|
||||
@@ -145,7 +146,7 @@ class RoundtableJobRepository:
|
||||
error=None,
|
||||
attempt=0,
|
||||
version=0,
|
||||
created_at=now,
|
||||
created_at=room_started_at or now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
@@ -204,6 +205,7 @@ class RoundtableJobRepository:
|
||||
request_id: str | None = None,
|
||||
request_hash: str | None = None,
|
||||
input_snapshot: dict[str, Any] | None = None,
|
||||
room_started_at: datetime | None = None,
|
||||
) -> tuple[dict[str, Any], bool]:
|
||||
"""原子地「创建活跃作业 → 冲突则幂等返回已有活跃作业」(Phase 3 唯一活跃键版)。
|
||||
|
||||
@@ -240,6 +242,7 @@ class RoundtableJobRepository:
|
||||
request_hash=request_hash,
|
||||
active_dedupe_key=dedupe_key,
|
||||
input_snapshot=input_snapshot,
|
||||
room_started_at=room_started_at,
|
||||
)
|
||||
|
||||
# 无 draft_id → 无法去重,直接创建(保持原有行为)。
|
||||
@@ -462,6 +465,30 @@ class RoundtableJobRepository:
|
||||
result = await session.execute(stmt)
|
||||
return [r[0] for r in result.all()]
|
||||
|
||||
async def request_cancel_expired_step2(
|
||||
self,
|
||||
*,
|
||||
created_before: datetime,
|
||||
non_occupying_phases: tuple[str, ...],
|
||||
) -> int:
|
||||
"""Cancel unfinished Step-2 jobs that exceeded the room hard lifetime."""
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(RoundtableJobRow)
|
||||
.where(
|
||||
RoundtableJobRow.status.in_(list(CANCELLABLE_STATUSES)),
|
||||
RoundtableJobRow.created_at <= created_before,
|
||||
RoundtableJobRow.phase.not_in(list(non_occupying_phases)),
|
||||
)
|
||||
.values(
|
||||
status="cancel_requested",
|
||||
version=RoundtableJobRow.version + 1,
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def list_cancel_reapable(self, *, now: datetime, limit: int = 20) -> list[str]:
|
||||
"""列出可以由 dispatcher 收口的取消请求。
|
||||
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"""Roundtable step-2 room leases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
|
||||
from deerflow.persistence.roundtable_rooms.sql import RoundtableRoomRepository
|
||||
|
||||
|
||||
def make_roundtable_room_store(session_factory) -> RoundtableRoomRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return RoundtableRoomRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RoundtableRoomLeaseRow",
|
||||
"RoundtableRoomRepository",
|
||||
"make_roundtable_room_store",
|
||||
]
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
"""Foreground roundtable room leases.
|
||||
|
||||
One row is one browser session that has entered step 2 and is still heartbeating.
|
||||
Background jobs are not stored here: they already occupy a room through
|
||||
``roundtable_jobs`` while their status is non-terminal. Both sides share the
|
||||
job active-dedupe key so a foreground session and its later background job
|
||||
count as one room.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class RoundtableRoomLeaseRow(Base):
|
||||
__tablename__ = "roundtable_room_leases"
|
||||
|
||||
room_key: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
draft_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
task_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
# Last account to heartbeat. Release only succeeds for this account, so one
|
||||
# viewer leaving a shared task room does not drop another viewer's lease.
|
||||
holder_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
+584
@@ -0,0 +1,584 @@
|
||||
"""Deployment-wide roundtable room leases and background-job occupancy.
|
||||
|
||||
A room is one live roundtable execution, not one saved draft, not one account,
|
||||
and not one seat. Different pages running the same draft have distinct hidden
|
||||
session ids; every seat and step of one execution shares one id. The active set
|
||||
is the union of every user's:
|
||||
|
||||
- a durable foreground/session lease renewed by the browser or a run entry
|
||||
- live ``extra_keys`` supplied by the Gateway as a reconciliation fallback
|
||||
only when no durable lease/job already represents that persisted scope
|
||||
- a background job that is executing or waiting for the user's Step-2
|
||||
clarification (``queued`` / ``running`` / ``awaiting_input`` /
|
||||
``cancel_requested``)
|
||||
|
||||
Historical jobs older than the hard room lifetime and merely viewing chat
|
||||
history are not rooms. There is no per-user pool.
|
||||
|
||||
Admission and lease creation happen while holding one database-backed gate
|
||||
row. Updating that row serializes contenders on SQLite, MySQL and PostgreSQL,
|
||||
so four simultaneous requests cannot all observe the same stale count.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
|
||||
from deerflow.persistence.roundtable_jobs.sql import compute_active_dedupe_key
|
||||
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
|
||||
|
||||
LEASE_SECONDS = 60
|
||||
JOB_LIVE_SECONDS = 15 * 60
|
||||
OCCUPYING_JOB_STATUSES = ("queued", "running", "awaiting_input", "cancel_requested")
|
||||
# A background job keeps running while Step 3 generates its artifacts, but the
|
||||
# concurrency cap is intentionally scoped to Step 2. ``consensus`` is written
|
||||
# before Step 3 begins, which releases the slot without waiting for reporting.
|
||||
NON_OCCUPYING_JOB_PHASES = frozenset({
|
||||
"consensus",
|
||||
"summary_writing",
|
||||
"report_drawing",
|
||||
"report_done",
|
||||
})
|
||||
GATE_ROW_KEY = "__roundtable_room_gate__"
|
||||
|
||||
|
||||
def compute_room_session_key(room_session_id: str | None) -> str | None:
|
||||
"""Stable key for one live roundtable execution (not one saved draft)."""
|
||||
clean = (room_session_id or "").strip()
|
||||
if not clean:
|
||||
return None
|
||||
return hashlib.sha256(f"roundtable-session:{clean}".encode()).hexdigest()
|
||||
|
||||
|
||||
def _aware(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=UTC)
|
||||
return value
|
||||
|
||||
|
||||
def _job_is_live(
|
||||
lease_until: datetime | None,
|
||||
updated_at: datetime | None,
|
||||
now: datetime,
|
||||
live_after: datetime,
|
||||
) -> bool:
|
||||
until = _aware(lease_until)
|
||||
if until is not None and until > now:
|
||||
return True
|
||||
updated = _aware(updated_at)
|
||||
return updated is not None and updated >= live_after
|
||||
|
||||
|
||||
def _hard_deadline_reached(
|
||||
created_at: datetime | None,
|
||||
now: datetime,
|
||||
max_hold_minutes: int,
|
||||
) -> bool:
|
||||
created = _aware(created_at)
|
||||
return created is not None and created + timedelta(minutes=max_hold_minutes) <= now
|
||||
|
||||
|
||||
class RoundtableRoomRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def admit(
|
||||
self,
|
||||
*,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
room_session_id: str | None = None,
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically reserve/renew a room, or reject when the pool is full."""
|
||||
key = compute_room_session_key(room_session_id) or compute_active_dedupe_key(
|
||||
draft_id, user_id, task_id
|
||||
)
|
||||
return await self.admit_room_key(
|
||||
room_key=key,
|
||||
draft_id=draft_id,
|
||||
user_id=user_id,
|
||||
task_id=task_id,
|
||||
holder_user_id=user_id,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
extra_keys=extra_keys,
|
||||
)
|
||||
|
||||
async def admit_room_key(
|
||||
self,
|
||||
*,
|
||||
room_key: str | None,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
holder_user_id: str | None,
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically admit a pre-resolved room key.
|
||||
|
||||
``room_key`` is used by ``/run/stream`` when it resolves the room from
|
||||
thread metadata. ``user_id`` remains the room owner while
|
||||
``holder_user_id`` is the account currently renewing a shared room.
|
||||
"""
|
||||
async with self._lock:
|
||||
await self._ensure_gate_row()
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
# This no-op-ish write is intentional: it is the portable DB
|
||||
# mutex protecting the subsequent count + lease upsert.
|
||||
await session.execute(
|
||||
update(RoundtableRoomLeaseRow)
|
||||
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
|
||||
.values(updated_at=now)
|
||||
)
|
||||
keys, durable_scopes, hard_expired_keys = await self._active_snapshot(
|
||||
session,
|
||||
now,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
)
|
||||
self._merge_extra_keys(keys, durable_scopes, extra_keys)
|
||||
if room_key is None:
|
||||
await session.commit()
|
||||
blocked = bool(enabled)
|
||||
return self._decision(
|
||||
ok=not blocked,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=False,
|
||||
code="DRAFT_REQUIRED" if blocked else "",
|
||||
message="" if not blocked else "进入第二步需要先保存会商草稿",
|
||||
)
|
||||
if room_key in hard_expired_keys:
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=False,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=False,
|
||||
code="ROOM_MAX_HOLD_EXCEEDED",
|
||||
message="本次会商已达到最长占用时间,房间已自动释放。请返回第一步后重新发起。",
|
||||
)
|
||||
owns = room_key in keys
|
||||
if enabled and not owns and len(keys) >= max_rooms:
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=False,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=False,
|
||||
code="ROUNDTABLE_ROOM_FULL",
|
||||
message="当前会商房间暂时已满,请稍候片刻后再试。",
|
||||
)
|
||||
await self._upsert_lease(
|
||||
session,
|
||||
room_key=room_key,
|
||||
draft_id=draft_id,
|
||||
user_id=user_id,
|
||||
task_id=task_id,
|
||||
holder_user_id=holder_user_id,
|
||||
now=now,
|
||||
)
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=True,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys) if owns else len(keys) + 1,
|
||||
renewed=owns,
|
||||
code="",
|
||||
message="",
|
||||
)
|
||||
|
||||
async def release(
|
||||
self,
|
||||
*,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
room_session_id: str | None = None,
|
||||
) -> bool:
|
||||
"""Drop a leftover foreground lease row if one still exists."""
|
||||
key = compute_room_session_key(room_session_id) or compute_active_dedupe_key(
|
||||
draft_id, user_id, task_id
|
||||
)
|
||||
if not key:
|
||||
return False
|
||||
async with self._lock:
|
||||
await self._ensure_gate_row()
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(RoundtableRoomLeaseRow)
|
||||
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
|
||||
.values(updated_at=datetime.now(UTC))
|
||||
)
|
||||
row = await session.get(RoundtableRoomLeaseRow, key)
|
||||
if row is None or (user_id and row.holder_user_id not in (None, user_id)):
|
||||
await session.commit()
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def started_at(
|
||||
self,
|
||||
*,
|
||||
room_session_id: str | None,
|
||||
holder_user_id: str | None,
|
||||
) -> datetime | None:
|
||||
"""Return the original admission time for a foreground-to-job handoff."""
|
||||
key = compute_room_session_key(room_session_id)
|
||||
if key is None:
|
||||
return None
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableRoomLeaseRow, key)
|
||||
if row is None or (
|
||||
holder_user_id
|
||||
and row.holder_user_id not in (None, holder_user_id)
|
||||
):
|
||||
return None
|
||||
return _aware(row.created_at)
|
||||
|
||||
async def renew(
|
||||
self,
|
||||
*,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
room_session_id: str | None,
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Renew one live foreground lease without ever creating it.
|
||||
|
||||
Heartbeats are deliberately weaker than admission. A delayed browser
|
||||
callback must not resurrect a room that was released after a pause,
|
||||
network disconnect or completed Step 2.
|
||||
"""
|
||||
key = compute_room_session_key(room_session_id)
|
||||
if key is None:
|
||||
return self._decision(
|
||||
ok=not enabled,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=0,
|
||||
renewed=False,
|
||||
code="ROOM_SESSION_REQUIRED" if enabled else "",
|
||||
message="会商会话标识缺失,请刷新页面后重试。" if enabled else "",
|
||||
)
|
||||
|
||||
return await self.renew_room_key(
|
||||
room_key=key,
|
||||
holder_user_id=user_id,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
extra_keys=extra_keys,
|
||||
)
|
||||
|
||||
async def renew_room_key(
|
||||
self,
|
||||
*,
|
||||
room_key: str,
|
||||
holder_user_id: str | None,
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
max_hold_minutes: int = 30,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Renew a resolved persisted key without creating a missing lease."""
|
||||
async with self._lock:
|
||||
await self._ensure_gate_row()
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(RoundtableRoomLeaseRow)
|
||||
.where(RoundtableRoomLeaseRow.room_key == GATE_ROW_KEY)
|
||||
.values(updated_at=now)
|
||||
)
|
||||
keys, durable_scopes, hard_expired_keys = await self._active_snapshot(
|
||||
session,
|
||||
now,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
)
|
||||
self._merge_extra_keys(keys, durable_scopes, extra_keys)
|
||||
row = await session.get(RoundtableRoomLeaseRow, room_key)
|
||||
if row is not None and room_key in hard_expired_keys:
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=False,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=False,
|
||||
code="ROOM_MAX_HOLD_EXCEEDED",
|
||||
message="本次会商已达到最长占用时间,房间已自动释放。请返回第一步后重新发起。",
|
||||
)
|
||||
owns = row is not None and room_key in keys and (
|
||||
not holder_user_id or row.holder_user_id in (None, holder_user_id)
|
||||
)
|
||||
if not owns:
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=not enabled,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=False,
|
||||
code="ROOM_LEASE_EXPIRED" if enabled else "",
|
||||
message="会商连接已断开,请重新恢复研讨。" if enabled else "",
|
||||
)
|
||||
|
||||
row.lease_until = now + timedelta(seconds=LEASE_SECONDS)
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
return self._decision(
|
||||
ok=True,
|
||||
enabled=enabled,
|
||||
max_rooms=max_rooms,
|
||||
active_count=len(keys),
|
||||
renewed=True,
|
||||
code="",
|
||||
message="",
|
||||
)
|
||||
|
||||
async def summary(
|
||||
self,
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None = None,
|
||||
*,
|
||||
max_hold_minutes: int = 30,
|
||||
) -> int:
|
||||
"""Return the number of distinct occupied rooms across every user."""
|
||||
async with self._lock:
|
||||
await self._ensure_gate_row()
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
keys, durable_scopes, _hard_expired_keys = await self._active_snapshot(
|
||||
session,
|
||||
now,
|
||||
max_hold_minutes=max_hold_minutes,
|
||||
)
|
||||
self._merge_extra_keys(keys, durable_scopes, extra_keys)
|
||||
await session.commit()
|
||||
return len(keys)
|
||||
|
||||
@staticmethod
|
||||
def _merge_extra_keys(
|
||||
keys: set[str],
|
||||
durable_scopes: set[str],
|
||||
extra_keys: set[str] | Mapping[str, str | None] | None,
|
||||
) -> None:
|
||||
"""Add run-manager fallbacks without double-counting durable occupancy.
|
||||
|
||||
The map value is the legacy draft/user-or-task scope. If a live page
|
||||
lease or background job already owns that scope, its stamped run key is
|
||||
merely evidence for the same room and must not become another room.
|
||||
"""
|
||||
if not extra_keys:
|
||||
return
|
||||
items = extra_keys.items() if isinstance(extra_keys, Mapping) else ((key, None) for key in extra_keys)
|
||||
for key, scope_key in items:
|
||||
if not key or (scope_key and scope_key in durable_scopes):
|
||||
continue
|
||||
keys.add(key)
|
||||
|
||||
async def _active_snapshot(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
now: datetime,
|
||||
*,
|
||||
max_hold_minutes: int,
|
||||
) -> tuple[set[str], set[str], set[str]]:
|
||||
lease_rows = (
|
||||
await session.execute(
|
||||
select(RoundtableRoomLeaseRow).where(
|
||||
RoundtableRoomLeaseRow.room_key != GATE_ROW_KEY
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
expired: list[str] = []
|
||||
keys: set[str] = set()
|
||||
durable_scopes: set[str] = set()
|
||||
hard_expired_keys: set[str] = set()
|
||||
for row in lease_rows:
|
||||
until = _aware(row.lease_until)
|
||||
if until is None or until <= now:
|
||||
expired.append(row.room_key)
|
||||
continue
|
||||
scope_key = compute_active_dedupe_key(row.draft_id, row.user_id, row.task_id)
|
||||
if _hard_deadline_reached(row.created_at, now, max_hold_minutes):
|
||||
# Keep the still-live row briefly as a non-occupying tombstone.
|
||||
# Its scope suppresses a lagging RunManager fallback until the
|
||||
# browser observes the rejected heartbeat and cancels the run.
|
||||
hard_expired_keys.add(row.room_key)
|
||||
if scope_key:
|
||||
durable_scopes.add(scope_key)
|
||||
continue
|
||||
keys.add(row.room_key)
|
||||
if scope_key:
|
||||
durable_scopes.add(scope_key)
|
||||
if expired:
|
||||
await session.execute(
|
||||
delete(RoundtableRoomLeaseRow).where(
|
||||
RoundtableRoomLeaseRow.room_key.in_(expired)
|
||||
)
|
||||
)
|
||||
|
||||
live_after = now - timedelta(seconds=JOB_LIVE_SECONDS)
|
||||
job_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
RoundtableJobRow.id,
|
||||
RoundtableJobRow.draft_id,
|
||||
RoundtableJobRow.user_id,
|
||||
RoundtableJobRow.task_id,
|
||||
RoundtableJobRow.status,
|
||||
RoundtableJobRow.phase,
|
||||
RoundtableJobRow.lease_until,
|
||||
RoundtableJobRow.updated_at,
|
||||
RoundtableJobRow.created_at,
|
||||
).where(RoundtableJobRow.status.in_(OCCUPYING_JOB_STATUSES))
|
||||
)
|
||||
).all()
|
||||
for job_id, draft_id, user_id, task_id, status, phase, lease_until, updated_at, created_at in job_rows:
|
||||
if phase in NON_OCCUPYING_JOB_PHASES:
|
||||
continue
|
||||
if _hard_deadline_reached(created_at, now, max_hold_minutes):
|
||||
# The worker cancellation watcher will finalize this state.
|
||||
# Marking the intent here also covers queued jobs and workers
|
||||
# that stopped renewing but have not yet been reaped.
|
||||
await session.execute(
|
||||
update(RoundtableJobRow)
|
||||
.where(
|
||||
RoundtableJobRow.id == job_id,
|
||||
RoundtableJobRow.status.in_(["queued", "running", "awaiting_input"]),
|
||||
)
|
||||
.values(
|
||||
status="cancel_requested",
|
||||
version=RoundtableJobRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
continue
|
||||
if status != "awaiting_input" and not _job_is_live(
|
||||
lease_until,
|
||||
updated_at,
|
||||
now,
|
||||
live_after,
|
||||
):
|
||||
continue
|
||||
job_key = compute_active_dedupe_key(draft_id, user_id, task_id)
|
||||
if job_key and job_key in durable_scopes:
|
||||
# Foreground → background handoff can overlap for a few
|
||||
# milliseconds. Both represent one room for the same scope.
|
||||
continue
|
||||
if not job_key:
|
||||
job_key = f"job:{job_id}"
|
||||
keys.add(job_key)
|
||||
scope_key = compute_active_dedupe_key(draft_id, user_id, task_id)
|
||||
if scope_key:
|
||||
durable_scopes.add(scope_key)
|
||||
return keys, durable_scopes, hard_expired_keys
|
||||
|
||||
async def _ensure_gate_row(self) -> None:
|
||||
"""Create the portable database mutex row once, tolerating races."""
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
if await session.get(RoundtableRoomLeaseRow, GATE_ROW_KEY) is not None:
|
||||
return
|
||||
session.add(
|
||||
RoundtableRoomLeaseRow(
|
||||
room_key=GATE_ROW_KEY,
|
||||
draft_id=None,
|
||||
user_id=None,
|
||||
task_id=None,
|
||||
holder_user_id=None,
|
||||
lease_until=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
|
||||
async def _upsert_lease(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
room_key: str,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
holder_user_id: str | None,
|
||||
now: datetime,
|
||||
) -> None:
|
||||
row = await session.get(RoundtableRoomLeaseRow, room_key)
|
||||
lease_until = now + timedelta(seconds=LEASE_SECONDS)
|
||||
if row is None:
|
||||
session.add(
|
||||
RoundtableRoomLeaseRow(
|
||||
room_key=room_key,
|
||||
draft_id=draft_id,
|
||||
user_id=user_id,
|
||||
task_id=task_id,
|
||||
holder_user_id=holder_user_id,
|
||||
lease_until=lease_until,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
return
|
||||
row.draft_id = draft_id
|
||||
row.user_id = user_id
|
||||
row.task_id = task_id
|
||||
row.holder_user_id = holder_user_id
|
||||
row.lease_until = lease_until
|
||||
row.updated_at = now
|
||||
|
||||
@staticmethod
|
||||
def _decision(
|
||||
*,
|
||||
ok: bool,
|
||||
enabled: bool,
|
||||
max_rooms: int,
|
||||
active_count: int,
|
||||
renewed: bool,
|
||||
code: str,
|
||||
message: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"ok": ok,
|
||||
"enabled": enabled,
|
||||
"max_rooms": max_rooms,
|
||||
"active_count": active_count,
|
||||
"renewed": renewed,
|
||||
"code": code,
|
||||
"message": message,
|
||||
}
|
||||
@@ -11,13 +11,15 @@ import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.runtime.runs.store.base import RunStore
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
_TERMINAL_RUN_STATUSES = ("success", "error", "timeout", "interrupted")
|
||||
|
||||
|
||||
class RunRepository(RunStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
@@ -137,6 +139,14 @@ class RunRepository(RunStore):
|
||||
await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(**values))
|
||||
await session.commit()
|
||||
|
||||
async def update_snapshot(self, run_id, *, metadata, kwargs):
|
||||
# Internal RunManager retention path; the run identity comes from its
|
||||
# live registry, never a client-supplied mutation endpoint.
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(metadata_json=self._safe_json(metadata), kwargs_json=self._safe_json(kwargs)))
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
run_id,
|
||||
@@ -205,21 +215,70 @@ class RunRepository(RunStore):
|
||||
await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(**values))
|
||||
await session.commit()
|
||||
|
||||
async def count_by_user(self) -> dict[str, int]:
|
||||
"""Return ``{user_id: run_count}`` via a single GROUP BY query.
|
||||
async def count_by_user(self, *, since: datetime | None = None) -> dict[str, int]:
|
||||
"""Return ``{user_id: run_count}`` via one bounded GROUP BY query.
|
||||
|
||||
Each run is one user question, so this is the source for the admin
|
||||
user center's 问答次数 column. Runs without a ``user_id`` are skipped.
|
||||
"""
|
||||
filters = [RunRow.user_id.is_not(None)]
|
||||
if since is not None:
|
||||
since_dt = since if since.tzinfo else since.replace(tzinfo=UTC)
|
||||
filters.append(RunRow.created_at >= since_dt)
|
||||
stmt = (
|
||||
select(RunRow.user_id, func.count().label("cnt"))
|
||||
.where(RunRow.user_id.is_not(None))
|
||||
.where(*filters)
|
||||
.group_by(RunRow.user_id)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return {r.user_id: r.cnt for r in rows}
|
||||
|
||||
async def purge_terminal_before(
|
||||
self,
|
||||
cutoff: datetime,
|
||||
*,
|
||||
batch_size: int = 500,
|
||||
) -> int:
|
||||
"""Delete one portable, primary-key-bounded batch of old run audits.
|
||||
|
||||
Pending and running records are intentionally excluded, so retention
|
||||
never removes work that could still be resumed or observed. A run must
|
||||
also have been *updated* before the cutoff: an unusually long-running
|
||||
job created months ago but completed today must remain visible. The
|
||||
compact run row is cleaned, while the conversation checkpoint stays
|
||||
under its separate user-history lifecycle.
|
||||
"""
|
||||
size = max(1, min(int(batch_size), 500))
|
||||
cutoff_dt = cutoff if cutoff.tzinfo else cutoff.replace(tzinfo=UTC)
|
||||
async with self._sf() as session:
|
||||
ids = list(
|
||||
(
|
||||
await session.scalars(
|
||||
select(RunRow.run_id)
|
||||
.where(
|
||||
RunRow.created_at < cutoff_dt,
|
||||
RunRow.updated_at < cutoff_dt,
|
||||
RunRow.status.in_(_TERMINAL_RUN_STATUSES),
|
||||
)
|
||||
.order_by(RunRow.created_at.asc(), RunRow.run_id.asc())
|
||||
.limit(size)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not ids:
|
||||
return 0
|
||||
result = await session.execute(
|
||||
delete(RunRow).where(
|
||||
RunRow.run_id.in_(ids),
|
||||
RunRow.created_at < cutoff_dt,
|
||||
RunRow.updated_at < cutoff_dt,
|
||||
RunRow.status.in_(_TERMINAL_RUN_STATUSES),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def aggregate_tokens_by_thread(self, thread_id: str) -> dict[str, Any]:
|
||||
"""Aggregate token usage via a single SQL GROUP BY query."""
|
||||
_completed = RunRow.status.in_(("success", "error"))
|
||||
|
||||
@@ -95,6 +95,15 @@ class SkillStore(ABC):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
|
||||
"""Map skill names to their configured Chinese display names.
|
||||
|
||||
Only names that have a non-empty ``name_zh`` are returned. The default
|
||||
is empty so stores that have not implemented the lookup stay safe.
|
||||
"""
|
||||
del names
|
||||
return {}
|
||||
|
||||
@abstractmethod
|
||||
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
|
||||
"""Return the skill names in ``user_id``'s "我的技能" collection.
|
||||
|
||||
+11
@@ -165,6 +165,17 @@ class MemorySkillStore(SkillStore):
|
||||
async def list_always_on_names(self) -> set[str]:
|
||||
return {name for name, record in self._skills.items() if record.get("always_on")}
|
||||
|
||||
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
|
||||
labels: dict[str, str] = {}
|
||||
for name in names:
|
||||
record = self._skills.get(name)
|
||||
if record is None:
|
||||
continue
|
||||
label = str(record.get("name_zh") or "").strip()
|
||||
if label:
|
||||
labels[name] = label
|
||||
return labels
|
||||
|
||||
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
|
||||
return set(self._favorites.get(user_id, set()))
|
||||
|
||||
|
||||
+36
-1
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, String, Text, UniqueConstraint
|
||||
from sqlalchemy import BigInteger, Boolean, DateTime, Index, Integer, String, Text, UniqueConstraint, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -19,10 +19,30 @@ from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
class SkillRow(Base):
|
||||
__tablename__ = "skills"
|
||||
__table_args__ = (
|
||||
Index("ix_skills_catalog_available_category_created", "available", "category", "created_at"),
|
||||
)
|
||||
|
||||
# Global unique name — must match the on-disk folder name under skills/custom/.
|
||||
# Hyphen-case or snake_case allowed; see deerflow.skills.validation.
|
||||
name: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Compact catalog projection fields. The list API reads these columns from
|
||||
# MySQL and deliberately excludes the long-form ``detail`` text column.
|
||||
category: Mapped[str] = mapped_column(String(16), nullable=False, default="custom", server_default="custom")
|
||||
description: Mapped[str] = mapped_column(String(1024), nullable=False, default="", server_default="")
|
||||
license: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
|
||||
available: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"), index=True)
|
||||
use_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
view_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
patch_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=text("0"))
|
||||
last_activity_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, default=None)
|
||||
state: Mapped[str] = mapped_column(String(16), nullable=False, default="active", server_default="active")
|
||||
pinned: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
|
||||
featured: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=text("0"))
|
||||
source: Mapped[str] = mapped_column(String(16), nullable=False, default="upload", server_default="upload")
|
||||
source_mtime_ns: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default=text("0"))
|
||||
detail_preview: Mapped[str | None] = mapped_column(String(1024), nullable=True, default=None)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# Human-authored long-form detail/introduction, editable by the skill owner
|
||||
@@ -83,3 +103,18 @@ class SkillDisplayAdapterRow(Base):
|
||||
sample_payload: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class SkillCatalogSnapshotRow(Base):
|
||||
"""MySQL-backed projection of filesystem skill metadata for fast list APIs.
|
||||
|
||||
Skill source files remain the authoring format. The Gateway refreshes this
|
||||
one-row projection during startup so homepage listing requests do not scan
|
||||
every SKILL.md or read per-skill usage sidecars.
|
||||
"""
|
||||
|
||||
__tablename__ = "skill_catalog_snapshots"
|
||||
|
||||
cache_key: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
catalog_json: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -109,6 +109,7 @@ class SkillRepository(SkillStore):
|
||||
row.published = bool(data["published"])
|
||||
if "detail" in data:
|
||||
row.detail = data["detail"]
|
||||
row.detail_preview = (str(data["detail"] or "")[:1024] or None)
|
||||
if "name_zh" in data:
|
||||
row.name_zh = data["name_zh"]
|
||||
if "always_on" in data:
|
||||
@@ -131,6 +132,7 @@ class SkillRepository(SkillStore):
|
||||
row.owner_user_id = data["owner_user_id"]
|
||||
if "detail" in data:
|
||||
row.detail = data["detail"]
|
||||
row.detail_preview = (str(data["detail"] or "")[:1024] or None)
|
||||
if "name_zh" in data:
|
||||
row.name_zh = data["name_zh"]
|
||||
if "always_on" in data:
|
||||
@@ -183,6 +185,20 @@ class SkillRepository(SkillStore):
|
||||
result = await session.execute(stmt)
|
||||
return {row[0] for row in result.all()}
|
||||
|
||||
async def list_name_zh(self, names: list[str]) -> dict[str, str]:
|
||||
wanted = list(dict.fromkeys(name for name in names if name))
|
||||
if not wanted:
|
||||
return {}
|
||||
stmt = select(SkillRow.name, SkillRow.name_zh).where(SkillRow.name.in_(wanted))
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
labels: dict[str, str] = {}
|
||||
for name, name_zh in rows:
|
||||
label = (name_zh or "").strip()
|
||||
if label:
|
||||
labels[name] = label
|
||||
return labels
|
||||
|
||||
async def list_favorite_skill_names(self, user_id: str) -> set[str]:
|
||||
stmt = select(SkillFavoriteRow.skill_name).where(SkillFavoriteRow.user_id == user_id)
|
||||
async with self._sf() as session:
|
||||
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
"""Persistence for model-stream truncation events (模型截断记录).
|
||||
|
||||
The patched OpenAI-compatible streaming layer calls
|
||||
:func:`record_stream_truncation_event` whenever a provider stream ends
|
||||
abnormally. The write is fire-and-forget: it is routed onto the engine's home
|
||||
loop via :func:`fire_and_forget_db_write` and must never disturb the model
|
||||
stream itself. Rows are capped at one month by the SQL repository.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from deerflow.persistence.stream_truncation.base import (
|
||||
RETENTION_DAYS,
|
||||
StreamTruncationEvent,
|
||||
StreamTruncationQuery,
|
||||
StreamTruncationStore,
|
||||
)
|
||||
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
|
||||
from deerflow.persistence.stream_truncation.sql import SqlStreamTruncationStore
|
||||
|
||||
__all__ = [
|
||||
"RETENTION_DAYS",
|
||||
"SqlStreamTruncationStore",
|
||||
"StreamTruncationEvent",
|
||||
"StreamTruncationEventRow",
|
||||
"StreamTruncationQuery",
|
||||
"StreamTruncationStore",
|
||||
"make_stream_truncation_store",
|
||||
"record_stream_truncation_event",
|
||||
]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_store_cache: SqlStreamTruncationStore | None = None
|
||||
_store_cache_failed = False
|
||||
|
||||
|
||||
def make_stream_truncation_store(session_factory) -> StreamTruncationStore | None:
|
||||
"""Return a SQL-backed truncation store, or ``None`` without a DB."""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return SqlStreamTruncationStore(session_factory)
|
||||
|
||||
|
||||
def record_stream_truncation_event(
|
||||
*,
|
||||
model_name: str | None,
|
||||
event_type: str,
|
||||
strict_mode: bool = False,
|
||||
finish_reason: str = "none",
|
||||
status: str = "success",
|
||||
error_type: str | None = None,
|
||||
chunk_count: int = 0,
|
||||
duration_ms: int = 0,
|
||||
first_chunk_ms: int = -1,
|
||||
provider_completion_id: str | None = None,
|
||||
run_id: str | None = None,
|
||||
detail: str | None = None,
|
||||
) -> None:
|
||||
"""Best-effort truncation-event write from any loop/thread. Never raises."""
|
||||
global _store_cache, _store_cache_failed
|
||||
if _store_cache is None:
|
||||
if _store_cache_failed:
|
||||
return
|
||||
try:
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
session_factory = get_session_factory()
|
||||
except Exception:
|
||||
session_factory = None
|
||||
if session_factory is None:
|
||||
# No DB configured (memory backend); do not retry on every stream.
|
||||
_store_cache_failed = True
|
||||
return
|
||||
_store_cache = SqlStreamTruncationStore(session_factory)
|
||||
run_id_value = run_id if run_id and run_id not in {"none", ""} else None
|
||||
event = StreamTruncationEvent(
|
||||
id=uuid.uuid4().hex,
|
||||
created_at=datetime.now(UTC),
|
||||
model_name=model_name,
|
||||
provider_completion_id=provider_completion_id
|
||||
if provider_completion_id and provider_completion_id != "none"
|
||||
else None,
|
||||
run_id=run_id_value,
|
||||
event_type=event_type,
|
||||
strict_mode=strict_mode,
|
||||
finish_reason=finish_reason,
|
||||
status=status,
|
||||
error_type=error_type,
|
||||
chunk_count=chunk_count,
|
||||
duration_ms=duration_ms,
|
||||
first_chunk_ms=first_chunk_ms,
|
||||
detail=detail,
|
||||
)
|
||||
try:
|
||||
from deerflow.persistence.engine import fire_and_forget_db_write
|
||||
|
||||
fire_and_forget_db_write(_store_cache.record(event))
|
||||
except Exception:
|
||||
logger.debug("stream truncation event drop", exc_info=True)
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
"""Repository contract for model-stream truncation events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
# Hard retention cap. The user-facing promise is "最多存一个月" of truncation
|
||||
# records; the SQL repository enforces it on every insert so no scheduler is
|
||||
# required for the cap to hold.
|
||||
RETENTION_DAYS = 30
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamTruncationEvent:
|
||||
"""In-memory shape of a truncation event before persistence."""
|
||||
|
||||
id: str
|
||||
created_at: datetime
|
||||
model_name: str | None = None
|
||||
provider_completion_id: str | None = None
|
||||
run_id: str | None = None
|
||||
# silent_no_terminal / incomplete_stream_error / length_truncated /
|
||||
# content_filter / midstream_error
|
||||
event_type: str = "silent_no_terminal"
|
||||
strict_mode: bool = False
|
||||
finish_reason: str = "none"
|
||||
status: str = "success"
|
||||
error_type: str | None = None
|
||||
chunk_count: int = 0
|
||||
duration_ms: int = 0
|
||||
first_chunk_ms: int = -1
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamTruncationQuery:
|
||||
"""Filter set for the admin list endpoint."""
|
||||
|
||||
model_name: str | None = None
|
||||
event_type: str | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
limit: int = 100
|
||||
offset: int = 0
|
||||
|
||||
|
||||
class StreamTruncationStore(ABC):
|
||||
"""Persistence contract for the model-layer recorder + admin API."""
|
||||
|
||||
@abstractmethod
|
||||
async def record(self, event: StreamTruncationEvent) -> None:
|
||||
"""Persist one event and enforce the retention cap. Must never raise."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
"""Return matching rows (newest first), paginated by ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count(self, query: StreamTruncationQuery) -> int:
|
||||
"""Return the unpaginated row count for ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def model_breakdown(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
"""Per-model aggregate: ``[{model_name, event_type counts..., total, last_at}]``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_before(self, cutoff: datetime) -> int:
|
||||
"""Delete rows older than ``cutoff``; return the deleted row count."""
|
||||
raise NotImplementedError
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
"""ORM model for model-stream truncation events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Index, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class StreamTruncationEventRow(Base):
|
||||
"""One row per detected model-stream truncation.
|
||||
|
||||
Written from the patched OpenAI-compatible streaming layer when a provider
|
||||
stream ends abnormally: without a terminal frame (silently committed half
|
||||
answer), via ``IncompleteChatStreamError`` (strict integrity mode, retried),
|
||||
on an output-length cap, or interrupted mid-stream by a transport error.
|
||||
Rows are retained for at most ``RETENTION_DAYS`` by the SQL repository.
|
||||
"""
|
||||
|
||||
__tablename__ = "stream_truncation_events"
|
||||
__table_args__ = (
|
||||
Index("ix_stream_truncation_created_model", "created_at", "model_name"),
|
||||
Index("ix_stream_truncation_created_event", "created_at", "event_type"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), index=True
|
||||
)
|
||||
|
||||
# Configured model name (the config.yaml entry name), sanitized by the model layer.
|
||||
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
# Provider-side completion id when the gateway supplied one; joins a row to
|
||||
# the upstream gateway's own request log.
|
||||
provider_completion_id: Mapped[str | None] = mapped_column(String(128))
|
||||
# LangGraph run id when the model layer could see one ("none" is dropped).
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
|
||||
# silent_no_terminal / incomplete_stream_error / length_truncated /
|
||||
# content_filter / midstream_error
|
||||
event_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
# Whether the model ran with strict_stream_integrity enabled.
|
||||
strict_mode: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
# Raw provider finish reason bucket: none / stop / length / tool_calls /
|
||||
# function_call / content_filter / end_turn / other.
|
||||
finish_reason: Mapped[str] = mapped_column(String(16), default="none")
|
||||
# Stream outcome from the model layer: success / error / cancelled.
|
||||
status: Mapped[str] = mapped_column(String(16), default="success")
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
chunk_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
first_chunk_ms: Mapped[int] = mapped_column(Integer, default=-1)
|
||||
|
||||
detail: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat()
|
||||
if isinstance(self.created_at, datetime)
|
||||
else self.created_at,
|
||||
"model_name": self.model_name,
|
||||
"provider_completion_id": self.provider_completion_id,
|
||||
"run_id": self.run_id,
|
||||
"event_type": self.event_type,
|
||||
"strict_mode": bool(self.strict_mode),
|
||||
"finish_reason": self.finish_reason,
|
||||
"status": self.status,
|
||||
"error_type": self.error_type,
|
||||
"chunk_count": self.chunk_count,
|
||||
"duration_ms": self.duration_ms,
|
||||
"first_chunk_ms": self.first_chunk_ms,
|
||||
"detail": self.detail,
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
"""SQLAlchemy-backed stream-truncation event repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, desc, func, select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.stream_truncation.base import (
|
||||
RETENTION_DAYS,
|
||||
StreamTruncationEvent,
|
||||
StreamTruncationQuery,
|
||||
StreamTruncationStore,
|
||||
)
|
||||
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SqlStreamTruncationStore(StreamTruncationStore):
|
||||
"""Persist truncation events with a hard one-month retention cap."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
async def _ensure_table(self, session) -> None:
|
||||
# Table creation follows the repo's create_all-on-first-use convention
|
||||
# (see e.g. task_buttons); safe to call repeatedly.
|
||||
try:
|
||||
await session.run_sync(
|
||||
lambda sync_session: StreamTruncationEventRow.__table__.create(
|
||||
sync_session.bind, checkfirst=True
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("stream_truncation table create skipped", exc_info=True)
|
||||
|
||||
async def record(self, event: StreamTruncationEvent) -> None:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
session.add(
|
||||
StreamTruncationEventRow(
|
||||
id=event.id,
|
||||
created_at=event.created_at,
|
||||
model_name=event.model_name,
|
||||
provider_completion_id=event.provider_completion_id,
|
||||
run_id=event.run_id,
|
||||
event_type=event.event_type,
|
||||
strict_mode=event.strict_mode,
|
||||
finish_reason=event.finish_reason,
|
||||
status=event.status,
|
||||
error_type=event.error_type,
|
||||
chunk_count=event.chunk_count,
|
||||
duration_ms=event.duration_ms,
|
||||
first_chunk_ms=event.first_chunk_ms,
|
||||
detail=event.detail,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
# Same session/transaction scope: enforce the retention cap on
|
||||
# every insert. Truncations are rare, so this stays cheap and
|
||||
# keeps the one-month promise without any scheduler.
|
||||
await self._enforce_retention(session)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to persist stream truncation event model=%s type=%s",
|
||||
event.model_name,
|
||||
event.event_type,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
async def _enforce_retention(self, session) -> None:
|
||||
"""Delete rows beyond the retention cap. Best-effort, never raises."""
|
||||
try:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=RETENTION_DAYS)
|
||||
await session.execute(
|
||||
delete(StreamTruncationEventRow).where(
|
||||
StreamTruncationEventRow.created_at < cutoff
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.debug("stream truncation retention sweep failed", exc_info=True)
|
||||
|
||||
def _apply_filters(self, query: StreamTruncationQuery):
|
||||
stmt = select(StreamTruncationEventRow)
|
||||
if query.model_name:
|
||||
stmt = stmt.where(StreamTruncationEventRow.model_name == query.model_name)
|
||||
if query.event_type:
|
||||
stmt = stmt.where(StreamTruncationEventRow.event_type == query.event_type)
|
||||
if query.since is not None:
|
||||
stmt = stmt.where(StreamTruncationEventRow.created_at >= query.since)
|
||||
if query.until is not None:
|
||||
stmt = stmt.where(StreamTruncationEventRow.created_at <= query.until)
|
||||
return stmt
|
||||
|
||||
async def list(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
stmt = self._apply_filters(query).order_by(
|
||||
desc(StreamTruncationEventRow.created_at)
|
||||
).limit(query.limit).offset(query.offset)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [row.to_dict() for row in rows]
|
||||
except Exception:
|
||||
logger.warning("Failed to list stream truncation events", exc_info=True)
|
||||
return []
|
||||
|
||||
async def count(self, query: StreamTruncationQuery) -> int:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
stmt = self._apply_filters(query).with_only_columns(
|
||||
func.count(StreamTruncationEventRow.id)
|
||||
)
|
||||
return int((await session.execute(stmt)).scalar() or 0)
|
||||
except Exception:
|
||||
logger.warning("Failed to count stream truncation events", exc_info=True)
|
||||
return 0
|
||||
|
||||
async def model_breakdown(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
stmt = self._apply_filters(query).with_only_columns(
|
||||
StreamTruncationEventRow.model_name,
|
||||
StreamTruncationEventRow.event_type,
|
||||
func.count(StreamTruncationEventRow.id).label("total"),
|
||||
func.max(StreamTruncationEventRow.created_at).label("last_at"),
|
||||
).group_by(StreamTruncationEventRow.model_name, StreamTruncationEventRow.event_type)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
merged: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
name = row.model_name or "(unknown)"
|
||||
bucket = merged.setdefault(
|
||||
name, {"model_name": name, "total": 0, "last_at": None}
|
||||
)
|
||||
bucket[row.event_type] = int(row.total or 0)
|
||||
bucket["total"] += int(row.total or 0)
|
||||
row_last = row.last_at
|
||||
if row_last is not None and (
|
||||
bucket["last_at"] is None or row_last > bucket["last_at"]
|
||||
):
|
||||
bucket["last_at"] = row_last
|
||||
result = sorted(
|
||||
merged.values(), key=lambda item: item["last_at"] or datetime.min.replace(tzinfo=UTC), reverse=True
|
||||
)
|
||||
for item in result:
|
||||
if isinstance(item["last_at"], datetime):
|
||||
item["last_at"] = item["last_at"].isoformat()
|
||||
return result
|
||||
except Exception:
|
||||
logger.warning("Failed to aggregate stream truncation events", exc_info=True)
|
||||
return []
|
||||
|
||||
async def purge_before(self, cutoff: datetime) -> int:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
result = await session.execute(
|
||||
delete(StreamTruncationEventRow).where(
|
||||
StreamTruncationEventRow.created_at < cutoff
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
rowcount = result.rowcount
|
||||
return 0 if rowcount is None else int(rowcount)
|
||||
except Exception:
|
||||
logger.warning("Failed to purge stream truncation events", exc_info=True)
|
||||
return 0
|
||||
+9
-2
@@ -14,7 +14,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, UniqueConstraint
|
||||
from sqlalchemy import Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -43,7 +43,14 @@ class TagRow(Base):
|
||||
|
||||
class TagAssignmentRow(Base):
|
||||
__tablename__ = "tag_assignments"
|
||||
__table_args__ = (UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),)
|
||||
__table_args__ = (
|
||||
UniqueConstraint("tag_id", "target_type", "target_id", name="uq_tag_assignments_tag_target"),
|
||||
# Gallery list joins assignments by (type, target); the tag filter
|
||||
# looks up (type, tag). The unique key leads with tag_id, so neither
|
||||
# query can use it as a prefix.
|
||||
Index("ix_tag_assignments_type_target", "target_type", "target_id"),
|
||||
Index("ix_tag_assignments_type_tag", "target_type", "tag_id", "target_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
tag_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
|
||||
+14
-2
@@ -7,10 +7,21 @@ from typing import Any
|
||||
|
||||
|
||||
class TaskDeeplinkDetailStore(ABC):
|
||||
"""Shared external task snapshots keyed by ``(detail_type, detail_id)``."""
|
||||
"""Shared external task snapshots keyed by ``(detail_type, detail_id, sub_type)``.
|
||||
|
||||
``sub_type`` is the integrator-supplied business type (the deep-link
|
||||
``?type=``): the same task id may hold one snapshot per type, each with its
|
||||
own version counter. The empty string is the legacy no-type partition.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def get_detail(self, detail_type: str, detail_id: str) -> dict[str, Any] | None:
|
||||
async def get_detail(
|
||||
self,
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
"""Return one persisted task snapshot, or ``None`` when it is unknown."""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -20,6 +31,7 @@ class TaskDeeplinkDetailStore(ABC):
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
task_name: str,
|
||||
task_content: str,
|
||||
overview: str,
|
||||
|
||||
+12
-4
@@ -11,10 +11,16 @@ from deerflow.persistence.task_deeplink_details.base import TaskDeeplinkDetailSt
|
||||
|
||||
class MemoryTaskDeeplinkDetailStore(TaskDeeplinkDetailStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
self._rows: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
|
||||
async def get_detail(self, detail_type: str, detail_id: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get((detail_type, detail_id))
|
||||
async def get_detail(
|
||||
self,
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
row = self._rows.get((detail_type, detail_id, sub_type))
|
||||
return deepcopy(row) if row is not None else None
|
||||
|
||||
async def upsert_detail(
|
||||
@@ -22,13 +28,14 @@ class MemoryTaskDeeplinkDetailStore(TaskDeeplinkDetailStore):
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
task_name: str,
|
||||
task_content: str,
|
||||
overview: str,
|
||||
extra: dict[str, Any],
|
||||
updated_by: str | None,
|
||||
) -> tuple[dict[str, Any], bool, bool]:
|
||||
key = (detail_type, detail_id)
|
||||
key = (detail_type, detail_id, sub_type)
|
||||
row = self._rows.get(key)
|
||||
created = row is None
|
||||
if row is None:
|
||||
@@ -36,6 +43,7 @@ class MemoryTaskDeeplinkDetailStore(TaskDeeplinkDetailStore):
|
||||
row = {
|
||||
"detail_type": detail_type,
|
||||
"detail_id": detail_id,
|
||||
"sub_type": sub_type,
|
||||
"version": 1,
|
||||
"created_by": updated_by,
|
||||
"created_at": now,
|
||||
|
||||
+3
@@ -18,6 +18,9 @@ class TaskDeeplinkDetailRow(Base):
|
||||
|
||||
detail_type: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
detail_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
# Integrator-supplied business type (the deep-link ``?type=``): one task id
|
||||
# may carry several typed snapshots; "" is the legacy no-type partition.
|
||||
sub_type: Mapped[str] = mapped_column(String(64), primary_key=True, default="")
|
||||
task_name: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
task_content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
|
||||
overview: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
|
||||
|
||||
+12
-3
@@ -15,6 +15,7 @@ def _to_dict(row: TaskDeeplinkDetailRow) -> dict[str, Any]:
|
||||
return {
|
||||
"detail_type": row.detail_type,
|
||||
"detail_id": row.detail_id,
|
||||
"sub_type": row.sub_type or "",
|
||||
"task_name": row.task_name,
|
||||
"task_content": row.task_content,
|
||||
"overview": row.overview,
|
||||
@@ -31,11 +32,17 @@ class TaskDeeplinkDetailRepository(TaskDeeplinkDetailStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def get_detail(self, detail_type: str, detail_id: str) -> dict[str, Any] | None:
|
||||
async def get_detail(
|
||||
self,
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(
|
||||
TaskDeeplinkDetailRow,
|
||||
{"detail_type": detail_type, "detail_id": detail_id},
|
||||
{"detail_type": detail_type, "detail_id": detail_id, "sub_type": sub_type},
|
||||
)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
@@ -44,6 +51,7 @@ class TaskDeeplinkDetailRepository(TaskDeeplinkDetailStore):
|
||||
detail_type: str,
|
||||
detail_id: str,
|
||||
*,
|
||||
sub_type: str = "",
|
||||
task_name: str,
|
||||
task_content: str,
|
||||
overview: str,
|
||||
@@ -53,7 +61,7 @@ class TaskDeeplinkDetailRepository(TaskDeeplinkDetailStore):
|
||||
async with self._sf() as session:
|
||||
row = await session.get(
|
||||
TaskDeeplinkDetailRow,
|
||||
{"detail_type": detail_type, "detail_id": detail_id},
|
||||
{"detail_type": detail_type, "detail_id": detail_id, "sub_type": sub_type},
|
||||
)
|
||||
created = row is None
|
||||
now = datetime.now(UTC)
|
||||
@@ -61,6 +69,7 @@ class TaskDeeplinkDetailRepository(TaskDeeplinkDetailStore):
|
||||
row = TaskDeeplinkDetailRow(
|
||||
detail_type=detail_type,
|
||||
detail_id=detail_id,
|
||||
sub_type=sub_type,
|
||||
task_name=task_name,
|
||||
task_content=task_content,
|
||||
overview=overview,
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""Dedicated taskId + goPath conversation-history index."""
|
||||
"""Dedicated taskId + goPath conversation-history cards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
+31
-5
@@ -1,8 +1,9 @@
|
||||
"""Shared helpers for task deep-link conversation indexing.
|
||||
"""Shared helpers for task deep-link conversation history.
|
||||
|
||||
The LangGraph thread/checkpoint remains the source of truth for messages and
|
||||
execution state. This package only owns a small, dedicated lookup index so the
|
||||
task workspaces never have to search/filter the global ``threads_meta`` table.
|
||||
execution state. This package owns a dedicated history card so the task
|
||||
workspaces never have to search, filter, or join the global ``threads_meta``
|
||||
catalogue when listing a task's conversations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -10,6 +11,12 @@ from __future__ import annotations
|
||||
from typing import Any, TypedDict
|
||||
|
||||
TASK_DEEPLINK_GO_PATHS = frozenset({"3qfx", "rwfx", "xdfx", "b5-1"})
|
||||
# Custom business mappings use their administrator-defined name as goPath.
|
||||
# Keep this in lock-step with ``business_mapping.business_code``.
|
||||
MAX_TASK_DEEPLINK_GO_PATH = 32
|
||||
|
||||
# Matches the ``task_deeplink_threads.sub_type`` column width.
|
||||
MAX_TASK_DEEPLINK_SUB_TYPE = 64
|
||||
|
||||
|
||||
class TaskDeeplinkBinding(TypedDict):
|
||||
@@ -17,6 +24,21 @@ class TaskDeeplinkBinding(TypedDict):
|
||||
source_task_id: str
|
||||
go_path: str
|
||||
agent_id: str | None
|
||||
# b5-1 integrator-supplied ?type= partition; "" when the link had none.
|
||||
sub_type: str
|
||||
|
||||
|
||||
def is_valid_task_deeplink_go_path(value: str) -> bool:
|
||||
"""Accept built-ins and validated custom mapping names.
|
||||
|
||||
The harness cannot import the app-level business-mapping store (the import
|
||||
firewall deliberately forbids that), so it validates the safe wire format
|
||||
here. Creation itself remains admin-only in the Gateway router.
|
||||
"""
|
||||
go_path = value.strip().lower()
|
||||
if go_path in TASK_DEEPLINK_GO_PATHS:
|
||||
return True
|
||||
return bool(go_path) and len(go_path) <= MAX_TASK_DEEPLINK_GO_PATH and not go_path.endswith("-a")
|
||||
|
||||
|
||||
def task_deeplink_binding(metadata: dict[str, Any] | None) -> TaskDeeplinkBinding | None:
|
||||
@@ -26,21 +48,25 @@ def task_deeplink_binding(metadata: dict[str, Any] | None) -> TaskDeeplinkBindin
|
||||
actual task id for 3qfx/rwfx and the action id for xdfx. Older callers may
|
||||
omit it, in which case ``taskId`` is a safe fallback. A ``taskGoPath`` is
|
||||
mandatory, deliberately preventing legacy task rows from being migrated or
|
||||
accidentally mixed into the new store.
|
||||
accidentally mixed into the new store. ``taskType`` (the b5-1 ``?type=``
|
||||
business type) further partitions one task's history so integrator-supplied
|
||||
types never mix; it is capped to the column width here.
|
||||
"""
|
||||
|
||||
md = metadata or {}
|
||||
go_path = str(md.get("taskGoPath") or "").strip().lower()
|
||||
if go_path not in TASK_DEEPLINK_GO_PATHS:
|
||||
if not is_valid_task_deeplink_go_path(go_path):
|
||||
return None
|
||||
source_task_id = str(md.get("taskId") or "").strip()
|
||||
task_id = str(md.get("taskRecordId") or source_task_id).strip()
|
||||
if not task_id or not source_task_id:
|
||||
return None
|
||||
agent_id = str(md.get("agent_id") or "").strip() or None
|
||||
sub_type = str(md.get("taskType") or "").strip()[:MAX_TASK_DEEPLINK_SUB_TYPE]
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"source_task_id": source_task_id,
|
||||
"go_path": go_path,
|
||||
"agent_id": agent_id,
|
||||
"sub_type": sub_type,
|
||||
}
|
||||
+32
-22
@@ -1,4 +1,4 @@
|
||||
"""In-memory task deep-link index backed by LangGraph BaseStore."""
|
||||
"""In-memory task deep-link history cards backed by LangGraph BaseStore."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -16,6 +16,19 @@ if TYPE_CHECKING:
|
||||
TASK_DEEPLINK_THREADS_NS: tuple[str, ...] = ("task_deeplink_threads",)
|
||||
|
||||
|
||||
def _card_record(thread: dict[str, Any], binding: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"thread_id": str(thread.get("thread_id") or ""),
|
||||
**binding,
|
||||
"display_name": thread.get("display_name"),
|
||||
"status": thread.get("status") or "idle",
|
||||
"metadata": thread.get("metadata") or {},
|
||||
"created_by": thread.get("user_id"),
|
||||
"created_at": thread.get("created_at") or now_iso(),
|
||||
"updated_at": thread.get("updated_at") or now_iso(),
|
||||
}
|
||||
|
||||
|
||||
async def sync_task_deeplink_index(store: BaseStore, thread: dict[str, Any]) -> None:
|
||||
metadata = thread.get("metadata") or {}
|
||||
binding = task_deeplink_binding(metadata)
|
||||
@@ -26,17 +39,7 @@ async def sync_task_deeplink_index(store: BaseStore, thread: dict[str, Any]) ->
|
||||
if "taskGoPath" in metadata or "taskRecordId" in metadata:
|
||||
await store.adelete(TASK_DEEPLINK_THREADS_NS, thread_id)
|
||||
return
|
||||
await store.aput(
|
||||
TASK_DEEPLINK_THREADS_NS,
|
||||
thread_id,
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
**binding,
|
||||
"created_by": thread.get("user_id"),
|
||||
"created_at": thread.get("created_at") or now_iso(),
|
||||
"updated_at": thread.get("updated_at") or now_iso(),
|
||||
},
|
||||
)
|
||||
await store.aput(TASK_DEEPLINK_THREADS_NS, thread_id, _card_record(thread, binding))
|
||||
|
||||
|
||||
async def delete_task_deeplink_index(store: BaseStore, thread_id: str) -> None:
|
||||
@@ -46,6 +49,8 @@ async def delete_task_deeplink_index(store: BaseStore, thread_id: str) -> None:
|
||||
class MemoryTaskDeeplinkThreadStore:
|
||||
def __init__(self, store: BaseStore, thread_store: ThreadMetaStore) -> None:
|
||||
self._store = store
|
||||
# Kept for constructor compatibility with the factory. Listing no
|
||||
# longer reads the generic thread catalogue.
|
||||
self._thread_store = thread_store
|
||||
|
||||
async def list_threads(
|
||||
@@ -54,28 +59,33 @@ class MemoryTaskDeeplinkThreadStore:
|
||||
go_path: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
sub_type: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
# ``sub_type`` semantics mirror the SQL repository: provided value (even
|
||||
# "") narrows to that b5-1 ?type= partition; None returns all sub-types.
|
||||
conditions = {"task_id": task_id, "go_path": go_path}
|
||||
if sub_type is not None:
|
||||
conditions["sub_type"] = sub_type
|
||||
items = await self._store.asearch(
|
||||
TASK_DEEPLINK_THREADS_NS,
|
||||
filter={"task_id": task_id, "go_path": go_path},
|
||||
filter=conditions,
|
||||
limit=limit,
|
||||
)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
thread_id = str(item.value.get("thread_id") or "")
|
||||
value = item.value or {}
|
||||
thread_id = str(value.get("thread_id") or "")
|
||||
if not thread_id:
|
||||
continue
|
||||
thread = await self._thread_store.get(thread_id, user_id=None)
|
||||
if thread is None:
|
||||
continue
|
||||
display_name = value.get("display_name")
|
||||
rows.append(
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"status": thread.get("status", "idle"),
|
||||
"created_at": thread.get("created_at", ""),
|
||||
"updated_at": thread.get("updated_at", ""),
|
||||
"metadata": thread.get("metadata") or {},
|
||||
"values": ({"title": thread.get("display_name")} if thread.get("display_name") else {}),
|
||||
"status": value.get("status") or "idle",
|
||||
"created_at": value.get("created_at", ""),
|
||||
"updated_at": value.get("updated_at", ""),
|
||||
"metadata": value.get("metadata") or {},
|
||||
"values": ({"title": display_name} if display_name else {}),
|
||||
"interrupts": {},
|
||||
}
|
||||
)
|
||||
|
||||
+24
-7
@@ -1,4 +1,4 @@
|
||||
"""Dedicated lookup rows for single-agent task deep-link conversations."""
|
||||
"""Dedicated history cards for single-agent task deep-link conversations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -8,15 +8,15 @@ from sqlalchemy import Index, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class TaskDeeplinkThreadRow(Base):
|
||||
"""One task-workspace history entry pointing at one LangGraph thread.
|
||||
"""One task-workspace history card pointing at one LangGraph thread.
|
||||
|
||||
The row intentionally stores no messages or title. Those remain in the
|
||||
canonical checkpoint / ``threads_meta`` record and are joined by primary
|
||||
key when listing a task's history.
|
||||
List queries read only this table. Title / status / metadata are copied
|
||||
here so the huge generic ``threads_meta`` catalogue is never joined.
|
||||
Messages stay in the LangGraph checkpoint.
|
||||
"""
|
||||
|
||||
__tablename__ = "task_deeplink_threads"
|
||||
@@ -25,6 +25,17 @@ class TaskDeeplinkThreadRow(Base):
|
||||
"ix_task_deeplink_threads_scope",
|
||||
"task_id",
|
||||
"go_path",
|
||||
"sub_type",
|
||||
"updated_at",
|
||||
"thread_id",
|
||||
),
|
||||
# 3qfx/rwfx/xdfx omit sub_type. The scope index is ordered by
|
||||
# sub_type before updated_at, so those listings cannot ride it for
|
||||
# "newest first" without a filesort. This prefix matches that query.
|
||||
Index(
|
||||
"ix_task_deeplink_threads_task_go_updated",
|
||||
"task_id",
|
||||
"go_path",
|
||||
"updated_at",
|
||||
"thread_id",
|
||||
),
|
||||
@@ -35,8 +46,14 @@ class TaskDeeplinkThreadRow(Base):
|
||||
task_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
# Resolved real task id used by model/run context (equals task_id except xdfx).
|
||||
source_task_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
go_path: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
go_path: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# b5-1 ?type= business type; "" for links without one (and for the other
|
||||
# go paths, which have no type dimension).
|
||||
sub_type: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
display_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(20), nullable=False, default="idle")
|
||||
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
|
||||
# Audit only. Task deep-link histories remain shared across authenticated users.
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
+67
-44
@@ -1,11 +1,11 @@
|
||||
"""SQL implementation of the dedicated task deep-link conversation index."""
|
||||
"""SQL implementation of the dedicated task deep-link conversation cards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.task_deeplink_threads.base import task_deeplink_binding
|
||||
@@ -15,8 +15,28 @@ if TYPE_CHECKING:
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
|
||||
|
||||
def _card_payload(thread: TaskDeeplinkThreadRow) -> dict[str, Any]:
|
||||
return {
|
||||
"thread_id": thread.thread_id,
|
||||
"status": thread.status or "idle",
|
||||
"created_at": thread.created_at,
|
||||
"updated_at": thread.updated_at,
|
||||
"metadata": thread.metadata_json or {},
|
||||
"values": ({"title": thread.display_name} if thread.display_name else {}),
|
||||
"interrupts": {},
|
||||
}
|
||||
|
||||
|
||||
def _apply_card_fields(row: TaskDeeplinkThreadRow, thread: ThreadMetaRow, now: datetime) -> None:
|
||||
row.display_name = thread.display_name
|
||||
row.status = thread.status or "idle"
|
||||
row.metadata_json = dict(thread.metadata_json or {})
|
||||
row.created_by = thread.user_id
|
||||
row.updated_at = now
|
||||
|
||||
|
||||
async def sync_task_deeplink_index(session: AsyncSession, thread: ThreadMetaRow) -> None:
|
||||
"""Synchronize one index row inside the caller's thread-meta transaction."""
|
||||
"""Synchronize one history card inside the caller's thread-meta transaction."""
|
||||
|
||||
metadata = thread.metadata_json or {}
|
||||
binding = task_deeplink_binding(metadata)
|
||||
@@ -30,25 +50,39 @@ async def sync_task_deeplink_index(session: AsyncSession, thread: ThreadMetaRow)
|
||||
existing = await session.get(TaskDeeplinkThreadRow, thread.thread_id)
|
||||
now = thread.updated_at if isinstance(thread.updated_at, datetime) else datetime.now(UTC)
|
||||
if existing is None:
|
||||
session.add(
|
||||
TaskDeeplinkThreadRow(
|
||||
thread_id=thread.thread_id,
|
||||
task_id=binding["task_id"],
|
||||
source_task_id=binding["source_task_id"],
|
||||
go_path=binding["go_path"],
|
||||
agent_id=binding["agent_id"],
|
||||
created_by=thread.user_id,
|
||||
created_at=(thread.created_at if isinstance(thread.created_at, datetime) else now),
|
||||
updated_at=now,
|
||||
)
|
||||
row = TaskDeeplinkThreadRow(
|
||||
thread_id=thread.thread_id,
|
||||
task_id=binding["task_id"],
|
||||
source_task_id=binding["source_task_id"],
|
||||
go_path=binding["go_path"],
|
||||
sub_type=binding["sub_type"],
|
||||
agent_id=binding["agent_id"],
|
||||
created_by=thread.user_id,
|
||||
created_at=(thread.created_at if isinstance(thread.created_at, datetime) else now),
|
||||
updated_at=now,
|
||||
)
|
||||
_apply_card_fields(row, thread, now)
|
||||
session.add(row)
|
||||
return
|
||||
|
||||
existing.task_id = binding["task_id"]
|
||||
existing.source_task_id = binding["source_task_id"]
|
||||
existing.go_path = binding["go_path"]
|
||||
existing.sub_type = binding["sub_type"]
|
||||
existing.agent_id = binding["agent_id"]
|
||||
existing.updated_at = now
|
||||
_apply_card_fields(existing, thread, now)
|
||||
|
||||
|
||||
async def update_task_deeplink_card(session: AsyncSession, thread_id: str, **values: Any) -> None:
|
||||
"""Patch a history card without reading ``threads_meta``.
|
||||
|
||||
Title / status / ``updated_at`` changes on ordinary chats no-op here
|
||||
because those ``thread_id`` values are absent from this table.
|
||||
"""
|
||||
|
||||
if not values:
|
||||
return
|
||||
await session.execute(update(TaskDeeplinkThreadRow).where(TaskDeeplinkThreadRow.thread_id == thread_id).values(**values))
|
||||
|
||||
|
||||
async def delete_task_deeplink_index(session: AsyncSession, thread_id: str) -> None:
|
||||
@@ -65,41 +99,30 @@ class TaskDeeplinkThreadRepository:
|
||||
go_path: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
sub_type: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List one task/goPath partition without scanning thread metadata JSON."""
|
||||
"""List one task/goPath partition from the dedicated card table only.
|
||||
|
||||
# Local import avoids a package-initialization cycle: thread_meta.sql
|
||||
# imports the sync helpers above for atomic dual writes.
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
``sub_type`` narrows the partition to one b5-1 ``?type=`` value when
|
||||
provided (an explicit "" selects the no-type partition); ``None`` keeps
|
||||
the untyped legacy behaviour of returning every sub-type at once.
|
||||
"""
|
||||
|
||||
conditions = [
|
||||
TaskDeeplinkThreadRow.task_id == task_id,
|
||||
TaskDeeplinkThreadRow.go_path == go_path,
|
||||
]
|
||||
if sub_type is not None:
|
||||
conditions.append(TaskDeeplinkThreadRow.sub_type == sub_type)
|
||||
stmt = (
|
||||
select(TaskDeeplinkThreadRow, ThreadMetaRow)
|
||||
.join(
|
||||
ThreadMetaRow,
|
||||
ThreadMetaRow.thread_id == TaskDeeplinkThreadRow.thread_id,
|
||||
)
|
||||
.where(
|
||||
TaskDeeplinkThreadRow.task_id == task_id,
|
||||
TaskDeeplinkThreadRow.go_path == go_path,
|
||||
)
|
||||
select(TaskDeeplinkThreadRow)
|
||||
.where(*conditions)
|
||||
.order_by(
|
||||
ThreadMetaRow.updated_at.desc(),
|
||||
ThreadMetaRow.thread_id.desc(),
|
||||
TaskDeeplinkThreadRow.updated_at.desc(),
|
||||
TaskDeeplinkThreadRow.thread_id.desc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
|
||||
return [
|
||||
{
|
||||
"thread_id": thread.thread_id,
|
||||
"status": thread.status,
|
||||
"created_at": thread.created_at,
|
||||
"updated_at": thread.updated_at,
|
||||
"metadata": thread.metadata_json or {},
|
||||
"values": ({"title": thread.display_name} if thread.display_name else {}),
|
||||
"interrupts": {},
|
||||
}
|
||||
for _index, thread in rows
|
||||
]
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [_card_payload(row) for row in rows]
|
||||
+8
-1
@@ -192,10 +192,14 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
item = await self._store.aget(THREADS_NS, thread_id)
|
||||
if item is None:
|
||||
return not require_existing
|
||||
record_user_id = item.value.get("user_id")
|
||||
if record_user_id:
|
||||
from deerflow.runtime.thread_paths import remember_thread_owner
|
||||
|
||||
remember_thread_owner(thread_id, str(record_user_id))
|
||||
# 任务深链 / 圆桌 / 任务分析系统线程:全局共享(与 SQL store 同口径)。
|
||||
if is_shared_access_thread(item.value.get("metadata") or {}):
|
||||
return True
|
||||
record_user_id = item.value.get("user_id")
|
||||
if record_user_id is None:
|
||||
return True
|
||||
return record_user_id == user_id
|
||||
@@ -207,6 +211,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
record["display_name"] = display_name
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
await sync_task_deeplink_index(self._store, record)
|
||||
|
||||
async def update_status(self, thread_id: str, status: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_status")
|
||||
@@ -215,6 +220,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
record["status"] = status
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
await sync_task_deeplink_index(self._store, record)
|
||||
|
||||
async def update_assistant_id(self, thread_id: str, assistant_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_assistant_id")
|
||||
@@ -223,6 +229,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
record["assistant_id"] = assistant_id
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
await sync_task_deeplink_index(self._store, record)
|
||||
|
||||
async def update_metadata(self, thread_id: str, metadata: dict, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_metadata")
|
||||
|
||||
+158
-167
@@ -5,72 +5,60 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.json_compat import JsonPredicate, supports_json_filter
|
||||
from deerflow.persistence.task_deeplink_threads.sql import (
|
||||
delete_task_deeplink_index,
|
||||
sync_task_deeplink_index,
|
||||
update_task_deeplink_card,
|
||||
)
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
|
||||
from deerflow.persistence.thread_meta.base import EXCLUDED_SYSTEM_THREAD_TYPES, ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
_INDEXED_METADATA_COLUMNS = {
|
||||
"taskId": ThreadMetaRow.task_id,
|
||||
"agent_id": ThreadMetaRow.agent_id,
|
||||
"actionId": ThreadMetaRow.action_id,
|
||||
}
|
||||
|
||||
_INDEXED_METADATA_ATTRIBUTES = {
|
||||
"taskId": "task_id",
|
||||
"agent_id": "agent_id",
|
||||
"actionId": "action_id",
|
||||
}
|
||||
|
||||
|
||||
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
|
||||
"""Return metadata predicates that have a maintained SQL projection.
|
||||
|
||||
Task deep-link callers use string ids. Restrict the SQL shortcut to
|
||||
strings so legacy callers that intentionally store another JSON type keep
|
||||
the exact Python-side equality semantics of the generic metadata filter.
|
||||
"""
|
||||
return {
|
||||
key: value
|
||||
for key, value in (metadata or {}).items()
|
||||
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
|
||||
}
|
||||
|
||||
|
||||
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
|
||||
"""Build ORM column values from authoritative thread metadata.
|
||||
|
||||
Always include every projection so a metadata update that changes a value
|
||||
from a string to another JSON type cannot leave a stale indexed value.
|
||||
"""
|
||||
values = _indexed_metadata_values(metadata)
|
||||
return {
|
||||
attribute: values.get(metadata_key)
|
||||
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
|
||||
}
|
||||
|
||||
|
||||
def _apply_indexed_metadata_filters(stmt: Any, metadata: dict[str, Any] | None) -> tuple[Any, set[str]]:
|
||||
"""Apply known metadata predicates in SQL and return their keys.
|
||||
|
||||
The caller still verifies the full metadata object in Python whenever a
|
||||
generic filter or system-thread exclusion is involved. This keeps the
|
||||
public API's exact-match contract while reducing that verification set to
|
||||
rows for the requested task/agent/action instead of the entire table.
|
||||
"""
|
||||
values = _indexed_metadata_values(metadata)
|
||||
for key, value in values.items():
|
||||
stmt = stmt.where(_INDEXED_METADATA_COLUMNS[key] == value)
|
||||
return stmt, set(values)
|
||||
|
||||
|
||||
class ThreadMetaRepository(ThreadMetaStore):
|
||||
_INDEXED_METADATA_COLUMNS = {
|
||||
"taskId": ThreadMetaRow.task_id,
|
||||
"agent_id": ThreadMetaRow.agent_id,
|
||||
"actionId": ThreadMetaRow.action_id,
|
||||
}
|
||||
|
||||
_INDEXED_METADATA_ATTRIBUTES = {
|
||||
"taskId": "task_id",
|
||||
"agent_id": "agent_id",
|
||||
"actionId": "action_id",
|
||||
}
|
||||
|
||||
|
||||
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
|
||||
"""Return metadata predicates that have a maintained SQL projection.
|
||||
|
||||
Task deep-link callers use string ids. Restrict the SQL shortcut to
|
||||
strings so legacy callers that intentionally store another JSON type keep
|
||||
the exact Python-side equality semantics of the generic metadata filter.
|
||||
"""
|
||||
return {
|
||||
key: value
|
||||
for key, value in (metadata or {}).items()
|
||||
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
|
||||
}
|
||||
|
||||
|
||||
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
|
||||
"""Build ORM column values from authoritative thread metadata.
|
||||
|
||||
Always include every projection so a metadata update that changes a value
|
||||
from a string to another JSON type cannot leave a stale indexed value.
|
||||
"""
|
||||
values = _indexed_metadata_values(metadata)
|
||||
return {
|
||||
attribute: values.get(metadata_key)
|
||||
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
|
||||
}
|
||||
|
||||
|
||||
class ThreadMetaRepository(ThreadMetaStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@@ -97,14 +85,14 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
# creates an orphan row (used by migration scripts).
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.create")
|
||||
now = datetime.now(UTC)
|
||||
row = ThreadMetaRow(
|
||||
thread_id=thread_id,
|
||||
assistant_id=assistant_id,
|
||||
user_id=resolved_user_id,
|
||||
**_indexed_thread_metadata(metadata),
|
||||
display_name=display_name,
|
||||
status="idle",
|
||||
metadata_json=metadata or {},
|
||||
row = ThreadMetaRow(
|
||||
thread_id=thread_id,
|
||||
assistant_id=assistant_id,
|
||||
user_id=resolved_user_id,
|
||||
**_indexed_thread_metadata(metadata),
|
||||
display_name=display_name,
|
||||
status="idle",
|
||||
metadata_json=metadata or {},
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -164,6 +152,10 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
row = await session.get(ThreadMetaRow, thread_id)
|
||||
if row is None:
|
||||
return not require_existing
|
||||
if row.user_id:
|
||||
from deerflow.runtime.thread_paths import remember_thread_owner
|
||||
|
||||
remember_thread_owner(thread_id, row.user_id)
|
||||
md = row.metadata_json or {}
|
||||
# 全局共享、不区分用户(任何登录用户都可读写/续聊):
|
||||
# ① 任务深链对话(metadata.taskId);
|
||||
@@ -176,6 +168,43 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
return True
|
||||
return row.user_id == user_id
|
||||
|
||||
@staticmethod
|
||||
def _query_conditions(*, dialect, metadata, status, user_id, exclude_system, query):
|
||||
"""Share exact search/count predicates without changing owner resolution.
|
||||
|
||||
Complex metadata and unsupported dialects retain their exact Python
|
||||
filter. No rejected key/value is silently ignored. Server numeric JSON
|
||||
comparisons retain that path: DECIMAL and DOUBLE cannot reproduce all
|
||||
Python float/int comparisons (including large numeric task ids).
|
||||
"""
|
||||
conditions = []
|
||||
if user_id is not None:
|
||||
conditions.append(ThreadMetaRow.user_id == user_id)
|
||||
if status:
|
||||
conditions.append(ThreadMetaRow.status == status)
|
||||
if query and query.strip():
|
||||
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
# Retain the master branch's indexed task/agent/action narrowing.
|
||||
# JSON predicates below still enforce exact case/type semantics.
|
||||
for key, value in _indexed_metadata_values(metadata).items():
|
||||
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
|
||||
supported = dialect in {"sqlite", "postgresql", "mysql"}
|
||||
residual = {}
|
||||
for key, value in (metadata or {}).items():
|
||||
numeric_server = dialect in {"mysql", "postgresql"} and isinstance(value, (bool, int, float))
|
||||
if supported and supports_json_filter(key, value) and not numeric_server:
|
||||
conditions.append(JsonPredicate(ThreadMetaRow.metadata_json, key, value=value))
|
||||
else:
|
||||
residual[key] = value
|
||||
if exclude_system and supported:
|
||||
column = ThreadMetaRow.metadata_json
|
||||
conditions.append(~or_(
|
||||
JsonPredicate(column, "system", "truthy"),
|
||||
JsonPredicate(column, "task_analysis_seat", "nonblank"),
|
||||
*(JsonPredicate(column, "thread_type", value=value) for value in sorted(EXCLUDED_SYSTEM_THREAD_TYPES)),
|
||||
))
|
||||
return conditions, residual, exclude_system and not supported
|
||||
|
||||
async def search(
|
||||
self,
|
||||
*,
|
||||
@@ -187,71 +216,46 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
exclude_system: bool = False,
|
||||
query: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Search threads with optional metadata and status filters.
|
||||
|
||||
Owner filter is enforced by default: caller must be in a user
|
||||
context. Pass ``user_id=None`` to bypass (migration/CLI).
|
||||
"""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.search")
|
||||
# 任务深链对话(按 metadata.taskId 过滤)全局共享、不区分用户:跳过 owner 过滤。
|
||||
# Preserve task-scoped sharing, still applying EVERY metadata filter
|
||||
# (taskId + agent_id + actionId) before the pagination window.
|
||||
if metadata and metadata.get("taskId"):
|
||||
resolved_user_id = None
|
||||
# thread_id tie-breaker keeps OFFSET pagination stable when many rows
|
||||
# share the same updated_at (bulk imports, scheduler bursts).
|
||||
stmt = select(ThreadMetaRow).order_by(ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc())
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
|
||||
if status:
|
||||
stmt = stmt.where(ThreadMetaRow.status == status)
|
||||
if query and query.strip():
|
||||
# Case-insensitive title search. display_name is a real column, so
|
||||
# this filter runs in SQL even when the JSON filters below fall
|
||||
# back to Python-side scanning.
|
||||
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
|
||||
# ``metadata_json`` is deliberately portable, so arbitrary filters
|
||||
# still receive Python-side equality checks below. These three task
|
||||
# identifiers are maintained projections and can safely narrow the
|
||||
# SQL result first. A deep-link query that used to scan every thread
|
||||
# now reads only that task's rows.
|
||||
stmt, indexed_metadata_keys = _apply_indexed_metadata_filters(stmt, metadata)
|
||||
remaining_metadata_keys = set(metadata or {}) - indexed_metadata_keys
|
||||
|
||||
if not remaining_metadata_keys and not exclude_system:
|
||||
stmt = stmt.limit(limit).offset(offset)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
# Metadata filters live inside the JSON blob, so rows are scanned in
|
||||
# batches and filtered in Python until the requested window is full.
|
||||
# A fixed over-fetch window would under-fill pages when most rows are
|
||||
# filtered out (e.g. heavy scheduled-task usage), making callers that
|
||||
# paginate believe the list is exhausted. TODO(Phase 2): use JSON DB
|
||||
# operators (Postgres @>, SQLite json_extract) for server-side filtering.
|
||||
def _matches(row: dict) -> bool:
|
||||
md = row.get("metadata") or {}
|
||||
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
|
||||
return False
|
||||
if exclude_system and is_excluded_system_thread(md):
|
||||
return False
|
||||
return True
|
||||
|
||||
needed = offset + limit
|
||||
batch_size = max(500, needed)
|
||||
matched: list[dict] = []
|
||||
scan_offset = 0
|
||||
async with self._sf() as session:
|
||||
conditions, residual, filter_system = self._query_conditions(
|
||||
dialect=session.get_bind().dialect.name,
|
||||
metadata=metadata, status=status, user_id=resolved_user_id,
|
||||
exclude_system=exclude_system, query=query,
|
||||
)
|
||||
stmt = select(ThreadMetaRow).where(*conditions).order_by(
|
||||
ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc(),
|
||||
)
|
||||
if not residual and not filter_system:
|
||||
result = await session.execute(stmt.limit(limit).offset(offset))
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
# Compatibility path for nested values, unusual literal keys,
|
||||
# numeric server filters, or the separate openGauss dialect.
|
||||
needed = offset + limit
|
||||
batch_size = max(500, needed)
|
||||
matched = []
|
||||
scan_offset = 0
|
||||
while len(matched) < needed:
|
||||
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
|
||||
rows = result.scalars().all()
|
||||
if not rows:
|
||||
break
|
||||
matched.extend(d for d in (self._row_to_dict(r) for r in rows) if _matches(d))
|
||||
for row in rows:
|
||||
record = self._row_to_dict(row)
|
||||
md = record.get("metadata") or {}
|
||||
if all(md.get(key) == value for key, value in residual.items()) and not (
|
||||
filter_system and is_excluded_system_thread(md)
|
||||
):
|
||||
matched.append(record)
|
||||
if len(rows) < batch_size:
|
||||
break
|
||||
scan_offset += len(rows)
|
||||
return matched[offset:needed]
|
||||
return matched[offset:needed]
|
||||
|
||||
async def count(
|
||||
self,
|
||||
@@ -262,56 +266,37 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
exclude_system: bool = False,
|
||||
query: str | None = None,
|
||||
) -> int:
|
||||
"""Count threads matching the same filters as :meth:`search`."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.count")
|
||||
# 任务深链对话(按 metadata.taskId 过滤)全局共享、不区分用户:跳过 owner 过滤。
|
||||
if metadata and metadata.get("taskId"):
|
||||
resolved_user_id = None
|
||||
conditions = []
|
||||
if resolved_user_id is not None:
|
||||
conditions.append(ThreadMetaRow.user_id == resolved_user_id)
|
||||
if status:
|
||||
conditions.append(ThreadMetaRow.status == status)
|
||||
if query and query.strip():
|
||||
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
|
||||
indexed_metadata = _indexed_metadata_values(metadata)
|
||||
for key, value in indexed_metadata.items():
|
||||
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
|
||||
remaining_metadata_keys = set(metadata or {}) - set(indexed_metadata)
|
||||
|
||||
if not remaining_metadata_keys and not exclude_system:
|
||||
stmt = select(func.count()).select_from(ThreadMetaRow)
|
||||
for condition in conditions:
|
||||
stmt = stmt.where(condition)
|
||||
async with self._sf() as session:
|
||||
async with self._sf() as session:
|
||||
conditions, residual, filter_system = self._query_conditions(
|
||||
dialect=session.get_bind().dialect.name,
|
||||
metadata=metadata, status=status, user_id=resolved_user_id,
|
||||
exclude_system=exclude_system, query=query,
|
||||
)
|
||||
if not residual and not filter_system:
|
||||
stmt = select(func.count()).select_from(ThreadMetaRow).where(*conditions)
|
||||
return int((await session.execute(stmt)).scalar_one())
|
||||
|
||||
# JSON filters can't run in portable SQL — scan just the metadata
|
||||
# column in batches and count in Python (mirrors search()).
|
||||
stmt = select(ThreadMetaRow.metadata_json).order_by(ThreadMetaRow.thread_id)
|
||||
for condition in conditions:
|
||||
stmt = stmt.where(condition)
|
||||
total = 0
|
||||
batch_size = 1000
|
||||
scan_offset = 0
|
||||
async with self._sf() as session:
|
||||
stmt = select(ThreadMetaRow.metadata_json).where(*conditions).order_by(ThreadMetaRow.thread_id)
|
||||
total = 0
|
||||
scan_offset = 0
|
||||
batch_size = 1000
|
||||
while True:
|
||||
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
|
||||
rows = result.scalars().all()
|
||||
rows = (await session.execute(stmt.limit(batch_size).offset(scan_offset))).scalars().all()
|
||||
if not rows:
|
||||
break
|
||||
for metadata_json in rows:
|
||||
md = metadata_json or {}
|
||||
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
|
||||
continue
|
||||
if exclude_system and is_excluded_system_thread(md):
|
||||
continue
|
||||
total += 1
|
||||
for md in rows:
|
||||
md = md or {}
|
||||
if all(md.get(key) == value for key, value in residual.items()) and not (
|
||||
filter_system and is_excluded_system_thread(md)
|
||||
):
|
||||
total += 1
|
||||
if len(rows) < batch_size:
|
||||
break
|
||||
scan_offset += len(rows)
|
||||
return total
|
||||
return total
|
||||
|
||||
async def _check_ownership(self, session: AsyncSession, thread_id: str, resolved_user_id: str | None) -> bool:
|
||||
"""Return True if the row exists and is owned (or filter bypassed)."""
|
||||
@@ -337,7 +322,9 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
async with self._sf() as session:
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id):
|
||||
return
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=datetime.now(UTC)))
|
||||
now = datetime.now(UTC)
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=now))
|
||||
await update_task_deeplink_card(session, thread_id, display_name=display_name, updated_at=now)
|
||||
await session.commit()
|
||||
|
||||
async def update_status(
|
||||
@@ -351,7 +338,9 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
async with self._sf() as session:
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id):
|
||||
return
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=datetime.now(UTC)))
|
||||
now = datetime.now(UTC)
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=now))
|
||||
await update_task_deeplink_card(session, thread_id, status=status, updated_at=now)
|
||||
await session.commit()
|
||||
|
||||
async def update_assistant_id(
|
||||
@@ -365,7 +354,9 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
async with self._sf() as session:
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id):
|
||||
return
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=datetime.now(UTC)))
|
||||
now = datetime.now(UTC)
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(assistant_id=assistant_id, updated_at=now))
|
||||
await update_task_deeplink_card(session, thread_id, updated_at=now)
|
||||
await session.commit()
|
||||
|
||||
async def update_metadata(
|
||||
@@ -390,14 +381,14 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
task_shared = bool((row.metadata_json or {}).get("taskId"))
|
||||
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
|
||||
return
|
||||
merged = dict(row.metadata_json or {})
|
||||
merged.update(metadata)
|
||||
row.metadata_json = merged
|
||||
for key, value in _indexed_thread_metadata(merged).items():
|
||||
setattr(row, key, value)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
merged = dict(row.metadata_json or {})
|
||||
merged.update(metadata)
|
||||
row.metadata_json = merged
|
||||
for key, value in _indexed_thread_metadata(merged).items():
|
||||
setattr(row, key, value)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await sync_task_deeplink_index(session, row)
|
||||
await session.commit()
|
||||
await session.commit()
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
|
||||
+8
@@ -82,3 +82,11 @@ class ToolMetricsStore(ABC):
|
||||
"which skills were invoked, and how often did each succeed/fail".
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
|
||||
"""Delete one bounded batch of audit rows older than ``cutoff``.
|
||||
|
||||
The default keeps alternate/in-memory implementations compatible; the
|
||||
SQL repository supplies the real retention operation.
|
||||
"""
|
||||
return 0
|
||||
+32
-1
@@ -8,7 +8,7 @@ from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy import delete, desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
|
||||
@@ -245,3 +245,34 @@ class SqlToolMetricsStore(ToolMetricsStore):
|
||||
{"skill_name": name, **counts}
|
||||
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
|
||||
]
|
||||
|
||||
async def purge_before(self, cutoff: datetime, *, batch_size: int = 1_000) -> int:
|
||||
"""Delete one small, oldest-first retention batch.
|
||||
|
||||
Key selection followed by a primary-key delete works consistently on
|
||||
every supported SQL dialect and prevents a three-month cleanup from
|
||||
holding a large audit-table write lock.
|
||||
"""
|
||||
# Keep below SQLite's traditional 999 bind-variable limit. The
|
||||
# retention worker requests batches repeatedly, so portability is more
|
||||
# valuable than one giant delete statement.
|
||||
size = max(1, min(int(batch_size), 500))
|
||||
normalized_cutoff = _normalize_created_at(cutoff)
|
||||
async with self._sf() as session:
|
||||
ids = list(
|
||||
(
|
||||
await session.scalars(
|
||||
select(ToolCallMetricRow.id)
|
||||
.where(ToolCallMetricRow.created_at < normalized_cutoff)
|
||||
.order_by(ToolCallMetricRow.created_at.asc(), ToolCallMetricRow.id.asc())
|
||||
.limit(size)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not ids:
|
||||
return 0
|
||||
result = await session.execute(
|
||||
delete(ToolCallMetricRow).where(ToolCallMetricRow.id.in_(ids))
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
@@ -43,7 +43,17 @@ class PortableJSON(TypeDecorator):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
return json.loads(value)
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
from deerflow.persistence.gauss_mysql_compat import (
|
||||
decode_gauss_mysql_json,
|
||||
is_gauss_mysql_dialect,
|
||||
)
|
||||
|
||||
if not is_gauss_mysql_dialect(dialect):
|
||||
raise
|
||||
return decode_gauss_mysql_json(value)
|
||||
|
||||
|
||||
class PortableLongText(TypeDecorator):
|
||||
|
||||
+104
-79
@@ -1,79 +1,104 @@
|
||||
"""ORM model for the users table.
|
||||
|
||||
Lives in the harness persistence package so it is picked up by
|
||||
``Base.metadata.create_all()`` alongside ``threads_meta``, ``runs``,
|
||||
``run_events``, and ``feedback``. Using the shared engine means:
|
||||
|
||||
- One SQLite/Postgres database, one connection pool
|
||||
- One schema initialisation codepath
|
||||
- Consistent async sessions across auth and persistence reads
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Index, String, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class UserRow(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
# UUIDs are stored as 36-char strings for cross-backend portability.
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
|
||||
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
|
||||
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
|
||||
# when new roles are introduced.
|
||||
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
# OAuth linkage (optional). A partial unique index enforces one
|
||||
# account per (provider, oauth_id) pair, leaving NULL/NULL rows
|
||||
# unconstrained so plain password accounts can coexist.
|
||||
oauth_provider: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
oauth_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# Auth lifecycle flags
|
||||
needs_setup: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
token_version: Mapped[int] = mapped_column(nullable=False, default=0)
|
||||
|
||||
# 登录白名单(放行)标记 —— **黑名单模式**。仅当系统设置 ``login_whitelist.enabled``
|
||||
# 打开时生效:关 → 忽略此列,人人可登录;开 → 仅 approved=False(被管理员显式
|
||||
# 「取消放行」)的用户被拦,其余(含从未登录过的新账号)放行。新账号默认 True
|
||||
# (首次登录即放行);存量用户由迁移回填 True。server_default=1 让迁移 ADD COLUMN
|
||||
# 时把已存在行置为已放行。
|
||||
approved: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
|
||||
|
||||
# 管理员在「白名单管理」页给个别账号设置的「登录口令」(**明文**)。非空 → 该账号的
|
||||
# 用户名直登(``POST /api/v1/auth/login/username``)必须带 ``?password=`` 且与此值
|
||||
# 完全一致才放行(优先于全局共享口令门);为空/NULL → 走原有逻辑(全局共享口令门
|
||||
# 或免口令)。明文存储是有意为之:口令本就以 URL query 明文传输,存明文才能让管理员
|
||||
# 随时复制完整登录链接 ``…/login/<用户名>?password=xxxx``。仅 admin 可读写。
|
||||
login_password: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
|
||||
# 岗位 (position) the user is assigned to (1:1). NULL = no explicit
|
||||
# position → the user inherits the ``is_default`` position's grants. Drives
|
||||
# the subscription sync engine (see ``deerflow.services.position_sync``).
|
||||
position_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"idx_users_oauth_identity",
|
||||
"oauth_provider",
|
||||
"oauth_id",
|
||||
unique=True,
|
||||
sqlite_where=text("oauth_provider IS NOT NULL AND oauth_id IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
"""ORM model for the users table.
|
||||
|
||||
Lives in the harness persistence package so it is picked up by
|
||||
``Base.metadata.create_all()`` alongside ``threads_meta``, ``runs``,
|
||||
``run_events``, and ``feedback``. Using the shared engine means:
|
||||
|
||||
- One SQLite/Postgres database, one connection pool
|
||||
- One schema initialisation codepath
|
||||
- Consistent async sessions across auth and persistence reads
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Index, String, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class AuthInitializationRow(Base):
|
||||
"""Singleton transaction claim for the public first-admin setup endpoint."""
|
||||
|
||||
__tablename__ = "auth_initialization"
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
|
||||
|
||||
class AuthUsernameRow(Base):
|
||||
"""Reserve new account local-parts across concurrent email-domain variants."""
|
||||
|
||||
__tablename__ = "auth_usernames"
|
||||
username: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
|
||||
|
||||
class AuthServiceNonceRow(Base):
|
||||
"""One-time service exchange nonce shared by every gateway worker."""
|
||||
|
||||
__tablename__ = "auth_service_nonces"
|
||||
nonce_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
client_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
|
||||
|
||||
|
||||
class UserRow(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
# UUIDs are stored as 36-char strings for cross-backend portability.
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
|
||||
email: Mapped[str] = mapped_column(String(191), unique=True, nullable=False, index=True)
|
||||
# Canonical local part used by SSO / username login. Keeping it separately
|
||||
# avoids applying LOWER()/LIKE to every users.email row on each login.
|
||||
username_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
|
||||
# when new roles are introduced.
|
||||
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
# OAuth linkage (optional). A partial unique index enforces one
|
||||
# account per (provider, oauth_id) pair, leaving NULL/NULL rows
|
||||
# unconstrained so plain password accounts can coexist.
|
||||
oauth_provider: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
oauth_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# Auth lifecycle flags
|
||||
needs_setup: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
token_version: Mapped[int] = mapped_column(nullable=False, default=0)
|
||||
|
||||
# 登录白名单(放行)标记 —— **黑名单模式**。仅当系统设置 ``login_whitelist.enabled``
|
||||
# 打开时生效:关 → 忽略此列,人人可登录;开 → 仅 approved=False(被管理员显式
|
||||
# 「取消放行」)的用户被拦,其余(含从未登录过的新账号)放行。新账号默认 True
|
||||
# (首次登录即放行);存量用户由迁移回填 True。server_default=1 让迁移 ADD COLUMN
|
||||
# 时把已存在行置为已放行。
|
||||
approved: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=text("1"))
|
||||
|
||||
# Unified password/passphrase. New writes store the same versioned hash as
|
||||
# password_hash; historical plaintext remains accepted until the next reset.
|
||||
# Never serialize hashes into user-list or login responses.
|
||||
login_password: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
|
||||
# 岗位 (position) the user is assigned to (1:1). NULL = no explicit
|
||||
# position → the user inherits the ``is_default`` position's grants. Drives
|
||||
# the subscription sync engine (see ``deerflow.services.position_sync``).
|
||||
position_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_users_username_key_created_at", "username_key", "created_at"),
|
||||
Index(
|
||||
"idx_users_oauth_identity",
|
||||
"oauth_provider",
|
||||
"oauth_id",
|
||||
unique=True,
|
||||
sqlite_where=text("oauth_provider IS NOT NULL AND oauth_id IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
Reference in new issue
Block a user