sync update from LAN repo (source ji @ 85078a31)
This commit is contained in:
1 parent
5d6627af50
commit
e602d88695
864 files changed
+99150
-6346
No files matched your search
+197
@@ -0,0 +1,197 @@
|
||||
"""User-owned analysis projects and immutable project/thread bindings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import ForeignKey, String, Text, delete, select
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
MODES = frozenset({"qa", "strategy", "solution"})
|
||||
CONFIG_FIELDS = frozenset({"agent_id", "report_config", "strategy_config"})
|
||||
EDITABLE_FIELDS = frozenset({"name", "background", "knowledge_base_ids", "assistant_knowledge_base_ids"}) | CONFIG_FIELDS
|
||||
|
||||
|
||||
class AnalysisProjectConfigRow(Base):
|
||||
"""Additive table: upgrades do not alter existing project/thread tables."""
|
||||
|
||||
__tablename__ = "analysis_project_configs"
|
||||
key: Mapped[str] = mapped_column(String(80), primary_key=True)
|
||||
config: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
|
||||
|
||||
class AnalysisProjectRow(Base):
|
||||
__tablename__ = "analysis_projects"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(255), index=True)
|
||||
name: Mapped[str] = mapped_column(String(160))
|
||||
mode: Mapped[str] = mapped_column(String(16))
|
||||
background: Mapped[str] = mapped_column(Text, default="")
|
||||
knowledge_base_ids: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
assistant_knowledge_base_ids: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime())
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime())
|
||||
|
||||
|
||||
class AnalysisProjectThreadRow(Base):
|
||||
__tablename__ = "analysis_project_threads"
|
||||
|
||||
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
project_id: Mapped[str] = mapped_column(ForeignKey("analysis_projects.id"), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime())
|
||||
|
||||
|
||||
def _dict(row):
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
if isinstance(data.get(key), datetime):
|
||||
data[key] = data[key].isoformat()
|
||||
return deepcopy(data)
|
||||
|
||||
|
||||
class AnalysisProjectStore:
|
||||
"""SQL in persistent deployments; isolated in-memory fallback for memory mode.
|
||||
|
||||
Every public project operation requires an explicit owner. The unscoped
|
||||
thread lookup is only for the Gateway's ownership and run-policy checks.
|
||||
Mode has no update path, and bindings are separate from client-editable metadata.
|
||||
"""
|
||||
|
||||
def __init__(self, session_factory=None):
|
||||
self._sf = session_factory
|
||||
self._projects: dict[str, dict] = {}
|
||||
self._threads: dict[str, dict] = {}
|
||||
self._configs: dict[str, dict] = {}
|
||||
|
||||
async def config_for(self, key: str) -> dict:
|
||||
if not self._sf:
|
||||
return deepcopy(self._configs.get(key, {}))
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AnalysisProjectConfigRow, key)
|
||||
return deepcopy(row.config) if row else {}
|
||||
|
||||
async def _enrich(self, project: dict | None) -> dict | None:
|
||||
if project is None:
|
||||
return None
|
||||
return {
|
||||
**project,
|
||||
"agent_id": None,
|
||||
"report_config": {},
|
||||
"strategy_config": {},
|
||||
**await self.config_for(f"project:{project['id']}"),
|
||||
}
|
||||
|
||||
async def create(self, user_id: str, *, mode: str, **fields) -> dict:
|
||||
if mode not in MODES or set(fields) - EDITABLE_FIELDS:
|
||||
raise ValueError("Invalid project fields")
|
||||
now = datetime.now(UTC)
|
||||
row = AnalysisProjectRow(
|
||||
id=str(uuid4()),
|
||||
user_id=user_id,
|
||||
mode=mode,
|
||||
name=fields["name"],
|
||||
background=fields.get("background", ""),
|
||||
knowledge_base_ids=list(fields.get("knowledge_base_ids", [])),
|
||||
assistant_knowledge_base_ids=list(fields.get("assistant_knowledge_base_ids", [])),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
if self._sf:
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
session.add(AnalysisProjectConfigRow(key=f"project:{row.id}", config={key: deepcopy(fields[key]) for key in CONFIG_FIELDS if key in fields}))
|
||||
await session.commit()
|
||||
else:
|
||||
self._projects[row.id] = _dict(row)
|
||||
self._configs[f"project:{row.id}"] = {key: deepcopy(fields[key]) for key in CONFIG_FIELDS if key in fields}
|
||||
return await self._enrich(_dict(row))
|
||||
|
||||
async def list(self, user_id: str) -> list[dict]:
|
||||
if not self._sf:
|
||||
return [await self._enrich(deepcopy(p)) for p in sorted(self._projects.values(), key=lambda p: p["updated_at"], reverse=True) if p["user_id"] == user_id]
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.user_id == user_id).order_by(AnalysisProjectRow.updated_at.desc()))
|
||||
rows = [_dict(row) for row in result.scalars()]
|
||||
return [await self._enrich(row) for row in rows]
|
||||
|
||||
async def get(self, project_id: str, user_id: str) -> dict | None:
|
||||
if not self._sf:
|
||||
row = self._projects.get(project_id)
|
||||
return await self._enrich(deepcopy(row)) if row and row["user_id"] == user_id else None
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.id == project_id, AnalysisProjectRow.user_id == user_id))).scalar_one_or_none()
|
||||
data = _dict(row) if row else None
|
||||
return await self._enrich(data)
|
||||
|
||||
async def update(self, project_id: str, user_id: str, **fields) -> dict | None:
|
||||
if set(fields) - EDITABLE_FIELDS:
|
||||
raise ValueError("项目模式创建后不可更改")
|
||||
if not self._sf:
|
||||
row = self._projects.get(project_id)
|
||||
if not row or row["user_id"] != user_id:
|
||||
return None
|
||||
row.update({key: deepcopy(value) for key, value in fields.items() if key not in CONFIG_FIELDS}, updated_at=datetime.now(UTC).isoformat())
|
||||
self._configs.setdefault(f"project:{project_id}", {}).update({key: deepcopy(value) for key, value in fields.items() if key in CONFIG_FIELDS})
|
||||
return await self._enrich(deepcopy(row))
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.id == project_id, AnalysisProjectRow.user_id == user_id).with_for_update())).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
for key, value in fields.items():
|
||||
if key not in CONFIG_FIELDS:
|
||||
setattr(row, key, deepcopy(value))
|
||||
options = await session.get(AnalysisProjectConfigRow, f"project:{project_id}")
|
||||
config = {**(options.config if options else {}), **{key: deepcopy(value) for key, value in fields.items() if key in CONFIG_FIELDS}}
|
||||
await session.merge(AnalysisProjectConfigRow(key=f"project:{project_id}", config=config))
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
data = _dict(row)
|
||||
return await self._enrich(data)
|
||||
|
||||
async def bind_thread(self, project_id: str, user_id: str, thread_id: str, *, agent_id: str | None = None) -> None:
|
||||
if await self.get(project_id, user_id) is None:
|
||||
raise ValueError("Project not found")
|
||||
row = AnalysisProjectThreadRow(thread_id=thread_id, project_id=project_id, created_at=datetime.now(UTC))
|
||||
if not self._sf:
|
||||
if thread_id in self._threads:
|
||||
raise ValueError("Thread already bound")
|
||||
self._threads[thread_id] = _dict(row)
|
||||
self._configs[f"thread:{thread_id}"] = {"agent_id": agent_id}
|
||||
return
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
session.add(AnalysisProjectConfigRow(key=f"thread:{thread_id}", config={"agent_id": agent_id}))
|
||||
await session.commit()
|
||||
|
||||
async def project_for_thread(self, thread_id: str) -> dict | None:
|
||||
if not self._sf:
|
||||
binding = self._threads.get(thread_id)
|
||||
return await self._enrich(deepcopy(self._projects.get(binding["project_id"]))) if binding else None
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(AnalysisProjectRow).join(AnalysisProjectThreadRow).where(AnalysisProjectThreadRow.thread_id == thread_id))).scalar_one_or_none()
|
||||
data = _dict(row) if row else None
|
||||
return await self._enrich(data)
|
||||
|
||||
async def thread_ids(self, project_id: str, user_id: str) -> list[str]:
|
||||
if await self.get(project_id, user_id) is None:
|
||||
return []
|
||||
if not self._sf:
|
||||
return [key for key, value in reversed(list(self._threads.items())) if value["project_id"] == project_id]
|
||||
async with self._sf() as session:
|
||||
return list((await session.execute(select(AnalysisProjectThreadRow.thread_id).where(AnalysisProjectThreadRow.project_id == project_id).order_by(AnalysisProjectThreadRow.created_at.desc()))).scalars())
|
||||
|
||||
async def unbind_failed_creation(self, thread_id: str) -> None:
|
||||
if not self._sf:
|
||||
self._threads.pop(thread_id, None)
|
||||
self._configs.pop(f"thread:{thread_id}", None)
|
||||
return
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(AnalysisProjectThreadRow).where(AnalysisProjectThreadRow.thread_id == thread_id))
|
||||
await session.execute(delete(AnalysisProjectConfigRow).where(AnalysisProjectConfigRow.key == f"thread:{thread_id}"))
|
||||
await session.commit()
|
||||
+18
-1
@@ -1,4 +1,21 @@
|
||||
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
|
||||
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
|
||||
|
||||
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
|
||||
_assistant_knowledge_store = None
|
||||
|
||||
|
||||
def register_assistant_knowledge_store(store) -> None:
|
||||
global _assistant_knowledge_store
|
||||
_assistant_knowledge_store = store
|
||||
|
||||
|
||||
def get_assistant_knowledge_store():
|
||||
return _assistant_knowledge_store
|
||||
|
||||
|
||||
__all__ = [
|
||||
name
|
||||
for name in globals()
|
||||
if name.startswith("AssistantKnowledge")
|
||||
or name in {"make_assistant_knowledge_store", "register_assistant_knowledge_store", "get_assistant_knowledge_store"}
|
||||
]
|
||||
+998
-137
File diff suppressed because it is too large.
Load diff
+2
-2
@@ -1,7 +1,7 @@
|
||||
"""ORM model for the global business → agent / chain mapping (业务链条管理).
|
||||
|
||||
A "business mapping" row binds a fixed business code (3Q / 6BF / 6-1…6-4 / 7BF /
|
||||
8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
|
||||
A "business mapping" row binds a fixed business code (3Q / 6BF / 3q-1…3q-5 /
|
||||
6-1…6-4 / 7BF / 7-1…7-4 / 8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
|
||||
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
|
||||
account can resolve it). Used to auto-pick the business chain when a task page
|
||||
launches the multi-agent roundtable.
|
||||
|
||||
+6
-1
@@ -21,7 +21,7 @@ _UNSET = object()
|
||||
# Fixed business rows (业务码 → 展示名). Seeded on first boot; agent/chain left empty.
|
||||
# 8BF 页面尚未建好,先占位保留行。顺序即管理表的展示顺序。
|
||||
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
|
||||
# 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
|
||||
# 3q-1…3q-5 / 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
|
||||
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
{"business_code": "3Q", "label": "3QFX"},
|
||||
{"business_code": "6BF", "label": "任务FX"},
|
||||
@@ -29,6 +29,11 @@ DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
{"business_code": "8BF", "label": "8BF"},
|
||||
{"business_code": "AFX", "label": "AGENTFX"},
|
||||
# 独立配置位,与上方业务码无关;固定排在表尾。
|
||||
{"business_code": "3q-1", "label": "3q-1"},
|
||||
{"business_code": "3q-2", "label": "3q-2"},
|
||||
{"business_code": "3q-3", "label": "3q-3"},
|
||||
{"business_code": "3q-4", "label": "3q-4"},
|
||||
{"business_code": "3q-5", "label": "3q-5"},
|
||||
{"business_code": "6-1", "label": "6-1"},
|
||||
{"business_code": "6-2", "label": "6-2"},
|
||||
{"business_code": "6-3", "label": "6-3"},
|
||||
|
||||
+5
-1
@@ -9,13 +9,17 @@ from deerflow.persistence.compact_menu_overrides.keys import (
|
||||
normalize_compact_menu_type,
|
||||
)
|
||||
from deerflow.persistence.compact_menu_overrides.memory import MemoryCompactMenuOverrideStore
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
|
||||
from deerflow.persistence.compact_menu_overrides.model import (
|
||||
CompactMenuOverrideRow,
|
||||
UserCompactMenuOverrideConfigRow,
|
||||
)
|
||||
from deerflow.persistence.compact_menu_overrides.sql import CompactMenuOverrideRepository
|
||||
|
||||
__all__ = [
|
||||
"ALLOWED_COMPACT_MENU_TYPES",
|
||||
"CompactMenuOverrideRepository",
|
||||
"CompactMenuOverrideRow",
|
||||
"UserCompactMenuOverrideConfigRow",
|
||||
"CompactMenuOverrideStore",
|
||||
"DEFAULT_COMPACT_MENU_TYPE",
|
||||
"MemoryCompactMenuOverrideStore",
|
||||
|
||||
+24
-1
@@ -1,4 +1,10 @@
|
||||
"""Abstract compact-sidebar menu override persistence API."""
|
||||
"""Abstract compact-sidebar menu override persistence API.
|
||||
|
||||
The regular rows are the administrator's default for a compact-menu variant.
|
||||
Personal layouts live in a separate complete snapshot per user and variant;
|
||||
the router overlays that snapshot while allowing newly-added default node ids
|
||||
to flow through.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -22,3 +28,20 @@ class CompactMenuOverrideStore(ABC):
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Atomically replace the compact-menu override set for one type."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
|
||||
"""Return one user's snapshot for a type, or ``None`` if absent."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
menu_type: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""Replace one user's snapshot; an empty list resets to defaults."""
|
||||
raise NotImplementedError
|
||||
+34
@@ -38,6 +38,7 @@ def _public(row: dict[str, Any]) -> dict[str, Any]:
|
||||
class MemoryCompactMenuOverrideStore(CompactMenuOverrideStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._user_rows: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
||||
|
||||
async def list_overrides(self, menu_type: str | None = None) -> list[dict[str, Any]]:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
@@ -62,3 +63,36 @@ class MemoryCompactMenuOverrideStore(CompactMenuOverrideStore):
|
||||
stored = _normalize(r, t, updated_by)
|
||||
self._rows[str(stored["node_id"])] = stored
|
||||
return await self.list_overrides(t)
|
||||
|
||||
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
rows = self._user_rows.get((str(user_id), t))
|
||||
return [dict(r) for r in rows] if rows is not None else None
|
||||
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
menu_type: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
key = (str(user_id), t)
|
||||
if not rows:
|
||||
self._user_rows.pop(key, None)
|
||||
return None
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
node_id = row.get("node_id")
|
||||
if not node_id:
|
||||
continue
|
||||
stored = _normalize(row, t, updated_by or str(user_id))
|
||||
# Expose the public id in a personal document just like the API.
|
||||
public = _public(stored)
|
||||
by_id[str(public["node_id"])] = public
|
||||
if not by_id:
|
||||
self._user_rows.pop(key, None)
|
||||
return None
|
||||
self._user_rows[key] = [dict(r) for r in by_id.values()]
|
||||
return [dict(r) for r in self._user_rows[key]]
|
||||
+15
-1
@@ -12,7 +12,7 @@ from sqlalchemy import Index, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class CompactMenuOverrideRow(Base):
|
||||
@@ -28,3 +28,17 @@ class CompactMenuOverrideRow(Base):
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
|
||||
class UserCompactMenuOverrideConfigRow(Base):
|
||||
"""A user's complete snapshot for one compact-menu variant."""
|
||||
|
||||
__tablename__ = "user_compact_menu_override_configs"
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
menu_type: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
overrides: Mapped[list[dict]] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
+88
-2
@@ -2,7 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, or_, select
|
||||
@@ -15,7 +15,10 @@ from deerflow.persistence.compact_menu_overrides.keys import (
|
||||
public_node_id,
|
||||
storage_node_id,
|
||||
)
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
|
||||
from deerflow.persistence.compact_menu_overrides.model import (
|
||||
CompactMenuOverrideRow,
|
||||
UserCompactMenuOverrideConfigRow,
|
||||
)
|
||||
|
||||
|
||||
def _to_dict(row: CompactMenuOverrideRow) -> dict[str, Any]:
|
||||
@@ -42,6 +45,41 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C
|
||||
)
|
||||
|
||||
|
||||
def _normalize_user_row(data: dict[str, Any], menu_type: str) -> dict[str, Any]:
|
||||
placement = str(data.get("placement") or "sidebar")
|
||||
return {
|
||||
"node_id": public_node_id(str(data["node_id"]), menu_type),
|
||||
"menu_type": menu_type,
|
||||
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"label": (data.get("label") or None),
|
||||
}
|
||||
|
||||
|
||||
def _user_rows(row: UserCompactMenuOverrideConfigRow) -> list[dict[str, Any]]:
|
||||
value = row.overrides
|
||||
if isinstance(value, str):
|
||||
import json
|
||||
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
value = []
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
menu_type = normalize_compact_menu_type(row.menu_type)
|
||||
updated_at = row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at
|
||||
return [
|
||||
{
|
||||
**_normalize_user_row(item, menu_type),
|
||||
"updated_by": row.updated_by,
|
||||
"updated_at": updated_at,
|
||||
}
|
||||
for item in value
|
||||
if isinstance(item, dict) and item.get("node_id")
|
||||
]
|
||||
|
||||
|
||||
class CompactMenuOverrideRepository(CompactMenuOverrideStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
@@ -76,6 +114,54 @@ class CompactMenuOverrideRepository(CompactMenuOverrideStore):
|
||||
await session.refresh(row)
|
||||
return [_to_dict(row) for row in new_rows]
|
||||
|
||||
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserCompactMenuOverrideConfigRow, (str(user_id), t))
|
||||
return _user_rows(row) if row is not None else None
|
||||
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
menu_type: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
key = str(user_id)
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for data in rows:
|
||||
node_id = data.get("node_id")
|
||||
if node_id:
|
||||
normalized = _normalize_user_row(data, t)
|
||||
by_id[normalized["node_id"]] = normalized
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserCompactMenuOverrideConfigRow, (key, t))
|
||||
if not by_id:
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
payload = list(by_id.values())
|
||||
if row is None:
|
||||
row = UserCompactMenuOverrideConfigRow(
|
||||
user_id=key,
|
||||
menu_type=t,
|
||||
overrides=payload,
|
||||
updated_by=updated_by or key,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.overrides = payload
|
||||
row.updated_by = updated_by or key
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
# ``expire_on_commit=False`` keeps this writer-side row usable and
|
||||
# avoids a replica-lag-prone post-commit refresh.
|
||||
return _user_rows(row)
|
||||
|
||||
|
||||
def _menu_type_filter(menu_type: str):
|
||||
if menu_type == DEFAULT_COMPACT_MENU_TYPE:
|
||||
|
||||
+38
-24
@@ -1,38 +1,52 @@
|
||||
"""ORM model for user feedback on runs."""
|
||||
"""ORM model for user feedback on assistant answers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
# Prefix-unique instead of UniqueConstraint: a plain unique over three
|
||||
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
|
||||
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
|
||||
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
|
||||
# mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
# ``target_key`` is a SHA-256 digest of user/thread/message/agent. A
|
||||
# digest keeps the idempotency index well below MySQL's legacy utf8mb4
|
||||
# key-size limit; indexing all four source strings would exceed it.
|
||||
__table_args__ = (
|
||||
Index("uq_feedback_target_key", "target_key", unique=True),
|
||||
Index("ix_feedback_agent_created", "agent_id", "created_at"),
|
||||
)
|
||||
|
||||
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
message_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# message_id is an optional RunEventStore event identifier —
|
||||
# allows feedback to target a specific message or the entire run
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
message_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# ``message_id`` is the LangChain/checkpoint message id serialized inside
|
||||
# a RunEventStore ``llm.ai.response`` event. It stays nullable solely for
|
||||
# legacy run-level feedback rows created before answer-level feedback.
|
||||
|
||||
agent_id: Mapped[str] = mapped_column(String(128), nullable=False, default="lead_agent")
|
||||
target_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
|
||||
rating: Mapped[int] = mapped_column(nullable=False)
|
||||
# +1 (thumbs-up) or -1 (thumbs-down)
|
||||
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional text feedback from the user
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional user detail. The API exposes this as ``detail`` and retains
|
||||
# ``comment`` as a compatibility alias for older clients.
|
||||
|
||||
reasons: Mapped[list[str]] = mapped_column(PortableJSON(), default=list)
|
||||
# Stable machine-readable reason codes for negative feedback.
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
"""Transparent user-feedback scoring for assistant answers.
|
||||
|
||||
This module deliberately uses no model or heuristic content grading. It
|
||||
summarizes explicit thumbs feedback and selected problem reasons only. The
|
||||
result is an answer-quality signal, not a complete evaluation of an agent's
|
||||
planning, reasoning trajectory, tool use, latency, reliability, or safety.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
FEEDBACK_REASONS = (
|
||||
"incorrect_or_incomplete",
|
||||
"unmet_expectations",
|
||||
"slow_or_problematic",
|
||||
"style_or_tone",
|
||||
"safety_or_legal",
|
||||
"other",
|
||||
)
|
||||
|
||||
# The five response-quality dimensions and weights are fixed product policy.
|
||||
# A negative reason is evidence only for the explicitly mapped dimensions;
|
||||
# operational, communication, safety, and uncategorized complaints remain in
|
||||
# the problem breakdown instead of being mislabeled as task-quality evidence.
|
||||
DIMENSION_CONFIG: dict[str, dict[str, Any]] = {
|
||||
"correctness": {
|
||||
"weight": 0.30,
|
||||
"mapped_reasons": ["incorrect_or_incomplete"],
|
||||
},
|
||||
"completeness": {
|
||||
"weight": 0.20,
|
||||
"mapped_reasons": ["incorrect_or_incomplete"],
|
||||
},
|
||||
"evidence_grounding": {
|
||||
"weight": 0.20,
|
||||
"mapped_reasons": ["incorrect_or_incomplete"],
|
||||
},
|
||||
"relevance": {
|
||||
"weight": 0.15,
|
||||
"mapped_reasons": ["unmet_expectations"],
|
||||
},
|
||||
"actionability": {
|
||||
"weight": 0.15,
|
||||
"mapped_reasons": ["unmet_expectations"],
|
||||
},
|
||||
}
|
||||
|
||||
# Beta(8, 2): prior mean 80/100 with ten equivalent observations. This keeps
|
||||
# one or two votes from producing an extreme public score while real evidence
|
||||
# steadily dominates. With no real sample the API returns ``score=null`` so
|
||||
# the prior is never presented as earned performance.
|
||||
PRIOR_POSITIVE = 8
|
||||
PRIOR_NEGATIVE = 2
|
||||
PRIOR_EQUIVALENT_SAMPLES = PRIOR_POSITIVE + PRIOR_NEGATIVE
|
||||
PRIOR_MEAN = PRIOR_POSITIVE / PRIOR_EQUIVALENT_SAMPLES
|
||||
|
||||
|
||||
def _posterior_score(successes: int, failures: int) -> float | None:
|
||||
evidence_count = successes + failures
|
||||
if evidence_count == 0:
|
||||
return None
|
||||
value = 100.0 * (successes + PRIOR_POSITIVE) / (
|
||||
evidence_count + PRIOR_EQUIVALENT_SAMPLES
|
||||
)
|
||||
return round(value, 1)
|
||||
|
||||
|
||||
def _score_band(score: float | None) -> dict[str, Any]:
|
||||
if score is None:
|
||||
return {
|
||||
"key": "insufficient_data",
|
||||
"label": "数据不足",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
}
|
||||
if score >= 90:
|
||||
return {
|
||||
"key": "excellent",
|
||||
"label": "优秀",
|
||||
"min": 90,
|
||||
"max": 100,
|
||||
}
|
||||
if score >= 80:
|
||||
return {
|
||||
"key": "good",
|
||||
"label": "良好",
|
||||
"min": 80,
|
||||
"max": 89.9,
|
||||
}
|
||||
if score >= 70:
|
||||
return {
|
||||
"key": "needs_attention",
|
||||
"label": "需关注",
|
||||
"min": 70,
|
||||
"max": 79.9,
|
||||
}
|
||||
return {
|
||||
"key": "poor",
|
||||
"label": "较差",
|
||||
"min": 0,
|
||||
"max": 69.9,
|
||||
}
|
||||
|
||||
|
||||
def _confidence_level(sample_count: int) -> str:
|
||||
if sample_count == 0:
|
||||
return "none"
|
||||
if sample_count < 10:
|
||||
return "low"
|
||||
if sample_count < 30:
|
||||
return "medium"
|
||||
return "high"
|
||||
|
||||
|
||||
def calculate_answer_quality_score(
|
||||
rows: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
agent_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Aggregate explicit answer feedback into a transparent 0-100 score."""
|
||||
|
||||
records = list(rows)
|
||||
positive_count = sum(1 for row in records if int(row.get("rating") or 0) == 1)
|
||||
negative_rows = [row for row in records if int(row.get("rating") or 0) == -1]
|
||||
negative_count = len(negative_rows)
|
||||
sample_count = positive_count + negative_count
|
||||
|
||||
reason_counts = {reason: 0 for reason in FEEDBACK_REASONS}
|
||||
negative_reason_sets: list[set[str]] = []
|
||||
for row in negative_rows:
|
||||
raw_reasons = row.get("reasons")
|
||||
if not isinstance(raw_reasons, list):
|
||||
raw_reasons = []
|
||||
reason_set = {
|
||||
str(reason)
|
||||
for reason in raw_reasons
|
||||
if str(reason) in reason_counts
|
||||
}
|
||||
negative_reason_sets.append(reason_set)
|
||||
for reason in reason_set:
|
||||
reason_counts[reason] += 1
|
||||
|
||||
dimensions: dict[str, dict[str, Any]] = {}
|
||||
weighted_total = 0.0
|
||||
available_weight = 0.0
|
||||
for name, config in DIMENSION_CONFIG.items():
|
||||
mapped = set(config["mapped_reasons"])
|
||||
failures = sum(1 for reasons in negative_reason_sets if reasons & mapped)
|
||||
evidence_count = positive_count + failures
|
||||
dimension_score = _posterior_score(positive_count, failures)
|
||||
weight = float(config["weight"])
|
||||
dimensions[name] = {
|
||||
"weight": weight,
|
||||
"score": dimension_score,
|
||||
"evidence_count": evidence_count,
|
||||
"positive_count": positive_count,
|
||||
"negative_count": failures,
|
||||
"mapped_reasons": list(config["mapped_reasons"]),
|
||||
}
|
||||
if dimension_score is not None:
|
||||
weighted_total += weight * dimension_score
|
||||
available_weight += weight
|
||||
|
||||
task_score = (
|
||||
round(weighted_total / available_weight, 1)
|
||||
if available_weight > 0
|
||||
else None
|
||||
)
|
||||
score = _posterior_score(positive_count, negative_count)
|
||||
task_success_rate = (
|
||||
round(positive_count / sample_count, 4) if sample_count else None
|
||||
)
|
||||
reason_breakdown = {
|
||||
reason: {
|
||||
"count": count,
|
||||
"rate": round(count / negative_count, 4) if negative_count else 0.0,
|
||||
}
|
||||
for reason, count in reason_counts.items()
|
||||
}
|
||||
|
||||
return {
|
||||
"agent_id": agent_id,
|
||||
"score": score,
|
||||
"score_band": _score_band(score),
|
||||
"confidence": round(sample_count / (sample_count + 10), 4),
|
||||
"confidence_level": _confidence_level(sample_count),
|
||||
"sample_count": sample_count,
|
||||
"positive_count": positive_count,
|
||||
"negative_count": negative_count,
|
||||
"task_success_rate": task_success_rate,
|
||||
"task_score": task_score,
|
||||
"dimensions": dimensions,
|
||||
"reason_breakdown": reason_breakdown,
|
||||
"prior": {
|
||||
"positive": PRIOR_POSITIVE,
|
||||
"negative": PRIOR_NEGATIVE,
|
||||
"mean": PRIOR_MEAN,
|
||||
"equivalent_samples": PRIOR_EQUIVALENT_SAMPLES,
|
||||
},
|
||||
"formula": {
|
||||
"scope": "user_feedback_answer_quality",
|
||||
"overall": (
|
||||
"100 * (positive_count + 8) / (sample_count + 10); "
|
||||
"null when sample_count is zero"
|
||||
),
|
||||
"confidence": "sample_count / (sample_count + 10)",
|
||||
"dimension": (
|
||||
"100 * (positive_count + 8) / "
|
||||
"(positive_count + mapped_negative_count + 10); "
|
||||
"null when dimension evidence_count is zero"
|
||||
),
|
||||
"task_score": (
|
||||
"weighted mean of evidenced dimensions using weights "
|
||||
"30/20/20/15/15, renormalized over available evidence"
|
||||
),
|
||||
"reason_rate_denominator": "negative_count",
|
||||
"limitations": (
|
||||
"This is not a complete agent-capability score and does not "
|
||||
"grade hidden reasoning, planning, skill/tool use, latency, "
|
||||
"reliability, or safety without separate evidence."
|
||||
),
|
||||
},
|
||||
}
|
||||
+279
-88
@@ -1,17 +1,23 @@
|
||||
"""SQLAlchemy-backed feedback storage.
|
||||
"""SQLAlchemy-backed assistant-answer feedback storage.
|
||||
|
||||
Each method acquires its own short-lived session.
|
||||
Each method acquires its own short-lived session. Writes are idempotent on a
|
||||
SHA-256 target key derived from user/thread/message/agent; the digest avoids
|
||||
oversized compound indexes on legacy MySQL utf8mb4 deployments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
from sqlalchemy import case, delete, func, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.feedback.scoring import FEEDBACK_REASONS, calculate_answer_quality_score
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
|
||||
@@ -20,12 +26,51 @@ class FeedbackRepository:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: FeedbackRow) -> dict:
|
||||
d = row.to_dict()
|
||||
val = d.get("created_at")
|
||||
if isinstance(val, datetime):
|
||||
d["created_at"] = val.isoformat()
|
||||
return d
|
||||
def _row_to_dict(row: FeedbackRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
data["reasons"] = list(data.get("reasons") or [])
|
||||
data["detail"] = data.get("comment")
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def make_target_key(
|
||||
*,
|
||||
user_id: str | None,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None,
|
||||
agent_id: str,
|
||||
) -> str:
|
||||
"""Build the portable idempotency identity for one answer rating.
|
||||
|
||||
Historical run-level callers have no message id. Their deterministic
|
||||
``run:<id>`` surrogate preserves the old one-rating-per-run behavior.
|
||||
"""
|
||||
|
||||
message_key = message_id or f"run:{run_id}"
|
||||
raw = "\x1f".join(
|
||||
(
|
||||
user_id or "",
|
||||
thread_id,
|
||||
message_key,
|
||||
agent_id or "lead_agent",
|
||||
)
|
||||
)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _normalize_reasons(reasons: list[str] | None) -> list[str]:
|
||||
allowed = set(FEEDBACK_REASONS)
|
||||
normalized: list[str] = []
|
||||
for reason in reasons or []:
|
||||
value = str(reason).strip()
|
||||
if value in allowed and value not in normalized:
|
||||
normalized.append(value)
|
||||
return normalized
|
||||
|
||||
async def create(
|
||||
self,
|
||||
@@ -35,34 +80,29 @@ class FeedbackRepository:
|
||||
rating: int,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
message_id: str | None = None,
|
||||
agent_id: str = "lead_agent",
|
||||
reasons: list[str] | None = None,
|
||||
comment: str | None = None,
|
||||
) -> dict:
|
||||
"""Create a feedback record. rating must be +1 or -1."""
|
||||
if rating not in (1, -1):
|
||||
raise ValueError(f"rating must be +1 or -1, got {rating}")
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.create")
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
) -> dict[str, Any]:
|
||||
"""Create or update the same answer target idempotently."""
|
||||
|
||||
return await self.upsert(
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
message_id=message_id,
|
||||
rating=rating,
|
||||
user_id=user_id,
|
||||
message_id=message_id,
|
||||
agent_id=agent_id,
|
||||
reasons=reasons,
|
||||
comment=comment,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
feedback_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict | None:
|
||||
) -> dict[str, Any] | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.get")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(FeedbackRow, feedback_id)
|
||||
@@ -72,38 +112,101 @@ class FeedbackRepository:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get_by_target(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None,
|
||||
agent_id: str,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict[str, Any] | None:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.get_by_target",
|
||||
)
|
||||
target_key = self.make_target_key(
|
||||
user_id=resolved_user_id,
|
||||
thread_id=thread_id,
|
||||
run_id=run_id,
|
||||
message_id=message_id,
|
||||
agent_id=agent_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = await session.scalar(
|
||||
select(FeedbackRow).where(FeedbackRow.target_key == target_key)
|
||||
)
|
||||
return self._row_to_dict(row) if row is not None else None
|
||||
|
||||
async def list_by_run(
|
||||
self,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
message_id: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_run")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
|
||||
) -> list[dict[str, Any]]:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.list_by_run",
|
||||
)
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
)
|
||||
if message_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.message_id == message_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_by_thread(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
limit: int = 1000,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread")
|
||||
) -> list[dict[str, Any]]:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.list_by_thread",
|
||||
)
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_by_message_ids(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
message_ids: list[str],
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Load current-user feedback for a bounded visible-message batch."""
|
||||
|
||||
if not message_ids:
|
||||
return []
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.list_by_message_ids",
|
||||
)
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.message_id.in_(message_ids),
|
||||
)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
@@ -111,7 +214,10 @@ class FeedbackRepository:
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete")
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.delete",
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = await session.get(FeedbackRow, feedback_id)
|
||||
if row is None:
|
||||
@@ -129,84 +235,152 @@ class FeedbackRepository:
|
||||
thread_id: str,
|
||||
rating: int,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
message_id: str | None = None,
|
||||
agent_id: str = "lead_agent",
|
||||
reasons: list[str] | None = None,
|
||||
comment: str | None = None,
|
||||
) -> dict:
|
||||
"""Create or update feedback for (thread_id, run_id, user_id). rating must be +1 or -1."""
|
||||
) -> dict[str, Any]:
|
||||
"""Create or update feedback for one user/thread/message/agent target."""
|
||||
|
||||
if rating not in (1, -1):
|
||||
raise ValueError(f"rating must be +1 or -1, got {rating}")
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.upsert")
|
||||
async with self._sf() as session:
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is not None:
|
||||
row.rating = rating
|
||||
row.comment = comment
|
||||
row.created_at = datetime.now(UTC)
|
||||
else:
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
rating=rating,
|
||||
comment=comment,
|
||||
created_at=datetime.now(UTC),
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.upsert",
|
||||
)
|
||||
normalized_agent_id = str(agent_id or "lead_agent")[:128]
|
||||
normalized_reasons = self._normalize_reasons(reasons)
|
||||
target_key = self.make_target_key(
|
||||
user_id=resolved_user_id,
|
||||
thread_id=thread_id,
|
||||
run_id=run_id,
|
||||
message_id=message_id,
|
||||
agent_id=normalized_agent_id,
|
||||
)
|
||||
|
||||
# The unique digest handles cross-worker races. On a simultaneous
|
||||
# insert, retry as an update in a fresh transaction.
|
||||
for attempt in range(2):
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
row = await session.scalar(
|
||||
select(FeedbackRow).where(FeedbackRow.target_key == target_key)
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
if row is None:
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
message_id=message_id,
|
||||
agent_id=normalized_agent_id,
|
||||
target_key=target_key,
|
||||
rating=rating,
|
||||
reasons=normalized_reasons,
|
||||
comment=comment,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.run_id = run_id
|
||||
row.rating = rating
|
||||
row.reasons = normalized_reasons
|
||||
row.comment = comment
|
||||
row.updated_at = now
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
if attempt == 0:
|
||||
continue
|
||||
raise
|
||||
# Runtime factories use expire_on_commit=False. Avoid a
|
||||
# post-commit refresh, unsafe behind a read replica.
|
||||
return self._row_to_dict(row)
|
||||
raise RuntimeError("unreachable feedback upsert retry state")
|
||||
|
||||
async def delete_by_run(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
"""Delete the current user's feedback for a run. Returns True if a record was deleted."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete_by_run")
|
||||
"""Delete this user's feedback for a run or one message in the run."""
|
||||
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.delete_by_run",
|
||||
)
|
||||
conditions = [
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
]
|
||||
if message_id is not None:
|
||||
conditions.append(FeedbackRow.message_id == message_id)
|
||||
async with self._sf() as session:
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
result = await session.execute(delete(FeedbackRow).where(*conditions))
|
||||
await session.commit()
|
||||
return True
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def delete_by_target(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str,
|
||||
agent_id: str,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.delete_by_target",
|
||||
)
|
||||
target_key = self.make_target_key(
|
||||
user_id=resolved_user_id,
|
||||
thread_id=thread_id,
|
||||
run_id=run_id,
|
||||
message_id=message_id,
|
||||
agent_id=agent_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(FeedbackRow).where(FeedbackRow.target_key == target_key)
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def list_by_thread_grouped(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict[str, dict]:
|
||||
"""Return feedback grouped by run_id for a thread: {run_id: feedback_dict}."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread_grouped")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return {row.run_id: self._row_to_dict(row) for row in result.scalars()}
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Legacy run-keyed view, retaining the newest row for each run."""
|
||||
|
||||
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict:
|
||||
records = await self.list_by_thread(thread_id, user_id=user_id)
|
||||
return {record["run_id"]: record for record in records}
|
||||
|
||||
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict[str, Any]:
|
||||
"""Aggregate feedback stats for a run using database-side counting."""
|
||||
|
||||
stmt = select(
|
||||
func.count().label("total"),
|
||||
func.coalesce(func.sum(case((FeedbackRow.rating == 1, 1), else_=0)), 0).label("positive"),
|
||||
func.coalesce(func.sum(case((FeedbackRow.rating == -1, 1), else_=0)), 0).label("negative"),
|
||||
).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
|
||||
func.coalesce(
|
||||
func.sum(case((FeedbackRow.rating == 1, 1), else_=0)),
|
||||
0,
|
||||
).label("positive"),
|
||||
func.coalesce(
|
||||
func.sum(case((FeedbackRow.rating == -1, 1), else_=0)),
|
||||
0,
|
||||
).label("negative"),
|
||||
).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).one()
|
||||
return {
|
||||
@@ -215,3 +389,20 @@ class FeedbackRepository:
|
||||
"positive": row.positive,
|
||||
"negative": row.negative,
|
||||
}
|
||||
|
||||
async def aggregate_by_agent(self, agent_id: str) -> dict[str, Any]:
|
||||
"""Return the documented user-feedback answer-quality score."""
|
||||
|
||||
normalized_agent_id = str(agent_id or "lead_agent")[:128]
|
||||
stmt = select(FeedbackRow.rating, FeedbackRow.reasons).where(
|
||||
FeedbackRow.agent_id == normalized_agent_id
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return calculate_answer_quality_score(
|
||||
(
|
||||
{"rating": row.rating, "reasons": list(row.reasons or [])}
|
||||
for row in rows
|
||||
),
|
||||
agent_id=normalized_agent_id,
|
||||
)
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Durable, owner-scoped records for the public-document workbench."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.gongwen_documents.memory import MemoryGongwenDocumentStore
|
||||
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
|
||||
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore, SqlGongwenDocumentStore
|
||||
|
||||
__all__ = [
|
||||
"GongwenDocumentRow",
|
||||
"GongwenDocumentStore",
|
||||
"GongwenDocumentVersionRow",
|
||||
"MemoryGongwenDocumentStore",
|
||||
"SqlGongwenDocumentStore",
|
||||
"make_gongwen_document_store",
|
||||
]
|
||||
|
||||
|
||||
def make_gongwen_document_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryGongwenDocumentStore()
|
||||
return SqlGongwenDocumentStore(session_factory)
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
"""Small in-memory fallback used when persistence is disabled in tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore
|
||||
|
||||
|
||||
class MemoryGongwenDocumentStore(GongwenDocumentStore):
|
||||
def __init__(self) -> None:
|
||||
self._documents: dict[str, dict[str, Any]] = {}
|
||||
self._versions: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _owned(document: dict[str, Any] | None, user_id: str | None) -> bool:
|
||||
return bool(document and document.get("user_id") == user_id)
|
||||
|
||||
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row = {
|
||||
"id": str(data["id"]), "user_id": data.get("user_id"), "title": str(data.get("title") or ""),
|
||||
"document_type": str(data.get("document_type") or "notice"), "source": str(data.get("source") or "upload"),
|
||||
"status": str(data.get("status") or "checking"), "template_key": data.get("template_key"),
|
||||
"research_session_id": data.get("research_session_id"), "original_version_id": None, "latest_version_id": None,
|
||||
"issue_counts": deepcopy(data.get("issue_counts")), "error": data.get("error"), "created_at": now, "updated_at": now,
|
||||
}
|
||||
self._documents[row["id"]] = row
|
||||
return deepcopy(row)
|
||||
|
||||
async def list_documents(self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._documents.values() if r.get("user_id") == user_id]
|
||||
if source:
|
||||
rows = [r for r in rows if r.get("source") == source]
|
||||
if status:
|
||||
rows = [r for r in rows if r.get("status") == status]
|
||||
return deepcopy(sorted(rows, key=lambda item: item["updated_at"], reverse=True)[:limit])
|
||||
|
||||
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
row = self._documents.get(document_id)
|
||||
return deepcopy(row) if self._owned(row, user_id) else None
|
||||
|
||||
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
|
||||
if not self._owned(self._documents.get(document_id), user_id):
|
||||
return []
|
||||
return deepcopy(list(reversed(self._versions.get(document_id, []))))
|
||||
|
||||
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
if not self._owned(self._documents.get(document_id), user_id):
|
||||
return None
|
||||
return next((deepcopy(row) for row in self._versions.get(document_id, []) if row["id"] == version_id), None)
|
||||
|
||||
async def append_version(self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
document = self._documents.get(document_id)
|
||||
if not self._owned(document, user_id):
|
||||
return None
|
||||
rows = self._versions.setdefault(document_id, [])
|
||||
row = {**deepcopy(version), "document_id": document_id, "version_no": len(rows) + 1, "created_at": datetime.now(UTC).isoformat()}
|
||||
rows.append(row)
|
||||
document.update(deepcopy(document_patch))
|
||||
document["latest_version_id"] = row["id"]
|
||||
if row["kind"] == "original" and not document.get("original_version_id"):
|
||||
document["original_version_id"] = row["id"]
|
||||
document["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return deepcopy(document), deepcopy(row)
|
||||
|
||||
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
document = self._documents.get(document_id)
|
||||
if not self._owned(document, user_id):
|
||||
return None
|
||||
document.update(deepcopy(patch))
|
||||
document["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return deepcopy(document)
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
"""ORM rows for the public-document library.
|
||||
|
||||
The binary files deliberately stay outside the database. A document row is
|
||||
the durable library entry; every uploaded original, generated output and
|
||||
format-repaired output is an immutable version row beneath it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class GongwenDocumentRow(Base):
|
||||
__tablename__ = "gongwen_documents"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
|
||||
document_type: Mapped[str] = mapped_column(String(48), nullable=False, default="notice", index=True)
|
||||
# upload | generated. Keep source separate from status so the library can
|
||||
# filter both axes independently.
|
||||
source: Mapped[str] = mapped_column(String(24), nullable=False, default="upload", index=True)
|
||||
# checking | review_required | verified | repaired | failed
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="checking", index=True)
|
||||
template_key: Mapped[str | None] = mapped_column(String(96), nullable=True)
|
||||
research_session_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
original_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
latest_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
issue_counts: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
|
||||
class GongwenDocumentVersionRow(Base):
|
||||
__tablename__ = "gongwen_document_versions"
|
||||
__table_args__ = (
|
||||
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
|
||||
sa.Index("ix_gongwen_document_versions_document_created", "document_id", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
document_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
version_no: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
# original | generated | repaired | markdown
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
file_name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
storage_key: Mapped[str] = mapped_column(String(1024), nullable=False)
|
||||
content_type: Mapped[str] = mapped_column(String(128), nullable=False, default="application/octet-stream")
|
||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
# Exact review snapshot for this version. Later rule updates must not
|
||||
# rewrite history or make past repair decisions impossible to explain.
|
||||
review_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
"""SQL and in-memory stores for public-document library metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
|
||||
|
||||
|
||||
def _json_load(value: Any) -> Any:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _json_dump(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def _as_dict(row: GongwenDocumentRow | GongwenDocumentVersionRow) -> dict[str, Any]:
|
||||
result = row.to_dict()
|
||||
for key, value in tuple(result.items()):
|
||||
if isinstance(value, datetime):
|
||||
result[key] = value.isoformat()
|
||||
if "issue_counts" in result:
|
||||
result["issue_counts"] = _json_load(result["issue_counts"])
|
||||
if "review_snapshot" in result:
|
||||
result["review_snapshot"] = _json_load(result["review_snapshot"])
|
||||
return result
|
||||
|
||||
|
||||
class GongwenDocumentStore(ABC):
|
||||
@abstractmethod
|
||||
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_documents(
|
||||
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def append_version(
|
||||
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
|
||||
) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SqlGongwenDocumentStore(GongwenDocumentStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _owned(stmt, user_id: str | None):
|
||||
return stmt.where(GongwenDocumentRow.user_id == user_id)
|
||||
|
||||
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = GongwenDocumentRow(
|
||||
id=str(data["id"]),
|
||||
user_id=data.get("user_id"),
|
||||
title=str(data.get("title") or ""),
|
||||
document_type=str(data.get("document_type") or "notice"),
|
||||
source=str(data.get("source") or "upload"),
|
||||
status=str(data.get("status") or "checking"),
|
||||
template_key=data.get("template_key"),
|
||||
research_session_id=data.get("research_session_id"),
|
||||
issue_counts=_json_dump(data.get("issue_counts")),
|
||||
error=data.get("error"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _as_dict(row)
|
||||
|
||||
async def list_documents(
|
||||
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = self._owned(select(GongwenDocumentRow), user_id)
|
||||
if source:
|
||||
stmt = stmt.where(GongwenDocumentRow.source == source)
|
||||
if status:
|
||||
stmt = stmt.where(GongwenDocumentRow.status == status)
|
||||
stmt = stmt.order_by(GongwenDocumentRow.updated_at.desc()).limit(max(1, min(limit, 200)))
|
||||
async with self._sf() as session:
|
||||
return [_as_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _as_dict(row) if row else None
|
||||
|
||||
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
|
||||
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
async with self._sf() as session:
|
||||
if (await session.execute(stmt)).scalar_one_or_none() is None:
|
||||
return []
|
||||
versions = await session.execute(
|
||||
select(GongwenDocumentVersionRow)
|
||||
.where(GongwenDocumentVersionRow.document_id == document_id)
|
||||
.order_by(GongwenDocumentVersionRow.version_no.desc())
|
||||
)
|
||||
return [_as_dict(row) for row in versions.scalars()]
|
||||
|
||||
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
async with self._sf() as session:
|
||||
if (await session.execute(stmt)).scalar_one_or_none() is None:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(GongwenDocumentVersionRow).where(
|
||||
GongwenDocumentVersionRow.document_id == document_id,
|
||||
GongwenDocumentVersionRow.id == version_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return _as_dict(row) if row else None
|
||||
|
||||
async def append_version(
|
||||
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
|
||||
) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
async with self._sf() as session:
|
||||
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
document = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if document is None:
|
||||
return None
|
||||
version_no = int(
|
||||
(await session.execute(
|
||||
select(func.coalesce(func.max(GongwenDocumentVersionRow.version_no), 0)).where(
|
||||
GongwenDocumentVersionRow.document_id == document_id
|
||||
)
|
||||
)).scalar_one()
|
||||
) + 1
|
||||
row = GongwenDocumentVersionRow(
|
||||
id=str(version["id"]),
|
||||
document_id=document_id,
|
||||
version_no=version_no,
|
||||
kind=str(version["kind"]),
|
||||
file_name=str(version["file_name"]),
|
||||
storage_key=str(version["storage_key"]),
|
||||
content_type=str(version.get("content_type") or "application/octet-stream"),
|
||||
size_bytes=int(version.get("size_bytes") or 0),
|
||||
sha256=str(version.get("sha256") or ""),
|
||||
review_snapshot=_json_dump(version.get("review_snapshot")),
|
||||
)
|
||||
for key, value in document_patch.items():
|
||||
if key == "issue_counts":
|
||||
value = _json_dump(value)
|
||||
if hasattr(document, key):
|
||||
setattr(document, key, value)
|
||||
document.latest_version_id = row.id
|
||||
if row.kind == "original" and not document.original_version_id:
|
||||
document.original_version_id = row.id
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(document)
|
||||
await session.refresh(row)
|
||||
return _as_dict(document), _as_dict(row)
|
||||
|
||||
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
for key, value in patch.items():
|
||||
if key == "issue_counts":
|
||||
value = _json_dump(value)
|
||||
if hasattr(row, key):
|
||||
setattr(row, key, value)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _as_dict(row)
|
||||
@@ -28,6 +28,9 @@ class LlmWikiStore(ABC):
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
test_questions: list[str] | None = None,
|
||||
test_questions_set: bool = False,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
+9
@@ -28,6 +28,8 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
"owner_user_id": owner_user_id,
|
||||
"name": name,
|
||||
"description": description,
|
||||
"test_questions": None,
|
||||
"retrieval_settings": None,
|
||||
"kb_type": kb_type,
|
||||
"publication_status": "private",
|
||||
"published_by": None,
|
||||
@@ -74,6 +76,9 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
test_questions: list[str] | None = None,
|
||||
test_questions_set: bool = False,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
@@ -84,6 +89,10 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
row["name"] = name
|
||||
if description is not None:
|
||||
row["description"] = description
|
||||
if test_questions_set:
|
||||
row["test_questions"] = list(test_questions or [])
|
||||
if retrieval_settings is not None:
|
||||
row["retrieval_settings"] = dict(retrieval_settings)
|
||||
if wiki_index_enabled is not None:
|
||||
row["wiki_index_enabled"] = wiki_index_enabled
|
||||
if external_search_enabled is not None:
|
||||
|
||||
+6
-1
@@ -8,7 +8,7 @@ from sqlalchemy import Boolean, String, UniqueConstraint, false, true
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class LlmWikiKnowledgeBaseRow(Base):
|
||||
@@ -19,6 +19,11 @@ class LlmWikiKnowledgeBaseRow(Base):
|
||||
owner_user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
name: Mapped[str] = mapped_column(String(255), default="")
|
||||
description: Mapped[str] = mapped_column(String(2048), default="")
|
||||
# ``None`` means "use the product defaults derived from the knowledge-base
|
||||
# name". An explicit empty list is meaningful: the owner disabled starter
|
||||
# questions for this library.
|
||||
test_questions: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
retrieval_settings: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
kb_type: Mapped[str] = mapped_column(String(32), default="document")
|
||||
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
|
||||
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
|
||||
@@ -88,6 +88,9 @@ class LlmWikiRepository(LlmWikiStore):
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
test_questions: list[str] | None = None,
|
||||
test_questions_set: bool = False,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
@@ -99,6 +102,10 @@ class LlmWikiRepository(LlmWikiStore):
|
||||
row.name = name
|
||||
if description is not None:
|
||||
row.description = description
|
||||
if test_questions_set:
|
||||
row.test_questions = list(test_questions or [])
|
||||
if retrieval_settings is not None:
|
||||
row.retrieval_settings = dict(retrieval_settings)
|
||||
if wiki_index_enabled is not None:
|
||||
row.wiki_index_enabled = wiki_index_enabled
|
||||
if external_search_enabled is not None:
|
||||
|
||||
+8
-6
@@ -2,16 +2,17 @@
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.model import (
|
||||
LlmWikiWikiEdgeRow,
|
||||
LlmWikiWikiPageRow,
|
||||
LlmWikiWikiSyncStateRow,
|
||||
LlmWikiWikiVectorRow,
|
||||
)
|
||||
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
|
||||
|
||||
|
||||
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
|
||||
return (
|
||||
MemoryLlmWikiIndexStore()
|
||||
if session_factory is None
|
||||
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
|
||||
)
|
||||
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
|
||||
|
||||
|
||||
__all__ = [
|
||||
@@ -19,6 +20,7 @@ __all__ = [
|
||||
"MemoryLlmWikiIndexStore",
|
||||
"SqlLlmWikiIndexStore",
|
||||
"LlmWikiWikiPageRow",
|
||||
"LlmWikiWikiEdgeRow",
|
||||
"LlmWikiWikiVectorRow",
|
||||
"LlmWikiWikiSyncStateRow",
|
||||
"make_llmwiki_index_store",
|
||||
|
||||
+113
-5
@@ -14,8 +14,85 @@ class LlmWikiIndexStore(ABC):
|
||||
@abstractmethod
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
|
||||
|
||||
async def search_pages_by_keywords(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
tokens: list[str],
|
||||
*,
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return local Wiki body pages containing any token.
|
||||
|
||||
The default keeps lightweight test doubles and third-party stores
|
||||
backward compatible. Database and in-memory stores override it.
|
||||
"""
|
||||
|
||||
return []
|
||||
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
edges: list[dict[str, Any]],
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Replace one mapping's directed Wiki graph snapshot.
|
||||
|
||||
The default keeps lightweight third-party/test stores compatible. An
|
||||
implementation must resolve both endpoints inside ``mapping_id`` and
|
||||
discard dangling or cross-mapping edges. ``None`` means the caller no
|
||||
longer owns the supplied sync lease and no mutation was made.
|
||||
"""
|
||||
|
||||
return {"edge_count": 0, "changed": False}
|
||||
|
||||
async def get_adjacent_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
seed_slugs: dict[str, list[str]] | list[str],
|
||||
*,
|
||||
direction: str = "both",
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return one-hop neighbors within the explicitly allowed mappings.
|
||||
|
||||
Passing a ``{mapping_id: [slug, ...]}`` mapping is preferred because it
|
||||
retains the exact seed provenance. A flat slug list is applied only to
|
||||
the supplied ``mapping_ids`` and never broadens authorization.
|
||||
"""
|
||||
|
||||
return []
|
||||
|
||||
@abstractmethod
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
|
||||
async def touch_page(
|
||||
self,
|
||||
page_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
mapping_id: str | None = None,
|
||||
owner: str | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def refresh_page_metadata(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
owner: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Refresh a mirrored page without replacing its vectors.
|
||||
|
||||
The write is fenced by the mapping sync lease. ``None`` means the
|
||||
caller no longer owns that lease (or the expected page disappeared),
|
||||
while the returned ``changed`` flag reports whether durable page
|
||||
metadata changed. The page is always marked as seen by ``sync_id``.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def replace_page_vectors(
|
||||
@@ -25,13 +102,34 @@ class LlmWikiIndexStore(ABC):
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any]: ...
|
||||
owner: str | None = None,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
|
||||
async def record_page_failure(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
error: str,
|
||||
owner: str | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
|
||||
async def mark_missing_pages_deleted(
|
||||
self,
|
||||
mapping_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
) -> int | None:
|
||||
"""Retire pages absent from a fenced full scan.
|
||||
|
||||
``None`` means the supplied owner/sync pair no longer owns the mapping
|
||||
lease and no mutation was made.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
|
||||
@@ -73,7 +171,17 @@ class LlmWikiIndexStore(ABC):
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
|
||||
async def finish_sync(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
success: bool,
|
||||
remote_page_count: int,
|
||||
error: str | None = None,
|
||||
result: dict[str, Any] | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
|
||||
|
||||
+275
-18
@@ -17,11 +17,28 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self) -> None:
|
||||
self.pages: dict[str, dict[str, Any]] = {}
|
||||
self.vectors: dict[str, list[dict[str, Any]]] = {}
|
||||
self.edges: dict[str, list[dict[str, Any]]] = {}
|
||||
self.states: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def _state(self, mapping_id: str) -> dict[str, Any]:
|
||||
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
|
||||
|
||||
def _has_current_sync_lease(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> bool:
|
||||
state = self.states.get(mapping_id)
|
||||
return bool(
|
||||
state is not None
|
||||
and state.get("lease_owner") == owner
|
||||
and state.get("active_sync_id") == sync_id
|
||||
and isinstance(state.get("lease_expires_at"), datetime)
|
||||
and state["lease_expires_at"] >= _now()
|
||||
)
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
|
||||
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
|
||||
@@ -30,12 +47,219 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
if page_id in self.pages:
|
||||
self.pages[page_id]["last_seen_sync_id"] = sync_id
|
||||
self.pages[page_id]["last_error"] = None
|
||||
async def search_pages_by_keywords(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
tokens: list[str],
|
||||
*,
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
wanted = set(mapping_ids)
|
||||
needles = [str(item).strip().lower() for item in tokens if str(item).strip()]
|
||||
rows = []
|
||||
for page in self.pages.values():
|
||||
if page.get("knowledge_base_mapping_id") not in wanted or page.get("is_remote_deleted"):
|
||||
continue
|
||||
if page.get("status") == "draft" and not include_drafts:
|
||||
continue
|
||||
if page.get("status") not in {"published", "draft"}:
|
||||
continue
|
||||
haystack = " ".join(str(page.get(key) or "") for key in ("title", "summary", "content_md")).lower()
|
||||
if needles and any(needle in haystack for needle in needles):
|
||||
rows.append(dict(page))
|
||||
if len(rows) >= max(1, limit):
|
||||
break
|
||||
return rows
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
edges: list[dict[str, Any]],
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return None
|
||||
pages_by_slug = {str(page.get("slug") or ""): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") == mapping_id and not page.get("is_remote_deleted") and page.get("status") in {"published", "draft"}}
|
||||
canonical = {
|
||||
(
|
||||
str(edge.get("source_slug") or "").strip(),
|
||||
str(edge.get("target_slug") or "").strip(),
|
||||
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
|
||||
)
|
||||
for edge in edges
|
||||
if isinstance(edge, dict)
|
||||
}
|
||||
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
|
||||
replacement = [
|
||||
{
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"source_page_id": pages_by_slug[source]["id"],
|
||||
"target_page_id": pages_by_slug[target]["id"],
|
||||
"source_slug": source,
|
||||
"target_slug": target,
|
||||
"edge_type": edge_type,
|
||||
}
|
||||
for source, target, edge_type in sorted(canonical)
|
||||
]
|
||||
current = self.edges.get(mapping_id, [])
|
||||
current_keys = {(row["source_slug"], row["target_slug"], row["edge_type"]) for row in current}
|
||||
changed = current_keys != canonical
|
||||
if changed:
|
||||
self.edges[mapping_id] = replacement
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return {"edge_count": len(replacement), "changed": changed}
|
||||
|
||||
async def get_adjacent_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
seed_slugs: dict[str, list[str]] | list[str],
|
||||
*,
|
||||
direction: str = "both",
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
allowed = set(str(item).strip() for item in mapping_ids if str(item).strip())
|
||||
if direction not in {"incoming", "outgoing", "both"}:
|
||||
raise ValueError("direction must be incoming, outgoing, or both")
|
||||
if isinstance(seed_slugs, dict):
|
||||
seeds = {mapping_id: {str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip()} for mapping_id in allowed}
|
||||
else:
|
||||
shared = {str(slug).strip() for slug in seed_slugs if str(slug).strip()}
|
||||
seeds = {mapping_id: shared for mapping_id in allowed}
|
||||
pages_by_id = {str(page["id"]): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") in allowed}
|
||||
result: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str, str, str]] = set()
|
||||
for mapping_id in sorted(allowed):
|
||||
mapping_seeds = seeds.get(mapping_id) or set()
|
||||
for edge in self.edges.get(mapping_id, []):
|
||||
traversals: list[tuple[str, str, str, str]] = []
|
||||
if direction in {"outgoing", "both"} and edge["source_slug"] in mapping_seeds:
|
||||
traversals.append(
|
||||
(
|
||||
edge["source_slug"],
|
||||
edge["target_slug"],
|
||||
"outgoing",
|
||||
edge["target_page_id"],
|
||||
)
|
||||
)
|
||||
if direction in {"incoming", "both"} and edge["target_slug"] in mapping_seeds:
|
||||
traversals.append(
|
||||
(
|
||||
edge["target_slug"],
|
||||
edge["source_slug"],
|
||||
"incoming",
|
||||
edge["source_page_id"],
|
||||
)
|
||||
)
|
||||
for seed_slug, neighbor_slug, traversal_direction, page_id in traversals:
|
||||
page = pages_by_id.get(page_id)
|
||||
if not page or page.get("is_remote_deleted"):
|
||||
continue
|
||||
if page.get("status") == "draft" and not include_drafts:
|
||||
continue
|
||||
if page.get("status") not in {"published", "draft"}:
|
||||
continue
|
||||
key = (mapping_id, seed_slug, neighbor_slug, traversal_direction)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(
|
||||
{
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"seed_slug": seed_slug,
|
||||
"neighbor_slug": neighbor_slug,
|
||||
"direction": traversal_direction,
|
||||
"edge_type": edge["edge_type"],
|
||||
"source_slug": edge["source_slug"],
|
||||
"target_slug": edge["target_slug"],
|
||||
"page": dict(page),
|
||||
}
|
||||
)
|
||||
result.sort(
|
||||
key=lambda row: (
|
||||
row["knowledge_base_mapping_id"],
|
||||
row["seed_slug"],
|
||||
row["neighbor_slug"],
|
||||
row["direction"],
|
||||
)
|
||||
)
|
||||
return result[: max(1, min(int(limit), 1000))]
|
||||
|
||||
async def touch_page(
|
||||
self,
|
||||
page_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
mapping_id: str | None = None,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
page = self.pages.get(page_id)
|
||||
if page is None:
|
||||
return False
|
||||
actual_mapping_id = str(page.get("knowledge_base_mapping_id") or "")
|
||||
if mapping_id is not None and mapping_id != actual_mapping_id:
|
||||
return False
|
||||
if owner is not None and not self._has_current_sync_lease(
|
||||
actual_mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
page["last_seen_sync_id"] = sync_id
|
||||
page["last_error"] = None
|
||||
return True
|
||||
|
||||
async def refresh_page_metadata(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
owner: str,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
if not self._has_current_sync_lease(
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
existing = await self.get_page_by_slug_hash(mapping_id, str(page["slug_hash"]))
|
||||
if existing is None:
|
||||
return None
|
||||
durable_changed = any(existing.get(key) != value for key, value in page.items())
|
||||
row = self.pages[str(existing["id"])]
|
||||
row.update(page)
|
||||
row.update(
|
||||
{
|
||||
"last_seen_sync_id": sync_id,
|
||||
"last_error": None,
|
||||
"is_remote_deleted": False,
|
||||
"updated_at": _now().isoformat(),
|
||||
}
|
||||
)
|
||||
if durable_changed:
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return {"changed": durable_changed, "page": dict(row)}
|
||||
|
||||
async def replace_page_vectors(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
owner: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
if owner is not None and not self._has_current_sync_lease(
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
page_id = existing["id"] if existing else str(uuid4())
|
||||
now = _now().isoformat()
|
||||
@@ -59,7 +283,21 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
state["embedding_fingerprint"] = fingerprint
|
||||
return dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
async def record_page_failure(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
error: str,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
if owner is not None and not self._has_current_sync_lease(
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
if existing:
|
||||
self.pages[existing["id"]].update(
|
||||
@@ -70,11 +308,20 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
|
||||
}
|
||||
)
|
||||
return
|
||||
return True
|
||||
page_id = str(uuid4())
|
||||
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
|
||||
return True
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
async def mark_missing_pages_deleted(
|
||||
self,
|
||||
mapping_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
) -> int | None:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return None
|
||||
changed = 0
|
||||
for page in self.pages.values():
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
|
||||
@@ -82,6 +329,8 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
self.vectors.pop(page["id"], None)
|
||||
changed += 1
|
||||
if changed:
|
||||
active_page_ids = {page["id"] for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")}
|
||||
self.edges[mapping_id] = [edge for edge in self.edges.get(mapping_id, []) if edge["source_page_id"] in active_page_ids and edge["target_page_id"] in active_page_ids]
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return changed
|
||||
|
||||
@@ -131,17 +380,27 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return False
|
||||
state = self._state(mapping_id)
|
||||
state["remote_page_count"] = max(0, remote_page_count)
|
||||
state["updated_at"] = _now()
|
||||
return True
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
async def finish_sync(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
success: bool,
|
||||
remote_page_count: int,
|
||||
error: str | None = None,
|
||||
result: dict[str, Any] | None = None,
|
||||
) -> bool:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return False
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
return
|
||||
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
|
||||
was_rebuilding = state.get("state") == "rebuilding"
|
||||
state.update(
|
||||
@@ -158,8 +417,10 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
|
||||
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
|
||||
"last_success_at": _now() if success else state.get("last_success_at"),
|
||||
"last_sync_result": dict(result) if result is not None else state.get("last_sync_result"),
|
||||
}
|
||||
)
|
||||
return True
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
wanted = set(mapping_ids or [])
|
||||
@@ -168,11 +429,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for page in sorted(
|
||||
(
|
||||
page
|
||||
for page in self.pages.values()
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
|
||||
),
|
||||
(page for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")),
|
||||
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
|
||||
):
|
||||
result.append(
|
||||
|
||||
+48
@@ -72,6 +72,53 @@ class LlmWikiWikiVectorRow(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
|
||||
|
||||
class LlmWikiWikiEdgeRow(Base):
|
||||
"""One directed, mapping-scoped link between two mirrored Wiki pages."""
|
||||
|
||||
__tablename__ = "llmwiki_wiki_edges"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"knowledge_base_mapping_id",
|
||||
"source_page_id",
|
||||
"target_page_id",
|
||||
"edge_type",
|
||||
name="uq_llmwiki_wiki_edge_mapping_pages_type",
|
||||
),
|
||||
Index(
|
||||
"ix_llmwiki_wiki_edge_mapping_source",
|
||||
"knowledge_base_mapping_id",
|
||||
"source_page_id",
|
||||
),
|
||||
Index(
|
||||
"ix_llmwiki_wiki_edge_mapping_target",
|
||||
"knowledge_base_mapping_id",
|
||||
"target_page_id",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
source_page_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
target_page_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
edge_type: Mapped[str] = mapped_column(String(32), nullable=False, default="wiki_link")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
|
||||
|
||||
class LlmWikiWikiSyncStateRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_sync_states"
|
||||
|
||||
@@ -91,4 +138,5 @@ class LlmWikiWikiSyncStateRow(Base):
|
||||
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
last_sync_result: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
|
||||
+396
-10
@@ -7,12 +7,18 @@ from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import case, delete, func, insert, or_, select, update
|
||||
from sqlalchemy import and_, case, delete, func, insert, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.model import (
|
||||
LlmWikiWikiEdgeRow,
|
||||
LlmWikiWikiPageRow,
|
||||
LlmWikiWikiSyncStateRow,
|
||||
LlmWikiWikiVectorRow,
|
||||
)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
@@ -40,6 +46,32 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
await session.flush()
|
||||
return row
|
||||
|
||||
async def _fence_current_sync_lease(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> bool:
|
||||
"""Fence destructive snapshot writes behind the current live lease."""
|
||||
|
||||
# Use a conditional UPDATE rather than SELECT ... FOR UPDATE. SQLite
|
||||
# ignores FOR UPDATE, while this statement obtains its write lock on
|
||||
# every supported backend before the page/edge mutation starts.
|
||||
stmt = (
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiSyncStateRow.lease_owner == owner,
|
||||
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
|
||||
)
|
||||
.values(updated_at=_now())
|
||||
)
|
||||
return bool((await session.execute(stmt)).rowcount)
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
|
||||
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
|
||||
@@ -53,18 +85,322 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
async def search_pages_by_keywords(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
tokens: list[str],
|
||||
*,
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
normalized_ids = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
|
||||
normalized_tokens = list(dict.fromkeys(str(item).strip().lower() for item in tokens if str(item).strip()))
|
||||
if not normalized_ids or not normalized_tokens:
|
||||
return []
|
||||
conditions = []
|
||||
for token in normalized_tokens[:32]:
|
||||
needle = f"%{token}%"
|
||||
conditions.extend(
|
||||
(
|
||||
func.lower(LlmWikiWikiPageRow.title).like(needle),
|
||||
func.lower(LlmWikiWikiPageRow.summary).like(needle),
|
||||
func.lower(LlmWikiWikiPageRow.content_md).like(needle),
|
||||
)
|
||||
)
|
||||
statuses = ["published", "draft"] if include_drafts else ["published"]
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(statuses),
|
||||
func.length(func.trim(LlmWikiWikiPageRow.content_md)) > 0,
|
||||
or_(*conditions),
|
||||
)
|
||||
.order_by(LlmWikiWikiPageRow.updated_at.desc())
|
||||
.limit(max(1, min(int(limit), 2000)))
|
||||
)
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
edges: list[dict[str, Any]],
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomically replace an authorized mapping's resolved edge snapshot."""
|
||||
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
page_stmt = select(LlmWikiWikiPageRow).where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
|
||||
)
|
||||
pages = list((await session.execute(page_stmt)).scalars())
|
||||
pages_by_slug = {page.slug: page for page in pages}
|
||||
canonical = {
|
||||
(
|
||||
str(edge.get("source_slug") or "").strip(),
|
||||
str(edge.get("target_slug") or "").strip(),
|
||||
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
|
||||
)
|
||||
for edge in edges
|
||||
if isinstance(edge, dict)
|
||||
}
|
||||
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
|
||||
existing_stmt = select(
|
||||
LlmWikiWikiEdgeRow.source_page_id,
|
||||
LlmWikiWikiEdgeRow.target_page_id,
|
||||
LlmWikiWikiEdgeRow.edge_type,
|
||||
).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id)
|
||||
existing = {(str(source_id), str(target_id), str(edge_type)) for source_id, target_id, edge_type in (await session.execute(existing_stmt)).all()}
|
||||
replacement_ids = {
|
||||
(
|
||||
pages_by_slug[source].id,
|
||||
pages_by_slug[target].id,
|
||||
edge_type,
|
||||
)
|
||||
for source, target, edge_type in canonical
|
||||
}
|
||||
changed = existing != replacement_ids
|
||||
if changed:
|
||||
await session.execute(delete(LlmWikiWikiEdgeRow).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id))
|
||||
now = _now()
|
||||
edge_rows = [
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"source_page_id": pages_by_slug[source].id,
|
||||
"target_page_id": pages_by_slug[target].id,
|
||||
"edge_type": edge_type,
|
||||
"created_at": now,
|
||||
}
|
||||
for source, target, edge_type in sorted(canonical)
|
||||
]
|
||||
for start in range(0, len(edge_rows), self.write_batch_size):
|
||||
await session.execute(
|
||||
insert(LlmWikiWikiEdgeRow),
|
||||
edge_rows[start : start + self.write_batch_size],
|
||||
)
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return {"edge_count": len(canonical), "changed": changed}
|
||||
|
||||
async def get_adjacent_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
seed_slugs: dict[str, list[str]] | list[str],
|
||||
*,
|
||||
direction: str = "both",
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
allowed = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
|
||||
if not allowed:
|
||||
return []
|
||||
if direction not in {"incoming", "outgoing", "both"}:
|
||||
raise ValueError("direction must be incoming, outgoing, or both")
|
||||
if isinstance(seed_slugs, dict):
|
||||
seeds = {mapping_id: list(dict.fromkeys(str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip())) for mapping_id in allowed}
|
||||
else:
|
||||
shared = list(dict.fromkeys(str(slug).strip() for slug in seed_slugs if str(slug).strip()))
|
||||
seeds = {mapping_id: shared for mapping_id in allowed}
|
||||
seeds = {mapping_id: slugs for mapping_id, slugs in seeds.items() if slugs}
|
||||
if not seeds:
|
||||
return []
|
||||
|
||||
source_page = aliased(LlmWikiWikiPageRow)
|
||||
target_page = aliased(LlmWikiWikiPageRow)
|
||||
statuses = ["published", "draft"] if include_drafts else ["published"]
|
||||
cap = max(1, min(int(limit), 1000))
|
||||
|
||||
async def _query(traversal_direction: str) -> list[dict[str, Any]]:
|
||||
seed_page = source_page if traversal_direction == "outgoing" else target_page
|
||||
neighbor_page = target_page if traversal_direction == "outgoing" else source_page
|
||||
seed_conditions = or_(
|
||||
*(
|
||||
and_(
|
||||
LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id,
|
||||
seed_page.slug.in_(slugs),
|
||||
)
|
||||
for mapping_id, slugs in seeds.items()
|
||||
)
|
||||
)
|
||||
stmt = (
|
||||
select(LlmWikiWikiEdgeRow, source_page, target_page)
|
||||
.join(source_page, source_page.id == LlmWikiWikiEdgeRow.source_page_id)
|
||||
.join(target_page, target_page.id == LlmWikiWikiEdgeRow.target_page_id)
|
||||
.where(
|
||||
LlmWikiWikiEdgeRow.knowledge_base_mapping_id.in_(allowed),
|
||||
source_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
|
||||
target_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
|
||||
source_page.is_remote_deleted.is_(False),
|
||||
target_page.is_remote_deleted.is_(False),
|
||||
source_page.status.in_(statuses),
|
||||
target_page.status.in_(statuses),
|
||||
seed_conditions,
|
||||
)
|
||||
.order_by(
|
||||
LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
|
||||
seed_page.slug,
|
||||
neighbor_page.slug,
|
||||
)
|
||||
.limit(cap)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
triples = list((await session.execute(stmt)).all())
|
||||
rows: list[dict[str, Any]] = []
|
||||
for edge, source, target in triples:
|
||||
seed = source if traversal_direction == "outgoing" else target
|
||||
neighbor = target if traversal_direction == "outgoing" else source
|
||||
rows.append(
|
||||
{
|
||||
"knowledge_base_mapping_id": edge.knowledge_base_mapping_id,
|
||||
"seed_slug": seed.slug,
|
||||
"neighbor_slug": neighbor.slug,
|
||||
"direction": traversal_direction,
|
||||
"edge_type": edge.edge_type,
|
||||
"source_slug": source.slug,
|
||||
"target_slug": target.slug,
|
||||
"page": _dict(neighbor),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
result: list[dict[str, Any]] = []
|
||||
if direction in {"outgoing", "both"}:
|
||||
result.extend(await _query("outgoing"))
|
||||
if direction in {"incoming", "both"}:
|
||||
result.extend(await _query("incoming"))
|
||||
result.sort(
|
||||
key=lambda row: (
|
||||
row["knowledge_base_mapping_id"],
|
||||
row["seed_slug"],
|
||||
row["neighbor_slug"],
|
||||
row["direction"],
|
||||
)
|
||||
)
|
||||
deduplicated: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str, str, str]] = set()
|
||||
for row in result:
|
||||
key = (
|
||||
row["knowledge_base_mapping_id"],
|
||||
row["seed_slug"],
|
||||
row["neighbor_slug"],
|
||||
row["direction"],
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduplicated.append(row)
|
||||
if len(deduplicated) >= cap:
|
||||
break
|
||||
return deduplicated
|
||||
|
||||
async def touch_page(
|
||||
self,
|
||||
page_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
mapping_id: str | None = None,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
async with self._sf() as session:
|
||||
if owner is not None:
|
||||
if not mapping_id or not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
conditions = [LlmWikiWikiPageRow.id == page_id]
|
||||
if mapping_id is not None:
|
||||
conditions.append(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id)
|
||||
result = await session.execute(
|
||||
update(LlmWikiWikiPageRow)
|
||||
.where(LlmWikiWikiPageRow.id == page_id)
|
||||
.where(*conditions)
|
||||
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
async def refresh_page_metadata(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
owner: str,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.slug_hash == page["slug_hash"],
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
if row.slug != page["slug"]:
|
||||
raise ValueError("Wiki slug hash collision detected")
|
||||
durable_changed = bool(row.is_remote_deleted or row.last_error) or any(
|
||||
getattr(row, key) != value for key, value in page.items()
|
||||
)
|
||||
for key, value in page.items():
|
||||
setattr(row, key, value)
|
||||
now = _now()
|
||||
row.last_seen_sync_id = sync_id
|
||||
row.last_error = None
|
||||
row.is_remote_deleted = False
|
||||
row.updated_at = now
|
||||
if durable_changed:
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return {"changed": durable_changed, "page": _dict(row)}
|
||||
|
||||
async def replace_page_vectors(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
owner: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
async with self._sf() as session:
|
||||
if owner is not None and not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is not None and row.slug != page["slug"]:
|
||||
@@ -115,10 +451,24 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
await session.commit()
|
||||
return _dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
async def record_page_failure(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
error: str,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
safe_error = error[:4000]
|
||||
async with self._sf() as session:
|
||||
if owner is not None and not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
now = _now()
|
||||
@@ -136,9 +486,23 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row.updated_at = now
|
||||
await self._ensure_state(session, mapping_id)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
async def mark_missing_pages_deleted(
|
||||
self,
|
||||
mapping_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
) -> int | None:
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
@@ -287,17 +651,36 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiSyncStateRow.lease_owner == owner,
|
||||
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
|
||||
)
|
||||
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
async def finish_sync(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
success: bool,
|
||||
remote_page_count: int,
|
||||
error: str | None = None,
|
||||
result: dict[str, Any] | None = None,
|
||||
) -> bool:
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
|
||||
return
|
||||
return False
|
||||
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
|
||||
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
|
||||
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
|
||||
@@ -329,12 +712,15 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
state.ready_page_count = ready_count
|
||||
state.failed_page_count = failed_count
|
||||
state.vector_count = vector_count
|
||||
if result is not None:
|
||||
state.last_sync_result = dict(result)
|
||||
if success and not error:
|
||||
state.last_completed_at = now
|
||||
if success:
|
||||
state.last_success_at = now
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmWikiWikiSyncStateRow)
|
||||
|
||||
+2
-1
@@ -4,13 +4,14 @@ from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.menu_overrides.base import MenuOverrideStore
|
||||
from deerflow.persistence.menu_overrides.memory import MemoryMenuOverrideStore
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
|
||||
from deerflow.persistence.menu_overrides.sql import MenuOverrideRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryMenuOverrideStore",
|
||||
"MenuOverrideRepository",
|
||||
"MenuOverrideRow",
|
||||
"UserMenuOverrideConfigRow",
|
||||
"MenuOverrideStore",
|
||||
"make_menu_override_store",
|
||||
]
|
||||
|
||||
+27
-6
@@ -1,11 +1,11 @@
|
||||
"""Abstract sidebar-menu override persistence API.
|
||||
|
||||
Global / shared store: no ``user_id`` scoping. Reads are open to any caller; the
|
||||
router gates writes behind admin. ``updated_by`` is audit-only.
|
||||
|
||||
The whole override set is treated as one document: the editor computes the full
|
||||
desired layout and replaces every row atomically via :meth:`replace_all`, so the
|
||||
stored set never drifts out of sync with what the admin sees.
|
||||
The original ``menu_overrides`` document remains the administrator's default
|
||||
layout. User layouts are stored as a separate, complete snapshot so that a
|
||||
user can customise the menu without changing the default for anybody else.
|
||||
The router overlays a user's snapshot on top of the default and lets default
|
||||
rows for node ids not present in that snapshot flow through (new administrator
|
||||
menus therefore appear for existing users).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -28,3 +28,24 @@ class MenuOverrideStore(ABC):
|
||||
Returns the persisted rows.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
|
||||
"""Return one user's snapshot, or ``None`` when no snapshot exists."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""Replace one user's snapshot.
|
||||
|
||||
An empty list removes the snapshot and returns ``None``. This is the
|
||||
reset-to-default operation and intentionally differs from the admin
|
||||
``replace_all`` document, where an empty list means an empty default.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
+27
@@ -23,6 +23,7 @@ def _normalize(data: dict[str, Any], updated_by: str | None) -> dict[str, Any]:
|
||||
class MemoryMenuOverrideStore(MenuOverrideStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._user_rows: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
async def list_overrides(self) -> list[dict[str, Any]]:
|
||||
return [dict(r) for r in sorted(self._rows.values(), key=lambda r: r.get("sort_order", 0))]
|
||||
@@ -35,3 +36,29 @@ class MemoryMenuOverrideStore(MenuOverrideStore):
|
||||
continue
|
||||
self._rows[str(node_id)] = _normalize(r, updated_by)
|
||||
return [dict(r) for r in self._rows.values()]
|
||||
|
||||
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
|
||||
rows = self._user_rows.get(str(user_id))
|
||||
return [dict(r) for r in rows] if rows is not None else None
|
||||
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
key = str(user_id)
|
||||
if not rows:
|
||||
self._user_rows.pop(key, None)
|
||||
return None
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
node_id = row.get("node_id")
|
||||
if node_id:
|
||||
by_id[str(node_id)] = _normalize(row, updated_by or key)
|
||||
if not by_id:
|
||||
self._user_rows.pop(key, None)
|
||||
return None
|
||||
self._user_rows[key] = [dict(row) for row in by_id.values()]
|
||||
return [dict(row) for row in self._user_rows[key]]
|
||||
+25
-5
@@ -1,9 +1,10 @@
|
||||
"""ORM model for sidebar-menu overrides (菜单管理).
|
||||
|
||||
Global / shared — there is no ``user_id`` scoping; every account sees the same
|
||||
menu layout. The static menu tree (icons / routes / adminOnly) lives in the
|
||||
frontend code; this table only stores per-node **overrides** keyed by the static
|
||||
node id (or an injected iframe light-app node id ``light-app-<appId>``):
|
||||
``menu_overrides`` stores the administrator's shared default. Personal layouts
|
||||
live separately in ``user_menu_override_configs`` as complete snapshots. The
|
||||
static menu tree (icons / routes / adminOnly) lives in the frontend code; the
|
||||
default table stores per-node **overrides** keyed by the static node id (or an
|
||||
injected iframe light-app node id ``light-app-<appId>``):
|
||||
|
||||
- ``parent_id`` — the effective parent node id (NULL = top level). Re-parenting
|
||||
a node moves its whole subtree.
|
||||
@@ -23,7 +24,7 @@ from sqlalchemy import Boolean, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class MenuOverrideRow(Base):
|
||||
@@ -40,3 +41,22 @@ class MenuOverrideRow(Base):
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
|
||||
class UserMenuOverrideConfigRow(Base):
|
||||
"""A complete personal sidebar-menu snapshot.
|
||||
|
||||
Keeping the snapshot in one row is deliberate: the presence of a row is
|
||||
the ``has_user_override`` bit, while the node ids in ``overrides`` are the
|
||||
user's known-node set. This lets us distinguish an administrator-added
|
||||
node from an old node the user intentionally left at its default value.
|
||||
"""
|
||||
|
||||
__tablename__ = "user_menu_override_configs"
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
overrides: Mapped[list[dict]] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
+84
-2
@@ -2,14 +2,14 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.menu_overrides.base import MenuOverrideStore
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
|
||||
|
||||
|
||||
def _to_dict(row: MenuOverrideRow) -> dict[str, Any]:
|
||||
@@ -31,6 +31,42 @@ def _row_from(data: dict[str, Any], updated_by: str | None) -> MenuOverrideRow:
|
||||
)
|
||||
|
||||
|
||||
def _normalize_user_row(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Keep only the public override fields in the personal JSON document."""
|
||||
return {
|
||||
"node_id": str(data["node_id"]),
|
||||
"parent_id": (str(data["parent_id"]) if data.get("parent_id") not in (None, "") else None),
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"label": (data.get("label") or None),
|
||||
"disabled": bool(data.get("disabled", False)),
|
||||
}
|
||||
|
||||
|
||||
def _user_rows(row: UserMenuOverrideConfigRow) -> list[dict[str, Any]]:
|
||||
value = row.overrides
|
||||
if isinstance(value, str):
|
||||
# PortableJSON handles MySQL automatically, but this keeps old rows
|
||||
# readable if a driver returned the raw TEXT value.
|
||||
import json
|
||||
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
value = []
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
updated_at = row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at
|
||||
return [
|
||||
{
|
||||
**_normalize_user_row(item),
|
||||
"updated_by": row.updated_by,
|
||||
"updated_at": updated_at,
|
||||
}
|
||||
for item in value
|
||||
if isinstance(item, dict) and item.get("node_id")
|
||||
]
|
||||
|
||||
|
||||
class MenuOverrideRepository(MenuOverrideStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
@@ -56,3 +92,49 @@ class MenuOverrideRepository(MenuOverrideStore):
|
||||
for row in new_rows:
|
||||
await session.refresh(row)
|
||||
return [_to_dict(row) for row in new_rows]
|
||||
|
||||
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserMenuOverrideConfigRow, str(user_id))
|
||||
return _user_rows(row) if row is not None else None
|
||||
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
key = str(user_id)
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for data in rows:
|
||||
node_id = data.get("node_id")
|
||||
if node_id:
|
||||
normalized = _normalize_user_row(data)
|
||||
by_id[normalized["node_id"]] = normalized
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserMenuOverrideConfigRow, key)
|
||||
if not by_id:
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
payload = list(by_id.values())
|
||||
if row is None:
|
||||
row = UserMenuOverrideConfigRow(
|
||||
user_id=key,
|
||||
overrides=payload,
|
||||
updated_by=updated_by or key,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.overrides = payload
|
||||
row.updated_by = updated_by or key
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
# The shared session factory uses ``expire_on_commit=False``. Build
|
||||
# the response from the writer-side instance instead of issuing a
|
||||
# post-commit refresh that a read/write-splitting endpoint could
|
||||
# route to a lagging replica.
|
||||
return _user_rows(row)
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
"""Merge assistant-wiki metadata and report-collaboration heads.
|
||||
|
||||
Revision ID: 20260909_01
|
||||
Revises: 20260903_01, 20260907_01
|
||||
Create Date: 2026-09-09
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
revision = "20260909_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
"""Create durable public-document library tables.
|
||||
|
||||
Revision ID: 20260914_01
|
||||
Revises: 20260903_01, 20260907_01
|
||||
Create Date: 2026-09-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260914_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("gongwen_documents"):
|
||||
op.create_table(
|
||||
"gongwen_documents",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("user_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("title", sa.String(length=256), nullable=False, server_default=""),
|
||||
sa.Column("document_type", sa.String(length=48), nullable=False, server_default="notice"),
|
||||
sa.Column("source", sa.String(length=24), nullable=False, server_default="upload"),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="checking"),
|
||||
sa.Column("template_key", sa.String(length=96), nullable=True),
|
||||
sa.Column("research_session_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("original_version_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("latest_version_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("issue_counts", sa.Text(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_gongwen_documents_user_id", "gongwen_documents", ["user_id"])
|
||||
op.create_index("ix_gongwen_documents_document_type", "gongwen_documents", ["document_type"])
|
||||
op.create_index("ix_gongwen_documents_source", "gongwen_documents", ["source"])
|
||||
op.create_index("ix_gongwen_documents_status", "gongwen_documents", ["status"])
|
||||
op.create_index("ix_gongwen_documents_research_session_id", "gongwen_documents", ["research_session_id"])
|
||||
|
||||
if not inspector.has_table("gongwen_document_versions"):
|
||||
op.create_table(
|
||||
"gongwen_document_versions",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("document_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("version_no", sa.Integer(), nullable=False),
|
||||
sa.Column("kind", sa.String(length=32), nullable=False),
|
||||
sa.Column("file_name", sa.String(length=512), nullable=False),
|
||||
sa.Column("storage_key", sa.String(length=1024), nullable=False),
|
||||
sa.Column("content_type", sa.String(length=128), nullable=False, server_default="application/octet-stream"),
|
||||
sa.Column("size_bytes", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("sha256", sa.String(length=64), nullable=False, server_default=""),
|
||||
sa.Column("review_snapshot", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
|
||||
)
|
||||
op.create_index("ix_gongwen_document_versions_document_id", "gongwen_document_versions", ["document_id"])
|
||||
op.create_index(
|
||||
"ix_gongwen_document_versions_document_created",
|
||||
"gongwen_document_versions",
|
||||
["document_id", "created_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if inspector.has_table("gongwen_document_versions"):
|
||||
op.drop_table("gongwen_document_versions")
|
||||
if inspector.has_table("gongwen_documents"):
|
||||
op.drop_table("gongwen_documents")
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
"""Persist mapping-scoped links for local LLMWiki graph expansion.
|
||||
|
||||
Revision ID: 20260916_01
|
||||
Revises: 20260903_01, 20260907_01
|
||||
Create Date: 2026-09-16
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260916_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if inspector.has_table("llmwiki_wiki_edges"):
|
||||
return
|
||||
op.create_table(
|
||||
"llmwiki_wiki_edges",
|
||||
sa.Column("id", sa.String(length=36), primary_key=True),
|
||||
sa.Column(
|
||||
"knowledge_base_mapping_id",
|
||||
sa.String(length=36),
|
||||
sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"source_page_id",
|
||||
sa.String(length=36),
|
||||
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"target_page_id",
|
||||
sa.String(length=36),
|
||||
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"edge_type",
|
||||
sa.String(length=32),
|
||||
nullable=False,
|
||||
server_default="wiki_link",
|
||||
),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint(
|
||||
"knowledge_base_mapping_id",
|
||||
"source_page_id",
|
||||
"target_page_id",
|
||||
"edge_type",
|
||||
name="uq_llmwiki_wiki_edge_mapping_pages_type",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edges_knowledge_base_mapping_id",
|
||||
"llmwiki_wiki_edges",
|
||||
["knowledge_base_mapping_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edges_source_page_id",
|
||||
"llmwiki_wiki_edges",
|
||||
["source_page_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edges_target_page_id",
|
||||
"llmwiki_wiki_edges",
|
||||
["target_page_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edge_mapping_source",
|
||||
"llmwiki_wiki_edges",
|
||||
["knowledge_base_mapping_id", "source_page_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edge_mapping_target",
|
||||
"llmwiki_wiki_edges",
|
||||
["knowledge_base_mapping_id", "target_page_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table("llmwiki_wiki_edges"):
|
||||
op.drop_table("llmwiki_wiki_edges")
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
"""Merge the public-document and assistant-knowledge migration heads.
|
||||
|
||||
Revision ID: 20260916_01
|
||||
Revises: 20260909_01, 20260914_01
|
||||
Create Date: 2026-09-16
|
||||
|
||||
The feedback-scoring migration was added as ``20260916_02`` and references
|
||||
this merge revision. Keeping the merge as an explicit no-op makes the
|
||||
Alembic graph linear again without changing either existing schema branch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
revision = "20260916_01"
|
||||
down_revision = ("20260909_01", "20260914_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Record the merge; both parent revisions already own their DDL."""
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Splitting the migration history requires no schema change."""
|
||||
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
"""Add answer-level feedback reasons and agent score dimensions.
|
||||
|
||||
Revision ID: 20260916_02
|
||||
Revises: 20260916_01
|
||||
Create Date: 2026-09-16
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260916_02"
|
||||
down_revision = "20260916_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "feedback"
|
||||
_OLD_INDEX = "uq_feedback_thread_run_user"
|
||||
_TARGET_INDEX = "uq_feedback_target_key"
|
||||
_AGENT_INDEX = "ix_feedback_agent_created"
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
return {
|
||||
str(column["name"])
|
||||
for column in sa.inspect(op.get_bind()).get_columns(_TABLE)
|
||||
}
|
||||
|
||||
|
||||
def _indexes() -> set[str]:
|
||||
return {
|
||||
str(index["name"])
|
||||
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE)
|
||||
if index.get("name")
|
||||
}
|
||||
|
||||
|
||||
def _json_type() -> sa.types.TypeEngine:
|
||||
return sa.Text() if op.get_bind().dialect.name == "mysql" else sa.JSON()
|
||||
|
||||
|
||||
def _empty_reasons_value() -> str | list[str]:
|
||||
# PortableJSON is TEXT on MySQL, where binding a Python list directly to
|
||||
# a raw migration Text column is invalid. SQLite/PostgreSQL JSON columns
|
||||
# should receive the native list so their type processors serialize it.
|
||||
return "[]" if op.get_bind().dialect.name == "mysql" else []
|
||||
|
||||
|
||||
def _target_key(
|
||||
*,
|
||||
user_id: str | None,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None,
|
||||
agent_id: str,
|
||||
) -> str:
|
||||
message_key = message_id or f"run:{run_id}"
|
||||
raw = "\x1f".join(
|
||||
(user_id or "", thread_id, message_key, agent_id or "lead_agent")
|
||||
)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _backfill() -> None:
|
||||
bind = op.get_bind()
|
||||
feedback = sa.table(
|
||||
_TABLE,
|
||||
sa.column("feedback_id", sa.String(64)),
|
||||
sa.column("run_id", sa.String(64)),
|
||||
sa.column("thread_id", sa.String(64)),
|
||||
sa.column("user_id", sa.String(64)),
|
||||
sa.column("message_id", sa.String(64)),
|
||||
sa.column("agent_id", sa.String(128)),
|
||||
sa.column("target_key", sa.String(64)),
|
||||
sa.column("reasons", _json_type()),
|
||||
sa.column("created_at", sa.DateTime(timezone=True)),
|
||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
rows = list(
|
||||
bind.execute(
|
||||
sa.select(
|
||||
feedback.c.feedback_id,
|
||||
feedback.c.run_id,
|
||||
feedback.c.thread_id,
|
||||
feedback.c.user_id,
|
||||
feedback.c.message_id,
|
||||
feedback.c.agent_id,
|
||||
feedback.c.created_at,
|
||||
)
|
||||
).mappings()
|
||||
)
|
||||
|
||||
run_agents: dict[str, str] = {}
|
||||
inspector = sa.inspect(bind)
|
||||
if rows and inspector.has_table("runs"):
|
||||
run_columns = {str(column["name"]) for column in inspector.get_columns("runs")}
|
||||
if "assistant_id" in run_columns:
|
||||
runs = sa.table(
|
||||
"runs",
|
||||
sa.column("run_id", sa.String(64)),
|
||||
sa.column("assistant_id", sa.String(128)),
|
||||
)
|
||||
run_ids = sorted({str(row["run_id"]) for row in rows})
|
||||
run_agents = {
|
||||
str(row.run_id): str(row.assistant_id)
|
||||
for row in bind.execute(
|
||||
sa.select(runs.c.run_id, runs.c.assistant_id).where(
|
||||
runs.c.run_id.in_(run_ids)
|
||||
)
|
||||
)
|
||||
if row.assistant_id
|
||||
}
|
||||
|
||||
seen_target_keys: set[str] = set()
|
||||
for row in rows:
|
||||
agent_id = str(
|
||||
run_agents.get(str(row["run_id"]))
|
||||
or row.get("agent_id")
|
||||
or "lead_agent"
|
||||
)[:128]
|
||||
target_key = _target_key(
|
||||
user_id=row.get("user_id"),
|
||||
thread_id=str(row["thread_id"]),
|
||||
run_id=str(row["run_id"]),
|
||||
message_id=row.get("message_id"),
|
||||
agent_id=agent_id,
|
||||
)
|
||||
# SQL unique indexes historically allowed several NULL-user rows.
|
||||
# Keep those legacy records without blocking the new unique digest;
|
||||
# one canonical row retains the deterministic future-upsert key.
|
||||
if target_key in seen_target_keys:
|
||||
legacy_raw = f"{target_key}\x1flegacy:{row['feedback_id']}"
|
||||
target_key = hashlib.sha256(legacy_raw.encode("utf-8")).hexdigest()
|
||||
else:
|
||||
seen_target_keys.add(target_key)
|
||||
bind.execute(
|
||||
sa.update(feedback)
|
||||
.where(feedback.c.feedback_id == row["feedback_id"])
|
||||
.values(
|
||||
agent_id=agent_id,
|
||||
target_key=target_key,
|
||||
reasons=_empty_reasons_value(),
|
||||
updated_at=row.get("created_at"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TABLE):
|
||||
return
|
||||
|
||||
columns = _columns()
|
||||
if "agent_id" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(
|
||||
"agent_id",
|
||||
sa.String(length=128),
|
||||
nullable=True,
|
||||
server_default="lead_agent",
|
||||
),
|
||||
)
|
||||
if "target_key" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("target_key", sa.String(length=64), nullable=True),
|
||||
)
|
||||
if "reasons" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("reasons", _json_type(), nullable=True),
|
||||
)
|
||||
if "updated_at" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
|
||||
_backfill()
|
||||
|
||||
# SQLite needs a batch table rebuild for nullability changes; Alembic uses
|
||||
# native ALTER statements on MySQL/PostgreSQL where possible.
|
||||
with op.batch_alter_table(_TABLE) as batch:
|
||||
batch.alter_column(
|
||||
"agent_id",
|
||||
existing_type=sa.String(length=128),
|
||||
nullable=False,
|
||||
server_default=None,
|
||||
)
|
||||
batch.alter_column(
|
||||
"target_key",
|
||||
existing_type=sa.String(length=64),
|
||||
nullable=False,
|
||||
)
|
||||
batch.alter_column(
|
||||
"reasons",
|
||||
existing_type=_json_type(),
|
||||
nullable=False,
|
||||
)
|
||||
batch.alter_column(
|
||||
"updated_at",
|
||||
existing_type=sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
indexes = _indexes()
|
||||
if _OLD_INDEX in indexes:
|
||||
op.drop_index(_OLD_INDEX, table_name=_TABLE)
|
||||
indexes = _indexes()
|
||||
if _TARGET_INDEX not in indexes:
|
||||
op.create_index(_TARGET_INDEX, _TABLE, ["target_key"], unique=True)
|
||||
if _AGENT_INDEX not in indexes:
|
||||
op.create_index(_AGENT_INDEX, _TABLE, ["agent_id", "created_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
return
|
||||
indexes = _indexes()
|
||||
if _AGENT_INDEX in indexes:
|
||||
op.drop_index(_AGENT_INDEX, table_name=_TABLE)
|
||||
if _TARGET_INDEX in indexes:
|
||||
op.drop_index(_TARGET_INDEX, table_name=_TABLE)
|
||||
if _OLD_INDEX not in _indexes():
|
||||
op.create_index(
|
||||
_OLD_INDEX,
|
||||
_TABLE,
|
||||
["thread_id", "run_id", "user_id"],
|
||||
unique=True,
|
||||
mysql_length={"user_id": 63},
|
||||
)
|
||||
columns = _columns()
|
||||
with op.batch_alter_table(_TABLE) as batch:
|
||||
for name in ("updated_at", "reasons", "target_key", "agent_id"):
|
||||
if name in columns:
|
||||
batch.drop_column(name)
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
"""Persist the latest LLMWiki incremental-sync result.
|
||||
|
||||
Revision ID: 20260920_01
|
||||
Revises: 20260916_02
|
||||
Create Date: 2026-09-20
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20260920_01"
|
||||
down_revision = "20260916_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("llmwiki_wiki_sync_states"):
|
||||
return
|
||||
columns = {
|
||||
str(column["name"])
|
||||
for column in inspector.get_columns("llmwiki_wiki_sync_states")
|
||||
}
|
||||
if "last_sync_result" not in columns:
|
||||
op.add_column(
|
||||
"llmwiki_wiki_sync_states",
|
||||
sa.Column("last_sync_result", PortableJSON(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("llmwiki_wiki_sync_states"):
|
||||
return
|
||||
columns = {
|
||||
str(column["name"])
|
||||
for column in inspector.get_columns("llmwiki_wiki_sync_states")
|
||||
}
|
||||
if "last_sync_result" in columns:
|
||||
op.drop_column("llmwiki_wiki_sync_states", "last_sync_result")
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
"""Add editable starter questions and retrieval settings to LLMWiki mappings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260920_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("llmwiki_knowledge_bases"):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspector.get_columns("llmwiki_knowledge_bases")}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
columns = _columns()
|
||||
if not columns:
|
||||
return
|
||||
if "test_questions" not in columns:
|
||||
op.add_column("llmwiki_knowledge_bases", sa.Column("test_questions", sa.JSON(), nullable=True))
|
||||
if "retrieval_settings" not in columns:
|
||||
op.add_column("llmwiki_knowledge_bases", sa.Column("retrieval_settings", sa.JSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
columns = _columns()
|
||||
if "retrieval_settings" in columns:
|
||||
op.drop_column("llmwiki_knowledge_bases", "retrieval_settings")
|
||||
if "test_questions" in columns:
|
||||
op.drop_column("llmwiki_knowledge_bases", "test_questions")
|
||||
+21
-3
@@ -35,6 +35,7 @@ from deerflow.persistence.agents.model import (
|
||||
AgentRow,
|
||||
AgentSkillRow,
|
||||
)
|
||||
from deerflow.persistence.analysis_projects import AnalysisProjectConfigRow, AnalysisProjectRow, AnalysisProjectThreadRow
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeBaseRow,
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
@@ -48,7 +49,7 @@ from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeWikiRevisionRow,
|
||||
)
|
||||
from deerflow.persistence.business_mapping.model import BusinessMappingRow
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow, UserCompactMenuOverrideConfigRow
|
||||
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
|
||||
from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow
|
||||
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
|
||||
@@ -60,13 +61,19 @@ from deerflow.persistence.document_rewrite_versions.model import DocumentRewrite
|
||||
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
|
||||
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
from deerflow.persistence.light_apps.model import LightAppRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.model import (
|
||||
LlmWikiWikiEdgeRow,
|
||||
LlmWikiWikiPageRow,
|
||||
LlmWikiWikiSyncStateRow,
|
||||
LlmWikiWikiVectorRow,
|
||||
)
|
||||
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.notifications.model import NotificationRow
|
||||
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
|
||||
@@ -133,6 +140,7 @@ from deerflow.persistence.taskcop_tasks.model import TaskCopTaskRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.upload_policy.model import UploadPolicyRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.user_preferences.model import UserPreferencesRow
|
||||
from deerflow.persistence.user_prompts.model import UserCustomPromptRow
|
||||
@@ -150,8 +158,11 @@ from deerflow.persistence.workflow_runs.model import (
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
|
||||
__all__ = [
|
||||
"AnalysisProjectConfigRow",
|
||||
"FeedbackRow",
|
||||
"FixedQuestionRow",
|
||||
"GongwenDocumentRow",
|
||||
"GongwenDocumentVersionRow",
|
||||
"AgentExtrasRow",
|
||||
"AgentFavoriteRow",
|
||||
"AgentRow",
|
||||
@@ -160,6 +171,7 @@ __all__ = [
|
||||
"BusinessMappingRow",
|
||||
"AssistantKnowledgeBaseRow",
|
||||
"AssistantKnowledgeSourceRow",
|
||||
"AssistantKnowledgeVectorRow",
|
||||
"AssistantKnowledgeImportJobRow",
|
||||
"AssistantKnowledgeWikiPageRow",
|
||||
"AssistantKnowledgeWikiRevisionRow",
|
||||
@@ -168,6 +180,7 @@ __all__ = [
|
||||
"AssistantKnowledgeRelationRow",
|
||||
"AssistantKnowledgeRelationContributionRow",
|
||||
"CompactMenuOverrideRow",
|
||||
"UserCompactMenuOverrideConfigRow",
|
||||
"ConcurrencySampleRow",
|
||||
"DashboardSessionRow",
|
||||
"DocumentRewriteVersionRow",
|
||||
@@ -201,9 +214,11 @@ __all__ = [
|
||||
"LlmWikiKnowledgeBaseRow",
|
||||
"LlmWikiConversationDepositRow",
|
||||
"LlmWikiWikiPageRow",
|
||||
"LlmWikiWikiEdgeRow",
|
||||
"LlmWikiWikiVectorRow",
|
||||
"LlmWikiWikiSyncStateRow",
|
||||
"MenuOverrideRow",
|
||||
"UserMenuOverrideConfigRow",
|
||||
"NotificationRow",
|
||||
"ParallelAgentTaskRow",
|
||||
"PositionRoundtableNodeRow",
|
||||
@@ -259,6 +274,8 @@ __all__ = [
|
||||
"TagAssignmentRow",
|
||||
"TagRow",
|
||||
"TaskButtonRow",
|
||||
"AnalysisProjectRow",
|
||||
"AnalysisProjectThreadRow",
|
||||
"TaskCopTaskRow",
|
||||
"TaskReportRow",
|
||||
"EmbedSessionRow",
|
||||
@@ -266,6 +283,7 @@ __all__ = [
|
||||
"ThreadMetaRow",
|
||||
"ThreadShareRow",
|
||||
"ToolCallMetricRow",
|
||||
"UploadPolicyRow",
|
||||
"UserCustomPromptRow",
|
||||
"UserMemoryConfigRow",
|
||||
"UserPreferencesRow",
|
||||
|
||||
+32
-4
@@ -57,12 +57,40 @@ class ParallelAgentTaskRepository:
|
||||
data["messages"] = _loads(row.messages) or []
|
||||
return data
|
||||
|
||||
async def list_tasks(self, user_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
|
||||
"""列出当前用户的任务(轻量元信息,不含 messages,供历史列表)。"""
|
||||
stmt = select(ParallelAgentTaskRow).where(ParallelAgentTaskRow.user_id == user_id).order_by(ParallelAgentTaskRow.updated_at.desc()).limit(limit)
|
||||
async def list_tasks(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
limit: int = 200,
|
||||
offset: int = 0,
|
||||
id_prefix: str | None = None,
|
||||
include_messages: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List the current user's tasks, optionally including saved snapshots.
|
||||
|
||||
History-list callers keep the lightweight default. Server-side
|
||||
projections such as the artifact library can opt into ``messages`` in
|
||||
one query instead of issuing an N+1 ``get_task`` sequence. ``id_prefix``
|
||||
and ``offset`` allow those projections to page only their own task
|
||||
namespace without loading unrelated, potentially large transcripts.
|
||||
"""
|
||||
stmt = select(ParallelAgentTaskRow).where(ParallelAgentTaskRow.user_id == user_id)
|
||||
if id_prefix:
|
||||
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix))
|
||||
stmt = (
|
||||
stmt.order_by(
|
||||
ParallelAgentTaskRow.updated_at.desc(),
|
||||
ParallelAgentTaskRow.id.asc(),
|
||||
)
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._to_dict(row, with_messages=False) for row in result.scalars()]
|
||||
return [
|
||||
self._to_dict(row, with_messages=include_messages)
|
||||
for row in result.scalars()
|
||||
]
|
||||
|
||||
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""External-task multi-agent scheduled analysis persistence (任务分析定时)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
|
||||
from deerflow.persistence.task_analysis.data_blocks import (
|
||||
DEFAULT_DATA_BLOCK_KEYS,
|
||||
DEFAULT_DATA_BLOCKS,
|
||||
merge_payload,
|
||||
)
|
||||
from deerflow.persistence.task_analysis.memory import MemoryTaskAnalysisStore
|
||||
from deerflow.persistence.task_analysis.model import (
|
||||
TaskAnalysisBindingRow,
|
||||
TaskAnalysisContentVersionRow,
|
||||
TaskAnalysisRunMessageRow,
|
||||
TaskAnalysisRunRow,
|
||||
TaskAnalysisTaskRow,
|
||||
TaskDataBlockDefRow,
|
||||
TaskDataBlockRow,
|
||||
TaskDataPeriodRow,
|
||||
TaskDataSubmissionRow,
|
||||
)
|
||||
from deerflow.persistence.task_analysis.sql import TaskAnalysisRepository
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_DATA_BLOCK_KEYS",
|
||||
"DEFAULT_DATA_BLOCKS",
|
||||
"MemoryTaskAnalysisStore",
|
||||
"TaskAnalysisBindingRow",
|
||||
"TaskAnalysisContentVersionRow",
|
||||
"TaskAnalysisRepository",
|
||||
"TaskAnalysisRunMessageRow",
|
||||
"TaskAnalysisRunRow",
|
||||
"TaskAnalysisStore",
|
||||
"TaskAnalysisTaskRow",
|
||||
"TaskDataBlockDefRow",
|
||||
"TaskDataBlockRow",
|
||||
"TaskDataPeriodRow",
|
||||
"TaskDataSubmissionRow",
|
||||
"make_task_analysis_store",
|
||||
"merge_payload",
|
||||
]
|
||||
|
||||
|
||||
def make_task_analysis_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryTaskAnalysisStore()
|
||||
return TaskAnalysisRepository(session_factory)
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
"""Abstract store API for the external-task scheduled analysis feature."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
class TaskAnalysisStore(ABC):
|
||||
"""Tasks / bindings / runs / run-messages / content versions."""
|
||||
|
||||
# ── external tasks ──────────────────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create a task. Raises ValueError when source_task_id already exists."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_task(self, task_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_tasks(self) -> list[dict[str, Any]]:
|
||||
"""All tasks newest first, each augmented with its binding (if any)."""
|
||||
|
||||
@abstractmethod
|
||||
async def delete_task_cascade(self, task_id: str) -> bool:
|
||||
"""Delete task + binding + runs + messages + versions. True if existed."""
|
||||
|
||||
# ── bindings ────────────────────────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create-or-update the 1:1 binding for a task (refresh chain snapshot)."""
|
||||
|
||||
@abstractmethod
|
||||
async def delete_binding(self, external_task_id: str) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
"""Enabled bindings with next_run_at <= now (execution candidates)."""
|
||||
|
||||
@abstractmethod
|
||||
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Patch mutable fields (next_run_at / last_status / direction_notes...)."""
|
||||
|
||||
@abstractmethod
|
||||
async def last_bound_chain_id(self) -> str | None:
|
||||
"""Most recently updated binding's chain id (default option for create API)."""
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Idempotent create: same binding_id + scheduled_for returns existing row."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
"""Runs still pending/running past ``older_than`` (watchdog recovery)."""
|
||||
|
||||
# ── run messages ────────────────────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
|
||||
"""Append batch of {seat_agent_id, stage_index, event_type, content}."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_run_messages(
|
||||
self, run_id: str, *, seat: str | None = None, limit: int = 500
|
||||
) -> list[dict[str, Any]]: ...
|
||||
|
||||
# ── content versions (timeline) ─────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""One version per run (idempotent on run_id: replaces existing)."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_versions(self, external_task_id: str, *, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:
|
||||
"""Timeline versions newest first (对方详情接口数据源)."""
|
||||
|
||||
# ── data delivery: block defs / periods / submissions ────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
"""Seed the 5 default block defs when absent (create/legacy tasks); return ordered."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
"""Block defs ordered by sort_order (empty list when never seeded)."""
|
||||
|
||||
@abstractmethod
|
||||
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Patch title/sort_order per block_key. ValueError on unknown block_key."""
|
||||
|
||||
@abstractmethod
|
||||
async def open_period(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
source: str,
|
||||
run_id: str | None = None,
|
||||
opened_at: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Append a new timeline period (period_index = max+1). Idempotent on run_id.
|
||||
|
||||
``opened_at`` 默认取当前时间;演示/回填场景可显式指定刻度时刻。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def current_period(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""Latest period (submissions always land here)."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
|
||||
"""Periods ascending by period_index (timeline ticks)."""
|
||||
|
||||
@abstractmethod
|
||||
async def append_submission(
|
||||
self,
|
||||
task_id: str,
|
||||
period_index: int,
|
||||
block_key: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
submitted_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Append to the submission log (seq per task+period+block)."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_submissions(
|
||||
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Submission log for one block of one period ordered by seq (merge source)."""
|
||||
|
||||
@abstractmethod
|
||||
async def upsert_period_block(
|
||||
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
"""Write the merged payload for (task, period, block); bumps updated_at."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
|
||||
"""All merged block rows of a task ({block_key, period_index, payload, updated_at})."""
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
"""Constants and pure helpers for the scheduled data-delivery blocks.
|
||||
|
||||
块 key 定死 ``block_1``~``block_6``(已定稿口径,不做席位↔块映射);payload 不做
|
||||
schema 校验(schema 会变),只要求是 JSON 对象。合并规则:顶层 key 合并、同 key
|
||||
后写覆盖 —— 多个智能体可各写一个块的不同部分,一个智能体也可写多个块。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# (block_key, 默认标题);建任务时自动种 6 条,标题/排序后续可经管理端修改。
|
||||
DEFAULT_DATA_BLOCKS: tuple[tuple[str, str], ...] = (
|
||||
("block_1", "数据块一"),
|
||||
("block_2", "数据块二"),
|
||||
("block_3", "数据块三"),
|
||||
("block_4", "数据块四"),
|
||||
("block_5", "数据块五"),
|
||||
("block_6", "数据块六"),
|
||||
)
|
||||
|
||||
DEFAULT_DATA_BLOCK_KEYS: tuple[str, ...] = tuple(key for key, _title in DEFAULT_DATA_BLOCKS)
|
||||
|
||||
# 期(时间轴刻度)触发来源。
|
||||
PERIOD_TRIGGER_CRON = "cron"
|
||||
PERIOD_TRIGGER_MANUAL = "manual"
|
||||
PERIOD_TRIGGER_EXTERNAL = "external"
|
||||
|
||||
|
||||
def default_block_defs() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"block_key": key, "title": title, "sort_order": index}
|
||||
for index, (key, title) in enumerate(DEFAULT_DATA_BLOCKS, start=1)
|
||||
]
|
||||
|
||||
|
||||
def is_default_block_key(block_key: str) -> bool:
|
||||
return block_key in DEFAULT_DATA_BLOCK_KEYS
|
||||
|
||||
|
||||
def merge_payload(base: dict[str, Any], incoming: dict[str, Any]) -> dict[str, Any]:
|
||||
"""顶层 key 合并、同 key 后写覆盖(纯函数,返回新 dict)。"""
|
||||
merged = dict(base)
|
||||
merged.update(incoming)
|
||||
return merged
|
||||
+458
@@ -0,0 +1,458 @@
|
||||
"""In-memory task-analysis store (no DB configured / tests).
|
||||
|
||||
Also serves as the behavioural reference for the SQL repository: field
|
||||
normalisation (JSON encode/decode, ISO datetime) lives in ``_normalize_*``
|
||||
helpers that the SQL layer mirrors.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
|
||||
|
||||
|
||||
def _iso(value: datetime | str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
return str(value)
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if value is None or isinstance(value, datetime):
|
||||
return value
|
||||
try:
|
||||
return datetime.fromisoformat(str(value))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _dump(value: Any) -> str | None:
|
||||
return None if value is None else json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _load(value: str | None, default: Any) -> Any:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
class MemoryTaskAnalysisStore(TaskAnalysisStore):
|
||||
def __init__(self) -> None:
|
||||
self._tasks: dict[str, dict[str, Any]] = {}
|
||||
self._bindings: dict[str, dict[str, Any]] = {} # keyed by external_task_id
|
||||
self._runs: dict[str, dict[str, Any]] = {}
|
||||
self._messages: list[dict[str, Any]] = []
|
||||
self._versions: dict[str, dict[str, Any]] = {} # keyed by run_id
|
||||
# 数据交付:块定义 / 期刻度 / 合并块 / 提交流水。
|
||||
self._block_defs: dict[str, dict[str, dict[str, Any]]] = {} # task_id -> block_key -> def
|
||||
self._periods: dict[str, list[dict[str, Any]]] = {} # task_id -> ordered by period_index
|
||||
self._task_blocks: dict[tuple[str, int, str], dict[str, Any]] = {} # (task, period, block)
|
||||
self._submissions: dict[tuple[str, int, str], list[dict[str, Any]]] = {}
|
||||
self._seq = 0
|
||||
# 绑定最近写入序号(时钟精度不足时仍能稳定判定“最近一次绑定”)。
|
||||
self._binding_rev = 0
|
||||
|
||||
def _next_id(self, prefix: str) -> str:
|
||||
self._seq += 1
|
||||
return f"{prefix}-{self._seq:06d}"
|
||||
|
||||
# ── data delivery helpers ───────────────────────────────────────────────
|
||||
|
||||
def _seed_block_defs(self, task_id: str) -> None:
|
||||
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
|
||||
|
||||
existing = self._block_defs.setdefault(task_id, {})
|
||||
now = datetime.now(UTC).isoformat()
|
||||
for definition in default_block_defs():
|
||||
if definition["block_key"] in existing:
|
||||
continue
|
||||
existing[definition["block_key"]] = {
|
||||
"id": self._next_id("tdbd"),
|
||||
"task_id": task_id,
|
||||
**definition,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
|
||||
def _ordered_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
rows = list(self._block_defs.get(task_id, {}).values())
|
||||
rows.sort(key=lambda d: (d.get("sort_order", 0), d.get("block_key", "")))
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def _period_rows(self, task_id: str) -> list[dict[str, Any]]:
|
||||
rows = list(self._periods.get(task_id, []))
|
||||
rows.sort(key=lambda p: p["period_index"])
|
||||
return rows
|
||||
|
||||
# ── tasks ───────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
source = str(data["source_task_id"])
|
||||
if await self.get_task_by_source(source) is not None:
|
||||
raise ValueError(f"source_task_id already exists: {source}")
|
||||
now = datetime.now(UTC).isoformat()
|
||||
task = {
|
||||
"id": self._next_id("ta"),
|
||||
"source_task_id": source,
|
||||
"name": str(data.get("name") or ""),
|
||||
"description": data.get("description"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._tasks[task["id"]] = dict(task)
|
||||
# 建任务即种 6 条数据块定义(块 key 定死,标题后续可改)。
|
||||
self._seed_block_defs(task["id"])
|
||||
return dict(task)
|
||||
|
||||
async def get_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
task = self._tasks.get(task_id)
|
||||
return dict(task) if task else None
|
||||
|
||||
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None:
|
||||
for task in self._tasks.values():
|
||||
if task["source_task_id"] == source_task_id:
|
||||
return dict(task)
|
||||
return None
|
||||
|
||||
async def list_tasks(self) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
for task in sorted(self._tasks.values(), key=lambda t: t["created_at"], reverse=True):
|
||||
row = dict(task)
|
||||
row["binding"] = dict(self._bindings[task["id"]]) if task["id"] in self._bindings else None
|
||||
out.append(row)
|
||||
return out
|
||||
|
||||
async def delete_task_cascade(self, task_id: str) -> bool:
|
||||
if task_id not in self._tasks:
|
||||
return False
|
||||
del self._tasks[task_id]
|
||||
self._bindings.pop(task_id, None)
|
||||
run_ids = [run_id for run_id, run in self._runs.items() if run["external_task_id"] == task_id]
|
||||
for run_id in run_ids:
|
||||
del self._runs[run_id]
|
||||
self._versions.pop(run_id, None)
|
||||
self._messages = [m for m in self._messages if m["run_id"] not in set(run_ids)]
|
||||
self._block_defs.pop(task_id, None)
|
||||
self._periods.pop(task_id, None)
|
||||
self._task_blocks = {
|
||||
key: row for key, row in self._task_blocks.items() if key[0] != task_id
|
||||
}
|
||||
self._submissions = {
|
||||
key: rows for key, rows in self._submissions.items() if key[0] != task_id
|
||||
}
|
||||
return True
|
||||
|
||||
# ── bindings ────────────────────────────────────────────────────────────
|
||||
|
||||
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None:
|
||||
binding = self._bindings.get(external_task_id)
|
||||
return dict(binding) if binding else None
|
||||
|
||||
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None:
|
||||
for binding in self._bindings.values():
|
||||
if binding["id"] == binding_id:
|
||||
return dict(binding)
|
||||
return None
|
||||
|
||||
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
existing = self._bindings.get(external_task_id)
|
||||
now = datetime.now(UTC).isoformat()
|
||||
self._binding_rev += 1
|
||||
if existing is None:
|
||||
row = {
|
||||
"id": self._next_id("tab"),
|
||||
"external_task_id": external_task_id,
|
||||
"chain_id": str(data["chain_id"]),
|
||||
"chain_snapshot": data.get("chain_snapshot") or {},
|
||||
"cron_expr": str(data.get("cron_expr") or ""),
|
||||
"timezone": str(data.get("timezone") or "Asia/Shanghai"),
|
||||
"schedule_type": str(data.get("schedule_type") or "cron"),
|
||||
"enabled": bool(data.get("enabled", True)),
|
||||
"next_run_at": _iso(data.get("next_run_at")),
|
||||
"coordinator_prompt": data.get("coordinator_prompt"),
|
||||
"direction_notes": list(data.get("direction_notes") or []),
|
||||
"last_run_at": None,
|
||||
"last_status": None,
|
||||
"model": data.get("model"),
|
||||
"created_by": data.get("created_by"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"_rev": self._binding_rev,
|
||||
}
|
||||
else:
|
||||
row = dict(existing)
|
||||
row["_rev"] = self._binding_rev
|
||||
row.update({k: v for k, v in data.items() if k in {
|
||||
"chain_id", "chain_snapshot", "cron_expr", "timezone", "schedule_type", "enabled",
|
||||
"next_run_at", "coordinator_prompt", "model",
|
||||
}})
|
||||
if "direction_notes" in data:
|
||||
row["direction_notes"] = list(data["direction_notes"])
|
||||
row["updated_at"] = now
|
||||
self._bindings[external_task_id] = row
|
||||
return dict(row)
|
||||
|
||||
async def delete_binding(self, external_task_id: str) -> bool:
|
||||
return self._bindings.pop(external_task_id, None) is not None
|
||||
|
||||
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
due = []
|
||||
for binding in self._bindings.values():
|
||||
if not binding.get("enabled"):
|
||||
continue
|
||||
next_at = _parse_dt(binding.get("next_run_at"))
|
||||
if next_at is not None and next_at <= now:
|
||||
due.append(binding)
|
||||
due.sort(key=lambda b: b.get("next_run_at") or "")
|
||||
return [dict(b) for b in due[:limit]]
|
||||
|
||||
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
for key, binding in self._bindings.items():
|
||||
if binding["id"] == binding_id:
|
||||
binding.update({k: v for k, v in patch.items() if k != "id"})
|
||||
binding["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return dict(binding)
|
||||
return None
|
||||
|
||||
async def last_bound_chain_id(self) -> str | None:
|
||||
if not self._bindings:
|
||||
return None
|
||||
latest = max(self._bindings.values(), key=lambda b: b.get("_rev", 0))
|
||||
return latest.get("chain_id")
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
binding_id = data["binding_id"]
|
||||
scheduled_for = _parse_dt(data["scheduled_for"]) or datetime.now(UTC)
|
||||
for run in self._runs.values():
|
||||
if run["binding_id"] == binding_id and run["scheduled_for"] == scheduled_for.isoformat():
|
||||
return dict(run)
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row = {
|
||||
"id": self._next_id("tar"),
|
||||
"binding_id": binding_id,
|
||||
"external_task_id": str(data["external_task_id"]),
|
||||
"trigger": str(data.get("trigger") or "schedule"),
|
||||
"scheduled_for": scheduled_for.isoformat(),
|
||||
"started_at": now,
|
||||
"finished_at": None,
|
||||
"status": str(data.get("status") or "pending"),
|
||||
"stages_snapshot": data.get("stages_snapshot") or [],
|
||||
"stage_goals_snapshot": data.get("stage_goals_snapshot") or [],
|
||||
"seat_names_snapshot": data.get("seat_names_snapshot") or {},
|
||||
"seat_results": data.get("seat_results") or {},
|
||||
"error": data.get("error"),
|
||||
"retry_count": 0,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._runs[row["id"]] = row
|
||||
return dict(row)
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
return dict(run) if run else None
|
||||
|
||||
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._runs.values() if r["external_task_id"] == external_task_id]
|
||||
rows.sort(key=lambda r: r["started_at"], reverse=True)
|
||||
return [dict(r) for r in rows[:limit]]
|
||||
|
||||
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
run.update({k: v for k, v in patch.items() if k not in {"id", "binding_id"}})
|
||||
run["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return dict(run)
|
||||
|
||||
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
for run in self._runs.values():
|
||||
if run["status"] in {"pending", "running"}:
|
||||
started = _parse_dt(run["started_at"])
|
||||
if started is not None and started < older_than:
|
||||
out.append(run)
|
||||
out.sort(key=lambda r: r["started_at"])
|
||||
return [dict(r) for r in out[:limit]]
|
||||
|
||||
# ── messages ────────────────────────────────────────────────────────────
|
||||
|
||||
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
|
||||
base_seq = sum(1 for m in self._messages if m["run_id"] == run_id)
|
||||
added = 0
|
||||
for index, message in enumerate(messages):
|
||||
self._messages.append({
|
||||
"id": self._next_id("tam"),
|
||||
"run_id": run_id,
|
||||
"seq": base_seq + index,
|
||||
"seat_agent_id": str(message.get("seat_agent_id") or ""),
|
||||
"stage_index": int(message.get("stage_index") or 0),
|
||||
"event_type": str(message.get("event_type") or "llm.ai"),
|
||||
"content": message.get("content"),
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
})
|
||||
added += 1
|
||||
return added
|
||||
|
||||
async def list_run_messages(
|
||||
self, run_id: str, *, seat: str | None = None, limit: int = 500
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = [m for m in self._messages if m["run_id"] == run_id]
|
||||
if seat is not None:
|
||||
rows = [m for m in rows if m["seat_agent_id"] == seat]
|
||||
rows.sort(key=lambda m: m["seq"])
|
||||
return [dict(m) for m in rows[:limit]]
|
||||
|
||||
# ── versions ────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(data["run_id"])
|
||||
row = {
|
||||
"id": self._next_id("tav"),
|
||||
"external_task_id": str(data["external_task_id"]),
|
||||
"run_id": run_id,
|
||||
"blocks": data.get("blocks") or {},
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
self._versions[run_id] = row
|
||||
return dict(row)
|
||||
|
||||
async def list_versions(
|
||||
self, external_task_id: str, *, limit: int = 50, offset: int = 0
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = [v for v in self._versions.values() if v["external_task_id"] == external_task_id]
|
||||
rows.sort(key=lambda v: v["created_at"], reverse=True)
|
||||
return [dict(v) for v in rows[offset : offset + limit]]
|
||||
|
||||
# ── data delivery: block defs / periods / submissions ────────────────────
|
||||
|
||||
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
self._seed_block_defs(task_id)
|
||||
return self._ordered_defs(task_id)
|
||||
|
||||
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
return self._ordered_defs(task_id)
|
||||
|
||||
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
definitions = self._block_defs.setdefault(task_id, {})
|
||||
now = datetime.now(UTC).isoformat()
|
||||
for update in updates:
|
||||
block_key = str(update.get("block_key") or "")
|
||||
row = definitions.get(block_key)
|
||||
if row is None:
|
||||
raise ValueError(f"未知数据块:{block_key}")
|
||||
if update.get("title") is not None:
|
||||
row["title"] = str(update["title"])
|
||||
if update.get("sort_order") is not None:
|
||||
row["sort_order"] = int(update["sort_order"])
|
||||
row["updated_at"] = now
|
||||
return self._ordered_defs(task_id)
|
||||
|
||||
async def open_period(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
source: str,
|
||||
run_id: str | None = None,
|
||||
opened_at: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if run_id:
|
||||
for period in self._periods.get(task_id, []):
|
||||
if period.get("run_id") == run_id:
|
||||
return dict(period)
|
||||
rows = self._periods.setdefault(task_id, [])
|
||||
index = (rows[-1]["period_index"] if rows else 0) + 1
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row = {
|
||||
"id": self._next_id("tdp"),
|
||||
"task_id": task_id,
|
||||
"period_index": index,
|
||||
"opened_at": _iso(opened_at) or now,
|
||||
"trigger": str(source or "cron"),
|
||||
"run_id": run_id,
|
||||
"created_at": now,
|
||||
}
|
||||
rows.append(row)
|
||||
return dict(row)
|
||||
|
||||
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None:
|
||||
for period in self._period_rows(task_id):
|
||||
if period["period_index"] == period_index:
|
||||
return dict(period)
|
||||
return None
|
||||
|
||||
async def current_period(self, task_id: str) -> dict[str, Any] | None:
|
||||
rows = self._period_rows(task_id)
|
||||
return dict(rows[-1]) if rows else None
|
||||
|
||||
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
|
||||
rows = self._period_rows(task_id)
|
||||
return [dict(p) for p in rows[-limit:]]
|
||||
|
||||
async def append_submission(
|
||||
self,
|
||||
task_id: str,
|
||||
period_index: int,
|
||||
block_key: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
submitted_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
key = (task_id, period_index, block_key)
|
||||
rows = self._submissions.setdefault(key, [])
|
||||
row = {
|
||||
"id": self._next_id("tds"),
|
||||
"task_id": task_id,
|
||||
"period_index": period_index,
|
||||
"block_key": block_key,
|
||||
"payload": dict(payload),
|
||||
"submitted_by": submitted_by,
|
||||
"seq": len(rows),
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
rows.append(row)
|
||||
return dict(row)
|
||||
|
||||
async def list_submissions(
|
||||
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = list(self._submissions.get((task_id, period_index, block_key), []))
|
||||
rows.sort(key=lambda s: s["seq"])
|
||||
return [dict(s) for s in rows[:limit]]
|
||||
|
||||
async def upsert_period_block(
|
||||
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
key = (task_id, period_index, block_key)
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row = self._task_blocks.get(key)
|
||||
if row is None:
|
||||
row = {
|
||||
"id": self._next_id("tdb"),
|
||||
"task_id": task_id,
|
||||
"period_index": period_index,
|
||||
"block_key": block_key,
|
||||
"payload": dict(payload),
|
||||
"updated_at": now,
|
||||
"created_at": now,
|
||||
}
|
||||
self._task_blocks[key] = row
|
||||
else:
|
||||
row["payload"] = dict(payload)
|
||||
row["updated_at"] = now
|
||||
return dict(row)
|
||||
|
||||
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
|
||||
rows = [dict(row) for key, row in self._task_blocks.items() if key[0] == task_id]
|
||||
rows.sort(key=lambda r: (r["period_index"], r["block_key"]))
|
||||
return rows[:limit]
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
"""ORM models for the external-task multi-agent scheduled analysis (任务分析定时).
|
||||
|
||||
Feature summary (design doc in the frontend ``official-task-analysis`` module):
|
||||
an external system pushes tasks via our create API; each task binds 1:1 to a
|
||||
business-chain orchestration plus a cron schedule; the scheduler runs the chain
|
||||
stage-by-stage (seats within a stage in parallel) and persists per-seat process
|
||||
messages plus a content "timeline" version per run. Manual runs wrap the
|
||||
roundtable Step-2 coordinator Q&A and are stored with the same message table
|
||||
(``seat_agent_id`` distinguishes seats; the coordinator uses ``__coordinator__``).
|
||||
|
||||
Tables:
|
||||
- ``task_analysis_tasks`` — external tasks (source_task_id unique)
|
||||
- ``task_analysis_bindings`` — 1:1 binding (chain snapshot + cron + notes)
|
||||
- ``task_analysis_runs`` — executions (trigger: schedule | manual)
|
||||
- ``task_analysis_run_messages`` — per-seat process messages for a run
|
||||
- ``task_analysis_content_versions`` — one timeline version per run
|
||||
- ``task_data_block_defs`` — the 5 fixed data blocks per task (title/sort editable)
|
||||
- ``task_data_periods`` — data-delivery timeline ticks (period_index per task)
|
||||
- ``task_data_blocks`` — merged payload per (task, period, block)
|
||||
- ``task_data_submissions`` — append-only submission log feeding the merge
|
||||
|
||||
JSON payloads are stored as LONGTEXT-backed JSON strings (PortableLongText),
|
||||
mirroring the sibling roundtable persistence modules for headroom.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class TaskAnalysisTaskRow(Base):
|
||||
__tablename__ = "task_analysis_tasks"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# 对方系统任务号(创建接口传入;唯一)。
|
||||
source_task_id: Mapped[str] = mapped_column(String(128), nullable=False, unique=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
# 任务简介:创建时落库的存量即执行时用的版本(不回拉对方接口)。
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
|
||||
class TaskAnalysisBindingRow(Base):
|
||||
__tablename__ = "task_analysis_bindings"
|
||||
__table_args__ = (UniqueConstraint("external_task_id", name="uq_task_analysis_binding_task"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# 引用 roundtable_chains.id;执行前校验,链缺失/停用 → 跳过并标记 chain_missing。
|
||||
chain_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# 执行编排快照 JSON:{seats, stages, stage_goals, coordinator_prompt}。
|
||||
# 每次执行前刷新(跟随链最新版),历史 run 用自己的 stages_snapshot 回放。
|
||||
chain_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
cron_expr: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
timezone: Mapped[str] = mapped_column(String(64), nullable=False, default="Asia/Shanghai")
|
||||
# 执行方式:cron = 按频率定时;once = 只执行一次(创建即执行,next_run_at 恒空)。
|
||||
schedule_type: Mapped[str] = mapped_column(String(16), nullable=False, default="cron", server_default="cron")
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="1")
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
|
||||
# 绑定级总控提示(与链内 coordinator_prompt 拼接使用)。
|
||||
coordinator_prompt: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# 问答沉淀的「执行方向备注」JSON 数组(逐条追加,注入后续每次定时执行)。
|
||||
direction_notes: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
# 执行模型(席位 run 的 model_name;NULL = 系统/席位默认)。
|
||||
model: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
|
||||
class TaskAnalysisRunRow(Base):
|
||||
__tablename__ = "task_analysis_runs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
binding_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# schedule = cron 定时执行;manual = 总控问答(对方详情页发起)。
|
||||
trigger: Mapped[str] = mapped_column(String(16), nullable=False, default="schedule")
|
||||
# 幂等键:同 binding + scheduled_for 只建一条(对齐 scheduled_tasks.create_run_once)。
|
||||
scheduled_for: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False)
|
||||
started_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
# pending | running | succeeded | failed | partial
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
|
||||
# 本次编排快照 JSON:[[seat_id,...],...](回放不受链后续修改影响)。
|
||||
stages_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
stage_goals_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
seat_names_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# 各席位结果 JSON:{agent_id: {state, duration_ms, thread_id, error}}。
|
||||
seat_results: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
|
||||
class TaskAnalysisRunMessageRow(Base):
|
||||
__tablename__ = "task_analysis_run_messages"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# 消息归属席位(agent_id);总控消息用固定标识 __coordinator__。
|
||||
seat_agent_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
stage_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# input | llm.ai | tool.result | artifact(对齐现有 run messages 事件分类)。
|
||||
event_type: Mapped[str] = mapped_column(String(32), nullable=False, default="llm.ai")
|
||||
content: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
|
||||
|
||||
class TaskAnalysisContentVersionRow(Base):
|
||||
__tablename__ = "task_analysis_content_versions"
|
||||
__table_args__ = (UniqueConstraint("run_id", name="uq_task_analysis_version_run"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# 一次执行 = 一期时间线版本(1:1)。
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# JSON:{key: {name, seat_agent_id, content, format}} —— 对方详情的各块内容。
|
||||
blocks: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
|
||||
|
||||
class TaskDataBlockDefRow(Base):
|
||||
"""数据块定义:块 key 定死 block_1~block_6,标题默认「数据块一~六」可改。"""
|
||||
|
||||
__tablename__ = "task_data_block_defs"
|
||||
__table_args__ = (UniqueConstraint("task_id", "block_key", name="uq_task_data_block_def_key"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# 关联 task_analysis_tasks.id(内部任务 id)。
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
title: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
|
||||
class TaskDataPeriodRow(Base):
|
||||
"""数据交付时间轴刻度:每个任务 period_index 从 1 递增;run_id 作幂等锚点。"""
|
||||
|
||||
__tablename__ = "task_data_periods"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("task_id", "period_index", name="uq_task_data_period_index"),
|
||||
UniqueConstraint("run_id", name="uq_task_data_period_run"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
opened_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
# cron = 定时执行开期;manual = 手动触发/总控问答开期;external = 首次提交兜底开期。
|
||||
trigger: Mapped[str] = mapped_column(String(16), nullable=False, default="cron")
|
||||
# 触发本期的执行记录(问答/定时执行共用 run 表);提交兜底开的期没有 run。
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
|
||||
|
||||
class TaskDataBlockRow(Base):
|
||||
"""每期每块的合并结果(由 task_data_submissions 流水按序合并得出)。"""
|
||||
|
||||
__tablename__ = "task_data_blocks"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("task_id", "period_index", "block_key", name="uq_task_data_block_tpb"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# JSON 对象(顶层 key 合并、同 key 后写覆盖)。
|
||||
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
|
||||
|
||||
class TaskDataSubmissionRow(Base):
|
||||
"""入库流水(追加只读):块 payload 的一致性来源。"""
|
||||
|
||||
__tablename__ = "task_data_submissions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# 本次提交的 JSON 对象片段(合并语义:顶层 key 合并、同 key 后写覆盖)。
|
||||
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# 提交方(席位 agent id / 用户 id,仅审计)。
|
||||
submitted_by: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
# 同 (task, period, block) 内的提交序号,从 0 递增 —— 合并顺序依据。
|
||||
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
+785
@@ -0,0 +1,785 @@
|
||||
"""SQLAlchemy-backed task-analysis store.
|
||||
|
||||
Mirrors the memory implementation's normalisation: JSON payloads
|
||||
(chain_snapshot / stages / seat_results / blocks / direction_notes) are stored
|
||||
as LONGTEXT JSON strings; datetimes are ISO strings in dict form.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
|
||||
from deerflow.persistence.task_analysis.model import (
|
||||
TaskAnalysisBindingRow,
|
||||
TaskAnalysisContentVersionRow,
|
||||
TaskAnalysisRunMessageRow,
|
||||
TaskAnalysisRunRow,
|
||||
TaskAnalysisTaskRow,
|
||||
TaskDataBlockDefRow,
|
||||
TaskDataBlockRow,
|
||||
TaskDataPeriodRow,
|
||||
TaskDataSubmissionRow,
|
||||
)
|
||||
|
||||
|
||||
def _iso(value: datetime | str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
return str(value)
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if value is None or isinstance(value, datetime):
|
||||
return value
|
||||
try:
|
||||
return datetime.fromisoformat(str(value))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _dump(value: Any) -> str | None:
|
||||
return None if value is None else json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _load(value: str | None, default: Any) -> Any:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _task_dict(row: TaskAnalysisTaskRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
data["updated_at"] = _iso(row.updated_at)
|
||||
return data
|
||||
|
||||
|
||||
def _binding_dict(row: TaskAnalysisBindingRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["chain_snapshot"] = _load(row.chain_snapshot, {})
|
||||
data["direction_notes"] = _load(row.direction_notes, [])
|
||||
for key in ("next_run_at", "last_run_at", "created_at", "updated_at"):
|
||||
data[key] = _iso(getattr(row, key))
|
||||
return data
|
||||
|
||||
|
||||
def _run_dict(row: TaskAnalysisRunRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["scheduled_for"] = _iso(row.scheduled_for)
|
||||
data["started_at"] = _iso(row.started_at)
|
||||
data["finished_at"] = _iso(row.finished_at)
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
data["updated_at"] = _iso(row.updated_at)
|
||||
for key, default in (
|
||||
("stages_snapshot", []),
|
||||
("stage_goals_snapshot", []),
|
||||
("seat_names_snapshot", {}),
|
||||
("seat_results", {}),
|
||||
):
|
||||
data[key] = _load(getattr(row, key), default)
|
||||
return data
|
||||
|
||||
|
||||
def _message_dict(row: TaskAnalysisRunMessageRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
return data
|
||||
|
||||
|
||||
def _version_dict(row: TaskAnalysisContentVersionRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["blocks"] = _load(row.blocks, {})
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
return data
|
||||
|
||||
|
||||
def _block_def_dict(row: TaskDataBlockDefRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
data["updated_at"] = _iso(row.updated_at)
|
||||
return data
|
||||
|
||||
|
||||
def _period_dict(row: TaskDataPeriodRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["opened_at"] = _iso(row.opened_at)
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
return data
|
||||
|
||||
|
||||
def _task_block_dict(row: TaskDataBlockRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["payload"] = _load(row.payload, {})
|
||||
data["updated_at"] = _iso(row.updated_at)
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
return data
|
||||
|
||||
|
||||
def _submission_dict(row: TaskDataSubmissionRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["payload"] = _load(row.payload, {})
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
return data
|
||||
|
||||
|
||||
class TaskAnalysisRepository(TaskAnalysisStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# ── tasks ───────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
source = str(data["source_task_id"])
|
||||
async with self._sf() as session:
|
||||
existing = await session.execute(
|
||||
select(TaskAnalysisTaskRow).where(TaskAnalysisTaskRow.source_task_id == source)
|
||||
)
|
||||
if existing.scalar_one_or_none() is not None:
|
||||
raise ValueError(f"source_task_id already exists: {source}")
|
||||
row = TaskAnalysisTaskRow(
|
||||
id=str(data.get("id") or f"ta-{source}"),
|
||||
source_task_id=source,
|
||||
name=str(data.get("name") or ""),
|
||||
description=data.get("description"),
|
||||
)
|
||||
session.add(row)
|
||||
# 建任务即种 6 条数据块定义(块 key 定死,标题后续可改)。
|
||||
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
|
||||
|
||||
for definition in default_block_defs():
|
||||
session.add(
|
||||
TaskDataBlockDefRow(
|
||||
id=f"tdbd-{row.id}-{definition['block_key']}",
|
||||
task_id=row.id,
|
||||
block_key=definition["block_key"],
|
||||
title=definition["title"],
|
||||
sort_order=definition["sort_order"],
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _task_dict(row)
|
||||
|
||||
async def get_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TaskAnalysisTaskRow, task_id)
|
||||
return _task_dict(row) if row else None
|
||||
|
||||
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisTaskRow).where(TaskAnalysisTaskRow.source_task_id == source_task_id)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _task_dict(row) if row else None
|
||||
|
||||
async def list_tasks(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisTaskRow).order_by(TaskAnalysisTaskRow.created_at.desc())
|
||||
)
|
||||
tasks = [_task_dict(row) for row in result.scalars()]
|
||||
bindings = await session.execute(select(TaskAnalysisBindingRow))
|
||||
by_task: dict[str, dict[str, Any]] = {}
|
||||
for row in bindings.scalars():
|
||||
by_task[row.external_task_id] = _binding_dict(row)
|
||||
for task in tasks:
|
||||
task["binding"] = by_task.get(task["id"])
|
||||
return tasks
|
||||
|
||||
async def delete_task_cascade(self, task_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TaskAnalysisTaskRow, task_id)
|
||||
if row is None:
|
||||
return False
|
||||
run_ids = (
|
||||
(
|
||||
await session.execute(
|
||||
select(TaskAnalysisRunRow.id).where(TaskAnalysisRunRow.external_task_id == task_id)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskAnalysisBindingRow).where(TaskAnalysisBindingRow.external_task_id == task_id)
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskDataPeriodRow).where(TaskDataPeriodRow.task_id == task_id)
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskDataBlockRow).where(TaskDataBlockRow.task_id == task_id)
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskDataSubmissionRow).where(TaskDataSubmissionRow.task_id == task_id)
|
||||
)
|
||||
if run_ids:
|
||||
await session.execute(
|
||||
delete(TaskAnalysisRunMessageRow).where(TaskAnalysisRunMessageRow.run_id.in_(run_ids))
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskAnalysisContentVersionRow).where(
|
||||
TaskAnalysisContentVersionRow.run_id.in_(run_ids)
|
||||
)
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskAnalysisRunRow).where(TaskAnalysisRunRow.external_task_id == task_id)
|
||||
)
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
# ── bindings ────────────────────────────────────────────────────────────
|
||||
|
||||
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisBindingRow).where(
|
||||
TaskAnalysisBindingRow.external_task_id == external_task_id
|
||||
)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _binding_dict(row) if row else None
|
||||
|
||||
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TaskAnalysisBindingRow, binding_id)
|
||||
return _binding_dict(row) if row else None
|
||||
|
||||
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisBindingRow).where(
|
||||
TaskAnalysisBindingRow.external_task_id == external_task_id
|
||||
)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
row = TaskAnalysisBindingRow(
|
||||
id=str(data.get("id") or f"tab-{external_task_id}"),
|
||||
external_task_id=external_task_id,
|
||||
chain_id=str(data["chain_id"]),
|
||||
created_by=data.get("created_by"),
|
||||
)
|
||||
session.add(row)
|
||||
row.chain_id = str(data.get("chain_id") or row.chain_id)
|
||||
row.chain_snapshot = _dump(data.get("chain_snapshot") or {})
|
||||
row.cron_expr = str(data.get("cron_expr") or row.cron_expr or "")
|
||||
row.timezone = str(data.get("timezone") or row.timezone or "Asia/Shanghai")
|
||||
if "schedule_type" in data:
|
||||
row.schedule_type = str(data.get("schedule_type") or "cron")
|
||||
row.model = data.get("model")
|
||||
if "enabled" in data:
|
||||
row.enabled = bool(data["enabled"])
|
||||
if "next_run_at" in data:
|
||||
row.next_run_at = _parse_dt(data.get("next_run_at"))
|
||||
row.coordinator_prompt = data.get("coordinator_prompt")
|
||||
if "direction_notes" in data:
|
||||
row.direction_notes = _dump(list(data["direction_notes"] or []))
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _binding_dict(row)
|
||||
|
||||
async def delete_binding(self, external_task_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(TaskAnalysisBindingRow).where(
|
||||
TaskAnalysisBindingRow.external_task_id == external_task_id
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisBindingRow)
|
||||
.where(
|
||||
TaskAnalysisBindingRow.enabled.is_(True),
|
||||
TaskAnalysisBindingRow.next_run_at.is_not(None),
|
||||
TaskAnalysisBindingRow.next_run_at <= now,
|
||||
)
|
||||
.order_by(TaskAnalysisBindingRow.next_run_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_binding_dict(row) for row in result.scalars()]
|
||||
|
||||
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
allowed = {
|
||||
"chain_id", "cron_expr", "timezone", "schedule_type", "enabled", "next_run_at",
|
||||
"last_run_at", "last_status", "coordinator_prompt", "model",
|
||||
}
|
||||
values: dict[str, Any] = {k: v for k, v in patch.items() if k in allowed}
|
||||
if "next_run_at" in values:
|
||||
values["next_run_at"] = _parse_dt(values["next_run_at"])
|
||||
if "last_run_at" in values:
|
||||
values["last_run_at"] = _parse_dt(values["last_run_at"])
|
||||
if "direction_notes" in patch:
|
||||
values["direction_notes"] = _dump(list(patch["direction_notes"] or []))
|
||||
if not values:
|
||||
row = await self._get_binding_row(binding_id)
|
||||
return _binding_dict(row) if row else None
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(TaskAnalysisBindingRow).where(TaskAnalysisBindingRow.id == binding_id).values(**values)
|
||||
)
|
||||
await session.commit()
|
||||
row = await session.get(TaskAnalysisBindingRow, binding_id)
|
||||
return _binding_dict(row) if row else None
|
||||
|
||||
async def _get_binding_row(self, binding_id: str) -> TaskAnalysisBindingRow | None:
|
||||
async with self._sf() as session:
|
||||
return await session.get(TaskAnalysisBindingRow, binding_id)
|
||||
|
||||
async def last_bound_chain_id(self) -> str | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisBindingRow.chain_id)
|
||||
.order_by(TaskAnalysisBindingRow.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
binding_id = str(data["binding_id"])
|
||||
scheduled_for = _parse_dt(data["scheduled_for"]) or datetime.now(UTC)
|
||||
import uuid as _uuid
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisRunRow).where(
|
||||
TaskAnalysisRunRow.binding_id == binding_id,
|
||||
TaskAnalysisRunRow.scheduled_for == scheduled_for,
|
||||
)
|
||||
)
|
||||
existing = result.scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return _run_dict(existing)
|
||||
row = TaskAnalysisRunRow(
|
||||
id=str(data.get("id") or f"tar-{_uuid.uuid4()}"),
|
||||
binding_id=binding_id,
|
||||
external_task_id=str(data["external_task_id"]),
|
||||
trigger=str(data.get("trigger") or "schedule"),
|
||||
scheduled_for=scheduled_for,
|
||||
status=str(data.get("status") or "pending"),
|
||||
stages_snapshot=_dump(data.get("stages_snapshot") or []),
|
||||
stage_goals_snapshot=_dump(data.get("stage_goals_snapshot") or []),
|
||||
seat_names_snapshot=_dump(data.get("seat_names_snapshot") or {}),
|
||||
seat_results=_dump(data.get("seat_results") or {}),
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _run_dict(row)
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TaskAnalysisRunRow, run_id)
|
||||
return _run_dict(row) if row else None
|
||||
|
||||
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisRunRow)
|
||||
.where(TaskAnalysisRunRow.external_task_id == external_task_id)
|
||||
.order_by(TaskAnalysisRunRow.started_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_run_dict(row) for row in result.scalars()]
|
||||
|
||||
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
allowed = {
|
||||
"trigger", "status", "finished_at", "error", "retry_count",
|
||||
"stages_snapshot", "stage_goals_snapshot", "seat_names_snapshot", "seat_results",
|
||||
}
|
||||
values: dict[str, Any] = {}
|
||||
for key, value in patch.items():
|
||||
if key not in allowed:
|
||||
continue
|
||||
if key == "finished_at":
|
||||
values[key] = _parse_dt(value)
|
||||
elif key in {"stages_snapshot", "stage_goals_snapshot", "seat_names_snapshot", "seat_results"}:
|
||||
values[key] = _dump(value)
|
||||
else:
|
||||
values[key] = value
|
||||
if not values:
|
||||
row = await self._get_run_row(run_id)
|
||||
return _run_dict(row) if row else None
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(TaskAnalysisRunRow).where(TaskAnalysisRunRow.id == run_id).values(**values)
|
||||
)
|
||||
await session.commit()
|
||||
row = await session.get(TaskAnalysisRunRow, run_id)
|
||||
return _run_dict(row) if row else None
|
||||
|
||||
async def _get_run_row(self, run_id: str) -> TaskAnalysisRunRow | None:
|
||||
async with self._sf() as session:
|
||||
return await session.get(TaskAnalysisRunRow, run_id)
|
||||
|
||||
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisRunRow)
|
||||
.where(
|
||||
TaskAnalysisRunRow.status.in_(("pending", "running")),
|
||||
TaskAnalysisRunRow.started_at < older_than,
|
||||
)
|
||||
.order_by(TaskAnalysisRunRow.started_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_run_dict(row) for row in result.scalars()]
|
||||
|
||||
# ── messages ────────────────────────────────────────────────────────────
|
||||
|
||||
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
|
||||
import uuid as _uuid
|
||||
|
||||
async with self._sf() as session:
|
||||
existing = (
|
||||
(
|
||||
await session.execute(
|
||||
select(TaskAnalysisRunMessageRow.id).where(
|
||||
TaskAnalysisRunMessageRow.run_id == run_id
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
base_seq = len(existing)
|
||||
rows = [
|
||||
TaskAnalysisRunMessageRow(
|
||||
id=str(_uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
seq=base_seq + index,
|
||||
seat_agent_id=str(message.get("seat_agent_id") or ""),
|
||||
stage_index=int(message.get("stage_index") or 0),
|
||||
event_type=str(message.get("event_type") or "llm.ai"),
|
||||
content=message.get("content"),
|
||||
)
|
||||
for index, message in enumerate(messages)
|
||||
]
|
||||
session.add_all(rows)
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
|
||||
async def list_run_messages(
|
||||
self, run_id: str, *, seat: str | None = None, limit: int = 500
|
||||
) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(TaskAnalysisRunMessageRow)
|
||||
.where(TaskAnalysisRunMessageRow.run_id == run_id)
|
||||
.order_by(TaskAnalysisRunMessageRow.seq.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
if seat is not None:
|
||||
stmt = stmt.where(TaskAnalysisRunMessageRow.seat_agent_id == seat)
|
||||
result = await session.execute(stmt)
|
||||
return [_message_dict(row) for row in result.scalars()]
|
||||
|
||||
# ── versions ────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
import uuid as _uuid
|
||||
|
||||
run_id = str(data["run_id"])
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
delete(TaskAnalysisContentVersionRow).where(
|
||||
TaskAnalysisContentVersionRow.run_id == run_id
|
||||
)
|
||||
)
|
||||
row = TaskAnalysisContentVersionRow(
|
||||
id=str(_uuid.uuid4()),
|
||||
external_task_id=str(data["external_task_id"]),
|
||||
run_id=run_id,
|
||||
blocks=_dump(data.get("blocks") or {}),
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _version_dict(row)
|
||||
|
||||
async def list_versions(
|
||||
self, external_task_id: str, *, limit: int = 50, offset: int = 0
|
||||
) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisContentVersionRow)
|
||||
.where(TaskAnalysisContentVersionRow.external_task_id == external_task_id)
|
||||
.order_by(TaskAnalysisContentVersionRow.created_at.desc())
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
return [_version_dict(row) for row in result.scalars()]
|
||||
|
||||
# ── data delivery: block defs / periods / submissions ────────────────────
|
||||
|
||||
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
|
||||
)
|
||||
existing_keys = {row.block_key for row in result.scalars()}
|
||||
missing = [
|
||||
definition
|
||||
for definition in default_block_defs()
|
||||
if definition["block_key"] not in existing_keys
|
||||
]
|
||||
if not missing:
|
||||
return await self._ordered_defs(session, task_id)
|
||||
for definition in missing:
|
||||
session.add(
|
||||
TaskDataBlockDefRow(
|
||||
id=f"tdbd-{task_id}-{definition['block_key']}",
|
||||
task_id=task_id,
|
||||
block_key=definition["block_key"],
|
||||
title=definition["title"],
|
||||
sort_order=definition["sort_order"],
|
||||
)
|
||||
)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# 并发补种:另一个请求先种好了,沿用其结果。
|
||||
await session.rollback()
|
||||
return await self._ordered_defs(session, task_id)
|
||||
|
||||
@staticmethod
|
||||
async def _ordered_defs(session: AsyncSession, task_id: str) -> list[dict[str, Any]]:
|
||||
result = await session.execute(
|
||||
select(TaskDataBlockDefRow)
|
||||
.where(TaskDataBlockDefRow.task_id == task_id)
|
||||
.order_by(TaskDataBlockDefRow.sort_order.asc(), TaskDataBlockDefRow.block_key.asc())
|
||||
)
|
||||
return [_block_def_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
return await self._ordered_defs(session, task_id)
|
||||
|
||||
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
|
||||
)
|
||||
rows = {row.block_key: row for row in result.scalars()}
|
||||
touched = False
|
||||
for item in updates:
|
||||
block_key = str(item.get("block_key") or "")
|
||||
row = rows.get(block_key)
|
||||
if row is None:
|
||||
raise ValueError(f"未知数据块:{block_key}")
|
||||
patch: dict[str, Any] = {}
|
||||
if item.get("title") is not None:
|
||||
patch["title"] = str(item["title"])
|
||||
if item.get("sort_order") is not None:
|
||||
patch["sort_order"] = int(item["sort_order"])
|
||||
if patch:
|
||||
await session.execute(
|
||||
update(TaskDataBlockDefRow)
|
||||
.where(TaskDataBlockDefRow.id == row.id)
|
||||
.values(**patch)
|
||||
)
|
||||
touched = True
|
||||
if touched:
|
||||
await session.commit()
|
||||
return await self._ordered_defs(session, task_id)
|
||||
|
||||
async def open_period(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
source: str,
|
||||
run_id: str | None = None,
|
||||
opened_at: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
import uuid as _uuid
|
||||
|
||||
for _attempt in range(3):
|
||||
async with self._sf() as session:
|
||||
if run_id:
|
||||
existing = await session.execute(
|
||||
select(TaskDataPeriodRow).where(TaskDataPeriodRow.run_id == run_id)
|
||||
)
|
||||
row = existing.scalar_one_or_none()
|
||||
if row is not None:
|
||||
return _period_dict(row)
|
||||
max_index = (
|
||||
await session.execute(
|
||||
select(func.max(TaskDataPeriodRow.period_index)).where(
|
||||
TaskDataPeriodRow.task_id == task_id
|
||||
)
|
||||
)
|
||||
).scalar()
|
||||
row = TaskDataPeriodRow(
|
||||
id=f"tdp-{_uuid.uuid4()}",
|
||||
task_id=task_id,
|
||||
period_index=(max_index or 0) + 1,
|
||||
opened_at=opened_at or _utcnow(),
|
||||
trigger=str(source or "cron"),
|
||||
run_id=run_id,
|
||||
)
|
||||
session.add(row)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# 并发开期撞了期号唯一键(或 run_id 幂等锚点)——重查/重试。
|
||||
await session.rollback()
|
||||
continue
|
||||
await session.refresh(row)
|
||||
return _period_dict(row)
|
||||
raise RuntimeError(f"open_period 连续冲突,task={task_id} source={source}")
|
||||
|
||||
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataPeriodRow).where(
|
||||
TaskDataPeriodRow.task_id == task_id,
|
||||
TaskDataPeriodRow.period_index == period_index,
|
||||
)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _period_dict(row) if row else None
|
||||
|
||||
async def current_period(self, task_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataPeriodRow)
|
||||
.where(TaskDataPeriodRow.task_id == task_id)
|
||||
.order_by(TaskDataPeriodRow.period_index.desc())
|
||||
.limit(1)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _period_dict(row) if row else None
|
||||
|
||||
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataPeriodRow)
|
||||
.where(TaskDataPeriodRow.task_id == task_id)
|
||||
.order_by(TaskDataPeriodRow.period_index.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
rows = [_period_dict(row) for row in result.scalars()]
|
||||
rows.reverse() # 时间轴语义:按期号升序返回
|
||||
return rows
|
||||
|
||||
async def append_submission(
|
||||
self,
|
||||
task_id: str,
|
||||
period_index: int,
|
||||
block_key: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
submitted_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
import uuid as _uuid
|
||||
|
||||
async with self._sf() as session:
|
||||
base_seq = (
|
||||
(
|
||||
await session.execute(
|
||||
select(func.count(TaskDataSubmissionRow.id)).where(
|
||||
TaskDataSubmissionRow.task_id == task_id,
|
||||
TaskDataSubmissionRow.period_index == period_index,
|
||||
TaskDataSubmissionRow.block_key == block_key,
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalar_one()
|
||||
)
|
||||
row = TaskDataSubmissionRow(
|
||||
id=f"tds-{_uuid.uuid4()}",
|
||||
task_id=task_id,
|
||||
period_index=period_index,
|
||||
block_key=block_key,
|
||||
payload=_dump(payload),
|
||||
submitted_by=submitted_by,
|
||||
seq=int(base_seq),
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _submission_dict(row)
|
||||
|
||||
async def list_submissions(
|
||||
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
|
||||
) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataSubmissionRow)
|
||||
.where(
|
||||
TaskDataSubmissionRow.task_id == task_id,
|
||||
TaskDataSubmissionRow.period_index == period_index,
|
||||
TaskDataSubmissionRow.block_key == block_key,
|
||||
)
|
||||
.order_by(TaskDataSubmissionRow.seq.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_submission_dict(row) for row in result.scalars()]
|
||||
|
||||
async def upsert_period_block(
|
||||
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
import uuid as _uuid
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataBlockRow).where(
|
||||
TaskDataBlockRow.task_id == task_id,
|
||||
TaskDataBlockRow.period_index == period_index,
|
||||
TaskDataBlockRow.block_key == block_key,
|
||||
)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
row = TaskDataBlockRow(
|
||||
id=f"tdb-{_uuid.uuid4()}",
|
||||
task_id=task_id,
|
||||
period_index=period_index,
|
||||
block_key=block_key,
|
||||
payload=_dump(payload),
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.payload = _dump(payload)
|
||||
row.updated_at = _utcnow()
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _task_block_dict(row)
|
||||
|
||||
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataBlockRow)
|
||||
.where(TaskDataBlockRow.task_id == task_id)
|
||||
.order_by(TaskDataBlockRow.period_index.asc(), TaskDataBlockRow.block_key.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_task_block_dict(row) for row in result.scalars()]
|
||||
+21
@@ -30,6 +30,18 @@ EXCLUDED_SYSTEM_THREAD_TYPES = frozenset(
|
||||
"deep_research",
|
||||
"writing_setup",
|
||||
"report_structure_setup",
|
||||
"task-analysis",
|
||||
"task-analysis-qa",
|
||||
}
|
||||
)
|
||||
|
||||
# 这些系统线程 id 只存在于编排内部,不进「最近的对话」;loopback 派活必须能跨
|
||||
# 「点击触发的 HTTP 用户」和「绑定 created_by 签发的 JWT」读写,否则会 404 not accessible。
|
||||
SHARED_ACCESS_THREAD_TYPES = frozenset(
|
||||
{
|
||||
"roundtable",
|
||||
"task-analysis",
|
||||
"task-analysis-qa",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -43,6 +55,15 @@ def is_excluded_system_thread(metadata: dict[str, Any] | None) -> bool:
|
||||
return isinstance(thread_type, str) and thread_type in EXCLUDED_SYSTEM_THREAD_TYPES
|
||||
|
||||
|
||||
def is_shared_access_thread(metadata: dict[str, Any] | None) -> bool:
|
||||
"""Return True when any authenticated caller may read/write this thread."""
|
||||
md = metadata or {}
|
||||
if md.get("taskId"):
|
||||
return True
|
||||
thread_type = md.get("thread_type")
|
||||
return isinstance(thread_type, str) and thread_type in SHARED_ACCESS_THREAD_TYPES
|
||||
|
||||
|
||||
class ThreadMetaStore(abc.ABC):
|
||||
@abc.abstractmethod
|
||||
async def create(
|
||||
|
||||
+7
-3
@@ -11,7 +11,11 @@ from typing import Any
|
||||
|
||||
from langgraph.store.base import BaseStore
|
||||
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread
|
||||
from deerflow.persistence.thread_meta.base import (
|
||||
ThreadMetaStore,
|
||||
is_excluded_system_thread,
|
||||
is_shared_access_thread,
|
||||
)
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
from deerflow.utils.time import coerce_iso, now_iso
|
||||
|
||||
@@ -183,8 +187,8 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
item = await self._store.aget(THREADS_NS, thread_id)
|
||||
if item is None:
|
||||
return not require_existing
|
||||
# 任务深链对话(metadata.taskId)全局共享、不区分用户:任何登录用户都可读写/续聊。
|
||||
if (item.value.get("metadata") or {}).get("taskId"):
|
||||
# 任务深链 / 圆桌 / 任务分析系统线程:全局共享(与 SQL store 同口径)。
|
||||
if is_shared_access_thread(item.value.get("metadata") or {}):
|
||||
return True
|
||||
record_user_id = item.value.get("user_id")
|
||||
if record_user_id is None:
|
||||
|
||||
+5
-5
@@ -8,7 +8,7 @@ from typing import Any
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
@@ -108,10 +108,10 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
md = row.metadata_json or {}
|
||||
# 全局共享、不区分用户(任何登录用户都可读写/续聊):
|
||||
# ① 任务深链对话(metadata.taskId);
|
||||
# ② 圆桌会商系统线程(metadata.thread_type=="roundtable")——其 Step2/Step3
|
||||
# 沙箱产物(md/流程图/报告)需让历史草稿或其他账号打开同一会商时也能查看,
|
||||
# 与任务深链对话一致(线程 id 仅存在于草稿中,不会被无关账号发现)。
|
||||
if md.get("taskId") or md.get("thread_type") == "roundtable":
|
||||
# ② 圆桌 / 任务分析等系统编排线程(thread_type 见 SHARED_ACCESS_THREAD_TYPES)
|
||||
# —— loopback 派活签发的 JWT 可能与建 thread 时 HTTP 上下文用户不是同一人,
|
||||
# 若按属主拦截会把刚建好的总控 thread 打成 404 not accessible。
|
||||
if is_shared_access_thread(md):
|
||||
return True
|
||||
if row.user_id is None:
|
||||
return True
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Global database-backed upload policy persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.upload_policy.base import UploadPolicyStore
|
||||
from deerflow.persistence.upload_policy.memory import MemoryUploadPolicyStore
|
||||
from deerflow.persistence.upload_policy.model import UploadPolicyRow
|
||||
from deerflow.persistence.upload_policy.sql import UploadPolicyRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryUploadPolicyStore",
|
||||
"UploadPolicyRepository",
|
||||
"UploadPolicyRow",
|
||||
"UploadPolicyStore",
|
||||
"make_upload_policy_store",
|
||||
]
|
||||
|
||||
|
||||
def make_upload_policy_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryUploadPolicyStore()
|
||||
return UploadPolicyRepository(session_factory)
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
"""Persistence contract for the global upload policy document."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class UploadPolicyStore(ABC):
|
||||
@abstractmethod
|
||||
async def ensure_seed(self, policy: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create the default row if missing without overwriting saved edits."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_policy(self) -> dict[str, Any] | None:
|
||||
"""Return the stored policy, or ``None`` before the first save."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_policy(self, policy: dict[str, Any], *, updated_by: str | None = None) -> dict[str, Any]:
|
||||
"""Atomically replace the global policy document."""
|
||||
|
||||
raise NotImplementedError
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""In-memory upload-policy store used without a configured SQL database."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.upload_policy.base import UploadPolicyStore
|
||||
|
||||
|
||||
class MemoryUploadPolicyStore(UploadPolicyStore):
|
||||
def __init__(self) -> None:
|
||||
self._policy: dict[str, Any] | None = None
|
||||
self._updated_by: str | None = None
|
||||
self._updated_at: str | None = None
|
||||
self._revision = 0
|
||||
|
||||
async def ensure_seed(self, policy: dict[str, Any]) -> dict[str, Any]:
|
||||
if self._policy is None:
|
||||
return await self.set_policy(policy, updated_by=None)
|
||||
return (await self.get_policy()) or {}
|
||||
|
||||
async def get_policy(self) -> dict[str, Any] | None:
|
||||
if self._policy is None:
|
||||
return None
|
||||
return {
|
||||
"policy": deepcopy(self._policy),
|
||||
"updated_by": self._updated_by,
|
||||
"updated_at": self._updated_at,
|
||||
"revision": self._revision,
|
||||
}
|
||||
|
||||
async def set_policy(self, policy: dict[str, Any], *, updated_by: str | None = None) -> dict[str, Any]:
|
||||
self._policy = deepcopy(policy)
|
||||
self._updated_by = updated_by
|
||||
self._updated_at = datetime.now(UTC).isoformat()
|
||||
self._revision += 1
|
||||
return (await self.get_policy()) or {}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
"""ORM row for the global, database-backed upload policy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class UploadPolicyRow(Base):
|
||||
__tablename__ = "upload_policy"
|
||||
|
||||
# A named singleton keeps the schema extensible if scoped policies are
|
||||
# introduced later without requiring a table redesign.
|
||||
scope: Mapped[str] = mapped_column(String(64), primary_key=True, default="global")
|
||||
policy_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
revision: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
"""SQLAlchemy-backed upload-policy store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.upload_policy.base import UploadPolicyStore
|
||||
from deerflow.persistence.upload_policy.model import UploadPolicyRow
|
||||
|
||||
_GLOBAL_SCOPE = "global"
|
||||
|
||||
|
||||
def _to_dict(row: UploadPolicyRow) -> dict[str, Any]:
|
||||
updated_at: datetime | str | None = row.updated_at
|
||||
if isinstance(updated_at, datetime):
|
||||
updated_at = updated_at.isoformat()
|
||||
return {
|
||||
"policy": deepcopy(row.policy_json or {}),
|
||||
"updated_by": row.updated_by,
|
||||
"updated_at": updated_at,
|
||||
"revision": int(row.revision or 1),
|
||||
}
|
||||
|
||||
|
||||
class UploadPolicyRepository(UploadPolicyStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def ensure_seed(self, policy: dict[str, Any]) -> dict[str, Any]:
|
||||
existing = await self.get_policy()
|
||||
if existing is not None:
|
||||
return existing
|
||||
async with self._sf() as session:
|
||||
row = UploadPolicyRow(
|
||||
scope=_GLOBAL_SCOPE,
|
||||
policy_json=deepcopy(policy),
|
||||
revision=1,
|
||||
updated_by=None,
|
||||
)
|
||||
session.add(row)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Another worker seeded the singleton concurrently.
|
||||
await session.rollback()
|
||||
winner = await self.get_policy()
|
||||
if winner is None:
|
||||
raise
|
||||
return winner
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def get_policy(self) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UploadPolicyRow, _GLOBAL_SCOPE)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def set_policy(self, policy: dict[str, Any], *, updated_by: str | None = None) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(UploadPolicyRow).where(UploadPolicyRow.scope == _GLOBAL_SCOPE).with_for_update()
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
row = UploadPolicyRow(
|
||||
scope=_GLOBAL_SCOPE,
|
||||
policy_json=deepcopy(policy),
|
||||
revision=1,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.policy_json = deepcopy(policy)
|
||||
row.updated_by = updated_by
|
||||
row.revision = int(row.revision or 0) + 1
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
Reference in new issue
Block a user